From d1a5bac213a51619f6e8d2095c90a221d38d2faa Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Fri, 21 Jan 2022 10:43:37 -0700 Subject: [PATCH 01/22] fix: extra brace typo in insert_by_period_materialization --- macros/materializations/insert_by_period_materialization.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 851afa3d..5a15c815 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -53,7 +53,7 @@ {% materialization insert_by_period, default -%} {%- set timestamp_field = config.require('timestamp_field') -%} {%- set start_date = config.require('start_date') -%} - {%- set stop_date = config.get('stop_date') or '' -%}} + {%- set stop_date = config.get('stop_date') or '' -%} {%- set period = config.get('period') or 'week' -%} {%- if sql.find('__PERIOD_FILTER__') == -1 -%} From 7c4d7f2a8dca533207b4e8a63a919f73eb0bb136 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Thu, 5 May 2022 09:08:14 -0600 Subject: [PATCH 02/22] chore: apply sqlfmt 0.8.0 --- .../macros/assert_equal_values.sql | 13 +- integration_tests/macros/limit_zero.sql | 10 +- .../models/cross_db_utils/test_any_value.sql | 41 +- .../models/cross_db_utils/test_bool_or.sql | 8 +- .../models/cross_db_utils/test_concat.sql | 10 +- .../cross_db_utils/test_current_timestamp.sql | 1 - .../test_current_timestamp_in_utc.sql | 2 +- .../models/cross_db_utils/test_date_trunc.sql | 10 +- .../models/cross_db_utils/test_dateadd.sql | 34 +- .../models/cross_db_utils/test_datediff.sql | 112 ++++-- .../models/cross_db_utils/test_hash.sql | 10 +- .../models/cross_db_utils/test_last_day.sql | 15 +- .../models/cross_db_utils/test_length.sql | 13 +- .../models/cross_db_utils/test_listagg.sql | 130 +++--- .../models/cross_db_utils/test_position.sql | 10 +- .../models/cross_db_utils/test_replace.sql | 21 +- .../models/cross_db_utils/test_right.sql | 10 +- .../models/cross_db_utils/test_safe_cast.sql | 8 +- .../models/cross_db_utils/test_split_part.sql | 15 +- .../cross_db_utils/test_width_bucket.sql | 9 +- .../models/datetime/test_date_spine.sql | 22 +- .../test_equal_column_subset.sql | 10 +- .../generic_tests/test_equal_rowcount.sql | 11 +- .../generic_tests/test_fewer_rows_than.sql | 11 +- .../models/generic_tests/test_recency.sql | 11 +- .../models/geo/test_haversine_distance_km.sql | 39 +- .../models/geo/test_haversine_distance_mi.sql | 65 ++- .../expected_insert_by_period.sql | 9 +- .../test_insert_by_period.sql | 26 +- .../models/sql/test_deduplicate.sql | 29 +- .../models/sql/test_generate_series.sql | 12 +- .../models/sql/test_get_column_values.sql | 19 +- .../test_get_filtered_columns_in_relation.sql | 21 +- .../sql/test_get_relations_by_pattern.sql | 13 +- ...test_get_relations_by_prefix_and_union.sql | 4 +- integration_tests/models/sql/test_groupby.sql | 28 +- .../models/sql/test_nullcheck_table.sql | 27 +- integration_tests/models/sql/test_pivot.sql | 16 +- .../models/sql/test_pivot_apostrophe.sql | 14 +- .../models/sql/test_safe_add.sql | 10 +- integration_tests/models/sql/test_star.sql | 15 +- .../models/sql/test_star_aggregate.sql | 19 +- .../models/sql/test_star_prefix_suffix.sql | 18 +- .../models/sql/test_star_uppercase.sql | 15 +- .../models/sql/test_surrogate_key.sql | 14 +- integration_tests/models/sql/test_union.sql | 8 +- .../models/sql/test_union_base.sql | 5 +- integration_tests/models/sql/test_unpivot.sql | 35 +- .../models/sql/test_unpivot_bool.sql | 33 +- .../models/sql/test_unpivot_original_api.sql | 28 +- .../models/web/test_url_host.sql | 13 +- .../models/web/test_url_path.sql | 12 +- integration_tests/models/web/test_urls.sql | 12 +- ...et_query_results_as_dict_objects_equal.sql | 52 ++- .../assert_pretty_output_msg_is_string.sql | 10 +- .../assert_pretty_time_is_string.sql | 10 +- .../tests/jinja_helpers/test_slugify.sql | 11 +- .../test_get_column_values_use_default.sql | 50 ++- macros/cross_db_utils/_is_ephemeral.sql | 17 +- macros/cross_db_utils/_is_relation.sql | 9 +- macros/cross_db_utils/any_value.sql | 15 +- macros/cross_db_utils/bool_or.sql | 20 +- macros/cross_db_utils/cast_bool_to_text.sql | 11 +- macros/cross_db_utils/concat.sql | 6 +- macros/cross_db_utils/current_timestamp.sql | 24 +- macros/cross_db_utils/datatypes.sql | 78 +--- macros/cross_db_utils/date_trunc.sql | 9 +- macros/cross_db_utils/dateadd.sql | 24 +- macros/cross_db_utils/datediff.sql | 115 ++++-- .../cross_db_utils/escape_single_quotes.sql | 4 +- macros/cross_db_utils/except.sql | 14 +- macros/cross_db_utils/hash.sql | 6 +- macros/cross_db_utils/identifier.sql | 23 +- macros/cross_db_utils/intersect.sql | 14 +- macros/cross_db_utils/last_day.sql | 44 +- macros/cross_db_utils/length.sql | 18 +- macros/cross_db_utils/listagg.sql | 140 +++---- macros/cross_db_utils/literal.sql | 6 +- macros/cross_db_utils/position.sql | 16 +- macros/cross_db_utils/replace.sql | 12 +- macros/cross_db_utils/right.sql | 37 +- macros/cross_db_utils/safe_cast.sql | 10 +- macros/cross_db_utils/split_part.sql | 17 +- macros/cross_db_utils/width_bucket.sql | 73 ++-- macros/generic_tests/accepted_range.sql | 53 ++- macros/generic_tests/at_least_one.sql | 14 +- macros/generic_tests/cardinality_equality.sql | 60 +-- macros/generic_tests/equal_rowcount.sql | 41 +- macros/generic_tests/equality.sql | 77 ++-- macros/generic_tests/expression_is_true.sql | 21 +- macros/generic_tests/fewer_rows_than.sql | 66 ++- .../mutually_exclusive_ranges.sql | 190 +++++---- macros/generic_tests/not_accepted_values.sql | 44 +- macros/generic_tests/not_constant.sql | 9 +- macros/generic_tests/not_null_proportion.sql | 36 +- macros/generic_tests/recency.sql | 22 +- macros/generic_tests/relationships_where.sql | 58 +-- macros/generic_tests/sequential_values.sql | 56 ++- macros/generic_tests/test_not_null_where.sql | 21 +- macros/generic_tests/test_unique_where.sql | 21 +- .../unique_combination_of_columns.sql | 49 ++- macros/jinja_helpers/log_info.sql | 4 +- macros/jinja_helpers/pretty_log_format.sql | 4 +- macros/jinja_helpers/pretty_time.sql | 8 +- macros/jinja_helpers/slugify.sql | 6 +- .../insert_by_period_materialization.sql | 379 ++++++++++-------- macros/sql/date_spine.sql | 77 ++-- macros/sql/deduplicate.sql | 40 +- macros/sql/generate_series.sql | 53 ++- macros/sql/get_column_values.sql | 75 ++-- .../sql/get_filtered_columns_in_relation.sql | 32 +- macros/sql/get_query_results_as_dict.sql | 25 +- macros/sql/get_relations_by_pattern.sql | 54 ++- macros/sql/get_relations_by_prefix.sql | 50 ++- macros/sql/get_table_types_sql.sql | 52 ++- macros/sql/get_tables_by_pattern_sql.sql | 63 +-- macros/sql/get_tables_by_prefix_sql.sql | 30 +- macros/sql/groupby.sql | 7 +- macros/sql/haversine_distance.sql | 56 +-- macros/sql/nullcheck.sql | 14 +- macros/sql/nullcheck_table.sql | 12 +- macros/sql/pivot.sql | 97 +++-- macros/sql/safe_add.sql | 12 +- macros/sql/star.sql | 25 +- macros/sql/surrogate_key.sql | 36 +- macros/sql/union.sql | 139 ++++--- macros/sql/unpivot.sql | 145 ++++--- macros/web/get_url_host.sql | 34 +- macros/web/get_url_parameter.sql | 8 +- macros/web/get_url_path.sql | 21 +- 130 files changed, 2192 insertions(+), 2205 deletions(-) diff --git a/integration_tests/macros/assert_equal_values.sql b/integration_tests/macros/assert_equal_values.sql index d4f02618..48c0ff81 100644 --- a/integration_tests/macros/assert_equal_values.sql +++ b/integration_tests/macros/assert_equal_values.sql @@ -1,8 +1,7 @@ {% macro assert_equal_values(actual_object, expected_object) %} {% if not execute %} - {# pass #} - +{# pass #} {% elif actual_object != expected_object %} {% set msg %} @@ -20,13 +19,11 @@ {% endset %} - {{ log(msg, info=True) }} - - select 'fail' +{{ log(msg, info=True) }} -{% else %} +select 'fail' - select 'ok' {{ limit_zero() }} +{% else %} select 'ok' {{ limit_zero() }} {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/integration_tests/macros/limit_zero.sql b/integration_tests/macros/limit_zero.sql index 697849d1..d08ca051 100644 --- a/integration_tests/macros/limit_zero.sql +++ b/integration_tests/macros/limit_zero.sql @@ -1,11 +1,7 @@ -{% macro my_custom_macro() %} - whatever -{% endmacro %} +{% macro my_custom_macro() %} whatever {% endmacro %} {% macro limit_zero() %} - {{ return(adapter.dispatch('limit_zero', 'dbt_utils')()) }} +{{ return adapter.dispatch("limit_zero", "dbt_utils")() }} {% endmacro %} -{% macro default__limit_zero() %} - {{ return('limit 0') }} -{% endmacro %} \ No newline at end of file +{% macro default__limit_zero() %} {{ return "limit 0" }} {% endmacro %} diff --git a/integration_tests/models/cross_db_utils/test_any_value.sql b/integration_tests/models/cross_db_utils/test_any_value.sql index 9b27ed1d..6781b41f 100644 --- a/integration_tests/models/cross_db_utils/test_any_value.sql +++ b/integration_tests/models/cross_db_utils/test_any_value.sql @@ -1,19 +1,26 @@ -with some_model as ( - select 1 as id, 'abc' as key_name, 'dbt' as static_col union all - select 2 as id, 'abc' as key_name, 'dbt' as static_col union all - select 3 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 4 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 5 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 6 as id, 'xyz' as key_name, 'test' as static_col -), +with + some_model as ( + select 1 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 2 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 3 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 4 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 5 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 6 as id, 'xyz' as key_name, 'test' as static_col + ), -final as ( - select - key_name, - {{ dbt_utils.any_value('static_col') }} as static_col, - count(id) as num_rows - from some_model - group by key_name -) + final as ( + select + key_name, + {{ dbt_utils.any_value("static_col") }} as static_col, + count(id) as num_rows + from some_model + group by key_name + ) -select * from final \ No newline at end of file +select * +from final diff --git a/integration_tests/models/cross_db_utils/test_bool_or.sql b/integration_tests/models/cross_db_utils/test_bool_or.sql index 7375d1e4..1d6f4ad5 100644 --- a/integration_tests/models/cross_db_utils/test_bool_or.sql +++ b/integration_tests/models/cross_db_utils/test_bool_or.sql @@ -1,5 +1,3 @@ -select - key, - {{ dbt_utils.bool_or('val1 = val2') }} as value -from {{ ref('data_bool_or' )}} -group by key \ No newline at end of file +select key, {{ dbt_utils.bool_or("val1 = val2") }} as value +from {{ ref("data_bool_or") }} +group by key diff --git a/integration_tests/models/cross_db_utils/test_concat.sql b/integration_tests/models/cross_db_utils/test_concat.sql index 9efeb748..8c036c65 100644 --- a/integration_tests/models/cross_db_utils/test_concat.sql +++ b/integration_tests/models/cross_db_utils/test_concat.sql @@ -1,12 +1,6 @@ -with data as ( +with data as (select * from {{ ref("data_concat") }}) - select * from {{ ref('data_concat') }} - -) - -select - {{ dbt_utils.concat(['input_1', 'input_2']) }} as actual, - output as expected +select {{ dbt_utils.concat(["input_1", "input_2"]) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp.sql b/integration_tests/models/cross_db_utils/test_current_timestamp.sql index 9a777f28..d6c2cf73 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp.sql @@ -3,4 +3,3 @@ select {{ dbt_utils.current_timestamp() }} as actual, {{ dbt_utils.current_timestamp() }} as expected - diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql index 55bc8e23..e88e934a 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql @@ -2,4 +2,4 @@ -- how can we test this better? select {{ dbt_utils.current_timestamp_in_utc() }} as actual, - {{ dbt_utils.current_timestamp_in_utc() }} as expected \ No newline at end of file + {{ dbt_utils.current_timestamp_in_utc() }} as expected diff --git a/integration_tests/models/cross_db_utils/test_date_trunc.sql b/integration_tests/models/cross_db_utils/test_date_trunc.sql index bac21fed..a5b88aa3 100644 --- a/integration_tests/models/cross_db_utils/test_date_trunc.sql +++ b/integration_tests/models/cross_db_utils/test_date_trunc.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_date_trunc') }} - -) +with data as (select * from {{ ref("data_date_trunc") }}) select - cast({{dbt_utils.date_trunc('day', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("day", "updated_at") }} as date) as actual, day as expected from data @@ -14,7 +10,7 @@ from data union all select - cast({{ dbt_utils.date_trunc('month', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("month", "updated_at") }} as date) as actual, month as expected from data diff --git a/integration_tests/models/cross_db_utils/test_dateadd.sql b/integration_tests/models/cross_db_utils/test_dateadd.sql index 3e199d86..2f2dee99 100644 --- a/integration_tests/models/cross_db_utils/test_dateadd.sql +++ b/integration_tests/models/cross_db_utils/test_dateadd.sql @@ -1,16 +1,32 @@ -with data as ( - - select * from {{ ref('data_dateadd') }} - -) +with data as (select * from {{ ref("data_dateadd") }}) select case - when datepart = 'hour' then cast({{ dbt_utils.dateadd('hour', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'day' then cast({{ dbt_utils.dateadd('day', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'month' then cast({{ dbt_utils.dateadd('month', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'year' then cast({{ dbt_utils.dateadd('year', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) + when datepart = 'hour' + then + cast( + {{ dbt_utils.dateadd("hour", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'day' + then + cast( + {{ dbt_utils.dateadd("day", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'month' + then + cast( + {{ dbt_utils.dateadd("month", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'year' + then + cast( + {{ dbt_utils.dateadd("year", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_datediff.sql b/integration_tests/models/cross_db_utils/test_datediff.sql index a2340456..4b6c95a6 100644 --- a/integration_tests/models/cross_db_utils/test_datediff.sql +++ b/integration_tests/models/cross_db_utils/test_datediff.sql @@ -1,20 +1,23 @@ -with data as ( - - select * from {{ ref('data_datediff') }} - -) +with data as (select * from {{ ref("data_datediff") }}) select case - when datepart = 'second' then {{ dbt_utils.datediff('first_date', 'second_date', 'second') }} - when datepart = 'minute' then {{ dbt_utils.datediff('first_date', 'second_date', 'minute') }} - when datepart = 'hour' then {{ dbt_utils.datediff('first_date', 'second_date', 'hour') }} - when datepart = 'day' then {{ dbt_utils.datediff('first_date', 'second_date', 'day') }} - when datepart = 'week' then {{ dbt_utils.datediff('first_date', 'second_date', 'week') }} - when datepart = 'month' then {{ dbt_utils.datediff('first_date', 'second_date', 'month') }} - when datepart = 'year' then {{ dbt_utils.datediff('first_date', 'second_date', 'year') }} + when datepart = 'second' + then {{ dbt_utils.datediff("first_date", "second_date", "second") }} + when datepart = 'minute' + then {{ dbt_utils.datediff("first_date", "second_date", "minute") }} + when datepart = 'hour' + then {{ dbt_utils.datediff("first_date", "second_date", "hour") }} + when datepart = 'day' + then {{ dbt_utils.datediff("first_date", "second_date", "day") }} + when datepart = 'week' + then {{ dbt_utils.datediff("first_date", "second_date", "week") }} + when datepart = 'month' + then {{ dbt_utils.datediff("first_date", "second_date", "month") }} + when datepart = 'year' + then {{ dbt_utils.datediff("first_date", "second_date", "year") }} else null end as actual, result as expected @@ -22,14 +25,77 @@ select from data -- Also test correct casting of literal values. - -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "microsecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "millisecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "microsecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "millisecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year" + ) + }} as actual, 1 as expected diff --git a/integration_tests/models/cross_db_utils/test_hash.sql b/integration_tests/models/cross_db_utils/test_hash.sql index 8f017780..6d526274 100644 --- a/integration_tests/models/cross_db_utils/test_hash.sql +++ b/integration_tests/models/cross_db_utils/test_hash.sql @@ -1,12 +1,6 @@ -with data as ( +with data as (select * from {{ ref("data_hash") }}) - select * from {{ ref('data_hash') }} - -) - -select - {{ dbt_utils.hash('input_1') }} as actual, - output as expected +select {{ dbt_utils.hash("input_1") }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_last_day.sql b/integration_tests/models/cross_db_utils/test_last_day.sql index fee3a823..4e6cec97 100644 --- a/integration_tests/models/cross_db_utils/test_last_day.sql +++ b/integration_tests/models/cross_db_utils/test_last_day.sql @@ -1,15 +1,14 @@ -with data as ( - - select * from {{ ref('data_last_day') }} - -) +with data as (select * from {{ ref("data_last_day") }}) select case - when date_part = 'month' then {{ dbt_utils.last_day('date_day', 'month') }} - when date_part = 'quarter' then {{ dbt_utils.last_day('date_day', 'quarter') }} - when date_part = 'year' then {{ dbt_utils.last_day('date_day', 'year') }} + when date_part = 'month' + then {{ dbt_utils.last_day("date_day", "month") }} + when date_part = 'quarter' + then {{ dbt_utils.last_day("date_day", "quarter") }} + when date_part = 'year' + then {{ dbt_utils.last_day("date_day", "year") }} else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_length.sql b/integration_tests/models/cross_db_utils/test_length.sql index f4ef099c..d6352d05 100644 --- a/integration_tests/models/cross_db_utils/test_length.sql +++ b/integration_tests/models/cross_db_utils/test_length.sql @@ -1,12 +1,5 @@ -with data as ( +with data as (select * from {{ ref("data_length") }}) - select * from {{ ref('data_length') }} +select {{ dbt_utils.length("expression") }} as actual, output as expected -) - -select - - {{ dbt_utils.length('expression') }} as actual, - output as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_listagg.sql b/integration_tests/models/cross_db_utils/test_listagg.sql index 006948de..27d96745 100644 --- a/integration_tests/models/cross_db_utils/test_listagg.sql +++ b/integration_tests/models/cross_db_utils/test_listagg.sql @@ -1,69 +1,63 @@ -with data as ( - - select * from {{ ref('data_listagg') }} - -), - -data_output as ( - - select * from {{ ref('data_listagg_output') }} - -), - -calculate as ( - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col") }} as actual, - 'bottom_ordered' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col", 2) }} as actual, - 'bottom_ordered_limited' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "', '") }} as actual, - 'comma_whitespace_unordered' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('DISTINCT string_text', "','") }} as actual, - 'distinct_comma' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text') }} as actual, - 'no_params' as version - from data - where group_col = 3 - group by group_col - -) - -select - calculate.actual, - data_output.expected +with + data as (select * from {{ ref("data_listagg") }}), + + data_output as (select * from {{ ref("data_listagg_output") }}), + + calculate as ( + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col") }} + as actual, + 'bottom_ordered' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col", 2) }} + as actual, + 'bottom_ordered_limited' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "', '") }} as actual, + 'comma_whitespace_unordered' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("DISTINCT string_text", "','") }} as actual, + 'distinct_comma' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text") }} as actual, + 'no_params' as version + from data + where group_col = 3 + group by group_col + + ) + +select calculate.actual, data_output.expected from calculate -left join data_output -on calculate.group_col = data_output.group_col -and calculate.version = data_output.version \ No newline at end of file +left join + data_output + on calculate.group_col = data_output.group_col + and calculate.version = data_output.version diff --git a/integration_tests/models/cross_db_utils/test_position.sql b/integration_tests/models/cross_db_utils/test_position.sql index b09be66d..d1785f14 100644 --- a/integration_tests/models/cross_db_utils/test_position.sql +++ b/integration_tests/models/cross_db_utils/test_position.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_position') }} - -) +with data as (select * from {{ ref("data_position") }}) select - {{ dbt_utils.position('substring_text', 'string_text') }} as actual, + {{ dbt_utils.position("substring_text", "string_text") }} as actual, result as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_replace.sql b/integration_tests/models/cross_db_utils/test_replace.sql index 6c8a8aa1..0e413c8a 100644 --- a/integration_tests/models/cross_db_utils/test_replace.sql +++ b/integration_tests/models/cross_db_utils/test_replace.sql @@ -1,18 +1,19 @@ -with data as ( +with + data as ( - select - - *, - coalesce(search_chars, '') as old_chars, - coalesce(replace_chars, '') as new_chars - - from {{ ref('data_replace') }} + select -) + *, + coalesce(search_chars, '') as old_chars, + coalesce(replace_chars, '') as new_chars + + from {{ ref("data_replace") }} + + ) select - {{ dbt_utils.replace('string_text', 'old_chars', 'new_chars') }} as actual, + {{ dbt_utils.replace("string_text", "old_chars", "new_chars") }} as actual, result as expected from data diff --git a/integration_tests/models/cross_db_utils/test_right.sql b/integration_tests/models/cross_db_utils/test_right.sql index eaad4f4f..de2df7dc 100644 --- a/integration_tests/models/cross_db_utils/test_right.sql +++ b/integration_tests/models/cross_db_utils/test_right.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_right') }} - -) +with data as (select * from {{ ref("data_right") }}) select - {{ dbt_utils.right('string_text', 'length_expression') }} as actual, + {{ dbt_utils.right("string_text", "length_expression") }} as actual, coalesce(output, '') as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_safe_cast.sql b/integration_tests/models/cross_db_utils/test_safe_cast.sql index 8ed74fc3..b2e6d82c 100644 --- a/integration_tests/models/cross_db_utils/test_safe_cast.sql +++ b/integration_tests/models/cross_db_utils/test_safe_cast.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_safe_cast') }} - -) +with data as (select * from {{ ref("data_safe_cast") }}) select - {{ dbt_utils.safe_cast('field', dbt_utils.type_string()) }} as actual, + {{ dbt_utils.safe_cast("field", dbt_utils.type_string()) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_split_part.sql b/integration_tests/models/cross_db_utils/test_split_part.sql index 6a10327c..05c0d70f 100644 --- a/integration_tests/models/cross_db_utils/test_split_part.sql +++ b/integration_tests/models/cross_db_utils/test_split_part.sql @@ -1,28 +1,21 @@ -with data as ( - - select * from {{ ref('data_split_part') }} - -) +with data as (select * from {{ ref("data_split_part") }}) select - {{ dbt_utils.split_part('parts', 'split_on', 1) }} as actual, - result_1 as expected + {{ dbt_utils.split_part("parts", "split_on", 1) }} as actual, result_1 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 2) }} as actual, - result_2 as expected + {{ dbt_utils.split_part("parts", "split_on", 2) }} as actual, result_2 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 3) }} as actual, - result_3 as expected + {{ dbt_utils.split_part("parts", "split_on", 3) }} as actual, result_3 as expected from data diff --git a/integration_tests/models/cross_db_utils/test_width_bucket.sql b/integration_tests/models/cross_db_utils/test_width_bucket.sql index dc2b8609..c2b79646 100644 --- a/integration_tests/models/cross_db_utils/test_width_bucket.sql +++ b/integration_tests/models/cross_db_utils/test_width_bucket.sql @@ -1,12 +1,9 @@ -with data as ( - - select * from {{ ref('data_width_bucket') }} - -) +with data as (select * from {{ ref("data_width_bucket") }}) select - {{ dbt_utils.width_bucket('amount', 'min_value', 'max_value', 'num_buckets') }} as actual, + {{ dbt_utils.width_bucket("amount", "min_value", "max_value", "num_buckets") }} + as actual, bucket as expected from data diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index fa4ae52b..f2c2dd12 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -3,24 +3,22 @@ -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} -{{ config(materialized='table') }} +with + date_spine as ( -with date_spine as ( - - {% if target.type == 'postgres' %} + {% if target.type == "postgres" %} {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} - - {% elif target.type == 'bigquery' %} + + {% elif target.type == "bigquery" %} select cast(date_day as date) as date_day from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) - - {% else %} - {{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} - {% endif %} -) + {% else %}{{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} + {% endif %} + + ) select date_day from date_spine - diff --git a/integration_tests/models/generic_tests/test_equal_column_subset.sql b/integration_tests/models/generic_tests/test_equal_column_subset.sql index b426bb63..7ecae303 100644 --- a/integration_tests/models/generic_tests/test_equal_column_subset.sql +++ b/integration_tests/models/generic_tests/test_equal_column_subset.sql @@ -1,9 +1,5 @@ -{{ config(materialized='ephemeral') }} +{{ config(materialized="ephemeral") }} -select +select first_name, last_name, email - first_name, - last_name, - email - -from {{ ref('data_people') }} +from {{ ref("data_people") }} diff --git a/integration_tests/models/generic_tests/test_equal_rowcount.sql b/integration_tests/models/generic_tests/test_equal_rowcount.sql index 01b1a549..621ccf61 100644 --- a/integration_tests/models/generic_tests/test_equal_rowcount.sql +++ b/integration_tests/models/generic_tests/test_equal_rowcount.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_equal_rowcount") }}) - select * from {{ ref('data_test_equal_rowcount') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_fewer_rows_than.sql b/integration_tests/models/generic_tests/test_fewer_rows_than.sql index c2ad4cbd..621fce6c 100644 --- a/integration_tests/models/generic_tests/test_fewer_rows_than.sql +++ b/integration_tests/models/generic_tests/test_fewer_rows_than.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_fewer_rows_than_table_1") }}) - select * from {{ ref('data_test_fewer_rows_than_table_1') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_recency.sql b/integration_tests/models/generic_tests/test_recency.sql index d300e572..142490f4 100644 --- a/integration_tests/models/generic_tests/test_recency.sql +++ b/integration_tests/models/generic_tests/test_recency.sql @@ -1,12 +1,13 @@ -{% if target.type == 'postgres' %} +{% if target.type == "postgres" %} -select - {{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as today +select {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as today {% else %} select - cast({{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as datetime) as today - + cast( + {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as datetime + ) as today + {% endif %} diff --git a/integration_tests/models/geo/test_haversine_distance_km.sql b/integration_tests/models/geo/test_haversine_distance_km.sql index e8ca4818..fce70682 100644 --- a/integration_tests/models/geo/test_haversine_distance_km.sql +++ b/integration_tests/models/geo/test_haversine_distance_km.sql @@ -1,23 +1,20 @@ -with data as ( - select * from {{ ref('data_haversine_km') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='km' +with + data as (select * from {{ ref("data_haversine_km") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="km", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/geo/test_haversine_distance_mi.sql b/integration_tests/models/geo/test_haversine_distance_mi.sql index 53f8172c..9ff4a1f7 100644 --- a/integration_tests/models/geo/test_haversine_distance_mi.sql +++ b/integration_tests/models/geo/test_haversine_distance_mi.sql @@ -1,39 +1,36 @@ -with data as ( - select * from {{ ref('data_haversine_mi') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='mi' +with + data as (select * from {{ ref("data_haversine_mi") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="mi", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data - union all + union all - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/materializations/expected_insert_by_period.sql b/integration_tests/models/materializations/expected_insert_by_period.sql index 0a198b6b..0d7d68c1 100644 --- a/integration_tests/models/materializations/expected_insert_by_period.sql +++ b/integration_tests/models/materializations/expected_insert_by_period.sql @@ -1,10 +1,5 @@ -{{ - config( - materialized = 'view', - enabled=(target.type == 'redshift') - ) -}} +{{ config(materialized="view", enabled=(target.type == "redshift")) }} select * -from {{ ref('data_insert_by_period') }} +from {{ ref("data_insert_by_period") }} where id in (2, 3, 4, 5, 6) diff --git a/integration_tests/models/materializations/test_insert_by_period.sql b/integration_tests/models/materializations/test_insert_by_period.sql index 8cc396d3..b48d22f6 100644 --- a/integration_tests/models/materializations/test_insert_by_period.sql +++ b/integration_tests/models/materializations/test_insert_by_period.sql @@ -1,18 +1,16 @@ {{ - config( - materialized = 'insert_by_period', - period = 'month', - timestamp_field = 'created_at', - start_date = '2018-01-01', - stop_date = '2018-06-01', - enabled=(target.type == 'redshift') - ) + config( + materialized="insert_by_period", + period="month", + timestamp_field="created_at", + start_date="2018-01-01", + stop_date="2018-06-01", + enabled=(target.type == "redshift"), + ) }} -with events as ( - select * - from {{ ref('data_insert_by_period') }} - where __PERIOD_FILTER__ -) +with + events as (select * from {{ ref("data_insert_by_period") }} where __period_filter__) -select * from events +select * +from events diff --git a/integration_tests/models/sql/test_deduplicate.sql b/integration_tests/models/sql/test_deduplicate.sql index 81fe81e7..47649a61 100644 --- a/integration_tests/models/sql/test_deduplicate.sql +++ b/integration_tests/models/sql/test_deduplicate.sql @@ -1,22 +1,19 @@ with -source as ( - select * - from {{ ref('data_deduplicate') }} - where user_id = 1 -), + source as (select * from {{ ref("data_deduplicate") }} where user_id = 1), -deduped as ( + deduped as ( - {{ - dbt_utils.deduplicate( - ref('data_deduplicate'), - group_by='user_id', - order_by='version desc', - relation_alias="source" - ) | indent - }} + {{ + dbt_utils.deduplicate( + ref("data_deduplicate"), + group_by="user_id", + order_by="version desc", + relation_alias="source", + ) | indent + }} -) + ) -select * from deduped +select * +from deduped diff --git a/integration_tests/models/sql/test_generate_series.sql b/integration_tests/models/sql/test_generate_series.sql index 11370b7b..2e5acf19 100644 --- a/integration_tests/models/sql/test_generate_series.sql +++ b/integration_tests/models/sql/test_generate_series.sql @@ -3,13 +3,9 @@ -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} -{{ config(materialized='table') }} +with data as ({{ dbt_utils.generate_series(10) }}) -with data as ( - - {{ dbt_utils.generate_series(10) }} - -) - -select generated_number from data +select generated_number +from data diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index 36214984..fe91fa57 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -1,29 +1,32 @@ -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values"), "field", default=[], order_by="field" +) %} -{% if target.type == 'snowflake' %} +{% if target.type == "snowflake" %} select {% for val in column_values -%} - sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} - {%- if not loop.last %},{% endif -%} + sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} + {%- if not loop.last %},{% endif -%} {%- endfor %} -from {{ ref('data_get_column_values') }} +from {{ ref("data_get_column_values") }} {% else %} select {% for val in column_values -%} - {{dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} as count_{{ val }} - {%- if not loop.last %},{% endif -%} + {{ dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} + as count_{{ val }} + {%- if not loop.last %},{% endif -%} {%- endfor %} -from {{ ref('data_get_column_values') }} +from {{ ref("data_get_column_values") }} {% endif %} diff --git a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql index 7b3ca72f..af04d517 100644 --- a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql +++ b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql @@ -1,16 +1,19 @@ -{% set exclude_field = 'field_1' %} +{% set exclude_field = "field_1" %} {% set column_names = dbt_utils.get_filtered_columns_in_relation(from= ref('data_filtered_columns_in_relation'), except=[exclude_field]) %} -with data as ( +with + data as ( - select + select - {% for column_name in column_names %} - max({{ column_name }}) as {{ column_name }} {% if not loop.last %},{% endif %} - {% endfor %} + {% for column_name in column_names %} + max({{ column_name }}) as {{ column_name }} + {% if not loop.last %},{% endif %} + {% endfor %} - from {{ ref('data_filtered_columns_in_relation') }} + from {{ ref("data_filtered_columns_in_relation") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index 52ab07a2..2c20559e 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,16 +1,9 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} {% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %} -with unioned as ( - - {{ dbt_utils.union_relations(relations) }} - -) +with unioned as ({{ dbt_utils.union_relations(relations) }}) -select - - user_id, - event +select user_id, event from unioned diff --git a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql index 481f233e..6e968e79 100644 --- a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql +++ b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql @@ -1,4 +1,4 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} -{% set relations = dbt_utils.get_relations_by_prefix(target.schema, 'data_events_') %} +{% set relations = dbt_utils.get_relations_by_prefix(target.schema, "data_events_") %} {{ dbt_utils.union_relations(relations) }} diff --git a/integration_tests/models/sql/test_groupby.sql b/integration_tests/models/sql/test_groupby.sql index 50a6ced7..a63f3a5c 100644 --- a/integration_tests/models/sql/test_groupby.sql +++ b/integration_tests/models/sql/test_groupby.sql @@ -1,24 +1,14 @@ -with test_data as ( - - select - - {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string() )}} as column_1, - {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string() )}} as column_2 - -), +with + test_data as ( -grouped as ( + select - select - *, - count(*) as total - - from test_data - {{ dbt_utils.group_by(2) }} - -) - -select * from grouped + {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string()) }} as column_1, + {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string()) }} as column_2 + ), + grouped as (select *, count(*) as total from test_data {{ dbt_utils.group_by(2) }}) +select * +from grouped diff --git a/integration_tests/models/sql/test_nullcheck_table.sql b/integration_tests/models/sql/test_nullcheck_table.sql index ea66c8ec..4eb60e4e 100644 --- a/integration_tests/models/sql/test_nullcheck_table.sql +++ b/integration_tests/models/sql/test_nullcheck_table.sql @@ -1,17 +1,12 @@ -{{ config( materialized = "table" ) }} - --- TO DO: remove if-statement - -{% set tbl = ref('data_nullcheck_table') %} - +{{ config(materialized="table") }} -with nulled as ( +-- TO DO: remove if-statement +{% set tbl = ref("data_nullcheck_table") %} - {{ dbt_utils.nullcheck_table(tbl) }} -) +with nulled as ({{ dbt_utils.nullcheck_table(tbl) }}) -{% if target.type == 'snowflake' %} +{% if target.type == "snowflake" %} select field_1::varchar as field_1, @@ -24,17 +19,11 @@ from nulled select - {{ dbt_utils.safe_cast('field_1', - dbt_utils.type_string() - )}} as field_1, + {{ dbt_utils.safe_cast("field_1", dbt_utils.type_string()) }} as field_1, - {{ dbt_utils.safe_cast('field_2', - dbt_utils.type_string() - )}} as field_2, + {{ dbt_utils.safe_cast("field_2", dbt_utils.type_string()) }} as field_2, - {{ dbt_utils.safe_cast('field_3', - dbt_utils.type_string() - )}} as field_3 + {{ dbt_utils.safe_cast("field_3", dbt_utils.type_string()) }} as field_3 from nulled diff --git a/integration_tests/models/sql/test_pivot.sql b/integration_tests/models/sql/test_pivot.sql index a63e6e3b..730b7bec 100644 --- a/integration_tests/models/sql/test_pivot.sql +++ b/integration_tests/models/sql/test_pivot.sql @@ -1,17 +1,11 @@ -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE'] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue'] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} +{% set column_values = ["RED", "BLUE"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue"] %} {% set cmp = "=" %} {% endif %} -select - size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp) }} +select size, {{ dbt_utils.pivot("color", column_values, cmp=cmp) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_pivot_apostrophe.sql b/integration_tests/models/sql/test_pivot_apostrophe.sql index 792f4686..e7a340a3 100644 --- a/integration_tests/models/sql/test_pivot_apostrophe.sql +++ b/integration_tests/models/sql/test_pivot_apostrophe.sql @@ -1,17 +1,13 @@ -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE', "BLUE'S"] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue', "blue's"] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} +{% set column_values = ["RED", "BLUE", "BLUE'S"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue", "blue's"] %} {% set cmp = "=" %} {% endif %} select size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp, quote_identifiers=False) }} + {{ dbt_utils.pivot("color", column_values, cmp=cmp, quote_identifiers=False) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_safe_add.sql b/integration_tests/models/sql/test_safe_add.sql index af3894d0..840871ff 100644 --- a/integration_tests/models/sql/test_safe_add.sql +++ b/integration_tests/models/sql/test_safe_add.sql @@ -1,12 +1,6 @@ -with data as ( +with data as (select * from {{ ref("data_safe_add") }}) - select * from {{ ref('data_safe_add') }} - -) - -select - {{ dbt_utils.safe_add('field_1', 'field_2', 'field_3') }} as actual, - expected +select {{ dbt_utils.safe_add("field_1", "field_2", "field_3") }} as actual, expected from data diff --git a/integration_tests/models/sql/test_star.sql b/integration_tests/models/sql/test_star.sql index 2092e16a..74566119 100644 --- a/integration_tests/models/sql/test_star.sql +++ b/integration_tests/models/sql/test_star.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'field_3' %} +{% set exclude_field = "field_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_aggregate.sql b/integration_tests/models/sql/test_star_aggregate.sql index 9dcd7c2d..45085a04 100644 --- a/integration_tests/models/sql/test_star_aggregate.sql +++ b/integration_tests/models/sql/test_star_aggregate.sql @@ -1,16 +1,15 @@ -/*This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ - +/* This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ {% set selected_columns = dbt_utils.star(from=ref('data_star_aggregate'), except=['value_field']) %} -with data as ( +with + data as ( - select - {{ selected_columns }}, - sum(value_field) as value_field_sum + select {{ selected_columns }}, sum(value_field) as value_field_sum - from {{ ref('data_star_aggregate') }} - group by {{ selected_columns }} + from {{ ref("data_star_aggregate") }} + group by {{ selected_columns }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_prefix_suffix.sql b/integration_tests/models/sql/test_star_prefix_suffix.sql index ae1c64fe..69794309 100644 --- a/integration_tests/models/sql/test_star_prefix_suffix.sql +++ b/integration_tests/models/sql/test_star_prefix_suffix.sql @@ -1,13 +1,15 @@ -{% set prefix_with = 'prefix_' if target.type != 'snowflake' else 'PREFIX_' %} -{% set suffix_with = '_suffix' if target.type != 'snowflake' else '_SUFFIX' %} +{% set prefix_with = "prefix_" if target.type != "snowflake" else "PREFIX_" %} +{% set suffix_with = "_suffix" if target.type != "snowflake" else "_SUFFIX" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} + select + {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data \ No newline at end of file +select * +from data diff --git a/integration_tests/models/sql/test_star_uppercase.sql b/integration_tests/models/sql/test_star_uppercase.sql index 6179e691..be5b69fe 100644 --- a/integration_tests/models/sql/test_star_uppercase.sql +++ b/integration_tests/models/sql/test_star_uppercase.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'FIELD_3' %} +{% set exclude_field = "FIELD_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_surrogate_key.sql b/integration_tests/models/sql/test_surrogate_key.sql index fd90939f..a0da48ef 100644 --- a/integration_tests/models/sql/test_surrogate_key.sql +++ b/integration_tests/models/sql/test_surrogate_key.sql @@ -1,15 +1,13 @@ -with data as ( - - select * from {{ ref('data_surrogate_key') }} - -) +with data as (select * from {{ ref("data_surrogate_key") }}) select - {{ dbt_utils.surrogate_key('column_1') }} as actual_column_1_only, + {{ dbt_utils.surrogate_key("column_1") }} as actual_column_1_only, expected_column_1_only, - {{ dbt_utils.surrogate_key('column_1', 'column_2', 'column_3') }} as actual_all_columns_arguments, - {{ dbt_utils.surrogate_key(['column_1', 'column_2', 'column_3']) }} as actual_all_columns_list, + {{ dbt_utils.surrogate_key("column_1", "column_2", "column_3") }} + as actual_all_columns_arguments, + {{ dbt_utils.surrogate_key(["column_1", "column_2", "column_3"]) }} + as actual_all_columns_list, expected_all_columns from data diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql index 69836833..1b14ff1f 100644 --- a/integration_tests/models/sql/test_union.sql +++ b/integration_tests/models/sql/test_union.sql @@ -1,8 +1,2 @@ -select - id, - name, - favorite_color - -from {{ ref('test_union_base') }} - +select id, name, favorite_color from {{ ref("test_union_base") }} diff --git a/integration_tests/models/sql/test_union_base.sql b/integration_tests/models/sql/test_union_base.sql index a51bd2b8..045e651b 100644 --- a/integration_tests/models/sql/test_union_base.sql +++ b/integration_tests/models/sql/test_union_base.sql @@ -1,5 +1,2 @@ -{{ dbt_utils.union_relations([ - ref('data_union_table_1'), - ref('data_union_table_2')] -) }} +{{ dbt_utils.union_relations([ref("data_union_table_1"), ref("data_union_table_2")]) }} diff --git a/integration_tests/models/sql/test_unpivot.sql b/integration_tests/models/sql/test_unpivot.sql index 8cf1ad6e..7198d0a3 100644 --- a/integration_tests/models/sql/test_unpivot.sql +++ b/integration_tests/models/sql/test_unpivot.sql @@ -4,30 +4,27 @@ -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - remove=['name'], - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + remove=["name"], + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_bool.sql b/integration_tests/models/sql/test_unpivot_bool.sql index 26842c01..82d7205b 100644 --- a/integration_tests/models/sql/test_unpivot_bool.sql +++ b/integration_tests/models/sql/test_unpivot_bool.sql @@ -4,29 +4,26 @@ -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot_bool'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot_bool"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_original_api.sql b/integration_tests/models/sql/test_unpivot_original_api.sql index f6b9395a..c10d016e 100644 --- a/integration_tests/models/sql/test_unpivot_original_api.sql +++ b/integration_tests/models/sql/test_unpivot_original_api.sql @@ -1,32 +1,30 @@ -- unpivot() was enhanced with 3 new parameters -- This test targets the original API. - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, case - when '{{ target.name }}' = 'snowflake' then lower(FIELD_NAME) - else field_name + when '{{ target.name }}' = 'snowflake' then lower(field_name) else field_name end as field_name, value -from ( - {{ dbt_utils.unpivot( - table=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + table=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + ) + }} + ) as sbq diff --git a/integration_tests/models/web/test_url_host.sql b/integration_tests/models/web/test_url_host.sql index 06a9f69c..2e18e65d 100644 --- a/integration_tests/models/web/test_url_host.sql +++ b/integration_tests/models/web/test_url_host.sql @@ -1,12 +1,5 @@ -with data as ( - - select * from {{ref('data_url_host')}} - -) +with data as (select * from {{ ref("data_url_host") }}) -select +select {{ dbt_utils.get_url_host("original_url") }} as actual, parsed_url as expected - {{ dbt_utils.get_url_host('original_url') }} as actual, - parsed_url as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/web/test_url_path.sql b/integration_tests/models/web/test_url_path.sql index c03e5547..16f45ae0 100644 --- a/integration_tests/models/web/test_url_path.sql +++ b/integration_tests/models/web/test_url_path.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ref('data_url_path')}} - -) +with data as (select * from {{ ref("data_url_path") }}) select - coalesce({{ dbt_utils.get_url_path('original_url') }}, '') as actual, + coalesce({{ dbt_utils.get_url_path("original_url") }}, '') as actual, coalesce(parsed_path, '') as expected - -from data \ No newline at end of file + +from data diff --git a/integration_tests/models/web/test_urls.sql b/integration_tests/models/web/test_urls.sql index 60353dbb..066a0fcd 100644 --- a/integration_tests/models/web/test_urls.sql +++ b/integration_tests/models/web/test_urls.sql @@ -1,20 +1,14 @@ -with data as ( - - select * from {{ ref('data_urls') }} - -) +with data as (select * from {{ ref("data_urls") }}) select - {{ dbt_utils.get_url_parameter('url', 'utm_medium') }} as actual, - medium as expected + {{ dbt_utils.get_url_parameter("url", "utm_medium") }} as actual, medium as expected from data union all select - {{ dbt_utils.get_url_parameter('url', 'utm_source') }} as actual, - source as expected + {{ dbt_utils.get_url_parameter("url", "utm_source") }} as actual, source as expected from data diff --git a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql index 55d5400e..eec6203c 100644 --- a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql +++ b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql @@ -1,17 +1,16 @@ -- depends_on: {{ ref('data_get_query_results_as_dict') }} - -{% set expected_dictionary={ - 'col_1': [1, 2, 3], - 'col_2': ['a', 'b', 'c'], - 'col_3': [True, False, none] +{% set expected_dictionary = { + "col_1": [1, 2, 3], + "col_2": ["a", "b", "c"], + "col_3": [True, False, none], } %} {#- Handle snowflake casing silliness -#} -{% if target.type == 'snowflake' %} -{% set expected_dictionary={ - 'COL_1': [1, 2, 3], - 'COL_2': ['a', 'b', 'c'], - 'COL_3': [True, False, none] +{% if target.type == "snowflake" %} +{% set expected_dictionary = { + "COL_1": [1, 2, 3], + "COL_2": ["a", "b", "c"], + "COL_3": [True, False, none], } %} {% endif %} @@ -23,18 +22,17 @@ For reasons that remain unclear, Jinja won't return True for actual_dictionary == expected_dictionary. Instead, we'll manually check that the values of these dictionaries are equivalent. -#} - {% set ns = namespace( pass=True, err_msg = "" ) %} {% if execute %} {#- Check that the dictionaries have the same keys -#} -{% set expected_keys=expected_dictionary.keys() | list | sort %} -{% set actual_keys=actual_dictionary.keys() | list | sort %} +{% set expected_keys = expected_dictionary.keys() | list | sort %} +{% set actual_keys = actual_dictionary.keys() | list | sort %} {% if expected_keys != actual_keys %} - {% set ns.pass=False %} +{% set ns.pass=False %} {% set ns.err_msg %} The two dictionaries have different keys: expected_dictionary has keys: {{ expected_keys }} @@ -44,33 +42,33 @@ Instead, we'll manually check that the values of these dictionaries are equivale {% else %} {% for key, value in expected_dictionary.items() %} - {% set expected_length=expected_dictionary[key] | length %} - {% set actual_length=actual_dictionary[key] | length %} +{% set expected_length = expected_dictionary[key] | length %} +{% set actual_length = actual_dictionary[key] | length %} - {% if expected_length != actual_length %} - {% set ns.pass=False %} +{% if expected_length != actual_length %} +{% set ns.pass=False %} {% set ns.err_msg %} The {{ key }} column has different lengths: expected_dictionary[{{ key }}] has length {{ expected_length }} actual_dictionary[{{ key }}] has length {{ actual_length }} {% endset %} - {% else %} +{% else %} - {% for i in range(value | length) %} - {% set expected_value=expected_dictionary[key][i] %} - {% set actual_value=actual_dictionary[key][i] %} - {% if expected_value != actual_value %} - {% set ns.pass=False %} +{% for i in range(value | length) %} +{% set expected_value = expected_dictionary[key][i] %} +{% set actual_value = actual_dictionary[key][i] %} +{% if expected_value != actual_value %} +{% set ns.pass=False %} {% set ns.err_msg %} The {{ key }} column has differing values: expected_dictionary[{{ key }}][{{ i }}] == {{ expected_value }} actual_dictionary[{{ key }}][{{ i }}] == {{ actual_value }} {% endset %} - {% endif %} - {% endfor %} - {% endif %} +{% endif %} +{% endfor %} +{% endif %} {% endfor %} diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql index fea33594..6782e8d1 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql @@ -1,7 +1,5 @@ -{% if dbt_utils.pretty_log_format() is string %} - {# Return 0 rows for the test to pass #} - select 1 as col_name {{ limit_zero() }} -{% else %} - {# Return >0 rows for the test to fail #} - select 1 +{# Return 0 rows for the test to pass #} +{% if dbt_utils.pretty_log_format() is string %} select 1 as col_name {{ limit_zero() }} +{# Return >0 rows for the test to fail #} +{% else %} select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql index a6347abd..a9ec4b19 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql @@ -1,7 +1,5 @@ -{% if dbt_utils.pretty_time() is string %} - {# Return 0 rows for the test to pass #} - select 1 as col_name {{ limit_zero() }} -{% else %} - {# Return >0 rows for the test to fail #} - select 1 +{# Return 0 rows for the test to pass #} +{% if dbt_utils.pretty_time() is string %} select 1 as col_name {{ limit_zero() }} +{# Return >0 rows for the test to fail #} +{% else %} select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/test_slugify.sql b/integration_tests/tests/jinja_helpers/test_slugify.sql index c0839f59..13d0a2bc 100644 --- a/integration_tests/tests/jinja_helpers/test_slugify.sql +++ b/integration_tests/tests/jinja_helpers/test_slugify.sql @@ -1,7 +1,6 @@ -{% if dbt_utils.slugify('!Hell0 world-hi') == 'hell0_world_hi' %} - {# Return 0 rows for the test to pass #} - select 1 as col_name {{ limit_zero() }} -{% else %} - {# Return >0 rows for the test to fail #} - select 1 as col_name +{% if dbt_utils.slugify("!Hell0 world-hi") == "hell0_world_hi" %} +{# Return 0 rows for the test to pass #} +select 1 as col_name {{ limit_zero() }} +{# Return >0 rows for the test to fail #} +{% else %} select 1 as col_name {% endif %} diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 7f9fa3a4..0fcd8ac0 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -1,26 +1,40 @@ -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values_dropped"), + "field", + default=["y", "z"], + order_by="field", +) %} -with expected as ( - select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected_column_value union all - select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected_column_value -), +with + expected as ( + select + {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} + as expected_column_value + union all + select + {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} + as expected_column_value + ), + + actual as ( -actual as ( - {% for val in column_values %} - select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual_column_value - {% if not loop.last %} - union all - {% endif %} + select + {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} + as actual_column_value + {% if not loop.last %} union all {% endif %} {% endfor %} -), + ), -failures as ( - select * from actual - where actual.actual_column_value not in ( - select expected.expected_column_value from expected + failures as ( + select * + from actual + where + actual.actual_column_value not in ( + select expected.expected_column_value from expected + ) ) -) -select * from failures \ No newline at end of file +select * +from failures diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index 34c70db9..54317376 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -1,16 +1,15 @@ {% macro _is_ephemeral(obj, macro) %} - {%- if obj.is_cte -%} - {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} - {% if obj.name.startswith(ephemeral_prefix) %} - {% set model_name = obj.name[(ephemeral_prefix|length):] %} - {% else %} - {% set model_name = obj.name %} - {%- endif -%} +{%- if obj.is_cte -%} +{% set ephemeral_prefix = api.Relation.add_ephemeral_prefix("") %} +{% if obj.name.startswith(ephemeral_prefix) %} +{% set model_name = obj.name[(ephemeral_prefix | length) :] %} +{% else %} {% set model_name = obj.name %} +{%- endif -%} {% set error_message %} The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. `{{ model_name }}` is an ephemeral model. Consider making it a view or table instead. {% endset %} - {%- do exceptions.raise_compiler_error(error_message) -%} - {%- endif -%} +{%- do exceptions.raise_compiler_error(error_message) -%} +{%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/_is_relation.sql b/macros/cross_db_utils/_is_relation.sql index bee6db97..af01c32c 100644 --- a/macros/cross_db_utils/_is_relation.sql +++ b/macros/cross_db_utils/_is_relation.sql @@ -1,5 +1,8 @@ {% macro _is_relation(obj, macro) %} - {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%} - {%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} - {%- endif -%} +{%- if not ( + obj is mapping + and obj.get("metadata", {}).get("type", "").endswith("Relation") +) -%} +{%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} +{%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/any_value.sql b/macros/cross_db_utils/any_value.sql index 78cb75ba..5ee1c0e9 100644 --- a/macros/cross_db_utils/any_value.sql +++ b/macros/cross_db_utils/any_value.sql @@ -1,17 +1,10 @@ {% macro any_value(expression) -%} - {{ return(adapter.dispatch('any_value', 'dbt_utils') (expression)) }} +{{ return adapter.dispatch("any_value", "dbt_utils")(expression) }} {% endmacro %} -{% macro default__any_value(expression) -%} - - any_value({{ expression }}) - -{%- endmacro %} +{% macro default__any_value(expression) -%} any_value({{ expression }}) {%- endmacro %} -{% macro postgres__any_value(expression) -%} - {#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} - min({{ expression }}) - -{%- endmacro %} \ No newline at end of file +{#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} +{% macro postgres__any_value(expression) -%} min({{ expression }}) {%- endmacro %} diff --git a/macros/cross_db_utils/bool_or.sql b/macros/cross_db_utils/bool_or.sql index ce0a6857..8187b012 100644 --- a/macros/cross_db_utils/bool_or.sql +++ b/macros/cross_db_utils/bool_or.sql @@ -1,24 +1,12 @@ {% macro bool_or(expression) -%} - {{ return(adapter.dispatch('bool_or', 'dbt_utils') (expression)) }} +{{ return adapter.dispatch("bool_or", "dbt_utils")(expression) }} {% endmacro %} -{% macro default__bool_or(expression) -%} - - bool_or({{ expression }}) - -{%- endmacro %} +{% macro default__bool_or(expression) -%} bool_or({{ expression }}) {%- endmacro %} -{% macro snowflake__bool_or(expression) -%} - - boolor_agg({{ expression }}) - -{%- endmacro %} +{% macro snowflake__bool_or(expression) -%} boolor_agg({{ expression }}) {%- endmacro %} -{% macro bigquery__bool_or(expression) -%} - - logical_or({{ expression }}) - -{%- endmacro %} \ No newline at end of file +{% macro bigquery__bool_or(expression) -%} logical_or({{ expression }}) {%- endmacro %} diff --git a/macros/cross_db_utils/cast_bool_to_text.sql b/macros/cross_db_utils/cast_bool_to_text.sql index 5efa9d60..eb8c768e 100644 --- a/macros/cross_db_utils/cast_bool_to_text.sql +++ b/macros/cross_db_utils/cast_bool_to_text.sql @@ -1,15 +1,14 @@ {% macro cast_bool_to_text(field) %} - {{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }} +{{ adapter.dispatch("cast_bool_to_text", "dbt_utils")(field) }} {% endmacro %} {% macro default__cast_bool_to_text(field) %} - cast({{ field }} as {{ dbt_utils.type_string() }}) +cast({{ field }} as {{ dbt_utils.type_string() }}) {% endmacro %} {% macro redshift__cast_bool_to_text(field) %} - case - when {{ field }} is true then 'true' - when {{ field }} is false then 'false' - end::text +case + when {{ field }} is true then 'true' when {{ field }} is false then 'false' +end::text {% endmacro %} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 7e97e8f8..c2a25a62 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,9 +1,7 @@ {% macro concat(fields) -%} - {{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }} +{{ return adapter.dispatch("concat", "dbt_utils")(fields) }} {%- endmacro %} -{% macro default__concat(fields) -%} - {{ fields|join(' || ') }} -{%- endmacro %} +{% macro default__concat(fields) -%} {{ fields | join(" || ") }} {%- endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index 66ad8dc6..157d432d 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,38 +1,36 @@ {% macro current_timestamp() -%} - {{ return(adapter.dispatch('current_timestamp', 'dbt_utils')()) }} +{{ return adapter.dispatch("current_timestamp", "dbt_utils")() }} {%- endmacro %} {% macro default__current_timestamp() %} - current_timestamp::{{dbt_utils.type_timestamp()}} +current_timestamp::{{ dbt_utils.type_timestamp() }} {% endmacro %} -{% macro redshift__current_timestamp() %} - getdate() -{% endmacro %} +{% macro redshift__current_timestamp() %} getdate() {% endmacro %} -{% macro bigquery__current_timestamp() %} - current_timestamp -{% endmacro %} +{% macro bigquery__current_timestamp() %} current_timestamp {% endmacro %} {% macro current_timestamp_in_utc() -%} - {{ return(adapter.dispatch('current_timestamp_in_utc', 'dbt_utils')()) }} +{{ return adapter.dispatch("current_timestamp_in_utc", "dbt_utils")() }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} - {{dbt_utils.current_timestamp()}} +{{ dbt_utils.current_timestamp() }} {% endmacro %} {% macro snowflake__current_timestamp_in_utc() %} - convert_timezone('UTC', {{dbt_utils.current_timestamp()}})::{{dbt_utils.type_timestamp()}} +convert_timezone( + 'UTC', {{ dbt_utils.current_timestamp() }} +)::{{ dbt_utils.type_timestamp() }} {% endmacro %} {% macro postgres__current_timestamp_in_utc() %} - (current_timestamp at time zone 'utc')::{{dbt_utils.type_timestamp()}} +(current_timestamp at time zone 'utc')::{{ dbt_utils.type_timestamp() }} {% endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__current_timestamp_in_utc() %} - {{ return(dbt_utils.default__current_timestamp_in_utc()) }} +{{ return dbt_utils.default__current_timestamp_in_utc() }} {% endmacro %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index f115b4e2..8ccd5580 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,99 +1,63 @@ {# string ------------------------------------------------- #} - {%- macro type_string() -%} - {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_string", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_string() %} - string -{% endmacro %} +{% macro default__type_string() %} string {% endmacro %} -{%- macro redshift__type_string() -%} - varchar -{%- endmacro -%} +{%- macro redshift__type_string() -%} varchar {%- endmacro -%} -{% macro postgres__type_string() %} - varchar -{% endmacro %} +{% macro postgres__type_string() %} varchar {% endmacro %} -{% macro snowflake__type_string() %} - varchar -{% endmacro %} +{% macro snowflake__type_string() %} varchar {% endmacro %} {# timestamp ------------------------------------------------- #} - {%- macro type_timestamp() -%} - {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_timestamp", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_timestamp() %} - timestamp -{% endmacro %} +{% macro default__type_timestamp() %} timestamp {% endmacro %} -{% macro postgres__type_timestamp() %} - timestamp without time zone -{% endmacro %} +{% macro postgres__type_timestamp() %} timestamp without time zone {% endmacro %} -{% macro snowflake__type_timestamp() %} - timestamp_ntz -{% endmacro %} +{% macro snowflake__type_timestamp() %} timestamp_ntz {% endmacro %} {# float ------------------------------------------------- #} - {%- macro type_float() -%} - {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_float", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_float() %} - float -{% endmacro %} +{% macro default__type_float() %} float {% endmacro %} -{% macro bigquery__type_float() %} - float64 -{% endmacro %} +{% macro bigquery__type_float() %} float64 {% endmacro %} {# numeric ------------------------------------------------ #} - {%- macro type_numeric() -%} - {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_numeric", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_numeric() %} - numeric(28, 6) -{% endmacro %} +{% macro default__type_numeric() %} numeric(28, 6) {% endmacro %} -{% macro bigquery__type_numeric() %} - numeric -{% endmacro %} +{% macro bigquery__type_numeric() %} numeric {% endmacro %} {# bigint ------------------------------------------------- #} - {%- macro type_bigint() -%} - {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_bigint", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_bigint() %} - bigint -{% endmacro %} +{% macro default__type_bigint() %} bigint {% endmacro %} -{% macro bigquery__type_bigint() %} - int64 -{% endmacro %} +{% macro bigquery__type_bigint() %} int64 {% endmacro %} {# int ------------------------------------------------- #} - {%- macro type_int() -%} - {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_int", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_int() %} - int -{% endmacro %} +{% macro default__type_int() %} int {% endmacro %} -{% macro bigquery__type_int() %} - int64 -{% endmacro %} +{% macro bigquery__type_int() %} int64 {% endmacro %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index f9d0364b..54ac4309 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,15 +1,12 @@ {% macro date_trunc(datepart, date) -%} - {{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }} +{{ return adapter.dispatch("date_trunc", "dbt_utils")(datepart, date) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) -%} - date_trunc('{{datepart}}', {{date}}) +date_trunc('{{datepart}}', {{ date }}) {%- endmacro %} {% macro bigquery__date_trunc(datepart, date) -%} - timestamp_trunc( - cast({{date}} as timestamp), - {{datepart}} - ) +timestamp_trunc(cast({{ date }} as timestamp), {{ datepart }}) {%- endmacro %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 09c0f115..5f019e45 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,37 +1,37 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }} +{{ + return adapter.dispatch("dateadd", "dbt_utils")( + datepart, interval, from_date_or_timestamp + ) +}} {% endmacro %} {% macro default__dateadd(datepart, interval, from_date_or_timestamp) %} - dateadd( - {{ datepart }}, - {{ interval }}, - {{ from_date_or_timestamp }} - ) +dateadd({{ datepart }}, {{ interval }}, {{ from_date_or_timestamp }}) {% endmacro %} {% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %} - datetime_add( - cast( {{ from_date_or_timestamp }} as datetime), - interval {{ interval }} {{ datepart }} - ) +datetime_add( + cast({{ from_date_or_timestamp }} as datetime), + interval {{ interval }} {{ datepart }} +) {% endmacro %} {% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} - {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) +{{ from_date_or_timestamp }} + ( (interval '1 {{ datepart }}') * ({{ interval }})) {% endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }} +{{ return dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp) }} {% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 2b5d6613..d0275619 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,59 +1,94 @@ {% macro datediff(first_date, second_date, datepart) %} - {{ return(adapter.dispatch('datediff', 'dbt_utils')(first_date, second_date, datepart)) }} +{{ return adapter.dispatch("datediff", "dbt_utils")(first_date, second_date, datepart) }} {% endmacro %} {% macro default__datediff(first_date, second_date, datepart) -%} - datediff( - {{ datepart }}, - {{ first_date }}, - {{ second_date }} - ) +datediff({{ datepart }}, {{ first_date }}, {{ second_date }}) {%- endmacro %} {% macro bigquery__datediff(first_date, second_date, datepart) -%} - datetime_diff( - cast({{second_date}} as datetime), - cast({{first_date}} as datetime), - {{datepart}} - ) +datetime_diff( + cast({{ second_date }} as datetime), + cast({{ first_date }} as datetime), + {{ datepart }} +) {%- endmacro %} {% macro postgres__datediff(first_date, second_date, datepart) -%} - {% if datepart == 'year' %} - (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) - {% elif datepart == 'quarter' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) - {% elif datepart == 'month' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) - {% elif datepart == 'day' %} - (({{second_date}})::date - ({{first_date}})::date) - {% elif datepart == 'week' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case - when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then - case when {{first_date}} <= {{second_date}} then 0 else -1 end - else - case when {{first_date}} <= {{second_date}} then 1 else 0 end - end) - {% elif datepart == 'hour' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) - {% elif datepart == 'minute' %} - ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) - {% elif datepart == 'second' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) - {% elif datepart == 'millisecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) - {% elif datepart == 'microsecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) - {% else %} - {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} - {% endif %} +{% if datepart == "year" %} +( + date_part('year', ({{ second_date }})::date) - date_part( + 'year', ({{ first_date }})::date + ) +) +{% elif datepart == "quarter" %} +( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + date_part( + 'quarter', ({{ second_date }})::date + ) - date_part('quarter', ({{ first_date }})::date) +) +{% elif datepart == "month" %} +( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + date_part( + 'month', ({{ second_date }})::date + ) - date_part('month', ({{ first_date }})::date) +) +{% elif datepart == "day" %} ( ({{ second_date }})::date - ({{ first_date }})::date) +{% elif datepart == "week" %} +( + {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case + when + date_part('dow', ({{ first_date }})::timestamp) <= date_part( + 'dow', ({{ second_date }})::timestamp + ) + then case when {{ first_date }} <= {{ second_date }} then 0 else -1 end + else case when {{ first_date }} <= {{ second_date }} then 1 else 0 end + end +) +{% elif datepart == "hour" %} +( + {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + date_part( + 'hour', ({{ second_date }})::timestamp + ) - date_part('hour', ({{ first_date }})::timestamp) +) +{% elif datepart == "minute" %} +( + {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + date_part( + 'minute', ({{ second_date }})::timestamp + ) - date_part('minute', ({{ first_date }})::timestamp) +) +{% elif datepart == "second" %} +( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + floor( + date_part('second', ({{ second_date }})::timestamp) + ) - floor(date_part('second', ({{ first_date }})::timestamp)) +) +{% elif datepart == "millisecond" %} +( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + floor( + date_part('millisecond', ({{ second_date }})::timestamp) + ) - floor(date_part('millisecond', ({{ first_date }})::timestamp)) +) +{% elif datepart == "microsecond" %} +( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + floor( + date_part('microsecond', ({{ second_date }})::timestamp) + ) - floor(date_part('microsecond', ({{ first_date }})::timestamp)) +) +{% else %} +{{ + exceptions.raise_compiler_error( + "Unsupported datepart for macro datediff in postgres: {!r}".format(datepart) + ) +}} +{% endif %} {%- endmacro %} @@ -61,6 +96,6 @@ {# redshift should use default instead of postgres #} {% macro redshift__datediff(first_date, second_date, datepart) -%} - {{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }} +{{ return dbt_utils.default__datediff(first_date, second_date, datepart) }} {%- endmacro %} diff --git a/macros/cross_db_utils/escape_single_quotes.sql b/macros/cross_db_utils/escape_single_quotes.sql index d024f16f..aa82ec01 100644 --- a/macros/cross_db_utils/escape_single_quotes.sql +++ b/macros/cross_db_utils/escape_single_quotes.sql @@ -1,10 +1,10 @@ {% macro escape_single_quotes(expression) %} - {{ return(adapter.dispatch('escape_single_quotes', 'dbt_utils') (expression)) }} +{{ return adapter.dispatch("escape_single_quotes", "dbt_utils")(expression) }} {% endmacro %} {# /*Default to replacing a single apostrophe with two apostrophes: they're -> they''re*/ #} {% macro default__escape_single_quotes(expression) -%} -{{ expression | replace("'","''") }} +{{ expression | replace("'", "''") }} {%- endmacro %} {# /*Snowflake uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 9bb75b86..facada67 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,16 +1,8 @@ {% macro except() %} - {{ return(adapter.dispatch('except', 'dbt_utils')()) }} +{{ return adapter.dispatch("except", "dbt_utils")() }} {% endmacro %} -{% macro default__except() %} +{% macro default__except() %} except {% endmacro %} - except - -{% endmacro %} - -{% macro bigquery__except() %} - - except distinct - -{% endmacro %} \ No newline at end of file +{% macro bigquery__except() %} except distinct {% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index d086c2e6..865a047a 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,13 +1,13 @@ {% macro hash(field) -%} - {{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }} +{{ return adapter.dispatch("hash", "dbt_utils")(field) }} {%- endmacro %} {% macro default__hash(field) -%} - md5(cast({{field}} as {{dbt_utils.type_string()}})) +md5(cast({{ field }} as {{ dbt_utils.type_string() }})) {%- endmacro %} {% macro bigquery__hash(field) -%} - to_hex({{dbt_utils.default__hash(field)}}) +to_hex({{ dbt_utils.default__hash(field) }}) {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index 619cb7cb..edc1bc06 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -1,16 +1,11 @@ -{% macro identifier(value) %} - {%- set error_message = ' - Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ - Use `adapter.quote` instead. The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - {{ return(adapter.dispatch('identifier', 'dbt_utils') (value)) }} -{% endmacro %} +{% macro identifier(value) %} +{%- set error_message = " Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name +) -%} +{%- do exceptions.warn(error_message) -%} +{{ return adapter.dispatch("identifier", "dbt_utils")(value) }} +{% endmacro %} -{% macro default__identifier(value) -%} - "{{ value }}" -{%- endmacro %} +{% macro default__identifier(value) -%} "{{ value }}" {%- endmacro %} -{% macro bigquery__identifier(value) -%} - `{{ value }}` -{%- endmacro %} +{% macro bigquery__identifier(value) -%} `{{ value }}` {%- endmacro %} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index b53c72f3..86b3c70c 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,16 +1,8 @@ {% macro intersect() %} - {{ return(adapter.dispatch('intersect', 'dbt_utils')()) }} +{{ return adapter.dispatch("intersect", "dbt_utils")() }} {% endmacro %} -{% macro default__intersect() %} +{% macro default__intersect() %} intersect {% endmacro %} - intersect - -{% endmacro %} - -{% macro bigquery__intersect() %} - - intersect distinct - -{% endmacro %} +{% macro bigquery__intersect() %} intersect distinct {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 18ca161c..c6f63954 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -2,44 +2,50 @@ This function has been tested with dateparts of month and quarters. Further testing is required to validate that it will work on other dateparts. */ - {% macro last_day(date, datepart) %} - {{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }} +{{ return adapter.dispatch("last_day", "dbt_utils")(date, datepart) }} {% endmacro %} {%- macro default_last_day(date, datepart) -%} - cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd(datepart, '1', dbt_utils.date_trunc(datepart, date)) - )}} - as date) +cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd(datepart, "1", dbt_utils.date_trunc(datepart, date)), + ) + }} as date +) {%- endmacro -%} {% macro default__last_day(date, datepart) -%} - {{dbt_utils.default_last_day(date, datepart)}} +{{ dbt_utils.default_last_day(date, datepart) }} {%- endmacro %} {% macro postgres__last_day(date, datepart) -%} - {%- if datepart == 'quarter' -%} - -- postgres dateadd does not support quarter interval. - cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd('month', '3', dbt_utils.date_trunc(datepart, date)) - )}} - as date) - {%- else -%} - {{dbt_utils.default_last_day(date, datepart)}} - {%- endif -%} +{%- if datepart == "quarter" -%} +-- postgres dateadd does not support quarter interval. +cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd("month", "3", dbt_utils.date_trunc(datepart, date)), + ) + }} as date +) +{%- else -%} {{ dbt_utils.default_last_day(date, datepart) }} +{%- endif -%} {%- endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__last_day(date, datepart) %} - {{ return(dbt_utils.default__last_day(date, datepart)) }} +{{ return dbt_utils.default__last_day(date, datepart) }} {% endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index 8c2e265c..95570736 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,21 +1,9 @@ {% macro length(expression) -%} - {{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }} +{{ return adapter.dispatch("length", "dbt_utils")(expression) }} {% endmacro %} -{% macro default__length(expression) %} - - length( - {{ expression }} - ) - -{%- endmacro -%} +{% macro default__length(expression) %} length({{ expression }}) {%- endmacro -%} -{% macro redshift__length(expression) %} - - len( - {{ expression }} - ) - -{%- endmacro -%} \ No newline at end of file +{% macro redshift__length(expression) %} len({{ expression }}) {%- endmacro -%} diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 1d19a54f..6179a870 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -1,104 +1,88 @@ -{% macro listagg(measure, delimiter_text="','", order_by_clause=none, limit_num=none) -%} - {{ return(adapter.dispatch('listagg', 'dbt_utils') (measure, delimiter_text, order_by_clause, limit_num)) }} +{% macro listagg( + measure, delimiter_text="','", order_by_clause=none, limit_num=none +) -%} +{{ + return adapter.dispatch("listagg", "dbt_utils")( + measure, delimiter_text, order_by_clause, limit_num + ) +}} {%- endmacro %} {% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - {% if limit_num -%} - array_to_string( - array_slice( - array_agg( - {{ measure }} - ){% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,0 - ,{{ limit_num }} - ), - {{ delimiter_text }} - ) - {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - {%- endif %} +{% if limit_num -%} +array_to_string( + array_slice( + array_agg({{ measure }}) + {% if order_by_clause -%} within group({{ order_by_clause }}) {%- endif %}, + 0, + {{ limit_num }} + ), + {{ delimiter_text }} +) +{%- else %} +listagg({{ measure }}, {{ delimiter_text }}) +{% if order_by_clause -%} within group({{ order_by_clause }}) +{%- endif %} +{%- endif %} {%- endmacro %} {% macro bigquery__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - {% if limit_num -%} - limit {{ limit_num }} - {%- endif %} - ) +string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + {% if limit_num -%} limit {{ limit_num }} {%- endif %} +) {%- endmacro %} {% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - - {% if limit_num -%} - array_to_string( - (array_agg( - {{ measure }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - ))[1:{{ limit_num }}], - {{ delimiter_text }} - ) - {%- else %} - string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} + +{% if limit_num -%} +array_to_string( + ( + array_agg( + {{ measure }} {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} ) + ) [1:{{ limit_num }}], + {{ delimiter_text }} +) +{%- else %} +string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} +) +{%- endif %} {%- endmacro %} {# if there are instances of delimiter_text within your measure, you cannot include a limit_num #} {% macro redshift__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - {% if limit_num -%} - {% set ns = namespace() %} - {% set ns.delimiter_text_regex = delimiter_text|trim("'") %} +{% if limit_num -%} +{% set ns = namespace() %} +{% set ns.delimiter_text_regex = delimiter_text | trim("'") %} {% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} - {%- for char in special_chars.split(',') -%} +{%- for char in special_chars.split(",") -%} {% set escape_char %}\\{{ char }}{% endset %} - {% set ns.delimiter_text_regex = ns.delimiter_text_regex|replace(char,escape_char) %} - {%- endfor -%} +{% set ns.delimiter_text_regex = ns.delimiter_text_regex | replace(char, escape_char) %} +{%- endfor -%} {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} - regexp_substr( - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,{{ regex }} - ) - {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - {%- endif %} +regexp_substr( + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group({{ order_by_clause }}) {%- endif %}, + {{ regex }} +) +{%- else %} +listagg({{ measure }}, {{ delimiter_text }}) +{% if order_by_clause -%} within group({{ order_by_clause }}) +{%- endif %} +{%- endif %} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index c2291467..01ce7188 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,8 +1,6 @@ {%- macro string_literal(value) -%} - {{ return(adapter.dispatch('string_literal', 'dbt_utils') (value)) }} +{{ return adapter.dispatch("string_literal", "dbt_utils")(value) }} {%- endmacro -%} -{% macro default__string_literal(value) -%} - '{{ value }}' -{%- endmacro %} +{% macro default__string_literal(value) -%} '{{ value }}' {%- endmacro %} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index a67b7aa2..803fa647 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,22 +1,16 @@ {% macro position(substring_text, string_text) -%} - {{ return(adapter.dispatch('position', 'dbt_utils') (substring_text, string_text)) }} +{{ return adapter.dispatch("position", "dbt_utils")(substring_text, string_text) }} {% endmacro %} {% macro default__position(substring_text, string_text) %} - position( - {{ substring_text }} in {{ string_text }} - ) - +position({{ substring_text }} in {{ string_text }}) + {%- endmacro -%} {% macro bigquery__position(substring_text, string_text) %} - strpos( - {{ string_text }}, - {{ substring_text }} - - ) - +strpos({{ string_text }}, {{ substring_text }}) + {%- endmacro -%} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index ef27d8fc..a2173cac 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,15 +1,11 @@ {% macro replace(field, old_chars, new_chars) -%} - {{ return(adapter.dispatch('replace', 'dbt_utils') (field, old_chars, new_chars)) }} +{{ return adapter.dispatch("replace", "dbt_utils")(field, old_chars, new_chars) }} {% endmacro %} {% macro default__replace(field, old_chars, new_chars) %} - replace( - {{ field }}, - {{ old_chars }}, - {{ new_chars }} - ) - +replace({{ field }}, {{ old_chars }}, {{ new_chars }}) -{% endmacro %} \ No newline at end of file + +{% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 8ae1640f..1479e760 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,38 +1,29 @@ {% macro right(string_text, length_expression) -%} - {{ return(adapter.dispatch('right', 'dbt_utils') (string_text, length_expression)) }} +{{ return adapter.dispatch("right", "dbt_utils")(string_text, length_expression) }} {% endmacro %} {% macro default__right(string_text, length_expression) %} - right( - {{ string_text }}, - {{ length_expression }} - ) - +right({{ string_text }}, {{ length_expression }}) + {%- endmacro -%} {% macro bigquery__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 - then '' - else - substr( - {{ string_text }}, - -1 * ({{ length_expression }}) - ) - end +case + when {{ length_expression }} = 0 + then '' + else substr({{ string_text }}, -1 * ({{ length_expression }})) +end {%- endmacro -%} {% macro snowflake__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 - then '' - else - right( - {{ string_text }}, - {{ length_expression }} - ) - end +case + when {{ length_expression }} = 0 + then '' + else right({{ string_text }}, {{ length_expression }}) +end -{%- endmacro -%} \ No newline at end of file +{%- endmacro -%} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index 8569eecd..bbfcb867 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,20 +1,20 @@ {% macro safe_cast(field, type) %} - {{ return(adapter.dispatch('safe_cast', 'dbt_utils') (field, type)) }} +{{ return adapter.dispatch("safe_cast", "dbt_utils")(field, type) }} {% endmacro %} {% macro default__safe_cast(field, type) %} - {# most databases don't support this function yet +{# most databases don't support this function yet so we just need to use cast #} - cast({{field}} as {{type}}) +cast({{ field }} as {{ type }}) {% endmacro %} {% macro snowflake__safe_cast(field, type) %} - try_cast({{field}} as {{type}}) +try_cast({{ field }} as {{ type }}) {% endmacro %} {% macro bigquery__safe_cast(field, type) %} - safe_cast({{field}} as {{type}}) +safe_cast({{ field }} as {{ type }}) {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index 036f7d5a..10035a94 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,24 +1,21 @@ {% macro split_part(string_text, delimiter_text, part_number) %} - {{ return(adapter.dispatch('split_part', 'dbt_utils') (string_text, delimiter_text, part_number)) }} +{{ + return adapter.dispatch("split_part", "dbt_utils")( + string_text, delimiter_text, part_number + ) +}} {% endmacro %} {% macro default__split_part(string_text, delimiter_text, part_number) %} - split_part( - {{ string_text }}, - {{ delimiter_text }}, - {{ part_number }} - ) +split_part({{ string_text }}, {{ delimiter_text }}, {{ part_number }}) {% endmacro %} {% macro bigquery__split_part(string_text, delimiter_text, part_number) %} - split( - {{ string_text }}, - {{ delimiter_text }} - )[safe_offset({{ part_number - 1 }})] +split({{ string_text }}, {{ delimiter_text }}) [safe_offset({{ part_number - 1 }})] {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index fc8ae81c..38afcc25 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,5 +1,9 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} - {{ return(adapter.dispatch('width_bucket', 'dbt_utils') (expr, min_value, max_value, num_buckets)) }} +{{ + return adapter.dispatch("width_bucket", "dbt_utils")( + expr, min_value, max_value, num_buckets + ) +}} {% endmacro %} @@ -8,25 +12,20 @@ {% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) {%- endset %} - ( - -- to break ties when the amount is eaxtly at the bucket egde - case - when - mod( - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }}, - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - ) = 0 - then 1 - else 0 - end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 - ) +( + -- to break ties when the amount is eaxtly at the bucket egde + case + when + mod( + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }}, + {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + ) = 0 + then 1 + else 0 + end +) + +-- Anything over max_value goes the N+1 bucket +least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} @@ -34,26 +33,22 @@ {% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) {%- endset %} - ( - -- to break ties when the amount is exactly at the bucket edge - case - when - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }} % - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - = 0 - then 1 - else 0 - end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 - ) +( + -- to break ties when the amount is exactly at the bucket edge + case + when + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }} + % + {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + = 0 + then 1 + else 0 + end +) + +-- Anything over max_value goes the N+1 bucket +least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} - width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }} ) +width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }}) {% endmacro %} diff --git a/macros/generic_tests/accepted_range.sql b/macros/generic_tests/accepted_range.sql index 49ab00d9..0e91c4a7 100644 --- a/macros/generic_tests/accepted_range.sql +++ b/macros/generic_tests/accepted_range.sql @@ -1,31 +1,40 @@ -{% test accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} - {{ return(adapter.dispatch('test_accepted_range', 'dbt_utils')(model, column_name, min_value, max_value, inclusive)) }} +{% test accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} +{{ + return adapter.dispatch("test_accepted_range", "dbt_utils")( + model, column_name, min_value, max_value, inclusive + ) +}} {% endtest %} -{% macro default__test_accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} +{% macro default__test_accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} -with meet_condition as( - select * - from {{ model }} -), +with + meet_condition as (select * from {{ model }}), -validation_errors as ( - select * - from meet_condition - where - -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds - 1 = 2 + validation_errors as ( + select * + from meet_condition + where + -- never true, defaults to an empty result set. Exists to ensure any combo + -- of the `or` clauses below succeeds + 1 = 2 - {%- if min_value is not none %} - -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} - {%- endif %} + {%- if min_value is not none %} + -- records with a value >= min_value are permitted. The `not` flips this + -- to find records that don't meet the rule. + or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} + {%- endif %} - {%- if max_value is not none %} - -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} - {%- endif %} -) + {%- if max_value is not none %} + -- records with a value <= max_value are permitted. The `not` flips this + -- to find records that don't meet the rule. + or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} + {%- endif %} + ) select * from validation_errors diff --git a/macros/generic_tests/at_least_one.sql b/macros/generic_tests/at_least_one.sql index ce02108f..4df140bc 100644 --- a/macros/generic_tests/at_least_one.sql +++ b/macros/generic_tests/at_least_one.sql @@ -1,20 +1,20 @@ {% test at_least_one(model, column_name) %} - {{ return(adapter.dispatch('test_at_least_one', 'dbt_utils')(model, column_name)) }} +{{ return adapter.dispatch("test_at_least_one", "dbt_utils")(model, column_name) }} {% endtest %} {% macro default__test_at_least_one(model, column_name) %} select * -from ( - select +from + ( {# In TSQL, subquery aggregate columns need aliases #} {# thus: a filler col name, 'filler_column' #} - count({{ column_name }}) as filler_column + select count({{ column_name }}) as filler_column - from {{ model }} + from {{ model }} - having count({{ column_name }}) = 0 + having count({{ column_name }}) = 0 -) validation_errors + ) validation_errors {% endmacro %} diff --git a/macros/generic_tests/cardinality_equality.sql b/macros/generic_tests/cardinality_equality.sql index 4ec7d87f..90df0c47 100644 --- a/macros/generic_tests/cardinality_equality.sql +++ b/macros/generic_tests/cardinality_equality.sql @@ -1,51 +1,31 @@ {% test cardinality_equality(model, column_name, to, field) %} - {{ return(adapter.dispatch('test_cardinality_equality', 'dbt_utils')(model, column_name, to, field)) }} +{{ + return adapter.dispatch("test_cardinality_equality", "dbt_utils")( + model, column_name, to, field + ) +}} {% endtest %} {% macro default__test_cardinality_equality(model, column_name, to, field) %} {# T-SQL does not let you use numbers as aliases for columns #} {# Thus, no "GROUP BY 1" #} +with + table_a as ( + select {{ column_name }}, count(*) as num_rows + from {{ model }} + group by {{ column_name }} + ), -with table_a as ( -select - {{ column_name }}, - count(*) as num_rows -from {{ model }} -group by {{ column_name }} -), - -table_b as ( -select - {{ field }}, - count(*) as num_rows -from {{ to }} -group by {{ field }} -), - -except_a as ( - select * - from table_a - {{ dbt_utils.except() }} - select * - from table_b -), - -except_b as ( - select * - from table_b - {{ dbt_utils.except() }} - select * - from table_a -), - -unioned as ( - select * - from except_a - union all - select * - from except_b -) + table_b as ( + select {{ field }}, count(*) as num_rows from {{ to }} group by {{ field }} + ), + + except_a as (select * from table_a {{ dbt_utils.except() }} select * from table_b), + + except_b as (select * from table_b {{ dbt_utils.except() }} select * from table_a), + + unioned as (select * from except_a union all select * from except_b) select * from unioned diff --git a/macros/generic_tests/equal_rowcount.sql b/macros/generic_tests/equal_rowcount.sql index 713cb12a..19eccae3 100644 --- a/macros/generic_tests/equal_rowcount.sql +++ b/macros/generic_tests/equal_rowcount.sql @@ -1,38 +1,27 @@ {% test equal_rowcount(model, compare_model) %} - {{ return(adapter.dispatch('test_equal_rowcount', 'dbt_utils')(model, compare_model)) }} +{{ return adapter.dispatch("test_equal_rowcount", "dbt_utils")(model, compare_model) }} {% endtest %} {% macro default__test_equal_rowcount(model, compare_model) %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = 'coalesce(diff_count, 0)') }} +{#- - Needs to be set at parse time, before we return '' below --#} +{{ config(fail_calc="coalesce(diff_count, 0)") }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return "" }} {% endif %} -with a as ( +with + a as (select count(*) as count_a from {{ model }}), + b as (select count(*) as count_b from {{ compare_model }}), + final as ( - select count(*) as count_a from {{ model }} + select count_a, count_b, abs(count_a - count_b) as diff_count + from a + cross join b -), -b as ( + ) - select count(*) as count_b from {{ compare_model }} - -), -final as ( - - select - count_a, - count_b, - abs(count_a - count_b) as diff_count - from a - cross join b - -) - -select * from final +select * +from final {% endmacro %} diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index bc302c18..8acec338 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -1,5 +1,9 @@ {% test equality(model, compare_model, compare_columns=None) %} - {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns)) }} +{{ + return adapter.dispatch("test_equality", "dbt_utils")( + model, compare_model, compare_columns + ) +}} {% endtest %} {% macro default__test_equality(model, compare_model, compare_columns=None) %} @@ -11,65 +15,62 @@ ), 0) {% endset %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = set_diff) }} +{#- - Needs to be set at parse time, before we return '' below --#} +{{ config(fail_calc=set_diff) }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return "" }} {% endif %} -- setup -{%- do dbt_utils._is_relation(model, 'test_equality') -%} +{%- do dbt_utils._is_relation(model, "test_equality") -%} {#- If the compare_cols arg is provided, we can run this test without querying the information schema — this allows the model to be an ephemeral model -#} - {%- if not compare_columns -%} - {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} +{%- do dbt_utils._is_ephemeral(model, "test_equality") -%} +{%- set compare_columns = adapter.get_columns_in_relation(model) | map( + attribute="quoted" +) -%} {%- endif -%} -{% set compare_cols_csv = compare_columns | join(', ') %} - -with a as ( - - select * from {{ model }} - -), - -b as ( +{% set compare_cols_csv = compare_columns | join(", ") %} - select * from {{ compare_model }} +with + a as (select * from {{ model }}), -), + b as (select * from {{ compare_model }}), -a_minus_b as ( + a_minus_b as ( - select {{compare_cols_csv}} from a - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from b + select {{ compare_cols_csv }} + from a {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from b -), + ), -b_minus_a as ( + b_minus_a as ( - select {{compare_cols_csv}} from b - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from a + select {{ compare_cols_csv }} + from b {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from a -), + ), -unioned as ( + unioned as ( - select 'a_minus_b' as which_diff, a_minus_b.* from a_minus_b - union all - select 'b_minus_a' as which_diff, b_minus_a.* from b_minus_a + select 'a_minus_b' as which_diff, a_minus_b.* + from a_minus_b + union all + select 'b_minus_a' as which_diff, b_minus_a.* + from b_minus_a -) + ) -select * from unioned +select * +from unioned {% endmacro %} diff --git a/macros/generic_tests/expression_is_true.sql b/macros/generic_tests/expression_is_true.sql index 611685b4..d3285004 100644 --- a/macros/generic_tests/expression_is_true.sql +++ b/macros/generic_tests/expression_is_true.sql @@ -1,22 +1,21 @@ -{% test expression_is_true(model, expression, column_name=None, condition='1=1') %} +{% test expression_is_true(model, expression, column_name=None, condition="1=1") %} {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} {# ref https://stackoverflow.com/a/7170753/3842610 #} - {{ return(adapter.dispatch('test_expression_is_true', 'dbt_utils')(model, expression, column_name, condition)) }} +{{ + return adapter.dispatch("test_expression_is_true", "dbt_utils")( + model, expression, column_name, condition + ) +}} {% endtest %} {% macro default__test_expression_is_true(model, expression, column_name, condition) %} -with meet_condition as ( - select * from {{ model }} where {{ condition }} -) +with meet_condition as (select * from {{ model }} where {{ condition }}) -select - * +select * from meet_condition -{% if column_name is none %} -where not({{ expression }}) -{%- else %} -where not({{ column_name }} {{ expression }}) +{% if column_name is none %} where not ({{ expression }}) +{%- else %} where not ({{ column_name }} {{ expression }}) {%- endif %} {% endmacro %} diff --git a/macros/generic_tests/fewer_rows_than.sql b/macros/generic_tests/fewer_rows_than.sql index 450148b7..dfcefa6e 100644 --- a/macros/generic_tests/fewer_rows_than.sql +++ b/macros/generic_tests/fewer_rows_than.sql @@ -1,45 +1,35 @@ {% test fewer_rows_than(model, compare_model) %} - {{ return(adapter.dispatch('test_fewer_rows_than', 'dbt_utils')(model, compare_model)) }} +{{ return adapter.dispatch("test_fewer_rows_than", "dbt_utils")(model, compare_model) }} {% endtest %} {% macro default__test_fewer_rows_than(model, compare_model) %} -{{ config(fail_calc = 'coalesce(row_count_delta, 0)') }} - -with a as ( - - select count(*) as count_our_model from {{ model }} - -), -b as ( - - select count(*) as count_comparison_model from {{ compare_model }} - -), -counts as ( - - select - count_our_model, - count_comparison_model - from a - cross join b - -), -final as ( - - select *, - case - -- fail the test if we have more rows than the reference model and return the row count delta - when count_our_model > count_comparison_model then (count_our_model - count_comparison_model) - -- fail the test if they are the same number - when count_our_model = count_comparison_model then 1 - -- pass the test if the delta is positive (i.e. return the number 0) - else 0 - end as row_count_delta - from counts - -) - -select * from final +{{ config(fail_calc="coalesce(row_count_delta, 0)") }} + +with + a as (select count(*) as count_our_model from {{ model }}), + b as (select count(*) as count_comparison_model from {{ compare_model }}), + counts as (select count_our_model, count_comparison_model from a cross join b), + final as ( + + select + *, + case + -- fail the test if we have more rows than the reference model and + -- return the row count delta + when count_our_model > count_comparison_model + then (count_our_model - count_comparison_model) + -- fail the test if they are the same number + when count_our_model = count_comparison_model + then 1 + -- pass the test if the delta is positive (i.e. return the number 0) + else 0 + end as row_count_delta + from counts + + ) + +select * +from final {% endmacro %} diff --git a/macros/generic_tests/mutually_exclusive_ranges.sql b/macros/generic_tests/mutually_exclusive_ranges.sql index f88efbe4..fd7c8155 100644 --- a/macros/generic_tests/mutually_exclusive_ranges.sql +++ b/macros/generic_tests/mutually_exclusive_ranges.sql @@ -1,97 +1,121 @@ -{% test mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} - {{ return(adapter.dispatch('test_mutually_exclusive_ranges', 'dbt_utils')(model, lower_bound_column, upper_bound_column, partition_by, gaps, zero_length_range_allowed)) }} +{% test mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False, +) %} +{{ + return adapter.dispatch("test_mutually_exclusive_ranges", "dbt_utils")( + model, + lower_bound_column, + upper_bound_column, + partition_by, + gaps, + zero_length_range_allowed, + ) +}} {% endtest %} -{% macro default__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} -{% if gaps == 'not_allowed' %} - {% set allow_gaps_operator='=' %} - {% set allow_gaps_operator_in_words='equal_to' %} -{% elif gaps == 'allowed' %} - {% set allow_gaps_operator='<=' %} - {% set allow_gaps_operator_in_words='less_than_or_equal_to' %} -{% elif gaps == 'required' %} - {% set allow_gaps_operator='<' %} - {% set allow_gaps_operator_in_words='less_than' %} +{% macro default__test_mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False, +) %} +{% if gaps == "not_allowed" %} +{% set allow_gaps_operator = "=" %} {% set allow_gaps_operator_in_words = "equal_to" %} +{% elif gaps == "allowed" %} +{% set allow_gaps_operator = "<=" %} +{% set allow_gaps_operator_in_words = "less_than_or_equal_to" %} +{% elif gaps == "required" %} +{% set allow_gaps_operator = "<" %} {% set allow_gaps_operator_in_words = "less_than" %} {% else %} - {{ exceptions.raise_compiler_error( - "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" - ) }} +{{ + exceptions.raise_compiler_error( + "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" + ) +}} {% endif %} {% if not zero_length_range_allowed %} - {% set allow_zero_length_operator='<' %} - {% set allow_zero_length_operator_in_words='less_than' %} +{% set allow_zero_length_operator = "<" %} +{% set allow_zero_length_operator_in_words = "less_than" %} {% elif zero_length_range_allowed %} - {% set allow_zero_length_operator='<=' %} - {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %} +{% set allow_zero_length_operator = "<=" %} +{% set allow_zero_length_operator_in_words = "less_than_or_equal_to" %} {% else %} - {{ exceptions.raise_compiler_error( - "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length_range_allowed ~"'.'" - ) }} +{{ + exceptions.raise_compiler_error( + "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length_range_allowed ~"'.'" + ) +}} {% endif %} {% set partition_clause="partition by " ~ partition_by if partition_by else '' %} -with window_functions as ( - - select - {% if partition_by %} - {{ partition_by }} as partition_by_col, - {% endif %} - {{ lower_bound_column }} as lower_bound, - {{ upper_bound_column }} as upper_bound, - - lead({{ lower_bound_column }}) over ( - {{ partition_clause }} - order by {{ lower_bound_column }} - ) as next_lower_bound, - - row_number() over ( - {{ partition_clause }} - order by {{ lower_bound_column }} desc - ) = 1 as is_last_record - - from {{ model }} - -), - -calc as ( - -- We want to return records where one of our assumptions fails, so we'll use - -- the `not` function with `and` statements so we can write our assumptions nore cleanly - select - *, - - -- For each record: lower_bound should be < upper_bound. - -- Coalesce it to return an error on the null case (implicit assumption - -- these columns are not_null) - coalesce( - lower_bound {{ allow_zero_length_operator }} upper_bound, - false - ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, - - -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. - -- Coalesce it to handle null cases for the last record. - coalesce( - upper_bound {{ allow_gaps_operator }} next_lower_bound, - is_last_record, - false - ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound - - from window_functions - -), - -validation_errors as ( - - select - * - from calc - - where not( - -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound - and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound +with + window_functions as ( + + select + {% if partition_by %} {{ partition_by }} as partition_by_col, {% endif %} + {{ lower_bound_column }} as lower_bound, + {{ upper_bound_column }} as upper_bound, + + lead({{ lower_bound_column }}) over ( + {{ partition_clause }} order by {{ lower_bound_column }} + ) as next_lower_bound, + + row_number() over ( + {{ partition_clause }} order by {{ lower_bound_column }} desc + ) = 1 as is_last_record + + from {{ model }} + + ), + + calc as ( + -- We want to return records where one of our assumptions fails, so we'll use + -- the `not` function with `and` statements so we can write our assumptions + -- nore cleanly + select + *, + + -- For each record: lower_bound should be < upper_bound. + -- Coalesce it to return an error on the null case (implicit assumption + -- these columns are not_null) + coalesce( + lower_bound {{ allow_zero_length_operator }} upper_bound, false + ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, + + -- For each record: upper_bound {{ allow_gaps_operator }} the next + -- lower_bound. + -- Coalesce it to handle null cases for the last record. + coalesce( + upper_bound {{ allow_gaps_operator }} next_lower_bound, + is_last_record, + false + ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + + from window_functions + + ), + + validation_errors as ( + + select * + from calc + + where + not ( + -- THE FOLLOWING SHOULD BE TRUE -- + lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound + and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + ) ) -) -select * from validation_errors +select * +from validation_errors {% endmacro %} diff --git a/macros/generic_tests/not_accepted_values.sql b/macros/generic_tests/not_accepted_values.sql index ab24188f..f618da35 100644 --- a/macros/generic_tests/not_accepted_values.sql +++ b/macros/generic_tests/not_accepted_values.sql @@ -1,35 +1,31 @@ {% test not_accepted_values(model, column_name, values, quote=True) %} - {{ return(adapter.dispatch('test_not_accepted_values', 'dbt_utils')(model, column_name, values, quote)) }} +{{ + return adapter.dispatch("test_not_accepted_values", "dbt_utils")( + model, column_name, values, quote + ) +}} {% endtest %} {% macro default__test_not_accepted_values(model, column_name, values, quote=True) %} -with all_values as ( +with + all_values as (select distinct {{ column_name }} as value_field from {{ model }}), - select distinct - {{ column_name }} as value_field + validation_errors as ( - from {{ model }} + select value_field -), + from all_values + where + value_field in ( + {% for value in values -%} + {% if quote -%}'{{ value }}' + {%- else -%}{{ value }} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {%- endfor %} + ) -validation_errors as ( - - select - value_field - - from all_values - where value_field in ( - {% for value in values -%} - {% if quote -%} - '{{ value }}' - {%- else -%} - {{ value }} - {%- endif -%} - {%- if not loop.last -%},{%- endif %} - {%- endfor %} - ) - -) + ) select * from validation_errors diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index 781ed7d9..bae4b54b 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -1,15 +1,14 @@ {% test not_constant(model, column_name) %} - {{ return(adapter.dispatch('test_not_constant', 'dbt_utils')(model, column_name)) }} +{{ return adapter.dispatch("test_not_constant", "dbt_utils")(model, column_name) }} {% endtest %} {% macro default__test_not_constant(model, column_name) %} -select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count(distinct {{ column_name }}) as filler_column +{# In TSQL, subquery aggregate columns need aliases #} +{# thus: a filler col name, 'filler_column' #} +select count(distinct {{ column_name }}) as filler_column from {{ model }} diff --git a/macros/generic_tests/not_null_proportion.sql b/macros/generic_tests/not_null_proportion.sql index 45b9050c..ef56bb65 100644 --- a/macros/generic_tests/not_null_proportion.sql +++ b/macros/generic_tests/not_null_proportion.sql @@ -1,26 +1,28 @@ {% macro test_not_null_proportion(model) %} - {{ return(adapter.dispatch('test_not_null_proportion', 'dbt_utils')(model, **kwargs)) }} +{{ return adapter.dispatch("test_not_null_proportion", "dbt_utils")(model, **kwargs) }} {% endmacro %} {% macro default__test_not_null_proportion(model) %} -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} -{% set at_least = kwargs.get('at_least', kwargs.get('arg')) %} -{% set at_most = kwargs.get('at_most', kwargs.get('arg', 1)) %} +{% set column_name = kwargs.get("column_name", kwargs.get("arg")) %} +{% set at_least = kwargs.get("at_least", kwargs.get("arg")) %} +{% set at_most = kwargs.get("at_most", kwargs.get("arg", 1)) %} -with validation as ( - select - sum(case when {{ column_name }} is null then 0 else 1 end) / cast(count(*) as numeric) as not_null_proportion - from {{ model }} -), -validation_errors as ( - select - not_null_proportion - from validation - where not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} -) -select - * +with + validation as ( + select + sum(case when {{ column_name }} is null then 0 else 1 end) / cast( + count(*) as numeric + ) as not_null_proportion + from {{ model }} + ), + validation_errors as ( + select not_null_proportion + from validation + where + not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} + ) +select * from validation_errors {% endmacro %} diff --git a/macros/generic_tests/recency.sql b/macros/generic_tests/recency.sql index cb9a8de6..44b02295 100644 --- a/macros/generic_tests/recency.sql +++ b/macros/generic_tests/recency.sql @@ -1,22 +1,20 @@ {% test recency(model, field, datepart, interval) %} - {{ return(adapter.dispatch('test_recency', 'dbt_utils')(model, field, datepart, interval)) }} +{{ + return adapter.dispatch("test_recency", "dbt_utils")( + model, field, datepart, interval + ) +}} {% endtest %} {% macro default__test_recency(model, field, datepart, interval) %} -{% set threshold = dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp()) %} +{% set threshold = dbt_utils.dateadd( + datepart, interval * -1, dbt_utils.current_timestamp() +) %} -with recency as ( +with recency as (select max({{ field }}) as most_recent from {{ model }}) - select max({{field}}) as most_recent - from {{ model }} - -) - -select - - most_recent, - {{ threshold }} as threshold +select most_recent, {{ threshold }} as threshold from recency where most_recent < {{ threshold }} diff --git a/macros/generic_tests/relationships_where.sql b/macros/generic_tests/relationships_where.sql index c35a380a..fd68632d 100644 --- a/macros/generic_tests/relationships_where.sql +++ b/macros/generic_tests/relationships_where.sql @@ -1,51 +1,53 @@ -{% test relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} - {{ return(adapter.dispatch('test_relationships_where', 'dbt_utils')(model, column_name, to, field, from_condition, to_condition)) }} +{% test relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} +{{ + return adapter.dispatch("test_relationships_where", "dbt_utils")( + model, column_name, to, field, from_condition, to_condition + ) +}} {% endtest %} -{% macro default__test_relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} +{% macro default__test_relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} {# ref https://stackoverflow.com/a/7170753/3842610 #} +with + left_table as ( -with left_table as ( + select {{ column_name }} as id - select - {{column_name}} as id + from {{ model }} - from {{model}} + where {{ column_name }} is not null and {{ from_condition }} - where {{column_name}} is not null - and {{from_condition}} + ), -), + right_table as ( -right_table as ( + select {{ field }} as id - select - {{field}} as id + from {{ to }} - from {{to}} + where {{ field }} is not null and {{ to_condition }} - where {{field}} is not null - and {{to_condition}} + ), -), + exceptions as ( -exceptions as ( + select left_table.id, right_table.id as right_id - select - left_table.id, - right_table.id as right_id + from left_table - from left_table + left join right_table on left_table.id = right_table.id - left join right_table - on left_table.id = right_table.id + where right_table.id is null - where right_table.id is null + ) -) - -select * from exceptions +select * +from exceptions {% endmacro %} diff --git a/macros/generic_tests/sequential_values.sql b/macros/generic_tests/sequential_values.sql index 8ddae707..4df9770b 100644 --- a/macros/generic_tests/sequential_values.sql +++ b/macros/generic_tests/sequential_values.sql @@ -1,33 +1,45 @@ {% test sequential_values(model, column_name, interval=1, datepart=None) %} - {{ return(adapter.dispatch('test_sequential_values', 'dbt_utils')(model, column_name, interval, datepart)) }} +{{ + return adapter.dispatch("test_sequential_values", "dbt_utils")( + model, column_name, interval, datepart + ) +}} {% endtest %} -{% macro default__test_sequential_values(model, column_name, interval=1, datepart=None) %} +{% macro default__test_sequential_values( + model, column_name, interval=1, datepart=None +) %} {% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} -with windowed as ( - - select - {{ column_name }}, - lag({{ column_name }}) over ( - order by {{ column_name }} - ) as {{ previous_column_name }} - from {{ model }} -), - -validation_errors as ( - select - * - from windowed - {% if datepart %} - where not(cast({{ column_name }} as {{ dbt_utils.type_timestamp() }})= cast({{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} as {{ dbt_utils.type_timestamp() }})) - {% else %} - where not({{ column_name }} = {{ previous_column_name }} + {{ interval }}) - {% endif %} -) +with + windowed as ( + + select + {{ column_name }}, + lag({{ column_name }}) over ( + order by {{ column_name }} + ) as {{ previous_column_name }} + from {{ model }} + ), + + validation_errors as ( + select * + from windowed + {% if datepart %} + where + not ( + cast({{ column_name }} as {{ dbt_utils.type_timestamp() }}) = cast( + {{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} + as {{ dbt_utils.type_timestamp() }} + ) + ) + {% else %} + where not ({{ column_name }} = {{ previous_column_name }} + {{ interval }}) + {% endif %} + ) select * from validation_errors diff --git a/macros/generic_tests/test_not_null_where.sql b/macros/generic_tests/test_not_null_where.sql index d5dbf6c5..44f695f6 100644 --- a/macros/generic_tests/test_not_null_where.sql +++ b/macros/generic_tests/test_not_null_where.sql @@ -1,12 +1,9 @@ -{% test not_null_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.not_null_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_not_null_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_not_null_where(model, column_name) %} - {{ return(test_not_null(model, column_name)) }} -{% endmacro %} +{% test not_null_where(model, column_name) %} +{%- set deprecation_warning = " Warning: `dbt_utils.not_null_where` is no longer supported. Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. " -%} +{%- do exceptions.warn(deprecation_warning) -%} +{{ return adapter.dispatch("test_not_null_where", "dbt_utils")(model, column_name) }} +{% endtest %} + +{% macro default__test_not_null_where(model, column_name) %} +{{ return test_not_null(model, column_name) }} +{% endmacro %} diff --git a/macros/generic_tests/test_unique_where.sql b/macros/generic_tests/test_unique_where.sql index e752d7b2..915e0918 100644 --- a/macros/generic_tests/test_unique_where.sql +++ b/macros/generic_tests/test_unique_where.sql @@ -1,12 +1,9 @@ -{% test unique_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.unique_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_unique_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_unique_where(model, column_name) %} - {{ return(test_unique(model, column_name)) }} -{% endmacro %} +{% test unique_where(model, column_name) %} +{%- set deprecation_warning = " Warning: `dbt_utils.unique_where` is no longer supported. Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. " -%} +{%- do exceptions.warn(deprecation_warning) -%} +{{ return adapter.dispatch("test_unique_where", "dbt_utils")(model, column_name) }} +{% endtest %} + +{% macro default__test_unique_where(model, column_name) %} +{{ return test_unique(model, column_name) }} +{% endmacro %} diff --git a/macros/generic_tests/unique_combination_of_columns.sql b/macros/generic_tests/unique_combination_of_columns.sql index 74ccf5c6..dbd1e1ff 100644 --- a/macros/generic_tests/unique_combination_of_columns.sql +++ b/macros/generic_tests/unique_combination_of_columns.sql @@ -1,34 +1,43 @@ -{% test unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} - {{ return(adapter.dispatch('test_unique_combination_of_columns', 'dbt_utils')(model, combination_of_columns, quote_columns)) }} +{% test unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} +{{ + return adapter.dispatch("test_unique_combination_of_columns", "dbt_utils")( + model, combination_of_columns, quote_columns + ) +}} {% endtest %} -{% macro default__test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} +{% macro default__test_unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} -{% if not quote_columns %} - {%- set column_list=combination_of_columns %} +{% if not quote_columns %} {%- set column_list = combination_of_columns %} {% elif quote_columns %} - {%- set column_list=[] %} - {% for column in combination_of_columns -%} - {% set column_list = column_list.append( adapter.quote(column) ) %} - {%- endfor %} +{%- set column_list = [] %} +{% for column in combination_of_columns -%} +{% set column_list = column_list.append(adapter.quote(column)) %} +{%- endfor %} {% else %} - {{ exceptions.raise_compiler_error( - "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" - ) }} +{{ + exceptions.raise_compiler_error( + "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" + ) +}} {% endif %} -{%- set columns_csv=column_list | join(', ') %} +{%- set columns_csv = column_list | join(", ") %} -with validation_errors as ( +with + validation_errors as ( - select - {{ columns_csv }} - from {{ model }} - group by {{ columns_csv }} - having count(*) > 1 + select {{ columns_csv }} + from {{ model }} + group by {{ columns_csv }} + having count(*) > 1 -) + ) select * from validation_errors diff --git a/macros/jinja_helpers/log_info.sql b/macros/jinja_helpers/log_info.sql index 52b4b4a5..ded63447 100644 --- a/macros/jinja_helpers/log_info.sql +++ b/macros/jinja_helpers/log_info.sql @@ -1,7 +1,7 @@ {% macro log_info(message) %} - {{ return(adapter.dispatch('log_info', 'dbt_utils')(message)) }} +{{ return adapter.dispatch("log_info", "dbt_utils")(message) }} {% endmacro %} {% macro default__log_info(message) %} - {{ log(dbt_utils.pretty_log_format(message), info=True) }} +{{ log(dbt_utils.pretty_log_format(message), info=True) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_log_format.sql b/macros/jinja_helpers/pretty_log_format.sql index fe580d13..ed354c41 100644 --- a/macros/jinja_helpers/pretty_log_format.sql +++ b/macros/jinja_helpers/pretty_log_format.sql @@ -1,7 +1,7 @@ {% macro pretty_log_format(message) %} - {{ return(adapter.dispatch('pretty_log_format', 'dbt_utils')(message)) }} +{{ return adapter.dispatch("pretty_log_format", "dbt_utils")(message) }} {% endmacro %} {% macro default__pretty_log_format(message) %} - {{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} +{{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_time.sql b/macros/jinja_helpers/pretty_time.sql index bad37efe..0b07689e 100644 --- a/macros/jinja_helpers/pretty_time.sql +++ b/macros/jinja_helpers/pretty_time.sql @@ -1,7 +1,7 @@ -{% macro pretty_time(format='%H:%M:%S') %} - {{ return(adapter.dispatch('pretty_time', 'dbt_utils')(format)) }} +{% macro pretty_time(format="%H:%M:%S") %} +{{ return adapter.dispatch("pretty_time", "dbt_utils")(format) }} {% endmacro %} -{% macro default__pretty_time(format='%H:%M:%S') %} - {{ return(modules.datetime.datetime.now().strftime(format)) }} +{% macro default__pretty_time(format="%H:%M:%S") %} +{{ return modules.datetime.datetime.now().strftime(format) }} {% endmacro %} diff --git a/macros/jinja_helpers/slugify.sql b/macros/jinja_helpers/slugify.sql index 1b3c7272..9131ccaf 100644 --- a/macros/jinja_helpers/slugify.sql +++ b/macros/jinja_helpers/slugify.sql @@ -3,10 +3,10 @@ {#- Lower case the string -#} {% set string = string | lower %} {#- Replace spaces and dashes with underscores -#} -{% set string = modules.re.sub('[ -]+', '_', string) %} +{% set string = modules.re.sub("[ -]+", "_", string) %} {#- Only take letters, numbers, and underscores -#} -{% set string = modules.re.sub('[^a-z0-9_]+', '', string) %} +{% set string = modules.re.sub("[^a-z0-9_]+", "", string) %} -{{ return(string) }} +{{ return string }} {% endmacro %} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 5a15c815..2fb03ad8 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -1,38 +1,86 @@ -{% macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - {{ return(adapter.dispatch('get_period_boundaries', 'dbt_utils')(target_schema, target_table, timestamp_field, start_date, stop_date, period)) }} +{% macro get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period, +) -%} +{{ + return adapter.dispatch("get_period_boundaries", "dbt_utils")( + target_schema, target_table, timestamp_field, start_date, stop_date, period + ) +}} {% endmacro %} -{% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - - {% call statement('period_boundaries', fetch_result=True) -%} - with data as ( - select - coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp as start_timestamp, - coalesce( - {{dbt_utils.dateadd('millisecond', - -1, - "nullif('" ~ stop_date ~ "','')::timestamp")}}, - {{dbt_utils.current_timestamp()}} - ) as stop_timestamp - from "{{target_schema}}"."{{target_table}}" +{% macro default__get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period, +) -%} + +{% call statement('period_boundaries', fetch_result=True) -%} +with + data as ( + select + coalesce( + max("{{timestamp_field}}"), '{{start_date}}' + )::timestamp as start_timestamp, + coalesce( + {{ + dbt_utils.dateadd('millisecond', + -1, + "nullif('" ~ stop_date ~ "','')::timestamp") + }}, + {{ dbt_utils.current_timestamp() }} + ) as stop_timestamp + from "{{target_schema}}"."{{target_table}}" ) - select - start_timestamp, - stop_timestamp, - {{dbt_utils.datediff('start_timestamp', - 'stop_timestamp', - period)}} + 1 as num_periods - from data - {%- endcall %} +select + start_timestamp, + stop_timestamp, + {{ dbt_utils.datediff("start_timestamp", "stop_timestamp", period) }} + + 1 + as num_periods +from data {%- endcall %} {%- endmacro %} -{% macro get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} - {{ return(adapter.dispatch('get_period_sql', 'dbt_utils')(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset)) }} -{% endmacro %} - -{% macro default__get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} + {% macro get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset, +) -%} + {{ + return adapter.dispatch("get_period_sql", "dbt_utils")( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset, + ) + }} + {% endmacro %} + +{% macro default__get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset, +) -%} {%- set period_filter -%} ("{{timestamp_field}}" > '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' and @@ -40,150 +88,159 @@ "{{timestamp_field}}" < '{{stop_timestamp}}'::timestamp) {%- endset -%} - {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} + {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} - select - {{target_cols_csv}} - from ( - {{filtered_sql}} - ) +select {{ target_cols_csv }} +from ({{ filtered_sql }}) {%- endmacro %} -{% materialization insert_by_period, default -%} - {%- set timestamp_field = config.require('timestamp_field') -%} - {%- set start_date = config.require('start_date') -%} - {%- set stop_date = config.get('stop_date') or '' -%} - {%- set period = config.get('period') or 'week' -%} + {% materialization insert_by_period, default -%} + {%- set timestamp_field = config.require("timestamp_field") -%} + {%- set start_date = config.require("start_date") -%} + {%- set stop_date = config.get("stop_date") or "" -%} + {%- set period = config.get("period") or "week" -%} - {%- if sql.find('__PERIOD_FILTER__') == -1 -%} + {%- if sql.find("__PERIOD_FILTER__") == -1 -%} {%- set error_message -%} Model '{{ model.unique_id }}' does not include the required string '__PERIOD_FILTER__' in its sql {%- endset -%} {{ exceptions.raise_compiler_error(error_message) }} - {%- endif -%} - - {%- set identifier = model['name'] -%} - - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} - - {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - - {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} - {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} + {%- endif -%} + + {%- set identifier = model["name"] -%} + + {%- set old_relation = adapter.get_relation( + database=database, schema=schema, identifier=identifier +) -%} + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, type="table" +) -%} + + {%- set non_destructive_mode = flags.NON_DESTRUCTIVE == True -%} + {%- set full_refresh_mode = flags.FULL_REFRESH == True -%} + + {%- set exists_as_table = old_relation is not none and old_relation.is_table -%} + {%- set exists_not_as_table = ( + old_relation is not none and not old_relation.is_table +) -%} + + {%- set should_truncate = ( + non_destructive_mode and full_refresh_mode and exists_as_table +) -%} + {%- set should_drop = not should_truncate and ( + full_refresh_mode or exists_not_as_table +) -%} + {%- set force_create = flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE -%} + + -- setup + {% if old_relation is none -%} + -- noop + {%- elif should_truncate -%} {{ adapter.truncate_relation(old_relation) }} + {%- elif should_drop -%} + {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} + {%- endif %} - {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%} - {%- set should_drop = (not should_truncate and (full_refresh_mode or exists_not_as_table)) -%} - {%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%} + {{ run_hooks(pre_hooks, inside_transaction=False) }} - -- setup - {% if old_relation is none -%} - -- noop - {%- elif should_truncate -%} - {{adapter.truncate_relation(old_relation)}} - {%- elif should_drop -%} - {{adapter.drop_relation(old_relation)}} - {%- set old_relation = none -%} - {%- endif %} - - {{run_hooks(pre_hooks, inside_transaction=False)}} - - -- `begin` happens here, so `commit` after it to finish the transaction - {{run_hooks(pre_hooks, inside_transaction=True)}} - {% call statement() -%} - begin; -- make extra sure we've closed out the transaction - commit; - {%- endcall %} - - -- build model - {% if force_create or old_relation is none -%} - {# Create an empty target table -#} - {% call statement('main') -%} - {%- set empty_sql = sql | replace("__PERIOD_FILTER__", 'false') -%} - {{create_table_as(False, target_relation, empty_sql)}} - {%- endcall %} - {%- endif %} - - {% set _ = dbt_utils.get_period_boundaries(schema, - identifier, - timestamp_field, - start_date, - stop_date, - period) %} - {%- set start_timestamp = load_result('period_boundaries')['data'][0][0] | string -%} - {%- set stop_timestamp = load_result('period_boundaries')['data'][0][1] | string -%} - {%- set num_periods = load_result('period_boundaries')['data'][0][2] | int -%} - - {% set target_columns = adapter.get_columns_in_relation(target_relation) %} - {%- set target_cols_csv = target_columns | map(attribute='quoted') | join(', ') -%} - {%- set loop_vars = {'sum_rows_inserted': 0} -%} - - -- commit each period as a separate transaction - {% for i in range(num_periods) -%} - {%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} - {{ dbt_utils.log_info(msg) }} - - {%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%} - {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, - schema=schema, type='table') -%} + -- `begin` happens here, so `commit` after it to finish the transaction + {{ run_hooks(pre_hooks, inside_transaction=True) }} {% call statement() -%} - {% set tmp_table_sql = dbt_utils.get_period_sql(target_cols_csv, - sql, - timestamp_field, - period, - start_timestamp, - stop_timestamp, - i) %} - {{dbt.create_table_as(True, tmp_relation, tmp_table_sql)}} - {%- endcall %} - - {{adapter.expand_target_column_types(from_relation=tmp_relation, - to_relation=target_relation)}} - {%- set name = 'main-' ~ i -%} - {% call statement(name, fetch_result=True) -%} - insert into {{target_relation}} ({{target_cols_csv}}) - ( - select - {{target_cols_csv}} - from {{tmp_relation.include(schema=False)}} - ); - {%- endcall %} - {% set result = load_result('main-' ~ i) %} - {% if 'response' in result.keys() %} {# added in v0.19.0 #} - {% set rows_inserted = result['response']['rows_affected'] %} - {% else %} {# older versions #} - {% set rows_inserted = result['status'].split(" ")[2] | int %} - {% endif %} - - {%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%} - {%- if loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %} {% endif -%} - - {%- set msg = "Ran for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) ~ "; " ~ rows_inserted ~ " records inserted" -%} - {{ dbt_utils.log_info(msg) }} - - {%- endfor %} - - {% call statement() -%} - begin; - {%- endcall %} - - {{run_hooks(post_hooks, inside_transaction=True)}} - - {% call statement() -%} - commit; - {%- endcall %} - - {{run_hooks(post_hooks, inside_transaction=False)}} - - {%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} - - {% call noop_statement('main', status_string) -%} - -- no-op - {%- endcall %} - - -- Return the relations created in this materialization - {{ return({'relations': [target_relation]}) }} + begin -- make extra sure we've closed out the transaction +; +commit +; +{%- endcall %} + +-- build model +{% if force_create or old_relation is none -%} +{# Create an empty target table -#} +{% call statement('main') -%} +{%- set empty_sql = sql | replace("__PERIOD_FILTER__", "false") -%} +{{ create_table_as(False, target_relation, empty_sql) }} +{%- endcall %} +{%- endif %} + +{% set _ = dbt_utils.get_period_boundaries( + schema, identifier, timestamp_field, start_date, stop_date, period +) %} +{%- set start_timestamp = load_result("period_boundaries")["data"][0][0] | string -%} +{%- set stop_timestamp = load_result("period_boundaries")["data"][0][1] | string -%} +{%- set num_periods = load_result("period_boundaries")["data"][0][2] | int -%} + +{% set target_columns = adapter.get_columns_in_relation(target_relation) %} +{%- set target_cols_csv = target_columns | map(attribute="quoted") | join(", ") -%} +{%- set loop_vars = {"sum_rows_inserted": 0} -%} + +-- commit each period as a separate transaction +{% for i in range(num_periods) -%} +{%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} +{{ dbt_utils.log_info(msg) }} + +{%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%} +{%- set tmp_relation = api.Relation.create( + identifier=tmp_identifier, schema=schema, type="table" +) -%} +{% call statement() -%} +{% set tmp_table_sql = dbt_utils.get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + i, +) %} +{{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} +{%- endcall %} + +{{ + adapter.expand_target_column_types( + from_relation=tmp_relation, to_relation=target_relation + ) +}} +{%- set name = 'main-' ~ i -%} +{% call statement(name, fetch_result=True) -%} +insert into {{ target_relation }} ({{ target_cols_csv }}) +(select {{ target_cols_csv }} from {{ tmp_relation.include(schema=False) }}) +; +{%- endcall %} +{% set result = load_result('main-' ~ i) %} +{% if "response" in result.keys() %} {# added in v0.19.0 #} +{% set rows_inserted = result["response"]["rows_affected"] %} +{# older versions #} +{% else %} {% set rows_inserted = result["status"].split(" ")[2] | int %} +{% endif %} + +{%- set sum_rows_inserted = loop_vars["sum_rows_inserted"] + rows_inserted -%} +{%- if loop_vars.update({"sum_rows_inserted": sum_rows_inserted}) %} {% endif -%} + +{%- set msg = "Ran for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) ~ "; " ~ rows_inserted ~ " records inserted" -%} +{{ dbt_utils.log_info(msg) }} + +{%- endfor %} + +{% call statement() -%} +begin +; +{%- endcall %} + +{{ run_hooks(post_hooks, inside_transaction=True) }} + +{% call statement() -%} +commit +; +{%- endcall %} + +{{ run_hooks(post_hooks, inside_transaction=False) }} + +{%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} + +{% call noop_statement('main', status_string) -%} +-- no-op +{%- endcall %} + +-- Return the relations created in this materialization +{{ return {"relations": [target_relation]} }} {%- endmaterialization %} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index 759f8d59..39f587a0 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -1,21 +1,25 @@ {% macro get_intervals_between(start_date, end_date, datepart) -%} - {{ return(adapter.dispatch('get_intervals_between', 'dbt_utils')(start_date, end_date, datepart)) }} +{{ + return adapter.dispatch("get_intervals_between", "dbt_utils")( + start_date, end_date, datepart + ) +}} {%- endmacro %} {% macro default__get_intervals_between(start_date, end_date, datepart) -%} - {%- call statement('get_intervals_between', fetch_result=True) %} +{%- call statement('get_intervals_between', fetch_result=True) %} - select {{dbt_utils.datediff(start_date, end_date, datepart)}} +select + {{ dbt_utils.datediff(start_date, end_date, datepart) }} {%- endcall -%} - {%- set value_list = load_result('get_intervals_between') -%} + {%- set value_list = load_result("get_intervals_between") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} - {{ return(values[0]) }} - {%- else -%} - {{ return(1) }} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} + {{ return values[0] }} + {%- else -%} {{ return 1 }} {%- endif -%} {%- endmacro %} @@ -23,9 +27,13 @@ -{% macro date_spine(datepart, start_date, end_date) %} - {{ return(adapter.dispatch('date_spine', 'dbt_utils')(datepart, start_date, end_date)) }} -{%- endmacro %} + {% macro date_spine(datepart, start_date, end_date) %} + {{ + return adapter.dispatch("date_spine", "dbt_utils")( + datepart, start_date, end_date + ) + }} + {%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} @@ -37,39 +45,34 @@ date_spine( "to_date('01/01/2016', 'mm/dd/yyyy')", "dateadd(week, 1, current_date)" ) #} +with + rawdata as ( - -with rawdata as ( - - {{dbt_utils.generate_series( - dbt_utils.get_intervals_between(start_date, end_date, datepart) - )}} - -), - -all_periods as ( - - select ( {{ - dbt_utils.dateadd( - datepart, - "row_number() over (order by 1) - 1", - start_date + dbt_utils.generate_series( + dbt_utils.get_intervals_between(start_date, end_date, datepart) ) }} - ) as date_{{datepart}} - from rawdata -), + ), + + all_periods as ( -filtered as ( + select + ( + {{ + dbt_utils.dateadd( + datepart, "row_number() over (order by 1) - 1", start_date + ) + }} + ) as date_{{ datepart }} + from rawdata - select * - from all_periods - where date_{{datepart}} <= {{ end_date }} + ), -) + filtered as (select * from all_periods where date_{{ datepart }} <= {{ end_date }}) -select * from filtered +select * +from filtered {% endmacro %} diff --git a/macros/sql/deduplicate.sql b/macros/sql/deduplicate.sql index 9a3571a2..04e8c546 100644 --- a/macros/sql/deduplicate.sql +++ b/macros/sql/deduplicate.sql @@ -1,23 +1,27 @@ {%- macro deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} - {{ return(adapter.dispatch('deduplicate', 'dbt_utils')(relation, group_by, order_by=order_by, relation_alias=relation_alias)) }} +{{ + return adapter.dispatch("deduplicate", "dbt_utils")( + relation, group_by, order_by=order_by, relation_alias=relation_alias + ) +}} {% endmacro %} -{%- macro default__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro default__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( +select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} +from + ( select _inner.*, row_number() over ( partition by {{ group_by }} - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} ) as rn from {{ relation if relation_alias is none else relation_alias }} as _inner ) as deduped - where deduped.rn = 1 +where deduped.rn = 1 {%- endmacro -%} @@ -26,19 +30,19 @@ -- clause in BigQuery: -- https://github.com/dbt-labs/dbt-utils/issues/335#issuecomment-788157572 #} -{%- macro bigquery__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro bigquery__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( +select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} +from + ( select - array_agg ( + array_agg( original - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} limit 1 - )[offset(0)] as deduped + ) [offset (0)] as deduped from {{ relation if relation_alias is none else relation_alias }} as original group by {{ group_by }} ) diff --git a/macros/sql/generate_series.sql b/macros/sql/generate_series.sql index efcbd8ac..417dbfb2 100644 --- a/macros/sql/generate_series.sql +++ b/macros/sql/generate_series.sql @@ -1,53 +1,50 @@ {% macro get_powers_of_two(upper_bound) %} - {{ return(adapter.dispatch('get_powers_of_two', 'dbt_utils')(upper_bound)) }} +{{ return adapter.dispatch("get_powers_of_two", "dbt_utils")(upper_bound) }} {% endmacro %} {% macro default__get_powers_of_two(upper_bound) %} - {% if upper_bound <= 0 %} - {{ exceptions.raise_compiler_error("upper bound must be positive") }} - {% endif %} +{% if upper_bound <= 0 %} +{{ exceptions.raise_compiler_error("upper bound must be positive") }} +{% endif %} - {% for _ in range(1, 100) %} - {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} - {% endfor %} +{% for _ in range(1, 100) %} +{% if upper_bound <= 2**loop.index %} {{ return loop.index }}{% endif %} +{% endfor %} {% endmacro %} {% macro generate_series(upper_bound) %} - {{ return(adapter.dispatch('generate_series', 'dbt_utils')(upper_bound)) }} +{{ return adapter.dispatch("generate_series", "dbt_utils")(upper_bound) }} {% endmacro %} {% macro default__generate_series(upper_bound) %} - {% set n = dbt_utils.get_powers_of_two(upper_bound) %} +{% set n = dbt_utils.get_powers_of_two(upper_bound) %} - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( +with + p as (select 0 as generated_number union all select 1), + unioned as ( - select + select - {% for i in range(n) %} - p{{i}}.generated_number * power(2, {{i}}) - {% if not loop.last %} + {% endif %} - {% endfor %} - + 1 - as generated_number + {% for i in range(n) %} + p{{ i }}.generated_number * power(2, {{ i }}) + {% if not loop.last %} + {% endif %} + {% endfor %} + 1 as generated_number - from + from - {% for i in range(n) %} - p as p{{i}} - {% if not loop.last %} cross join {% endif %} - {% endfor %} + {% for i in range(n) %} + p as p{{ i }} {% if not loop.last %} cross join {% endif %} + {% endfor %} ) - select * - from unioned - where generated_number <= {{upper_bound}} - order by generated_number +select * +from unioned +where generated_number <= {{ upper_bound }} +order by generated_number {% endmacro %} diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index f70890e2..4aa08820 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -1,60 +1,63 @@ -{% macro get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {{ return(adapter.dispatch('get_column_values', 'dbt_utils')(table, column, order_by, max_records, default)) }} +{% macro get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} +{{ + return adapter.dispatch("get_column_values", "dbt_utils")( + table, column, order_by, max_records, default + ) +}} {% endmacro %} -{% macro default__get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {% set default = [] if not default %} - {{ return(default) }} - {% endif %} +{% macro default__get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} +{% set default = [] if not default %} {{ return default }} +{% endif %} - {%- do dbt_utils._is_ephemeral(table, 'get_column_values') -%} +{%- do dbt_utils._is_ephemeral(table, "get_column_values") -%} - {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} - {# TODO: Change the method signature in a future 0.x.0 release #} - {%- set target_relation = table -%} +{# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} +{# TODO: Change the method signature in a future 0.x.0 release #} +{%- set target_relation = table -%} - {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} - {% set relation_exists = (load_relation(target_relation)) is not none %} +{# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} +{% set relation_exists = (load_relation(target_relation)) is not none %} - {%- call statement('get_column_values', fetch_result=true) %} +{%- call statement('get_column_values', fetch_result=true) %} - {%- if not relation_exists and default is none -%} +{%- if not relation_exists and default is none -%} - {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} +{{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} - {%- elif not relation_exists and default is not none -%} +{%- elif not relation_exists and default is not none -%} - {{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} +{{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} - {{ return(default) }} +{{ return default }} - {%- else -%} +{%- else -%} - select - {{ column }} as value +select {{ column }} as value - from {{ target_relation }} - group by {{ column }} - order by {{ order_by }} +from {{ target_relation }} +group by {{ column }} +order by {{ order_by }} - {% if max_records is not none %} - limit {{ max_records }} - {% endif %} +{% if max_records is not none %} limit {{ max_records }} +{% endif %} - {% endif %} +{% endif %} {%- endcall -%} - {%- set value_list = load_result('get_column_values') -%} + {%- set value_list = load_result("get_column_values") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} - {{ return(values) }} - {%- else -%} - {{ return(default) }} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} {{ return values }} + {%- else -%} {{ return default }} {%- endif -%} {%- endmacro %} diff --git a/macros/sql/get_filtered_columns_in_relation.sql b/macros/sql/get_filtered_columns_in_relation.sql index 7f4af889..4885c473 100644 --- a/macros/sql/get_filtered_columns_in_relation.sql +++ b/macros/sql/get_filtered_columns_in_relation.sql @@ -1,25 +1,23 @@ {% macro get_filtered_columns_in_relation(from, except=[]) -%} - {{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} +{{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} {% endmacro %} {% macro default__get_filtered_columns_in_relation(from, except=[]) -%} - {%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} - {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} +{%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} +{%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} - {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('') }} - {% endif %} +{# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return "" }} {% endif %} - {%- set include_cols = [] %} - {%- set cols = adapter.get_columns_in_relation(from) -%} - {%- set except = except | map("lower") | list %} - {%- for col in cols -%} - {%- if col.column|lower not in except -%} - {% do include_cols.append(col.column) %} - {%- endif %} - {%- endfor %} +{%- set include_cols = [] %} +{%- set cols = adapter.get_columns_in_relation(from) -%} +{%- set except = except | map("lower") | list %} +{%- for col in cols -%} +{%- if col.column|lower not in except -%} +{% do include_cols.append(col.column) %} +{%- endif %} +{%- endfor %} - {{ return(include_cols) }} +{{ return include_cols }} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/sql/get_query_results_as_dict.sql b/macros/sql/get_query_results_as_dict.sql index 6548f2dd..e0136922 100644 --- a/macros/sql/get_query_results_as_dict.sql +++ b/macros/sql/get_query_results_as_dict.sql @@ -1,26 +1,25 @@ {% macro get_query_results_as_dict(query) %} - {{ return(adapter.dispatch('get_query_results_as_dict', 'dbt_utils')(query)) }} +{{ return adapter.dispatch("get_query_results_as_dict", "dbt_utils")(query) }} {% endmacro %} {% macro default__get_query_results_as_dict(query) %} {# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} +{%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} - {%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} +{{ query }} - {{ query }} +{%- endcall -%} - {%- endcall -%} +{% set sql_results = {} %} - {% set sql_results={} %} +{%- if execute -%} +{% set sql_results_table = load_result("get_query_results").table.columns %} +{% for column_name, column in sql_results_table.items() %} +{% do sql_results.update({column_name: column.values()}) %} +{% endfor %} +{%- endif -%} - {%- if execute -%} - {% set sql_results_table = load_result('get_query_results').table.columns %} - {% for column_name, column in sql_results_table.items() %} - {% do sql_results.update({column_name: column.values()}) %} - {% endfor %} - {%- endif -%} - - {{ return(sql_results) }} +{{ return sql_results }} {% endmacro %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 9325a883..749f83f3 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -1,32 +1,42 @@ -{% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_pattern', 'dbt_utils')(schema_pattern, table_pattern, exclude, database)) }} +{% macro get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} +{{ + return adapter.dispatch("get_relations_by_pattern", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) +}} {% endmacro %} -{% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro default__get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} +{%- call statement('get_tables', fetch_result=True) %} - {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} +{{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude, database + ) +}} - {%- endcall -%} +{%- endcall -%} - {%- set table_list = load_result('get_tables') -%} +{%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} - {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} - {%- set tbl_relation = api.Relation.create( - database=database, - schema=row.table_schema, - identifier=row.table_name, - type=row.table_type - ) -%} - {%- do tbl_relations.append(tbl_relation) -%} - {%- endfor -%} +{%- if table_list and table_list["table"] -%} +{%- set tbl_relations = [] -%} +{%- for row in table_list["table"] -%} +{%- set tbl_relation = api.Relation.create( + database=database, + schema=row.table_schema, + identifier=row.table_name, + type=row.table_type, +) -%} {%- do tbl_relations.append(tbl_relation) -%} +{%- endfor -%} - {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} - {%- endif -%} +{{ return tbl_relations }} +{%- else -%} {{ return [] }} +{%- endif -%} {% endmacro %} diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index b6733c4b..286ca672 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -1,32 +1,38 @@ -{% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_prefix', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} +{{ + return adapter.dispatch("get_relations_by_prefix", "dbt_utils")( + schema, prefix, exclude, database + ) +}} {% endmacro %} -{% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} +{%- call statement('get_tables', fetch_result=True) %} - {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} +{{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} - {%- endcall -%} +{%- endcall -%} - {%- set table_list = load_result('get_tables') -%} +{%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} - {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} - {%- set tbl_relation = api.Relation.create( - database=database, - schema=row.table_schema, - identifier=row.table_name, - type=row.table_type - ) -%} - {%- do tbl_relations.append(tbl_relation) -%} - {%- endfor -%} +{%- if table_list and table_list["table"] -%} +{%- set tbl_relations = [] -%} +{%- for row in table_list["table"] -%} +{%- set tbl_relation = api.Relation.create( + database=database, + schema=row.table_schema, + identifier=row.table_name, + type=row.table_type, +) -%} {%- do tbl_relations.append(tbl_relation) -%} +{%- endfor -%} - {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} - {%- endif -%} +{{ return tbl_relations }} +{%- else -%} {{ return [] }} +{%- endif -%} {% endmacro %} diff --git a/macros/sql/get_table_types_sql.sql b/macros/sql/get_table_types_sql.sql index 91573779..fede5af4 100644 --- a/macros/sql/get_table_types_sql.sql +++ b/macros/sql/get_table_types_sql.sql @@ -1,31 +1,43 @@ {%- macro get_table_types_sql() -%} - {{ return(adapter.dispatch('get_table_types_sql', 'dbt_utils')()) }} +{{ return adapter.dispatch("get_table_types_sql", "dbt_utils")() }} {%- endmacro -%} {% macro default__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" +case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) +end as "table_type" {% endmacro %} {% macro postgres__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'FOREIGN' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" +case + table_type + when 'BASE TABLE' + then 'table' + when 'FOREIGN' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) +end as "table_type" {% endmacro %} {% macro bigquery__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as `table_type` -{% endmacro %} \ No newline at end of file +case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) +end as `table_type` +{% endmacro %} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index 4d5a8fc9..72356757 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -1,28 +1,39 @@ -{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_pattern_sql', 'dbt_utils') - (schema_pattern, table_pattern, exclude, database)) }} +{% macro get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} +{{ + return adapter.dispatch("get_tables_by_pattern_sql", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) +}} {% endmacro %} -{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro default__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - select distinct - table_schema as "table_schema", - table_name as "table_name", - {{ dbt_utils.get_table_types_sql() }} - from {{ database }}.information_schema.tables - where table_schema ilike '{{ schema_pattern }}' - and table_name ilike '{{ table_pattern }}' - and table_name not ilike '{{ exclude }}' +select distinct + table_schema as "table_schema", + table_name as "table_name", + {{ dbt_utils.get_table_types_sql() }} +from {{ database }}.information_schema.tables +where + table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' {% endmacro %} -{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + {% macro bigquery__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {% if '%' in schema_pattern %} - {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %} - {% else %} - {% set schemata=[schema_pattern] %} + {% if "%" in schema_pattern %} + {% set schemata = dbt_utils._bigquery__get_matching_schemata( + schema_pattern, database +) %} + {% else %} {% set schemata = [schema_pattern] %} {% endif %} {% set sql %} @@ -41,12 +52,12 @@ {% endfor %} {% endset %} - {{ return(sql) }} + {{ return sql }} -{% endmacro %} + {% endmacro %} -{% macro _bigquery__get_matching_schemata(schema_pattern, database) %} + {% macro _bigquery__get_matching_schemata(schema_pattern, database) %} {% if execute %} {% set sql %} @@ -54,17 +65,15 @@ where lower(schema_name) like lower('{{ schema_pattern }}') {% endset %} - {% set results=run_query(sql) %} - - {% set schemata=results.columns['schema_name'].values() %} + {% set results = run_query(sql) %} - {{ return(schemata) }} + {% set schemata = results.columns["schema_name"].values() %} - {% else %} + {{ return schemata }} - {{ return([]) }} + {% else %} {{ return [] }} {% endif %} -{% endmacro %} + {% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index f8fdfba6..5d956e24 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,14 +1,24 @@ -{% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_prefix_sql', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} +{{ + return adapter.dispatch("get_tables_by_prefix_sql", "dbt_utils")( + schema, prefix, exclude, database + ) +}} {% endmacro %} -{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} + +{{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern = schema, + table_pattern = prefix ~ '%', + exclude = exclude, + database = database + ) +}} - {{ dbt_utils.get_tables_by_pattern_sql( - schema_pattern = schema, - table_pattern = prefix ~ '%', - exclude = exclude, - database = database - ) }} - {% endmacro %} diff --git a/macros/sql/groupby.sql b/macros/sql/groupby.sql index 68a68cf2..ad6645b5 100644 --- a/macros/sql/groupby.sql +++ b/macros/sql/groupby.sql @@ -1,11 +1,10 @@ {%- macro group_by(n) -%} - {{ return(adapter.dispatch('group_by', 'dbt_utils')(n)) }} +{{ return adapter.dispatch("group_by", "dbt_utils")(n) }} {% endmacro %} {%- macro default__group_by(n) -%} - group by {% for i in range(1, n + 1) -%} - {{ i }}{{ ',' if not loop.last }} - {%- endfor -%} +group by + {% for i in range(1, n + 1) -%} {{ i }}{{ ',' if not loop.last }} {%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 70f276ec..55522a3c 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -5,47 +5,51 @@ http://daynebatten.com/2015/09/latitude-longitude-distance-sql/ The arguments should be float type. #} - -{% macro degrees_to_radians(degrees) -%} - acos(-1) * {{degrees}} / 180 -{%- endmacro %} - -{% macro haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} - {{ return(adapter.dispatch('haversine_distance', 'dbt_utils')(lat1,lon1,lat2,lon2,unit)) }} +{% macro degrees_to_radians(degrees) -%} acos(-1) * {{ degrees }} / 180 {%- endmacro %} + +{% macro haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} +{{ + return adapter.dispatch("haversine_distance", "dbt_utils")( + lat1, lon1, lat2, lon2, unit + ) +}} {% endmacro %} -{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} +{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} +{%- if unit == "mi" %} {% set conversion_rate = 1 %} +{% elif unit == "km" %} {% set conversion_rate = 1.60934 %} {% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} +{{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} {% endif %} - 2 * 3961 * asin(sqrt(power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + - cos(radians({{lat1}})) * cos(radians({{lat2}})) * - power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2))) * {{ conversion_rate }} +2 * 3961 * asin( + sqrt( + power( (sin(radians( ({{ lat2 }} - {{ lat1 }}) / 2))), 2) + + cos(radians({{ lat1 }})) * cos(radians({{ lat2 }})) * + power( (sin(radians( ({{ lon2 }} - {{ lon1 }}) / 2))), 2) + ) +) * {{ conversion_rate }} {%- endmacro %} -{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} +{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} {% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} {% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} {% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} {% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} +{%- if unit == "mi" %} {% set conversion_rate = 1 %} +{% elif unit == "km" %} {% set conversion_rate = 1.60934 %} {% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} +{{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} {% endif %} - 2 * 3961 * asin(sqrt(power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + - cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * - power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2))) * {{ conversion_rate }} +2 * 3961 * asin( + sqrt( + power(sin( ({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + + cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * + power(sin( ({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2) + ) +) * {{ conversion_rate }} {%- endmacro %} - diff --git a/macros/sql/nullcheck.sql b/macros/sql/nullcheck.sql index 509d24f7..b6ed126b 100644 --- a/macros/sql/nullcheck.sql +++ b/macros/sql/nullcheck.sql @@ -1,21 +1,17 @@ {% macro nullcheck(cols) %} - {{ return(adapter.dispatch('nullcheck', 'dbt_utils')(cols)) }} +{{ return adapter.dispatch("nullcheck", "dbt_utils")(cols) }} {% endmacro %} {% macro default__nullcheck(cols) %} {%- for col in cols %} - {% if col.is_string() -%} +{% if col.is_string() -%} nullif({{ col.name }}, '') as {{ col.name }} - nullif({{col.name}},'') as {{col.name}} +{%- else -%} {{ col.name }} - {%- else -%} +{%- endif -%} - {{col.name}} - - {%- endif -%} - -{%- if not loop.last -%} , {%- endif -%} +{%- if not loop.last -%}, {%- endif -%} {%- endfor -%} {% endmacro %} diff --git a/macros/sql/nullcheck_table.sql b/macros/sql/nullcheck_table.sql index c9ab5838..65fa7b81 100644 --- a/macros/sql/nullcheck_table.sql +++ b/macros/sql/nullcheck_table.sql @@ -1,14 +1,14 @@ {% macro nullcheck_table(relation) %} - {{ return(adapter.dispatch('nullcheck_table', 'dbt_utils')(relation)) }} +{{ return adapter.dispatch("nullcheck_table", "dbt_utils")(relation) }} {% endmacro %} {% macro default__nullcheck_table(relation) %} - {%- do dbt_utils._is_relation(relation, 'nullcheck_table') -%} - {%- do dbt_utils._is_ephemeral(relation, 'nullcheck_table') -%} - {% set cols = adapter.get_columns_in_relation(relation) %} +{%- do dbt_utils._is_relation(relation, "nullcheck_table") -%} +{%- do dbt_utils._is_ephemeral(relation, "nullcheck_table") -%} +{% set cols = adapter.get_columns_in_relation(relation) %} - select {{ dbt_utils.nullcheck(cols) }} - from {{relation}} +select {{ dbt_utils.nullcheck(cols) }} +from {{ relation }} {% endmacro %} diff --git a/macros/sql/pivot.sql b/macros/sql/pivot.sql index 88751062..2d09a52d 100644 --- a/macros/sql/pivot.sql +++ b/macros/sql/pivot.sql @@ -39,48 +39,63 @@ Arguments: quote_identifiers: Whether to surround column aliases with double quotes, default is true distinct: Whether to use distinct in the aggregation, default is False #} - -{% macro pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {{ return(adapter.dispatch('pivot', 'dbt_utils')(column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers, distinct)) }} +{% macro pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False, +) %} +{{ + return adapter.dispatch("pivot", "dbt_utils")( + column, + values, + alias, + agg, + cmp, + prefix, + suffix, + then_value, + else_value, + quote_identifiers, + distinct, + ) +}} {% endmacro %} -{% macro default__pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {% for value in values %} - {{ agg }}( - {% if distinct %} distinct {% endif %} - case - when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' +{% macro default__pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False, +) %} +{% for value in values %} +{{ agg }} ( + {% if distinct %}distinct {% endif %} + case + when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' then {{ then_value }} - else {{ else_value }} - end - ) - {% if alias %} - {% if quote_identifiers %} - as {{ adapter.quote(prefix ~ value ~ suffix) }} - {% else %} - as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} - {% endif %} - {% endif %} - {% if not loop.last %},{% endif %} - {% endfor %} + else {{ else_value }} + end +) +{% if alias %} +{% if quote_identifiers %} as {{ adapter.quote(prefix ~ value ~ suffix) }} +{% else %} as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} +{% endif %} +{% endif %} +{% if not loop.last %},{% endif %} +{% endfor %} {% endmacro %} diff --git a/macros/sql/safe_add.sql b/macros/sql/safe_add.sql index 3b6195c8..d75f9b26 100644 --- a/macros/sql/safe_add.sql +++ b/macros/sql/safe_add.sql @@ -1,8 +1,8 @@ {%- macro safe_add() -%} - {# needed for safe_add to allow for non-keyword arguments see SO post #} - {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} - {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('safe_add', 'dbt_utils')(*varargs)) }} +{# needed for safe_add to allow for non-keyword arguments see SO post #} +{# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} +{% set frustrating_jinja_feature = varargs %} +{{ return adapter.dispatch("safe_add", "dbt_utils")(*varargs) }} {% endmacro %} {%- macro default__safe_add() -%} @@ -11,10 +11,10 @@ {%- for field in varargs -%} - {% do fields.append("coalesce(" ~ field ~ ", 0)") %} +{% do fields.append("coalesce(" ~ field ~ ", 0)") %} {%- endfor -%} -{{ fields|join(' +\n ') }} +{{ fields | join(" +\n ") }} {%- endmacro -%} diff --git a/macros/sql/star.sql b/macros/sql/star.sql index 25e58969..9a65b80e 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -1,20 +1,23 @@ {% macro star(from, relation_alias=False, except=[], prefix='', suffix='') -%} - {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} +{{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} {% endmacro %} {% macro default__star(from, relation_alias=False, except=[], prefix='', suffix='') -%} - {%- do dbt_utils._is_relation(from, 'star') -%} - {%- do dbt_utils._is_ephemeral(from, 'star') -%} +{%- do dbt_utils._is_relation(from, 'star') -%} +{%- do dbt_utils._is_ephemeral(from, 'star') -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('*') }} - {% endif %} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return "*" }} {% endif %} - {%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} +{%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} - {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} {%- if prefix!='' or suffix!='' %} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- endif -%} - {%- if not loop.last %},{{ '\n ' }}{% endif %} +{%- if relation_alias %} {{ relation_alias }}. +{% else %} +{%- endif -%} {{ adapter.quote(col) | trim }} +{%- if prefix != "" or suffix != "" %} +as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} +{%- endif -%} +{%- if not loop.last %},{{ "\n " }}{% endif %} - {%- endfor -%} +{%- endfor -%} {%- endmacro %} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index ba5c3e44..199cbd42 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -1,33 +1,27 @@ {%- macro surrogate_key(field_list) -%} - {# needed for safe_add to allow for non-keyword arguments see SO post #} - {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} - {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('surrogate_key', 'dbt_utils')(field_list, *varargs)) }} +{# needed for safe_add to allow for non-keyword arguments see SO post #} +{# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} +{% set frustrating_jinja_feature = varargs %} +{{ return adapter.dispatch("surrogate_key", "dbt_utils")(field_list, *varargs) }} {% endmacro %} {%- macro default__surrogate_key(field_list) -%} -{%- if varargs|length >= 1 or field_list is string %} +{%- if varargs | length >= 1 or field_list is string %} -{%- set error_message = ' -Warning: the `surrogate_key` macro now takes a single list argument instead of \ -multiple string arguments. Support for multiple string arguments will be \ -deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ -'.format(model.package_name, model.name) -%} +{%- set error_message = " Warning: the `surrogate_key` macro now takes a single list argument instead of \ multiple string arguments. Support for multiple string arguments will be \ deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name +) -%} {%- do exceptions.warn(error_message) -%} {# first argument is not included in varargs, so add first element to field_list_xf #} {%- set field_list_xf = [field_list] -%} -{%- for field in varargs %} -{%- set _ = field_list_xf.append(field) -%} -{%- endfor -%} - -{%- else -%} +{%- for field in varargs %} {%- set _ = field_list_xf.append(field) -%} {%- endfor -%} {# if using list, just set field_list_xf as field_list #} -{%- set field_list_xf = field_list -%} +{%- else -%} {%- set field_list_xf = field_list -%} {%- endif -%} @@ -36,16 +30,12 @@ deprecated in a future release of dbt-utils. The {}.{} model triggered this warn {%- for field in field_list_xf -%} - {%- set _ = fields.append( +{%- set _ = fields.append( "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" - ) -%} - - {%- if not loop.last %} - {%- set _ = fields.append("'-'") -%} - {%- endif -%} + ) -%} {%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} {%- endfor -%} -{{dbt_utils.hash(dbt_utils.concat(fields))}} +{{ dbt_utils.hash(dbt_utils.concat(fields)) }} {%- endmacro -%} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index a7bf1d95..c71de746 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -1,67 +1,83 @@ -{%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} - {{ return(adapter.dispatch('union_relations', 'dbt_utils')(relations, column_override, include, exclude, source_column_name)) }} +{%- macro union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation", +) -%} +{{ + return adapter.dispatch("union_relations", "dbt_utils")( + relations, column_override, include, exclude, source_column_name + ) +}} {% endmacro %} -{%- macro default__union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} +{%- macro default__union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation", +) -%} - {%- if exclude and include -%} - {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} - {%- endif -%} +{%- if exclude and include -%} +{{ + exceptions.raise_compiler_error( + "Both an exclude and include list were provided to the `union` macro. Only one is allowed" + ) +}} +{%- endif -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} - {%- if not execute %} - {{ return('') }} - {% endif -%} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} +{%- if not execute %} {{ return "" }} {% endif -%} - {%- set column_override = column_override if column_override is not none else {} -%} +{%- set column_override = column_override if column_override is not none else {} -%} - {%- set relation_columns = {} -%} - {%- set column_superset = {} -%} +{%- set relation_columns = {} -%} +{%- set column_superset = {} -%} - {%- for relation in relations -%} +{%- for relation in relations -%} - {%- do relation_columns.update({relation: []}) -%} +{%- do relation_columns.update({relation: []}) -%} - {%- do dbt_utils._is_relation(relation, 'union_relations') -%} - {%- do dbt_utils._is_ephemeral(relation, 'union_relations') -%} - {%- set cols = adapter.get_columns_in_relation(relation) -%} - {%- for col in cols -%} +{%- do dbt_utils._is_relation(relation, "union_relations") -%} +{%- do dbt_utils._is_ephemeral(relation, "union_relations") -%} +{%- set cols = adapter.get_columns_in_relation(relation) -%} +{%- for col in cols -%} - {#- If an exclude list was provided and the column is in the list, do nothing -#} - {%- if exclude and col.column in exclude -%} +{#- If an exclude list was provided and the column is in the list, do nothing -#} +{%- if exclude and col.column in exclude -%} - {#- If an include list was provided and the column is not in the list, do nothing -#} - {%- elif include and col.column not in include -%} +{#- If an include list was provided and the column is not in the list, do nothing -#} +{%- elif include and col.column not in include -%} - {#- Otherwise add the column to the column superset -#} - {%- else -%} +{#- Otherwise add the column to the column superset -#} +{%- else -%} - {#- update the list of columns in this relation -#} - {%- do relation_columns[relation].append(col.column) -%} +{#- update the list of columns in this relation -#} +{%- do relation_columns[relation].append(col.column) -%} - {%- if col.column in column_superset -%} +{%- if col.column in column_superset -%} - {%- set stored = column_superset[col.column] -%} - {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} +{%- set stored = column_superset[col.column] -%} +{%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} - {%- do column_superset.update({col.column: col}) -%} +{%- do column_superset.update({col.column: col}) -%} - {%- endif %} +{%- endif %} - {%- else -%} +{%- else -%} {%- do column_superset.update({col.column: col}) -%} - {%- do column_superset.update({col.column: col}) -%} +{%- endif -%} - {%- endif -%} +{%- endif -%} - {%- endif -%} +{%- endfor -%} +{%- endfor -%} - {%- endfor -%} - {%- endfor -%} +{%- set ordered_column_names = column_superset.keys() -%} - {%- set ordered_column_names = column_superset.keys() -%} - - {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} +{% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} {%- set relations_string -%} {%- for relation in relations -%} {{ relation.name }} @@ -73,31 +89,36 @@ There were no columns found to union for relations {{ relations_string }} {%- endset -%} - {{ exceptions.raise_compiler_error(error_message) }} - {%- endif -%} +{{ exceptions.raise_compiler_error(error_message) }} +{%- endif -%} - {%- for relation in relations %} +{%- for relation in relations %} - ( - select +( + select - cast({{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }}) as {{ source_column_name }}, - {% for col_name in ordered_column_names -%} + cast( + {{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }} + ) as {{ source_column_name }}, + {% for col_name in ordered_column_names -%} - {%- set col = column_superset[col_name] %} - {%- set col_type = column_override.get(col.column, col.data_type) %} - {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %} - cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%} + {%- set col = column_superset[col_name] %} + {%- set col_type = column_override.get(col.column, col.data_type) %} + {%- set col_name = ( + adapter.quote(col_name) + if col_name in relation_columns[relation] + else "null" +) %} + cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} + {% if not loop.last %},{% endif -%} - {%- endfor %} + {%- endfor %} - from {{ relation }} - ) + from {{ relation }} +) - {% if not loop.last -%} - union all - {% endif -%} +{% if not loop.last -%} union all {% endif -%} - {%- endfor -%} +{%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index 8821aad2..8ee6af08 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -11,69 +11,90 @@ Arguments: field_name: Destination table column name for the source table column names. value_name: Destination table column name for the pivoted values #} - -{% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} - {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name, table)) }} +{% macro unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none, +) -%} +{{ + return adapter.dispatch("unpivot", "dbt_utils")( + relation, cast_to, exclude, remove, field_name, value_name, table + ) +}} {% endmacro %} -{% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} - - {% if table %} - {%- set error_message = ' - Warning: the `unpivot` macro no longer accepts a `table` parameter. \ - This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ - The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - {% endif %} - - {% if relation and table %} - {{ exceptions.raise_compiler_error("Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`).") }} - {% elif not relation and table %} - {% set relation=table %} - {% elif not relation and not table %} - {{ exceptions.raise_compiler_error("Error: argument `relation` is required for `unpivot` macro.") }} - {% endif %} - - {%- set exclude = exclude if exclude is not none else [] %} - {%- set remove = remove if remove is not none else [] %} - - {%- set include_cols = [] %} - - {%- set table_columns = {} %} - - {%- do table_columns.update({relation: []}) %} - - {%- do dbt_utils._is_relation(relation, 'unpivot') -%} - {%- do dbt_utils._is_ephemeral(relation, 'unpivot') -%} - {%- set cols = adapter.get_columns_in_relation(relation) %} - - {%- for col in cols -%} - {%- if col.column.lower() not in remove|map('lower') and col.column.lower() not in exclude|map('lower') -%} - {% do include_cols.append(col) %} - {%- endif %} - {%- endfor %} - - - {%- for col in include_cols -%} - select - {%- for exclude_col in exclude %} - {{ exclude_col }}, - {%- endfor %} - - cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, - cast( {% if col.data_type == 'boolean' %} - {{ dbt_utils.cast_bool_to_text(col.column) }} - {% else %} - {{ col.column }} - {% endif %} - as {{ cast_to }}) as {{ value_name }} - - from {{ relation }} - - {% if not loop.last -%} - union all - {% endif -%} - {%- endfor -%} +{% macro default__unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none, +) -%} + +{% if table %} +{%- set error_message = " Warning: the `unpivot` macro no longer accepts a `table` parameter. \ This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name +) -%} +{%- do exceptions.warn(error_message) -%} +{% endif %} + +{% if relation and table %} +{{ + exceptions.raise_compiler_error( + "Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`)." + ) +}} +{% elif not relation and table %} {% set relation = table %} +{% elif not relation and not table %} +{{ + exceptions.raise_compiler_error( + "Error: argument `relation` is required for `unpivot` macro." + ) +}} +{% endif %} + +{%- set exclude = exclude if exclude is not none else [] %} +{%- set remove = remove if remove is not none else [] %} + +{%- set include_cols = [] %} + +{%- set table_columns = {} %} + +{%- do table_columns.update({relation: []}) %} + +{%- do dbt_utils._is_relation(relation, "unpivot") -%} +{%- do dbt_utils._is_ephemeral(relation, "unpivot") -%} +{%- set cols = adapter.get_columns_in_relation(relation) %} + +{%- for col in cols -%} +{%- if col.column.lower() not in remove | map( + "lower" +) and col.column.lower() not in exclude | map("lower") -%} +{% do include_cols.append(col) %} +{%- endif %} +{%- endfor %} + + +{%- for col in include_cols -%} +select + {%- for exclude_col in exclude %} {{ exclude_col }}, {%- endfor %} + + cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, + cast( + {% if col.data_type == "boolean" %}{{ dbt_utils.cast_bool_to_text(col.column) }} + {% else %}{{ col.column }} + {% endif %} as {{ cast_to }} + ) as {{ value_name }} + +from {{ relation }} + +{% if not loop.last -%} union all {% endif -%} +{%- endfor -%} {%- endmacro %} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index 43a58cc4..5320d5bb 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -1,27 +1,25 @@ {% macro get_url_host(field) -%} - {{ return(adapter.dispatch('get_url_host', 'dbt_utils')(field)) }} +{{ return adapter.dispatch("get_url_host", "dbt_utils")(field) }} {% endmacro %} {% macro default__get_url_host(field) -%} -{%- set parsed = +{%- set parsed = dbt_utils.split_part( dbt_utils.split_part( - dbt_utils.split_part( + dbt_utils.replace( dbt_utils.replace( - dbt_utils.replace( - dbt_utils.replace(field, "'android-app://'", "''" - ), "'http://'", "''" - ), "'https://'", "''" - ), "'/'", 1 - ), "'?'", 1 - ) - --%} - - - {{ dbt_utils.safe_cast( - parsed, - dbt_utils.type_string() - )}} + dbt_utils.replace(field, "'android-app://'", "''"), + "'http://'", + "''", + ), + "'https://'", + "''", + ), + "'/'", + 1, + ), + "'?'", + 1, +) -%} {{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} {%- endmacro %} diff --git a/macros/web/get_url_parameter.sql b/macros/web/get_url_parameter.sql index f363fcbb..ab183b17 100644 --- a/macros/web/get_url_parameter.sql +++ b/macros/web/get_url_parameter.sql @@ -1,13 +1,15 @@ {% macro get_url_parameter(field, url_parameter) -%} - {{ return(adapter.dispatch('get_url_parameter', 'dbt_utils')(field, url_parameter)) }} +{{ return adapter.dispatch("get_url_parameter", "dbt_utils")(field, url_parameter) }} {% endmacro %} {% macro default__get_url_parameter(field, url_parameter) -%} {%- set formatted_url_parameter = "'" + url_parameter + "='" -%} -{%- set split = dbt_utils.split_part(dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1) -%} +{%- set split = dbt_utils.split_part( + dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1 +) -%} -nullif({{ split }},'') +nullif({{ split }}, '') {%- endmacro %} diff --git a/macros/web/get_url_path.sql b/macros/web/get_url_path.sql index 4d224277..0548fd23 100644 --- a/macros/web/get_url_path.sql +++ b/macros/web/get_url_path.sql @@ -1,13 +1,12 @@ {% macro get_url_path(field) -%} - {{ return(adapter.dispatch('get_url_path', 'dbt_utils')(field)) }} +{{ return adapter.dispatch("get_url_path", "dbt_utils")(field) }} {% endmacro %} {% macro default__get_url_path(field) -%} - {%- set stripped_url = - dbt_utils.replace( - dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''") - -%} +{%- set stripped_url = dbt_utils.replace( + dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''" +) -%} {%- set first_slash_pos -%} coalesce( @@ -16,19 +15,15 @@ ) {%- endset -%} - {%- set parsed_path = +{%- set parsed_path = dbt_utils.split_part( dbt_utils.right( stripped_url, dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos ), "'?'", 1 - ) - -%} + ) -%} + +{{ dbt_utils.safe_cast(parsed_path, dbt_utils.type_string()) }} - {{ dbt_utils.safe_cast( - parsed_path, - dbt_utils.type_string() - )}} - {%- endmacro %} From dbbc827e9f1e92414c991f85d9216d4bb889561b Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Wed, 11 May 2022 16:19:33 -0600 Subject: [PATCH 03/22] chore: apply sqlfmt 59e52a1073180668ddc6c53a2d1eb9450ebff329 --- .../macros/assert_equal_values.sql | 13 +- integration_tests/macros/limit_zero.sql | 10 +- .../models/cross_db_utils/test_any_value.sql | 41 +- .../models/cross_db_utils/test_bool_or.sql | 8 +- .../models/cross_db_utils/test_concat.sql | 10 +- .../cross_db_utils/test_current_timestamp.sql | 1 - .../test_current_timestamp_in_utc.sql | 2 +- .../models/cross_db_utils/test_date_trunc.sql | 10 +- .../models/cross_db_utils/test_dateadd.sql | 34 +- .../models/cross_db_utils/test_datediff.sql | 112 ++++-- .../models/cross_db_utils/test_hash.sql | 10 +- .../models/cross_db_utils/test_last_day.sql | 15 +- .../models/cross_db_utils/test_length.sql | 13 +- .../models/cross_db_utils/test_listagg.sql | 130 +++--- .../models/cross_db_utils/test_position.sql | 10 +- .../models/cross_db_utils/test_replace.sql | 21 +- .../models/cross_db_utils/test_right.sql | 10 +- .../models/cross_db_utils/test_safe_cast.sql | 8 +- .../models/cross_db_utils/test_split_part.sql | 15 +- .../cross_db_utils/test_width_bucket.sql | 9 +- .../models/datetime/test_date_spine.sql | 22 +- .../test_equal_column_subset.sql | 10 +- .../generic_tests/test_equal_rowcount.sql | 11 +- .../generic_tests/test_fewer_rows_than.sql | 11 +- .../models/generic_tests/test_recency.sql | 11 +- .../models/geo/test_haversine_distance_km.sql | 39 +- .../models/geo/test_haversine_distance_mi.sql | 65 ++- .../expected_insert_by_period.sql | 9 +- .../test_insert_by_period.sql | 26 +- .../models/sql/test_deduplicate.sql | 29 +- .../models/sql/test_generate_series.sql | 12 +- .../models/sql/test_get_column_values.sql | 19 +- .../test_get_filtered_columns_in_relation.sql | 21 +- .../sql/test_get_relations_by_pattern.sql | 13 +- ...test_get_relations_by_prefix_and_union.sql | 4 +- integration_tests/models/sql/test_groupby.sql | 28 +- .../models/sql/test_nullcheck_table.sql | 27 +- integration_tests/models/sql/test_pivot.sql | 16 +- .../models/sql/test_pivot_apostrophe.sql | 14 +- .../models/sql/test_safe_add.sql | 10 +- integration_tests/models/sql/test_star.sql | 15 +- .../models/sql/test_star_aggregate.sql | 19 +- .../models/sql/test_star_prefix_suffix.sql | 18 +- .../models/sql/test_star_uppercase.sql | 15 +- .../models/sql/test_surrogate_key.sql | 14 +- integration_tests/models/sql/test_union.sql | 8 +- .../models/sql/test_union_base.sql | 5 +- integration_tests/models/sql/test_unpivot.sql | 35 +- .../models/sql/test_unpivot_bool.sql | 33 +- .../models/sql/test_unpivot_original_api.sql | 28 +- .../models/web/test_url_host.sql | 13 +- .../models/web/test_url_path.sql | 12 +- integration_tests/models/web/test_urls.sql | 12 +- ...et_query_results_as_dict_objects_equal.sql | 52 ++- .../assert_pretty_output_msg_is_string.sql | 10 +- .../assert_pretty_time_is_string.sql | 10 +- .../tests/jinja_helpers/test_slugify.sql | 11 +- .../test_get_column_values_use_default.sql | 50 ++- macros/cross_db_utils/_is_ephemeral.sql | 17 +- macros/cross_db_utils/_is_relation.sql | 9 +- macros/cross_db_utils/any_value.sql | 15 +- macros/cross_db_utils/bool_or.sql | 20 +- macros/cross_db_utils/cast_bool_to_text.sql | 11 +- macros/cross_db_utils/concat.sql | 6 +- macros/cross_db_utils/current_timestamp.sql | 24 +- macros/cross_db_utils/datatypes.sql | 78 +--- macros/cross_db_utils/date_trunc.sql | 9 +- macros/cross_db_utils/dateadd.sql | 24 +- macros/cross_db_utils/datediff.sql | 115 ++++-- .../cross_db_utils/escape_single_quotes.sql | 4 +- macros/cross_db_utils/except.sql | 14 +- macros/cross_db_utils/hash.sql | 6 +- macros/cross_db_utils/identifier.sql | 23 +- macros/cross_db_utils/intersect.sql | 14 +- macros/cross_db_utils/last_day.sql | 44 ++- macros/cross_db_utils/length.sql | 18 +- macros/cross_db_utils/listagg.sql | 140 +++---- macros/cross_db_utils/literal.sql | 6 +- macros/cross_db_utils/position.sql | 16 +- macros/cross_db_utils/replace.sql | 12 +- macros/cross_db_utils/right.sql | 37 +- macros/cross_db_utils/safe_cast.sql | 10 +- macros/cross_db_utils/split_part.sql | 17 +- macros/cross_db_utils/width_bucket.sql | 73 ++-- macros/generic_tests/accepted_range.sql | 53 +-- macros/generic_tests/at_least_one.sql | 14 +- macros/generic_tests/cardinality_equality.sql | 60 +-- macros/generic_tests/equal_rowcount.sql | 41 +- macros/generic_tests/equality.sql | 77 ++-- macros/generic_tests/expression_is_true.sql | 21 +- macros/generic_tests/fewer_rows_than.sql | 66 ++-- .../mutually_exclusive_ranges.sql | 178 +++++---- macros/generic_tests/not_accepted_values.sql | 44 +-- macros/generic_tests/not_constant.sql | 9 +- macros/generic_tests/not_null_proportion.sql | 36 +- macros/generic_tests/recency.sql | 22 +- macros/generic_tests/relationships_where.sql | 58 +-- macros/generic_tests/sequential_values.sql | 56 +-- macros/generic_tests/test_not_null_where.sql | 21 +- macros/generic_tests/test_unique_where.sql | 21 +- .../unique_combination_of_columns.sql | 43 +- macros/jinja_helpers/log_info.sql | 4 +- macros/jinja_helpers/pretty_log_format.sql | 4 +- macros/jinja_helpers/pretty_time.sql | 8 +- macros/jinja_helpers/slugify.sql | 6 +- .../insert_by_period_materialization.sql | 373 ++++++++++-------- macros/sql/date_spine.sql | 77 ++-- macros/sql/deduplicate.sql | 40 +- macros/sql/generate_series.sql | 53 ++- macros/sql/get_column_values.sql | 75 ++-- .../sql/get_filtered_columns_in_relation.sql | 32 +- macros/sql/get_query_results_as_dict.sql | 25 +- macros/sql/get_relations_by_pattern.sql | 54 +-- macros/sql/get_relations_by_prefix.sql | 50 +-- macros/sql/get_table_types_sql.sql | 52 ++- macros/sql/get_tables_by_pattern_sql.sql | 63 +-- macros/sql/get_tables_by_prefix_sql.sql | 18 +- macros/sql/groupby.sql | 7 +- macros/sql/haversine_distance.sql | 56 +-- macros/sql/nullcheck.sql | 14 +- macros/sql/nullcheck_table.sql | 12 +- macros/sql/pivot.sql | 97 +++-- macros/sql/safe_add.sql | 12 +- macros/sql/star.sql | 25 +- macros/sql/surrogate_key.sql | 36 +- macros/sql/union.sql | 139 ++++--- macros/sql/unpivot.sql | 145 ++++--- macros/web/get_url_host.sql | 34 +- macros/web/get_url_parameter.sql | 8 +- macros/web/get_url_path.sql | 21 +- 130 files changed, 2169 insertions(+), 2192 deletions(-) diff --git a/integration_tests/macros/assert_equal_values.sql b/integration_tests/macros/assert_equal_values.sql index d4f02618..48c0ff81 100644 --- a/integration_tests/macros/assert_equal_values.sql +++ b/integration_tests/macros/assert_equal_values.sql @@ -1,8 +1,7 @@ {% macro assert_equal_values(actual_object, expected_object) %} {% if not execute %} - {# pass #} - +{# pass #} {% elif actual_object != expected_object %} {% set msg %} @@ -20,13 +19,11 @@ {% endset %} - {{ log(msg, info=True) }} - - select 'fail' +{{ log(msg, info=True) }} -{% else %} +select 'fail' - select 'ok' {{ limit_zero() }} +{% else %} select 'ok' {{ limit_zero() }} {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/integration_tests/macros/limit_zero.sql b/integration_tests/macros/limit_zero.sql index 697849d1..d08ca051 100644 --- a/integration_tests/macros/limit_zero.sql +++ b/integration_tests/macros/limit_zero.sql @@ -1,11 +1,7 @@ -{% macro my_custom_macro() %} - whatever -{% endmacro %} +{% macro my_custom_macro() %} whatever {% endmacro %} {% macro limit_zero() %} - {{ return(adapter.dispatch('limit_zero', 'dbt_utils')()) }} +{{ return adapter.dispatch("limit_zero", "dbt_utils")() }} {% endmacro %} -{% macro default__limit_zero() %} - {{ return('limit 0') }} -{% endmacro %} \ No newline at end of file +{% macro default__limit_zero() %} {{ return "limit 0" }} {% endmacro %} diff --git a/integration_tests/models/cross_db_utils/test_any_value.sql b/integration_tests/models/cross_db_utils/test_any_value.sql index 9b27ed1d..6781b41f 100644 --- a/integration_tests/models/cross_db_utils/test_any_value.sql +++ b/integration_tests/models/cross_db_utils/test_any_value.sql @@ -1,19 +1,26 @@ -with some_model as ( - select 1 as id, 'abc' as key_name, 'dbt' as static_col union all - select 2 as id, 'abc' as key_name, 'dbt' as static_col union all - select 3 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 4 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 5 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 6 as id, 'xyz' as key_name, 'test' as static_col -), +with + some_model as ( + select 1 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 2 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 3 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 4 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 5 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 6 as id, 'xyz' as key_name, 'test' as static_col + ), -final as ( - select - key_name, - {{ dbt_utils.any_value('static_col') }} as static_col, - count(id) as num_rows - from some_model - group by key_name -) + final as ( + select + key_name, + {{ dbt_utils.any_value("static_col") }} as static_col, + count(id) as num_rows + from some_model + group by key_name + ) -select * from final \ No newline at end of file +select * +from final diff --git a/integration_tests/models/cross_db_utils/test_bool_or.sql b/integration_tests/models/cross_db_utils/test_bool_or.sql index 7375d1e4..1d6f4ad5 100644 --- a/integration_tests/models/cross_db_utils/test_bool_or.sql +++ b/integration_tests/models/cross_db_utils/test_bool_or.sql @@ -1,5 +1,3 @@ -select - key, - {{ dbt_utils.bool_or('val1 = val2') }} as value -from {{ ref('data_bool_or' )}} -group by key \ No newline at end of file +select key, {{ dbt_utils.bool_or("val1 = val2") }} as value +from {{ ref("data_bool_or") }} +group by key diff --git a/integration_tests/models/cross_db_utils/test_concat.sql b/integration_tests/models/cross_db_utils/test_concat.sql index 9efeb748..8c036c65 100644 --- a/integration_tests/models/cross_db_utils/test_concat.sql +++ b/integration_tests/models/cross_db_utils/test_concat.sql @@ -1,12 +1,6 @@ -with data as ( +with data as (select * from {{ ref("data_concat") }}) - select * from {{ ref('data_concat') }} - -) - -select - {{ dbt_utils.concat(['input_1', 'input_2']) }} as actual, - output as expected +select {{ dbt_utils.concat(["input_1", "input_2"]) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp.sql b/integration_tests/models/cross_db_utils/test_current_timestamp.sql index 9a777f28..d6c2cf73 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp.sql @@ -3,4 +3,3 @@ select {{ dbt_utils.current_timestamp() }} as actual, {{ dbt_utils.current_timestamp() }} as expected - diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql index 55bc8e23..e88e934a 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql @@ -2,4 +2,4 @@ -- how can we test this better? select {{ dbt_utils.current_timestamp_in_utc() }} as actual, - {{ dbt_utils.current_timestamp_in_utc() }} as expected \ No newline at end of file + {{ dbt_utils.current_timestamp_in_utc() }} as expected diff --git a/integration_tests/models/cross_db_utils/test_date_trunc.sql b/integration_tests/models/cross_db_utils/test_date_trunc.sql index bac21fed..a5b88aa3 100644 --- a/integration_tests/models/cross_db_utils/test_date_trunc.sql +++ b/integration_tests/models/cross_db_utils/test_date_trunc.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_date_trunc') }} - -) +with data as (select * from {{ ref("data_date_trunc") }}) select - cast({{dbt_utils.date_trunc('day', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("day", "updated_at") }} as date) as actual, day as expected from data @@ -14,7 +10,7 @@ from data union all select - cast({{ dbt_utils.date_trunc('month', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("month", "updated_at") }} as date) as actual, month as expected from data diff --git a/integration_tests/models/cross_db_utils/test_dateadd.sql b/integration_tests/models/cross_db_utils/test_dateadd.sql index 3e199d86..2f2dee99 100644 --- a/integration_tests/models/cross_db_utils/test_dateadd.sql +++ b/integration_tests/models/cross_db_utils/test_dateadd.sql @@ -1,16 +1,32 @@ -with data as ( - - select * from {{ ref('data_dateadd') }} - -) +with data as (select * from {{ ref("data_dateadd") }}) select case - when datepart = 'hour' then cast({{ dbt_utils.dateadd('hour', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'day' then cast({{ dbt_utils.dateadd('day', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'month' then cast({{ dbt_utils.dateadd('month', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'year' then cast({{ dbt_utils.dateadd('year', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) + when datepart = 'hour' + then + cast( + {{ dbt_utils.dateadd("hour", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'day' + then + cast( + {{ dbt_utils.dateadd("day", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'month' + then + cast( + {{ dbt_utils.dateadd("month", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'year' + then + cast( + {{ dbt_utils.dateadd("year", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_datediff.sql b/integration_tests/models/cross_db_utils/test_datediff.sql index a2340456..4b6c95a6 100644 --- a/integration_tests/models/cross_db_utils/test_datediff.sql +++ b/integration_tests/models/cross_db_utils/test_datediff.sql @@ -1,20 +1,23 @@ -with data as ( - - select * from {{ ref('data_datediff') }} - -) +with data as (select * from {{ ref("data_datediff") }}) select case - when datepart = 'second' then {{ dbt_utils.datediff('first_date', 'second_date', 'second') }} - when datepart = 'minute' then {{ dbt_utils.datediff('first_date', 'second_date', 'minute') }} - when datepart = 'hour' then {{ dbt_utils.datediff('first_date', 'second_date', 'hour') }} - when datepart = 'day' then {{ dbt_utils.datediff('first_date', 'second_date', 'day') }} - when datepart = 'week' then {{ dbt_utils.datediff('first_date', 'second_date', 'week') }} - when datepart = 'month' then {{ dbt_utils.datediff('first_date', 'second_date', 'month') }} - when datepart = 'year' then {{ dbt_utils.datediff('first_date', 'second_date', 'year') }} + when datepart = 'second' + then {{ dbt_utils.datediff("first_date", "second_date", "second") }} + when datepart = 'minute' + then {{ dbt_utils.datediff("first_date", "second_date", "minute") }} + when datepart = 'hour' + then {{ dbt_utils.datediff("first_date", "second_date", "hour") }} + when datepart = 'day' + then {{ dbt_utils.datediff("first_date", "second_date", "day") }} + when datepart = 'week' + then {{ dbt_utils.datediff("first_date", "second_date", "week") }} + when datepart = 'month' + then {{ dbt_utils.datediff("first_date", "second_date", "month") }} + when datepart = 'year' + then {{ dbt_utils.datediff("first_date", "second_date", "year") }} else null end as actual, result as expected @@ -22,14 +25,77 @@ select from data -- Also test correct casting of literal values. - -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "microsecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "millisecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "microsecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "millisecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year" + ) + }} as actual, 1 as expected diff --git a/integration_tests/models/cross_db_utils/test_hash.sql b/integration_tests/models/cross_db_utils/test_hash.sql index 8f017780..6d526274 100644 --- a/integration_tests/models/cross_db_utils/test_hash.sql +++ b/integration_tests/models/cross_db_utils/test_hash.sql @@ -1,12 +1,6 @@ -with data as ( +with data as (select * from {{ ref("data_hash") }}) - select * from {{ ref('data_hash') }} - -) - -select - {{ dbt_utils.hash('input_1') }} as actual, - output as expected +select {{ dbt_utils.hash("input_1") }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_last_day.sql b/integration_tests/models/cross_db_utils/test_last_day.sql index fee3a823..4e6cec97 100644 --- a/integration_tests/models/cross_db_utils/test_last_day.sql +++ b/integration_tests/models/cross_db_utils/test_last_day.sql @@ -1,15 +1,14 @@ -with data as ( - - select * from {{ ref('data_last_day') }} - -) +with data as (select * from {{ ref("data_last_day") }}) select case - when date_part = 'month' then {{ dbt_utils.last_day('date_day', 'month') }} - when date_part = 'quarter' then {{ dbt_utils.last_day('date_day', 'quarter') }} - when date_part = 'year' then {{ dbt_utils.last_day('date_day', 'year') }} + when date_part = 'month' + then {{ dbt_utils.last_day("date_day", "month") }} + when date_part = 'quarter' + then {{ dbt_utils.last_day("date_day", "quarter") }} + when date_part = 'year' + then {{ dbt_utils.last_day("date_day", "year") }} else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_length.sql b/integration_tests/models/cross_db_utils/test_length.sql index f4ef099c..d6352d05 100644 --- a/integration_tests/models/cross_db_utils/test_length.sql +++ b/integration_tests/models/cross_db_utils/test_length.sql @@ -1,12 +1,5 @@ -with data as ( +with data as (select * from {{ ref("data_length") }}) - select * from {{ ref('data_length') }} +select {{ dbt_utils.length("expression") }} as actual, output as expected -) - -select - - {{ dbt_utils.length('expression') }} as actual, - output as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_listagg.sql b/integration_tests/models/cross_db_utils/test_listagg.sql index 006948de..27d96745 100644 --- a/integration_tests/models/cross_db_utils/test_listagg.sql +++ b/integration_tests/models/cross_db_utils/test_listagg.sql @@ -1,69 +1,63 @@ -with data as ( - - select * from {{ ref('data_listagg') }} - -), - -data_output as ( - - select * from {{ ref('data_listagg_output') }} - -), - -calculate as ( - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col") }} as actual, - 'bottom_ordered' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col", 2) }} as actual, - 'bottom_ordered_limited' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "', '") }} as actual, - 'comma_whitespace_unordered' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('DISTINCT string_text', "','") }} as actual, - 'distinct_comma' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text') }} as actual, - 'no_params' as version - from data - where group_col = 3 - group by group_col - -) - -select - calculate.actual, - data_output.expected +with + data as (select * from {{ ref("data_listagg") }}), + + data_output as (select * from {{ ref("data_listagg_output") }}), + + calculate as ( + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col") }} + as actual, + 'bottom_ordered' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col", 2) }} + as actual, + 'bottom_ordered_limited' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "', '") }} as actual, + 'comma_whitespace_unordered' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("DISTINCT string_text", "','") }} as actual, + 'distinct_comma' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text") }} as actual, + 'no_params' as version + from data + where group_col = 3 + group by group_col + + ) + +select calculate.actual, data_output.expected from calculate -left join data_output -on calculate.group_col = data_output.group_col -and calculate.version = data_output.version \ No newline at end of file +left join + data_output + on calculate.group_col = data_output.group_col + and calculate.version = data_output.version diff --git a/integration_tests/models/cross_db_utils/test_position.sql b/integration_tests/models/cross_db_utils/test_position.sql index b09be66d..d1785f14 100644 --- a/integration_tests/models/cross_db_utils/test_position.sql +++ b/integration_tests/models/cross_db_utils/test_position.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_position') }} - -) +with data as (select * from {{ ref("data_position") }}) select - {{ dbt_utils.position('substring_text', 'string_text') }} as actual, + {{ dbt_utils.position("substring_text", "string_text") }} as actual, result as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_replace.sql b/integration_tests/models/cross_db_utils/test_replace.sql index 6c8a8aa1..0e413c8a 100644 --- a/integration_tests/models/cross_db_utils/test_replace.sql +++ b/integration_tests/models/cross_db_utils/test_replace.sql @@ -1,18 +1,19 @@ -with data as ( +with + data as ( - select - - *, - coalesce(search_chars, '') as old_chars, - coalesce(replace_chars, '') as new_chars - - from {{ ref('data_replace') }} + select -) + *, + coalesce(search_chars, '') as old_chars, + coalesce(replace_chars, '') as new_chars + + from {{ ref("data_replace") }} + + ) select - {{ dbt_utils.replace('string_text', 'old_chars', 'new_chars') }} as actual, + {{ dbt_utils.replace("string_text", "old_chars", "new_chars") }} as actual, result as expected from data diff --git a/integration_tests/models/cross_db_utils/test_right.sql b/integration_tests/models/cross_db_utils/test_right.sql index eaad4f4f..de2df7dc 100644 --- a/integration_tests/models/cross_db_utils/test_right.sql +++ b/integration_tests/models/cross_db_utils/test_right.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_right') }} - -) +with data as (select * from {{ ref("data_right") }}) select - {{ dbt_utils.right('string_text', 'length_expression') }} as actual, + {{ dbt_utils.right("string_text", "length_expression") }} as actual, coalesce(output, '') as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_safe_cast.sql b/integration_tests/models/cross_db_utils/test_safe_cast.sql index 8ed74fc3..b2e6d82c 100644 --- a/integration_tests/models/cross_db_utils/test_safe_cast.sql +++ b/integration_tests/models/cross_db_utils/test_safe_cast.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_safe_cast') }} - -) +with data as (select * from {{ ref("data_safe_cast") }}) select - {{ dbt_utils.safe_cast('field', dbt_utils.type_string()) }} as actual, + {{ dbt_utils.safe_cast("field", dbt_utils.type_string()) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_split_part.sql b/integration_tests/models/cross_db_utils/test_split_part.sql index 6a10327c..05c0d70f 100644 --- a/integration_tests/models/cross_db_utils/test_split_part.sql +++ b/integration_tests/models/cross_db_utils/test_split_part.sql @@ -1,28 +1,21 @@ -with data as ( - - select * from {{ ref('data_split_part') }} - -) +with data as (select * from {{ ref("data_split_part") }}) select - {{ dbt_utils.split_part('parts', 'split_on', 1) }} as actual, - result_1 as expected + {{ dbt_utils.split_part("parts", "split_on", 1) }} as actual, result_1 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 2) }} as actual, - result_2 as expected + {{ dbt_utils.split_part("parts", "split_on", 2) }} as actual, result_2 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 3) }} as actual, - result_3 as expected + {{ dbt_utils.split_part("parts", "split_on", 3) }} as actual, result_3 as expected from data diff --git a/integration_tests/models/cross_db_utils/test_width_bucket.sql b/integration_tests/models/cross_db_utils/test_width_bucket.sql index dc2b8609..c2b79646 100644 --- a/integration_tests/models/cross_db_utils/test_width_bucket.sql +++ b/integration_tests/models/cross_db_utils/test_width_bucket.sql @@ -1,12 +1,9 @@ -with data as ( - - select * from {{ ref('data_width_bucket') }} - -) +with data as (select * from {{ ref("data_width_bucket") }}) select - {{ dbt_utils.width_bucket('amount', 'min_value', 'max_value', 'num_buckets') }} as actual, + {{ dbt_utils.width_bucket("amount", "min_value", "max_value", "num_buckets") }} + as actual, bucket as expected from data diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index fa4ae52b..f2c2dd12 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -3,24 +3,22 @@ -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} -{{ config(materialized='table') }} +with + date_spine as ( -with date_spine as ( - - {% if target.type == 'postgres' %} + {% if target.type == "postgres" %} {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} - - {% elif target.type == 'bigquery' %} + + {% elif target.type == "bigquery" %} select cast(date_day as date) as date_day from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) - - {% else %} - {{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} - {% endif %} -) + {% else %}{{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} + {% endif %} + + ) select date_day from date_spine - diff --git a/integration_tests/models/generic_tests/test_equal_column_subset.sql b/integration_tests/models/generic_tests/test_equal_column_subset.sql index b426bb63..7ecae303 100644 --- a/integration_tests/models/generic_tests/test_equal_column_subset.sql +++ b/integration_tests/models/generic_tests/test_equal_column_subset.sql @@ -1,9 +1,5 @@ -{{ config(materialized='ephemeral') }} +{{ config(materialized="ephemeral") }} -select +select first_name, last_name, email - first_name, - last_name, - email - -from {{ ref('data_people') }} +from {{ ref("data_people") }} diff --git a/integration_tests/models/generic_tests/test_equal_rowcount.sql b/integration_tests/models/generic_tests/test_equal_rowcount.sql index 01b1a549..621ccf61 100644 --- a/integration_tests/models/generic_tests/test_equal_rowcount.sql +++ b/integration_tests/models/generic_tests/test_equal_rowcount.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_equal_rowcount") }}) - select * from {{ ref('data_test_equal_rowcount') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_fewer_rows_than.sql b/integration_tests/models/generic_tests/test_fewer_rows_than.sql index c2ad4cbd..621fce6c 100644 --- a/integration_tests/models/generic_tests/test_fewer_rows_than.sql +++ b/integration_tests/models/generic_tests/test_fewer_rows_than.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_fewer_rows_than_table_1") }}) - select * from {{ ref('data_test_fewer_rows_than_table_1') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_recency.sql b/integration_tests/models/generic_tests/test_recency.sql index d300e572..142490f4 100644 --- a/integration_tests/models/generic_tests/test_recency.sql +++ b/integration_tests/models/generic_tests/test_recency.sql @@ -1,12 +1,13 @@ -{% if target.type == 'postgres' %} +{% if target.type == "postgres" %} -select - {{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as today +select {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as today {% else %} select - cast({{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as datetime) as today - + cast( + {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as datetime + ) as today + {% endif %} diff --git a/integration_tests/models/geo/test_haversine_distance_km.sql b/integration_tests/models/geo/test_haversine_distance_km.sql index e8ca4818..fce70682 100644 --- a/integration_tests/models/geo/test_haversine_distance_km.sql +++ b/integration_tests/models/geo/test_haversine_distance_km.sql @@ -1,23 +1,20 @@ -with data as ( - select * from {{ ref('data_haversine_km') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='km' +with + data as (select * from {{ ref("data_haversine_km") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="km", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/geo/test_haversine_distance_mi.sql b/integration_tests/models/geo/test_haversine_distance_mi.sql index 53f8172c..9ff4a1f7 100644 --- a/integration_tests/models/geo/test_haversine_distance_mi.sql +++ b/integration_tests/models/geo/test_haversine_distance_mi.sql @@ -1,39 +1,36 @@ -with data as ( - select * from {{ ref('data_haversine_mi') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='mi' +with + data as (select * from {{ ref("data_haversine_mi") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="mi", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data - union all + union all - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/materializations/expected_insert_by_period.sql b/integration_tests/models/materializations/expected_insert_by_period.sql index 0a198b6b..0d7d68c1 100644 --- a/integration_tests/models/materializations/expected_insert_by_period.sql +++ b/integration_tests/models/materializations/expected_insert_by_period.sql @@ -1,10 +1,5 @@ -{{ - config( - materialized = 'view', - enabled=(target.type == 'redshift') - ) -}} +{{ config(materialized="view", enabled=(target.type == "redshift")) }} select * -from {{ ref('data_insert_by_period') }} +from {{ ref("data_insert_by_period") }} where id in (2, 3, 4, 5, 6) diff --git a/integration_tests/models/materializations/test_insert_by_period.sql b/integration_tests/models/materializations/test_insert_by_period.sql index 8cc396d3..b48d22f6 100644 --- a/integration_tests/models/materializations/test_insert_by_period.sql +++ b/integration_tests/models/materializations/test_insert_by_period.sql @@ -1,18 +1,16 @@ {{ - config( - materialized = 'insert_by_period', - period = 'month', - timestamp_field = 'created_at', - start_date = '2018-01-01', - stop_date = '2018-06-01', - enabled=(target.type == 'redshift') - ) + config( + materialized="insert_by_period", + period="month", + timestamp_field="created_at", + start_date="2018-01-01", + stop_date="2018-06-01", + enabled=(target.type == "redshift"), + ) }} -with events as ( - select * - from {{ ref('data_insert_by_period') }} - where __PERIOD_FILTER__ -) +with + events as (select * from {{ ref("data_insert_by_period") }} where __period_filter__) -select * from events +select * +from events diff --git a/integration_tests/models/sql/test_deduplicate.sql b/integration_tests/models/sql/test_deduplicate.sql index 81fe81e7..47649a61 100644 --- a/integration_tests/models/sql/test_deduplicate.sql +++ b/integration_tests/models/sql/test_deduplicate.sql @@ -1,22 +1,19 @@ with -source as ( - select * - from {{ ref('data_deduplicate') }} - where user_id = 1 -), + source as (select * from {{ ref("data_deduplicate") }} where user_id = 1), -deduped as ( + deduped as ( - {{ - dbt_utils.deduplicate( - ref('data_deduplicate'), - group_by='user_id', - order_by='version desc', - relation_alias="source" - ) | indent - }} + {{ + dbt_utils.deduplicate( + ref("data_deduplicate"), + group_by="user_id", + order_by="version desc", + relation_alias="source", + ) | indent + }} -) + ) -select * from deduped +select * +from deduped diff --git a/integration_tests/models/sql/test_generate_series.sql b/integration_tests/models/sql/test_generate_series.sql index 11370b7b..2e5acf19 100644 --- a/integration_tests/models/sql/test_generate_series.sql +++ b/integration_tests/models/sql/test_generate_series.sql @@ -3,13 +3,9 @@ -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} -{{ config(materialized='table') }} +with data as ({{ dbt_utils.generate_series(10) }}) -with data as ( - - {{ dbt_utils.generate_series(10) }} - -) - -select generated_number from data +select generated_number +from data diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index 36214984..fe91fa57 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -1,29 +1,32 @@ -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values"), "field", default=[], order_by="field" +) %} -{% if target.type == 'snowflake' %} +{% if target.type == "snowflake" %} select {% for val in column_values -%} - sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} - {%- if not loop.last %},{% endif -%} + sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} + {%- if not loop.last %},{% endif -%} {%- endfor %} -from {{ ref('data_get_column_values') }} +from {{ ref("data_get_column_values") }} {% else %} select {% for val in column_values -%} - {{dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} as count_{{ val }} - {%- if not loop.last %},{% endif -%} + {{ dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} + as count_{{ val }} + {%- if not loop.last %},{% endif -%} {%- endfor %} -from {{ ref('data_get_column_values') }} +from {{ ref("data_get_column_values") }} {% endif %} diff --git a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql index 7b3ca72f..af04d517 100644 --- a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql +++ b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql @@ -1,16 +1,19 @@ -{% set exclude_field = 'field_1' %} +{% set exclude_field = "field_1" %} {% set column_names = dbt_utils.get_filtered_columns_in_relation(from= ref('data_filtered_columns_in_relation'), except=[exclude_field]) %} -with data as ( +with + data as ( - select + select - {% for column_name in column_names %} - max({{ column_name }}) as {{ column_name }} {% if not loop.last %},{% endif %} - {% endfor %} + {% for column_name in column_names %} + max({{ column_name }}) as {{ column_name }} + {% if not loop.last %},{% endif %} + {% endfor %} - from {{ ref('data_filtered_columns_in_relation') }} + from {{ ref("data_filtered_columns_in_relation") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index 52ab07a2..2c20559e 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,16 +1,9 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} {% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %} -with unioned as ( - - {{ dbt_utils.union_relations(relations) }} - -) +with unioned as ({{ dbt_utils.union_relations(relations) }}) -select - - user_id, - event +select user_id, event from unioned diff --git a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql index 481f233e..6e968e79 100644 --- a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql +++ b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql @@ -1,4 +1,4 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} -{% set relations = dbt_utils.get_relations_by_prefix(target.schema, 'data_events_') %} +{% set relations = dbt_utils.get_relations_by_prefix(target.schema, "data_events_") %} {{ dbt_utils.union_relations(relations) }} diff --git a/integration_tests/models/sql/test_groupby.sql b/integration_tests/models/sql/test_groupby.sql index 50a6ced7..a63f3a5c 100644 --- a/integration_tests/models/sql/test_groupby.sql +++ b/integration_tests/models/sql/test_groupby.sql @@ -1,24 +1,14 @@ -with test_data as ( - - select - - {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string() )}} as column_1, - {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string() )}} as column_2 - -), +with + test_data as ( -grouped as ( + select - select - *, - count(*) as total - - from test_data - {{ dbt_utils.group_by(2) }} - -) - -select * from grouped + {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string()) }} as column_1, + {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string()) }} as column_2 + ), + grouped as (select *, count(*) as total from test_data {{ dbt_utils.group_by(2) }}) +select * +from grouped diff --git a/integration_tests/models/sql/test_nullcheck_table.sql b/integration_tests/models/sql/test_nullcheck_table.sql index ea66c8ec..4eb60e4e 100644 --- a/integration_tests/models/sql/test_nullcheck_table.sql +++ b/integration_tests/models/sql/test_nullcheck_table.sql @@ -1,17 +1,12 @@ -{{ config( materialized = "table" ) }} - --- TO DO: remove if-statement - -{% set tbl = ref('data_nullcheck_table') %} - +{{ config(materialized="table") }} -with nulled as ( +-- TO DO: remove if-statement +{% set tbl = ref("data_nullcheck_table") %} - {{ dbt_utils.nullcheck_table(tbl) }} -) +with nulled as ({{ dbt_utils.nullcheck_table(tbl) }}) -{% if target.type == 'snowflake' %} +{% if target.type == "snowflake" %} select field_1::varchar as field_1, @@ -24,17 +19,11 @@ from nulled select - {{ dbt_utils.safe_cast('field_1', - dbt_utils.type_string() - )}} as field_1, + {{ dbt_utils.safe_cast("field_1", dbt_utils.type_string()) }} as field_1, - {{ dbt_utils.safe_cast('field_2', - dbt_utils.type_string() - )}} as field_2, + {{ dbt_utils.safe_cast("field_2", dbt_utils.type_string()) }} as field_2, - {{ dbt_utils.safe_cast('field_3', - dbt_utils.type_string() - )}} as field_3 + {{ dbt_utils.safe_cast("field_3", dbt_utils.type_string()) }} as field_3 from nulled diff --git a/integration_tests/models/sql/test_pivot.sql b/integration_tests/models/sql/test_pivot.sql index a63e6e3b..730b7bec 100644 --- a/integration_tests/models/sql/test_pivot.sql +++ b/integration_tests/models/sql/test_pivot.sql @@ -1,17 +1,11 @@ -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE'] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue'] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} +{% set column_values = ["RED", "BLUE"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue"] %} {% set cmp = "=" %} {% endif %} -select - size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp) }} +select size, {{ dbt_utils.pivot("color", column_values, cmp=cmp) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_pivot_apostrophe.sql b/integration_tests/models/sql/test_pivot_apostrophe.sql index 792f4686..e7a340a3 100644 --- a/integration_tests/models/sql/test_pivot_apostrophe.sql +++ b/integration_tests/models/sql/test_pivot_apostrophe.sql @@ -1,17 +1,13 @@ -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE', "BLUE'S"] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue', "blue's"] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} +{% set column_values = ["RED", "BLUE", "BLUE'S"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue", "blue's"] %} {% set cmp = "=" %} {% endif %} select size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp, quote_identifiers=False) }} + {{ dbt_utils.pivot("color", column_values, cmp=cmp, quote_identifiers=False) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_safe_add.sql b/integration_tests/models/sql/test_safe_add.sql index af3894d0..840871ff 100644 --- a/integration_tests/models/sql/test_safe_add.sql +++ b/integration_tests/models/sql/test_safe_add.sql @@ -1,12 +1,6 @@ -with data as ( +with data as (select * from {{ ref("data_safe_add") }}) - select * from {{ ref('data_safe_add') }} - -) - -select - {{ dbt_utils.safe_add('field_1', 'field_2', 'field_3') }} as actual, - expected +select {{ dbt_utils.safe_add("field_1", "field_2", "field_3") }} as actual, expected from data diff --git a/integration_tests/models/sql/test_star.sql b/integration_tests/models/sql/test_star.sql index 2092e16a..74566119 100644 --- a/integration_tests/models/sql/test_star.sql +++ b/integration_tests/models/sql/test_star.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'field_3' %} +{% set exclude_field = "field_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_aggregate.sql b/integration_tests/models/sql/test_star_aggregate.sql index 9dcd7c2d..45085a04 100644 --- a/integration_tests/models/sql/test_star_aggregate.sql +++ b/integration_tests/models/sql/test_star_aggregate.sql @@ -1,16 +1,15 @@ -/*This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ - +/* This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ {% set selected_columns = dbt_utils.star(from=ref('data_star_aggregate'), except=['value_field']) %} -with data as ( +with + data as ( - select - {{ selected_columns }}, - sum(value_field) as value_field_sum + select {{ selected_columns }}, sum(value_field) as value_field_sum - from {{ ref('data_star_aggregate') }} - group by {{ selected_columns }} + from {{ ref("data_star_aggregate") }} + group by {{ selected_columns }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_prefix_suffix.sql b/integration_tests/models/sql/test_star_prefix_suffix.sql index ae1c64fe..69794309 100644 --- a/integration_tests/models/sql/test_star_prefix_suffix.sql +++ b/integration_tests/models/sql/test_star_prefix_suffix.sql @@ -1,13 +1,15 @@ -{% set prefix_with = 'prefix_' if target.type != 'snowflake' else 'PREFIX_' %} -{% set suffix_with = '_suffix' if target.type != 'snowflake' else '_SUFFIX' %} +{% set prefix_with = "prefix_" if target.type != "snowflake" else "PREFIX_" %} +{% set suffix_with = "_suffix" if target.type != "snowflake" else "_SUFFIX" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} + select + {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data \ No newline at end of file +select * +from data diff --git a/integration_tests/models/sql/test_star_uppercase.sql b/integration_tests/models/sql/test_star_uppercase.sql index 6179e691..be5b69fe 100644 --- a/integration_tests/models/sql/test_star_uppercase.sql +++ b/integration_tests/models/sql/test_star_uppercase.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'FIELD_3' %} +{% set exclude_field = "FIELD_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_surrogate_key.sql b/integration_tests/models/sql/test_surrogate_key.sql index fd90939f..a0da48ef 100644 --- a/integration_tests/models/sql/test_surrogate_key.sql +++ b/integration_tests/models/sql/test_surrogate_key.sql @@ -1,15 +1,13 @@ -with data as ( - - select * from {{ ref('data_surrogate_key') }} - -) +with data as (select * from {{ ref("data_surrogate_key") }}) select - {{ dbt_utils.surrogate_key('column_1') }} as actual_column_1_only, + {{ dbt_utils.surrogate_key("column_1") }} as actual_column_1_only, expected_column_1_only, - {{ dbt_utils.surrogate_key('column_1', 'column_2', 'column_3') }} as actual_all_columns_arguments, - {{ dbt_utils.surrogate_key(['column_1', 'column_2', 'column_3']) }} as actual_all_columns_list, + {{ dbt_utils.surrogate_key("column_1", "column_2", "column_3") }} + as actual_all_columns_arguments, + {{ dbt_utils.surrogate_key(["column_1", "column_2", "column_3"]) }} + as actual_all_columns_list, expected_all_columns from data diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql index 69836833..1b14ff1f 100644 --- a/integration_tests/models/sql/test_union.sql +++ b/integration_tests/models/sql/test_union.sql @@ -1,8 +1,2 @@ -select - id, - name, - favorite_color - -from {{ ref('test_union_base') }} - +select id, name, favorite_color from {{ ref("test_union_base") }} diff --git a/integration_tests/models/sql/test_union_base.sql b/integration_tests/models/sql/test_union_base.sql index a51bd2b8..045e651b 100644 --- a/integration_tests/models/sql/test_union_base.sql +++ b/integration_tests/models/sql/test_union_base.sql @@ -1,5 +1,2 @@ -{{ dbt_utils.union_relations([ - ref('data_union_table_1'), - ref('data_union_table_2')] -) }} +{{ dbt_utils.union_relations([ref("data_union_table_1"), ref("data_union_table_2")]) }} diff --git a/integration_tests/models/sql/test_unpivot.sql b/integration_tests/models/sql/test_unpivot.sql index 8cf1ad6e..7198d0a3 100644 --- a/integration_tests/models/sql/test_unpivot.sql +++ b/integration_tests/models/sql/test_unpivot.sql @@ -4,30 +4,27 @@ -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - remove=['name'], - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + remove=["name"], + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_bool.sql b/integration_tests/models/sql/test_unpivot_bool.sql index 26842c01..82d7205b 100644 --- a/integration_tests/models/sql/test_unpivot_bool.sql +++ b/integration_tests/models/sql/test_unpivot_bool.sql @@ -4,29 +4,26 @@ -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot_bool'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot_bool"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_original_api.sql b/integration_tests/models/sql/test_unpivot_original_api.sql index f6b9395a..c10d016e 100644 --- a/integration_tests/models/sql/test_unpivot_original_api.sql +++ b/integration_tests/models/sql/test_unpivot_original_api.sql @@ -1,32 +1,30 @@ -- unpivot() was enhanced with 3 new parameters -- This test targets the original API. - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, case - when '{{ target.name }}' = 'snowflake' then lower(FIELD_NAME) - else field_name + when '{{ target.name }}' = 'snowflake' then lower(field_name) else field_name end as field_name, value -from ( - {{ dbt_utils.unpivot( - table=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + table=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + ) + }} + ) as sbq diff --git a/integration_tests/models/web/test_url_host.sql b/integration_tests/models/web/test_url_host.sql index 06a9f69c..2e18e65d 100644 --- a/integration_tests/models/web/test_url_host.sql +++ b/integration_tests/models/web/test_url_host.sql @@ -1,12 +1,5 @@ -with data as ( - - select * from {{ref('data_url_host')}} - -) +with data as (select * from {{ ref("data_url_host") }}) -select +select {{ dbt_utils.get_url_host("original_url") }} as actual, parsed_url as expected - {{ dbt_utils.get_url_host('original_url') }} as actual, - parsed_url as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/web/test_url_path.sql b/integration_tests/models/web/test_url_path.sql index c03e5547..16f45ae0 100644 --- a/integration_tests/models/web/test_url_path.sql +++ b/integration_tests/models/web/test_url_path.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ref('data_url_path')}} - -) +with data as (select * from {{ ref("data_url_path") }}) select - coalesce({{ dbt_utils.get_url_path('original_url') }}, '') as actual, + coalesce({{ dbt_utils.get_url_path("original_url") }}, '') as actual, coalesce(parsed_path, '') as expected - -from data \ No newline at end of file + +from data diff --git a/integration_tests/models/web/test_urls.sql b/integration_tests/models/web/test_urls.sql index 60353dbb..066a0fcd 100644 --- a/integration_tests/models/web/test_urls.sql +++ b/integration_tests/models/web/test_urls.sql @@ -1,20 +1,14 @@ -with data as ( - - select * from {{ ref('data_urls') }} - -) +with data as (select * from {{ ref("data_urls") }}) select - {{ dbt_utils.get_url_parameter('url', 'utm_medium') }} as actual, - medium as expected + {{ dbt_utils.get_url_parameter("url", "utm_medium") }} as actual, medium as expected from data union all select - {{ dbt_utils.get_url_parameter('url', 'utm_source') }} as actual, - source as expected + {{ dbt_utils.get_url_parameter("url", "utm_source") }} as actual, source as expected from data diff --git a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql index 55d5400e..eec6203c 100644 --- a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql +++ b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql @@ -1,17 +1,16 @@ -- depends_on: {{ ref('data_get_query_results_as_dict') }} - -{% set expected_dictionary={ - 'col_1': [1, 2, 3], - 'col_2': ['a', 'b', 'c'], - 'col_3': [True, False, none] +{% set expected_dictionary = { + "col_1": [1, 2, 3], + "col_2": ["a", "b", "c"], + "col_3": [True, False, none], } %} {#- Handle snowflake casing silliness -#} -{% if target.type == 'snowflake' %} -{% set expected_dictionary={ - 'COL_1': [1, 2, 3], - 'COL_2': ['a', 'b', 'c'], - 'COL_3': [True, False, none] +{% if target.type == "snowflake" %} +{% set expected_dictionary = { + "COL_1": [1, 2, 3], + "COL_2": ["a", "b", "c"], + "COL_3": [True, False, none], } %} {% endif %} @@ -23,18 +22,17 @@ For reasons that remain unclear, Jinja won't return True for actual_dictionary == expected_dictionary. Instead, we'll manually check that the values of these dictionaries are equivalent. -#} - {% set ns = namespace( pass=True, err_msg = "" ) %} {% if execute %} {#- Check that the dictionaries have the same keys -#} -{% set expected_keys=expected_dictionary.keys() | list | sort %} -{% set actual_keys=actual_dictionary.keys() | list | sort %} +{% set expected_keys = expected_dictionary.keys() | list | sort %} +{% set actual_keys = actual_dictionary.keys() | list | sort %} {% if expected_keys != actual_keys %} - {% set ns.pass=False %} +{% set ns.pass=False %} {% set ns.err_msg %} The two dictionaries have different keys: expected_dictionary has keys: {{ expected_keys }} @@ -44,33 +42,33 @@ Instead, we'll manually check that the values of these dictionaries are equivale {% else %} {% for key, value in expected_dictionary.items() %} - {% set expected_length=expected_dictionary[key] | length %} - {% set actual_length=actual_dictionary[key] | length %} +{% set expected_length = expected_dictionary[key] | length %} +{% set actual_length = actual_dictionary[key] | length %} - {% if expected_length != actual_length %} - {% set ns.pass=False %} +{% if expected_length != actual_length %} +{% set ns.pass=False %} {% set ns.err_msg %} The {{ key }} column has different lengths: expected_dictionary[{{ key }}] has length {{ expected_length }} actual_dictionary[{{ key }}] has length {{ actual_length }} {% endset %} - {% else %} +{% else %} - {% for i in range(value | length) %} - {% set expected_value=expected_dictionary[key][i] %} - {% set actual_value=actual_dictionary[key][i] %} - {% if expected_value != actual_value %} - {% set ns.pass=False %} +{% for i in range(value | length) %} +{% set expected_value = expected_dictionary[key][i] %} +{% set actual_value = actual_dictionary[key][i] %} +{% if expected_value != actual_value %} +{% set ns.pass=False %} {% set ns.err_msg %} The {{ key }} column has differing values: expected_dictionary[{{ key }}][{{ i }}] == {{ expected_value }} actual_dictionary[{{ key }}][{{ i }}] == {{ actual_value }} {% endset %} - {% endif %} - {% endfor %} - {% endif %} +{% endif %} +{% endfor %} +{% endif %} {% endfor %} diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql index fea33594..6782e8d1 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql @@ -1,7 +1,5 @@ -{% if dbt_utils.pretty_log_format() is string %} - {# Return 0 rows for the test to pass #} - select 1 as col_name {{ limit_zero() }} -{% else %} - {# Return >0 rows for the test to fail #} - select 1 +{# Return 0 rows for the test to pass #} +{% if dbt_utils.pretty_log_format() is string %} select 1 as col_name {{ limit_zero() }} +{# Return >0 rows for the test to fail #} +{% else %} select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql index a6347abd..a9ec4b19 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql @@ -1,7 +1,5 @@ -{% if dbt_utils.pretty_time() is string %} - {# Return 0 rows for the test to pass #} - select 1 as col_name {{ limit_zero() }} -{% else %} - {# Return >0 rows for the test to fail #} - select 1 +{# Return 0 rows for the test to pass #} +{% if dbt_utils.pretty_time() is string %} select 1 as col_name {{ limit_zero() }} +{# Return >0 rows for the test to fail #} +{% else %} select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/test_slugify.sql b/integration_tests/tests/jinja_helpers/test_slugify.sql index c0839f59..13d0a2bc 100644 --- a/integration_tests/tests/jinja_helpers/test_slugify.sql +++ b/integration_tests/tests/jinja_helpers/test_slugify.sql @@ -1,7 +1,6 @@ -{% if dbt_utils.slugify('!Hell0 world-hi') == 'hell0_world_hi' %} - {# Return 0 rows for the test to pass #} - select 1 as col_name {{ limit_zero() }} -{% else %} - {# Return >0 rows for the test to fail #} - select 1 as col_name +{% if dbt_utils.slugify("!Hell0 world-hi") == "hell0_world_hi" %} +{# Return 0 rows for the test to pass #} +select 1 as col_name {{ limit_zero() }} +{# Return >0 rows for the test to fail #} +{% else %} select 1 as col_name {% endif %} diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 7f9fa3a4..0fcd8ac0 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -1,26 +1,40 @@ -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values_dropped"), + "field", + default=["y", "z"], + order_by="field", +) %} -with expected as ( - select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected_column_value union all - select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected_column_value -), +with + expected as ( + select + {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} + as expected_column_value + union all + select + {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} + as expected_column_value + ), + + actual as ( -actual as ( - {% for val in column_values %} - select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual_column_value - {% if not loop.last %} - union all - {% endif %} + select + {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} + as actual_column_value + {% if not loop.last %} union all {% endif %} {% endfor %} -), + ), -failures as ( - select * from actual - where actual.actual_column_value not in ( - select expected.expected_column_value from expected + failures as ( + select * + from actual + where + actual.actual_column_value not in ( + select expected.expected_column_value from expected + ) ) -) -select * from failures \ No newline at end of file +select * +from failures diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index 34c70db9..54317376 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -1,16 +1,15 @@ {% macro _is_ephemeral(obj, macro) %} - {%- if obj.is_cte -%} - {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} - {% if obj.name.startswith(ephemeral_prefix) %} - {% set model_name = obj.name[(ephemeral_prefix|length):] %} - {% else %} - {% set model_name = obj.name %} - {%- endif -%} +{%- if obj.is_cte -%} +{% set ephemeral_prefix = api.Relation.add_ephemeral_prefix("") %} +{% if obj.name.startswith(ephemeral_prefix) %} +{% set model_name = obj.name[(ephemeral_prefix | length) :] %} +{% else %} {% set model_name = obj.name %} +{%- endif -%} {% set error_message %} The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. `{{ model_name }}` is an ephemeral model. Consider making it a view or table instead. {% endset %} - {%- do exceptions.raise_compiler_error(error_message) -%} - {%- endif -%} +{%- do exceptions.raise_compiler_error(error_message) -%} +{%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/_is_relation.sql b/macros/cross_db_utils/_is_relation.sql index bee6db97..af01c32c 100644 --- a/macros/cross_db_utils/_is_relation.sql +++ b/macros/cross_db_utils/_is_relation.sql @@ -1,5 +1,8 @@ {% macro _is_relation(obj, macro) %} - {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%} - {%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} - {%- endif -%} +{%- if not ( + obj is mapping + and obj.get("metadata", {}).get("type", "").endswith("Relation") +) -%} +{%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} +{%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/any_value.sql b/macros/cross_db_utils/any_value.sql index 78cb75ba..5ee1c0e9 100644 --- a/macros/cross_db_utils/any_value.sql +++ b/macros/cross_db_utils/any_value.sql @@ -1,17 +1,10 @@ {% macro any_value(expression) -%} - {{ return(adapter.dispatch('any_value', 'dbt_utils') (expression)) }} +{{ return adapter.dispatch("any_value", "dbt_utils")(expression) }} {% endmacro %} -{% macro default__any_value(expression) -%} - - any_value({{ expression }}) - -{%- endmacro %} +{% macro default__any_value(expression) -%} any_value({{ expression }}) {%- endmacro %} -{% macro postgres__any_value(expression) -%} - {#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} - min({{ expression }}) - -{%- endmacro %} \ No newline at end of file +{#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} +{% macro postgres__any_value(expression) -%} min({{ expression }}) {%- endmacro %} diff --git a/macros/cross_db_utils/bool_or.sql b/macros/cross_db_utils/bool_or.sql index ce0a6857..8187b012 100644 --- a/macros/cross_db_utils/bool_or.sql +++ b/macros/cross_db_utils/bool_or.sql @@ -1,24 +1,12 @@ {% macro bool_or(expression) -%} - {{ return(adapter.dispatch('bool_or', 'dbt_utils') (expression)) }} +{{ return adapter.dispatch("bool_or", "dbt_utils")(expression) }} {% endmacro %} -{% macro default__bool_or(expression) -%} - - bool_or({{ expression }}) - -{%- endmacro %} +{% macro default__bool_or(expression) -%} bool_or({{ expression }}) {%- endmacro %} -{% macro snowflake__bool_or(expression) -%} - - boolor_agg({{ expression }}) - -{%- endmacro %} +{% macro snowflake__bool_or(expression) -%} boolor_agg({{ expression }}) {%- endmacro %} -{% macro bigquery__bool_or(expression) -%} - - logical_or({{ expression }}) - -{%- endmacro %} \ No newline at end of file +{% macro bigquery__bool_or(expression) -%} logical_or({{ expression }}) {%- endmacro %} diff --git a/macros/cross_db_utils/cast_bool_to_text.sql b/macros/cross_db_utils/cast_bool_to_text.sql index 5efa9d60..eb8c768e 100644 --- a/macros/cross_db_utils/cast_bool_to_text.sql +++ b/macros/cross_db_utils/cast_bool_to_text.sql @@ -1,15 +1,14 @@ {% macro cast_bool_to_text(field) %} - {{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }} +{{ adapter.dispatch("cast_bool_to_text", "dbt_utils")(field) }} {% endmacro %} {% macro default__cast_bool_to_text(field) %} - cast({{ field }} as {{ dbt_utils.type_string() }}) +cast({{ field }} as {{ dbt_utils.type_string() }}) {% endmacro %} {% macro redshift__cast_bool_to_text(field) %} - case - when {{ field }} is true then 'true' - when {{ field }} is false then 'false' - end::text +case + when {{ field }} is true then 'true' when {{ field }} is false then 'false' +end::text {% endmacro %} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 7e97e8f8..c2a25a62 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,9 +1,7 @@ {% macro concat(fields) -%} - {{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }} +{{ return adapter.dispatch("concat", "dbt_utils")(fields) }} {%- endmacro %} -{% macro default__concat(fields) -%} - {{ fields|join(' || ') }} -{%- endmacro %} +{% macro default__concat(fields) -%} {{ fields | join(" || ") }} {%- endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index 66ad8dc6..157d432d 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,38 +1,36 @@ {% macro current_timestamp() -%} - {{ return(adapter.dispatch('current_timestamp', 'dbt_utils')()) }} +{{ return adapter.dispatch("current_timestamp", "dbt_utils")() }} {%- endmacro %} {% macro default__current_timestamp() %} - current_timestamp::{{dbt_utils.type_timestamp()}} +current_timestamp::{{ dbt_utils.type_timestamp() }} {% endmacro %} -{% macro redshift__current_timestamp() %} - getdate() -{% endmacro %} +{% macro redshift__current_timestamp() %} getdate() {% endmacro %} -{% macro bigquery__current_timestamp() %} - current_timestamp -{% endmacro %} +{% macro bigquery__current_timestamp() %} current_timestamp {% endmacro %} {% macro current_timestamp_in_utc() -%} - {{ return(adapter.dispatch('current_timestamp_in_utc', 'dbt_utils')()) }} +{{ return adapter.dispatch("current_timestamp_in_utc", "dbt_utils")() }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} - {{dbt_utils.current_timestamp()}} +{{ dbt_utils.current_timestamp() }} {% endmacro %} {% macro snowflake__current_timestamp_in_utc() %} - convert_timezone('UTC', {{dbt_utils.current_timestamp()}})::{{dbt_utils.type_timestamp()}} +convert_timezone( + 'UTC', {{ dbt_utils.current_timestamp() }} +)::{{ dbt_utils.type_timestamp() }} {% endmacro %} {% macro postgres__current_timestamp_in_utc() %} - (current_timestamp at time zone 'utc')::{{dbt_utils.type_timestamp()}} +(current_timestamp at time zone 'utc')::{{ dbt_utils.type_timestamp() }} {% endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__current_timestamp_in_utc() %} - {{ return(dbt_utils.default__current_timestamp_in_utc()) }} +{{ return dbt_utils.default__current_timestamp_in_utc() }} {% endmacro %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index f115b4e2..8ccd5580 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,99 +1,63 @@ {# string ------------------------------------------------- #} - {%- macro type_string() -%} - {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_string", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_string() %} - string -{% endmacro %} +{% macro default__type_string() %} string {% endmacro %} -{%- macro redshift__type_string() -%} - varchar -{%- endmacro -%} +{%- macro redshift__type_string() -%} varchar {%- endmacro -%} -{% macro postgres__type_string() %} - varchar -{% endmacro %} +{% macro postgres__type_string() %} varchar {% endmacro %} -{% macro snowflake__type_string() %} - varchar -{% endmacro %} +{% macro snowflake__type_string() %} varchar {% endmacro %} {# timestamp ------------------------------------------------- #} - {%- macro type_timestamp() -%} - {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_timestamp", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_timestamp() %} - timestamp -{% endmacro %} +{% macro default__type_timestamp() %} timestamp {% endmacro %} -{% macro postgres__type_timestamp() %} - timestamp without time zone -{% endmacro %} +{% macro postgres__type_timestamp() %} timestamp without time zone {% endmacro %} -{% macro snowflake__type_timestamp() %} - timestamp_ntz -{% endmacro %} +{% macro snowflake__type_timestamp() %} timestamp_ntz {% endmacro %} {# float ------------------------------------------------- #} - {%- macro type_float() -%} - {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_float", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_float() %} - float -{% endmacro %} +{% macro default__type_float() %} float {% endmacro %} -{% macro bigquery__type_float() %} - float64 -{% endmacro %} +{% macro bigquery__type_float() %} float64 {% endmacro %} {# numeric ------------------------------------------------ #} - {%- macro type_numeric() -%} - {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_numeric", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_numeric() %} - numeric(28, 6) -{% endmacro %} +{% macro default__type_numeric() %} numeric(28, 6) {% endmacro %} -{% macro bigquery__type_numeric() %} - numeric -{% endmacro %} +{% macro bigquery__type_numeric() %} numeric {% endmacro %} {# bigint ------------------------------------------------- #} - {%- macro type_bigint() -%} - {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_bigint", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_bigint() %} - bigint -{% endmacro %} +{% macro default__type_bigint() %} bigint {% endmacro %} -{% macro bigquery__type_bigint() %} - int64 -{% endmacro %} +{% macro bigquery__type_bigint() %} int64 {% endmacro %} {# int ------------------------------------------------- #} - {%- macro type_int() -%} - {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }} +{{ return adapter.dispatch("type_int", "dbt_utils")() }} {%- endmacro -%} -{% macro default__type_int() %} - int -{% endmacro %} +{% macro default__type_int() %} int {% endmacro %} -{% macro bigquery__type_int() %} - int64 -{% endmacro %} +{% macro bigquery__type_int() %} int64 {% endmacro %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index f9d0364b..54ac4309 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,15 +1,12 @@ {% macro date_trunc(datepart, date) -%} - {{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }} +{{ return adapter.dispatch("date_trunc", "dbt_utils")(datepart, date) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) -%} - date_trunc('{{datepart}}', {{date}}) +date_trunc('{{datepart}}', {{ date }}) {%- endmacro %} {% macro bigquery__date_trunc(datepart, date) -%} - timestamp_trunc( - cast({{date}} as timestamp), - {{datepart}} - ) +timestamp_trunc(cast({{ date }} as timestamp), {{ datepart }}) {%- endmacro %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 09c0f115..5f019e45 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,37 +1,37 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }} +{{ + return adapter.dispatch("dateadd", "dbt_utils")( + datepart, interval, from_date_or_timestamp + ) +}} {% endmacro %} {% macro default__dateadd(datepart, interval, from_date_or_timestamp) %} - dateadd( - {{ datepart }}, - {{ interval }}, - {{ from_date_or_timestamp }} - ) +dateadd({{ datepart }}, {{ interval }}, {{ from_date_or_timestamp }}) {% endmacro %} {% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %} - datetime_add( - cast( {{ from_date_or_timestamp }} as datetime), - interval {{ interval }} {{ datepart }} - ) +datetime_add( + cast({{ from_date_or_timestamp }} as datetime), + interval {{ interval }} {{ datepart }} +) {% endmacro %} {% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} - {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) +{{ from_date_or_timestamp }} + ( (interval '1 {{ datepart }}') * ({{ interval }})) {% endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }} +{{ return dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp) }} {% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 2b5d6613..d0275619 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,59 +1,94 @@ {% macro datediff(first_date, second_date, datepart) %} - {{ return(adapter.dispatch('datediff', 'dbt_utils')(first_date, second_date, datepart)) }} +{{ return adapter.dispatch("datediff", "dbt_utils")(first_date, second_date, datepart) }} {% endmacro %} {% macro default__datediff(first_date, second_date, datepart) -%} - datediff( - {{ datepart }}, - {{ first_date }}, - {{ second_date }} - ) +datediff({{ datepart }}, {{ first_date }}, {{ second_date }}) {%- endmacro %} {% macro bigquery__datediff(first_date, second_date, datepart) -%} - datetime_diff( - cast({{second_date}} as datetime), - cast({{first_date}} as datetime), - {{datepart}} - ) +datetime_diff( + cast({{ second_date }} as datetime), + cast({{ first_date }} as datetime), + {{ datepart }} +) {%- endmacro %} {% macro postgres__datediff(first_date, second_date, datepart) -%} - {% if datepart == 'year' %} - (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) - {% elif datepart == 'quarter' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) - {% elif datepart == 'month' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) - {% elif datepart == 'day' %} - (({{second_date}})::date - ({{first_date}})::date) - {% elif datepart == 'week' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case - when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then - case when {{first_date}} <= {{second_date}} then 0 else -1 end - else - case when {{first_date}} <= {{second_date}} then 1 else 0 end - end) - {% elif datepart == 'hour' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) - {% elif datepart == 'minute' %} - ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) - {% elif datepart == 'second' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) - {% elif datepart == 'millisecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) - {% elif datepart == 'microsecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) - {% else %} - {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} - {% endif %} +{% if datepart == "year" %} +( + date_part('year', ({{ second_date }})::date) - date_part( + 'year', ({{ first_date }})::date + ) +) +{% elif datepart == "quarter" %} +( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + date_part( + 'quarter', ({{ second_date }})::date + ) - date_part('quarter', ({{ first_date }})::date) +) +{% elif datepart == "month" %} +( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + date_part( + 'month', ({{ second_date }})::date + ) - date_part('month', ({{ first_date }})::date) +) +{% elif datepart == "day" %} ( ({{ second_date }})::date - ({{ first_date }})::date) +{% elif datepart == "week" %} +( + {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case + when + date_part('dow', ({{ first_date }})::timestamp) <= date_part( + 'dow', ({{ second_date }})::timestamp + ) + then case when {{ first_date }} <= {{ second_date }} then 0 else -1 end + else case when {{ first_date }} <= {{ second_date }} then 1 else 0 end + end +) +{% elif datepart == "hour" %} +( + {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + date_part( + 'hour', ({{ second_date }})::timestamp + ) - date_part('hour', ({{ first_date }})::timestamp) +) +{% elif datepart == "minute" %} +( + {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + date_part( + 'minute', ({{ second_date }})::timestamp + ) - date_part('minute', ({{ first_date }})::timestamp) +) +{% elif datepart == "second" %} +( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + floor( + date_part('second', ({{ second_date }})::timestamp) + ) - floor(date_part('second', ({{ first_date }})::timestamp)) +) +{% elif datepart == "millisecond" %} +( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + floor( + date_part('millisecond', ({{ second_date }})::timestamp) + ) - floor(date_part('millisecond', ({{ first_date }})::timestamp)) +) +{% elif datepart == "microsecond" %} +( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + floor( + date_part('microsecond', ({{ second_date }})::timestamp) + ) - floor(date_part('microsecond', ({{ first_date }})::timestamp)) +) +{% else %} +{{ + exceptions.raise_compiler_error( + "Unsupported datepart for macro datediff in postgres: {!r}".format(datepart) + ) +}} +{% endif %} {%- endmacro %} @@ -61,6 +96,6 @@ {# redshift should use default instead of postgres #} {% macro redshift__datediff(first_date, second_date, datepart) -%} - {{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }} +{{ return dbt_utils.default__datediff(first_date, second_date, datepart) }} {%- endmacro %} diff --git a/macros/cross_db_utils/escape_single_quotes.sql b/macros/cross_db_utils/escape_single_quotes.sql index d024f16f..aa82ec01 100644 --- a/macros/cross_db_utils/escape_single_quotes.sql +++ b/macros/cross_db_utils/escape_single_quotes.sql @@ -1,10 +1,10 @@ {% macro escape_single_quotes(expression) %} - {{ return(adapter.dispatch('escape_single_quotes', 'dbt_utils') (expression)) }} +{{ return adapter.dispatch("escape_single_quotes", "dbt_utils")(expression) }} {% endmacro %} {# /*Default to replacing a single apostrophe with two apostrophes: they're -> they''re*/ #} {% macro default__escape_single_quotes(expression) -%} -{{ expression | replace("'","''") }} +{{ expression | replace("'", "''") }} {%- endmacro %} {# /*Snowflake uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 9bb75b86..facada67 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,16 +1,8 @@ {% macro except() %} - {{ return(adapter.dispatch('except', 'dbt_utils')()) }} +{{ return adapter.dispatch("except", "dbt_utils")() }} {% endmacro %} -{% macro default__except() %} +{% macro default__except() %} except {% endmacro %} - except - -{% endmacro %} - -{% macro bigquery__except() %} - - except distinct - -{% endmacro %} \ No newline at end of file +{% macro bigquery__except() %} except distinct {% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index d086c2e6..865a047a 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,13 +1,13 @@ {% macro hash(field) -%} - {{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }} +{{ return adapter.dispatch("hash", "dbt_utils")(field) }} {%- endmacro %} {% macro default__hash(field) -%} - md5(cast({{field}} as {{dbt_utils.type_string()}})) +md5(cast({{ field }} as {{ dbt_utils.type_string() }})) {%- endmacro %} {% macro bigquery__hash(field) -%} - to_hex({{dbt_utils.default__hash(field)}}) +to_hex({{ dbt_utils.default__hash(field) }}) {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index 619cb7cb..edc1bc06 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -1,16 +1,11 @@ -{% macro identifier(value) %} - {%- set error_message = ' - Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ - Use `adapter.quote` instead. The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - {{ return(adapter.dispatch('identifier', 'dbt_utils') (value)) }} -{% endmacro %} +{% macro identifier(value) %} +{%- set error_message = " Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name +) -%} +{%- do exceptions.warn(error_message) -%} +{{ return adapter.dispatch("identifier", "dbt_utils")(value) }} +{% endmacro %} -{% macro default__identifier(value) -%} - "{{ value }}" -{%- endmacro %} +{% macro default__identifier(value) -%} "{{ value }}" {%- endmacro %} -{% macro bigquery__identifier(value) -%} - `{{ value }}` -{%- endmacro %} +{% macro bigquery__identifier(value) -%} `{{ value }}` {%- endmacro %} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index b53c72f3..86b3c70c 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,16 +1,8 @@ {% macro intersect() %} - {{ return(adapter.dispatch('intersect', 'dbt_utils')()) }} +{{ return adapter.dispatch("intersect", "dbt_utils")() }} {% endmacro %} -{% macro default__intersect() %} +{% macro default__intersect() %} intersect {% endmacro %} - intersect - -{% endmacro %} - -{% macro bigquery__intersect() %} - - intersect distinct - -{% endmacro %} +{% macro bigquery__intersect() %} intersect distinct {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 18ca161c..c6f63954 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -2,44 +2,50 @@ This function has been tested with dateparts of month and quarters. Further testing is required to validate that it will work on other dateparts. */ - {% macro last_day(date, datepart) %} - {{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }} +{{ return adapter.dispatch("last_day", "dbt_utils")(date, datepart) }} {% endmacro %} {%- macro default_last_day(date, datepart) -%} - cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd(datepart, '1', dbt_utils.date_trunc(datepart, date)) - )}} - as date) +cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd(datepart, "1", dbt_utils.date_trunc(datepart, date)), + ) + }} as date +) {%- endmacro -%} {% macro default__last_day(date, datepart) -%} - {{dbt_utils.default_last_day(date, datepart)}} +{{ dbt_utils.default_last_day(date, datepart) }} {%- endmacro %} {% macro postgres__last_day(date, datepart) -%} - {%- if datepart == 'quarter' -%} - -- postgres dateadd does not support quarter interval. - cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd('month', '3', dbt_utils.date_trunc(datepart, date)) - )}} - as date) - {%- else -%} - {{dbt_utils.default_last_day(date, datepart)}} - {%- endif -%} +{%- if datepart == "quarter" -%} +-- postgres dateadd does not support quarter interval. +cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd("month", "3", dbt_utils.date_trunc(datepart, date)), + ) + }} as date +) +{%- else -%} {{ dbt_utils.default_last_day(date, datepart) }} +{%- endif -%} {%- endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__last_day(date, datepart) %} - {{ return(dbt_utils.default__last_day(date, datepart)) }} +{{ return dbt_utils.default__last_day(date, datepart) }} {% endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index 8c2e265c..95570736 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,21 +1,9 @@ {% macro length(expression) -%} - {{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }} +{{ return adapter.dispatch("length", "dbt_utils")(expression) }} {% endmacro %} -{% macro default__length(expression) %} - - length( - {{ expression }} - ) - -{%- endmacro -%} +{% macro default__length(expression) %} length({{ expression }}) {%- endmacro -%} -{% macro redshift__length(expression) %} - - len( - {{ expression }} - ) - -{%- endmacro -%} \ No newline at end of file +{% macro redshift__length(expression) %} len({{ expression }}) {%- endmacro -%} diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 1d19a54f..6179a870 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -1,104 +1,88 @@ -{% macro listagg(measure, delimiter_text="','", order_by_clause=none, limit_num=none) -%} - {{ return(adapter.dispatch('listagg', 'dbt_utils') (measure, delimiter_text, order_by_clause, limit_num)) }} +{% macro listagg( + measure, delimiter_text="','", order_by_clause=none, limit_num=none +) -%} +{{ + return adapter.dispatch("listagg", "dbt_utils")( + measure, delimiter_text, order_by_clause, limit_num + ) +}} {%- endmacro %} {% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - {% if limit_num -%} - array_to_string( - array_slice( - array_agg( - {{ measure }} - ){% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,0 - ,{{ limit_num }} - ), - {{ delimiter_text }} - ) - {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - {%- endif %} +{% if limit_num -%} +array_to_string( + array_slice( + array_agg({{ measure }}) + {% if order_by_clause -%} within group({{ order_by_clause }}) {%- endif %}, + 0, + {{ limit_num }} + ), + {{ delimiter_text }} +) +{%- else %} +listagg({{ measure }}, {{ delimiter_text }}) +{% if order_by_clause -%} within group({{ order_by_clause }}) +{%- endif %} +{%- endif %} {%- endmacro %} {% macro bigquery__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - {% if limit_num -%} - limit {{ limit_num }} - {%- endif %} - ) +string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + {% if limit_num -%} limit {{ limit_num }} {%- endif %} +) {%- endmacro %} {% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - - {% if limit_num -%} - array_to_string( - (array_agg( - {{ measure }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - ))[1:{{ limit_num }}], - {{ delimiter_text }} - ) - {%- else %} - string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} + +{% if limit_num -%} +array_to_string( + ( + array_agg( + {{ measure }} {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} ) + ) [1:{{ limit_num }}], + {{ delimiter_text }} +) +{%- else %} +string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} +) +{%- endif %} {%- endmacro %} {# if there are instances of delimiter_text within your measure, you cannot include a limit_num #} {% macro redshift__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - {% if limit_num -%} - {% set ns = namespace() %} - {% set ns.delimiter_text_regex = delimiter_text|trim("'") %} +{% if limit_num -%} +{% set ns = namespace() %} +{% set ns.delimiter_text_regex = delimiter_text | trim("'") %} {% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} - {%- for char in special_chars.split(',') -%} +{%- for char in special_chars.split(",") -%} {% set escape_char %}\\{{ char }}{% endset %} - {% set ns.delimiter_text_regex = ns.delimiter_text_regex|replace(char,escape_char) %} - {%- endfor -%} +{% set ns.delimiter_text_regex = ns.delimiter_text_regex | replace(char, escape_char) %} +{%- endfor -%} {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} - regexp_substr( - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,{{ regex }} - ) - {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - {%- endif %} +regexp_substr( + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group({{ order_by_clause }}) {%- endif %}, + {{ regex }} +) +{%- else %} +listagg({{ measure }}, {{ delimiter_text }}) +{% if order_by_clause -%} within group({{ order_by_clause }}) +{%- endif %} +{%- endif %} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index c2291467..01ce7188 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,8 +1,6 @@ {%- macro string_literal(value) -%} - {{ return(adapter.dispatch('string_literal', 'dbt_utils') (value)) }} +{{ return adapter.dispatch("string_literal", "dbt_utils")(value) }} {%- endmacro -%} -{% macro default__string_literal(value) -%} - '{{ value }}' -{%- endmacro %} +{% macro default__string_literal(value) -%} '{{ value }}' {%- endmacro %} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index a67b7aa2..803fa647 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,22 +1,16 @@ {% macro position(substring_text, string_text) -%} - {{ return(adapter.dispatch('position', 'dbt_utils') (substring_text, string_text)) }} +{{ return adapter.dispatch("position", "dbt_utils")(substring_text, string_text) }} {% endmacro %} {% macro default__position(substring_text, string_text) %} - position( - {{ substring_text }} in {{ string_text }} - ) - +position({{ substring_text }} in {{ string_text }}) + {%- endmacro -%} {% macro bigquery__position(substring_text, string_text) %} - strpos( - {{ string_text }}, - {{ substring_text }} - - ) - +strpos({{ string_text }}, {{ substring_text }}) + {%- endmacro -%} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index ef27d8fc..a2173cac 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,15 +1,11 @@ {% macro replace(field, old_chars, new_chars) -%} - {{ return(adapter.dispatch('replace', 'dbt_utils') (field, old_chars, new_chars)) }} +{{ return adapter.dispatch("replace", "dbt_utils")(field, old_chars, new_chars) }} {% endmacro %} {% macro default__replace(field, old_chars, new_chars) %} - replace( - {{ field }}, - {{ old_chars }}, - {{ new_chars }} - ) - +replace({{ field }}, {{ old_chars }}, {{ new_chars }}) -{% endmacro %} \ No newline at end of file + +{% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 8ae1640f..1479e760 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,38 +1,29 @@ {% macro right(string_text, length_expression) -%} - {{ return(adapter.dispatch('right', 'dbt_utils') (string_text, length_expression)) }} +{{ return adapter.dispatch("right", "dbt_utils")(string_text, length_expression) }} {% endmacro %} {% macro default__right(string_text, length_expression) %} - right( - {{ string_text }}, - {{ length_expression }} - ) - +right({{ string_text }}, {{ length_expression }}) + {%- endmacro -%} {% macro bigquery__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 - then '' - else - substr( - {{ string_text }}, - -1 * ({{ length_expression }}) - ) - end +case + when {{ length_expression }} = 0 + then '' + else substr({{ string_text }}, -1 * ({{ length_expression }})) +end {%- endmacro -%} {% macro snowflake__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 - then '' - else - right( - {{ string_text }}, - {{ length_expression }} - ) - end +case + when {{ length_expression }} = 0 + then '' + else right({{ string_text }}, {{ length_expression }}) +end -{%- endmacro -%} \ No newline at end of file +{%- endmacro -%} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index 8569eecd..bbfcb867 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,20 +1,20 @@ {% macro safe_cast(field, type) %} - {{ return(adapter.dispatch('safe_cast', 'dbt_utils') (field, type)) }} +{{ return adapter.dispatch("safe_cast", "dbt_utils")(field, type) }} {% endmacro %} {% macro default__safe_cast(field, type) %} - {# most databases don't support this function yet +{# most databases don't support this function yet so we just need to use cast #} - cast({{field}} as {{type}}) +cast({{ field }} as {{ type }}) {% endmacro %} {% macro snowflake__safe_cast(field, type) %} - try_cast({{field}} as {{type}}) +try_cast({{ field }} as {{ type }}) {% endmacro %} {% macro bigquery__safe_cast(field, type) %} - safe_cast({{field}} as {{type}}) +safe_cast({{ field }} as {{ type }}) {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index 036f7d5a..10035a94 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,24 +1,21 @@ {% macro split_part(string_text, delimiter_text, part_number) %} - {{ return(adapter.dispatch('split_part', 'dbt_utils') (string_text, delimiter_text, part_number)) }} +{{ + return adapter.dispatch("split_part", "dbt_utils")( + string_text, delimiter_text, part_number + ) +}} {% endmacro %} {% macro default__split_part(string_text, delimiter_text, part_number) %} - split_part( - {{ string_text }}, - {{ delimiter_text }}, - {{ part_number }} - ) +split_part({{ string_text }}, {{ delimiter_text }}, {{ part_number }}) {% endmacro %} {% macro bigquery__split_part(string_text, delimiter_text, part_number) %} - split( - {{ string_text }}, - {{ delimiter_text }} - )[safe_offset({{ part_number - 1 }})] +split({{ string_text }}, {{ delimiter_text }}) [safe_offset({{ part_number - 1 }})] {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index fc8ae81c..38afcc25 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,5 +1,9 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} - {{ return(adapter.dispatch('width_bucket', 'dbt_utils') (expr, min_value, max_value, num_buckets)) }} +{{ + return adapter.dispatch("width_bucket", "dbt_utils")( + expr, min_value, max_value, num_buckets + ) +}} {% endmacro %} @@ -8,25 +12,20 @@ {% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) {%- endset %} - ( - -- to break ties when the amount is eaxtly at the bucket egde - case - when - mod( - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }}, - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - ) = 0 - then 1 - else 0 - end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 - ) +( + -- to break ties when the amount is eaxtly at the bucket egde + case + when + mod( + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }}, + {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + ) = 0 + then 1 + else 0 + end +) + +-- Anything over max_value goes the N+1 bucket +least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} @@ -34,26 +33,22 @@ {% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) {%- endset %} - ( - -- to break ties when the amount is exactly at the bucket edge - case - when - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }} % - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - = 0 - then 1 - else 0 - end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 - ) +( + -- to break ties when the amount is exactly at the bucket edge + case + when + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }} + % + {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + = 0 + then 1 + else 0 + end +) + +-- Anything over max_value goes the N+1 bucket +least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} - width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }} ) +width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }}) {% endmacro %} diff --git a/macros/generic_tests/accepted_range.sql b/macros/generic_tests/accepted_range.sql index 49ab00d9..0e91c4a7 100644 --- a/macros/generic_tests/accepted_range.sql +++ b/macros/generic_tests/accepted_range.sql @@ -1,31 +1,40 @@ -{% test accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} - {{ return(adapter.dispatch('test_accepted_range', 'dbt_utils')(model, column_name, min_value, max_value, inclusive)) }} +{% test accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} +{{ + return adapter.dispatch("test_accepted_range", "dbt_utils")( + model, column_name, min_value, max_value, inclusive + ) +}} {% endtest %} -{% macro default__test_accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} +{% macro default__test_accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} -with meet_condition as( - select * - from {{ model }} -), +with + meet_condition as (select * from {{ model }}), -validation_errors as ( - select * - from meet_condition - where - -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds - 1 = 2 + validation_errors as ( + select * + from meet_condition + where + -- never true, defaults to an empty result set. Exists to ensure any combo + -- of the `or` clauses below succeeds + 1 = 2 - {%- if min_value is not none %} - -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} - {%- endif %} + {%- if min_value is not none %} + -- records with a value >= min_value are permitted. The `not` flips this + -- to find records that don't meet the rule. + or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} + {%- endif %} - {%- if max_value is not none %} - -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} - {%- endif %} -) + {%- if max_value is not none %} + -- records with a value <= max_value are permitted. The `not` flips this + -- to find records that don't meet the rule. + or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} + {%- endif %} + ) select * from validation_errors diff --git a/macros/generic_tests/at_least_one.sql b/macros/generic_tests/at_least_one.sql index ce02108f..4df140bc 100644 --- a/macros/generic_tests/at_least_one.sql +++ b/macros/generic_tests/at_least_one.sql @@ -1,20 +1,20 @@ {% test at_least_one(model, column_name) %} - {{ return(adapter.dispatch('test_at_least_one', 'dbt_utils')(model, column_name)) }} +{{ return adapter.dispatch("test_at_least_one", "dbt_utils")(model, column_name) }} {% endtest %} {% macro default__test_at_least_one(model, column_name) %} select * -from ( - select +from + ( {# In TSQL, subquery aggregate columns need aliases #} {# thus: a filler col name, 'filler_column' #} - count({{ column_name }}) as filler_column + select count({{ column_name }}) as filler_column - from {{ model }} + from {{ model }} - having count({{ column_name }}) = 0 + having count({{ column_name }}) = 0 -) validation_errors + ) validation_errors {% endmacro %} diff --git a/macros/generic_tests/cardinality_equality.sql b/macros/generic_tests/cardinality_equality.sql index 4ec7d87f..90df0c47 100644 --- a/macros/generic_tests/cardinality_equality.sql +++ b/macros/generic_tests/cardinality_equality.sql @@ -1,51 +1,31 @@ {% test cardinality_equality(model, column_name, to, field) %} - {{ return(adapter.dispatch('test_cardinality_equality', 'dbt_utils')(model, column_name, to, field)) }} +{{ + return adapter.dispatch("test_cardinality_equality", "dbt_utils")( + model, column_name, to, field + ) +}} {% endtest %} {% macro default__test_cardinality_equality(model, column_name, to, field) %} {# T-SQL does not let you use numbers as aliases for columns #} {# Thus, no "GROUP BY 1" #} +with + table_a as ( + select {{ column_name }}, count(*) as num_rows + from {{ model }} + group by {{ column_name }} + ), -with table_a as ( -select - {{ column_name }}, - count(*) as num_rows -from {{ model }} -group by {{ column_name }} -), - -table_b as ( -select - {{ field }}, - count(*) as num_rows -from {{ to }} -group by {{ field }} -), - -except_a as ( - select * - from table_a - {{ dbt_utils.except() }} - select * - from table_b -), - -except_b as ( - select * - from table_b - {{ dbt_utils.except() }} - select * - from table_a -), - -unioned as ( - select * - from except_a - union all - select * - from except_b -) + table_b as ( + select {{ field }}, count(*) as num_rows from {{ to }} group by {{ field }} + ), + + except_a as (select * from table_a {{ dbt_utils.except() }} select * from table_b), + + except_b as (select * from table_b {{ dbt_utils.except() }} select * from table_a), + + unioned as (select * from except_a union all select * from except_b) select * from unioned diff --git a/macros/generic_tests/equal_rowcount.sql b/macros/generic_tests/equal_rowcount.sql index 713cb12a..19eccae3 100644 --- a/macros/generic_tests/equal_rowcount.sql +++ b/macros/generic_tests/equal_rowcount.sql @@ -1,38 +1,27 @@ {% test equal_rowcount(model, compare_model) %} - {{ return(adapter.dispatch('test_equal_rowcount', 'dbt_utils')(model, compare_model)) }} +{{ return adapter.dispatch("test_equal_rowcount", "dbt_utils")(model, compare_model) }} {% endtest %} {% macro default__test_equal_rowcount(model, compare_model) %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = 'coalesce(diff_count, 0)') }} +{#- - Needs to be set at parse time, before we return '' below --#} +{{ config(fail_calc="coalesce(diff_count, 0)") }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return "" }} {% endif %} -with a as ( +with + a as (select count(*) as count_a from {{ model }}), + b as (select count(*) as count_b from {{ compare_model }}), + final as ( - select count(*) as count_a from {{ model }} + select count_a, count_b, abs(count_a - count_b) as diff_count + from a + cross join b -), -b as ( + ) - select count(*) as count_b from {{ compare_model }} - -), -final as ( - - select - count_a, - count_b, - abs(count_a - count_b) as diff_count - from a - cross join b - -) - -select * from final +select * +from final {% endmacro %} diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index bc302c18..8acec338 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -1,5 +1,9 @@ {% test equality(model, compare_model, compare_columns=None) %} - {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns)) }} +{{ + return adapter.dispatch("test_equality", "dbt_utils")( + model, compare_model, compare_columns + ) +}} {% endtest %} {% macro default__test_equality(model, compare_model, compare_columns=None) %} @@ -11,65 +15,62 @@ ), 0) {% endset %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = set_diff) }} +{#- - Needs to be set at parse time, before we return '' below --#} +{{ config(fail_calc=set_diff) }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return "" }} {% endif %} -- setup -{%- do dbt_utils._is_relation(model, 'test_equality') -%} +{%- do dbt_utils._is_relation(model, "test_equality") -%} {#- If the compare_cols arg is provided, we can run this test without querying the information schema — this allows the model to be an ephemeral model -#} - {%- if not compare_columns -%} - {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} +{%- do dbt_utils._is_ephemeral(model, "test_equality") -%} +{%- set compare_columns = adapter.get_columns_in_relation(model) | map( + attribute="quoted" +) -%} {%- endif -%} -{% set compare_cols_csv = compare_columns | join(', ') %} - -with a as ( - - select * from {{ model }} - -), - -b as ( +{% set compare_cols_csv = compare_columns | join(", ") %} - select * from {{ compare_model }} +with + a as (select * from {{ model }}), -), + b as (select * from {{ compare_model }}), -a_minus_b as ( + a_minus_b as ( - select {{compare_cols_csv}} from a - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from b + select {{ compare_cols_csv }} + from a {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from b -), + ), -b_minus_a as ( + b_minus_a as ( - select {{compare_cols_csv}} from b - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from a + select {{ compare_cols_csv }} + from b {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from a -), + ), -unioned as ( + unioned as ( - select 'a_minus_b' as which_diff, a_minus_b.* from a_minus_b - union all - select 'b_minus_a' as which_diff, b_minus_a.* from b_minus_a + select 'a_minus_b' as which_diff, a_minus_b.* + from a_minus_b + union all + select 'b_minus_a' as which_diff, b_minus_a.* + from b_minus_a -) + ) -select * from unioned +select * +from unioned {% endmacro %} diff --git a/macros/generic_tests/expression_is_true.sql b/macros/generic_tests/expression_is_true.sql index 611685b4..d3285004 100644 --- a/macros/generic_tests/expression_is_true.sql +++ b/macros/generic_tests/expression_is_true.sql @@ -1,22 +1,21 @@ -{% test expression_is_true(model, expression, column_name=None, condition='1=1') %} +{% test expression_is_true(model, expression, column_name=None, condition="1=1") %} {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} {# ref https://stackoverflow.com/a/7170753/3842610 #} - {{ return(adapter.dispatch('test_expression_is_true', 'dbt_utils')(model, expression, column_name, condition)) }} +{{ + return adapter.dispatch("test_expression_is_true", "dbt_utils")( + model, expression, column_name, condition + ) +}} {% endtest %} {% macro default__test_expression_is_true(model, expression, column_name, condition) %} -with meet_condition as ( - select * from {{ model }} where {{ condition }} -) +with meet_condition as (select * from {{ model }} where {{ condition }}) -select - * +select * from meet_condition -{% if column_name is none %} -where not({{ expression }}) -{%- else %} -where not({{ column_name }} {{ expression }}) +{% if column_name is none %} where not ({{ expression }}) +{%- else %} where not ({{ column_name }} {{ expression }}) {%- endif %} {% endmacro %} diff --git a/macros/generic_tests/fewer_rows_than.sql b/macros/generic_tests/fewer_rows_than.sql index 450148b7..dfcefa6e 100644 --- a/macros/generic_tests/fewer_rows_than.sql +++ b/macros/generic_tests/fewer_rows_than.sql @@ -1,45 +1,35 @@ {% test fewer_rows_than(model, compare_model) %} - {{ return(adapter.dispatch('test_fewer_rows_than', 'dbt_utils')(model, compare_model)) }} +{{ return adapter.dispatch("test_fewer_rows_than", "dbt_utils")(model, compare_model) }} {% endtest %} {% macro default__test_fewer_rows_than(model, compare_model) %} -{{ config(fail_calc = 'coalesce(row_count_delta, 0)') }} - -with a as ( - - select count(*) as count_our_model from {{ model }} - -), -b as ( - - select count(*) as count_comparison_model from {{ compare_model }} - -), -counts as ( - - select - count_our_model, - count_comparison_model - from a - cross join b - -), -final as ( - - select *, - case - -- fail the test if we have more rows than the reference model and return the row count delta - when count_our_model > count_comparison_model then (count_our_model - count_comparison_model) - -- fail the test if they are the same number - when count_our_model = count_comparison_model then 1 - -- pass the test if the delta is positive (i.e. return the number 0) - else 0 - end as row_count_delta - from counts - -) - -select * from final +{{ config(fail_calc="coalesce(row_count_delta, 0)") }} + +with + a as (select count(*) as count_our_model from {{ model }}), + b as (select count(*) as count_comparison_model from {{ compare_model }}), + counts as (select count_our_model, count_comparison_model from a cross join b), + final as ( + + select + *, + case + -- fail the test if we have more rows than the reference model and + -- return the row count delta + when count_our_model > count_comparison_model + then (count_our_model - count_comparison_model) + -- fail the test if they are the same number + when count_our_model = count_comparison_model + then 1 + -- pass the test if the delta is positive (i.e. return the number 0) + else 0 + end as row_count_delta + from counts + + ) + +select * +from final {% endmacro %} diff --git a/macros/generic_tests/mutually_exclusive_ranges.sql b/macros/generic_tests/mutually_exclusive_ranges.sql index f88efbe4..1dbf9fde 100644 --- a/macros/generic_tests/mutually_exclusive_ranges.sql +++ b/macros/generic_tests/mutually_exclusive_ranges.sql @@ -1,97 +1,117 @@ -{% test mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} - {{ return(adapter.dispatch('test_mutually_exclusive_ranges', 'dbt_utils')(model, lower_bound_column, upper_bound_column, partition_by, gaps, zero_length_range_allowed)) }} +{% test mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False +) %} +{{ + return adapter.dispatch("test_mutually_exclusive_ranges", "dbt_utils")( + model, + lower_bound_column, + upper_bound_column, + partition_by, + gaps, + zero_length_range_allowed, + ) +}} {% endtest %} -{% macro default__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} -{% if gaps == 'not_allowed' %} - {% set allow_gaps_operator='=' %} - {% set allow_gaps_operator_in_words='equal_to' %} -{% elif gaps == 'allowed' %} - {% set allow_gaps_operator='<=' %} - {% set allow_gaps_operator_in_words='less_than_or_equal_to' %} -{% elif gaps == 'required' %} - {% set allow_gaps_operator='<' %} - {% set allow_gaps_operator_in_words='less_than' %} +{% macro default__test_mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False +) %} +{% if gaps == "not_allowed" %} +{% set allow_gaps_operator = "=" %} {% set allow_gaps_operator_in_words = "equal_to" %} +{% elif gaps == "allowed" %} +{% set allow_gaps_operator = "<=" %} +{% set allow_gaps_operator_in_words = "less_than_or_equal_to" %} +{% elif gaps == "required" %} +{% set allow_gaps_operator = "<" %} {% set allow_gaps_operator_in_words = "less_than" %} {% else %} - {{ exceptions.raise_compiler_error( +{{ exceptions.raise_compiler_error( "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" ) }} {% endif %} {% if not zero_length_range_allowed %} - {% set allow_zero_length_operator='<' %} - {% set allow_zero_length_operator_in_words='less_than' %} +{% set allow_zero_length_operator = "<" %} +{% set allow_zero_length_operator_in_words = "less_than" %} {% elif zero_length_range_allowed %} - {% set allow_zero_length_operator='<=' %} - {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %} +{% set allow_zero_length_operator = "<=" %} +{% set allow_zero_length_operator_in_words = "less_than_or_equal_to" %} {% else %} - {{ exceptions.raise_compiler_error( +{{ exceptions.raise_compiler_error( "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length_range_allowed ~"'.'" ) }} {% endif %} {% set partition_clause="partition by " ~ partition_by if partition_by else '' %} -with window_functions as ( - - select - {% if partition_by %} - {{ partition_by }} as partition_by_col, - {% endif %} - {{ lower_bound_column }} as lower_bound, - {{ upper_bound_column }} as upper_bound, - - lead({{ lower_bound_column }}) over ( - {{ partition_clause }} - order by {{ lower_bound_column }} - ) as next_lower_bound, - - row_number() over ( - {{ partition_clause }} - order by {{ lower_bound_column }} desc - ) = 1 as is_last_record - - from {{ model }} - -), - -calc as ( - -- We want to return records where one of our assumptions fails, so we'll use - -- the `not` function with `and` statements so we can write our assumptions nore cleanly - select - *, - - -- For each record: lower_bound should be < upper_bound. - -- Coalesce it to return an error on the null case (implicit assumption - -- these columns are not_null) - coalesce( - lower_bound {{ allow_zero_length_operator }} upper_bound, - false - ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, - - -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. - -- Coalesce it to handle null cases for the last record. - coalesce( - upper_bound {{ allow_gaps_operator }} next_lower_bound, - is_last_record, - false - ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound - - from window_functions - -), - -validation_errors as ( - - select - * - from calc - - where not( - -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound - and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound +with + window_functions as ( + + select + {% if partition_by %} {{ partition_by }} as partition_by_col, {% endif %} + {{ lower_bound_column }} as lower_bound, + {{ upper_bound_column }} as upper_bound, + + lead({{ lower_bound_column }}) over ( + {{ partition_clause }} order by {{ lower_bound_column }} + ) as next_lower_bound, + + row_number() over ( + {{ partition_clause }} order by {{ lower_bound_column }} desc + ) = 1 as is_last_record + + from {{ model }} + + ), + + calc as ( + -- We want to return records where one of our assumptions fails, so we'll use + -- the `not` function with `and` statements so we can write our assumptions + -- nore cleanly + select + *, + + -- For each record: lower_bound should be < upper_bound. + -- Coalesce it to return an error on the null case (implicit assumption + -- these columns are not_null) + coalesce( + lower_bound {{ allow_zero_length_operator }} upper_bound, false + ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, + + -- For each record: upper_bound {{ allow_gaps_operator }} the next + -- lower_bound. + -- Coalesce it to handle null cases for the last record. + coalesce( + upper_bound {{ allow_gaps_operator }} next_lower_bound, + is_last_record, + false + ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + + from window_functions + + ), + + validation_errors as ( + + select * + from calc + + where + not ( + -- THE FOLLOWING SHOULD BE TRUE -- + lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound + and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + ) ) -) -select * from validation_errors +select * +from validation_errors {% endmacro %} diff --git a/macros/generic_tests/not_accepted_values.sql b/macros/generic_tests/not_accepted_values.sql index ab24188f..f618da35 100644 --- a/macros/generic_tests/not_accepted_values.sql +++ b/macros/generic_tests/not_accepted_values.sql @@ -1,35 +1,31 @@ {% test not_accepted_values(model, column_name, values, quote=True) %} - {{ return(adapter.dispatch('test_not_accepted_values', 'dbt_utils')(model, column_name, values, quote)) }} +{{ + return adapter.dispatch("test_not_accepted_values", "dbt_utils")( + model, column_name, values, quote + ) +}} {% endtest %} {% macro default__test_not_accepted_values(model, column_name, values, quote=True) %} -with all_values as ( +with + all_values as (select distinct {{ column_name }} as value_field from {{ model }}), - select distinct - {{ column_name }} as value_field + validation_errors as ( - from {{ model }} + select value_field -), + from all_values + where + value_field in ( + {% for value in values -%} + {% if quote -%}'{{ value }}' + {%- else -%}{{ value }} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {%- endfor %} + ) -validation_errors as ( - - select - value_field - - from all_values - where value_field in ( - {% for value in values -%} - {% if quote -%} - '{{ value }}' - {%- else -%} - {{ value }} - {%- endif -%} - {%- if not loop.last -%},{%- endif %} - {%- endfor %} - ) - -) + ) select * from validation_errors diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index 781ed7d9..bae4b54b 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -1,15 +1,14 @@ {% test not_constant(model, column_name) %} - {{ return(adapter.dispatch('test_not_constant', 'dbt_utils')(model, column_name)) }} +{{ return adapter.dispatch("test_not_constant", "dbt_utils")(model, column_name) }} {% endtest %} {% macro default__test_not_constant(model, column_name) %} -select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count(distinct {{ column_name }}) as filler_column +{# In TSQL, subquery aggregate columns need aliases #} +{# thus: a filler col name, 'filler_column' #} +select count(distinct {{ column_name }}) as filler_column from {{ model }} diff --git a/macros/generic_tests/not_null_proportion.sql b/macros/generic_tests/not_null_proportion.sql index 45b9050c..ef56bb65 100644 --- a/macros/generic_tests/not_null_proportion.sql +++ b/macros/generic_tests/not_null_proportion.sql @@ -1,26 +1,28 @@ {% macro test_not_null_proportion(model) %} - {{ return(adapter.dispatch('test_not_null_proportion', 'dbt_utils')(model, **kwargs)) }} +{{ return adapter.dispatch("test_not_null_proportion", "dbt_utils")(model, **kwargs) }} {% endmacro %} {% macro default__test_not_null_proportion(model) %} -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} -{% set at_least = kwargs.get('at_least', kwargs.get('arg')) %} -{% set at_most = kwargs.get('at_most', kwargs.get('arg', 1)) %} +{% set column_name = kwargs.get("column_name", kwargs.get("arg")) %} +{% set at_least = kwargs.get("at_least", kwargs.get("arg")) %} +{% set at_most = kwargs.get("at_most", kwargs.get("arg", 1)) %} -with validation as ( - select - sum(case when {{ column_name }} is null then 0 else 1 end) / cast(count(*) as numeric) as not_null_proportion - from {{ model }} -), -validation_errors as ( - select - not_null_proportion - from validation - where not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} -) -select - * +with + validation as ( + select + sum(case when {{ column_name }} is null then 0 else 1 end) / cast( + count(*) as numeric + ) as not_null_proportion + from {{ model }} + ), + validation_errors as ( + select not_null_proportion + from validation + where + not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} + ) +select * from validation_errors {% endmacro %} diff --git a/macros/generic_tests/recency.sql b/macros/generic_tests/recency.sql index cb9a8de6..44b02295 100644 --- a/macros/generic_tests/recency.sql +++ b/macros/generic_tests/recency.sql @@ -1,22 +1,20 @@ {% test recency(model, field, datepart, interval) %} - {{ return(adapter.dispatch('test_recency', 'dbt_utils')(model, field, datepart, interval)) }} +{{ + return adapter.dispatch("test_recency", "dbt_utils")( + model, field, datepart, interval + ) +}} {% endtest %} {% macro default__test_recency(model, field, datepart, interval) %} -{% set threshold = dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp()) %} +{% set threshold = dbt_utils.dateadd( + datepart, interval * -1, dbt_utils.current_timestamp() +) %} -with recency as ( +with recency as (select max({{ field }}) as most_recent from {{ model }}) - select max({{field}}) as most_recent - from {{ model }} - -) - -select - - most_recent, - {{ threshold }} as threshold +select most_recent, {{ threshold }} as threshold from recency where most_recent < {{ threshold }} diff --git a/macros/generic_tests/relationships_where.sql b/macros/generic_tests/relationships_where.sql index c35a380a..fd68632d 100644 --- a/macros/generic_tests/relationships_where.sql +++ b/macros/generic_tests/relationships_where.sql @@ -1,51 +1,53 @@ -{% test relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} - {{ return(adapter.dispatch('test_relationships_where', 'dbt_utils')(model, column_name, to, field, from_condition, to_condition)) }} +{% test relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} +{{ + return adapter.dispatch("test_relationships_where", "dbt_utils")( + model, column_name, to, field, from_condition, to_condition + ) +}} {% endtest %} -{% macro default__test_relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} +{% macro default__test_relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} {# ref https://stackoverflow.com/a/7170753/3842610 #} +with + left_table as ( -with left_table as ( + select {{ column_name }} as id - select - {{column_name}} as id + from {{ model }} - from {{model}} + where {{ column_name }} is not null and {{ from_condition }} - where {{column_name}} is not null - and {{from_condition}} + ), -), + right_table as ( -right_table as ( + select {{ field }} as id - select - {{field}} as id + from {{ to }} - from {{to}} + where {{ field }} is not null and {{ to_condition }} - where {{field}} is not null - and {{to_condition}} + ), -), + exceptions as ( -exceptions as ( + select left_table.id, right_table.id as right_id - select - left_table.id, - right_table.id as right_id + from left_table - from left_table + left join right_table on left_table.id = right_table.id - left join right_table - on left_table.id = right_table.id + where right_table.id is null - where right_table.id is null + ) -) - -select * from exceptions +select * +from exceptions {% endmacro %} diff --git a/macros/generic_tests/sequential_values.sql b/macros/generic_tests/sequential_values.sql index 8ddae707..4df9770b 100644 --- a/macros/generic_tests/sequential_values.sql +++ b/macros/generic_tests/sequential_values.sql @@ -1,33 +1,45 @@ {% test sequential_values(model, column_name, interval=1, datepart=None) %} - {{ return(adapter.dispatch('test_sequential_values', 'dbt_utils')(model, column_name, interval, datepart)) }} +{{ + return adapter.dispatch("test_sequential_values", "dbt_utils")( + model, column_name, interval, datepart + ) +}} {% endtest %} -{% macro default__test_sequential_values(model, column_name, interval=1, datepart=None) %} +{% macro default__test_sequential_values( + model, column_name, interval=1, datepart=None +) %} {% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} -with windowed as ( - - select - {{ column_name }}, - lag({{ column_name }}) over ( - order by {{ column_name }} - ) as {{ previous_column_name }} - from {{ model }} -), - -validation_errors as ( - select - * - from windowed - {% if datepart %} - where not(cast({{ column_name }} as {{ dbt_utils.type_timestamp() }})= cast({{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} as {{ dbt_utils.type_timestamp() }})) - {% else %} - where not({{ column_name }} = {{ previous_column_name }} + {{ interval }}) - {% endif %} -) +with + windowed as ( + + select + {{ column_name }}, + lag({{ column_name }}) over ( + order by {{ column_name }} + ) as {{ previous_column_name }} + from {{ model }} + ), + + validation_errors as ( + select * + from windowed + {% if datepart %} + where + not ( + cast({{ column_name }} as {{ dbt_utils.type_timestamp() }}) = cast( + {{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} + as {{ dbt_utils.type_timestamp() }} + ) + ) + {% else %} + where not ({{ column_name }} = {{ previous_column_name }} + {{ interval }}) + {% endif %} + ) select * from validation_errors diff --git a/macros/generic_tests/test_not_null_where.sql b/macros/generic_tests/test_not_null_where.sql index d5dbf6c5..44f695f6 100644 --- a/macros/generic_tests/test_not_null_where.sql +++ b/macros/generic_tests/test_not_null_where.sql @@ -1,12 +1,9 @@ -{% test not_null_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.not_null_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_not_null_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_not_null_where(model, column_name) %} - {{ return(test_not_null(model, column_name)) }} -{% endmacro %} +{% test not_null_where(model, column_name) %} +{%- set deprecation_warning = " Warning: `dbt_utils.not_null_where` is no longer supported. Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. " -%} +{%- do exceptions.warn(deprecation_warning) -%} +{{ return adapter.dispatch("test_not_null_where", "dbt_utils")(model, column_name) }} +{% endtest %} + +{% macro default__test_not_null_where(model, column_name) %} +{{ return test_not_null(model, column_name) }} +{% endmacro %} diff --git a/macros/generic_tests/test_unique_where.sql b/macros/generic_tests/test_unique_where.sql index e752d7b2..915e0918 100644 --- a/macros/generic_tests/test_unique_where.sql +++ b/macros/generic_tests/test_unique_where.sql @@ -1,12 +1,9 @@ -{% test unique_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.unique_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_unique_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_unique_where(model, column_name) %} - {{ return(test_unique(model, column_name)) }} -{% endmacro %} +{% test unique_where(model, column_name) %} +{%- set deprecation_warning = " Warning: `dbt_utils.unique_where` is no longer supported. Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. " -%} +{%- do exceptions.warn(deprecation_warning) -%} +{{ return adapter.dispatch("test_unique_where", "dbt_utils")(model, column_name) }} +{% endtest %} + +{% macro default__test_unique_where(model, column_name) %} +{{ return test_unique(model, column_name) }} +{% endmacro %} diff --git a/macros/generic_tests/unique_combination_of_columns.sql b/macros/generic_tests/unique_combination_of_columns.sql index 74ccf5c6..db16a3fd 100644 --- a/macros/generic_tests/unique_combination_of_columns.sql +++ b/macros/generic_tests/unique_combination_of_columns.sql @@ -1,34 +1,41 @@ -{% test unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} - {{ return(adapter.dispatch('test_unique_combination_of_columns', 'dbt_utils')(model, combination_of_columns, quote_columns)) }} +{% test unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} +{{ + return adapter.dispatch("test_unique_combination_of_columns", "dbt_utils")( + model, combination_of_columns, quote_columns + ) +}} {% endtest %} -{% macro default__test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} +{% macro default__test_unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} -{% if not quote_columns %} - {%- set column_list=combination_of_columns %} +{% if not quote_columns %} {%- set column_list = combination_of_columns %} {% elif quote_columns %} - {%- set column_list=[] %} - {% for column in combination_of_columns -%} - {% set column_list = column_list.append( adapter.quote(column) ) %} - {%- endfor %} +{%- set column_list = [] %} +{% for column in combination_of_columns -%} +{% set column_list = column_list.append(adapter.quote(column)) %} +{%- endfor %} {% else %} - {{ exceptions.raise_compiler_error( +{{ exceptions.raise_compiler_error( "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" ) }} {% endif %} -{%- set columns_csv=column_list | join(', ') %} +{%- set columns_csv = column_list | join(", ") %} -with validation_errors as ( +with + validation_errors as ( - select - {{ columns_csv }} - from {{ model }} - group by {{ columns_csv }} - having count(*) > 1 + select {{ columns_csv }} + from {{ model }} + group by {{ columns_csv }} + having count(*) > 1 -) + ) select * from validation_errors diff --git a/macros/jinja_helpers/log_info.sql b/macros/jinja_helpers/log_info.sql index 52b4b4a5..ded63447 100644 --- a/macros/jinja_helpers/log_info.sql +++ b/macros/jinja_helpers/log_info.sql @@ -1,7 +1,7 @@ {% macro log_info(message) %} - {{ return(adapter.dispatch('log_info', 'dbt_utils')(message)) }} +{{ return adapter.dispatch("log_info", "dbt_utils")(message) }} {% endmacro %} {% macro default__log_info(message) %} - {{ log(dbt_utils.pretty_log_format(message), info=True) }} +{{ log(dbt_utils.pretty_log_format(message), info=True) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_log_format.sql b/macros/jinja_helpers/pretty_log_format.sql index fe580d13..ed354c41 100644 --- a/macros/jinja_helpers/pretty_log_format.sql +++ b/macros/jinja_helpers/pretty_log_format.sql @@ -1,7 +1,7 @@ {% macro pretty_log_format(message) %} - {{ return(adapter.dispatch('pretty_log_format', 'dbt_utils')(message)) }} +{{ return adapter.dispatch("pretty_log_format", "dbt_utils")(message) }} {% endmacro %} {% macro default__pretty_log_format(message) %} - {{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} +{{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_time.sql b/macros/jinja_helpers/pretty_time.sql index bad37efe..0b07689e 100644 --- a/macros/jinja_helpers/pretty_time.sql +++ b/macros/jinja_helpers/pretty_time.sql @@ -1,7 +1,7 @@ -{% macro pretty_time(format='%H:%M:%S') %} - {{ return(adapter.dispatch('pretty_time', 'dbt_utils')(format)) }} +{% macro pretty_time(format="%H:%M:%S") %} +{{ return adapter.dispatch("pretty_time", "dbt_utils")(format) }} {% endmacro %} -{% macro default__pretty_time(format='%H:%M:%S') %} - {{ return(modules.datetime.datetime.now().strftime(format)) }} +{% macro default__pretty_time(format="%H:%M:%S") %} +{{ return modules.datetime.datetime.now().strftime(format) }} {% endmacro %} diff --git a/macros/jinja_helpers/slugify.sql b/macros/jinja_helpers/slugify.sql index 1b3c7272..9131ccaf 100644 --- a/macros/jinja_helpers/slugify.sql +++ b/macros/jinja_helpers/slugify.sql @@ -3,10 +3,10 @@ {#- Lower case the string -#} {% set string = string | lower %} {#- Replace spaces and dashes with underscores -#} -{% set string = modules.re.sub('[ -]+', '_', string) %} +{% set string = modules.re.sub("[ -]+", "_", string) %} {#- Only take letters, numbers, and underscores -#} -{% set string = modules.re.sub('[^a-z0-9_]+', '', string) %} +{% set string = modules.re.sub("[^a-z0-9_]+", "", string) %} -{{ return(string) }} +{{ return string }} {% endmacro %} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 5a15c815..206082ad 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -1,38 +1,84 @@ -{% macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - {{ return(adapter.dispatch('get_period_boundaries', 'dbt_utils')(target_schema, target_table, timestamp_field, start_date, stop_date, period)) }} +{% macro get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period +) -%} +{{ + return adapter.dispatch("get_period_boundaries", "dbt_utils")( + target_schema, target_table, timestamp_field, start_date, stop_date, period + ) +}} {% endmacro %} -{% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - - {% call statement('period_boundaries', fetch_result=True) -%} - with data as ( - select - coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp as start_timestamp, - coalesce( - {{dbt_utils.dateadd('millisecond', +{% macro default__get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period +) -%} + +{% call statement('period_boundaries', fetch_result=True) -%} +with + data as ( + select + coalesce( + max("{{timestamp_field}}"), '{{start_date}}' + )::timestamp as start_timestamp, + coalesce( + {{dbt_utils.dateadd('millisecond', -1, "nullif('" ~ stop_date ~ "','')::timestamp")}}, - {{dbt_utils.current_timestamp()}} - ) as stop_timestamp - from "{{target_schema}}"."{{target_table}}" + {{ dbt_utils.current_timestamp() }} + ) as stop_timestamp + from "{{target_schema}}"."{{target_table}}" ) - select - start_timestamp, - stop_timestamp, - {{dbt_utils.datediff('start_timestamp', - 'stop_timestamp', - period)}} + 1 as num_periods - from data - {%- endcall %} +select + start_timestamp, + stop_timestamp, + {{ dbt_utils.datediff("start_timestamp", "stop_timestamp", period) }} + + 1 + as num_periods +from data {%- endcall %} {%- endmacro %} -{% macro get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} - {{ return(adapter.dispatch('get_period_sql', 'dbt_utils')(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset)) }} -{% endmacro %} - -{% macro default__get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} + {% macro get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset +) -%} + {{ + return adapter.dispatch("get_period_sql", "dbt_utils")( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset, + ) + }} + {% endmacro %} + +{% macro default__get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset +) -%} {%- set period_filter -%} ("{{timestamp_field}}" > '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' and @@ -40,150 +86,159 @@ "{{timestamp_field}}" < '{{stop_timestamp}}'::timestamp) {%- endset -%} - {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} + {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} - select - {{target_cols_csv}} - from ( - {{filtered_sql}} - ) +select {{ target_cols_csv }} +from ({{ filtered_sql }}) {%- endmacro %} -{% materialization insert_by_period, default -%} - {%- set timestamp_field = config.require('timestamp_field') -%} - {%- set start_date = config.require('start_date') -%} - {%- set stop_date = config.get('stop_date') or '' -%} - {%- set period = config.get('period') or 'week' -%} + {% materialization insert_by_period, default -%} + {%- set timestamp_field = config.require("timestamp_field") -%} + {%- set start_date = config.require("start_date") -%} + {%- set stop_date = config.get("stop_date") or "" -%} + {%- set period = config.get("period") or "week" -%} - {%- if sql.find('__PERIOD_FILTER__') == -1 -%} + {%- if sql.find("__PERIOD_FILTER__") == -1 -%} {%- set error_message -%} Model '{{ model.unique_id }}' does not include the required string '__PERIOD_FILTER__' in its sql {%- endset -%} {{ exceptions.raise_compiler_error(error_message) }} - {%- endif -%} - - {%- set identifier = model['name'] -%} - - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} - - {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - - {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} - {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} + {%- endif -%} + + {%- set identifier = model["name"] -%} + + {%- set old_relation = adapter.get_relation( + database=database, schema=schema, identifier=identifier +) -%} + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, type="table" +) -%} + + {%- set non_destructive_mode = flags.NON_DESTRUCTIVE == True -%} + {%- set full_refresh_mode = flags.FULL_REFRESH == True -%} + + {%- set exists_as_table = old_relation is not none and old_relation.is_table -%} + {%- set exists_not_as_table = ( + old_relation is not none and not old_relation.is_table +) -%} + + {%- set should_truncate = ( + non_destructive_mode and full_refresh_mode and exists_as_table +) -%} + {%- set should_drop = not should_truncate and ( + full_refresh_mode or exists_not_as_table +) -%} + {%- set force_create = flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE -%} + + -- setup + {% if old_relation is none -%} + -- noop + {%- elif should_truncate -%} {{ adapter.truncate_relation(old_relation) }} + {%- elif should_drop -%} + {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} + {%- endif %} - {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%} - {%- set should_drop = (not should_truncate and (full_refresh_mode or exists_not_as_table)) -%} - {%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%} + {{ run_hooks(pre_hooks, inside_transaction=False) }} - -- setup - {% if old_relation is none -%} - -- noop - {%- elif should_truncate -%} - {{adapter.truncate_relation(old_relation)}} - {%- elif should_drop -%} - {{adapter.drop_relation(old_relation)}} - {%- set old_relation = none -%} - {%- endif %} - - {{run_hooks(pre_hooks, inside_transaction=False)}} - - -- `begin` happens here, so `commit` after it to finish the transaction - {{run_hooks(pre_hooks, inside_transaction=True)}} - {% call statement() -%} - begin; -- make extra sure we've closed out the transaction - commit; - {%- endcall %} - - -- build model - {% if force_create or old_relation is none -%} - {# Create an empty target table -#} - {% call statement('main') -%} - {%- set empty_sql = sql | replace("__PERIOD_FILTER__", 'false') -%} - {{create_table_as(False, target_relation, empty_sql)}} - {%- endcall %} - {%- endif %} - - {% set _ = dbt_utils.get_period_boundaries(schema, - identifier, - timestamp_field, - start_date, - stop_date, - period) %} - {%- set start_timestamp = load_result('period_boundaries')['data'][0][0] | string -%} - {%- set stop_timestamp = load_result('period_boundaries')['data'][0][1] | string -%} - {%- set num_periods = load_result('period_boundaries')['data'][0][2] | int -%} - - {% set target_columns = adapter.get_columns_in_relation(target_relation) %} - {%- set target_cols_csv = target_columns | map(attribute='quoted') | join(', ') -%} - {%- set loop_vars = {'sum_rows_inserted': 0} -%} - - -- commit each period as a separate transaction - {% for i in range(num_periods) -%} - {%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} - {{ dbt_utils.log_info(msg) }} - - {%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%} - {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, - schema=schema, type='table') -%} + -- `begin` happens here, so `commit` after it to finish the transaction + {{ run_hooks(pre_hooks, inside_transaction=True) }} {% call statement() -%} - {% set tmp_table_sql = dbt_utils.get_period_sql(target_cols_csv, - sql, - timestamp_field, - period, - start_timestamp, - stop_timestamp, - i) %} - {{dbt.create_table_as(True, tmp_relation, tmp_table_sql)}} - {%- endcall %} - - {{adapter.expand_target_column_types(from_relation=tmp_relation, - to_relation=target_relation)}} - {%- set name = 'main-' ~ i -%} - {% call statement(name, fetch_result=True) -%} - insert into {{target_relation}} ({{target_cols_csv}}) - ( - select - {{target_cols_csv}} - from {{tmp_relation.include(schema=False)}} - ); - {%- endcall %} - {% set result = load_result('main-' ~ i) %} - {% if 'response' in result.keys() %} {# added in v0.19.0 #} - {% set rows_inserted = result['response']['rows_affected'] %} - {% else %} {# older versions #} - {% set rows_inserted = result['status'].split(" ")[2] | int %} - {% endif %} - - {%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%} - {%- if loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %} {% endif -%} - - {%- set msg = "Ran for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) ~ "; " ~ rows_inserted ~ " records inserted" -%} - {{ dbt_utils.log_info(msg) }} - - {%- endfor %} - - {% call statement() -%} - begin; - {%- endcall %} - - {{run_hooks(post_hooks, inside_transaction=True)}} - - {% call statement() -%} - commit; - {%- endcall %} - - {{run_hooks(post_hooks, inside_transaction=False)}} - - {%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} - - {% call noop_statement('main', status_string) -%} - -- no-op - {%- endcall %} - - -- Return the relations created in this materialization - {{ return({'relations': [target_relation]}) }} + begin -- make extra sure we've closed out the transaction +; +commit +; +{%- endcall %} + +-- build model +{% if force_create or old_relation is none -%} +{# Create an empty target table -#} +{% call statement('main') -%} +{%- set empty_sql = sql | replace("__PERIOD_FILTER__", "false") -%} +{{ create_table_as(False, target_relation, empty_sql) }} +{%- endcall %} +{%- endif %} + +{% set _ = dbt_utils.get_period_boundaries( + schema, identifier, timestamp_field, start_date, stop_date, period +) %} +{%- set start_timestamp = load_result("period_boundaries")["data"][0][0] | string -%} +{%- set stop_timestamp = load_result("period_boundaries")["data"][0][1] | string -%} +{%- set num_periods = load_result("period_boundaries")["data"][0][2] | int -%} + +{% set target_columns = adapter.get_columns_in_relation(target_relation) %} +{%- set target_cols_csv = target_columns | map(attribute="quoted") | join(", ") -%} +{%- set loop_vars = {"sum_rows_inserted": 0} -%} + +-- commit each period as a separate transaction +{% for i in range(num_periods) -%} +{%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} +{{ dbt_utils.log_info(msg) }} + +{%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%} +{%- set tmp_relation = api.Relation.create( + identifier=tmp_identifier, schema=schema, type="table" +) -%} +{% call statement() -%} +{% set tmp_table_sql = dbt_utils.get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + i, +) %} +{{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} +{%- endcall %} + +{{ + adapter.expand_target_column_types( + from_relation=tmp_relation, to_relation=target_relation + ) +}} +{%- set name = 'main-' ~ i -%} +{% call statement(name, fetch_result=True) -%} +insert into {{ target_relation }} ({{ target_cols_csv }}) +(select {{ target_cols_csv }} from {{ tmp_relation.include(schema=False) }}) +; +{%- endcall %} +{% set result = load_result('main-' ~ i) %} +{% if "response" in result.keys() %} {# added in v0.19.0 #} +{% set rows_inserted = result["response"]["rows_affected"] %} +{# older versions #} +{% else %} {% set rows_inserted = result["status"].split(" ")[2] | int %} +{% endif %} + +{%- set sum_rows_inserted = loop_vars["sum_rows_inserted"] + rows_inserted -%} +{%- if loop_vars.update({"sum_rows_inserted": sum_rows_inserted}) %} {% endif -%} + +{%- set msg = "Ran for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) ~ "; " ~ rows_inserted ~ " records inserted" -%} +{{ dbt_utils.log_info(msg) }} + +{%- endfor %} + +{% call statement() -%} +begin +; +{%- endcall %} + +{{ run_hooks(post_hooks, inside_transaction=True) }} + +{% call statement() -%} +commit +; +{%- endcall %} + +{{ run_hooks(post_hooks, inside_transaction=False) }} + +{%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} + +{% call noop_statement('main', status_string) -%} +-- no-op +{%- endcall %} + +-- Return the relations created in this materialization +{{ return {"relations": [target_relation]} }} {%- endmaterialization %} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index 759f8d59..39f587a0 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -1,21 +1,25 @@ {% macro get_intervals_between(start_date, end_date, datepart) -%} - {{ return(adapter.dispatch('get_intervals_between', 'dbt_utils')(start_date, end_date, datepart)) }} +{{ + return adapter.dispatch("get_intervals_between", "dbt_utils")( + start_date, end_date, datepart + ) +}} {%- endmacro %} {% macro default__get_intervals_between(start_date, end_date, datepart) -%} - {%- call statement('get_intervals_between', fetch_result=True) %} +{%- call statement('get_intervals_between', fetch_result=True) %} - select {{dbt_utils.datediff(start_date, end_date, datepart)}} +select + {{ dbt_utils.datediff(start_date, end_date, datepart) }} {%- endcall -%} - {%- set value_list = load_result('get_intervals_between') -%} + {%- set value_list = load_result("get_intervals_between") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} - {{ return(values[0]) }} - {%- else -%} - {{ return(1) }} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} + {{ return values[0] }} + {%- else -%} {{ return 1 }} {%- endif -%} {%- endmacro %} @@ -23,9 +27,13 @@ -{% macro date_spine(datepart, start_date, end_date) %} - {{ return(adapter.dispatch('date_spine', 'dbt_utils')(datepart, start_date, end_date)) }} -{%- endmacro %} + {% macro date_spine(datepart, start_date, end_date) %} + {{ + return adapter.dispatch("date_spine", "dbt_utils")( + datepart, start_date, end_date + ) + }} + {%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} @@ -37,39 +45,34 @@ date_spine( "to_date('01/01/2016', 'mm/dd/yyyy')", "dateadd(week, 1, current_date)" ) #} +with + rawdata as ( - -with rawdata as ( - - {{dbt_utils.generate_series( - dbt_utils.get_intervals_between(start_date, end_date, datepart) - )}} - -), - -all_periods as ( - - select ( {{ - dbt_utils.dateadd( - datepart, - "row_number() over (order by 1) - 1", - start_date + dbt_utils.generate_series( + dbt_utils.get_intervals_between(start_date, end_date, datepart) ) }} - ) as date_{{datepart}} - from rawdata -), + ), + + all_periods as ( -filtered as ( + select + ( + {{ + dbt_utils.dateadd( + datepart, "row_number() over (order by 1) - 1", start_date + ) + }} + ) as date_{{ datepart }} + from rawdata - select * - from all_periods - where date_{{datepart}} <= {{ end_date }} + ), -) + filtered as (select * from all_periods where date_{{ datepart }} <= {{ end_date }}) -select * from filtered +select * +from filtered {% endmacro %} diff --git a/macros/sql/deduplicate.sql b/macros/sql/deduplicate.sql index 9a3571a2..04e8c546 100644 --- a/macros/sql/deduplicate.sql +++ b/macros/sql/deduplicate.sql @@ -1,23 +1,27 @@ {%- macro deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} - {{ return(adapter.dispatch('deduplicate', 'dbt_utils')(relation, group_by, order_by=order_by, relation_alias=relation_alias)) }} +{{ + return adapter.dispatch("deduplicate", "dbt_utils")( + relation, group_by, order_by=order_by, relation_alias=relation_alias + ) +}} {% endmacro %} -{%- macro default__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro default__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( +select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} +from + ( select _inner.*, row_number() over ( partition by {{ group_by }} - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} ) as rn from {{ relation if relation_alias is none else relation_alias }} as _inner ) as deduped - where deduped.rn = 1 +where deduped.rn = 1 {%- endmacro -%} @@ -26,19 +30,19 @@ -- clause in BigQuery: -- https://github.com/dbt-labs/dbt-utils/issues/335#issuecomment-788157572 #} -{%- macro bigquery__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro bigquery__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( +select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} +from + ( select - array_agg ( + array_agg( original - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} limit 1 - )[offset(0)] as deduped + ) [offset (0)] as deduped from {{ relation if relation_alias is none else relation_alias }} as original group by {{ group_by }} ) diff --git a/macros/sql/generate_series.sql b/macros/sql/generate_series.sql index efcbd8ac..417dbfb2 100644 --- a/macros/sql/generate_series.sql +++ b/macros/sql/generate_series.sql @@ -1,53 +1,50 @@ {% macro get_powers_of_two(upper_bound) %} - {{ return(adapter.dispatch('get_powers_of_two', 'dbt_utils')(upper_bound)) }} +{{ return adapter.dispatch("get_powers_of_two", "dbt_utils")(upper_bound) }} {% endmacro %} {% macro default__get_powers_of_two(upper_bound) %} - {% if upper_bound <= 0 %} - {{ exceptions.raise_compiler_error("upper bound must be positive") }} - {% endif %} +{% if upper_bound <= 0 %} +{{ exceptions.raise_compiler_error("upper bound must be positive") }} +{% endif %} - {% for _ in range(1, 100) %} - {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} - {% endfor %} +{% for _ in range(1, 100) %} +{% if upper_bound <= 2**loop.index %} {{ return loop.index }}{% endif %} +{% endfor %} {% endmacro %} {% macro generate_series(upper_bound) %} - {{ return(adapter.dispatch('generate_series', 'dbt_utils')(upper_bound)) }} +{{ return adapter.dispatch("generate_series", "dbt_utils")(upper_bound) }} {% endmacro %} {% macro default__generate_series(upper_bound) %} - {% set n = dbt_utils.get_powers_of_two(upper_bound) %} +{% set n = dbt_utils.get_powers_of_two(upper_bound) %} - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( +with + p as (select 0 as generated_number union all select 1), + unioned as ( - select + select - {% for i in range(n) %} - p{{i}}.generated_number * power(2, {{i}}) - {% if not loop.last %} + {% endif %} - {% endfor %} - + 1 - as generated_number + {% for i in range(n) %} + p{{ i }}.generated_number * power(2, {{ i }}) + {% if not loop.last %} + {% endif %} + {% endfor %} + 1 as generated_number - from + from - {% for i in range(n) %} - p as p{{i}} - {% if not loop.last %} cross join {% endif %} - {% endfor %} + {% for i in range(n) %} + p as p{{ i }} {% if not loop.last %} cross join {% endif %} + {% endfor %} ) - select * - from unioned - where generated_number <= {{upper_bound}} - order by generated_number +select * +from unioned +where generated_number <= {{ upper_bound }} +order by generated_number {% endmacro %} diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index f70890e2..4aa08820 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -1,60 +1,63 @@ -{% macro get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {{ return(adapter.dispatch('get_column_values', 'dbt_utils')(table, column, order_by, max_records, default)) }} +{% macro get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} +{{ + return adapter.dispatch("get_column_values", "dbt_utils")( + table, column, order_by, max_records, default + ) +}} {% endmacro %} -{% macro default__get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {% set default = [] if not default %} - {{ return(default) }} - {% endif %} +{% macro default__get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} +{% set default = [] if not default %} {{ return default }} +{% endif %} - {%- do dbt_utils._is_ephemeral(table, 'get_column_values') -%} +{%- do dbt_utils._is_ephemeral(table, "get_column_values") -%} - {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} - {# TODO: Change the method signature in a future 0.x.0 release #} - {%- set target_relation = table -%} +{# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} +{# TODO: Change the method signature in a future 0.x.0 release #} +{%- set target_relation = table -%} - {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} - {% set relation_exists = (load_relation(target_relation)) is not none %} +{# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} +{% set relation_exists = (load_relation(target_relation)) is not none %} - {%- call statement('get_column_values', fetch_result=true) %} +{%- call statement('get_column_values', fetch_result=true) %} - {%- if not relation_exists and default is none -%} +{%- if not relation_exists and default is none -%} - {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} +{{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} - {%- elif not relation_exists and default is not none -%} +{%- elif not relation_exists and default is not none -%} - {{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} +{{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} - {{ return(default) }} +{{ return default }} - {%- else -%} +{%- else -%} - select - {{ column }} as value +select {{ column }} as value - from {{ target_relation }} - group by {{ column }} - order by {{ order_by }} +from {{ target_relation }} +group by {{ column }} +order by {{ order_by }} - {% if max_records is not none %} - limit {{ max_records }} - {% endif %} +{% if max_records is not none %} limit {{ max_records }} +{% endif %} - {% endif %} +{% endif %} {%- endcall -%} - {%- set value_list = load_result('get_column_values') -%} + {%- set value_list = load_result("get_column_values") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} - {{ return(values) }} - {%- else -%} - {{ return(default) }} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} {{ return values }} + {%- else -%} {{ return default }} {%- endif -%} {%- endmacro %} diff --git a/macros/sql/get_filtered_columns_in_relation.sql b/macros/sql/get_filtered_columns_in_relation.sql index 7f4af889..4885c473 100644 --- a/macros/sql/get_filtered_columns_in_relation.sql +++ b/macros/sql/get_filtered_columns_in_relation.sql @@ -1,25 +1,23 @@ {% macro get_filtered_columns_in_relation(from, except=[]) -%} - {{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} +{{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} {% endmacro %} {% macro default__get_filtered_columns_in_relation(from, except=[]) -%} - {%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} - {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} +{%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} +{%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} - {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('') }} - {% endif %} +{# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return "" }} {% endif %} - {%- set include_cols = [] %} - {%- set cols = adapter.get_columns_in_relation(from) -%} - {%- set except = except | map("lower") | list %} - {%- for col in cols -%} - {%- if col.column|lower not in except -%} - {% do include_cols.append(col.column) %} - {%- endif %} - {%- endfor %} +{%- set include_cols = [] %} +{%- set cols = adapter.get_columns_in_relation(from) -%} +{%- set except = except | map("lower") | list %} +{%- for col in cols -%} +{%- if col.column|lower not in except -%} +{% do include_cols.append(col.column) %} +{%- endif %} +{%- endfor %} - {{ return(include_cols) }} +{{ return include_cols }} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/sql/get_query_results_as_dict.sql b/macros/sql/get_query_results_as_dict.sql index 6548f2dd..e0136922 100644 --- a/macros/sql/get_query_results_as_dict.sql +++ b/macros/sql/get_query_results_as_dict.sql @@ -1,26 +1,25 @@ {% macro get_query_results_as_dict(query) %} - {{ return(adapter.dispatch('get_query_results_as_dict', 'dbt_utils')(query)) }} +{{ return adapter.dispatch("get_query_results_as_dict", "dbt_utils")(query) }} {% endmacro %} {% macro default__get_query_results_as_dict(query) %} {# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} +{%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} - {%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} +{{ query }} - {{ query }} +{%- endcall -%} - {%- endcall -%} +{% set sql_results = {} %} - {% set sql_results={} %} +{%- if execute -%} +{% set sql_results_table = load_result("get_query_results").table.columns %} +{% for column_name, column in sql_results_table.items() %} +{% do sql_results.update({column_name: column.values()}) %} +{% endfor %} +{%- endif -%} - {%- if execute -%} - {% set sql_results_table = load_result('get_query_results').table.columns %} - {% for column_name, column in sql_results_table.items() %} - {% do sql_results.update({column_name: column.values()}) %} - {% endfor %} - {%- endif -%} - - {{ return(sql_results) }} +{{ return sql_results }} {% endmacro %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 9325a883..749f83f3 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -1,32 +1,42 @@ -{% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_pattern', 'dbt_utils')(schema_pattern, table_pattern, exclude, database)) }} +{% macro get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} +{{ + return adapter.dispatch("get_relations_by_pattern", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) +}} {% endmacro %} -{% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro default__get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} +{%- call statement('get_tables', fetch_result=True) %} - {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} +{{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude, database + ) +}} - {%- endcall -%} +{%- endcall -%} - {%- set table_list = load_result('get_tables') -%} +{%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} - {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} - {%- set tbl_relation = api.Relation.create( - database=database, - schema=row.table_schema, - identifier=row.table_name, - type=row.table_type - ) -%} - {%- do tbl_relations.append(tbl_relation) -%} - {%- endfor -%} +{%- if table_list and table_list["table"] -%} +{%- set tbl_relations = [] -%} +{%- for row in table_list["table"] -%} +{%- set tbl_relation = api.Relation.create( + database=database, + schema=row.table_schema, + identifier=row.table_name, + type=row.table_type, +) -%} {%- do tbl_relations.append(tbl_relation) -%} +{%- endfor -%} - {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} - {%- endif -%} +{{ return tbl_relations }} +{%- else -%} {{ return [] }} +{%- endif -%} {% endmacro %} diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index b6733c4b..286ca672 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -1,32 +1,38 @@ -{% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_prefix', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} +{{ + return adapter.dispatch("get_relations_by_prefix", "dbt_utils")( + schema, prefix, exclude, database + ) +}} {% endmacro %} -{% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} +{%- call statement('get_tables', fetch_result=True) %} - {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} +{{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} - {%- endcall -%} +{%- endcall -%} - {%- set table_list = load_result('get_tables') -%} +{%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} - {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} - {%- set tbl_relation = api.Relation.create( - database=database, - schema=row.table_schema, - identifier=row.table_name, - type=row.table_type - ) -%} - {%- do tbl_relations.append(tbl_relation) -%} - {%- endfor -%} +{%- if table_list and table_list["table"] -%} +{%- set tbl_relations = [] -%} +{%- for row in table_list["table"] -%} +{%- set tbl_relation = api.Relation.create( + database=database, + schema=row.table_schema, + identifier=row.table_name, + type=row.table_type, +) -%} {%- do tbl_relations.append(tbl_relation) -%} +{%- endfor -%} - {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} - {%- endif -%} +{{ return tbl_relations }} +{%- else -%} {{ return [] }} +{%- endif -%} {% endmacro %} diff --git a/macros/sql/get_table_types_sql.sql b/macros/sql/get_table_types_sql.sql index 91573779..fede5af4 100644 --- a/macros/sql/get_table_types_sql.sql +++ b/macros/sql/get_table_types_sql.sql @@ -1,31 +1,43 @@ {%- macro get_table_types_sql() -%} - {{ return(adapter.dispatch('get_table_types_sql', 'dbt_utils')()) }} +{{ return adapter.dispatch("get_table_types_sql", "dbt_utils")() }} {%- endmacro -%} {% macro default__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" +case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) +end as "table_type" {% endmacro %} {% macro postgres__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'FOREIGN' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" +case + table_type + when 'BASE TABLE' + then 'table' + when 'FOREIGN' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) +end as "table_type" {% endmacro %} {% macro bigquery__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as `table_type` -{% endmacro %} \ No newline at end of file +case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) +end as `table_type` +{% endmacro %} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index 4d5a8fc9..72356757 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -1,28 +1,39 @@ -{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_pattern_sql', 'dbt_utils') - (schema_pattern, table_pattern, exclude, database)) }} +{% macro get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} +{{ + return adapter.dispatch("get_tables_by_pattern_sql", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) +}} {% endmacro %} -{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro default__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - select distinct - table_schema as "table_schema", - table_name as "table_name", - {{ dbt_utils.get_table_types_sql() }} - from {{ database }}.information_schema.tables - where table_schema ilike '{{ schema_pattern }}' - and table_name ilike '{{ table_pattern }}' - and table_name not ilike '{{ exclude }}' +select distinct + table_schema as "table_schema", + table_name as "table_name", + {{ dbt_utils.get_table_types_sql() }} +from {{ database }}.information_schema.tables +where + table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' {% endmacro %} -{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + {% macro bigquery__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {% if '%' in schema_pattern %} - {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %} - {% else %} - {% set schemata=[schema_pattern] %} + {% if "%" in schema_pattern %} + {% set schemata = dbt_utils._bigquery__get_matching_schemata( + schema_pattern, database +) %} + {% else %} {% set schemata = [schema_pattern] %} {% endif %} {% set sql %} @@ -41,12 +52,12 @@ {% endfor %} {% endset %} - {{ return(sql) }} + {{ return sql }} -{% endmacro %} + {% endmacro %} -{% macro _bigquery__get_matching_schemata(schema_pattern, database) %} + {% macro _bigquery__get_matching_schemata(schema_pattern, database) %} {% if execute %} {% set sql %} @@ -54,17 +65,15 @@ where lower(schema_name) like lower('{{ schema_pattern }}') {% endset %} - {% set results=run_query(sql) %} - - {% set schemata=results.columns['schema_name'].values() %} + {% set results = run_query(sql) %} - {{ return(schemata) }} + {% set schemata = results.columns["schema_name"].values() %} - {% else %} + {{ return schemata }} - {{ return([]) }} + {% else %} {{ return [] }} {% endif %} -{% endmacro %} + {% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index f8fdfba6..82729388 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,14 +1,22 @@ -{% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_prefix_sql', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} +{{ + return adapter.dispatch("get_tables_by_prefix_sql", "dbt_utils")( + schema, prefix, exclude, database + ) +}} {% endmacro %} -{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} - {{ dbt_utils.get_tables_by_pattern_sql( +{{ dbt_utils.get_tables_by_pattern_sql( schema_pattern = schema, table_pattern = prefix ~ '%', exclude = exclude, database = database ) }} - + {% endmacro %} diff --git a/macros/sql/groupby.sql b/macros/sql/groupby.sql index 68a68cf2..ad6645b5 100644 --- a/macros/sql/groupby.sql +++ b/macros/sql/groupby.sql @@ -1,11 +1,10 @@ {%- macro group_by(n) -%} - {{ return(adapter.dispatch('group_by', 'dbt_utils')(n)) }} +{{ return adapter.dispatch("group_by", "dbt_utils")(n) }} {% endmacro %} {%- macro default__group_by(n) -%} - group by {% for i in range(1, n + 1) -%} - {{ i }}{{ ',' if not loop.last }} - {%- endfor -%} +group by + {% for i in range(1, n + 1) -%} {{ i }}{{ ',' if not loop.last }} {%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 70f276ec..55522a3c 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -5,47 +5,51 @@ http://daynebatten.com/2015/09/latitude-longitude-distance-sql/ The arguments should be float type. #} - -{% macro degrees_to_radians(degrees) -%} - acos(-1) * {{degrees}} / 180 -{%- endmacro %} - -{% macro haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} - {{ return(adapter.dispatch('haversine_distance', 'dbt_utils')(lat1,lon1,lat2,lon2,unit)) }} +{% macro degrees_to_radians(degrees) -%} acos(-1) * {{ degrees }} / 180 {%- endmacro %} + +{% macro haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} +{{ + return adapter.dispatch("haversine_distance", "dbt_utils")( + lat1, lon1, lat2, lon2, unit + ) +}} {% endmacro %} -{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} +{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} +{%- if unit == "mi" %} {% set conversion_rate = 1 %} +{% elif unit == "km" %} {% set conversion_rate = 1.60934 %} {% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} +{{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} {% endif %} - 2 * 3961 * asin(sqrt(power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + - cos(radians({{lat1}})) * cos(radians({{lat2}})) * - power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2))) * {{ conversion_rate }} +2 * 3961 * asin( + sqrt( + power( (sin(radians( ({{ lat2 }} - {{ lat1 }}) / 2))), 2) + + cos(radians({{ lat1 }})) * cos(radians({{ lat2 }})) * + power( (sin(radians( ({{ lon2 }} - {{ lon1 }}) / 2))), 2) + ) +) * {{ conversion_rate }} {%- endmacro %} -{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} +{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} {% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} {% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} {% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} {% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} +{%- if unit == "mi" %} {% set conversion_rate = 1 %} +{% elif unit == "km" %} {% set conversion_rate = 1.60934 %} {% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} +{{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} {% endif %} - 2 * 3961 * asin(sqrt(power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + - cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * - power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2))) * {{ conversion_rate }} +2 * 3961 * asin( + sqrt( + power(sin( ({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + + cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * + power(sin( ({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2) + ) +) * {{ conversion_rate }} {%- endmacro %} - diff --git a/macros/sql/nullcheck.sql b/macros/sql/nullcheck.sql index 509d24f7..b6ed126b 100644 --- a/macros/sql/nullcheck.sql +++ b/macros/sql/nullcheck.sql @@ -1,21 +1,17 @@ {% macro nullcheck(cols) %} - {{ return(adapter.dispatch('nullcheck', 'dbt_utils')(cols)) }} +{{ return adapter.dispatch("nullcheck", "dbt_utils")(cols) }} {% endmacro %} {% macro default__nullcheck(cols) %} {%- for col in cols %} - {% if col.is_string() -%} +{% if col.is_string() -%} nullif({{ col.name }}, '') as {{ col.name }} - nullif({{col.name}},'') as {{col.name}} +{%- else -%} {{ col.name }} - {%- else -%} +{%- endif -%} - {{col.name}} - - {%- endif -%} - -{%- if not loop.last -%} , {%- endif -%} +{%- if not loop.last -%}, {%- endif -%} {%- endfor -%} {% endmacro %} diff --git a/macros/sql/nullcheck_table.sql b/macros/sql/nullcheck_table.sql index c9ab5838..65fa7b81 100644 --- a/macros/sql/nullcheck_table.sql +++ b/macros/sql/nullcheck_table.sql @@ -1,14 +1,14 @@ {% macro nullcheck_table(relation) %} - {{ return(adapter.dispatch('nullcheck_table', 'dbt_utils')(relation)) }} +{{ return adapter.dispatch("nullcheck_table", "dbt_utils")(relation) }} {% endmacro %} {% macro default__nullcheck_table(relation) %} - {%- do dbt_utils._is_relation(relation, 'nullcheck_table') -%} - {%- do dbt_utils._is_ephemeral(relation, 'nullcheck_table') -%} - {% set cols = adapter.get_columns_in_relation(relation) %} +{%- do dbt_utils._is_relation(relation, "nullcheck_table") -%} +{%- do dbt_utils._is_ephemeral(relation, "nullcheck_table") -%} +{% set cols = adapter.get_columns_in_relation(relation) %} - select {{ dbt_utils.nullcheck(cols) }} - from {{relation}} +select {{ dbt_utils.nullcheck(cols) }} +from {{ relation }} {% endmacro %} diff --git a/macros/sql/pivot.sql b/macros/sql/pivot.sql index 88751062..c15b06e3 100644 --- a/macros/sql/pivot.sql +++ b/macros/sql/pivot.sql @@ -39,48 +39,63 @@ Arguments: quote_identifiers: Whether to surround column aliases with double quotes, default is true distinct: Whether to use distinct in the aggregation, default is False #} - -{% macro pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {{ return(adapter.dispatch('pivot', 'dbt_utils')(column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers, distinct)) }} +{% macro pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False +) %} +{{ + return adapter.dispatch("pivot", "dbt_utils")( + column, + values, + alias, + agg, + cmp, + prefix, + suffix, + then_value, + else_value, + quote_identifiers, + distinct, + ) +}} {% endmacro %} -{% macro default__pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {% for value in values %} - {{ agg }}( - {% if distinct %} distinct {% endif %} - case - when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' +{% macro default__pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False +) %} +{% for value in values %} +{{ agg }} ( + {% if distinct %}distinct {% endif %} + case + when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' then {{ then_value }} - else {{ else_value }} - end - ) - {% if alias %} - {% if quote_identifiers %} - as {{ adapter.quote(prefix ~ value ~ suffix) }} - {% else %} - as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} - {% endif %} - {% endif %} - {% if not loop.last %},{% endif %} - {% endfor %} + else {{ else_value }} + end +) +{% if alias %} +{% if quote_identifiers %} as {{ adapter.quote(prefix ~ value ~ suffix) }} +{% else %} as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} +{% endif %} +{% endif %} +{% if not loop.last %},{% endif %} +{% endfor %} {% endmacro %} diff --git a/macros/sql/safe_add.sql b/macros/sql/safe_add.sql index 3b6195c8..d75f9b26 100644 --- a/macros/sql/safe_add.sql +++ b/macros/sql/safe_add.sql @@ -1,8 +1,8 @@ {%- macro safe_add() -%} - {# needed for safe_add to allow for non-keyword arguments see SO post #} - {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} - {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('safe_add', 'dbt_utils')(*varargs)) }} +{# needed for safe_add to allow for non-keyword arguments see SO post #} +{# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} +{% set frustrating_jinja_feature = varargs %} +{{ return adapter.dispatch("safe_add", "dbt_utils")(*varargs) }} {% endmacro %} {%- macro default__safe_add() -%} @@ -11,10 +11,10 @@ {%- for field in varargs -%} - {% do fields.append("coalesce(" ~ field ~ ", 0)") %} +{% do fields.append("coalesce(" ~ field ~ ", 0)") %} {%- endfor -%} -{{ fields|join(' +\n ') }} +{{ fields | join(" +\n ") }} {%- endmacro -%} diff --git a/macros/sql/star.sql b/macros/sql/star.sql index 25e58969..9a65b80e 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -1,20 +1,23 @@ {% macro star(from, relation_alias=False, except=[], prefix='', suffix='') -%} - {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} +{{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} {% endmacro %} {% macro default__star(from, relation_alias=False, except=[], prefix='', suffix='') -%} - {%- do dbt_utils._is_relation(from, 'star') -%} - {%- do dbt_utils._is_ephemeral(from, 'star') -%} +{%- do dbt_utils._is_relation(from, 'star') -%} +{%- do dbt_utils._is_ephemeral(from, 'star') -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('*') }} - {% endif %} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return "*" }} {% endif %} - {%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} +{%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} - {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} {%- if prefix!='' or suffix!='' %} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- endif -%} - {%- if not loop.last %},{{ '\n ' }}{% endif %} +{%- if relation_alias %} {{ relation_alias }}. +{% else %} +{%- endif -%} {{ adapter.quote(col) | trim }} +{%- if prefix != "" or suffix != "" %} +as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} +{%- endif -%} +{%- if not loop.last %},{{ "\n " }}{% endif %} - {%- endfor -%} +{%- endfor -%} {%- endmacro %} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index ba5c3e44..199cbd42 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -1,33 +1,27 @@ {%- macro surrogate_key(field_list) -%} - {# needed for safe_add to allow for non-keyword arguments see SO post #} - {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} - {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('surrogate_key', 'dbt_utils')(field_list, *varargs)) }} +{# needed for safe_add to allow for non-keyword arguments see SO post #} +{# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} +{% set frustrating_jinja_feature = varargs %} +{{ return adapter.dispatch("surrogate_key", "dbt_utils")(field_list, *varargs) }} {% endmacro %} {%- macro default__surrogate_key(field_list) -%} -{%- if varargs|length >= 1 or field_list is string %} +{%- if varargs | length >= 1 or field_list is string %} -{%- set error_message = ' -Warning: the `surrogate_key` macro now takes a single list argument instead of \ -multiple string arguments. Support for multiple string arguments will be \ -deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ -'.format(model.package_name, model.name) -%} +{%- set error_message = " Warning: the `surrogate_key` macro now takes a single list argument instead of \ multiple string arguments. Support for multiple string arguments will be \ deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name +) -%} {%- do exceptions.warn(error_message) -%} {# first argument is not included in varargs, so add first element to field_list_xf #} {%- set field_list_xf = [field_list] -%} -{%- for field in varargs %} -{%- set _ = field_list_xf.append(field) -%} -{%- endfor -%} - -{%- else -%} +{%- for field in varargs %} {%- set _ = field_list_xf.append(field) -%} {%- endfor -%} {# if using list, just set field_list_xf as field_list #} -{%- set field_list_xf = field_list -%} +{%- else -%} {%- set field_list_xf = field_list -%} {%- endif -%} @@ -36,16 +30,12 @@ deprecated in a future release of dbt-utils. The {}.{} model triggered this warn {%- for field in field_list_xf -%} - {%- set _ = fields.append( +{%- set _ = fields.append( "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" - ) -%} - - {%- if not loop.last %} - {%- set _ = fields.append("'-'") -%} - {%- endif -%} + ) -%} {%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} {%- endfor -%} -{{dbt_utils.hash(dbt_utils.concat(fields))}} +{{ dbt_utils.hash(dbt_utils.concat(fields)) }} {%- endmacro -%} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index a7bf1d95..227acb55 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -1,67 +1,83 @@ -{%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} - {{ return(adapter.dispatch('union_relations', 'dbt_utils')(relations, column_override, include, exclude, source_column_name)) }} +{%- macro union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation" +) -%} +{{ + return adapter.dispatch("union_relations", "dbt_utils")( + relations, column_override, include, exclude, source_column_name + ) +}} {% endmacro %} -{%- macro default__union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} +{%- macro default__union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation" +) -%} - {%- if exclude and include -%} - {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} - {%- endif -%} +{%- if exclude and include -%} +{{ + exceptions.raise_compiler_error( + "Both an exclude and include list were provided to the `union` macro. Only one is allowed" + ) +}} +{%- endif -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} - {%- if not execute %} - {{ return('') }} - {% endif -%} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} +{%- if not execute %} {{ return "" }} {% endif -%} - {%- set column_override = column_override if column_override is not none else {} -%} +{%- set column_override = column_override if column_override is not none else {} -%} - {%- set relation_columns = {} -%} - {%- set column_superset = {} -%} +{%- set relation_columns = {} -%} +{%- set column_superset = {} -%} - {%- for relation in relations -%} +{%- for relation in relations -%} - {%- do relation_columns.update({relation: []}) -%} +{%- do relation_columns.update({relation: []}) -%} - {%- do dbt_utils._is_relation(relation, 'union_relations') -%} - {%- do dbt_utils._is_ephemeral(relation, 'union_relations') -%} - {%- set cols = adapter.get_columns_in_relation(relation) -%} - {%- for col in cols -%} +{%- do dbt_utils._is_relation(relation, "union_relations") -%} +{%- do dbt_utils._is_ephemeral(relation, "union_relations") -%} +{%- set cols = adapter.get_columns_in_relation(relation) -%} +{%- for col in cols -%} - {#- If an exclude list was provided and the column is in the list, do nothing -#} - {%- if exclude and col.column in exclude -%} +{#- If an exclude list was provided and the column is in the list, do nothing -#} +{%- if exclude and col.column in exclude -%} - {#- If an include list was provided and the column is not in the list, do nothing -#} - {%- elif include and col.column not in include -%} +{#- If an include list was provided and the column is not in the list, do nothing -#} +{%- elif include and col.column not in include -%} - {#- Otherwise add the column to the column superset -#} - {%- else -%} +{#- Otherwise add the column to the column superset -#} +{%- else -%} - {#- update the list of columns in this relation -#} - {%- do relation_columns[relation].append(col.column) -%} +{#- update the list of columns in this relation -#} +{%- do relation_columns[relation].append(col.column) -%} - {%- if col.column in column_superset -%} +{%- if col.column in column_superset -%} - {%- set stored = column_superset[col.column] -%} - {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} +{%- set stored = column_superset[col.column] -%} +{%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} - {%- do column_superset.update({col.column: col}) -%} +{%- do column_superset.update({col.column: col}) -%} - {%- endif %} +{%- endif %} - {%- else -%} +{%- else -%} {%- do column_superset.update({col.column: col}) -%} - {%- do column_superset.update({col.column: col}) -%} +{%- endif -%} - {%- endif -%} +{%- endif -%} - {%- endif -%} +{%- endfor -%} +{%- endfor -%} - {%- endfor -%} - {%- endfor -%} +{%- set ordered_column_names = column_superset.keys() -%} - {%- set ordered_column_names = column_superset.keys() -%} - - {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} +{% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} {%- set relations_string -%} {%- for relation in relations -%} {{ relation.name }} @@ -73,31 +89,36 @@ There were no columns found to union for relations {{ relations_string }} {%- endset -%} - {{ exceptions.raise_compiler_error(error_message) }} - {%- endif -%} +{{ exceptions.raise_compiler_error(error_message) }} +{%- endif -%} - {%- for relation in relations %} +{%- for relation in relations %} - ( - select +( + select - cast({{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }}) as {{ source_column_name }}, - {% for col_name in ordered_column_names -%} + cast( + {{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }} + ) as {{ source_column_name }}, + {% for col_name in ordered_column_names -%} - {%- set col = column_superset[col_name] %} - {%- set col_type = column_override.get(col.column, col.data_type) %} - {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %} - cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%} + {%- set col = column_superset[col_name] %} + {%- set col_type = column_override.get(col.column, col.data_type) %} + {%- set col_name = ( + adapter.quote(col_name) + if col_name in relation_columns[relation] + else "null" +) %} + cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} + {% if not loop.last %},{% endif -%} - {%- endfor %} + {%- endfor %} - from {{ relation }} - ) + from {{ relation }} +) - {% if not loop.last -%} - union all - {% endif -%} +{% if not loop.last -%} union all {% endif -%} - {%- endfor -%} +{%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index 8821aad2..1c606815 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -11,69 +11,90 @@ Arguments: field_name: Destination table column name for the source table column names. value_name: Destination table column name for the pivoted values #} - -{% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} - {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name, table)) }} +{% macro unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none +) -%} +{{ + return adapter.dispatch("unpivot", "dbt_utils")( + relation, cast_to, exclude, remove, field_name, value_name, table + ) +}} {% endmacro %} -{% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} - - {% if table %} - {%- set error_message = ' - Warning: the `unpivot` macro no longer accepts a `table` parameter. \ - This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ - The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - {% endif %} - - {% if relation and table %} - {{ exceptions.raise_compiler_error("Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`).") }} - {% elif not relation and table %} - {% set relation=table %} - {% elif not relation and not table %} - {{ exceptions.raise_compiler_error("Error: argument `relation` is required for `unpivot` macro.") }} - {% endif %} - - {%- set exclude = exclude if exclude is not none else [] %} - {%- set remove = remove if remove is not none else [] %} - - {%- set include_cols = [] %} - - {%- set table_columns = {} %} - - {%- do table_columns.update({relation: []}) %} - - {%- do dbt_utils._is_relation(relation, 'unpivot') -%} - {%- do dbt_utils._is_ephemeral(relation, 'unpivot') -%} - {%- set cols = adapter.get_columns_in_relation(relation) %} - - {%- for col in cols -%} - {%- if col.column.lower() not in remove|map('lower') and col.column.lower() not in exclude|map('lower') -%} - {% do include_cols.append(col) %} - {%- endif %} - {%- endfor %} - - - {%- for col in include_cols -%} - select - {%- for exclude_col in exclude %} - {{ exclude_col }}, - {%- endfor %} - - cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, - cast( {% if col.data_type == 'boolean' %} - {{ dbt_utils.cast_bool_to_text(col.column) }} - {% else %} - {{ col.column }} - {% endif %} - as {{ cast_to }}) as {{ value_name }} - - from {{ relation }} - - {% if not loop.last -%} - union all - {% endif -%} - {%- endfor -%} +{% macro default__unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none +) -%} + +{% if table %} +{%- set error_message = " Warning: the `unpivot` macro no longer accepts a `table` parameter. \ This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name +) -%} +{%- do exceptions.warn(error_message) -%} +{% endif %} + +{% if relation and table %} +{{ + exceptions.raise_compiler_error( + "Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`)." + ) +}} +{% elif not relation and table %} {% set relation = table %} +{% elif not relation and not table %} +{{ + exceptions.raise_compiler_error( + "Error: argument `relation` is required for `unpivot` macro." + ) +}} +{% endif %} + +{%- set exclude = exclude if exclude is not none else [] %} +{%- set remove = remove if remove is not none else [] %} + +{%- set include_cols = [] %} + +{%- set table_columns = {} %} + +{%- do table_columns.update({relation: []}) %} + +{%- do dbt_utils._is_relation(relation, "unpivot") -%} +{%- do dbt_utils._is_ephemeral(relation, "unpivot") -%} +{%- set cols = adapter.get_columns_in_relation(relation) %} + +{%- for col in cols -%} +{%- if col.column.lower() not in remove | map( + "lower" +) and col.column.lower() not in exclude | map("lower") -%} +{% do include_cols.append(col) %} +{%- endif %} +{%- endfor %} + + +{%- for col in include_cols -%} +select + {%- for exclude_col in exclude %} {{ exclude_col }}, {%- endfor %} + + cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, + cast( + {% if col.data_type == "boolean" %}{{ dbt_utils.cast_bool_to_text(col.column) }} + {% else %}{{ col.column }} + {% endif %} as {{ cast_to }} + ) as {{ value_name }} + +from {{ relation }} + +{% if not loop.last -%} union all {% endif -%} +{%- endfor -%} {%- endmacro %} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index 43a58cc4..5320d5bb 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -1,27 +1,25 @@ {% macro get_url_host(field) -%} - {{ return(adapter.dispatch('get_url_host', 'dbt_utils')(field)) }} +{{ return adapter.dispatch("get_url_host", "dbt_utils")(field) }} {% endmacro %} {% macro default__get_url_host(field) -%} -{%- set parsed = +{%- set parsed = dbt_utils.split_part( dbt_utils.split_part( - dbt_utils.split_part( + dbt_utils.replace( dbt_utils.replace( - dbt_utils.replace( - dbt_utils.replace(field, "'android-app://'", "''" - ), "'http://'", "''" - ), "'https://'", "''" - ), "'/'", 1 - ), "'?'", 1 - ) - --%} - - - {{ dbt_utils.safe_cast( - parsed, - dbt_utils.type_string() - )}} + dbt_utils.replace(field, "'android-app://'", "''"), + "'http://'", + "''", + ), + "'https://'", + "''", + ), + "'/'", + 1, + ), + "'?'", + 1, +) -%} {{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} {%- endmacro %} diff --git a/macros/web/get_url_parameter.sql b/macros/web/get_url_parameter.sql index f363fcbb..ab183b17 100644 --- a/macros/web/get_url_parameter.sql +++ b/macros/web/get_url_parameter.sql @@ -1,13 +1,15 @@ {% macro get_url_parameter(field, url_parameter) -%} - {{ return(adapter.dispatch('get_url_parameter', 'dbt_utils')(field, url_parameter)) }} +{{ return adapter.dispatch("get_url_parameter", "dbt_utils")(field, url_parameter) }} {% endmacro %} {% macro default__get_url_parameter(field, url_parameter) -%} {%- set formatted_url_parameter = "'" + url_parameter + "='" -%} -{%- set split = dbt_utils.split_part(dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1) -%} +{%- set split = dbt_utils.split_part( + dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1 +) -%} -nullif({{ split }},'') +nullif({{ split }}, '') {%- endmacro %} diff --git a/macros/web/get_url_path.sql b/macros/web/get_url_path.sql index 4d224277..0548fd23 100644 --- a/macros/web/get_url_path.sql +++ b/macros/web/get_url_path.sql @@ -1,13 +1,12 @@ {% macro get_url_path(field) -%} - {{ return(adapter.dispatch('get_url_path', 'dbt_utils')(field)) }} +{{ return adapter.dispatch("get_url_path", "dbt_utils")(field) }} {% endmacro %} {% macro default__get_url_path(field) -%} - {%- set stripped_url = - dbt_utils.replace( - dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''") - -%} +{%- set stripped_url = dbt_utils.replace( + dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''" +) -%} {%- set first_slash_pos -%} coalesce( @@ -16,19 +15,15 @@ ) {%- endset -%} - {%- set parsed_path = +{%- set parsed_path = dbt_utils.split_part( dbt_utils.right( stripped_url, dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos ), "'?'", 1 - ) - -%} + ) -%} + +{{ dbt_utils.safe_cast(parsed_path, dbt_utils.type_string()) }} - {{ dbt_utils.safe_cast( - parsed_path, - dbt_utils.type_string() - )}} - {%- endmacro %} From f6671f5ea588ea8580401dd3c57f980368effd9d Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Sat, 14 May 2022 12:40:37 -0600 Subject: [PATCH 04/22] chore: apply sqlfmt 138e98e --- .../macros/assert_equal_values.sql | 13 +- integration_tests/macros/limit_zero.sql | 10 +- .../models/cross_db_utils/test_any_value.sql | 41 +- .../models/cross_db_utils/test_bool_or.sql | 8 +- .../models/cross_db_utils/test_concat.sql | 10 +- .../cross_db_utils/test_current_timestamp.sql | 1 - .../test_current_timestamp_in_utc.sql | 2 +- .../models/cross_db_utils/test_date_trunc.sql | 10 +- .../models/cross_db_utils/test_dateadd.sql | 34 +- .../models/cross_db_utils/test_datediff.sql | 112 ++++-- .../models/cross_db_utils/test_hash.sql | 10 +- .../models/cross_db_utils/test_last_day.sql | 15 +- .../models/cross_db_utils/test_length.sql | 13 +- .../models/cross_db_utils/test_listagg.sql | 130 +++--- .../models/cross_db_utils/test_position.sql | 10 +- .../models/cross_db_utils/test_replace.sql | 21 +- .../models/cross_db_utils/test_right.sql | 10 +- .../models/cross_db_utils/test_safe_cast.sql | 8 +- .../models/cross_db_utils/test_split_part.sql | 15 +- .../cross_db_utils/test_width_bucket.sql | 9 +- .../models/datetime/test_date_spine.sql | 22 +- .../test_equal_column_subset.sql | 10 +- .../generic_tests/test_equal_rowcount.sql | 11 +- .../generic_tests/test_fewer_rows_than.sql | 11 +- .../models/generic_tests/test_recency.sql | 11 +- .../models/geo/test_haversine_distance_km.sql | 39 +- .../models/geo/test_haversine_distance_mi.sql | 65 ++- .../expected_insert_by_period.sql | 9 +- .../test_insert_by_period.sql | 26 +- .../models/sql/test_deduplicate.sql | 29 +- .../models/sql/test_generate_series.sql | 12 +- .../models/sql/test_get_column_values.sql | 19 +- .../test_get_filtered_columns_in_relation.sql | 25 +- .../sql/test_get_relations_by_pattern.sql | 13 +- ...test_get_relations_by_prefix_and_union.sql | 4 +- integration_tests/models/sql/test_groupby.sql | 28 +- .../models/sql/test_nullcheck_table.sql | 27 +- integration_tests/models/sql/test_pivot.sql | 16 +- .../models/sql/test_pivot_apostrophe.sql | 14 +- .../models/sql/test_safe_add.sql | 10 +- integration_tests/models/sql/test_star.sql | 15 +- .../models/sql/test_star_aggregate.sql | 23 +- .../models/sql/test_star_prefix_suffix.sql | 22 +- .../models/sql/test_star_uppercase.sql | 15 +- .../models/sql/test_surrogate_key.sql | 14 +- integration_tests/models/sql/test_union.sql | 8 +- .../models/sql/test_union_base.sql | 5 +- integration_tests/models/sql/test_unpivot.sql | 35 +- .../models/sql/test_unpivot_bool.sql | 33 +- .../models/sql/test_unpivot_original_api.sql | 28 +- .../models/web/test_url_host.sql | 13 +- .../models/web/test_url_path.sql | 12 +- integration_tests/models/web/test_urls.sql | 12 +- ...et_query_results_as_dict_objects_equal.sql | 57 ++- .../assert_pretty_output_msg_is_string.sql | 10 +- .../assert_pretty_time_is_string.sql | 10 +- .../tests/jinja_helpers/test_slugify.sql | 11 +- .../test_get_column_values_use_default.sql | 50 ++- macros/cross_db_utils/_is_ephemeral.sql | 17 +- macros/cross_db_utils/_is_relation.sql | 9 +- macros/cross_db_utils/any_value.sql | 15 +- macros/cross_db_utils/bool_or.sql | 20 +- macros/cross_db_utils/cast_bool_to_text.sql | 11 +- macros/cross_db_utils/concat.sql | 6 +- macros/cross_db_utils/current_timestamp.sql | 24 +- macros/cross_db_utils/datatypes.sql | 78 +--- macros/cross_db_utils/date_trunc.sql | 9 +- macros/cross_db_utils/dateadd.sql | 26 +- macros/cross_db_utils/datediff.sql | 119 ++++-- .../cross_db_utils/escape_single_quotes.sql | 4 +- macros/cross_db_utils/except.sql | 14 +- macros/cross_db_utils/hash.sql | 6 +- macros/cross_db_utils/identifier.sql | 23 +- macros/cross_db_utils/intersect.sql | 14 +- macros/cross_db_utils/last_day.sql | 44 +- macros/cross_db_utils/length.sql | 18 +- macros/cross_db_utils/listagg.sql | 142 +++---- macros/cross_db_utils/literal.sql | 6 +- macros/cross_db_utils/position.sql | 16 +- macros/cross_db_utils/replace.sql | 12 +- macros/cross_db_utils/right.sql | 37 +- macros/cross_db_utils/safe_cast.sql | 10 +- macros/cross_db_utils/split_part.sql | 19 +- macros/cross_db_utils/width_bucket.sql | 75 ++-- macros/generic_tests/accepted_range.sql | 55 ++- macros/generic_tests/at_least_one.sql | 14 +- macros/generic_tests/cardinality_equality.sql | 62 +-- macros/generic_tests/equal_rowcount.sql | 41 +- macros/generic_tests/equality.sql | 79 ++-- macros/generic_tests/expression_is_true.sql | 23 +- macros/generic_tests/fewer_rows_than.sql | 70 ++-- .../mutually_exclusive_ranges.sql | 180 +++++---- macros/generic_tests/not_accepted_values.sql | 46 +-- macros/generic_tests/not_constant.sql | 9 +- macros/generic_tests/not_null_proportion.sql | 36 +- macros/generic_tests/recency.sql | 22 +- macros/generic_tests/relationships_where.sql | 60 +-- macros/generic_tests/sequential_values.sql | 58 ++- macros/generic_tests/test_not_null_where.sql | 21 +- macros/generic_tests/test_unique_where.sql | 21 +- .../unique_combination_of_columns.sql | 45 ++- macros/jinja_helpers/log_info.sql | 4 +- macros/jinja_helpers/pretty_log_format.sql | 4 +- macros/jinja_helpers/pretty_time.sql | 8 +- macros/jinja_helpers/slugify.sql | 4 +- .../insert_by_period_materialization.sql | 377 ++++++++++-------- macros/sql/date_spine.sql | 79 ++-- macros/sql/deduplicate.sql | 42 +- macros/sql/generate_series.sql | 53 ++- macros/sql/get_column_values.sql | 78 ++-- .../sql/get_filtered_columns_in_relation.sql | 32 +- macros/sql/get_query_results_as_dict.sql | 25 +- macros/sql/get_relations_by_pattern.sql | 56 ++- macros/sql/get_relations_by_prefix.sql | 52 ++- macros/sql/get_table_types_sql.sql | 52 ++- macros/sql/get_tables_by_pattern_sql.sql | 63 +-- macros/sql/get_tables_by_prefix_sql.sql | 20 +- macros/sql/groupby.sql | 7 +- macros/sql/haversine_distance.sql | 58 +-- macros/sql/nullcheck.sql | 14 +- macros/sql/nullcheck_table.sql | 12 +- macros/sql/pivot.sql | 99 +++-- macros/sql/safe_add.sql | 12 +- macros/sql/star.sql | 25 +- macros/sql/surrogate_key.sql | 36 +- macros/sql/union.sql | 141 ++++--- macros/sql/unpivot.sql | 147 ++++--- macros/web/get_url_host.sql | 34 +- macros/web/get_url_parameter.sql | 8 +- macros/web/get_url_path.sql | 21 +- 130 files changed, 2239 insertions(+), 2196 deletions(-) diff --git a/integration_tests/macros/assert_equal_values.sql b/integration_tests/macros/assert_equal_values.sql index d4f02618..48c0ff81 100644 --- a/integration_tests/macros/assert_equal_values.sql +++ b/integration_tests/macros/assert_equal_values.sql @@ -1,8 +1,7 @@ {% macro assert_equal_values(actual_object, expected_object) %} {% if not execute %} - {# pass #} - +{# pass #} {% elif actual_object != expected_object %} {% set msg %} @@ -20,13 +19,11 @@ {% endset %} - {{ log(msg, info=True) }} - - select 'fail' +{{ log(msg, info=True) }} -{% else %} +select 'fail' - select 'ok' {{ limit_zero() }} +{% else %} select 'ok' {{ limit_zero() }} {% endif %} -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/integration_tests/macros/limit_zero.sql b/integration_tests/macros/limit_zero.sql index 697849d1..0752e234 100644 --- a/integration_tests/macros/limit_zero.sql +++ b/integration_tests/macros/limit_zero.sql @@ -1,11 +1,7 @@ -{% macro my_custom_macro() %} - whatever -{% endmacro %} +{% macro my_custom_macro() %} whatever {% endmacro %} {% macro limit_zero() %} - {{ return(adapter.dispatch('limit_zero', 'dbt_utils')()) }} +{{ return(adapter.dispatch("limit_zero", "dbt_utils")()) }} {% endmacro %} -{% macro default__limit_zero() %} - {{ return('limit 0') }} -{% endmacro %} \ No newline at end of file +{% macro default__limit_zero() %} {{ return("limit 0") }} {% endmacro %} diff --git a/integration_tests/models/cross_db_utils/test_any_value.sql b/integration_tests/models/cross_db_utils/test_any_value.sql index 9b27ed1d..6781b41f 100644 --- a/integration_tests/models/cross_db_utils/test_any_value.sql +++ b/integration_tests/models/cross_db_utils/test_any_value.sql @@ -1,19 +1,26 @@ -with some_model as ( - select 1 as id, 'abc' as key_name, 'dbt' as static_col union all - select 2 as id, 'abc' as key_name, 'dbt' as static_col union all - select 3 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 4 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 5 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 6 as id, 'xyz' as key_name, 'test' as static_col -), +with + some_model as ( + select 1 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 2 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 3 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 4 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 5 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 6 as id, 'xyz' as key_name, 'test' as static_col + ), -final as ( - select - key_name, - {{ dbt_utils.any_value('static_col') }} as static_col, - count(id) as num_rows - from some_model - group by key_name -) + final as ( + select + key_name, + {{ dbt_utils.any_value("static_col") }} as static_col, + count(id) as num_rows + from some_model + group by key_name + ) -select * from final \ No newline at end of file +select * +from final diff --git a/integration_tests/models/cross_db_utils/test_bool_or.sql b/integration_tests/models/cross_db_utils/test_bool_or.sql index 7375d1e4..1d6f4ad5 100644 --- a/integration_tests/models/cross_db_utils/test_bool_or.sql +++ b/integration_tests/models/cross_db_utils/test_bool_or.sql @@ -1,5 +1,3 @@ -select - key, - {{ dbt_utils.bool_or('val1 = val2') }} as value -from {{ ref('data_bool_or' )}} -group by key \ No newline at end of file +select key, {{ dbt_utils.bool_or("val1 = val2") }} as value +from {{ ref("data_bool_or") }} +group by key diff --git a/integration_tests/models/cross_db_utils/test_concat.sql b/integration_tests/models/cross_db_utils/test_concat.sql index 9efeb748..8c036c65 100644 --- a/integration_tests/models/cross_db_utils/test_concat.sql +++ b/integration_tests/models/cross_db_utils/test_concat.sql @@ -1,12 +1,6 @@ -with data as ( +with data as (select * from {{ ref("data_concat") }}) - select * from {{ ref('data_concat') }} - -) - -select - {{ dbt_utils.concat(['input_1', 'input_2']) }} as actual, - output as expected +select {{ dbt_utils.concat(["input_1", "input_2"]) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp.sql b/integration_tests/models/cross_db_utils/test_current_timestamp.sql index 9a777f28..d6c2cf73 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp.sql @@ -3,4 +3,3 @@ select {{ dbt_utils.current_timestamp() }} as actual, {{ dbt_utils.current_timestamp() }} as expected - diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql index 55bc8e23..e88e934a 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql @@ -2,4 +2,4 @@ -- how can we test this better? select {{ dbt_utils.current_timestamp_in_utc() }} as actual, - {{ dbt_utils.current_timestamp_in_utc() }} as expected \ No newline at end of file + {{ dbt_utils.current_timestamp_in_utc() }} as expected diff --git a/integration_tests/models/cross_db_utils/test_date_trunc.sql b/integration_tests/models/cross_db_utils/test_date_trunc.sql index bac21fed..a5b88aa3 100644 --- a/integration_tests/models/cross_db_utils/test_date_trunc.sql +++ b/integration_tests/models/cross_db_utils/test_date_trunc.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_date_trunc') }} - -) +with data as (select * from {{ ref("data_date_trunc") }}) select - cast({{dbt_utils.date_trunc('day', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("day", "updated_at") }} as date) as actual, day as expected from data @@ -14,7 +10,7 @@ from data union all select - cast({{ dbt_utils.date_trunc('month', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("month", "updated_at") }} as date) as actual, month as expected from data diff --git a/integration_tests/models/cross_db_utils/test_dateadd.sql b/integration_tests/models/cross_db_utils/test_dateadd.sql index 3e199d86..2f2dee99 100644 --- a/integration_tests/models/cross_db_utils/test_dateadd.sql +++ b/integration_tests/models/cross_db_utils/test_dateadd.sql @@ -1,16 +1,32 @@ -with data as ( - - select * from {{ ref('data_dateadd') }} - -) +with data as (select * from {{ ref("data_dateadd") }}) select case - when datepart = 'hour' then cast({{ dbt_utils.dateadd('hour', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'day' then cast({{ dbt_utils.dateadd('day', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'month' then cast({{ dbt_utils.dateadd('month', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'year' then cast({{ dbt_utils.dateadd('year', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) + when datepart = 'hour' + then + cast( + {{ dbt_utils.dateadd("hour", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'day' + then + cast( + {{ dbt_utils.dateadd("day", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'month' + then + cast( + {{ dbt_utils.dateadd("month", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'year' + then + cast( + {{ dbt_utils.dateadd("year", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_datediff.sql b/integration_tests/models/cross_db_utils/test_datediff.sql index a2340456..4b6c95a6 100644 --- a/integration_tests/models/cross_db_utils/test_datediff.sql +++ b/integration_tests/models/cross_db_utils/test_datediff.sql @@ -1,20 +1,23 @@ -with data as ( - - select * from {{ ref('data_datediff') }} - -) +with data as (select * from {{ ref("data_datediff") }}) select case - when datepart = 'second' then {{ dbt_utils.datediff('first_date', 'second_date', 'second') }} - when datepart = 'minute' then {{ dbt_utils.datediff('first_date', 'second_date', 'minute') }} - when datepart = 'hour' then {{ dbt_utils.datediff('first_date', 'second_date', 'hour') }} - when datepart = 'day' then {{ dbt_utils.datediff('first_date', 'second_date', 'day') }} - when datepart = 'week' then {{ dbt_utils.datediff('first_date', 'second_date', 'week') }} - when datepart = 'month' then {{ dbt_utils.datediff('first_date', 'second_date', 'month') }} - when datepart = 'year' then {{ dbt_utils.datediff('first_date', 'second_date', 'year') }} + when datepart = 'second' + then {{ dbt_utils.datediff("first_date", "second_date", "second") }} + when datepart = 'minute' + then {{ dbt_utils.datediff("first_date", "second_date", "minute") }} + when datepart = 'hour' + then {{ dbt_utils.datediff("first_date", "second_date", "hour") }} + when datepart = 'day' + then {{ dbt_utils.datediff("first_date", "second_date", "day") }} + when datepart = 'week' + then {{ dbt_utils.datediff("first_date", "second_date", "week") }} + when datepart = 'month' + then {{ dbt_utils.datediff("first_date", "second_date", "month") }} + when datepart = 'year' + then {{ dbt_utils.datediff("first_date", "second_date", "year") }} else null end as actual, result as expected @@ -22,14 +25,77 @@ select from data -- Also test correct casting of literal values. - -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "microsecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "millisecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "microsecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "millisecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year" + ) + }} as actual, 1 as expected diff --git a/integration_tests/models/cross_db_utils/test_hash.sql b/integration_tests/models/cross_db_utils/test_hash.sql index 8f017780..6d526274 100644 --- a/integration_tests/models/cross_db_utils/test_hash.sql +++ b/integration_tests/models/cross_db_utils/test_hash.sql @@ -1,12 +1,6 @@ -with data as ( +with data as (select * from {{ ref("data_hash") }}) - select * from {{ ref('data_hash') }} - -) - -select - {{ dbt_utils.hash('input_1') }} as actual, - output as expected +select {{ dbt_utils.hash("input_1") }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_last_day.sql b/integration_tests/models/cross_db_utils/test_last_day.sql index fee3a823..4e6cec97 100644 --- a/integration_tests/models/cross_db_utils/test_last_day.sql +++ b/integration_tests/models/cross_db_utils/test_last_day.sql @@ -1,15 +1,14 @@ -with data as ( - - select * from {{ ref('data_last_day') }} - -) +with data as (select * from {{ ref("data_last_day") }}) select case - when date_part = 'month' then {{ dbt_utils.last_day('date_day', 'month') }} - when date_part = 'quarter' then {{ dbt_utils.last_day('date_day', 'quarter') }} - when date_part = 'year' then {{ dbt_utils.last_day('date_day', 'year') }} + when date_part = 'month' + then {{ dbt_utils.last_day("date_day", "month") }} + when date_part = 'quarter' + then {{ dbt_utils.last_day("date_day", "quarter") }} + when date_part = 'year' + then {{ dbt_utils.last_day("date_day", "year") }} else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_length.sql b/integration_tests/models/cross_db_utils/test_length.sql index f4ef099c..d6352d05 100644 --- a/integration_tests/models/cross_db_utils/test_length.sql +++ b/integration_tests/models/cross_db_utils/test_length.sql @@ -1,12 +1,5 @@ -with data as ( +with data as (select * from {{ ref("data_length") }}) - select * from {{ ref('data_length') }} +select {{ dbt_utils.length("expression") }} as actual, output as expected -) - -select - - {{ dbt_utils.length('expression') }} as actual, - output as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_listagg.sql b/integration_tests/models/cross_db_utils/test_listagg.sql index 006948de..27d96745 100644 --- a/integration_tests/models/cross_db_utils/test_listagg.sql +++ b/integration_tests/models/cross_db_utils/test_listagg.sql @@ -1,69 +1,63 @@ -with data as ( - - select * from {{ ref('data_listagg') }} - -), - -data_output as ( - - select * from {{ ref('data_listagg_output') }} - -), - -calculate as ( - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col") }} as actual, - 'bottom_ordered' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col", 2) }} as actual, - 'bottom_ordered_limited' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "', '") }} as actual, - 'comma_whitespace_unordered' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('DISTINCT string_text', "','") }} as actual, - 'distinct_comma' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text') }} as actual, - 'no_params' as version - from data - where group_col = 3 - group by group_col - -) - -select - calculate.actual, - data_output.expected +with + data as (select * from {{ ref("data_listagg") }}), + + data_output as (select * from {{ ref("data_listagg_output") }}), + + calculate as ( + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col") }} + as actual, + 'bottom_ordered' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col", 2) }} + as actual, + 'bottom_ordered_limited' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "', '") }} as actual, + 'comma_whitespace_unordered' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("DISTINCT string_text", "','") }} as actual, + 'distinct_comma' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text") }} as actual, + 'no_params' as version + from data + where group_col = 3 + group by group_col + + ) + +select calculate.actual, data_output.expected from calculate -left join data_output -on calculate.group_col = data_output.group_col -and calculate.version = data_output.version \ No newline at end of file +left join + data_output + on calculate.group_col = data_output.group_col + and calculate.version = data_output.version diff --git a/integration_tests/models/cross_db_utils/test_position.sql b/integration_tests/models/cross_db_utils/test_position.sql index b09be66d..d1785f14 100644 --- a/integration_tests/models/cross_db_utils/test_position.sql +++ b/integration_tests/models/cross_db_utils/test_position.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_position') }} - -) +with data as (select * from {{ ref("data_position") }}) select - {{ dbt_utils.position('substring_text', 'string_text') }} as actual, + {{ dbt_utils.position("substring_text", "string_text") }} as actual, result as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_replace.sql b/integration_tests/models/cross_db_utils/test_replace.sql index 6c8a8aa1..0e413c8a 100644 --- a/integration_tests/models/cross_db_utils/test_replace.sql +++ b/integration_tests/models/cross_db_utils/test_replace.sql @@ -1,18 +1,19 @@ -with data as ( +with + data as ( - select - - *, - coalesce(search_chars, '') as old_chars, - coalesce(replace_chars, '') as new_chars - - from {{ ref('data_replace') }} + select -) + *, + coalesce(search_chars, '') as old_chars, + coalesce(replace_chars, '') as new_chars + + from {{ ref("data_replace") }} + + ) select - {{ dbt_utils.replace('string_text', 'old_chars', 'new_chars') }} as actual, + {{ dbt_utils.replace("string_text", "old_chars", "new_chars") }} as actual, result as expected from data diff --git a/integration_tests/models/cross_db_utils/test_right.sql b/integration_tests/models/cross_db_utils/test_right.sql index eaad4f4f..de2df7dc 100644 --- a/integration_tests/models/cross_db_utils/test_right.sql +++ b/integration_tests/models/cross_db_utils/test_right.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_right') }} - -) +with data as (select * from {{ ref("data_right") }}) select - {{ dbt_utils.right('string_text', 'length_expression') }} as actual, + {{ dbt_utils.right("string_text", "length_expression") }} as actual, coalesce(output, '') as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_safe_cast.sql b/integration_tests/models/cross_db_utils/test_safe_cast.sql index 8ed74fc3..b2e6d82c 100644 --- a/integration_tests/models/cross_db_utils/test_safe_cast.sql +++ b/integration_tests/models/cross_db_utils/test_safe_cast.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_safe_cast') }} - -) +with data as (select * from {{ ref("data_safe_cast") }}) select - {{ dbt_utils.safe_cast('field', dbt_utils.type_string()) }} as actual, + {{ dbt_utils.safe_cast("field", dbt_utils.type_string()) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_split_part.sql b/integration_tests/models/cross_db_utils/test_split_part.sql index 6a10327c..05c0d70f 100644 --- a/integration_tests/models/cross_db_utils/test_split_part.sql +++ b/integration_tests/models/cross_db_utils/test_split_part.sql @@ -1,28 +1,21 @@ -with data as ( - - select * from {{ ref('data_split_part') }} - -) +with data as (select * from {{ ref("data_split_part") }}) select - {{ dbt_utils.split_part('parts', 'split_on', 1) }} as actual, - result_1 as expected + {{ dbt_utils.split_part("parts", "split_on", 1) }} as actual, result_1 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 2) }} as actual, - result_2 as expected + {{ dbt_utils.split_part("parts", "split_on", 2) }} as actual, result_2 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 3) }} as actual, - result_3 as expected + {{ dbt_utils.split_part("parts", "split_on", 3) }} as actual, result_3 as expected from data diff --git a/integration_tests/models/cross_db_utils/test_width_bucket.sql b/integration_tests/models/cross_db_utils/test_width_bucket.sql index dc2b8609..c2b79646 100644 --- a/integration_tests/models/cross_db_utils/test_width_bucket.sql +++ b/integration_tests/models/cross_db_utils/test_width_bucket.sql @@ -1,12 +1,9 @@ -with data as ( - - select * from {{ ref('data_width_bucket') }} - -) +with data as (select * from {{ ref("data_width_bucket") }}) select - {{ dbt_utils.width_bucket('amount', 'min_value', 'max_value', 'num_buckets') }} as actual, + {{ dbt_utils.width_bucket("amount", "min_value", "max_value", "num_buckets") }} + as actual, bucket as expected from data diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index fa4ae52b..f2c2dd12 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -3,24 +3,22 @@ -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} -{{ config(materialized='table') }} +with + date_spine as ( -with date_spine as ( - - {% if target.type == 'postgres' %} + {% if target.type == "postgres" %} {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} - - {% elif target.type == 'bigquery' %} + + {% elif target.type == "bigquery" %} select cast(date_day as date) as date_day from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) - - {% else %} - {{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} - {% endif %} -) + {% else %}{{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} + {% endif %} + + ) select date_day from date_spine - diff --git a/integration_tests/models/generic_tests/test_equal_column_subset.sql b/integration_tests/models/generic_tests/test_equal_column_subset.sql index b426bb63..7ecae303 100644 --- a/integration_tests/models/generic_tests/test_equal_column_subset.sql +++ b/integration_tests/models/generic_tests/test_equal_column_subset.sql @@ -1,9 +1,5 @@ -{{ config(materialized='ephemeral') }} +{{ config(materialized="ephemeral") }} -select +select first_name, last_name, email - first_name, - last_name, - email - -from {{ ref('data_people') }} +from {{ ref("data_people") }} diff --git a/integration_tests/models/generic_tests/test_equal_rowcount.sql b/integration_tests/models/generic_tests/test_equal_rowcount.sql index 01b1a549..621ccf61 100644 --- a/integration_tests/models/generic_tests/test_equal_rowcount.sql +++ b/integration_tests/models/generic_tests/test_equal_rowcount.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_equal_rowcount") }}) - select * from {{ ref('data_test_equal_rowcount') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_fewer_rows_than.sql b/integration_tests/models/generic_tests/test_fewer_rows_than.sql index c2ad4cbd..621fce6c 100644 --- a/integration_tests/models/generic_tests/test_fewer_rows_than.sql +++ b/integration_tests/models/generic_tests/test_fewer_rows_than.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_fewer_rows_than_table_1") }}) - select * from {{ ref('data_test_fewer_rows_than_table_1') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_recency.sql b/integration_tests/models/generic_tests/test_recency.sql index d300e572..142490f4 100644 --- a/integration_tests/models/generic_tests/test_recency.sql +++ b/integration_tests/models/generic_tests/test_recency.sql @@ -1,12 +1,13 @@ -{% if target.type == 'postgres' %} +{% if target.type == "postgres" %} -select - {{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as today +select {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as today {% else %} select - cast({{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as datetime) as today - + cast( + {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as datetime + ) as today + {% endif %} diff --git a/integration_tests/models/geo/test_haversine_distance_km.sql b/integration_tests/models/geo/test_haversine_distance_km.sql index e8ca4818..fce70682 100644 --- a/integration_tests/models/geo/test_haversine_distance_km.sql +++ b/integration_tests/models/geo/test_haversine_distance_km.sql @@ -1,23 +1,20 @@ -with data as ( - select * from {{ ref('data_haversine_km') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='km' +with + data as (select * from {{ ref("data_haversine_km") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="km", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/geo/test_haversine_distance_mi.sql b/integration_tests/models/geo/test_haversine_distance_mi.sql index 53f8172c..9ff4a1f7 100644 --- a/integration_tests/models/geo/test_haversine_distance_mi.sql +++ b/integration_tests/models/geo/test_haversine_distance_mi.sql @@ -1,39 +1,36 @@ -with data as ( - select * from {{ ref('data_haversine_mi') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='mi' +with + data as (select * from {{ ref("data_haversine_mi") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="mi", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data - union all + union all - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/materializations/expected_insert_by_period.sql b/integration_tests/models/materializations/expected_insert_by_period.sql index 0a198b6b..0d7d68c1 100644 --- a/integration_tests/models/materializations/expected_insert_by_period.sql +++ b/integration_tests/models/materializations/expected_insert_by_period.sql @@ -1,10 +1,5 @@ -{{ - config( - materialized = 'view', - enabled=(target.type == 'redshift') - ) -}} +{{ config(materialized="view", enabled=(target.type == "redshift")) }} select * -from {{ ref('data_insert_by_period') }} +from {{ ref("data_insert_by_period") }} where id in (2, 3, 4, 5, 6) diff --git a/integration_tests/models/materializations/test_insert_by_period.sql b/integration_tests/models/materializations/test_insert_by_period.sql index 8cc396d3..b48d22f6 100644 --- a/integration_tests/models/materializations/test_insert_by_period.sql +++ b/integration_tests/models/materializations/test_insert_by_period.sql @@ -1,18 +1,16 @@ {{ - config( - materialized = 'insert_by_period', - period = 'month', - timestamp_field = 'created_at', - start_date = '2018-01-01', - stop_date = '2018-06-01', - enabled=(target.type == 'redshift') - ) + config( + materialized="insert_by_period", + period="month", + timestamp_field="created_at", + start_date="2018-01-01", + stop_date="2018-06-01", + enabled=(target.type == "redshift"), + ) }} -with events as ( - select * - from {{ ref('data_insert_by_period') }} - where __PERIOD_FILTER__ -) +with + events as (select * from {{ ref("data_insert_by_period") }} where __period_filter__) -select * from events +select * +from events diff --git a/integration_tests/models/sql/test_deduplicate.sql b/integration_tests/models/sql/test_deduplicate.sql index 81fe81e7..47649a61 100644 --- a/integration_tests/models/sql/test_deduplicate.sql +++ b/integration_tests/models/sql/test_deduplicate.sql @@ -1,22 +1,19 @@ with -source as ( - select * - from {{ ref('data_deduplicate') }} - where user_id = 1 -), + source as (select * from {{ ref("data_deduplicate") }} where user_id = 1), -deduped as ( + deduped as ( - {{ - dbt_utils.deduplicate( - ref('data_deduplicate'), - group_by='user_id', - order_by='version desc', - relation_alias="source" - ) | indent - }} + {{ + dbt_utils.deduplicate( + ref("data_deduplicate"), + group_by="user_id", + order_by="version desc", + relation_alias="source", + ) | indent + }} -) + ) -select * from deduped +select * +from deduped diff --git a/integration_tests/models/sql/test_generate_series.sql b/integration_tests/models/sql/test_generate_series.sql index 11370b7b..2e5acf19 100644 --- a/integration_tests/models/sql/test_generate_series.sql +++ b/integration_tests/models/sql/test_generate_series.sql @@ -3,13 +3,9 @@ -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} -{{ config(materialized='table') }} +with data as ({{ dbt_utils.generate_series(10) }}) -with data as ( - - {{ dbt_utils.generate_series(10) }} - -) - -select generated_number from data +select generated_number +from data diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index 36214984..fe91fa57 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -1,29 +1,32 @@ -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values"), "field", default=[], order_by="field" +) %} -{% if target.type == 'snowflake' %} +{% if target.type == "snowflake" %} select {% for val in column_values -%} - sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} - {%- if not loop.last %},{% endif -%} + sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} + {%- if not loop.last %},{% endif -%} {%- endfor %} -from {{ ref('data_get_column_values') }} +from {{ ref("data_get_column_values") }} {% else %} select {% for val in column_values -%} - {{dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} as count_{{ val }} - {%- if not loop.last %},{% endif -%} + {{ dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} + as count_{{ val }} + {%- if not loop.last %},{% endif -%} {%- endfor %} -from {{ ref('data_get_column_values') }} +from {{ ref("data_get_column_values") }} {% endif %} diff --git a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql index 7b3ca72f..afc3ccaa 100644 --- a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql +++ b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql @@ -1,16 +1,21 @@ -{% set exclude_field = 'field_1' %} -{% set column_names = dbt_utils.get_filtered_columns_in_relation(from= ref('data_filtered_columns_in_relation'), except=[exclude_field]) %} +{% set exclude_field = "field_1" %} +{% set column_names = dbt_utils.get_filtered_columns_in_relation( + from=ref("data_filtered_columns_in_relation"), except=[exclude_field] +) %} -with data as ( +with + data as ( - select + select - {% for column_name in column_names %} - max({{ column_name }}) as {{ column_name }} {% if not loop.last %},{% endif %} - {% endfor %} + {% for column_name in column_names %} + max({{ column_name }}) as {{ column_name }} + {% if not loop.last %},{% endif %} + {% endfor %} - from {{ ref('data_filtered_columns_in_relation') }} + from {{ ref("data_filtered_columns_in_relation") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index 52ab07a2..2c20559e 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,16 +1,9 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} {% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %} -with unioned as ( - - {{ dbt_utils.union_relations(relations) }} - -) +with unioned as ({{ dbt_utils.union_relations(relations) }}) -select - - user_id, - event +select user_id, event from unioned diff --git a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql index 481f233e..6e968e79 100644 --- a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql +++ b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql @@ -1,4 +1,4 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} -{% set relations = dbt_utils.get_relations_by_prefix(target.schema, 'data_events_') %} +{% set relations = dbt_utils.get_relations_by_prefix(target.schema, "data_events_") %} {{ dbt_utils.union_relations(relations) }} diff --git a/integration_tests/models/sql/test_groupby.sql b/integration_tests/models/sql/test_groupby.sql index 50a6ced7..a63f3a5c 100644 --- a/integration_tests/models/sql/test_groupby.sql +++ b/integration_tests/models/sql/test_groupby.sql @@ -1,24 +1,14 @@ -with test_data as ( - - select - - {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string() )}} as column_1, - {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string() )}} as column_2 - -), +with + test_data as ( -grouped as ( + select - select - *, - count(*) as total - - from test_data - {{ dbt_utils.group_by(2) }} - -) - -select * from grouped + {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string()) }} as column_1, + {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string()) }} as column_2 + ), + grouped as (select *, count(*) as total from test_data {{ dbt_utils.group_by(2) }}) +select * +from grouped diff --git a/integration_tests/models/sql/test_nullcheck_table.sql b/integration_tests/models/sql/test_nullcheck_table.sql index ea66c8ec..4eb60e4e 100644 --- a/integration_tests/models/sql/test_nullcheck_table.sql +++ b/integration_tests/models/sql/test_nullcheck_table.sql @@ -1,17 +1,12 @@ -{{ config( materialized = "table" ) }} - --- TO DO: remove if-statement - -{% set tbl = ref('data_nullcheck_table') %} - +{{ config(materialized="table") }} -with nulled as ( +-- TO DO: remove if-statement +{% set tbl = ref("data_nullcheck_table") %} - {{ dbt_utils.nullcheck_table(tbl) }} -) +with nulled as ({{ dbt_utils.nullcheck_table(tbl) }}) -{% if target.type == 'snowflake' %} +{% if target.type == "snowflake" %} select field_1::varchar as field_1, @@ -24,17 +19,11 @@ from nulled select - {{ dbt_utils.safe_cast('field_1', - dbt_utils.type_string() - )}} as field_1, + {{ dbt_utils.safe_cast("field_1", dbt_utils.type_string()) }} as field_1, - {{ dbt_utils.safe_cast('field_2', - dbt_utils.type_string() - )}} as field_2, + {{ dbt_utils.safe_cast("field_2", dbt_utils.type_string()) }} as field_2, - {{ dbt_utils.safe_cast('field_3', - dbt_utils.type_string() - )}} as field_3 + {{ dbt_utils.safe_cast("field_3", dbt_utils.type_string()) }} as field_3 from nulled diff --git a/integration_tests/models/sql/test_pivot.sql b/integration_tests/models/sql/test_pivot.sql index a63e6e3b..730b7bec 100644 --- a/integration_tests/models/sql/test_pivot.sql +++ b/integration_tests/models/sql/test_pivot.sql @@ -1,17 +1,11 @@ -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE'] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue'] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} +{% set column_values = ["RED", "BLUE"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue"] %} {% set cmp = "=" %} {% endif %} -select - size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp) }} +select size, {{ dbt_utils.pivot("color", column_values, cmp=cmp) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_pivot_apostrophe.sql b/integration_tests/models/sql/test_pivot_apostrophe.sql index 792f4686..e7a340a3 100644 --- a/integration_tests/models/sql/test_pivot_apostrophe.sql +++ b/integration_tests/models/sql/test_pivot_apostrophe.sql @@ -1,17 +1,13 @@ -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE', "BLUE'S"] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue', "blue's"] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} +{% set column_values = ["RED", "BLUE", "BLUE'S"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue", "blue's"] %} {% set cmp = "=" %} {% endif %} select size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp, quote_identifiers=False) }} + {{ dbt_utils.pivot("color", column_values, cmp=cmp, quote_identifiers=False) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_safe_add.sql b/integration_tests/models/sql/test_safe_add.sql index af3894d0..840871ff 100644 --- a/integration_tests/models/sql/test_safe_add.sql +++ b/integration_tests/models/sql/test_safe_add.sql @@ -1,12 +1,6 @@ -with data as ( +with data as (select * from {{ ref("data_safe_add") }}) - select * from {{ ref('data_safe_add') }} - -) - -select - {{ dbt_utils.safe_add('field_1', 'field_2', 'field_3') }} as actual, - expected +select {{ dbt_utils.safe_add("field_1", "field_2", "field_3") }} as actual, expected from data diff --git a/integration_tests/models/sql/test_star.sql b/integration_tests/models/sql/test_star.sql index 2092e16a..5932635c 100644 --- a/integration_tests/models/sql/test_star.sql +++ b/integration_tests/models/sql/test_star.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'field_3' %} +{% set exclude_field = "field_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref("data_star"), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_aggregate.sql b/integration_tests/models/sql/test_star_aggregate.sql index 9dcd7c2d..16b6c94b 100644 --- a/integration_tests/models/sql/test_star_aggregate.sql +++ b/integration_tests/models/sql/test_star_aggregate.sql @@ -1,16 +1,17 @@ -/*This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ +/* This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ +{% set selected_columns = dbt_utils.star( + from=ref("data_star_aggregate"), except=["value_field"] +) %} -{% set selected_columns = dbt_utils.star(from=ref('data_star_aggregate'), except=['value_field']) %} +with + data as ( -with data as ( + select {{ selected_columns }}, sum(value_field) as value_field_sum - select - {{ selected_columns }}, - sum(value_field) as value_field_sum + from {{ ref("data_star_aggregate") }} + group by {{ selected_columns }} - from {{ ref('data_star_aggregate') }} - group by {{ selected_columns }} + ) -) - -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_prefix_suffix.sql b/integration_tests/models/sql/test_star_prefix_suffix.sql index ae1c64fe..5d66ad06 100644 --- a/integration_tests/models/sql/test_star_prefix_suffix.sql +++ b/integration_tests/models/sql/test_star_prefix_suffix.sql @@ -1,13 +1,19 @@ -{% set prefix_with = 'prefix_' if target.type != 'snowflake' else 'PREFIX_' %} -{% set suffix_with = '_suffix' if target.type != 'snowflake' else '_SUFFIX' %} +{% set prefix_with = "prefix_" if target.type != "snowflake" else "PREFIX_" %} +{% set suffix_with = "_suffix" if target.type != "snowflake" else "_SUFFIX" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} + select + {{ + dbt_utils.star( + from=ref("data_star"), prefix=prefix_with, suffix=suffix_with + ) + }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data \ No newline at end of file +select * +from data diff --git a/integration_tests/models/sql/test_star_uppercase.sql b/integration_tests/models/sql/test_star_uppercase.sql index 6179e691..f8656abe 100644 --- a/integration_tests/models/sql/test_star_uppercase.sql +++ b/integration_tests/models/sql/test_star_uppercase.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'FIELD_3' %} +{% set exclude_field = "FIELD_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref("data_star"), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_surrogate_key.sql b/integration_tests/models/sql/test_surrogate_key.sql index fd90939f..a0da48ef 100644 --- a/integration_tests/models/sql/test_surrogate_key.sql +++ b/integration_tests/models/sql/test_surrogate_key.sql @@ -1,15 +1,13 @@ -with data as ( - - select * from {{ ref('data_surrogate_key') }} - -) +with data as (select * from {{ ref("data_surrogate_key") }}) select - {{ dbt_utils.surrogate_key('column_1') }} as actual_column_1_only, + {{ dbt_utils.surrogate_key("column_1") }} as actual_column_1_only, expected_column_1_only, - {{ dbt_utils.surrogate_key('column_1', 'column_2', 'column_3') }} as actual_all_columns_arguments, - {{ dbt_utils.surrogate_key(['column_1', 'column_2', 'column_3']) }} as actual_all_columns_list, + {{ dbt_utils.surrogate_key("column_1", "column_2", "column_3") }} + as actual_all_columns_arguments, + {{ dbt_utils.surrogate_key(["column_1", "column_2", "column_3"]) }} + as actual_all_columns_list, expected_all_columns from data diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql index 69836833..1b14ff1f 100644 --- a/integration_tests/models/sql/test_union.sql +++ b/integration_tests/models/sql/test_union.sql @@ -1,8 +1,2 @@ -select - id, - name, - favorite_color - -from {{ ref('test_union_base') }} - +select id, name, favorite_color from {{ ref("test_union_base") }} diff --git a/integration_tests/models/sql/test_union_base.sql b/integration_tests/models/sql/test_union_base.sql index a51bd2b8..045e651b 100644 --- a/integration_tests/models/sql/test_union_base.sql +++ b/integration_tests/models/sql/test_union_base.sql @@ -1,5 +1,2 @@ -{{ dbt_utils.union_relations([ - ref('data_union_table_1'), - ref('data_union_table_2')] -) }} +{{ dbt_utils.union_relations([ref("data_union_table_1"), ref("data_union_table_2")]) }} diff --git a/integration_tests/models/sql/test_unpivot.sql b/integration_tests/models/sql/test_unpivot.sql index 8cf1ad6e..7198d0a3 100644 --- a/integration_tests/models/sql/test_unpivot.sql +++ b/integration_tests/models/sql/test_unpivot.sql @@ -4,30 +4,27 @@ -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - remove=['name'], - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + remove=["name"], + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_bool.sql b/integration_tests/models/sql/test_unpivot_bool.sql index 26842c01..82d7205b 100644 --- a/integration_tests/models/sql/test_unpivot_bool.sql +++ b/integration_tests/models/sql/test_unpivot_bool.sql @@ -4,29 +4,26 @@ -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot_bool'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot_bool"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_original_api.sql b/integration_tests/models/sql/test_unpivot_original_api.sql index f6b9395a..c10d016e 100644 --- a/integration_tests/models/sql/test_unpivot_original_api.sql +++ b/integration_tests/models/sql/test_unpivot_original_api.sql @@ -1,32 +1,30 @@ -- unpivot() was enhanced with 3 new parameters -- This test targets the original API. - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, case - when '{{ target.name }}' = 'snowflake' then lower(FIELD_NAME) - else field_name + when '{{ target.name }}' = 'snowflake' then lower(field_name) else field_name end as field_name, value -from ( - {{ dbt_utils.unpivot( - table=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + table=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + ) + }} + ) as sbq diff --git a/integration_tests/models/web/test_url_host.sql b/integration_tests/models/web/test_url_host.sql index 06a9f69c..2e18e65d 100644 --- a/integration_tests/models/web/test_url_host.sql +++ b/integration_tests/models/web/test_url_host.sql @@ -1,12 +1,5 @@ -with data as ( - - select * from {{ref('data_url_host')}} - -) +with data as (select * from {{ ref("data_url_host") }}) -select +select {{ dbt_utils.get_url_host("original_url") }} as actual, parsed_url as expected - {{ dbt_utils.get_url_host('original_url') }} as actual, - parsed_url as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/web/test_url_path.sql b/integration_tests/models/web/test_url_path.sql index c03e5547..16f45ae0 100644 --- a/integration_tests/models/web/test_url_path.sql +++ b/integration_tests/models/web/test_url_path.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ref('data_url_path')}} - -) +with data as (select * from {{ ref("data_url_path") }}) select - coalesce({{ dbt_utils.get_url_path('original_url') }}, '') as actual, + coalesce({{ dbt_utils.get_url_path("original_url") }}, '') as actual, coalesce(parsed_path, '') as expected - -from data \ No newline at end of file + +from data diff --git a/integration_tests/models/web/test_urls.sql b/integration_tests/models/web/test_urls.sql index 60353dbb..066a0fcd 100644 --- a/integration_tests/models/web/test_urls.sql +++ b/integration_tests/models/web/test_urls.sql @@ -1,20 +1,14 @@ -with data as ( - - select * from {{ ref('data_urls') }} - -) +with data as (select * from {{ ref("data_urls") }}) select - {{ dbt_utils.get_url_parameter('url', 'utm_medium') }} as actual, - medium as expected + {{ dbt_utils.get_url_parameter("url", "utm_medium") }} as actual, medium as expected from data union all select - {{ dbt_utils.get_url_parameter('url', 'utm_source') }} as actual, - source as expected + {{ dbt_utils.get_url_parameter("url", "utm_source") }} as actual, source as expected from data diff --git a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql index 55d5400e..88c0dc9e 100644 --- a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql +++ b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql @@ -1,17 +1,16 @@ -- depends_on: {{ ref('data_get_query_results_as_dict') }} - -{% set expected_dictionary={ - 'col_1': [1, 2, 3], - 'col_2': ['a', 'b', 'c'], - 'col_3': [True, False, none] +{% set expected_dictionary = { + "col_1": [1, 2, 3], + "col_2": ["a", "b", "c"], + "col_3": [True, False, none], } %} {#- Handle snowflake casing silliness -#} -{% if target.type == 'snowflake' %} -{% set expected_dictionary={ - 'COL_1': [1, 2, 3], - 'COL_2': ['a', 'b', 'c'], - 'COL_3': [True, False, none] +{% if target.type == "snowflake" %} +{% set expected_dictionary = { + "COL_1": [1, 2, 3], + "COL_2": ["a", "b", "c"], + "COL_3": [True, False, none], } %} {% endif %} @@ -23,18 +22,14 @@ For reasons that remain unclear, Jinja won't return True for actual_dictionary == expected_dictionary. Instead, we'll manually check that the values of these dictionaries are equivalent. -#} - -{% set ns = namespace( - pass=True, - err_msg = "" -) %} +{% set ns = namespace(pass=True, err_msg="") %} {% if execute %} {#- Check that the dictionaries have the same keys -#} -{% set expected_keys=expected_dictionary.keys() | list | sort %} -{% set actual_keys=actual_dictionary.keys() | list | sort %} +{% set expected_keys = expected_dictionary.keys() | list | sort %} +{% set actual_keys = actual_dictionary.keys() | list | sort %} {% if expected_keys != actual_keys %} - {% set ns.pass=False %} +{% set ns.pass = False %} {% set ns.err_msg %} The two dictionaries have different keys: expected_dictionary has keys: {{ expected_keys }} @@ -44,33 +39,33 @@ Instead, we'll manually check that the values of these dictionaries are equivale {% else %} {% for key, value in expected_dictionary.items() %} - {% set expected_length=expected_dictionary[key] | length %} - {% set actual_length=actual_dictionary[key] | length %} +{% set expected_length = expected_dictionary[key] | length %} +{% set actual_length = actual_dictionary[key] | length %} - {% if expected_length != actual_length %} - {% set ns.pass=False %} +{% if expected_length != actual_length %} +{% set ns.pass = False %} {% set ns.err_msg %} The {{ key }} column has different lengths: expected_dictionary[{{ key }}] has length {{ expected_length }} actual_dictionary[{{ key }}] has length {{ actual_length }} {% endset %} - {% else %} +{% else %} - {% for i in range(value | length) %} - {% set expected_value=expected_dictionary[key][i] %} - {% set actual_value=actual_dictionary[key][i] %} - {% if expected_value != actual_value %} - {% set ns.pass=False %} +{% for i in range(value | length) %} +{% set expected_value = expected_dictionary[key][i] %} +{% set actual_value = actual_dictionary[key][i] %} +{% if expected_value != actual_value %} +{% set ns.pass = False %} {% set ns.err_msg %} The {{ key }} column has differing values: expected_dictionary[{{ key }}][{{ i }}] == {{ expected_value }} actual_dictionary[{{ key }}][{{ i }}] == {{ actual_value }} {% endset %} - {% endif %} - {% endfor %} - {% endif %} +{% endif %} +{% endfor %} +{% endif %} {% endfor %} diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql index fea33594..6782e8d1 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql @@ -1,7 +1,5 @@ -{% if dbt_utils.pretty_log_format() is string %} - {# Return 0 rows for the test to pass #} - select 1 as col_name {{ limit_zero() }} -{% else %} - {# Return >0 rows for the test to fail #} - select 1 +{# Return 0 rows for the test to pass #} +{% if dbt_utils.pretty_log_format() is string %} select 1 as col_name {{ limit_zero() }} +{# Return >0 rows for the test to fail #} +{% else %} select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql index a6347abd..a9ec4b19 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql @@ -1,7 +1,5 @@ -{% if dbt_utils.pretty_time() is string %} - {# Return 0 rows for the test to pass #} - select 1 as col_name {{ limit_zero() }} -{% else %} - {# Return >0 rows for the test to fail #} - select 1 +{# Return 0 rows for the test to pass #} +{% if dbt_utils.pretty_time() is string %} select 1 as col_name {{ limit_zero() }} +{# Return >0 rows for the test to fail #} +{% else %} select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/test_slugify.sql b/integration_tests/tests/jinja_helpers/test_slugify.sql index c0839f59..13d0a2bc 100644 --- a/integration_tests/tests/jinja_helpers/test_slugify.sql +++ b/integration_tests/tests/jinja_helpers/test_slugify.sql @@ -1,7 +1,6 @@ -{% if dbt_utils.slugify('!Hell0 world-hi') == 'hell0_world_hi' %} - {# Return 0 rows for the test to pass #} - select 1 as col_name {{ limit_zero() }} -{% else %} - {# Return >0 rows for the test to fail #} - select 1 as col_name +{% if dbt_utils.slugify("!Hell0 world-hi") == "hell0_world_hi" %} +{# Return 0 rows for the test to pass #} +select 1 as col_name {{ limit_zero() }} +{# Return >0 rows for the test to fail #} +{% else %} select 1 as col_name {% endif %} diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 7f9fa3a4..0fcd8ac0 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -1,26 +1,40 @@ -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values_dropped"), + "field", + default=["y", "z"], + order_by="field", +) %} -with expected as ( - select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected_column_value union all - select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected_column_value -), +with + expected as ( + select + {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} + as expected_column_value + union all + select + {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} + as expected_column_value + ), + + actual as ( -actual as ( - {% for val in column_values %} - select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual_column_value - {% if not loop.last %} - union all - {% endif %} + select + {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} + as actual_column_value + {% if not loop.last %} union all {% endif %} {% endfor %} -), + ), -failures as ( - select * from actual - where actual.actual_column_value not in ( - select expected.expected_column_value from expected + failures as ( + select * + from actual + where + actual.actual_column_value not in ( + select expected.expected_column_value from expected + ) ) -) -select * from failures \ No newline at end of file +select * +from failures diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index 34c70db9..54317376 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -1,16 +1,15 @@ {% macro _is_ephemeral(obj, macro) %} - {%- if obj.is_cte -%} - {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} - {% if obj.name.startswith(ephemeral_prefix) %} - {% set model_name = obj.name[(ephemeral_prefix|length):] %} - {% else %} - {% set model_name = obj.name %} - {%- endif -%} +{%- if obj.is_cte -%} +{% set ephemeral_prefix = api.Relation.add_ephemeral_prefix("") %} +{% if obj.name.startswith(ephemeral_prefix) %} +{% set model_name = obj.name[(ephemeral_prefix | length) :] %} +{% else %} {% set model_name = obj.name %} +{%- endif -%} {% set error_message %} The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. `{{ model_name }}` is an ephemeral model. Consider making it a view or table instead. {% endset %} - {%- do exceptions.raise_compiler_error(error_message) -%} - {%- endif -%} +{%- do exceptions.raise_compiler_error(error_message) -%} +{%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/_is_relation.sql b/macros/cross_db_utils/_is_relation.sql index bee6db97..af01c32c 100644 --- a/macros/cross_db_utils/_is_relation.sql +++ b/macros/cross_db_utils/_is_relation.sql @@ -1,5 +1,8 @@ {% macro _is_relation(obj, macro) %} - {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%} - {%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} - {%- endif -%} +{%- if not ( + obj is mapping + and obj.get("metadata", {}).get("type", "").endswith("Relation") +) -%} +{%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} +{%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/any_value.sql b/macros/cross_db_utils/any_value.sql index 78cb75ba..8f0668b0 100644 --- a/macros/cross_db_utils/any_value.sql +++ b/macros/cross_db_utils/any_value.sql @@ -1,17 +1,10 @@ {% macro any_value(expression) -%} - {{ return(adapter.dispatch('any_value', 'dbt_utils') (expression)) }} +{{ return(adapter.dispatch("any_value", "dbt_utils")(expression)) }} {% endmacro %} -{% macro default__any_value(expression) -%} - - any_value({{ expression }}) - -{%- endmacro %} +{% macro default__any_value(expression) -%} any_value({{ expression }}) {%- endmacro %} -{% macro postgres__any_value(expression) -%} - {#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} - min({{ expression }}) - -{%- endmacro %} \ No newline at end of file +{#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} +{% macro postgres__any_value(expression) -%} min({{ expression }}) {%- endmacro %} diff --git a/macros/cross_db_utils/bool_or.sql b/macros/cross_db_utils/bool_or.sql index ce0a6857..10f4fbb6 100644 --- a/macros/cross_db_utils/bool_or.sql +++ b/macros/cross_db_utils/bool_or.sql @@ -1,24 +1,12 @@ {% macro bool_or(expression) -%} - {{ return(adapter.dispatch('bool_or', 'dbt_utils') (expression)) }} +{{ return(adapter.dispatch("bool_or", "dbt_utils")(expression)) }} {% endmacro %} -{% macro default__bool_or(expression) -%} - - bool_or({{ expression }}) - -{%- endmacro %} +{% macro default__bool_or(expression) -%} bool_or({{ expression }}) {%- endmacro %} -{% macro snowflake__bool_or(expression) -%} - - boolor_agg({{ expression }}) - -{%- endmacro %} +{% macro snowflake__bool_or(expression) -%} boolor_agg({{ expression }}) {%- endmacro %} -{% macro bigquery__bool_or(expression) -%} - - logical_or({{ expression }}) - -{%- endmacro %} \ No newline at end of file +{% macro bigquery__bool_or(expression) -%} logical_or({{ expression }}) {%- endmacro %} diff --git a/macros/cross_db_utils/cast_bool_to_text.sql b/macros/cross_db_utils/cast_bool_to_text.sql index 5efa9d60..eb8c768e 100644 --- a/macros/cross_db_utils/cast_bool_to_text.sql +++ b/macros/cross_db_utils/cast_bool_to_text.sql @@ -1,15 +1,14 @@ {% macro cast_bool_to_text(field) %} - {{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }} +{{ adapter.dispatch("cast_bool_to_text", "dbt_utils")(field) }} {% endmacro %} {% macro default__cast_bool_to_text(field) %} - cast({{ field }} as {{ dbt_utils.type_string() }}) +cast({{ field }} as {{ dbt_utils.type_string() }}) {% endmacro %} {% macro redshift__cast_bool_to_text(field) %} - case - when {{ field }} is true then 'true' - when {{ field }} is false then 'false' - end::text +case + when {{ field }} is true then 'true' when {{ field }} is false then 'false' +end::text {% endmacro %} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 7e97e8f8..74b9fb30 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,9 +1,7 @@ {% macro concat(fields) -%} - {{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }} +{{ return(adapter.dispatch("concat", "dbt_utils")(fields)) }} {%- endmacro %} -{% macro default__concat(fields) -%} - {{ fields|join(' || ') }} -{%- endmacro %} +{% macro default__concat(fields) -%} {{ fields | join(" || ") }} {%- endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index 66ad8dc6..767c5d07 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,38 +1,36 @@ {% macro current_timestamp() -%} - {{ return(adapter.dispatch('current_timestamp', 'dbt_utils')()) }} +{{ return(adapter.dispatch("current_timestamp", "dbt_utils")()) }} {%- endmacro %} {% macro default__current_timestamp() %} - current_timestamp::{{dbt_utils.type_timestamp()}} +current_timestamp::{{ dbt_utils.type_timestamp() }} {% endmacro %} -{% macro redshift__current_timestamp() %} - getdate() -{% endmacro %} +{% macro redshift__current_timestamp() %} getdate() {% endmacro %} -{% macro bigquery__current_timestamp() %} - current_timestamp -{% endmacro %} +{% macro bigquery__current_timestamp() %} current_timestamp {% endmacro %} {% macro current_timestamp_in_utc() -%} - {{ return(adapter.dispatch('current_timestamp_in_utc', 'dbt_utils')()) }} +{{ return(adapter.dispatch("current_timestamp_in_utc", "dbt_utils")()) }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} - {{dbt_utils.current_timestamp()}} +{{ dbt_utils.current_timestamp() }} {% endmacro %} {% macro snowflake__current_timestamp_in_utc() %} - convert_timezone('UTC', {{dbt_utils.current_timestamp()}})::{{dbt_utils.type_timestamp()}} +convert_timezone( + 'UTC', {{ dbt_utils.current_timestamp() }} +)::{{ dbt_utils.type_timestamp() }} {% endmacro %} {% macro postgres__current_timestamp_in_utc() %} - (current_timestamp at time zone 'utc')::{{dbt_utils.type_timestamp()}} +(current_timestamp at time zone 'utc')::{{ dbt_utils.type_timestamp() }} {% endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__current_timestamp_in_utc() %} - {{ return(dbt_utils.default__current_timestamp_in_utc()) }} +{{ return(dbt_utils.default__current_timestamp_in_utc()) }} {% endmacro %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index f115b4e2..a981ee22 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,99 +1,63 @@ {# string ------------------------------------------------- #} - {%- macro type_string() -%} - {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }} +{{ return(adapter.dispatch("type_string", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_string() %} - string -{% endmacro %} +{% macro default__type_string() %} string {% endmacro %} -{%- macro redshift__type_string() -%} - varchar -{%- endmacro -%} +{%- macro redshift__type_string() -%} varchar {%- endmacro -%} -{% macro postgres__type_string() %} - varchar -{% endmacro %} +{% macro postgres__type_string() %} varchar {% endmacro %} -{% macro snowflake__type_string() %} - varchar -{% endmacro %} +{% macro snowflake__type_string() %} varchar {% endmacro %} {# timestamp ------------------------------------------------- #} - {%- macro type_timestamp() -%} - {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }} +{{ return(adapter.dispatch("type_timestamp", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_timestamp() %} - timestamp -{% endmacro %} +{% macro default__type_timestamp() %} timestamp {% endmacro %} -{% macro postgres__type_timestamp() %} - timestamp without time zone -{% endmacro %} +{% macro postgres__type_timestamp() %} timestamp without time zone {% endmacro %} -{% macro snowflake__type_timestamp() %} - timestamp_ntz -{% endmacro %} +{% macro snowflake__type_timestamp() %} timestamp_ntz {% endmacro %} {# float ------------------------------------------------- #} - {%- macro type_float() -%} - {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }} +{{ return(adapter.dispatch("type_float", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_float() %} - float -{% endmacro %} +{% macro default__type_float() %} float {% endmacro %} -{% macro bigquery__type_float() %} - float64 -{% endmacro %} +{% macro bigquery__type_float() %} float64 {% endmacro %} {# numeric ------------------------------------------------ #} - {%- macro type_numeric() -%} - {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }} +{{ return(adapter.dispatch("type_numeric", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_numeric() %} - numeric(28, 6) -{% endmacro %} +{% macro default__type_numeric() %} numeric(28, 6) {% endmacro %} -{% macro bigquery__type_numeric() %} - numeric -{% endmacro %} +{% macro bigquery__type_numeric() %} numeric {% endmacro %} {# bigint ------------------------------------------------- #} - {%- macro type_bigint() -%} - {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }} +{{ return(adapter.dispatch("type_bigint", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_bigint() %} - bigint -{% endmacro %} +{% macro default__type_bigint() %} bigint {% endmacro %} -{% macro bigquery__type_bigint() %} - int64 -{% endmacro %} +{% macro bigquery__type_bigint() %} int64 {% endmacro %} {# int ------------------------------------------------- #} - {%- macro type_int() -%} - {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }} +{{ return(adapter.dispatch("type_int", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_int() %} - int -{% endmacro %} +{% macro default__type_int() %} int {% endmacro %} -{% macro bigquery__type_int() %} - int64 -{% endmacro %} +{% macro bigquery__type_int() %} int64 {% endmacro %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index f9d0364b..80c6d100 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,15 +1,12 @@ {% macro date_trunc(datepart, date) -%} - {{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }} +{{ return(adapter.dispatch("date_trunc", "dbt_utils")(datepart, date)) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) -%} - date_trunc('{{datepart}}', {{date}}) +date_trunc('{{datepart}}', {{ date }}) {%- endmacro %} {% macro bigquery__date_trunc(datepart, date) -%} - timestamp_trunc( - cast({{date}} as timestamp), - {{datepart}} - ) +timestamp_trunc(cast({{ date }} as timestamp), {{ datepart }}) {%- endmacro %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 09c0f115..57cbaa25 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,37 +1,39 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }} +{{ + return( + adapter.dispatch("dateadd", "dbt_utils")( + datepart, interval, from_date_or_timestamp + ) + ) +}} {% endmacro %} {% macro default__dateadd(datepart, interval, from_date_or_timestamp) %} - dateadd( - {{ datepart }}, - {{ interval }}, - {{ from_date_or_timestamp }} - ) +dateadd({{ datepart }}, {{ interval }}, {{ from_date_or_timestamp }}) {% endmacro %} {% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %} - datetime_add( - cast( {{ from_date_or_timestamp }} as datetime), - interval {{ interval }} {{ datepart }} - ) +datetime_add( + cast({{ from_date_or_timestamp }} as datetime), + interval {{ interval }} {{ datepart }} +) {% endmacro %} {% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} - {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) +{{ from_date_or_timestamp }} + ( (interval '1 {{ datepart }}') * ({{ interval }})) {% endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }} +{{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }} {% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 2b5d6613..d95ae1ce 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,59 +1,98 @@ {% macro datediff(first_date, second_date, datepart) %} - {{ return(adapter.dispatch('datediff', 'dbt_utils')(first_date, second_date, datepart)) }} +{{ + return( + adapter.dispatch("datediff", "dbt_utils")(first_date, second_date, datepart) + ) +}} {% endmacro %} {% macro default__datediff(first_date, second_date, datepart) -%} - datediff( - {{ datepart }}, - {{ first_date }}, - {{ second_date }} - ) +datediff({{ datepart }}, {{ first_date }}, {{ second_date }}) {%- endmacro %} {% macro bigquery__datediff(first_date, second_date, datepart) -%} - datetime_diff( - cast({{second_date}} as datetime), - cast({{first_date}} as datetime), - {{datepart}} - ) +datetime_diff( + cast({{ second_date }} as datetime), + cast({{ first_date }} as datetime), + {{ datepart }} +) {%- endmacro %} {% macro postgres__datediff(first_date, second_date, datepart) -%} - {% if datepart == 'year' %} - (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) - {% elif datepart == 'quarter' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) - {% elif datepart == 'month' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) - {% elif datepart == 'day' %} - (({{second_date}})::date - ({{first_date}})::date) - {% elif datepart == 'week' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case - when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then - case when {{first_date}} <= {{second_date}} then 0 else -1 end - else - case when {{first_date}} <= {{second_date}} then 1 else 0 end - end) - {% elif datepart == 'hour' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) - {% elif datepart == 'minute' %} - ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) - {% elif datepart == 'second' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) - {% elif datepart == 'millisecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) - {% elif datepart == 'microsecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) - {% else %} - {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} - {% endif %} +{% if datepart == "year" %} +( + date_part('year', ({{ second_date }})::date) - date_part( + 'year', ({{ first_date }})::date + ) +) +{% elif datepart == "quarter" %} +( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + date_part( + 'quarter', ({{ second_date }})::date + ) - date_part('quarter', ({{ first_date }})::date) +) +{% elif datepart == "month" %} +( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + date_part( + 'month', ({{ second_date }})::date + ) - date_part('month', ({{ first_date }})::date) +) +{% elif datepart == "day" %} ( ({{ second_date }})::date - ({{ first_date }})::date) +{% elif datepart == "week" %} +( + {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case + when + date_part('dow', ({{ first_date }})::timestamp) <= date_part( + 'dow', ({{ second_date }})::timestamp + ) + then case when {{ first_date }} <= {{ second_date }} then 0 else -1 end + else case when {{ first_date }} <= {{ second_date }} then 1 else 0 end + end +) +{% elif datepart == "hour" %} +( + {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + date_part( + 'hour', ({{ second_date }})::timestamp + ) - date_part('hour', ({{ first_date }})::timestamp) +) +{% elif datepart == "minute" %} +( + {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + date_part( + 'minute', ({{ second_date }})::timestamp + ) - date_part('minute', ({{ first_date }})::timestamp) +) +{% elif datepart == "second" %} +( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + floor( + date_part('second', ({{ second_date }})::timestamp) + ) - floor(date_part('second', ({{ first_date }})::timestamp)) +) +{% elif datepart == "millisecond" %} +( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + floor( + date_part('millisecond', ({{ second_date }})::timestamp) + ) - floor(date_part('millisecond', ({{ first_date }})::timestamp)) +) +{% elif datepart == "microsecond" %} +( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + floor( + date_part('microsecond', ({{ second_date }})::timestamp) + ) - floor(date_part('microsecond', ({{ first_date }})::timestamp)) +) +{% else %} +{{ + exceptions.raise_compiler_error( + "Unsupported datepart for macro datediff in postgres: {!r}".format(datepart) + ) +}} +{% endif %} {%- endmacro %} @@ -61,6 +100,6 @@ {# redshift should use default instead of postgres #} {% macro redshift__datediff(first_date, second_date, datepart) -%} - {{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }} +{{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }} {%- endmacro %} diff --git a/macros/cross_db_utils/escape_single_quotes.sql b/macros/cross_db_utils/escape_single_quotes.sql index d024f16f..c42a3866 100644 --- a/macros/cross_db_utils/escape_single_quotes.sql +++ b/macros/cross_db_utils/escape_single_quotes.sql @@ -1,10 +1,10 @@ {% macro escape_single_quotes(expression) %} - {{ return(adapter.dispatch('escape_single_quotes', 'dbt_utils') (expression)) }} +{{ return(adapter.dispatch("escape_single_quotes", "dbt_utils")(expression)) }} {% endmacro %} {# /*Default to replacing a single apostrophe with two apostrophes: they're -> they''re*/ #} {% macro default__escape_single_quotes(expression) -%} -{{ expression | replace("'","''") }} +{{ expression | replace("'", "''") }} {%- endmacro %} {# /*Snowflake uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 9bb75b86..2799095c 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,16 +1,8 @@ {% macro except() %} - {{ return(adapter.dispatch('except', 'dbt_utils')()) }} +{{ return(adapter.dispatch("except", "dbt_utils")()) }} {% endmacro %} -{% macro default__except() %} +{% macro default__except() %} except {% endmacro %} - except - -{% endmacro %} - -{% macro bigquery__except() %} - - except distinct - -{% endmacro %} \ No newline at end of file +{% macro bigquery__except() %} except distinct {% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index d086c2e6..aa1a166c 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,13 +1,13 @@ {% macro hash(field) -%} - {{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }} +{{ return(adapter.dispatch("hash", "dbt_utils")(field)) }} {%- endmacro %} {% macro default__hash(field) -%} - md5(cast({{field}} as {{dbt_utils.type_string()}})) +md5(cast({{ field }} as {{ dbt_utils.type_string() }})) {%- endmacro %} {% macro bigquery__hash(field) -%} - to_hex({{dbt_utils.default__hash(field)}}) +to_hex({{ dbt_utils.default__hash(field) }}) {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index 619cb7cb..8340e864 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -1,16 +1,11 @@ -{% macro identifier(value) %} - {%- set error_message = ' - Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ - Use `adapter.quote` instead. The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - {{ return(adapter.dispatch('identifier', 'dbt_utils') (value)) }} -{% endmacro %} +{% macro identifier(value) %} +{%- set error_message = " Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name +) -%} +{%- do exceptions.warn(error_message) -%} +{{ return(adapter.dispatch("identifier", "dbt_utils")(value)) }} +{% endmacro %} -{% macro default__identifier(value) -%} - "{{ value }}" -{%- endmacro %} +{% macro default__identifier(value) -%} "{{ value }}" {%- endmacro %} -{% macro bigquery__identifier(value) -%} - `{{ value }}` -{%- endmacro %} +{% macro bigquery__identifier(value) -%} `{{ value }}` {%- endmacro %} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index b53c72f3..c224e44f 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,16 +1,8 @@ {% macro intersect() %} - {{ return(adapter.dispatch('intersect', 'dbt_utils')()) }} +{{ return(adapter.dispatch("intersect", "dbt_utils")()) }} {% endmacro %} -{% macro default__intersect() %} +{% macro default__intersect() %} intersect {% endmacro %} - intersect - -{% endmacro %} - -{% macro bigquery__intersect() %} - - intersect distinct - -{% endmacro %} +{% macro bigquery__intersect() %} intersect distinct {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 18ca161c..0e9f6502 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -2,44 +2,50 @@ This function has been tested with dateparts of month and quarters. Further testing is required to validate that it will work on other dateparts. */ - {% macro last_day(date, datepart) %} - {{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }} +{{ return(adapter.dispatch("last_day", "dbt_utils")(date, datepart)) }} {% endmacro %} {%- macro default_last_day(date, datepart) -%} - cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd(datepart, '1', dbt_utils.date_trunc(datepart, date)) - )}} - as date) +cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd(datepart, "1", dbt_utils.date_trunc(datepart, date)), + ) + }} as date +) {%- endmacro -%} {% macro default__last_day(date, datepart) -%} - {{dbt_utils.default_last_day(date, datepart)}} +{{ dbt_utils.default_last_day(date, datepart) }} {%- endmacro %} {% macro postgres__last_day(date, datepart) -%} - {%- if datepart == 'quarter' -%} - -- postgres dateadd does not support quarter interval. - cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd('month', '3', dbt_utils.date_trunc(datepart, date)) - )}} - as date) - {%- else -%} - {{dbt_utils.default_last_day(date, datepart)}} - {%- endif -%} +{%- if datepart == "quarter" -%} +-- postgres dateadd does not support quarter interval. +cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd("month", "3", dbt_utils.date_trunc(datepart, date)), + ) + }} as date +) +{%- else -%} {{ dbt_utils.default_last_day(date, datepart) }} +{%- endif -%} {%- endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__last_day(date, datepart) %} - {{ return(dbt_utils.default__last_day(date, datepart)) }} +{{ return(dbt_utils.default__last_day(date, datepart)) }} {% endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index 8c2e265c..2afdb89b 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,21 +1,9 @@ {% macro length(expression) -%} - {{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }} +{{ return(adapter.dispatch("length", "dbt_utils")(expression)) }} {% endmacro %} -{% macro default__length(expression) %} - - length( - {{ expression }} - ) - -{%- endmacro -%} +{% macro default__length(expression) %} length({{ expression }}) {%- endmacro -%} -{% macro redshift__length(expression) %} - - len( - {{ expression }} - ) - -{%- endmacro -%} \ No newline at end of file +{% macro redshift__length(expression) %} len({{ expression }}) {%- endmacro -%} diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 1d19a54f..6cf8374e 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -1,104 +1,90 @@ -{% macro listagg(measure, delimiter_text="','", order_by_clause=none, limit_num=none) -%} - {{ return(adapter.dispatch('listagg', 'dbt_utils') (measure, delimiter_text, order_by_clause, limit_num)) }} +{% macro listagg( + measure, delimiter_text="','", order_by_clause=none, limit_num=none +) -%} +{{ + return( + adapter.dispatch("listagg", "dbt_utils")( + measure, delimiter_text, order_by_clause, limit_num + ) + ) +}} {%- endmacro %} {% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - {% if limit_num -%} - array_to_string( - array_slice( - array_agg( - {{ measure }} - ){% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,0 - ,{{ limit_num }} - ), - {{ delimiter_text }} - ) - {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - {%- endif %} +{% if limit_num -%} +array_to_string( + array_slice( + array_agg({{ measure }}) + {% if order_by_clause -%} within group({{ order_by_clause }}) {%- endif %}, + 0, + {{ limit_num }} + ), + {{ delimiter_text }} +) +{%- else %} +listagg({{ measure }}, {{ delimiter_text }}) +{% if order_by_clause -%} within group({{ order_by_clause }}) +{%- endif %} +{%- endif %} {%- endmacro %} {% macro bigquery__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - {% if limit_num -%} - limit {{ limit_num }} - {%- endif %} - ) +string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + {% if limit_num -%} limit {{ limit_num }} {%- endif %} +) {%- endmacro %} {% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - - {% if limit_num -%} - array_to_string( - (array_agg( - {{ measure }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - ))[1:{{ limit_num }}], - {{ delimiter_text }} - ) - {%- else %} - string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} + +{% if limit_num -%} +array_to_string( + ( + array_agg( + {{ measure }} {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} ) + ) [1:{{ limit_num }}], + {{ delimiter_text }} +) +{%- else %} +string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} +) +{%- endif %} {%- endmacro %} {# if there are instances of delimiter_text within your measure, you cannot include a limit_num #} {% macro redshift__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - {% if limit_num -%} - {% set ns = namespace() %} - {% set ns.delimiter_text_regex = delimiter_text|trim("'") %} +{% if limit_num -%} +{% set ns = namespace() %} +{% set ns.delimiter_text_regex = delimiter_text | trim("'") %} {% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} - {%- for char in special_chars.split(',') -%} +{%- for char in special_chars.split(",") -%} {% set escape_char %}\\{{ char }}{% endset %} - {% set ns.delimiter_text_regex = ns.delimiter_text_regex|replace(char,escape_char) %} - {%- endfor -%} +{% set ns.delimiter_text_regex = ns.delimiter_text_regex | replace(char, escape_char) %} +{%- endfor -%} {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} - regexp_substr( - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,{{ regex }} - ) - {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - {%- endif %} +regexp_substr( + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group({{ order_by_clause }}) {%- endif %}, + {{ regex }} +) +{%- else %} +listagg({{ measure }}, {{ delimiter_text }}) +{% if order_by_clause -%} within group({{ order_by_clause }}) +{%- endif %} +{%- endif %} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index c2291467..cf4c577a 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,8 +1,6 @@ {%- macro string_literal(value) -%} - {{ return(adapter.dispatch('string_literal', 'dbt_utils') (value)) }} +{{ return(adapter.dispatch("string_literal", "dbt_utils")(value)) }} {%- endmacro -%} -{% macro default__string_literal(value) -%} - '{{ value }}' -{%- endmacro %} +{% macro default__string_literal(value) -%} '{{ value }}' {%- endmacro %} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index a67b7aa2..b8eac334 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,22 +1,16 @@ {% macro position(substring_text, string_text) -%} - {{ return(adapter.dispatch('position', 'dbt_utils') (substring_text, string_text)) }} +{{ return(adapter.dispatch("position", "dbt_utils")(substring_text, string_text)) }} {% endmacro %} {% macro default__position(substring_text, string_text) %} - position( - {{ substring_text }} in {{ string_text }} - ) - +position({{ substring_text }} in {{ string_text }}) + {%- endmacro -%} {% macro bigquery__position(substring_text, string_text) %} - strpos( - {{ string_text }}, - {{ substring_text }} - - ) - +strpos({{ string_text }}, {{ substring_text }}) + {%- endmacro -%} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index ef27d8fc..46b5fc91 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,15 +1,11 @@ {% macro replace(field, old_chars, new_chars) -%} - {{ return(adapter.dispatch('replace', 'dbt_utils') (field, old_chars, new_chars)) }} +{{ return(adapter.dispatch("replace", "dbt_utils")(field, old_chars, new_chars)) }} {% endmacro %} {% macro default__replace(field, old_chars, new_chars) %} - replace( - {{ field }}, - {{ old_chars }}, - {{ new_chars }} - ) - +replace({{ field }}, {{ old_chars }}, {{ new_chars }}) -{% endmacro %} \ No newline at end of file + +{% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 8ae1640f..379012b7 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,38 +1,29 @@ {% macro right(string_text, length_expression) -%} - {{ return(adapter.dispatch('right', 'dbt_utils') (string_text, length_expression)) }} +{{ return(adapter.dispatch("right", "dbt_utils")(string_text, length_expression)) }} {% endmacro %} {% macro default__right(string_text, length_expression) %} - right( - {{ string_text }}, - {{ length_expression }} - ) - +right({{ string_text }}, {{ length_expression }}) + {%- endmacro -%} {% macro bigquery__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 - then '' - else - substr( - {{ string_text }}, - -1 * ({{ length_expression }}) - ) - end +case + when {{ length_expression }} = 0 + then '' + else substr({{ string_text }}, -1 * ({{ length_expression }})) +end {%- endmacro -%} {% macro snowflake__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 - then '' - else - right( - {{ string_text }}, - {{ length_expression }} - ) - end +case + when {{ length_expression }} = 0 + then '' + else right({{ string_text }}, {{ length_expression }}) +end -{%- endmacro -%} \ No newline at end of file +{%- endmacro -%} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index 8569eecd..cd8dccda 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,20 +1,20 @@ {% macro safe_cast(field, type) %} - {{ return(adapter.dispatch('safe_cast', 'dbt_utils') (field, type)) }} +{{ return(adapter.dispatch("safe_cast", "dbt_utils")(field, type)) }} {% endmacro %} {% macro default__safe_cast(field, type) %} - {# most databases don't support this function yet +{# most databases don't support this function yet so we just need to use cast #} - cast({{field}} as {{type}}) +cast({{ field }} as {{ type }}) {% endmacro %} {% macro snowflake__safe_cast(field, type) %} - try_cast({{field}} as {{type}}) +try_cast({{ field }} as {{ type }}) {% endmacro %} {% macro bigquery__safe_cast(field, type) %} - safe_cast({{field}} as {{type}}) +safe_cast({{ field }} as {{ type }}) {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index 036f7d5a..d4441290 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,24 +1,23 @@ {% macro split_part(string_text, delimiter_text, part_number) %} - {{ return(adapter.dispatch('split_part', 'dbt_utils') (string_text, delimiter_text, part_number)) }} +{{ + return( + adapter.dispatch("split_part", "dbt_utils")( + string_text, delimiter_text, part_number + ) + ) +}} {% endmacro %} {% macro default__split_part(string_text, delimiter_text, part_number) %} - split_part( - {{ string_text }}, - {{ delimiter_text }}, - {{ part_number }} - ) +split_part({{ string_text }}, {{ delimiter_text }}, {{ part_number }}) {% endmacro %} {% macro bigquery__split_part(string_text, delimiter_text, part_number) %} - split( - {{ string_text }}, - {{ delimiter_text }} - )[safe_offset({{ part_number - 1 }})] +split({{ string_text }}, {{ delimiter_text }}) [safe_offset({{ part_number - 1 }})] {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index fc8ae81c..34818584 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,5 +1,11 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} - {{ return(adapter.dispatch('width_bucket', 'dbt_utils') (expr, min_value, max_value, num_buckets)) }} +{{ + return( + adapter.dispatch("width_bucket", "dbt_utils")( + expr, min_value, max_value, num_buckets + ) + ) +}} {% endmacro %} @@ -8,25 +14,20 @@ {% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) {%- endset %} - ( - -- to break ties when the amount is eaxtly at the bucket egde - case - when - mod( - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }}, - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - ) = 0 - then 1 - else 0 - end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 - ) +( + -- to break ties when the amount is eaxtly at the bucket egde + case + when + mod( + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }}, + {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + ) = 0 + then 1 + else 0 + end +) + +-- Anything over max_value goes the N+1 bucket +least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} @@ -34,26 +35,22 @@ {% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) {%- endset %} - ( - -- to break ties when the amount is exactly at the bucket edge - case - when - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }} % - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - = 0 - then 1 - else 0 - end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 - ) +( + -- to break ties when the amount is exactly at the bucket edge + case + when + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }} + % + {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + = 0 + then 1 + else 0 + end +) + +-- Anything over max_value goes the N+1 bucket +least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} - width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }} ) +width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }}) {% endmacro %} diff --git a/macros/generic_tests/accepted_range.sql b/macros/generic_tests/accepted_range.sql index 49ab00d9..68599ba6 100644 --- a/macros/generic_tests/accepted_range.sql +++ b/macros/generic_tests/accepted_range.sql @@ -1,31 +1,42 @@ -{% test accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} - {{ return(adapter.dispatch('test_accepted_range', 'dbt_utils')(model, column_name, min_value, max_value, inclusive)) }} +{% test accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} +{{ + return( + adapter.dispatch("test_accepted_range", "dbt_utils")( + model, column_name, min_value, max_value, inclusive + ) + ) +}} {% endtest %} -{% macro default__test_accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} +{% macro default__test_accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} -with meet_condition as( - select * - from {{ model }} -), +with + meet_condition as (select * from {{ model }}), -validation_errors as ( - select * - from meet_condition - where - -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds - 1 = 2 + validation_errors as ( + select * + from meet_condition + where + -- never true, defaults to an empty result set. Exists to ensure any combo + -- of the `or` clauses below succeeds + 1 = 2 - {%- if min_value is not none %} - -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} - {%- endif %} + {%- if min_value is not none %} + -- records with a value >= min_value are permitted. The `not` flips this + -- to find records that don't meet the rule. + or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} + {%- endif %} - {%- if max_value is not none %} - -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} - {%- endif %} -) + {%- if max_value is not none %} + -- records with a value <= max_value are permitted. The `not` flips this + -- to find records that don't meet the rule. + or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} + {%- endif %} + ) select * from validation_errors diff --git a/macros/generic_tests/at_least_one.sql b/macros/generic_tests/at_least_one.sql index ce02108f..a516e103 100644 --- a/macros/generic_tests/at_least_one.sql +++ b/macros/generic_tests/at_least_one.sql @@ -1,20 +1,20 @@ {% test at_least_one(model, column_name) %} - {{ return(adapter.dispatch('test_at_least_one', 'dbt_utils')(model, column_name)) }} +{{ return(adapter.dispatch("test_at_least_one", "dbt_utils")(model, column_name)) }} {% endtest %} {% macro default__test_at_least_one(model, column_name) %} select * -from ( - select +from + ( {# In TSQL, subquery aggregate columns need aliases #} {# thus: a filler col name, 'filler_column' #} - count({{ column_name }}) as filler_column + select count({{ column_name }}) as filler_column - from {{ model }} + from {{ model }} - having count({{ column_name }}) = 0 + having count({{ column_name }}) = 0 -) validation_errors + ) validation_errors {% endmacro %} diff --git a/macros/generic_tests/cardinality_equality.sql b/macros/generic_tests/cardinality_equality.sql index 4ec7d87f..be319a19 100644 --- a/macros/generic_tests/cardinality_equality.sql +++ b/macros/generic_tests/cardinality_equality.sql @@ -1,51 +1,33 @@ {% test cardinality_equality(model, column_name, to, field) %} - {{ return(adapter.dispatch('test_cardinality_equality', 'dbt_utils')(model, column_name, to, field)) }} +{{ + return( + adapter.dispatch("test_cardinality_equality", "dbt_utils")( + model, column_name, to, field + ) + ) +}} {% endtest %} {% macro default__test_cardinality_equality(model, column_name, to, field) %} {# T-SQL does not let you use numbers as aliases for columns #} {# Thus, no "GROUP BY 1" #} +with + table_a as ( + select {{ column_name }}, count(*) as num_rows + from {{ model }} + group by {{ column_name }} + ), -with table_a as ( -select - {{ column_name }}, - count(*) as num_rows -from {{ model }} -group by {{ column_name }} -), - -table_b as ( -select - {{ field }}, - count(*) as num_rows -from {{ to }} -group by {{ field }} -), - -except_a as ( - select * - from table_a - {{ dbt_utils.except() }} - select * - from table_b -), - -except_b as ( - select * - from table_b - {{ dbt_utils.except() }} - select * - from table_a -), - -unioned as ( - select * - from except_a - union all - select * - from except_b -) + table_b as ( + select {{ field }}, count(*) as num_rows from {{ to }} group by {{ field }} + ), + + except_a as (select * from table_a {{ dbt_utils.except() }} select * from table_b), + + except_b as (select * from table_b {{ dbt_utils.except() }} select * from table_a), + + unioned as (select * from except_a union all select * from except_b) select * from unioned diff --git a/macros/generic_tests/equal_rowcount.sql b/macros/generic_tests/equal_rowcount.sql index 713cb12a..c04363ef 100644 --- a/macros/generic_tests/equal_rowcount.sql +++ b/macros/generic_tests/equal_rowcount.sql @@ -1,38 +1,27 @@ {% test equal_rowcount(model, compare_model) %} - {{ return(adapter.dispatch('test_equal_rowcount', 'dbt_utils')(model, compare_model)) }} +{{ return(adapter.dispatch("test_equal_rowcount", "dbt_utils")(model, compare_model)) }} {% endtest %} {% macro default__test_equal_rowcount(model, compare_model) %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = 'coalesce(diff_count, 0)') }} +{#- - Needs to be set at parse time, before we return '' below --#} +{{ config(fail_calc="coalesce(diff_count, 0)") }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return("") }} {% endif %} -with a as ( +with + a as (select count(*) as count_a from {{ model }}), + b as (select count(*) as count_b from {{ compare_model }}), + final as ( - select count(*) as count_a from {{ model }} + select count_a, count_b, abs(count_a - count_b) as diff_count + from a + cross join b -), -b as ( + ) - select count(*) as count_b from {{ compare_model }} - -), -final as ( - - select - count_a, - count_b, - abs(count_a - count_b) as diff_count - from a - cross join b - -) - -select * from final +select * +from final {% endmacro %} diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index bc302c18..cd3c033f 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -1,5 +1,11 @@ {% test equality(model, compare_model, compare_columns=None) %} - {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns)) }} +{{ + return( + adapter.dispatch("test_equality", "dbt_utils")( + model, compare_model, compare_columns + ) + ) +}} {% endtest %} {% macro default__test_equality(model, compare_model, compare_columns=None) %} @@ -11,65 +17,62 @@ ), 0) {% endset %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = set_diff) }} +{#- - Needs to be set at parse time, before we return '' below --#} +{{ config(fail_calc=set_diff) }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return("") }} {% endif %} -- setup -{%- do dbt_utils._is_relation(model, 'test_equality') -%} +{%- do dbt_utils._is_relation(model, "test_equality") -%} {#- If the compare_cols arg is provided, we can run this test without querying the information schema — this allows the model to be an ephemeral model -#} - {%- if not compare_columns -%} - {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} +{%- do dbt_utils._is_ephemeral(model, "test_equality") -%} +{%- set compare_columns = adapter.get_columns_in_relation(model) | map( + attribute="quoted" +) -%} {%- endif -%} -{% set compare_cols_csv = compare_columns | join(', ') %} - -with a as ( - - select * from {{ model }} - -), - -b as ( +{% set compare_cols_csv = compare_columns | join(", ") %} - select * from {{ compare_model }} +with + a as (select * from {{ model }}), -), + b as (select * from {{ compare_model }}), -a_minus_b as ( + a_minus_b as ( - select {{compare_cols_csv}} from a - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from b + select {{ compare_cols_csv }} + from a {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from b -), + ), -b_minus_a as ( + b_minus_a as ( - select {{compare_cols_csv}} from b - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from a + select {{ compare_cols_csv }} + from b {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from a -), + ), -unioned as ( + unioned as ( - select 'a_minus_b' as which_diff, a_minus_b.* from a_minus_b - union all - select 'b_minus_a' as which_diff, b_minus_a.* from b_minus_a + select 'a_minus_b' as which_diff, a_minus_b.* + from a_minus_b + union all + select 'b_minus_a' as which_diff, b_minus_a.* + from b_minus_a -) + ) -select * from unioned +select * +from unioned {% endmacro %} diff --git a/macros/generic_tests/expression_is_true.sql b/macros/generic_tests/expression_is_true.sql index 611685b4..fdf854d1 100644 --- a/macros/generic_tests/expression_is_true.sql +++ b/macros/generic_tests/expression_is_true.sql @@ -1,22 +1,23 @@ -{% test expression_is_true(model, expression, column_name=None, condition='1=1') %} +{% test expression_is_true(model, expression, column_name=None, condition="1=1") %} {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} {# ref https://stackoverflow.com/a/7170753/3842610 #} - {{ return(adapter.dispatch('test_expression_is_true', 'dbt_utils')(model, expression, column_name, condition)) }} +{{ + return( + adapter.dispatch("test_expression_is_true", "dbt_utils")( + model, expression, column_name, condition + ) + ) +}} {% endtest %} {% macro default__test_expression_is_true(model, expression, column_name, condition) %} -with meet_condition as ( - select * from {{ model }} where {{ condition }} -) +with meet_condition as (select * from {{ model }} where {{ condition }}) -select - * +select * from meet_condition -{% if column_name is none %} -where not({{ expression }}) -{%- else %} -where not({{ column_name }} {{ expression }}) +{% if column_name is none %} where not ({{ expression }}) +{%- else %} where not ({{ column_name }} {{ expression }}) {%- endif %} {% endmacro %} diff --git a/macros/generic_tests/fewer_rows_than.sql b/macros/generic_tests/fewer_rows_than.sql index 450148b7..acd8dd23 100644 --- a/macros/generic_tests/fewer_rows_than.sql +++ b/macros/generic_tests/fewer_rows_than.sql @@ -1,45 +1,39 @@ {% test fewer_rows_than(model, compare_model) %} - {{ return(adapter.dispatch('test_fewer_rows_than', 'dbt_utils')(model, compare_model)) }} +{{ + return( + adapter.dispatch("test_fewer_rows_than", "dbt_utils")(model, compare_model) + ) +}} {% endtest %} {% macro default__test_fewer_rows_than(model, compare_model) %} -{{ config(fail_calc = 'coalesce(row_count_delta, 0)') }} - -with a as ( - - select count(*) as count_our_model from {{ model }} - -), -b as ( - - select count(*) as count_comparison_model from {{ compare_model }} - -), -counts as ( - - select - count_our_model, - count_comparison_model - from a - cross join b - -), -final as ( - - select *, - case - -- fail the test if we have more rows than the reference model and return the row count delta - when count_our_model > count_comparison_model then (count_our_model - count_comparison_model) - -- fail the test if they are the same number - when count_our_model = count_comparison_model then 1 - -- pass the test if the delta is positive (i.e. return the number 0) - else 0 - end as row_count_delta - from counts - -) - -select * from final +{{ config(fail_calc="coalesce(row_count_delta, 0)") }} + +with + a as (select count(*) as count_our_model from {{ model }}), + b as (select count(*) as count_comparison_model from {{ compare_model }}), + counts as (select count_our_model, count_comparison_model from a cross join b), + final as ( + + select + *, + case + -- fail the test if we have more rows than the reference model and + -- return the row count delta + when count_our_model > count_comparison_model + then (count_our_model - count_comparison_model) + -- fail the test if they are the same number + when count_our_model = count_comparison_model + then 1 + -- pass the test if the delta is positive (i.e. return the number 0) + else 0 + end as row_count_delta + from counts + + ) + +select * +from final {% endmacro %} diff --git a/macros/generic_tests/mutually_exclusive_ranges.sql b/macros/generic_tests/mutually_exclusive_ranges.sql index f88efbe4..6206ea01 100644 --- a/macros/generic_tests/mutually_exclusive_ranges.sql +++ b/macros/generic_tests/mutually_exclusive_ranges.sql @@ -1,97 +1,119 @@ -{% test mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} - {{ return(adapter.dispatch('test_mutually_exclusive_ranges', 'dbt_utils')(model, lower_bound_column, upper_bound_column, partition_by, gaps, zero_length_range_allowed)) }} +{% test mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False +) %} +{{ + return( + adapter.dispatch("test_mutually_exclusive_ranges", "dbt_utils")( + model, + lower_bound_column, + upper_bound_column, + partition_by, + gaps, + zero_length_range_allowed, + ) + ) +}} {% endtest %} -{% macro default__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} -{% if gaps == 'not_allowed' %} - {% set allow_gaps_operator='=' %} - {% set allow_gaps_operator_in_words='equal_to' %} -{% elif gaps == 'allowed' %} - {% set allow_gaps_operator='<=' %} - {% set allow_gaps_operator_in_words='less_than_or_equal_to' %} -{% elif gaps == 'required' %} - {% set allow_gaps_operator='<' %} - {% set allow_gaps_operator_in_words='less_than' %} +{% macro default__test_mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False +) %} +{% if gaps == "not_allowed" %} +{% set allow_gaps_operator = "=" %} {% set allow_gaps_operator_in_words = "equal_to" %} +{% elif gaps == "allowed" %} +{% set allow_gaps_operator = "<=" %} +{% set allow_gaps_operator_in_words = "less_than_or_equal_to" %} +{% elif gaps == "required" %} +{% set allow_gaps_operator = "<" %} {% set allow_gaps_operator_in_words = "less_than" %} {% else %} - {{ exceptions.raise_compiler_error( +{{ exceptions.raise_compiler_error( "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" ) }} {% endif %} {% if not zero_length_range_allowed %} - {% set allow_zero_length_operator='<' %} - {% set allow_zero_length_operator_in_words='less_than' %} +{% set allow_zero_length_operator = "<" %} +{% set allow_zero_length_operator_in_words = "less_than" %} {% elif zero_length_range_allowed %} - {% set allow_zero_length_operator='<=' %} - {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %} +{% set allow_zero_length_operator = "<=" %} +{% set allow_zero_length_operator_in_words = "less_than_or_equal_to" %} {% else %} - {{ exceptions.raise_compiler_error( +{{ exceptions.raise_compiler_error( "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length_range_allowed ~"'.'" ) }} {% endif %} {% set partition_clause="partition by " ~ partition_by if partition_by else '' %} -with window_functions as ( - - select - {% if partition_by %} - {{ partition_by }} as partition_by_col, - {% endif %} - {{ lower_bound_column }} as lower_bound, - {{ upper_bound_column }} as upper_bound, - - lead({{ lower_bound_column }}) over ( - {{ partition_clause }} - order by {{ lower_bound_column }} - ) as next_lower_bound, - - row_number() over ( - {{ partition_clause }} - order by {{ lower_bound_column }} desc - ) = 1 as is_last_record - - from {{ model }} - -), - -calc as ( - -- We want to return records where one of our assumptions fails, so we'll use - -- the `not` function with `and` statements so we can write our assumptions nore cleanly - select - *, - - -- For each record: lower_bound should be < upper_bound. - -- Coalesce it to return an error on the null case (implicit assumption - -- these columns are not_null) - coalesce( - lower_bound {{ allow_zero_length_operator }} upper_bound, - false - ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, - - -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. - -- Coalesce it to handle null cases for the last record. - coalesce( - upper_bound {{ allow_gaps_operator }} next_lower_bound, - is_last_record, - false - ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound - - from window_functions - -), - -validation_errors as ( - - select - * - from calc - - where not( - -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound - and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound +with + window_functions as ( + + select + {% if partition_by %} {{ partition_by }} as partition_by_col, {% endif %} + {{ lower_bound_column }} as lower_bound, + {{ upper_bound_column }} as upper_bound, + + lead({{ lower_bound_column }}) over ( + {{ partition_clause }} order by {{ lower_bound_column }} + ) as next_lower_bound, + + row_number() over ( + {{ partition_clause }} order by {{ lower_bound_column }} desc + ) = 1 as is_last_record + + from {{ model }} + + ), + + calc as ( + -- We want to return records where one of our assumptions fails, so we'll use + -- the `not` function with `and` statements so we can write our assumptions + -- nore cleanly + select + *, + + -- For each record: lower_bound should be < upper_bound. + -- Coalesce it to return an error on the null case (implicit assumption + -- these columns are not_null) + coalesce( + lower_bound {{ allow_zero_length_operator }} upper_bound, false + ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, + + -- For each record: upper_bound {{ allow_gaps_operator }} the next + -- lower_bound. + -- Coalesce it to handle null cases for the last record. + coalesce( + upper_bound {{ allow_gaps_operator }} next_lower_bound, + is_last_record, + false + ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + + from window_functions + + ), + + validation_errors as ( + + select * + from calc + + where + not ( + -- THE FOLLOWING SHOULD BE TRUE -- + lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound + and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + ) ) -) -select * from validation_errors +select * +from validation_errors {% endmacro %} diff --git a/macros/generic_tests/not_accepted_values.sql b/macros/generic_tests/not_accepted_values.sql index ab24188f..47d49e97 100644 --- a/macros/generic_tests/not_accepted_values.sql +++ b/macros/generic_tests/not_accepted_values.sql @@ -1,35 +1,33 @@ {% test not_accepted_values(model, column_name, values, quote=True) %} - {{ return(adapter.dispatch('test_not_accepted_values', 'dbt_utils')(model, column_name, values, quote)) }} +{{ + return( + adapter.dispatch("test_not_accepted_values", "dbt_utils")( + model, column_name, values, quote + ) + ) +}} {% endtest %} {% macro default__test_not_accepted_values(model, column_name, values, quote=True) %} -with all_values as ( - - select distinct - {{ column_name }} as value_field - - from {{ model }} +with + all_values as (select distinct {{ column_name }} as value_field from {{ model }}), -), + validation_errors as ( -validation_errors as ( + select value_field - select - value_field - - from all_values - where value_field in ( - {% for value in values -%} - {% if quote -%} - '{{ value }}' - {%- else -%} - {{ value }} - {%- endif -%} - {%- if not loop.last -%},{%- endif %} - {%- endfor %} - ) + from all_values + where + value_field in ( + {% for value in values -%} + {% if quote -%}'{{ value }}' + {%- else -%}{{ value }} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {%- endfor %} + ) -) + ) select * from validation_errors diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index 781ed7d9..842eaf5c 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -1,15 +1,14 @@ {% test not_constant(model, column_name) %} - {{ return(adapter.dispatch('test_not_constant', 'dbt_utils')(model, column_name)) }} +{{ return(adapter.dispatch("test_not_constant", "dbt_utils")(model, column_name)) }} {% endtest %} {% macro default__test_not_constant(model, column_name) %} -select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count(distinct {{ column_name }}) as filler_column +{# In TSQL, subquery aggregate columns need aliases #} +{# thus: a filler col name, 'filler_column' #} +select count(distinct {{ column_name }}) as filler_column from {{ model }} diff --git a/macros/generic_tests/not_null_proportion.sql b/macros/generic_tests/not_null_proportion.sql index 45b9050c..37df9a84 100644 --- a/macros/generic_tests/not_null_proportion.sql +++ b/macros/generic_tests/not_null_proportion.sql @@ -1,26 +1,28 @@ {% macro test_not_null_proportion(model) %} - {{ return(adapter.dispatch('test_not_null_proportion', 'dbt_utils')(model, **kwargs)) }} +{{ return(adapter.dispatch("test_not_null_proportion", "dbt_utils")(model, **kwargs)) }} {% endmacro %} {% macro default__test_not_null_proportion(model) %} -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} -{% set at_least = kwargs.get('at_least', kwargs.get('arg')) %} -{% set at_most = kwargs.get('at_most', kwargs.get('arg', 1)) %} +{% set column_name = kwargs.get("column_name", kwargs.get("arg")) %} +{% set at_least = kwargs.get("at_least", kwargs.get("arg")) %} +{% set at_most = kwargs.get("at_most", kwargs.get("arg", 1)) %} -with validation as ( - select - sum(case when {{ column_name }} is null then 0 else 1 end) / cast(count(*) as numeric) as not_null_proportion - from {{ model }} -), -validation_errors as ( - select - not_null_proportion - from validation - where not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} -) -select - * +with + validation as ( + select + sum(case when {{ column_name }} is null then 0 else 1 end) / cast( + count(*) as numeric + ) as not_null_proportion + from {{ model }} + ), + validation_errors as ( + select not_null_proportion + from validation + where + not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} + ) +select * from validation_errors {% endmacro %} diff --git a/macros/generic_tests/recency.sql b/macros/generic_tests/recency.sql index cb9a8de6..4005c9e7 100644 --- a/macros/generic_tests/recency.sql +++ b/macros/generic_tests/recency.sql @@ -1,22 +1,20 @@ {% test recency(model, field, datepart, interval) %} - {{ return(adapter.dispatch('test_recency', 'dbt_utils')(model, field, datepart, interval)) }} +{{ + return( + adapter.dispatch("test_recency", "dbt_utils")(model, field, datepart, interval) + ) +}} {% endtest %} {% macro default__test_recency(model, field, datepart, interval) %} -{% set threshold = dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp()) %} +{% set threshold = dbt_utils.dateadd( + datepart, interval * -1, dbt_utils.current_timestamp() +) %} -with recency as ( +with recency as (select max({{ field }}) as most_recent from {{ model }}) - select max({{field}}) as most_recent - from {{ model }} - -) - -select - - most_recent, - {{ threshold }} as threshold +select most_recent, {{ threshold }} as threshold from recency where most_recent < {{ threshold }} diff --git a/macros/generic_tests/relationships_where.sql b/macros/generic_tests/relationships_where.sql index c35a380a..16fbaf7c 100644 --- a/macros/generic_tests/relationships_where.sql +++ b/macros/generic_tests/relationships_where.sql @@ -1,51 +1,55 @@ -{% test relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} - {{ return(adapter.dispatch('test_relationships_where', 'dbt_utils')(model, column_name, to, field, from_condition, to_condition)) }} +{% test relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} +{{ + return( + adapter.dispatch("test_relationships_where", "dbt_utils")( + model, column_name, to, field, from_condition, to_condition + ) + ) +}} {% endtest %} -{% macro default__test_relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} +{% macro default__test_relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} {# ref https://stackoverflow.com/a/7170753/3842610 #} +with + left_table as ( -with left_table as ( + select {{ column_name }} as id - select - {{column_name}} as id + from {{ model }} - from {{model}} + where {{ column_name }} is not null and {{ from_condition }} - where {{column_name}} is not null - and {{from_condition}} + ), -), + right_table as ( -right_table as ( + select {{ field }} as id - select - {{field}} as id + from {{ to }} - from {{to}} + where {{ field }} is not null and {{ to_condition }} - where {{field}} is not null - and {{to_condition}} + ), -), + exceptions as ( -exceptions as ( + select left_table.id, right_table.id as right_id - select - left_table.id, - right_table.id as right_id + from left_table - from left_table + left join right_table on left_table.id = right_table.id - left join right_table - on left_table.id = right_table.id + where right_table.id is null - where right_table.id is null + ) -) - -select * from exceptions +select * +from exceptions {% endmacro %} diff --git a/macros/generic_tests/sequential_values.sql b/macros/generic_tests/sequential_values.sql index 8ddae707..4693373c 100644 --- a/macros/generic_tests/sequential_values.sql +++ b/macros/generic_tests/sequential_values.sql @@ -1,33 +1,47 @@ {% test sequential_values(model, column_name, interval=1, datepart=None) %} - {{ return(adapter.dispatch('test_sequential_values', 'dbt_utils')(model, column_name, interval, datepart)) }} +{{ + return( + adapter.dispatch("test_sequential_values", "dbt_utils")( + model, column_name, interval, datepart + ) + ) +}} {% endtest %} -{% macro default__test_sequential_values(model, column_name, interval=1, datepart=None) %} +{% macro default__test_sequential_values( + model, column_name, interval=1, datepart=None +) %} {% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} -with windowed as ( - - select - {{ column_name }}, - lag({{ column_name }}) over ( - order by {{ column_name }} - ) as {{ previous_column_name }} - from {{ model }} -), - -validation_errors as ( - select - * - from windowed - {% if datepart %} - where not(cast({{ column_name }} as {{ dbt_utils.type_timestamp() }})= cast({{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} as {{ dbt_utils.type_timestamp() }})) - {% else %} - where not({{ column_name }} = {{ previous_column_name }} + {{ interval }}) - {% endif %} -) +with + windowed as ( + + select + {{ column_name }}, + lag({{ column_name }}) over ( + order by {{ column_name }} + ) as {{ previous_column_name }} + from {{ model }} + ), + + validation_errors as ( + select * + from windowed + {% if datepart %} + where + not ( + cast({{ column_name }} as {{ dbt_utils.type_timestamp() }}) = cast( + {{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} + as {{ dbt_utils.type_timestamp() }} + ) + ) + {% else %} + where not ({{ column_name }} = {{ previous_column_name }} + {{ interval }}) + {% endif %} + ) select * from validation_errors diff --git a/macros/generic_tests/test_not_null_where.sql b/macros/generic_tests/test_not_null_where.sql index d5dbf6c5..9b602a78 100644 --- a/macros/generic_tests/test_not_null_where.sql +++ b/macros/generic_tests/test_not_null_where.sql @@ -1,12 +1,9 @@ -{% test not_null_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.not_null_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_not_null_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_not_null_where(model, column_name) %} - {{ return(test_not_null(model, column_name)) }} -{% endmacro %} +{% test not_null_where(model, column_name) %} +{%- set deprecation_warning = " Warning: `dbt_utils.not_null_where` is no longer supported. Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. " -%} +{%- do exceptions.warn(deprecation_warning) -%} +{{ return(adapter.dispatch("test_not_null_where", "dbt_utils")(model, column_name)) }} +{% endtest %} + +{% macro default__test_not_null_where(model, column_name) %} +{{ return(test_not_null(model, column_name)) }} +{% endmacro %} diff --git a/macros/generic_tests/test_unique_where.sql b/macros/generic_tests/test_unique_where.sql index e752d7b2..73442631 100644 --- a/macros/generic_tests/test_unique_where.sql +++ b/macros/generic_tests/test_unique_where.sql @@ -1,12 +1,9 @@ -{% test unique_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.unique_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_unique_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_unique_where(model, column_name) %} - {{ return(test_unique(model, column_name)) }} -{% endmacro %} +{% test unique_where(model, column_name) %} +{%- set deprecation_warning = " Warning: `dbt_utils.unique_where` is no longer supported. Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. " -%} +{%- do exceptions.warn(deprecation_warning) -%} +{{ return(adapter.dispatch("test_unique_where", "dbt_utils")(model, column_name)) }} +{% endtest %} + +{% macro default__test_unique_where(model, column_name) %} +{{ return(test_unique(model, column_name)) }} +{% endmacro %} diff --git a/macros/generic_tests/unique_combination_of_columns.sql b/macros/generic_tests/unique_combination_of_columns.sql index 74ccf5c6..97b8ccd5 100644 --- a/macros/generic_tests/unique_combination_of_columns.sql +++ b/macros/generic_tests/unique_combination_of_columns.sql @@ -1,34 +1,43 @@ -{% test unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} - {{ return(adapter.dispatch('test_unique_combination_of_columns', 'dbt_utils')(model, combination_of_columns, quote_columns)) }} +{% test unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} +{{ + return( + adapter.dispatch("test_unique_combination_of_columns", "dbt_utils")( + model, combination_of_columns, quote_columns + ) + ) +}} {% endtest %} -{% macro default__test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} +{% macro default__test_unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} -{% if not quote_columns %} - {%- set column_list=combination_of_columns %} +{% if not quote_columns %} {%- set column_list = combination_of_columns %} {% elif quote_columns %} - {%- set column_list=[] %} - {% for column in combination_of_columns -%} - {% set column_list = column_list.append( adapter.quote(column) ) %} - {%- endfor %} +{%- set column_list = [] %} +{% for column in combination_of_columns -%} +{% set column_list = column_list.append(adapter.quote(column)) %} +{%- endfor %} {% else %} - {{ exceptions.raise_compiler_error( +{{ exceptions.raise_compiler_error( "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" ) }} {% endif %} -{%- set columns_csv=column_list | join(', ') %} +{%- set columns_csv = column_list | join(", ") %} -with validation_errors as ( +with + validation_errors as ( - select - {{ columns_csv }} - from {{ model }} - group by {{ columns_csv }} - having count(*) > 1 + select {{ columns_csv }} + from {{ model }} + group by {{ columns_csv }} + having count(*) > 1 -) + ) select * from validation_errors diff --git a/macros/jinja_helpers/log_info.sql b/macros/jinja_helpers/log_info.sql index 52b4b4a5..44fe5d1c 100644 --- a/macros/jinja_helpers/log_info.sql +++ b/macros/jinja_helpers/log_info.sql @@ -1,7 +1,7 @@ {% macro log_info(message) %} - {{ return(adapter.dispatch('log_info', 'dbt_utils')(message)) }} +{{ return(adapter.dispatch("log_info", "dbt_utils")(message)) }} {% endmacro %} {% macro default__log_info(message) %} - {{ log(dbt_utils.pretty_log_format(message), info=True) }} +{{ log(dbt_utils.pretty_log_format(message), info=True) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_log_format.sql b/macros/jinja_helpers/pretty_log_format.sql index fe580d13..d5d6ea6c 100644 --- a/macros/jinja_helpers/pretty_log_format.sql +++ b/macros/jinja_helpers/pretty_log_format.sql @@ -1,7 +1,7 @@ {% macro pretty_log_format(message) %} - {{ return(adapter.dispatch('pretty_log_format', 'dbt_utils')(message)) }} +{{ return(adapter.dispatch("pretty_log_format", "dbt_utils")(message)) }} {% endmacro %} {% macro default__pretty_log_format(message) %} - {{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} +{{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_time.sql b/macros/jinja_helpers/pretty_time.sql index bad37efe..a342b9ce 100644 --- a/macros/jinja_helpers/pretty_time.sql +++ b/macros/jinja_helpers/pretty_time.sql @@ -1,7 +1,7 @@ -{% macro pretty_time(format='%H:%M:%S') %} - {{ return(adapter.dispatch('pretty_time', 'dbt_utils')(format)) }} +{% macro pretty_time(format="%H:%M:%S") %} +{{ return(adapter.dispatch("pretty_time", "dbt_utils")(format)) }} {% endmacro %} -{% macro default__pretty_time(format='%H:%M:%S') %} - {{ return(modules.datetime.datetime.now().strftime(format)) }} +{% macro default__pretty_time(format="%H:%M:%S") %} +{{ return(modules.datetime.datetime.now().strftime(format)) }} {% endmacro %} diff --git a/macros/jinja_helpers/slugify.sql b/macros/jinja_helpers/slugify.sql index 1b3c7272..01cce92d 100644 --- a/macros/jinja_helpers/slugify.sql +++ b/macros/jinja_helpers/slugify.sql @@ -3,9 +3,9 @@ {#- Lower case the string -#} {% set string = string | lower %} {#- Replace spaces and dashes with underscores -#} -{% set string = modules.re.sub('[ -]+', '_', string) %} +{% set string = modules.re.sub("[ -]+", "_", string) %} {#- Only take letters, numbers, and underscores -#} -{% set string = modules.re.sub('[^a-z0-9_]+', '', string) %} +{% set string = modules.re.sub("[^a-z0-9_]+", "", string) %} {{ return(string) }} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 5a15c815..e6c328cf 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -1,38 +1,88 @@ -{% macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - {{ return(adapter.dispatch('get_period_boundaries', 'dbt_utils')(target_schema, target_table, timestamp_field, start_date, stop_date, period)) }} +{% macro get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period +) -%} +{{ + return( + adapter.dispatch("get_period_boundaries", "dbt_utils")( + target_schema, target_table, timestamp_field, start_date, stop_date, period + ) + ) +}} {% endmacro %} -{% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - - {% call statement('period_boundaries', fetch_result=True) -%} - with data as ( - select - coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp as start_timestamp, - coalesce( - {{dbt_utils.dateadd('millisecond', +{% macro default__get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period +) -%} + +{% call statement('period_boundaries', fetch_result=True) -%} +with + data as ( + select + coalesce( + max("{{timestamp_field}}"), '{{start_date}}' + )::timestamp as start_timestamp, + coalesce( + {{dbt_utils.dateadd('millisecond', -1, "nullif('" ~ stop_date ~ "','')::timestamp")}}, - {{dbt_utils.current_timestamp()}} - ) as stop_timestamp - from "{{target_schema}}"."{{target_table}}" + {{ dbt_utils.current_timestamp() }} + ) as stop_timestamp + from "{{target_schema}}"."{{target_table}}" ) - select - start_timestamp, - stop_timestamp, - {{dbt_utils.datediff('start_timestamp', - 'stop_timestamp', - period)}} + 1 as num_periods - from data - {%- endcall %} +select + start_timestamp, + stop_timestamp, + {{ dbt_utils.datediff("start_timestamp", "stop_timestamp", period) }} + + 1 + as num_periods +from data {%- endcall %} {%- endmacro %} -{% macro get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} - {{ return(adapter.dispatch('get_period_sql', 'dbt_utils')(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset)) }} -{% endmacro %} - -{% macro default__get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} + {% macro get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset +) -%} + {{ + return( + adapter.dispatch("get_period_sql", "dbt_utils")( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset, + ) + ) + }} + {% endmacro %} + +{% macro default__get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset +) -%} {%- set period_filter -%} ("{{timestamp_field}}" > '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' and @@ -40,150 +90,159 @@ "{{timestamp_field}}" < '{{stop_timestamp}}'::timestamp) {%- endset -%} - {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} + {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} - select - {{target_cols_csv}} - from ( - {{filtered_sql}} - ) +select {{ target_cols_csv }} +from ({{ filtered_sql }}) {%- endmacro %} -{% materialization insert_by_period, default -%} - {%- set timestamp_field = config.require('timestamp_field') -%} - {%- set start_date = config.require('start_date') -%} - {%- set stop_date = config.get('stop_date') or '' -%} - {%- set period = config.get('period') or 'week' -%} + {% materialization insert_by_period, default -%} + {%- set timestamp_field = config.require("timestamp_field") -%} + {%- set start_date = config.require("start_date") -%} + {%- set stop_date = config.get("stop_date") or "" -%} + {%- set period = config.get("period") or "week" -%} - {%- if sql.find('__PERIOD_FILTER__') == -1 -%} + {%- if sql.find("__PERIOD_FILTER__") == -1 -%} {%- set error_message -%} Model '{{ model.unique_id }}' does not include the required string '__PERIOD_FILTER__' in its sql {%- endset -%} {{ exceptions.raise_compiler_error(error_message) }} - {%- endif -%} - - {%- set identifier = model['name'] -%} - - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} - - {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - - {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} - {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} + {%- endif -%} + + {%- set identifier = model["name"] -%} + + {%- set old_relation = adapter.get_relation( + database=database, schema=schema, identifier=identifier +) -%} + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, type="table" +) -%} + + {%- set non_destructive_mode = flags.NON_DESTRUCTIVE == True -%} + {%- set full_refresh_mode = flags.FULL_REFRESH == True -%} + + {%- set exists_as_table = old_relation is not none and old_relation.is_table -%} + {%- set exists_not_as_table = ( + old_relation is not none and not old_relation.is_table +) -%} + + {%- set should_truncate = ( + non_destructive_mode and full_refresh_mode and exists_as_table +) -%} + {%- set should_drop = not should_truncate and ( + full_refresh_mode or exists_not_as_table +) -%} + {%- set force_create = flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE -%} + + -- setup + {% if old_relation is none -%} + -- noop + {%- elif should_truncate -%} {{ adapter.truncate_relation(old_relation) }} + {%- elif should_drop -%} + {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} + {%- endif %} - {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%} - {%- set should_drop = (not should_truncate and (full_refresh_mode or exists_not_as_table)) -%} - {%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%} + {{ run_hooks(pre_hooks, inside_transaction=False) }} - -- setup - {% if old_relation is none -%} - -- noop - {%- elif should_truncate -%} - {{adapter.truncate_relation(old_relation)}} - {%- elif should_drop -%} - {{adapter.drop_relation(old_relation)}} - {%- set old_relation = none -%} - {%- endif %} - - {{run_hooks(pre_hooks, inside_transaction=False)}} - - -- `begin` happens here, so `commit` after it to finish the transaction - {{run_hooks(pre_hooks, inside_transaction=True)}} - {% call statement() -%} - begin; -- make extra sure we've closed out the transaction - commit; - {%- endcall %} - - -- build model - {% if force_create or old_relation is none -%} - {# Create an empty target table -#} - {% call statement('main') -%} - {%- set empty_sql = sql | replace("__PERIOD_FILTER__", 'false') -%} - {{create_table_as(False, target_relation, empty_sql)}} - {%- endcall %} - {%- endif %} - - {% set _ = dbt_utils.get_period_boundaries(schema, - identifier, - timestamp_field, - start_date, - stop_date, - period) %} - {%- set start_timestamp = load_result('period_boundaries')['data'][0][0] | string -%} - {%- set stop_timestamp = load_result('period_boundaries')['data'][0][1] | string -%} - {%- set num_periods = load_result('period_boundaries')['data'][0][2] | int -%} - - {% set target_columns = adapter.get_columns_in_relation(target_relation) %} - {%- set target_cols_csv = target_columns | map(attribute='quoted') | join(', ') -%} - {%- set loop_vars = {'sum_rows_inserted': 0} -%} - - -- commit each period as a separate transaction - {% for i in range(num_periods) -%} - {%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} - {{ dbt_utils.log_info(msg) }} - - {%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%} - {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, - schema=schema, type='table') -%} + -- `begin` happens here, so `commit` after it to finish the transaction + {{ run_hooks(pre_hooks, inside_transaction=True) }} {% call statement() -%} - {% set tmp_table_sql = dbt_utils.get_period_sql(target_cols_csv, - sql, - timestamp_field, - period, - start_timestamp, - stop_timestamp, - i) %} - {{dbt.create_table_as(True, tmp_relation, tmp_table_sql)}} - {%- endcall %} - - {{adapter.expand_target_column_types(from_relation=tmp_relation, - to_relation=target_relation)}} - {%- set name = 'main-' ~ i -%} - {% call statement(name, fetch_result=True) -%} - insert into {{target_relation}} ({{target_cols_csv}}) - ( - select - {{target_cols_csv}} - from {{tmp_relation.include(schema=False)}} - ); - {%- endcall %} - {% set result = load_result('main-' ~ i) %} - {% if 'response' in result.keys() %} {# added in v0.19.0 #} - {% set rows_inserted = result['response']['rows_affected'] %} - {% else %} {# older versions #} - {% set rows_inserted = result['status'].split(" ")[2] | int %} - {% endif %} - - {%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%} - {%- if loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %} {% endif -%} - - {%- set msg = "Ran for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) ~ "; " ~ rows_inserted ~ " records inserted" -%} - {{ dbt_utils.log_info(msg) }} - - {%- endfor %} - - {% call statement() -%} - begin; - {%- endcall %} - - {{run_hooks(post_hooks, inside_transaction=True)}} - - {% call statement() -%} - commit; - {%- endcall %} - - {{run_hooks(post_hooks, inside_transaction=False)}} - - {%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} - - {% call noop_statement('main', status_string) -%} - -- no-op - {%- endcall %} - - -- Return the relations created in this materialization - {{ return({'relations': [target_relation]}) }} + begin -- make extra sure we've closed out the transaction +; +commit +; +{%- endcall %} + +-- build model +{% if force_create or old_relation is none -%} +{# Create an empty target table -#} +{% call statement('main') -%} +{%- set empty_sql = sql | replace("__PERIOD_FILTER__", "false") -%} +{{ create_table_as(False, target_relation, empty_sql) }} +{%- endcall %} +{%- endif %} + +{% set _ = dbt_utils.get_period_boundaries( + schema, identifier, timestamp_field, start_date, stop_date, period +) %} +{%- set start_timestamp = load_result("period_boundaries")["data"][0][0] | string -%} +{%- set stop_timestamp = load_result("period_boundaries")["data"][0][1] | string -%} +{%- set num_periods = load_result("period_boundaries")["data"][0][2] | int -%} + +{% set target_columns = adapter.get_columns_in_relation(target_relation) %} +{%- set target_cols_csv = target_columns | map(attribute="quoted") | join(", ") -%} +{%- set loop_vars = {"sum_rows_inserted": 0} -%} + +-- commit each period as a separate transaction +{% for i in range(num_periods) -%} +{%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} +{{ dbt_utils.log_info(msg) }} + +{%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%} +{%- set tmp_relation = api.Relation.create( + identifier=tmp_identifier, schema=schema, type="table" +) -%} +{% call statement() -%} +{% set tmp_table_sql = dbt_utils.get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + i, +) %} +{{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} +{%- endcall %} + +{{ + adapter.expand_target_column_types( + from_relation=tmp_relation, to_relation=target_relation + ) +}} +{%- set name = 'main-' ~ i -%} +{% call statement(name, fetch_result=True) -%} +insert into {{ target_relation }} ({{ target_cols_csv }}) +(select {{ target_cols_csv }} from {{ tmp_relation.include(schema=False) }}) +; +{%- endcall %} +{% set result = load_result('main-' ~ i) %} +{% if "response" in result.keys() %} {# added in v0.19.0 #} +{% set rows_inserted = result["response"]["rows_affected"] %} +{# older versions #} +{% else %} {% set rows_inserted = result["status"].split(" ")[2] | int %} +{% endif %} + +{%- set sum_rows_inserted = loop_vars["sum_rows_inserted"] + rows_inserted -%} +{%- if loop_vars.update({"sum_rows_inserted": sum_rows_inserted}) %} {% endif -%} + +{%- set msg = "Ran for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) ~ "; " ~ rows_inserted ~ " records inserted" -%} +{{ dbt_utils.log_info(msg) }} + +{%- endfor %} + +{% call statement() -%} +begin +; +{%- endcall %} + +{{ run_hooks(post_hooks, inside_transaction=True) }} + +{% call statement() -%} +commit +; +{%- endcall %} + +{{ run_hooks(post_hooks, inside_transaction=False) }} + +{%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} + +{% call noop_statement('main', status_string) -%} +-- no-op +{%- endcall %} + +-- Return the relations created in this materialization +{{ return({"relations": [target_relation]}) }} {%- endmaterialization %} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index 759f8d59..b256dfde 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -1,21 +1,27 @@ {% macro get_intervals_between(start_date, end_date, datepart) -%} - {{ return(adapter.dispatch('get_intervals_between', 'dbt_utils')(start_date, end_date, datepart)) }} +{{ + return( + adapter.dispatch("get_intervals_between", "dbt_utils")( + start_date, end_date, datepart + ) + ) +}} {%- endmacro %} {% macro default__get_intervals_between(start_date, end_date, datepart) -%} - {%- call statement('get_intervals_between', fetch_result=True) %} +{%- call statement('get_intervals_between', fetch_result=True) %} - select {{dbt_utils.datediff(start_date, end_date, datepart)}} +select + {{ dbt_utils.datediff(start_date, end_date, datepart) }} {%- endcall -%} - {%- set value_list = load_result('get_intervals_between') -%} + {%- set value_list = load_result("get_intervals_between") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} - {{ return(values[0]) }} - {%- else -%} - {{ return(1) }} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} + {{ return(values[0]) }} + {%- else -%} {{ return(1) }} {%- endif -%} {%- endmacro %} @@ -23,9 +29,13 @@ -{% macro date_spine(datepart, start_date, end_date) %} - {{ return(adapter.dispatch('date_spine', 'dbt_utils')(datepart, start_date, end_date)) }} -{%- endmacro %} + {% macro date_spine(datepart, start_date, end_date) %} + {{ + return( + adapter.dispatch("date_spine", "dbt_utils")(datepart, start_date, end_date) + ) + }} + {%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} @@ -37,39 +47,34 @@ date_spine( "to_date('01/01/2016', 'mm/dd/yyyy')", "dateadd(week, 1, current_date)" ) #} +with + rawdata as ( - -with rawdata as ( - - {{dbt_utils.generate_series( - dbt_utils.get_intervals_between(start_date, end_date, datepart) - )}} - -), - -all_periods as ( - - select ( {{ - dbt_utils.dateadd( - datepart, - "row_number() over (order by 1) - 1", - start_date + dbt_utils.generate_series( + dbt_utils.get_intervals_between(start_date, end_date, datepart) ) }} - ) as date_{{datepart}} - from rawdata -), + ), + + all_periods as ( -filtered as ( + select + ( + {{ + dbt_utils.dateadd( + datepart, "row_number() over (order by 1) - 1", start_date + ) + }} + ) as date_{{ datepart }} + from rawdata - select * - from all_periods - where date_{{datepart}} <= {{ end_date }} + ), -) + filtered as (select * from all_periods where date_{{ datepart }} <= {{ end_date }}) -select * from filtered +select * +from filtered {% endmacro %} diff --git a/macros/sql/deduplicate.sql b/macros/sql/deduplicate.sql index 9a3571a2..949a15b2 100644 --- a/macros/sql/deduplicate.sql +++ b/macros/sql/deduplicate.sql @@ -1,23 +1,29 @@ {%- macro deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} - {{ return(adapter.dispatch('deduplicate', 'dbt_utils')(relation, group_by, order_by=order_by, relation_alias=relation_alias)) }} +{{ + return( + adapter.dispatch("deduplicate", "dbt_utils")( + relation, group_by, order_by=order_by, relation_alias=relation_alias + ) + ) +}} {% endmacro %} -{%- macro default__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro default__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( +select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} +from + ( select _inner.*, row_number() over ( partition by {{ group_by }} - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} ) as rn from {{ relation if relation_alias is none else relation_alias }} as _inner ) as deduped - where deduped.rn = 1 +where deduped.rn = 1 {%- endmacro -%} @@ -26,19 +32,19 @@ -- clause in BigQuery: -- https://github.com/dbt-labs/dbt-utils/issues/335#issuecomment-788157572 #} -{%- macro bigquery__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro bigquery__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( +select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} +from + ( select - array_agg ( + array_agg( original - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} limit 1 - )[offset(0)] as deduped + ) [offset (0)] as deduped from {{ relation if relation_alias is none else relation_alias }} as original group by {{ group_by }} ) diff --git a/macros/sql/generate_series.sql b/macros/sql/generate_series.sql index efcbd8ac..f2b1a76e 100644 --- a/macros/sql/generate_series.sql +++ b/macros/sql/generate_series.sql @@ -1,53 +1,50 @@ {% macro get_powers_of_two(upper_bound) %} - {{ return(adapter.dispatch('get_powers_of_two', 'dbt_utils')(upper_bound)) }} +{{ return(adapter.dispatch("get_powers_of_two", "dbt_utils")(upper_bound)) }} {% endmacro %} {% macro default__get_powers_of_two(upper_bound) %} - {% if upper_bound <= 0 %} - {{ exceptions.raise_compiler_error("upper bound must be positive") }} - {% endif %} +{% if upper_bound <= 0 %} +{{ exceptions.raise_compiler_error("upper bound must be positive") }} +{% endif %} - {% for _ in range(1, 100) %} - {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} - {% endfor %} +{% for _ in range(1, 100) %} +{% if upper_bound <= 2**loop.index %} {{ return(loop.index) }}{% endif %} +{% endfor %} {% endmacro %} {% macro generate_series(upper_bound) %} - {{ return(adapter.dispatch('generate_series', 'dbt_utils')(upper_bound)) }} +{{ return(adapter.dispatch("generate_series", "dbt_utils")(upper_bound)) }} {% endmacro %} {% macro default__generate_series(upper_bound) %} - {% set n = dbt_utils.get_powers_of_two(upper_bound) %} +{% set n = dbt_utils.get_powers_of_two(upper_bound) %} - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( +with + p as (select 0 as generated_number union all select 1), + unioned as ( - select + select - {% for i in range(n) %} - p{{i}}.generated_number * power(2, {{i}}) - {% if not loop.last %} + {% endif %} - {% endfor %} - + 1 - as generated_number + {% for i in range(n) %} + p{{ i }}.generated_number * power(2, {{ i }}) + {% if not loop.last %} + {% endif %} + {% endfor %} + 1 as generated_number - from + from - {% for i in range(n) %} - p as p{{i}} - {% if not loop.last %} cross join {% endif %} - {% endfor %} + {% for i in range(n) %} + p as p{{ i }} {% if not loop.last %} cross join {% endif %} + {% endfor %} ) - select * - from unioned - where generated_number <= {{upper_bound}} - order by generated_number +select * +from unioned +where generated_number <= {{ upper_bound }} +order by generated_number {% endmacro %} diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index f70890e2..5c4d0ac3 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -1,60 +1,66 @@ -{% macro get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {{ return(adapter.dispatch('get_column_values', 'dbt_utils')(table, column, order_by, max_records, default)) }} +{% macro get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} +{{ + return( + adapter.dispatch("get_column_values", "dbt_utils")( + table, column, order_by, max_records, default + ) + ) +}} {% endmacro %} -{% macro default__get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {% set default = [] if not default %} - {{ return(default) }} - {% endif %} +{% macro default__get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} +{% set default = [] if not default %} {{ return(default) }} +{% endif %} - {%- do dbt_utils._is_ephemeral(table, 'get_column_values') -%} +{%- do dbt_utils._is_ephemeral(table, "get_column_values") -%} - {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} - {# TODO: Change the method signature in a future 0.x.0 release #} - {%- set target_relation = table -%} +{# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} +{# TODO: Change the method signature in a future 0.x.0 release #} +{%- set target_relation = table -%} - {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} - {% set relation_exists = (load_relation(target_relation)) is not none %} +{# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} +{% set relation_exists = (load_relation(target_relation)) is not none %} - {%- call statement('get_column_values', fetch_result=true) %} +{%- call statement('get_column_values', fetch_result=true) %} - {%- if not relation_exists and default is none -%} +{%- if not relation_exists and default is none -%} - {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} +{{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} - {%- elif not relation_exists and default is not none -%} +{%- elif not relation_exists and default is not none -%} - {{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} +{{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} - {{ return(default) }} +{{ return(default) }} - {%- else -%} +{%- else -%} - select - {{ column }} as value +select {{ column }} as value - from {{ target_relation }} - group by {{ column }} - order by {{ order_by }} +from {{ target_relation }} +group by {{ column }} +order by {{ order_by }} - {% if max_records is not none %} - limit {{ max_records }} - {% endif %} +{% if max_records is not none %} limit {{ max_records }} +{% endif %} - {% endif %} +{% endif %} {%- endcall -%} - {%- set value_list = load_result('get_column_values') -%} + {%- set value_list = load_result("get_column_values") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} - {{ return(values) }} - {%- else -%} - {{ return(default) }} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} + {{ return(values) }} + {%- else -%} {{ return(default) }} {%- endif -%} {%- endmacro %} diff --git a/macros/sql/get_filtered_columns_in_relation.sql b/macros/sql/get_filtered_columns_in_relation.sql index 7f4af889..454fd18c 100644 --- a/macros/sql/get_filtered_columns_in_relation.sql +++ b/macros/sql/get_filtered_columns_in_relation.sql @@ -1,25 +1,23 @@ {% macro get_filtered_columns_in_relation(from, except=[]) -%} - {{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} +{{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} {% endmacro %} {% macro default__get_filtered_columns_in_relation(from, except=[]) -%} - {%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} - {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} +{%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} +{%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} - {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('') }} - {% endif %} +{# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return("") }} {% endif %} - {%- set include_cols = [] %} - {%- set cols = adapter.get_columns_in_relation(from) -%} - {%- set except = except | map("lower") | list %} - {%- for col in cols -%} - {%- if col.column|lower not in except -%} - {% do include_cols.append(col.column) %} - {%- endif %} - {%- endfor %} +{%- set include_cols = [] %} +{%- set cols = adapter.get_columns_in_relation(from) -%} +{%- set except = except | map("lower") | list %} +{%- for col in cols -%} +{%- if col.column|lower not in except -%} +{% do include_cols.append(col.column) %} +{%- endif %} +{%- endfor %} - {{ return(include_cols) }} +{{ return(include_cols) }} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/sql/get_query_results_as_dict.sql b/macros/sql/get_query_results_as_dict.sql index 6548f2dd..b4fc3e61 100644 --- a/macros/sql/get_query_results_as_dict.sql +++ b/macros/sql/get_query_results_as_dict.sql @@ -1,26 +1,25 @@ {% macro get_query_results_as_dict(query) %} - {{ return(adapter.dispatch('get_query_results_as_dict', 'dbt_utils')(query)) }} +{{ return(adapter.dispatch("get_query_results_as_dict", "dbt_utils")(query)) }} {% endmacro %} {% macro default__get_query_results_as_dict(query) %} {# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} +{%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} - {%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} +{{ query }} - {{ query }} +{%- endcall -%} - {%- endcall -%} +{% set sql_results = {} %} - {% set sql_results={} %} +{%- if execute -%} +{% set sql_results_table = load_result("get_query_results").table.columns %} +{% for column_name, column in sql_results_table.items() %} +{% do sql_results.update({column_name: column.values()}) %} +{% endfor %} +{%- endif -%} - {%- if execute -%} - {% set sql_results_table = load_result('get_query_results').table.columns %} - {% for column_name, column in sql_results_table.items() %} - {% do sql_results.update({column_name: column.values()}) %} - {% endfor %} - {%- endif -%} - - {{ return(sql_results) }} +{{ return(sql_results) }} {% endmacro %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 9325a883..41e02ae6 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -1,32 +1,44 @@ -{% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_pattern', 'dbt_utils')(schema_pattern, table_pattern, exclude, database)) }} +{% macro get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} +{{ + return( + adapter.dispatch("get_relations_by_pattern", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) + ) +}} {% endmacro %} -{% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro default__get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} +{%- call statement('get_tables', fetch_result=True) %} - {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} +{{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude, database + ) +}} - {%- endcall -%} +{%- endcall -%} - {%- set table_list = load_result('get_tables') -%} +{%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} - {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} - {%- set tbl_relation = api.Relation.create( - database=database, - schema=row.table_schema, - identifier=row.table_name, - type=row.table_type - ) -%} - {%- do tbl_relations.append(tbl_relation) -%} - {%- endfor -%} +{%- if table_list and table_list["table"] -%} +{%- set tbl_relations = [] -%} +{%- for row in table_list["table"] -%} +{%- set tbl_relation = api.Relation.create( + database=database, + schema=row.table_schema, + identifier=row.table_name, + type=row.table_type, +) -%} {%- do tbl_relations.append(tbl_relation) -%} +{%- endfor -%} - {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} - {%- endif -%} +{{ return(tbl_relations) }} +{%- else -%} {{ return([]) }} +{%- endif -%} {% endmacro %} diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index b6733c4b..cc458914 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -1,32 +1,40 @@ -{% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_prefix', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} +{{ + return( + adapter.dispatch("get_relations_by_prefix", "dbt_utils")( + schema, prefix, exclude, database + ) + ) +}} {% endmacro %} -{% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} +{%- call statement('get_tables', fetch_result=True) %} - {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} +{{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} - {%- endcall -%} +{%- endcall -%} - {%- set table_list = load_result('get_tables') -%} +{%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} - {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} - {%- set tbl_relation = api.Relation.create( - database=database, - schema=row.table_schema, - identifier=row.table_name, - type=row.table_type - ) -%} - {%- do tbl_relations.append(tbl_relation) -%} - {%- endfor -%} +{%- if table_list and table_list["table"] -%} +{%- set tbl_relations = [] -%} +{%- for row in table_list["table"] -%} +{%- set tbl_relation = api.Relation.create( + database=database, + schema=row.table_schema, + identifier=row.table_name, + type=row.table_type, +) -%} {%- do tbl_relations.append(tbl_relation) -%} +{%- endfor -%} - {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} - {%- endif -%} +{{ return(tbl_relations) }} +{%- else -%} {{ return([]) }} +{%- endif -%} {% endmacro %} diff --git a/macros/sql/get_table_types_sql.sql b/macros/sql/get_table_types_sql.sql index 91573779..72c9a535 100644 --- a/macros/sql/get_table_types_sql.sql +++ b/macros/sql/get_table_types_sql.sql @@ -1,31 +1,43 @@ {%- macro get_table_types_sql() -%} - {{ return(adapter.dispatch('get_table_types_sql', 'dbt_utils')()) }} +{{ return(adapter.dispatch("get_table_types_sql", "dbt_utils")()) }} {%- endmacro -%} {% macro default__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" +case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) +end as "table_type" {% endmacro %} {% macro postgres__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'FOREIGN' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" +case + table_type + when 'BASE TABLE' + then 'table' + when 'FOREIGN' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) +end as "table_type" {% endmacro %} {% macro bigquery__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as `table_type` -{% endmacro %} \ No newline at end of file +case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) +end as `table_type` +{% endmacro %} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index 4d5a8fc9..9a747655 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -1,28 +1,41 @@ -{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_pattern_sql', 'dbt_utils') - (schema_pattern, table_pattern, exclude, database)) }} +{% macro get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} +{{ + return( + adapter.dispatch("get_tables_by_pattern_sql", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) + ) +}} {% endmacro %} -{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro default__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - select distinct - table_schema as "table_schema", - table_name as "table_name", - {{ dbt_utils.get_table_types_sql() }} - from {{ database }}.information_schema.tables - where table_schema ilike '{{ schema_pattern }}' - and table_name ilike '{{ table_pattern }}' - and table_name not ilike '{{ exclude }}' +select distinct + table_schema as "table_schema", + table_name as "table_name", + {{ dbt_utils.get_table_types_sql() }} +from {{ database }}.information_schema.tables +where + table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' {% endmacro %} -{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} + {% macro bigquery__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {% if '%' in schema_pattern %} - {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %} - {% else %} - {% set schemata=[schema_pattern] %} + {% if "%" in schema_pattern %} + {% set schemata = dbt_utils._bigquery__get_matching_schemata( + schema_pattern, database +) %} + {% else %} {% set schemata = [schema_pattern] %} {% endif %} {% set sql %} @@ -43,10 +56,10 @@ {{ return(sql) }} -{% endmacro %} + {% endmacro %} -{% macro _bigquery__get_matching_schemata(schema_pattern, database) %} + {% macro _bigquery__get_matching_schemata(schema_pattern, database) %} {% if execute %} {% set sql %} @@ -54,17 +67,15 @@ where lower(schema_name) like lower('{{ schema_pattern }}') {% endset %} - {% set results=run_query(sql) %} - - {% set schemata=results.columns['schema_name'].values() %} + {% set results = run_query(sql) %} - {{ return(schemata) }} + {% set schemata = results.columns["schema_name"].values() %} - {% else %} + {{ return(schemata) }} - {{ return([]) }} + {% else %} {{ return([]) }} {% endif %} -{% endmacro %} + {% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index f8fdfba6..c3fd5f03 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,14 +1,24 @@ -{% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_prefix_sql', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} +{{ + return( + adapter.dispatch("get_tables_by_prefix_sql", "dbt_utils")( + schema, prefix, exclude, database + ) + ) +}} {% endmacro %} -{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} - {{ dbt_utils.get_tables_by_pattern_sql( +{{ dbt_utils.get_tables_by_pattern_sql( schema_pattern = schema, table_pattern = prefix ~ '%', exclude = exclude, database = database ) }} - + {% endmacro %} diff --git a/macros/sql/groupby.sql b/macros/sql/groupby.sql index 68a68cf2..b46201d1 100644 --- a/macros/sql/groupby.sql +++ b/macros/sql/groupby.sql @@ -1,11 +1,10 @@ {%- macro group_by(n) -%} - {{ return(adapter.dispatch('group_by', 'dbt_utils')(n)) }} +{{ return(adapter.dispatch("group_by", "dbt_utils")(n)) }} {% endmacro %} {%- macro default__group_by(n) -%} - group by {% for i in range(1, n + 1) -%} - {{ i }}{{ ',' if not loop.last }} - {%- endfor -%} +group by + {% for i in range(1, n + 1) -%} {{ i }}{{ ',' if not loop.last }} {%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 70f276ec..75ff834a 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -5,47 +5,53 @@ http://daynebatten.com/2015/09/latitude-longitude-distance-sql/ The arguments should be float type. #} - -{% macro degrees_to_radians(degrees) -%} - acos(-1) * {{degrees}} / 180 -{%- endmacro %} - -{% macro haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} - {{ return(adapter.dispatch('haversine_distance', 'dbt_utils')(lat1,lon1,lat2,lon2,unit)) }} +{% macro degrees_to_radians(degrees) -%} acos(-1) * {{ degrees }} / 180 {%- endmacro %} + +{% macro haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} +{{ + return( + adapter.dispatch("haversine_distance", "dbt_utils")( + lat1, lon1, lat2, lon2, unit + ) + ) +}} {% endmacro %} -{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} +{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} +{%- if unit == "mi" %} {% set conversion_rate = 1 %} +{% elif unit == "km" %} {% set conversion_rate = 1.60934 %} {% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} +{{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} {% endif %} - 2 * 3961 * asin(sqrt(power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + - cos(radians({{lat1}})) * cos(radians({{lat2}})) * - power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2))) * {{ conversion_rate }} +2 * 3961 * asin( + sqrt( + power( (sin(radians( ({{ lat2 }} - {{ lat1 }}) / 2))), 2) + + cos(radians({{ lat1 }})) * cos(radians({{ lat2 }})) * + power( (sin(radians( ({{ lon2 }} - {{ lon1 }}) / 2))), 2) + ) +) * {{ conversion_rate }} {%- endmacro %} -{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} +{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} {% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} {% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} {% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} {% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} +{%- if unit == "mi" %} {% set conversion_rate = 1 %} +{% elif unit == "km" %} {% set conversion_rate = 1.60934 %} {% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} +{{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} {% endif %} - 2 * 3961 * asin(sqrt(power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + - cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * - power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2))) * {{ conversion_rate }} +2 * 3961 * asin( + sqrt( + power(sin( ({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + + cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * + power(sin( ({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2) + ) +) * {{ conversion_rate }} {%- endmacro %} - diff --git a/macros/sql/nullcheck.sql b/macros/sql/nullcheck.sql index 509d24f7..9e0aed1f 100644 --- a/macros/sql/nullcheck.sql +++ b/macros/sql/nullcheck.sql @@ -1,21 +1,17 @@ {% macro nullcheck(cols) %} - {{ return(adapter.dispatch('nullcheck', 'dbt_utils')(cols)) }} +{{ return(adapter.dispatch("nullcheck", "dbt_utils")(cols)) }} {% endmacro %} {% macro default__nullcheck(cols) %} {%- for col in cols %} - {% if col.is_string() -%} +{% if col.is_string() -%} nullif({{ col.name }}, '') as {{ col.name }} - nullif({{col.name}},'') as {{col.name}} +{%- else -%} {{ col.name }} - {%- else -%} +{%- endif -%} - {{col.name}} - - {%- endif -%} - -{%- if not loop.last -%} , {%- endif -%} +{%- if not loop.last -%}, {%- endif -%} {%- endfor -%} {% endmacro %} diff --git a/macros/sql/nullcheck_table.sql b/macros/sql/nullcheck_table.sql index c9ab5838..a01f2e86 100644 --- a/macros/sql/nullcheck_table.sql +++ b/macros/sql/nullcheck_table.sql @@ -1,14 +1,14 @@ {% macro nullcheck_table(relation) %} - {{ return(adapter.dispatch('nullcheck_table', 'dbt_utils')(relation)) }} +{{ return(adapter.dispatch("nullcheck_table", "dbt_utils")(relation)) }} {% endmacro %} {% macro default__nullcheck_table(relation) %} - {%- do dbt_utils._is_relation(relation, 'nullcheck_table') -%} - {%- do dbt_utils._is_ephemeral(relation, 'nullcheck_table') -%} - {% set cols = adapter.get_columns_in_relation(relation) %} +{%- do dbt_utils._is_relation(relation, "nullcheck_table") -%} +{%- do dbt_utils._is_ephemeral(relation, "nullcheck_table") -%} +{% set cols = adapter.get_columns_in_relation(relation) %} - select {{ dbt_utils.nullcheck(cols) }} - from {{relation}} +select {{ dbt_utils.nullcheck(cols) }} +from {{ relation }} {% endmacro %} diff --git a/macros/sql/pivot.sql b/macros/sql/pivot.sql index 88751062..ff20b17e 100644 --- a/macros/sql/pivot.sql +++ b/macros/sql/pivot.sql @@ -39,48 +39,65 @@ Arguments: quote_identifiers: Whether to surround column aliases with double quotes, default is true distinct: Whether to use distinct in the aggregation, default is False #} - -{% macro pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {{ return(adapter.dispatch('pivot', 'dbt_utils')(column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers, distinct)) }} +{% macro pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False +) %} +{{ + return( + adapter.dispatch("pivot", "dbt_utils")( + column, + values, + alias, + agg, + cmp, + prefix, + suffix, + then_value, + else_value, + quote_identifiers, + distinct, + ) + ) +}} {% endmacro %} -{% macro default__pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {% for value in values %} - {{ agg }}( - {% if distinct %} distinct {% endif %} - case - when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' +{% macro default__pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False +) %} +{% for value in values %} +{{ agg }} ( + {% if distinct %}distinct {% endif %} + case + when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' then {{ then_value }} - else {{ else_value }} - end - ) - {% if alias %} - {% if quote_identifiers %} - as {{ adapter.quote(prefix ~ value ~ suffix) }} - {% else %} - as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} - {% endif %} - {% endif %} - {% if not loop.last %},{% endif %} - {% endfor %} + else {{ else_value }} + end +) +{% if alias %} +{% if quote_identifiers %} as {{ adapter.quote(prefix ~ value ~ suffix) }} +{% else %} as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} +{% endif %} +{% endif %} +{% if not loop.last %},{% endif %} +{% endfor %} {% endmacro %} diff --git a/macros/sql/safe_add.sql b/macros/sql/safe_add.sql index 3b6195c8..62ff9d7f 100644 --- a/macros/sql/safe_add.sql +++ b/macros/sql/safe_add.sql @@ -1,8 +1,8 @@ {%- macro safe_add() -%} - {# needed for safe_add to allow for non-keyword arguments see SO post #} - {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} - {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('safe_add', 'dbt_utils')(*varargs)) }} +{# needed for safe_add to allow for non-keyword arguments see SO post #} +{# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} +{% set frustrating_jinja_feature = varargs %} +{{ return(adapter.dispatch("safe_add", "dbt_utils")(*varargs)) }} {% endmacro %} {%- macro default__safe_add() -%} @@ -11,10 +11,10 @@ {%- for field in varargs -%} - {% do fields.append("coalesce(" ~ field ~ ", 0)") %} +{% do fields.append("coalesce(" ~ field ~ ", 0)") %} {%- endfor -%} -{{ fields|join(' +\n ') }} +{{ fields | join(" +\n ") }} {%- endmacro -%} diff --git a/macros/sql/star.sql b/macros/sql/star.sql index 25e58969..da42668d 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -1,20 +1,23 @@ {% macro star(from, relation_alias=False, except=[], prefix='', suffix='') -%} - {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} +{{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} {% endmacro %} {% macro default__star(from, relation_alias=False, except=[], prefix='', suffix='') -%} - {%- do dbt_utils._is_relation(from, 'star') -%} - {%- do dbt_utils._is_ephemeral(from, 'star') -%} +{%- do dbt_utils._is_relation(from, 'star') -%} +{%- do dbt_utils._is_ephemeral(from, 'star') -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('*') }} - {% endif %} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{%- if not execute -%} {{ return("*") }} {% endif %} - {%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} +{%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} - {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} {%- if prefix!='' or suffix!='' %} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- endif -%} - {%- if not loop.last %},{{ '\n ' }}{% endif %} +{%- if relation_alias %} {{ relation_alias }}. +{% else %} +{%- endif -%} {{ adapter.quote(col) | trim }} +{%- if prefix != "" or suffix != "" %} +as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} +{%- endif -%} +{%- if not loop.last %},{{ "\n " }}{% endif %} - {%- endfor -%} +{%- endfor -%} {%- endmacro %} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index ba5c3e44..a93cd101 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -1,33 +1,27 @@ {%- macro surrogate_key(field_list) -%} - {# needed for safe_add to allow for non-keyword arguments see SO post #} - {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} - {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('surrogate_key', 'dbt_utils')(field_list, *varargs)) }} +{# needed for safe_add to allow for non-keyword arguments see SO post #} +{# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} +{% set frustrating_jinja_feature = varargs %} +{{ return(adapter.dispatch("surrogate_key", "dbt_utils")(field_list, *varargs)) }} {% endmacro %} {%- macro default__surrogate_key(field_list) -%} -{%- if varargs|length >= 1 or field_list is string %} +{%- if varargs | length >= 1 or field_list is string %} -{%- set error_message = ' -Warning: the `surrogate_key` macro now takes a single list argument instead of \ -multiple string arguments. Support for multiple string arguments will be \ -deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ -'.format(model.package_name, model.name) -%} +{%- set error_message = " Warning: the `surrogate_key` macro now takes a single list argument instead of \ multiple string arguments. Support for multiple string arguments will be \ deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name +) -%} {%- do exceptions.warn(error_message) -%} {# first argument is not included in varargs, so add first element to field_list_xf #} {%- set field_list_xf = [field_list] -%} -{%- for field in varargs %} -{%- set _ = field_list_xf.append(field) -%} -{%- endfor -%} - -{%- else -%} +{%- for field in varargs %} {%- set _ = field_list_xf.append(field) -%} {%- endfor -%} {# if using list, just set field_list_xf as field_list #} -{%- set field_list_xf = field_list -%} +{%- else -%} {%- set field_list_xf = field_list -%} {%- endif -%} @@ -36,16 +30,12 @@ deprecated in a future release of dbt-utils. The {}.{} model triggered this warn {%- for field in field_list_xf -%} - {%- set _ = fields.append( +{%- set _ = fields.append( "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" - ) -%} - - {%- if not loop.last %} - {%- set _ = fields.append("'-'") -%} - {%- endif -%} + ) -%} {%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} {%- endfor -%} -{{dbt_utils.hash(dbt_utils.concat(fields))}} +{{ dbt_utils.hash(dbt_utils.concat(fields)) }} {%- endmacro -%} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index a7bf1d95..409fd0de 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -1,67 +1,85 @@ -{%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} - {{ return(adapter.dispatch('union_relations', 'dbt_utils')(relations, column_override, include, exclude, source_column_name)) }} +{%- macro union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation" +) -%} +{{ + return( + adapter.dispatch("union_relations", "dbt_utils")( + relations, column_override, include, exclude, source_column_name + ) + ) +}} {% endmacro %} -{%- macro default__union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} - - {%- if exclude and include -%} - {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} - {%- endif -%} +{%- macro default__union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation" +) -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} - {%- if not execute %} - {{ return('') }} - {% endif -%} +{%- if exclude and include -%} +{{ + exceptions.raise_compiler_error( + "Both an exclude and include list were provided to the `union` macro. Only one is allowed" + ) +}} +{%- endif -%} - {%- set column_override = column_override if column_override is not none else {} -%} +{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} +{%- if not execute %} {{ return("") }} {% endif -%} - {%- set relation_columns = {} -%} - {%- set column_superset = {} -%} +{%- set column_override = column_override if column_override is not none else {} -%} - {%- for relation in relations -%} +{%- set relation_columns = {} -%} +{%- set column_superset = {} -%} - {%- do relation_columns.update({relation: []}) -%} +{%- for relation in relations -%} - {%- do dbt_utils._is_relation(relation, 'union_relations') -%} - {%- do dbt_utils._is_ephemeral(relation, 'union_relations') -%} - {%- set cols = adapter.get_columns_in_relation(relation) -%} - {%- for col in cols -%} +{%- do relation_columns.update({relation: []}) -%} - {#- If an exclude list was provided and the column is in the list, do nothing -#} - {%- if exclude and col.column in exclude -%} +{%- do dbt_utils._is_relation(relation, "union_relations") -%} +{%- do dbt_utils._is_ephemeral(relation, "union_relations") -%} +{%- set cols = adapter.get_columns_in_relation(relation) -%} +{%- for col in cols -%} - {#- If an include list was provided and the column is not in the list, do nothing -#} - {%- elif include and col.column not in include -%} +{#- If an exclude list was provided and the column is in the list, do nothing -#} +{%- if exclude and col.column in exclude -%} - {#- Otherwise add the column to the column superset -#} - {%- else -%} +{#- If an include list was provided and the column is not in the list, do nothing -#} +{%- elif include and col.column not in include -%} - {#- update the list of columns in this relation -#} - {%- do relation_columns[relation].append(col.column) -%} +{#- Otherwise add the column to the column superset -#} +{%- else -%} - {%- if col.column in column_superset -%} +{#- update the list of columns in this relation -#} +{%- do relation_columns[relation].append(col.column) -%} - {%- set stored = column_superset[col.column] -%} - {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} +{%- if col.column in column_superset -%} - {%- do column_superset.update({col.column: col}) -%} +{%- set stored = column_superset[col.column] -%} +{%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} - {%- endif %} +{%- do column_superset.update({col.column: col}) -%} - {%- else -%} +{%- endif %} - {%- do column_superset.update({col.column: col}) -%} +{%- else -%} {%- do column_superset.update({col.column: col}) -%} - {%- endif -%} +{%- endif -%} - {%- endif -%} +{%- endif -%} - {%- endfor -%} - {%- endfor -%} +{%- endfor -%} +{%- endfor -%} - {%- set ordered_column_names = column_superset.keys() -%} +{%- set ordered_column_names = column_superset.keys() -%} - {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} +{% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} {%- set relations_string -%} {%- for relation in relations -%} {{ relation.name }} @@ -73,31 +91,36 @@ There were no columns found to union for relations {{ relations_string }} {%- endset -%} - {{ exceptions.raise_compiler_error(error_message) }} - {%- endif -%} +{{ exceptions.raise_compiler_error(error_message) }} +{%- endif -%} - {%- for relation in relations %} +{%- for relation in relations %} - ( - select +( + select - cast({{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }}) as {{ source_column_name }}, - {% for col_name in ordered_column_names -%} + cast( + {{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }} + ) as {{ source_column_name }}, + {% for col_name in ordered_column_names -%} - {%- set col = column_superset[col_name] %} - {%- set col_type = column_override.get(col.column, col.data_type) %} - {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %} - cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%} + {%- set col = column_superset[col_name] %} + {%- set col_type = column_override.get(col.column, col.data_type) %} + {%- set col_name = ( + adapter.quote(col_name) + if col_name in relation_columns[relation] + else "null" +) %} + cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} + {% if not loop.last %},{% endif -%} - {%- endfor %} + {%- endfor %} - from {{ relation }} - ) + from {{ relation }} +) - {% if not loop.last -%} - union all - {% endif -%} +{% if not loop.last -%} union all {% endif -%} - {%- endfor -%} +{%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index 8821aad2..e8cbe093 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -11,69 +11,92 @@ Arguments: field_name: Destination table column name for the source table column names. value_name: Destination table column name for the pivoted values #} - -{% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} - {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name, table)) }} +{% macro unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none +) -%} +{{ + return( + adapter.dispatch("unpivot", "dbt_utils")( + relation, cast_to, exclude, remove, field_name, value_name, table + ) + ) +}} {% endmacro %} -{% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} - - {% if table %} - {%- set error_message = ' - Warning: the `unpivot` macro no longer accepts a `table` parameter. \ - This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ - The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - {% endif %} - - {% if relation and table %} - {{ exceptions.raise_compiler_error("Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`).") }} - {% elif not relation and table %} - {% set relation=table %} - {% elif not relation and not table %} - {{ exceptions.raise_compiler_error("Error: argument `relation` is required for `unpivot` macro.") }} - {% endif %} - - {%- set exclude = exclude if exclude is not none else [] %} - {%- set remove = remove if remove is not none else [] %} - - {%- set include_cols = [] %} - - {%- set table_columns = {} %} - - {%- do table_columns.update({relation: []}) %} - - {%- do dbt_utils._is_relation(relation, 'unpivot') -%} - {%- do dbt_utils._is_ephemeral(relation, 'unpivot') -%} - {%- set cols = adapter.get_columns_in_relation(relation) %} - - {%- for col in cols -%} - {%- if col.column.lower() not in remove|map('lower') and col.column.lower() not in exclude|map('lower') -%} - {% do include_cols.append(col) %} - {%- endif %} - {%- endfor %} - - - {%- for col in include_cols -%} - select - {%- for exclude_col in exclude %} - {{ exclude_col }}, - {%- endfor %} - - cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, - cast( {% if col.data_type == 'boolean' %} - {{ dbt_utils.cast_bool_to_text(col.column) }} - {% else %} - {{ col.column }} - {% endif %} - as {{ cast_to }}) as {{ value_name }} - - from {{ relation }} - - {% if not loop.last -%} - union all - {% endif -%} - {%- endfor -%} +{% macro default__unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none +) -%} + +{% if table %} +{%- set error_message = " Warning: the `unpivot` macro no longer accepts a `table` parameter. \ This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name +) -%} +{%- do exceptions.warn(error_message) -%} +{% endif %} + +{% if relation and table %} +{{ + exceptions.raise_compiler_error( + "Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`)." + ) +}} +{% elif not relation and table %} {% set relation = table %} +{% elif not relation and not table %} +{{ + exceptions.raise_compiler_error( + "Error: argument `relation` is required for `unpivot` macro." + ) +}} +{% endif %} + +{%- set exclude = exclude if exclude is not none else [] %} +{%- set remove = remove if remove is not none else [] %} + +{%- set include_cols = [] %} + +{%- set table_columns = {} %} + +{%- do table_columns.update({relation: []}) %} + +{%- do dbt_utils._is_relation(relation, "unpivot") -%} +{%- do dbt_utils._is_ephemeral(relation, "unpivot") -%} +{%- set cols = adapter.get_columns_in_relation(relation) %} + +{%- for col in cols -%} +{%- if col.column.lower() not in remove | map( + "lower" +) and col.column.lower() not in exclude | map("lower") -%} +{% do include_cols.append(col) %} +{%- endif %} +{%- endfor %} + + +{%- for col in include_cols -%} +select + {%- for exclude_col in exclude %} {{ exclude_col }}, {%- endfor %} + + cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, + cast( + {% if col.data_type == "boolean" %}{{ dbt_utils.cast_bool_to_text(col.column) }} + {% else %}{{ col.column }} + {% endif %} as {{ cast_to }} + ) as {{ value_name }} + +from {{ relation }} + +{% if not loop.last -%} union all {% endif -%} +{%- endfor -%} {%- endmacro %} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index 43a58cc4..b49a9f42 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -1,27 +1,25 @@ {% macro get_url_host(field) -%} - {{ return(adapter.dispatch('get_url_host', 'dbt_utils')(field)) }} +{{ return(adapter.dispatch("get_url_host", "dbt_utils")(field)) }} {% endmacro %} {% macro default__get_url_host(field) -%} -{%- set parsed = +{%- set parsed = dbt_utils.split_part( dbt_utils.split_part( - dbt_utils.split_part( + dbt_utils.replace( dbt_utils.replace( - dbt_utils.replace( - dbt_utils.replace(field, "'android-app://'", "''" - ), "'http://'", "''" - ), "'https://'", "''" - ), "'/'", 1 - ), "'?'", 1 - ) - --%} - - - {{ dbt_utils.safe_cast( - parsed, - dbt_utils.type_string() - )}} + dbt_utils.replace(field, "'android-app://'", "''"), + "'http://'", + "''", + ), + "'https://'", + "''", + ), + "'/'", + 1, + ), + "'?'", + 1, +) -%} {{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} {%- endmacro %} diff --git a/macros/web/get_url_parameter.sql b/macros/web/get_url_parameter.sql index f363fcbb..64b21ba4 100644 --- a/macros/web/get_url_parameter.sql +++ b/macros/web/get_url_parameter.sql @@ -1,13 +1,15 @@ {% macro get_url_parameter(field, url_parameter) -%} - {{ return(adapter.dispatch('get_url_parameter', 'dbt_utils')(field, url_parameter)) }} +{{ return(adapter.dispatch("get_url_parameter", "dbt_utils")(field, url_parameter)) }} {% endmacro %} {% macro default__get_url_parameter(field, url_parameter) -%} {%- set formatted_url_parameter = "'" + url_parameter + "='" -%} -{%- set split = dbt_utils.split_part(dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1) -%} +{%- set split = dbt_utils.split_part( + dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1 +) -%} -nullif({{ split }},'') +nullif({{ split }}, '') {%- endmacro %} diff --git a/macros/web/get_url_path.sql b/macros/web/get_url_path.sql index 4d224277..dc2919cf 100644 --- a/macros/web/get_url_path.sql +++ b/macros/web/get_url_path.sql @@ -1,13 +1,12 @@ {% macro get_url_path(field) -%} - {{ return(adapter.dispatch('get_url_path', 'dbt_utils')(field)) }} +{{ return(adapter.dispatch("get_url_path", "dbt_utils")(field)) }} {% endmacro %} {% macro default__get_url_path(field) -%} - {%- set stripped_url = - dbt_utils.replace( - dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''") - -%} +{%- set stripped_url = dbt_utils.replace( + dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''" +) -%} {%- set first_slash_pos -%} coalesce( @@ -16,19 +15,15 @@ ) {%- endset -%} - {%- set parsed_path = +{%- set parsed_path = dbt_utils.split_part( dbt_utils.right( stripped_url, dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos ), "'?'", 1 - ) - -%} + ) -%} + +{{ dbt_utils.safe_cast(parsed_path, dbt_utils.type_string()) }} - {{ dbt_utils.safe_cast( - parsed_path, - dbt_utils.type_string() - )}} - {%- endmacro %} From cc17535ad47862605da51cc19777fe8736d6a886 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Wed, 18 May 2022 16:51:11 -0600 Subject: [PATCH 05/22] chore: apply sqlfmt d0e14b9 (#3) --- .../models/sql/test_get_column_values.sql | 9 ++++-- .../sql/test_get_relations_by_pattern.sql | 4 ++- ...et_query_results_as_dict_objects_equal.sql | 4 +-- macros/cross_db_utils/_is_relation.sql | 4 ++- .../mutually_exclusive_ranges.sql | 22 ++++++++----- .../unique_combination_of_columns.sql | 10 ++++-- macros/jinja_helpers/pretty_log_format.sql | 2 +- .../insert_by_period_materialization.sql | 31 +++++++++++++------ macros/sql/get_column_values.sql | 19 +++++++++--- macros/sql/get_tables_by_prefix_sql.sql | 14 +++++---- macros/sql/haversine_distance.sql | 12 +++++-- macros/sql/star.sql | 2 +- macros/sql/surrogate_key.sql | 4 +-- macros/web/get_url_path.sql | 15 +++++---- 14 files changed, 102 insertions(+), 50 deletions(-) diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index fe91fa57..dfb17366 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -21,9 +21,12 @@ from {{ ref("data_get_column_values") }} select {% for val in column_values -%} - {{ dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} - as count_{{ val }} - {%- if not loop.last %},{% endif -%} + {{ + dbt_utils.safe_cast( + "sum(case when field = '" ~ val ~ "' then 1 else 0 end)", + dbt_utils.type_string(), + ) + }} as count_{{ val }} {%- if not loop.last %},{% endif -%} {%- endfor %} diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index 2c20559e..653f3598 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,6 +1,8 @@ {{ config(materialized="table") }} -{% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %} +{% set relations = dbt_utils.get_relations_by_pattern( + target.schema ~ "%", "data_events_%" +) %} with unioned as ({{ dbt_utils.union_relations(relations) }}) diff --git a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql index 88c0dc9e..3578af3e 100644 --- a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql +++ b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql @@ -15,8 +15,8 @@ {% endif %} -{% set actual_dictionary=dbt_utils.get_query_results_as_dict( - "select * from " ~ ref('data_get_query_results_as_dict') ~ " order by 1" +{% set actual_dictionary = dbt_utils.get_query_results_as_dict( + "select * from " ~ ref("data_get_query_results_as_dict") ~ " order by 1" ) %} {#- For reasons that remain unclear, Jinja won't return True for actual_dictionary == expected_dictionary. diff --git a/macros/cross_db_utils/_is_relation.sql b/macros/cross_db_utils/_is_relation.sql index af01c32c..2d46dfc5 100644 --- a/macros/cross_db_utils/_is_relation.sql +++ b/macros/cross_db_utils/_is_relation.sql @@ -3,6 +3,8 @@ obj is mapping and obj.get("metadata", {}).get("type", "").endswith("Relation") ) -%} -{%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} +{%- do exceptions.raise_compiler_error( + "Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj +) -%} {%- endif -%} {% endmacro %} diff --git a/macros/generic_tests/mutually_exclusive_ranges.sql b/macros/generic_tests/mutually_exclusive_ranges.sql index 6206ea01..799b2d8e 100644 --- a/macros/generic_tests/mutually_exclusive_ranges.sql +++ b/macros/generic_tests/mutually_exclusive_ranges.sql @@ -36,9 +36,13 @@ {% elif gaps == "required" %} {% set allow_gaps_operator = "<" %} {% set allow_gaps_operator_in_words = "less_than" %} {% else %} -{{ exceptions.raise_compiler_error( - "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" - ) }} +{{ + exceptions.raise_compiler_error( + "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" + ~ gaps + ~ "'.'" + ) +}} {% endif %} {% if not zero_length_range_allowed %} {% set allow_zero_length_operator = "<" %} @@ -47,12 +51,16 @@ {% set allow_zero_length_operator = "<=" %} {% set allow_zero_length_operator_in_words = "less_than_or_equal_to" %} {% else %} -{{ exceptions.raise_compiler_error( - "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length_range_allowed ~"'.'" - ) }} +{{ + exceptions.raise_compiler_error( + "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" + ~ zero_length_range_allowed + ~ "'.'" + ) +}} {% endif %} -{% set partition_clause="partition by " ~ partition_by if partition_by else '' %} +{% set partition_clause = "partition by " ~ partition_by if partition_by else "" %} with window_functions as ( diff --git a/macros/generic_tests/unique_combination_of_columns.sql b/macros/generic_tests/unique_combination_of_columns.sql index 97b8ccd5..3507b33c 100644 --- a/macros/generic_tests/unique_combination_of_columns.sql +++ b/macros/generic_tests/unique_combination_of_columns.sql @@ -21,9 +21,13 @@ {% set column_list = column_list.append(adapter.quote(column)) %} {%- endfor %} {% else %} -{{ exceptions.raise_compiler_error( - "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" - ) }} +{{ + exceptions.raise_compiler_error( + "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" + ~ quote + ~ "'.'" + ) +}} {% endif %} {%- set columns_csv = column_list | join(", ") %} diff --git a/macros/jinja_helpers/pretty_log_format.sql b/macros/jinja_helpers/pretty_log_format.sql index d5d6ea6c..67096f48 100644 --- a/macros/jinja_helpers/pretty_log_format.sql +++ b/macros/jinja_helpers/pretty_log_format.sql @@ -3,5 +3,5 @@ {% endmacro %} {% macro default__pretty_log_format(message) %} -{{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} +{{ return(dbt_utils.pretty_time() ~ " + " ~ message) }} {% endmacro %} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index e6c328cf..c51f557e 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -32,10 +32,13 @@ with max("{{timestamp_field}}"), '{{start_date}}' )::timestamp as start_timestamp, coalesce( - {{dbt_utils.dateadd('millisecond', - -1, - "nullif('" ~ stop_date ~ "','')::timestamp")}}, - {{ dbt_utils.current_timestamp() }} + {{ + dbt_utils.dateadd( + "millisecond", + -1, + "nullif('" ~ stop_date ~ "','')::timestamp", + ) + }}, {{ dbt_utils.current_timestamp() }} ) as stop_timestamp from "{{target_schema}}"."{{target_table}}" ) @@ -179,7 +182,7 @@ commit {%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} {{ dbt_utils.log_info(msg) }} -{%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%} +{%- set tmp_identifier = model["name"] ~ "__dbt_incremental_period" ~ i ~ "_tmp" -%} {%- set tmp_relation = api.Relation.create( identifier=tmp_identifier, schema=schema, type="table" ) -%} @@ -201,13 +204,13 @@ commit from_relation=tmp_relation, to_relation=target_relation ) }} -{%- set name = 'main-' ~ i -%} +{%- set name = "main-" ~ i -%} {% call statement(name, fetch_result=True) -%} insert into {{ target_relation }} ({{ target_cols_csv }}) (select {{ target_cols_csv }} from {{ tmp_relation.include(schema=False) }}) ; {%- endcall %} -{% set result = load_result('main-' ~ i) %} +{% set result = load_result("main-" ~ i) %} {% if "response" in result.keys() %} {# added in v0.19.0 #} {% set rows_inserted = result["response"]["rows_affected"] %} {# older versions #} @@ -217,7 +220,17 @@ insert into {{ target_relation }} ({{ target_cols_csv }}) {%- set sum_rows_inserted = loop_vars["sum_rows_inserted"] + rows_inserted -%} {%- if loop_vars.update({"sum_rows_inserted": sum_rows_inserted}) %} {% endif -%} -{%- set msg = "Ran for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) ~ "; " ~ rows_inserted ~ " records inserted" -%} +{%- set msg = ( + "Ran for " + ~ period + ~ " " + ~ (i + 1) + ~ " of " + ~ (num_periods) + ~ "; " + ~ rows_inserted + ~ " records inserted" +) -%} {{ dbt_utils.log_info(msg) }} {%- endfor %} @@ -236,7 +249,7 @@ commit {{ run_hooks(post_hooks, inside_transaction=False) }} -{%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} +{%- set status_string = "INSERT " ~ loop_vars["sum_rows_inserted"] -%} {% call noop_statement('main', status_string) -%} -- no-op diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index 5c4d0ac3..0de8523b 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -31,13 +31,24 @@ {%- if not relation_exists and default is none -%} -{{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} +{{ + exceptions.raise_compiler_error( + "In get_column_values(): relation " + ~ target_relation + ~ " does not exist and no default value was provided." + ) +}} {%- elif not relation_exists and default is not none -%} -{{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} - -{{ return(default) }} +{{ + log( + "Relation " + ~ target_relation + ~ " does not exist. Returning the default value: " + ~ default + ) +}} {{ return(default) }} {%- else -%} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index c3fd5f03..68234b55 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -14,11 +14,13 @@ schema, prefix, exclude="", database=target.database ) %} -{{ dbt_utils.get_tables_by_pattern_sql( - schema_pattern = schema, - table_pattern = prefix ~ '%', - exclude = exclude, - database = database - ) }} +{{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern=schema, + table_pattern=prefix ~ "%", + exclude=exclude, + database=database, + ) +}} {% endmacro %} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 75ff834a..112caa1a 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -21,7 +21,11 @@ The arguments should be float type. {%- if unit == "mi" %} {% set conversion_rate = 1 %} {% elif unit == "km" %} {% set conversion_rate = 1.60934 %} {% else %} -{{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} +{{ + exceptions.raise_compiler_error( + "unit input must be one of 'mi' or 'km'. Got " ~ unit + ) +}} {% endif %} 2 * 3961 * asin( @@ -44,7 +48,11 @@ The arguments should be float type. {%- if unit == "mi" %} {% set conversion_rate = 1 %} {% elif unit == "km" %} {% set conversion_rate = 1.60934 %} {% else %} -{{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} +{{ + exceptions.raise_compiler_error( + "unit input must be one of 'mi' or 'km'. Got " ~ unit + ) +}} {% endif %} 2 * 3961 * asin( sqrt( diff --git a/macros/sql/star.sql b/macros/sql/star.sql index da42668d..8d989d78 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -15,7 +15,7 @@ {% else %} {%- endif -%} {{ adapter.quote(col) | trim }} {%- if prefix != "" or suffix != "" %} -as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} +as {{ adapter.quote(prefix ~ col ~ suffix) | trim }} {%- endif -%} {%- if not loop.last %},{{ "\n " }}{% endif %} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index a93cd101..56f69893 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -31,8 +31,8 @@ {%- for field in field_list_xf -%} {%- set _ = fields.append( - "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" - ) -%} {%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} + "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" +) -%} {%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} {%- endfor -%} diff --git a/macros/web/get_url_path.sql b/macros/web/get_url_path.sql index dc2919cf..bb518959 100644 --- a/macros/web/get_url_path.sql +++ b/macros/web/get_url_path.sql @@ -15,14 +15,13 @@ ) {%- endset -%} -{%- set parsed_path = - dbt_utils.split_part( - dbt_utils.right( - stripped_url, - dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos - ), - "'?'", 1 - ) -%} +{%- set parsed_path = dbt_utils.split_part( + dbt_utils.right( + stripped_url, dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos + ), + "'?'", + 1, +) -%} {{ dbt_utils.safe_cast(parsed_path, dbt_utils.type_string()) }} From 0695ce6084f39b0de761c357f43c9d6220f96d68 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Mon, 13 Jun 2022 13:01:09 -0600 Subject: [PATCH 06/22] chore: apply sqlfmt aca4b6a (#4) * chore: apply sqlfmt aca4b6a * fix: del + --- .../sql/test_get_column_values_use_default.sql | 4 +++- macros/cross_db_utils/except.sql | 13 +++++++++++-- macros/cross_db_utils/intersect.sql | 13 +++++++++++-- macros/generic_tests/cardinality_equality.sql | 8 +++++++- macros/sql/generate_series.sql | 6 +++++- macros/sql/union.sql | 4 +++- macros/sql/unpivot.sql | 4 +++- 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 0fcd8ac0..3140021a 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -23,7 +23,9 @@ with select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual_column_value - {% if not loop.last %} union all {% endif %} + {% if not loop.last %} + union all + {% endif %} {% endfor %} ), diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 2799095c..1ddde746 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -3,6 +3,15 @@ {% endmacro %} -{% macro default__except() %} except {% endmacro %} +{% macro default__except() %} -{% macro bigquery__except() %} except distinct {% endmacro %} +except + +{% endmacro %} + +{% macro bigquery__except() %} + +except +distinct + +{% endmacro %} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index c224e44f..d2eb752b 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -3,6 +3,15 @@ {% endmacro %} -{% macro default__intersect() %} intersect {% endmacro %} +{% macro default__intersect() %} -{% macro bigquery__intersect() %} intersect distinct {% endmacro %} +intersect + +{% endmacro %} + +{% macro bigquery__intersect() %} + +intersect +distinct + +{% endmacro %} diff --git a/macros/generic_tests/cardinality_equality.sql b/macros/generic_tests/cardinality_equality.sql index be319a19..06b30a6c 100644 --- a/macros/generic_tests/cardinality_equality.sql +++ b/macros/generic_tests/cardinality_equality.sql @@ -27,7 +27,13 @@ with except_b as (select * from table_b {{ dbt_utils.except() }} select * from table_a), - unioned as (select * from except_a union all select * from except_b) + unioned as ( + select * + from except_a + union all + select * + from except_b + ) select * from unioned diff --git a/macros/sql/generate_series.sql b/macros/sql/generate_series.sql index f2b1a76e..28a298d5 100644 --- a/macros/sql/generate_series.sql +++ b/macros/sql/generate_series.sql @@ -24,7 +24,11 @@ {% set n = dbt_utils.get_powers_of_two(upper_bound) %} with - p as (select 0 as generated_number union all select 1), + p as ( + select 0 as generated_number + union all + select 1 + ), unioned as ( select diff --git a/macros/sql/union.sql b/macros/sql/union.sql index 409fd0de..4023b569 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -119,7 +119,9 @@ from {{ relation }} ) -{% if not loop.last -%} union all {% endif -%} +{% if not loop.last -%} +union all +{% endif -%} {%- endfor -%} diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index e8cbe093..4cbd357a 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -96,7 +96,9 @@ select from {{ relation }} -{% if not loop.last -%} union all {% endif -%} +{% if not loop.last -%} +union all +{% endif -%} {%- endfor -%} {%- endmacro %} From a617b80efbf1c847d2fd1db339220cca3c601a46 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Fri, 15 Jul 2022 15:08:07 -0600 Subject: [PATCH 07/22] chore: apply sqlfmt e406e35 (#5) --- macros/cross_db_utils/concat.sql | 2 - macros/cross_db_utils/datediff.sql | 52 +++++++++++++------ macros/cross_db_utils/listagg.sql | 3 +- macros/cross_db_utils/width_bucket.sql | 12 ++--- .../mutually_exclusive_ranges.sql | 3 +- macros/generic_tests/not_constant.sql | 2 - macros/generic_tests/not_null_proportion.sql | 5 +- .../insert_by_period_materialization.sql | 8 ++- macros/sql/date_spine.sql | 6 --- macros/sql/deduplicate.sql | 3 +- macros/sql/generate_series.sql | 3 +- macros/sql/get_column_values.sql | 2 - macros/sql/get_tables_by_pattern_sql.sql | 2 - macros/sql/haversine_distance.sql | 32 ++++++++---- 14 files changed, 77 insertions(+), 58 deletions(-) diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 74b9fb30..140a7787 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,5 +1,3 @@ - - {% macro concat(fields) -%} {{ return(adapter.dispatch("concat", "dbt_utils")(fields)) }} {%- endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index d95ae1ce..fc2e2c89 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -34,20 +34,27 @@ datetime_diff( ) {% elif datepart == "quarter" %} ( - {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + date_part( + {{ dbt_utils.datediff(first_date, second_date, "year") }} + * 4 + date_part( 'quarter', ({{ second_date }})::date - ) - date_part('quarter', ({{ first_date }})::date) + ) - date_part( + 'quarter', ({{ first_date }})::date + ) ) {% elif datepart == "month" %} ( - {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + date_part( + {{ dbt_utils.datediff(first_date, second_date, "year") }} + * 12 + date_part( 'month', ({{ second_date }})::date - ) - date_part('month', ({{ first_date }})::date) + ) - date_part( + 'month', ({{ first_date }})::date + ) ) {% elif datepart == "day" %} ( ({{ second_date }})::date - ({{ first_date }})::date) {% elif datepart == "week" %} ( - {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case + {{ dbt_utils.datediff(first_date, second_date, "day") }} + / 7 + case when date_part('dow', ({{ first_date }})::timestamp) <= date_part( 'dow', ({{ second_date }})::timestamp @@ -58,33 +65,48 @@ datetime_diff( ) {% elif datepart == "hour" %} ( - {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + date_part( + {{ dbt_utils.datediff(first_date, second_date, "day") }} + * 24 + date_part( 'hour', ({{ second_date }})::timestamp - ) - date_part('hour', ({{ first_date }})::timestamp) + ) - date_part( + 'hour', ({{ first_date }})::timestamp + ) ) {% elif datepart == "minute" %} ( - {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + date_part( + {{ dbt_utils.datediff(first_date, second_date, "hour") }} + * 60 + date_part( 'minute', ({{ second_date }})::timestamp - ) - date_part('minute', ({{ first_date }})::timestamp) + ) - date_part( + 'minute', ({{ first_date }})::timestamp + ) ) {% elif datepart == "second" %} ( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + floor( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} + * 60 + floor( date_part('second', ({{ second_date }})::timestamp) - ) - floor(date_part('second', ({{ first_date }})::timestamp)) + ) - floor( + date_part('second', ({{ first_date }})::timestamp) + ) ) {% elif datepart == "millisecond" %} ( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + floor( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} + * 60000 + floor( date_part('millisecond', ({{ second_date }})::timestamp) - ) - floor(date_part('millisecond', ({{ first_date }})::timestamp)) + ) - floor( + date_part('millisecond', ({{ first_date }})::timestamp) + ) ) {% elif datepart == "microsecond" %} ( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + floor( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} + * 60000000 + floor( date_part('microsecond', ({{ second_date }})::timestamp) - ) - floor(date_part('microsecond', ({{ first_date }})::timestamp)) + ) - floor( + date_part('microsecond', ({{ first_date }})::timestamp) + ) ) {% else %} {{ diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 6cf8374e..5828a3dc 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -49,7 +49,8 @@ array_to_string( array_agg( {{ measure }} {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} ) - ) [1:{{ limit_num }}], + ) [1:{{ limit_num }} + ], {{ delimiter_text }} ) {%- else %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index 34818584..ca6e3abf 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -21,13 +21,13 @@ mod( {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }}, {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} - ) = 0 + ) + = 0 then 1 else 0 end -) + -- Anything over max_value goes the N+1 bucket -least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) +) + least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} @@ -40,15 +40,13 @@ least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + case when {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }} - % - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + % {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} = 0 then 1 else 0 end -) + -- Anything over max_value goes the N+1 bucket -least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) +) + least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} diff --git a/macros/generic_tests/mutually_exclusive_ranges.sql b/macros/generic_tests/mutually_exclusive_ranges.sql index 799b2d8e..f62cefb6 100644 --- a/macros/generic_tests/mutually_exclusive_ranges.sql +++ b/macros/generic_tests/mutually_exclusive_ranges.sql @@ -76,7 +76,8 @@ with row_number() over ( {{ partition_clause }} order by {{ lower_bound_column }} desc - ) = 1 as is_last_record + ) + = 1 as is_last_record from {{ model }} diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index 842eaf5c..d72f6d93 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -4,8 +4,6 @@ {% endtest %} {% macro default__test_not_constant(model, column_name) %} - - {# In TSQL, subquery aggregate columns need aliases #} {# thus: a filler col name, 'filler_column' #} select count(distinct {{ column_name }}) as filler_column diff --git a/macros/generic_tests/not_null_proportion.sql b/macros/generic_tests/not_null_proportion.sql index 37df9a84..87a6e964 100644 --- a/macros/generic_tests/not_null_proportion.sql +++ b/macros/generic_tests/not_null_proportion.sql @@ -11,9 +11,8 @@ with validation as ( select - sum(case when {{ column_name }} is null then 0 else 1 end) / cast( - count(*) as numeric - ) as not_null_proportion + sum(case when {{ column_name }} is null then 0 else 1 end) + / cast(count(*) as numeric) as not_null_proportion from {{ model }} ), validation_errors as ( diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index c51f557e..75d9b631 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -28,9 +28,8 @@ with data as ( select - coalesce( - max("{{timestamp_field}}"), '{{start_date}}' - )::timestamp as start_timestamp, + coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp + as start_timestamp, coalesce( {{ dbt_utils.dateadd( @@ -47,8 +46,7 @@ select start_timestamp, stop_timestamp, {{ dbt_utils.datediff("start_timestamp", "stop_timestamp", period) }} - + 1 - as num_periods + + 1 as num_periods from data {%- endcall %} {%- endmacro %} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index b256dfde..d4363c7d 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -25,10 +25,6 @@ select {%- endif -%} {%- endmacro %} - - - - {% macro date_spine(datepart, start_date, end_date) %} {{ return( @@ -38,8 +34,6 @@ select {%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} - - {# call as follows: date_spine( diff --git a/macros/sql/deduplicate.sql b/macros/sql/deduplicate.sql index 949a15b2..529baa92 100644 --- a/macros/sql/deduplicate.sql +++ b/macros/sql/deduplicate.sql @@ -44,7 +44,8 @@ from original {% if order_by is not none -%} order by {{ order_by }} {%- endif %} limit 1 - ) [offset (0)] as deduped + ) [offset (0) + ] as deduped from {{ relation if relation_alias is none else relation_alias }} as original group by {{ group_by }} ) diff --git a/macros/sql/generate_series.sql b/macros/sql/generate_series.sql index 28a298d5..f8ff6590 100644 --- a/macros/sql/generate_series.sql +++ b/macros/sql/generate_series.sql @@ -36,7 +36,8 @@ with {% for i in range(n) %} p{{ i }}.generated_number * power(2, {{ i }}) {% if not loop.last %} + {% endif %} - {% endfor %} + 1 as generated_number + {% endfor %} + + 1 as generated_number from diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index 0de8523b..0515e748 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -51,8 +51,6 @@ }} {{ return(default) }} {%- else -%} - - select {{ column }} as value from {{ target_relation }} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index 9a747655..8a579f29 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -25,8 +25,6 @@ where and table_name not ilike '{{ exclude }}' {% endmacro %} - - {% macro bigquery__get_tables_by_pattern_sql( schema_pattern, table_pattern, exclude="", database=target.database ) %} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 112caa1a..28491fe0 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -28,13 +28,19 @@ The arguments should be float type. }} {% endif %} -2 * 3961 * asin( +2 +* 3961 * asin( sqrt( - power( (sin(radians( ({{ lat2 }} - {{ lat1 }}) / 2))), 2) + - cos(radians({{ lat1 }})) * cos(radians({{ lat2 }})) * - power( (sin(radians( ({{ lon2 }} - {{ lon1 }}) / 2))), 2) + power( (sin(radians( ({{ lat2 }} - {{ lat1 }}) / 2))), 2) + cos( + radians({{ lat1 }}) + ) * cos( + radians({{ lat2 }}) + ) * power( + (sin(radians( ({{ lon2 }} - {{ lon1 }}) / 2))), 2 + ) ) -) * {{ conversion_rate }} +) +* {{ conversion_rate }} {%- endmacro %} @@ -54,12 +60,18 @@ The arguments should be float type. ) }} {% endif %} -2 * 3961 * asin( +2 +* 3961 * asin( sqrt( - power(sin( ({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + - cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * - power(sin( ({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2) + power(sin( ({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + cos( + {{ radians_lat1 }} + ) * cos( + {{ radians_lat2 }} + ) * power( + sin( ({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2 + ) ) -) * {{ conversion_rate }} +) +* {{ conversion_rate }} {%- endmacro %} From 1afba0e868e4d6221bb14e3c27dde1fe3fa3c405 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Tue, 19 Jul 2022 12:21:28 -0600 Subject: [PATCH 08/22] chore: apply sqlfmt fb99833 (#6) --- macros/cross_db_utils/dateadd.sql | 2 +- macros/cross_db_utils/datediff.sql | 61 ++++++++++---------------- macros/cross_db_utils/listagg.sql | 3 +- macros/cross_db_utils/split_part.sql | 2 +- macros/cross_db_utils/width_bucket.sql | 4 +- macros/sql/deduplicate.sql | 3 +- macros/sql/haversine_distance.sql | 22 +++++----- 7 files changed, 40 insertions(+), 57 deletions(-) diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 57cbaa25..133846c6 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -27,7 +27,7 @@ datetime_add( {% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} -{{ from_date_or_timestamp }} + ( (interval '1 {{ datepart }}') * ({{ interval }})) +{{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) {% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index fc2e2c89..87c1a726 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -35,26 +35,23 @@ datetime_diff( {% elif datepart == "quarter" %} ( {{ dbt_utils.datediff(first_date, second_date, "year") }} - * 4 + date_part( - 'quarter', ({{ second_date }})::date - ) - date_part( - 'quarter', ({{ first_date }})::date - ) + * 4 + + date_part('quarter', ({{ second_date }})::date) + - date_part('quarter', ({{ first_date }})::date) ) {% elif datepart == "month" %} ( {{ dbt_utils.datediff(first_date, second_date, "year") }} - * 12 + date_part( - 'month', ({{ second_date }})::date - ) - date_part( - 'month', ({{ first_date }})::date - ) + * 12 + + date_part('month', ({{ second_date }})::date) + - date_part('month', ({{ first_date }})::date) ) -{% elif datepart == "day" %} ( ({{ second_date }})::date - ({{ first_date }})::date) +{% elif datepart == "day" %}(({{ second_date }})::date - ({{ first_date }})::date) {% elif datepart == "week" %} ( {{ dbt_utils.datediff(first_date, second_date, "day") }} - / 7 + case + / 7 + + case when date_part('dow', ({{ first_date }})::timestamp) <= date_part( 'dow', ({{ second_date }})::timestamp @@ -66,47 +63,37 @@ datetime_diff( {% elif datepart == "hour" %} ( {{ dbt_utils.datediff(first_date, second_date, "day") }} - * 24 + date_part( - 'hour', ({{ second_date }})::timestamp - ) - date_part( - 'hour', ({{ first_date }})::timestamp - ) + * 24 + + date_part('hour', ({{ second_date }})::timestamp) + - date_part('hour', ({{ first_date }})::timestamp) ) {% elif datepart == "minute" %} ( {{ dbt_utils.datediff(first_date, second_date, "hour") }} - * 60 + date_part( - 'minute', ({{ second_date }})::timestamp - ) - date_part( - 'minute', ({{ first_date }})::timestamp - ) + * 60 + + date_part('minute', ({{ second_date }})::timestamp) + - date_part('minute', ({{ first_date }})::timestamp) ) {% elif datepart == "second" %} ( {{ dbt_utils.datediff(first_date, second_date, "minute") }} - * 60 + floor( - date_part('second', ({{ second_date }})::timestamp) - ) - floor( - date_part('second', ({{ first_date }})::timestamp) - ) + * 60 + + floor(date_part('second', ({{ second_date }})::timestamp)) + - floor(date_part('second', ({{ first_date }})::timestamp)) ) {% elif datepart == "millisecond" %} ( {{ dbt_utils.datediff(first_date, second_date, "minute") }} - * 60000 + floor( - date_part('millisecond', ({{ second_date }})::timestamp) - ) - floor( - date_part('millisecond', ({{ first_date }})::timestamp) - ) + * 60000 + + floor(date_part('millisecond', ({{ second_date }})::timestamp)) + - floor(date_part('millisecond', ({{ first_date }})::timestamp)) ) {% elif datepart == "microsecond" %} ( {{ dbt_utils.datediff(first_date, second_date, "minute") }} - * 60000000 + floor( - date_part('microsecond', ({{ second_date }})::timestamp) - ) - floor( - date_part('microsecond', ({{ first_date }})::timestamp) - ) + * 60000000 + + floor(date_part('microsecond', ({{ second_date }})::timestamp)) + - floor(date_part('microsecond', ({{ first_date }})::timestamp)) ) {% else %} {{ diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 5828a3dc..465ddb40 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -49,8 +49,7 @@ array_to_string( array_agg( {{ measure }} {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} ) - ) [1:{{ limit_num }} - ], + )[1:{{ limit_num }}], {{ delimiter_text }} ) {%- else %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index d4441290..e5c204ae 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -18,6 +18,6 @@ split_part({{ string_text }}, {{ delimiter_text }}, {{ part_number }}) {% macro bigquery__split_part(string_text, delimiter_text, part_number) %} -split({{ string_text }}, {{ delimiter_text }}) [safe_offset({{ part_number - 1 }})] +split({{ string_text }}, {{ delimiter_text }})[safe_offset({{ part_number - 1 }})] {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index ca6e3abf..4c63adcf 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -27,7 +27,7 @@ else 0 end -- Anything over max_value goes the N+1 bucket -) + least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) +) + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} @@ -46,7 +46,7 @@ else 0 end -- Anything over max_value goes the N+1 bucket -) + least(ceil( ({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) +) + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} diff --git a/macros/sql/deduplicate.sql b/macros/sql/deduplicate.sql index 529baa92..51ebce05 100644 --- a/macros/sql/deduplicate.sql +++ b/macros/sql/deduplicate.sql @@ -44,8 +44,7 @@ from original {% if order_by is not none -%} order by {{ order_by }} {%- endif %} limit 1 - ) [offset (0) - ] as deduped + )[offset(0)] as deduped from {{ relation if relation_alias is none else relation_alias }} as original group by {{ group_by }} ) diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 28491fe0..3527cd73 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -29,14 +29,13 @@ The arguments should be float type. {% endif %} 2 -* 3961 * asin( +* 3961 +* asin( sqrt( - power( (sin(radians( ({{ lat2 }} - {{ lat1 }}) / 2))), 2) + cos( + power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + cos( radians({{ lat1 }}) - ) * cos( - radians({{ lat2 }}) - ) * power( - (sin(radians( ({{ lon2 }} - {{ lon1 }}) / 2))), 2 + ) * cos(radians({{ lat2 }})) * power( + (sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2 ) ) ) @@ -61,14 +60,13 @@ The arguments should be float type. }} {% endif %} 2 -* 3961 * asin( +* 3961 +* asin( sqrt( - power(sin( ({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + cos( + power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + cos( {{ radians_lat1 }} - ) * cos( - {{ radians_lat2 }} - ) * power( - sin( ({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2 + ) * cos({{ radians_lat2 }}) * power( + sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2 ) ) ) From bfe3e2d33bea6a3cbc07d27e5d6c21eaf3273126 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Tue, 2 Aug 2022 15:14:45 -0600 Subject: [PATCH 09/22] chore: apply sqlfmt bcf660d (#8) --- macros/cross_db_utils/listagg.sql | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 465ddb40..75e8e8ff 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -16,7 +16,7 @@ array_to_string( array_slice( array_agg({{ measure }}) - {% if order_by_clause -%} within group({{ order_by_clause }}) {%- endif %}, + {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %}, 0, {{ limit_num }} ), @@ -24,7 +24,7 @@ array_to_string( ) {%- else %} listagg({{ measure }}, {{ delimiter_text }}) -{% if order_by_clause -%} within group({{ order_by_clause }}) +{% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %} {%- endif %} @@ -78,12 +78,12 @@ string_agg( {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} regexp_substr( listagg({{ measure }}, {{ delimiter_text }}) - {% if order_by_clause -%} within group({{ order_by_clause }}) {%- endif %}, + {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %}, {{ regex }} ) {%- else %} listagg({{ measure }}, {{ delimiter_text }}) -{% if order_by_clause -%} within group({{ order_by_clause }}) +{% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %} {%- endif %} From 98ef57072ad4c67343c5704c8a3667d42be1eaad Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Mon, 8 Aug 2022 13:48:37 -0600 Subject: [PATCH 10/22] chore: apply sqlfmt 6d33371 (#9) --- macros/generic_tests/equal_rowcount.sql | 6 +++++- macros/generic_tests/not_null_proportion.sql | 6 +++++- macros/generic_tests/recency.sql | 4 +++- .../materializations/insert_by_period_materialization.sql | 7 ++++++- macros/sql/date_spine.sql | 4 +++- macros/web/get_url_host.sql | 8 +++++++- 6 files changed, 29 insertions(+), 6 deletions(-) diff --git a/macros/generic_tests/equal_rowcount.sql b/macros/generic_tests/equal_rowcount.sql index c04363ef..e0c638cb 100644 --- a/macros/generic_tests/equal_rowcount.sql +++ b/macros/generic_tests/equal_rowcount.sql @@ -1,5 +1,9 @@ {% test equal_rowcount(model, compare_model) %} -{{ return(adapter.dispatch("test_equal_rowcount", "dbt_utils")(model, compare_model)) }} +{{ + return( + adapter.dispatch("test_equal_rowcount", "dbt_utils")(model, compare_model) + ) +}} {% endtest %} {% macro default__test_equal_rowcount(model, compare_model) %} diff --git a/macros/generic_tests/not_null_proportion.sql b/macros/generic_tests/not_null_proportion.sql index 87a6e964..a9369808 100644 --- a/macros/generic_tests/not_null_proportion.sql +++ b/macros/generic_tests/not_null_proportion.sql @@ -1,5 +1,9 @@ {% macro test_not_null_proportion(model) %} -{{ return(adapter.dispatch("test_not_null_proportion", "dbt_utils")(model, **kwargs)) }} +{{ + return( + adapter.dispatch("test_not_null_proportion", "dbt_utils")(model, **kwargs) + ) +}} {% endmacro %} {% macro default__test_not_null_proportion(model) %} diff --git a/macros/generic_tests/recency.sql b/macros/generic_tests/recency.sql index 4005c9e7..ed02969c 100644 --- a/macros/generic_tests/recency.sql +++ b/macros/generic_tests/recency.sql @@ -1,7 +1,9 @@ {% test recency(model, field, datepart, interval) %} {{ return( - adapter.dispatch("test_recency", "dbt_utils")(model, field, datepart, interval) + adapter.dispatch("test_recency", "dbt_utils")( + model, field, datepart, interval + ) ) }} {% endtest %} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 75d9b631..ff8145e0 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -9,7 +9,12 @@ {{ return( adapter.dispatch("get_period_boundaries", "dbt_utils")( - target_schema, target_table, timestamp_field, start_date, stop_date, period + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period, ) ) }} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index d4363c7d..8eef70d1 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -28,7 +28,9 @@ select {% macro date_spine(datepart, start_date, end_date) %} {{ return( - adapter.dispatch("date_spine", "dbt_utils")(datepart, start_date, end_date) + adapter.dispatch("date_spine", "dbt_utils")( + datepart, start_date, end_date + ) ) }} {%- endmacro %} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index b49a9f42..ca25016a 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -20,6 +20,12 @@ ), "'?'", 1, -) -%} {{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} +) -%} +{{ + dbt_utils.safe_cast( + parsed, + dbt_utils.type_string(), + ) +}} {%- endmacro %} From f67624110d9511e19aa2340eff8d85df7e6dd7b6 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Wed, 17 Aug 2022 15:49:03 -0600 Subject: [PATCH 11/22] chore: apply sqlfmt 8b20379 (#10) --- macros/cross_db_utils/datediff.sql | 53 +++++++++++++----------------- macros/web/get_url_host.sql | 8 +---- 2 files changed, 23 insertions(+), 38 deletions(-) diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 87c1a726..48be529d 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -34,24 +34,20 @@ datetime_diff( ) {% elif datepart == "quarter" %} ( - {{ dbt_utils.datediff(first_date, second_date, "year") }} - * 4 - + date_part('quarter', ({{ second_date }})::date) - - date_part('quarter', ({{ first_date }})::date) + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + date_part( + 'quarter', ({{ second_date }})::date + ) - date_part('quarter', ({{ first_date }})::date) ) {% elif datepart == "month" %} ( - {{ dbt_utils.datediff(first_date, second_date, "year") }} - * 12 - + date_part('month', ({{ second_date }})::date) - - date_part('month', ({{ first_date }})::date) + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + date_part( + 'month', ({{ second_date }})::date + ) - date_part('month', ({{ first_date }})::date) ) {% elif datepart == "day" %}(({{ second_date }})::date - ({{ first_date }})::date) {% elif datepart == "week" %} ( - {{ dbt_utils.datediff(first_date, second_date, "day") }} - / 7 - + case + {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case when date_part('dow', ({{ first_date }})::timestamp) <= date_part( 'dow', ({{ second_date }})::timestamp @@ -62,38 +58,33 @@ datetime_diff( ) {% elif datepart == "hour" %} ( - {{ dbt_utils.datediff(first_date, second_date, "day") }} - * 24 - + date_part('hour', ({{ second_date }})::timestamp) - - date_part('hour', ({{ first_date }})::timestamp) + {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + date_part( + 'hour', ({{ second_date }})::timestamp + ) - date_part('hour', ({{ first_date }})::timestamp) ) {% elif datepart == "minute" %} ( - {{ dbt_utils.datediff(first_date, second_date, "hour") }} - * 60 - + date_part('minute', ({{ second_date }})::timestamp) - - date_part('minute', ({{ first_date }})::timestamp) + {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + date_part( + 'minute', ({{ second_date }})::timestamp + ) - date_part('minute', ({{ first_date }})::timestamp) ) {% elif datepart == "second" %} ( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} - * 60 - + floor(date_part('second', ({{ second_date }})::timestamp)) - - floor(date_part('second', ({{ first_date }})::timestamp)) + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + floor( + date_part('second', ({{ second_date }})::timestamp) + ) - floor(date_part('second', ({{ first_date }})::timestamp)) ) {% elif datepart == "millisecond" %} ( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} - * 60000 - + floor(date_part('millisecond', ({{ second_date }})::timestamp)) - - floor(date_part('millisecond', ({{ first_date }})::timestamp)) + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + floor( + date_part('millisecond', ({{ second_date }})::timestamp) + ) - floor(date_part('millisecond', ({{ first_date }})::timestamp)) ) {% elif datepart == "microsecond" %} ( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} - * 60000000 - + floor(date_part('microsecond', ({{ second_date }})::timestamp)) - - floor(date_part('microsecond', ({{ first_date }})::timestamp)) + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + floor( + date_part('microsecond', ({{ second_date }})::timestamp) + ) - floor(date_part('microsecond', ({{ first_date }})::timestamp)) ) {% else %} {{ diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index ca25016a..b49a9f42 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -20,12 +20,6 @@ ), "'?'", 1, -) -%} -{{ - dbt_utils.safe_cast( - parsed, - dbt_utils.type_string(), - ) -}} +) -%} {{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} {%- endmacro %} From 55c91993041ace09e170f970ac1c775d5aacf3c6 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Fri, 19 Aug 2022 16:06:33 -0600 Subject: [PATCH 12/22] chore: apply sqlfmt 3e0f900 (#11) --- .../test_get_column_values_use_default.sql | 5 +- macros/cross_db_utils/datediff.sql | 52 +++++++++---------- macros/cross_db_utils/width_bucket.sql | 6 ++- macros/sql/haversine_distance.sql | 18 +++---- 4 files changed, 39 insertions(+), 42 deletions(-) diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 3140021a..eada09bd 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -33,9 +33,8 @@ with select * from actual where - actual.actual_column_value not in ( - select expected.expected_column_value from expected - ) + actual.actual_column_value + not in (select expected.expected_column_value from expected) ) select * diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 48be529d..9d3d17fa 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -28,63 +28,61 @@ datetime_diff( {% if datepart == "year" %} ( - date_part('year', ({{ second_date }})::date) - date_part( - 'year', ({{ first_date }})::date - ) + date_part('year', ({{ second_date }})::date) + - date_part('year', ({{ first_date }})::date) ) {% elif datepart == "quarter" %} ( - {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + date_part( - 'quarter', ({{ second_date }})::date - ) - date_part('quarter', ({{ first_date }})::date) + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + + date_part('quarter', ({{ second_date }})::date) + - date_part('quarter', ({{ first_date }})::date) ) {% elif datepart == "month" %} ( - {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + date_part( - 'month', ({{ second_date }})::date - ) - date_part('month', ({{ first_date }})::date) + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + + date_part('month', ({{ second_date }})::date) + - date_part('month', ({{ first_date }})::date) ) {% elif datepart == "day" %}(({{ second_date }})::date - ({{ first_date }})::date) {% elif datepart == "week" %} ( {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case when - date_part('dow', ({{ first_date }})::timestamp) <= date_part( - 'dow', ({{ second_date }})::timestamp - ) + date_part('dow', ({{ first_date }})::timestamp) + <= date_part('dow', ({{ second_date }})::timestamp) then case when {{ first_date }} <= {{ second_date }} then 0 else -1 end else case when {{ first_date }} <= {{ second_date }} then 1 else 0 end end ) {% elif datepart == "hour" %} ( - {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + date_part( - 'hour', ({{ second_date }})::timestamp - ) - date_part('hour', ({{ first_date }})::timestamp) + {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + + date_part('hour', ({{ second_date }})::timestamp) + - date_part('hour', ({{ first_date }})::timestamp) ) {% elif datepart == "minute" %} ( - {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + date_part( - 'minute', ({{ second_date }})::timestamp - ) - date_part('minute', ({{ first_date }})::timestamp) + {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + + date_part('minute', ({{ second_date }})::timestamp) + - date_part('minute', ({{ first_date }})::timestamp) ) {% elif datepart == "second" %} ( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + floor( - date_part('second', ({{ second_date }})::timestamp) - ) - floor(date_part('second', ({{ first_date }})::timestamp)) + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + + floor(date_part('second', ({{ second_date }})::timestamp)) + - floor(date_part('second', ({{ first_date }})::timestamp)) ) {% elif datepart == "millisecond" %} ( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + floor( - date_part('millisecond', ({{ second_date }})::timestamp) - ) - floor(date_part('millisecond', ({{ first_date }})::timestamp)) + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + + floor(date_part('millisecond', ({{ second_date }})::timestamp)) + - floor(date_part('millisecond', ({{ first_date }})::timestamp)) ) {% elif datepart == "microsecond" %} ( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + floor( - date_part('microsecond', ({{ second_date }})::timestamp) - ) - floor(date_part('microsecond', ({{ first_date }})::timestamp)) + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + + floor(date_part('microsecond', ({{ second_date }})::timestamp)) + - floor(date_part('microsecond', ({{ first_date }})::timestamp)) ) {% else %} {{ diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index 4c63adcf..7bafb8cd 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -26,8 +26,9 @@ then 1 else 0 end +) -- Anything over max_value goes the N+1 bucket -) + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) ++ least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} @@ -45,8 +46,9 @@ then 1 else 0 end +) -- Anything over max_value goes the N+1 bucket -) + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) ++ least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 3527cd73..3730ff4c 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -32,11 +32,10 @@ The arguments should be float type. * 3961 * asin( sqrt( - power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + cos( - radians({{ lat1 }}) - ) * cos(radians({{ lat2 }})) * power( - (sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2 - ) + power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + + cos(radians({{ lat1 }})) + * cos(radians({{ lat2 }})) + * power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2) ) ) * {{ conversion_rate }} @@ -63,11 +62,10 @@ The arguments should be float type. * 3961 * asin( sqrt( - power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + cos( - {{ radians_lat1 }} - ) * cos({{ radians_lat2 }}) * power( - sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2 - ) + power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + + cos({{ radians_lat1 }}) + * cos({{ radians_lat2 }}) + * power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2) ) ) * {{ conversion_rate }} From a70e75d7a89e64a8f8b0a5641384cbaeec6a34ad Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Mon, 14 Nov 2022 16:30:48 -0700 Subject: [PATCH 13/22] chore: apply sqlfmt b792a79 (#13) --- .../insert_by_period_materialization.sql | 120 +++++++++--------- macros/sql/date_spine.sql | 38 +++--- macros/sql/get_column_values.sql | 15 +-- macros/sql/get_query_results_as_dict.sql | 2 +- macros/sql/get_relations_by_pattern.sql | 2 +- macros/sql/get_relations_by_prefix.sql | 2 +- macros/sql/get_tables_by_pattern_sql.sql | 32 ++--- 7 files changed, 106 insertions(+), 105 deletions(-) diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index ff8145e0..df9676e9 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -29,7 +29,7 @@ period ) -%} -{% call statement('period_boundaries', fetch_result=True) -%} +{% call statement("period_boundaries", fetch_result=True) -%} with data as ( select @@ -52,11 +52,12 @@ select stop_timestamp, {{ dbt_utils.datediff("start_timestamp", "stop_timestamp", period) }} + 1 as num_periods -from data {%- endcall %} +from data +{%- endcall %} {%- endmacro %} - {% macro get_period_sql( +{% macro get_period_sql( target_cols_csv, sql, timestamp_field, @@ -65,20 +66,20 @@ from data {%- endcall %} stop_timestamp, offset ) -%} - {{ - return( - adapter.dispatch("get_period_sql", "dbt_utils")( - target_cols_csv, - sql, - timestamp_field, - period, - start_timestamp, - stop_timestamp, - offset, - ) +{{ + return( + adapter.dispatch("get_period_sql", "dbt_utils")( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset, ) - }} - {% endmacro %} + ) +}} +{% endmacro %} {% macro default__get_period_sql( target_cols_csv, @@ -96,65 +97,63 @@ from data {%- endcall %} "{{timestamp_field}}" < '{{stop_timestamp}}'::timestamp) {%- endset -%} - {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} +{%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} select {{ target_cols_csv }} from ({{ filtered_sql }}) {%- endmacro %} - {% materialization insert_by_period, default -%} - {%- set timestamp_field = config.require("timestamp_field") -%} - {%- set start_date = config.require("start_date") -%} - {%- set stop_date = config.get("stop_date") or "" -%} - {%- set period = config.get("period") or "week" -%} +{% materialization insert_by_period, default -%} +{%- set timestamp_field = config.require("timestamp_field") -%} +{%- set start_date = config.require("start_date") -%} +{%- set stop_date = config.get("stop_date") or "" -%} +{%- set period = config.get("period") or "week" -%} - {%- if sql.find("__PERIOD_FILTER__") == -1 -%} +{%- if sql.find("__PERIOD_FILTER__") == -1 -%} {%- set error_message -%} Model '{{ model.unique_id }}' does not include the required string '__PERIOD_FILTER__' in its sql {%- endset -%} - {{ exceptions.raise_compiler_error(error_message) }} - {%- endif -%} +{{ exceptions.raise_compiler_error(error_message) }} +{%- endif -%} - {%- set identifier = model["name"] -%} +{%- set identifier = model["name"] -%} - {%- set old_relation = adapter.get_relation( +{%- set old_relation = adapter.get_relation( database=database, schema=schema, identifier=identifier ) -%} - {%- set target_relation = api.Relation.create( +{%- set target_relation = api.Relation.create( identifier=identifier, schema=schema, type="table" ) -%} - {%- set non_destructive_mode = flags.NON_DESTRUCTIVE == True -%} - {%- set full_refresh_mode = flags.FULL_REFRESH == True -%} +{%- set non_destructive_mode = flags.NON_DESTRUCTIVE == True -%} +{%- set full_refresh_mode = flags.FULL_REFRESH == True -%} - {%- set exists_as_table = old_relation is not none and old_relation.is_table -%} - {%- set exists_not_as_table = ( - old_relation is not none and not old_relation.is_table -) -%} +{%- set exists_as_table = old_relation is not none and old_relation.is_table -%} +{%- set exists_not_as_table = old_relation is not none and not old_relation.is_table -%} - {%- set should_truncate = ( +{%- set should_truncate = ( non_destructive_mode and full_refresh_mode and exists_as_table ) -%} - {%- set should_drop = not should_truncate and ( +{%- set should_drop = not should_truncate and ( full_refresh_mode or exists_not_as_table ) -%} - {%- set force_create = flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE -%} - - -- setup - {% if old_relation is none -%} - -- noop - {%- elif should_truncate -%} {{ adapter.truncate_relation(old_relation) }} - {%- elif should_drop -%} - {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} - {%- endif %} - - {{ run_hooks(pre_hooks, inside_transaction=False) }} - - -- `begin` happens here, so `commit` after it to finish the transaction - {{ run_hooks(pre_hooks, inside_transaction=True) }} - {% call statement() -%} - begin -- make extra sure we've closed out the transaction +{%- set force_create = flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE -%} + +-- setup +{% if old_relation is none -%} +-- noop +{%- elif should_truncate -%} {{ adapter.truncate_relation(old_relation) }} +{%- elif should_drop -%} +{{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} +{%- endif %} + +{{ run_hooks(pre_hooks, inside_transaction=False) }} + +-- `begin` happens here, so `commit` after it to finish the transaction +{{ run_hooks(pre_hooks, inside_transaction=True) }} +{% call statement() -%} +begin -- make extra sure we've closed out the transaction ; commit ; @@ -163,7 +162,7 @@ commit -- build model {% if force_create or old_relation is none -%} {# Create an empty target table -#} -{% call statement('main') -%} +{% call statement("main") -%} {%- set empty_sql = sql | replace("__PERIOD_FILTER__", "false") -%} {{ create_table_as(False, target_relation, empty_sql) }} {%- endcall %} @@ -198,8 +197,7 @@ commit start_timestamp, stop_timestamp, i, -) %} -{{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} +) %} {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} {%- endcall %} {{ @@ -209,9 +207,12 @@ commit }} {%- set name = "main-" ~ i -%} {% call statement(name, fetch_result=True) -%} -insert into {{ target_relation }} ({{ target_cols_csv }}) -(select {{ target_cols_csv }} from {{ tmp_relation.include(schema=False) }}) -; + insert into {{target_relation}} ({{target_cols_csv}}) + ( + select + {{target_cols_csv}} + from {{tmp_relation.include(schema=False)}} + ); {%- endcall %} {% set result = load_result("main-" ~ i) %} {% if "response" in result.keys() %} {# added in v0.19.0 #} @@ -254,9 +255,8 @@ commit {%- set status_string = "INSERT " ~ loop_vars["sum_rows_inserted"] -%} -{% call noop_statement('main', status_string) -%} -- no-op -{%- endcall %} +{% call noop_statement("main", status_string) -%} {%- endcall %} -- Return the relations created in this materialization {{ return({"relations": [target_relation]}) }} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index 8eef70d1..5cb83f36 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -9,31 +9,31 @@ {%- endmacro %} {% macro default__get_intervals_between(start_date, end_date, datepart) -%} -{%- call statement('get_intervals_between', fetch_result=True) %} +{%- call statement("get_intervals_between", fetch_result=True) %} -select - {{ dbt_utils.datediff(start_date, end_date, datepart) }} +select {{ dbt_utils.datediff(start_date, end_date, datepart) }} - {%- endcall -%} +{%- endcall -%} - {%- set value_list = load_result("get_intervals_between") -%} +{%- set value_list = load_result("get_intervals_between") -%} - {%- if value_list and value_list["data"] -%} - {%- set values = value_list["data"] | map(attribute=0) | list %} - {{ return(values[0]) }} - {%- else -%} {{ return(1) }} - {%- endif -%} +{%- if value_list and value_list["data"] -%} +{%- set values = value_list["data"] | map(attribute=0) | list %} {{ return(values[0]) }} +{%- else -%} {{ return(1) }} +{%- endif -%} {%- endmacro %} - {% macro date_spine(datepart, start_date, end_date) %} - {{ - return( - adapter.dispatch("date_spine", "dbt_utils")( - datepart, start_date, end_date - ) - ) - }} - {%- endmacro %} + + + + +{% macro date_spine(datepart, start_date, end_date) %} +{{ + return( + adapter.dispatch("date_spine", "dbt_utils")(datepart, start_date, end_date) + ) +}} +{%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} {# call as follows: diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index 0515e748..df3d98d0 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -27,7 +27,7 @@ {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} {% set relation_exists = (load_relation(target_relation)) is not none %} -{%- call statement('get_column_values', fetch_result=true) %} +{%- call statement("get_column_values", fetch_result=true) %} {%- if not relation_exists and default is none -%} @@ -62,14 +62,13 @@ order by {{ order_by }} {% endif %} - {%- endcall -%} +{%- endcall -%} - {%- set value_list = load_result("get_column_values") -%} +{%- set value_list = load_result("get_column_values") -%} - {%- if value_list and value_list["data"] -%} - {%- set values = value_list["data"] | map(attribute=0) | list %} - {{ return(values) }} - {%- else -%} {{ return(default) }} - {%- endif -%} +{%- if value_list and value_list["data"] -%} +{%- set values = value_list["data"] | map(attribute=0) | list %} {{ return(values) }} +{%- else -%} {{ return(default) }} +{%- endif -%} {%- endmacro %} diff --git a/macros/sql/get_query_results_as_dict.sql b/macros/sql/get_query_results_as_dict.sql index b4fc3e61..f2199767 100644 --- a/macros/sql/get_query_results_as_dict.sql +++ b/macros/sql/get_query_results_as_dict.sql @@ -5,7 +5,7 @@ {% macro default__get_query_results_as_dict(query) %} {# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} -{%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} +{%- call statement("get_query_results", fetch_result=True, auto_begin=false) -%} {{ query }} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 41e02ae6..c1937a39 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -14,7 +14,7 @@ schema_pattern, table_pattern, exclude="", database=target.database ) %} -{%- call statement('get_tables', fetch_result=True) %} +{%- call statement("get_tables", fetch_result=True) %} {{ dbt_utils.get_tables_by_pattern_sql( diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index cc458914..d5c7af09 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -14,7 +14,7 @@ schema, prefix, exclude="", database=target.database ) %} -{%- call statement('get_tables', fetch_result=True) %} +{%- call statement("get_tables", fetch_result=True) %} {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index 8a579f29..a639b868 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -25,16 +25,18 @@ where and table_name not ilike '{{ exclude }}' {% endmacro %} - {% macro bigquery__get_tables_by_pattern_sql( + + +{% macro bigquery__get_tables_by_pattern_sql( schema_pattern, table_pattern, exclude="", database=target.database ) %} - {% if "%" in schema_pattern %} - {% set schemata = dbt_utils._bigquery__get_matching_schemata( +{% if "%" in schema_pattern %} +{% set schemata = dbt_utils._bigquery__get_matching_schemata( schema_pattern, database ) %} - {% else %} {% set schemata = [schema_pattern] %} - {% endif %} +{% else %} {% set schemata = [schema_pattern] %} +{% endif %} {% set sql %} {% for schema in schemata %} @@ -52,28 +54,28 @@ where {% endfor %} {% endset %} - {{ return(sql) }} +{{ return(sql) }} - {% endmacro %} +{% endmacro %} - {% macro _bigquery__get_matching_schemata(schema_pattern, database) %} - {% if execute %} +{% macro _bigquery__get_matching_schemata(schema_pattern, database) %} +{% if execute %} {% set sql %} select schema_name from {{ adapter.quote(database) }}.INFORMATION_SCHEMA.SCHEMATA where lower(schema_name) like lower('{{ schema_pattern }}') {% endset %} - {% set results = run_query(sql) %} +{% set results = run_query(sql) %} - {% set schemata = results.columns["schema_name"].values() %} +{% set schemata = results.columns["schema_name"].values() %} - {{ return(schemata) }} +{{ return(schemata) }} - {% else %} {{ return([]) }} +{% else %} {{ return([]) }} - {% endif %} +{% endif %} - {% endmacro %} +{% endmacro %} From 749446c83ca8d024cfd4e39a4d7b389aff44db70 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Tue, 29 Nov 2022 20:07:53 -0700 Subject: [PATCH 14/22] chore: apply sqlfmt a7ed980 (#14) --- macros/cross_db_utils/concat.sql | 2 ++ macros/generic_tests/not_constant.sql | 2 ++ macros/sql/date_spine.sql | 2 ++ macros/sql/get_column_values.sql | 2 ++ 4 files changed, 8 insertions(+) diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 140a7787..74b9fb30 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,3 +1,5 @@ + + {% macro concat(fields) -%} {{ return(adapter.dispatch("concat", "dbt_utils")(fields)) }} {%- endmacro %} diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index d72f6d93..842eaf5c 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -4,6 +4,8 @@ {% endtest %} {% macro default__test_not_constant(model, column_name) %} + + {# In TSQL, subquery aggregate columns need aliases #} {# thus: a filler col name, 'filler_column' #} select count(distinct {{ column_name }}) as filler_column diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index 5cb83f36..f04e4d6c 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -36,6 +36,8 @@ select {{ dbt_utils.datediff(start_date, end_date, datepart) }} {%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} + + {# call as follows: date_spine( diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index df3d98d0..f5a0f53d 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -51,6 +51,8 @@ }} {{ return(default) }} {%- else -%} + + select {{ column }} as value from {{ target_relation }} From eb7aec4500fef340f2f98865a6c03bcc775b4fa3 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Sun, 11 Dec 2022 16:50:11 -0600 Subject: [PATCH 15/22] chore: apply sqlfmt 4f9a917 (#16) --- integration_tests/macros/assert_equal_values.sql | 4 ++-- ...ssert_get_query_results_as_dict_objects_equal.sql | 12 ++++++------ macros/cross_db_utils/_is_ephemeral.sql | 4 ++-- macros/cross_db_utils/listagg.sql | 6 +++--- macros/cross_db_utils/width_bucket.sql | 8 ++++---- .../insert_by_period_materialization.sql | 8 ++++---- macros/sql/get_tables_by_pattern_sql.sql | 8 ++++---- macros/sql/union.sql | 8 ++++---- macros/web/get_url_path.sql | 4 ++-- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/integration_tests/macros/assert_equal_values.sql b/integration_tests/macros/assert_equal_values.sql index 48c0ff81..2eee477d 100644 --- a/integration_tests/macros/assert_equal_values.sql +++ b/integration_tests/macros/assert_equal_values.sql @@ -4,7 +4,7 @@ {# pass #} {% elif actual_object != expected_object %} - {% set msg %} +{% set msg %} Expected did not match actual ----------- @@ -17,7 +17,7 @@ ----------- --->{{ expected_object }}<--- - {% endset %} +{% endset %} {{ log(msg, info=True) }} diff --git a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql index 3578af3e..c7065506 100644 --- a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql +++ b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql @@ -30,11 +30,11 @@ Instead, we'll manually check that the values of these dictionaries are equivale {% if expected_keys != actual_keys %} {% set ns.pass = False %} - {% set ns.err_msg %} +{% set ns.err_msg %} The two dictionaries have different keys: expected_dictionary has keys: {{ expected_keys }} actual_dictionary has keys: {{ actual_keys }} - {% endset %} +{% endset %} {% else %} @@ -44,11 +44,11 @@ Instead, we'll manually check that the values of these dictionaries are equivale {% if expected_length != actual_length %} {% set ns.pass = False %} - {% set ns.err_msg %} +{% set ns.err_msg %} The {{ key }} column has different lengths: expected_dictionary[{{ key }}] has length {{ expected_length }} actual_dictionary[{{ key }}] has length {{ actual_length }} - {% endset %} +{% endset %} {% else %} @@ -57,11 +57,11 @@ Instead, we'll manually check that the values of these dictionaries are equivale {% set actual_value = actual_dictionary[key][i] %} {% if expected_value != actual_value %} {% set ns.pass = False %} - {% set ns.err_msg %} +{% set ns.err_msg %} The {{ key }} column has differing values: expected_dictionary[{{ key }}][{{ i }}] == {{ expected_value }} actual_dictionary[{{ key }}][{{ i }}] == {{ actual_value }} - {% endset %} +{% endset %} {% endif %} {% endfor %} diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index 54317376..fc078e51 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -5,11 +5,11 @@ {% set model_name = obj.name[(ephemeral_prefix | length) :] %} {% else %} {% set model_name = obj.name %} {%- endif -%} - {% set error_message %} +{% set error_message %} The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. `{{ model_name }}` is an ephemeral model. Consider making it a view or table instead. - {% endset %} +{% endset %} {%- do exceptions.raise_compiler_error(error_message) -%} {%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 75e8e8ff..29ca9cff 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -69,13 +69,13 @@ string_agg( {% if limit_num -%} {% set ns = namespace() %} {% set ns.delimiter_text_regex = delimiter_text | trim("'") %} - {% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} +{% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} {%- for char in special_chars.split(",") -%} - {% set escape_char %}\\{{ char }}{% endset %} +{% set escape_char %}\\{{ char }}{% endset %} {% set ns.delimiter_text_regex = ns.delimiter_text_regex | replace(char, escape_char) %} {%- endfor -%} - {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} +{% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} regexp_substr( listagg({{ measure }}, {{ delimiter_text }}) {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %}, diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index 7bafb8cd..39a232b4 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -11,9 +11,9 @@ {% macro default__width_bucket(expr, min_value, max_value, num_buckets) -%} - {% set bin_size -%} +{% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) - {%- endset %} +{%- endset %} ( -- to break ties when the amount is eaxtly at the bucket egde case @@ -33,9 +33,9 @@ {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} - {% set bin_size -%} +{% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) - {%- endset %} +{%- endset %} ( -- to break ties when the amount is exactly at the bucket edge case diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index df9676e9..02b56f3a 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -91,11 +91,11 @@ from data offset ) -%} - {%- set period_filter -%} +{%- set period_filter -%} ("{{timestamp_field}}" > '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' and "{{timestamp_field}}" <= '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' + interval '1 {{period}}' and "{{timestamp_field}}" < '{{stop_timestamp}}'::timestamp) - {%- endset -%} +{%- endset -%} {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} @@ -111,9 +111,9 @@ from ({{ filtered_sql }}) {%- set period = config.get("period") or "week" -%} {%- if sql.find("__PERIOD_FILTER__") == -1 -%} - {%- set error_message -%} +{%- set error_message -%} Model '{{ model.unique_id }}' does not include the required string '__PERIOD_FILTER__' in its sql - {%- endset -%} +{%- endset -%} {{ exceptions.raise_compiler_error(error_message) }} {%- endif -%} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index a639b868..5ec9b921 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -38,7 +38,7 @@ where {% else %} {% set schemata = [schema_pattern] %} {% endif %} - {% set sql %} +{% set sql %} {% for schema in schemata %} select distinct table_schema, @@ -52,7 +52,7 @@ where {% if not loop.last %} union all {% endif %} {% endfor %} - {% endset %} +{% endset %} {{ return(sql) }} @@ -62,10 +62,10 @@ where {% macro _bigquery__get_matching_schemata(schema_pattern, database) %} {% if execute %} - {% set sql %} +{% set sql %} select schema_name from {{ adapter.quote(database) }}.INFORMATION_SCHEMA.SCHEMATA where lower(schema_name) like lower('{{ schema_pattern }}') - {% endset %} +{% endset %} {% set results = run_query(sql) %} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index 4023b569..413a63c9 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -80,16 +80,16 @@ {%- set ordered_column_names = column_superset.keys() -%} {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} - {%- set relations_string -%} +{%- set relations_string -%} {%- for relation in relations -%} {{ relation.name }} {%- if not loop.last %}, {% endif -%} {%- endfor -%} - {%- endset -%} +{%- endset -%} - {%- set error_message -%} +{%- set error_message -%} There were no columns found to union for relations {{ relations_string }} - {%- endset -%} +{%- endset -%} {{ exceptions.raise_compiler_error(error_message) }} {%- endif -%} diff --git a/macros/web/get_url_path.sql b/macros/web/get_url_path.sql index bb518959..c295a13d 100644 --- a/macros/web/get_url_path.sql +++ b/macros/web/get_url_path.sql @@ -8,12 +8,12 @@ dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''" ) -%} - {%- set first_slash_pos -%} +{%- set first_slash_pos -%} coalesce( nullif({{dbt_utils.position("'/'", stripped_url)}}, 0), {{dbt_utils.position("'?'", stripped_url)}} - 1 ) - {%- endset -%} +{%- endset -%} {%- set parsed_path = dbt_utils.split_part( dbt_utils.right( From 6170f7740b69c95d932492dc32c05c2e2b5b807c Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Tue, 17 Jan 2023 15:18:45 -0700 Subject: [PATCH 16/22] chore: apply sqlfmt d620ac8 (#17) --- macros/cross_db_utils/current_timestamp.sql | 1 - macros/cross_db_utils/datatypes.sql | 1 - macros/cross_db_utils/replace.sql | 1 - macros/generic_tests/not_constant.sql | 2 -- macros/generic_tests/unique_combination_of_columns.sql | 2 -- macros/sql/date_spine.sql | 3 --- macros/sql/get_column_values.sql | 1 - macros/sql/get_tables_by_pattern_sql.sql | 1 - macros/sql/haversine_distance.sql | 1 - macros/sql/surrogate_key.sql | 1 - macros/sql/unpivot.sql | 1 - 11 files changed, 15 deletions(-) diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index 767c5d07..266a5948 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -11,7 +11,6 @@ current_timestamp::{{ dbt_utils.type_timestamp() }} {% macro bigquery__current_timestamp() %} current_timestamp {% endmacro %} - {% macro current_timestamp_in_utc() -%} {{ return(adapter.dispatch("current_timestamp_in_utc", "dbt_utils")()) }} {%- endmacro %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index a981ee22..bfcc8baa 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -12,7 +12,6 @@ {% macro snowflake__type_string() %} varchar {% endmacro %} - {# timestamp ------------------------------------------------- #} {%- macro type_timestamp() -%} {{ return(adapter.dispatch("type_timestamp", "dbt_utils")()) }} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index 46b5fc91..8a0e2375 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -7,5 +7,4 @@ replace({{ field }}, {{ old_chars }}, {{ new_chars }}) - {% endmacro %} diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index 842eaf5c..d85a3152 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -5,7 +5,6 @@ {% macro default__test_not_constant(model, column_name) %} - {# In TSQL, subquery aggregate columns need aliases #} {# thus: a filler col name, 'filler_column' #} select count(distinct {{ column_name }}) as filler_column @@ -14,5 +13,4 @@ from {{ model }} having count(distinct {{ column_name }}) = 1 - {% endmacro %} diff --git a/macros/generic_tests/unique_combination_of_columns.sql b/macros/generic_tests/unique_combination_of_columns.sql index 3507b33c..a0ab359b 100644 --- a/macros/generic_tests/unique_combination_of_columns.sql +++ b/macros/generic_tests/unique_combination_of_columns.sql @@ -32,7 +32,6 @@ {%- set columns_csv = column_list | join(", ") %} - with validation_errors as ( @@ -46,5 +45,4 @@ with select * from validation_errors - {% endmacro %} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index f04e4d6c..30064b79 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -25,8 +25,6 @@ select {{ dbt_utils.datediff(start_date, end_date, datepart) }} {%- endmacro %} - - {% macro date_spine(datepart, start_date, end_date) %} {{ return( @@ -37,7 +35,6 @@ select {{ dbt_utils.datediff(start_date, end_date, datepart) }} {% macro default__date_spine(datepart, start_date, end_date) %} - {# call as follows: date_spine( diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index f5a0f53d..8df1bc2a 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -52,7 +52,6 @@ {%- else -%} - select {{ column }} as value from {{ target_relation }} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index 5ec9b921..c7f85513 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -77,5 +77,4 @@ where {% endif %} - {% endmacro %} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 3730ff4c..e2585c13 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -43,7 +43,6 @@ The arguments should be float type. {%- endmacro %} - {% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} {% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} {% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index 56f69893..d54a177b 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -25,7 +25,6 @@ {%- endif -%} - {%- set fields = [] -%} {%- for field in field_list_xf -%} diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index 4cbd357a..8c133a35 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -82,7 +82,6 @@ Arguments: {%- endif %} {%- endfor %} - {%- for col in include_cols -%} select {%- for exclude_col in exclude %} {{ exclude_col }}, {%- endfor %} From 664fd6853c785b3838b96a886d0f822814801ff1 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Tue, 17 Jan 2023 15:38:23 -0700 Subject: [PATCH 17/22] chore: apply sqlfmt e6f9f4c (#18) --- integration_tests/macros/tests.sql | 1 - integration_tests/models/cross_db_utils/test_concat.sql | 1 - .../models/cross_db_utils/test_current_timestamp.sql | 1 - .../models/cross_db_utils/test_current_timestamp_in_utc.sql | 1 - integration_tests/models/cross_db_utils/test_date_trunc.sql | 1 - integration_tests/models/cross_db_utils/test_dateadd.sql | 1 - integration_tests/models/cross_db_utils/test_datediff.sql | 1 - integration_tests/models/cross_db_utils/test_hash.sql | 1 - integration_tests/models/cross_db_utils/test_last_day.sql | 1 - integration_tests/models/cross_db_utils/test_safe_cast.sql | 1 - integration_tests/models/cross_db_utils/test_split_part.sql | 1 - integration_tests/models/cross_db_utils/test_width_bucket.sql | 1 - integration_tests/models/datetime/test_date_spine.sql | 1 - integration_tests/models/generic_tests/test_recency.sql | 1 - integration_tests/models/sql/test_generate_series.sql | 1 - integration_tests/models/sql/test_get_column_values.sql | 1 - integration_tests/models/sql/test_pivot.sql | 1 - integration_tests/models/sql/test_pivot_apostrophe.sql | 1 - integration_tests/models/sql/test_safe_add.sql | 1 - integration_tests/models/sql/test_surrogate_key.sql | 1 - integration_tests/models/sql/test_union.sql | 1 - integration_tests/models/sql/test_union_base.sql | 1 - integration_tests/models/sql/test_unpivot.sql | 1 - integration_tests/models/sql/test_unpivot_bool.sql | 1 - integration_tests/models/sql/test_unpivot_original_api.sql | 1 - integration_tests/models/web/test_urls.sql | 1 - .../tests/sql/test_get_column_values_use_default.sql | 1 - macros/cross_db_utils/concat.sql | 2 -- macros/cross_db_utils/literal.sql | 1 - macros/generic_tests/not_constant.sql | 1 - 30 files changed, 31 deletions(-) diff --git a/integration_tests/macros/tests.sql b/integration_tests/macros/tests.sql index 046fab3a..be561059 100644 --- a/integration_tests/macros/tests.sql +++ b/integration_tests/macros/tests.sql @@ -1,4 +1,3 @@ - {% test assert_equal(model, actual, expected) %} select * from {{ model }} where {{ actual }} != {{ expected }} diff --git a/integration_tests/models/cross_db_utils/test_concat.sql b/integration_tests/models/cross_db_utils/test_concat.sql index 8c036c65..c48697a2 100644 --- a/integration_tests/models/cross_db_utils/test_concat.sql +++ b/integration_tests/models/cross_db_utils/test_concat.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_concat") }}) select {{ dbt_utils.concat(["input_1", "input_2"]) }} as actual, output as expected diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp.sql b/integration_tests/models/cross_db_utils/test_current_timestamp.sql index d6c2cf73..8d5e99df 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp.sql @@ -1,4 +1,3 @@ - -- how can we test this better? select {{ dbt_utils.current_timestamp() }} as actual, diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql index e88e934a..b3607bb9 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql @@ -1,4 +1,3 @@ - -- how can we test this better? select {{ dbt_utils.current_timestamp_in_utc() }} as actual, diff --git a/integration_tests/models/cross_db_utils/test_date_trunc.sql b/integration_tests/models/cross_db_utils/test_date_trunc.sql index a5b88aa3..307495ee 100644 --- a/integration_tests/models/cross_db_utils/test_date_trunc.sql +++ b/integration_tests/models/cross_db_utils/test_date_trunc.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_date_trunc") }}) select diff --git a/integration_tests/models/cross_db_utils/test_dateadd.sql b/integration_tests/models/cross_db_utils/test_dateadd.sql index 2f2dee99..7adacfa8 100644 --- a/integration_tests/models/cross_db_utils/test_dateadd.sql +++ b/integration_tests/models/cross_db_utils/test_dateadd.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_dateadd") }}) select diff --git a/integration_tests/models/cross_db_utils/test_datediff.sql b/integration_tests/models/cross_db_utils/test_datediff.sql index 4b6c95a6..bad3cbf1 100644 --- a/integration_tests/models/cross_db_utils/test_datediff.sql +++ b/integration_tests/models/cross_db_utils/test_datediff.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_datediff") }}) select diff --git a/integration_tests/models/cross_db_utils/test_hash.sql b/integration_tests/models/cross_db_utils/test_hash.sql index 6d526274..5f0cca83 100644 --- a/integration_tests/models/cross_db_utils/test_hash.sql +++ b/integration_tests/models/cross_db_utils/test_hash.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_hash") }}) select {{ dbt_utils.hash("input_1") }} as actual, output as expected diff --git a/integration_tests/models/cross_db_utils/test_last_day.sql b/integration_tests/models/cross_db_utils/test_last_day.sql index 4e6cec97..080e286b 100644 --- a/integration_tests/models/cross_db_utils/test_last_day.sql +++ b/integration_tests/models/cross_db_utils/test_last_day.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_last_day") }}) select diff --git a/integration_tests/models/cross_db_utils/test_safe_cast.sql b/integration_tests/models/cross_db_utils/test_safe_cast.sql index b2e6d82c..95a3acde 100644 --- a/integration_tests/models/cross_db_utils/test_safe_cast.sql +++ b/integration_tests/models/cross_db_utils/test_safe_cast.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_safe_cast") }}) select diff --git a/integration_tests/models/cross_db_utils/test_split_part.sql b/integration_tests/models/cross_db_utils/test_split_part.sql index 05c0d70f..79b98da0 100644 --- a/integration_tests/models/cross_db_utils/test_split_part.sql +++ b/integration_tests/models/cross_db_utils/test_split_part.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_split_part") }}) select diff --git a/integration_tests/models/cross_db_utils/test_width_bucket.sql b/integration_tests/models/cross_db_utils/test_width_bucket.sql index c2b79646..ef445ed1 100644 --- a/integration_tests/models/cross_db_utils/test_width_bucket.sql +++ b/integration_tests/models/cross_db_utils/test_width_bucket.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_width_bucket") }}) select diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index f2c2dd12..172e1208 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -1,4 +1,3 @@ - -- snowflake doesn't like this as a view because the `generate_series` -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are diff --git a/integration_tests/models/generic_tests/test_recency.sql b/integration_tests/models/generic_tests/test_recency.sql index 142490f4..fc91c3be 100644 --- a/integration_tests/models/generic_tests/test_recency.sql +++ b/integration_tests/models/generic_tests/test_recency.sql @@ -1,4 +1,3 @@ - {% if target.type == "postgres" %} select {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as today diff --git a/integration_tests/models/sql/test_generate_series.sql b/integration_tests/models/sql/test_generate_series.sql index 2e5acf19..fd104edf 100644 --- a/integration_tests/models/sql/test_generate_series.sql +++ b/integration_tests/models/sql/test_generate_series.sql @@ -1,4 +1,3 @@ - -- snowflake doesn't like this as a view because the `generate_series` -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index dfb17366..4445ecf8 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -1,4 +1,3 @@ - {% set column_values = dbt_utils.get_column_values( ref("data_get_column_values"), "field", default=[], order_by="field" ) %} diff --git a/integration_tests/models/sql/test_pivot.sql b/integration_tests/models/sql/test_pivot.sql index 730b7bec..b84b7345 100644 --- a/integration_tests/models/sql/test_pivot.sql +++ b/integration_tests/models/sql/test_pivot.sql @@ -1,4 +1,3 @@ - -- TODO: How do we make this work nicely on Snowflake too? {% if target.type == "snowflake" %} {% set column_values = ["RED", "BLUE"] %} {% set cmp = "ilike" %} diff --git a/integration_tests/models/sql/test_pivot_apostrophe.sql b/integration_tests/models/sql/test_pivot_apostrophe.sql index e7a340a3..b9f2274f 100644 --- a/integration_tests/models/sql/test_pivot_apostrophe.sql +++ b/integration_tests/models/sql/test_pivot_apostrophe.sql @@ -1,4 +1,3 @@ - -- TODO: How do we make this work nicely on Snowflake too? {% if target.type == "snowflake" %} {% set column_values = ["RED", "BLUE", "BLUE'S"] %} {% set cmp = "ilike" %} diff --git a/integration_tests/models/sql/test_safe_add.sql b/integration_tests/models/sql/test_safe_add.sql index 840871ff..2d6eb778 100644 --- a/integration_tests/models/sql/test_safe_add.sql +++ b/integration_tests/models/sql/test_safe_add.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_safe_add") }}) select {{ dbt_utils.safe_add("field_1", "field_2", "field_3") }} as actual, expected diff --git a/integration_tests/models/sql/test_surrogate_key.sql b/integration_tests/models/sql/test_surrogate_key.sql index a0da48ef..8f694083 100644 --- a/integration_tests/models/sql/test_surrogate_key.sql +++ b/integration_tests/models/sql/test_surrogate_key.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_surrogate_key") }}) select diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql index 1b14ff1f..e1b5e34c 100644 --- a/integration_tests/models/sql/test_union.sql +++ b/integration_tests/models/sql/test_union.sql @@ -1,2 +1 @@ - select id, name, favorite_color from {{ ref("test_union_base") }} diff --git a/integration_tests/models/sql/test_union_base.sql b/integration_tests/models/sql/test_union_base.sql index 045e651b..869baecd 100644 --- a/integration_tests/models/sql/test_union_base.sql +++ b/integration_tests/models/sql/test_union_base.sql @@ -1,2 +1 @@ - {{ dbt_utils.union_relations([ref("data_union_table_1"), ref("data_union_table_2")]) }} diff --git a/integration_tests/models/sql/test_unpivot.sql b/integration_tests/models/sql/test_unpivot.sql index 7198d0a3..a9081a6c 100644 --- a/integration_tests/models/sql/test_unpivot.sql +++ b/integration_tests/models/sql/test_unpivot.sql @@ -1,4 +1,3 @@ - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, diff --git a/integration_tests/models/sql/test_unpivot_bool.sql b/integration_tests/models/sql/test_unpivot_bool.sql index 82d7205b..2642afaf 100644 --- a/integration_tests/models/sql/test_unpivot_bool.sql +++ b/integration_tests/models/sql/test_unpivot_bool.sql @@ -1,4 +1,3 @@ - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, diff --git a/integration_tests/models/sql/test_unpivot_original_api.sql b/integration_tests/models/sql/test_unpivot_original_api.sql index c10d016e..83f6ddf6 100644 --- a/integration_tests/models/sql/test_unpivot_original_api.sql +++ b/integration_tests/models/sql/test_unpivot_original_api.sql @@ -1,4 +1,3 @@ - -- unpivot() was enhanced with 3 new parameters -- This test targets the original API. -- snowflake messes with these tests pretty badly since the diff --git a/integration_tests/models/web/test_urls.sql b/integration_tests/models/web/test_urls.sql index 066a0fcd..321dce5c 100644 --- a/integration_tests/models/web/test_urls.sql +++ b/integration_tests/models/web/test_urls.sql @@ -1,4 +1,3 @@ - with data as (select * from {{ ref("data_urls") }}) select diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index eada09bd..4394fe39 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -1,4 +1,3 @@ - {% set column_values = dbt_utils.get_column_values( ref("data_get_column_values_dropped"), "field", diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 74b9fb30..140a7787 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,5 +1,3 @@ - - {% macro concat(fields) -%} {{ return(adapter.dispatch("concat", "dbt_utils")(fields)) }} {%- endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index cf4c577a..d1b066e8 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,4 +1,3 @@ - {%- macro string_literal(value) -%} {{ return(adapter.dispatch("string_literal", "dbt_utils")(value)) }} {%- endmacro -%} diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index d85a3152..ab140e85 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -1,4 +1,3 @@ - {% test not_constant(model, column_name) %} {{ return(adapter.dispatch("test_not_constant", "dbt_utils")(model, column_name)) }} {% endtest %} From 8d818998dd7730bce71d3be6bb0eed885c5e53f1 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Tue, 24 Jan 2023 10:10:59 -0700 Subject: [PATCH 18/22] chore: apply sqlfmt d97af4e (#19) --- .../materializations/insert_by_period_materialization.sql | 6 ++++-- macros/sql/get_column_values.sql | 4 +++- macros/sql/get_relations_by_pattern.sql | 3 ++- macros/sql/get_relations_by_prefix.sql | 3 ++- macros/sql/surrogate_key.sql | 4 +++- macros/web/get_url_host.sql | 4 +++- 6 files changed, 17 insertions(+), 7 deletions(-) diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 02b56f3a..1a1f91aa 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -42,7 +42,8 @@ with -1, "nullif('" ~ stop_date ~ "','')::timestamp", ) - }}, {{ dbt_utils.current_timestamp() }} + }}, + {{ dbt_utils.current_timestamp() }} ) as stop_timestamp from "{{target_schema}}"."{{target_table}}" ) @@ -197,7 +198,8 @@ commit start_timestamp, stop_timestamp, i, -) %} {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} +) %} +{{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} {%- endcall %} {{ diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index 8df1bc2a..239478ed 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -48,7 +48,9 @@ ~ " does not exist. Returning the default value: " ~ default ) -}} {{ return(default) }} +}} + +{{ return(default) }} {%- else -%} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index c1937a39..457a2fe4 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -34,7 +34,8 @@ schema=row.table_schema, identifier=row.table_name, type=row.table_type, -) -%} {%- do tbl_relations.append(tbl_relation) -%} +) -%} +{%- do tbl_relations.append(tbl_relation) -%} {%- endfor -%} {{ return(tbl_relations) }} diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index d5c7af09..0f5aee98 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -30,7 +30,8 @@ schema=row.table_schema, identifier=row.table_name, type=row.table_type, -) -%} {%- do tbl_relations.append(tbl_relation) -%} +) -%} +{%- do tbl_relations.append(tbl_relation) -%} {%- endfor -%} {{ return(tbl_relations) }} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index d54a177b..7f42403f 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -31,7 +31,9 @@ {%- set _ = fields.append( "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" -) -%} {%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} +) -%} + +{%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} {%- endfor -%} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index b49a9f42..1de744b6 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -20,6 +20,8 @@ ), "'?'", 1, -) -%} {{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} +) -%} + +{{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} {%- endmacro %} From ab55e1086cd1d880820ef18682f3f9964ac85cae Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Thu, 26 Jan 2023 16:44:30 -0700 Subject: [PATCH 19/22] chore: apply sqlfmt 124890d (#20) --- .../assert_pretty_output_msg_is_string.sql | 6 ++++-- .../jinja_helpers/assert_pretty_time_is_string.sql | 6 ++++-- .../tests/jinja_helpers/test_slugify.sql | 3 ++- macros/cross_db_utils/any_value.sql | 5 ++++- macros/generic_tests/at_least_one.sql | 7 ++++--- macros/generic_tests/not_constant.sql | 7 ++++--- .../insert_by_period_materialization.sql | 11 ++++++----- macros/sql/surrogate_key.sql | 4 +++- 8 files changed, 31 insertions(+), 18 deletions(-) diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql index 6782e8d1..85509c83 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql @@ -1,5 +1,7 @@ +{% if dbt_utils.pretty_log_format() is string %} {# Return 0 rows for the test to pass #} -{% if dbt_utils.pretty_log_format() is string %} select 1 as col_name {{ limit_zero() }} +select 1 as col_name {{ limit_zero() }} +{% else %} {# Return >0 rows for the test to fail #} -{% else %} select 1 +select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql index a9ec4b19..ae151a86 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql @@ -1,5 +1,7 @@ +{% if dbt_utils.pretty_time() is string %} {# Return 0 rows for the test to pass #} -{% if dbt_utils.pretty_time() is string %} select 1 as col_name {{ limit_zero() }} +select 1 as col_name {{ limit_zero() }} +{% else %} {# Return >0 rows for the test to fail #} -{% else %} select 1 +select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/test_slugify.sql b/integration_tests/tests/jinja_helpers/test_slugify.sql index 13d0a2bc..48ac16ea 100644 --- a/integration_tests/tests/jinja_helpers/test_slugify.sql +++ b/integration_tests/tests/jinja_helpers/test_slugify.sql @@ -1,6 +1,7 @@ {% if dbt_utils.slugify("!Hell0 world-hi") == "hell0_world_hi" %} {# Return 0 rows for the test to pass #} select 1 as col_name {{ limit_zero() }} +{% else %} {# Return >0 rows for the test to fail #} -{% else %} select 1 as col_name +select 1 as col_name {% endif %} diff --git a/macros/cross_db_utils/any_value.sql b/macros/cross_db_utils/any_value.sql index 8f0668b0..6259e019 100644 --- a/macros/cross_db_utils/any_value.sql +++ b/macros/cross_db_utils/any_value.sql @@ -6,5 +6,8 @@ {% macro default__any_value(expression) -%} any_value({{ expression }}) {%- endmacro %} +{% macro postgres__any_value(expression) -%} {#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} -{% macro postgres__any_value(expression) -%} min({{ expression }}) {%- endmacro %} +min({{ expression }}) + +{%- endmacro %} diff --git a/macros/generic_tests/at_least_one.sql b/macros/generic_tests/at_least_one.sql index a516e103..5741c660 100644 --- a/macros/generic_tests/at_least_one.sql +++ b/macros/generic_tests/at_least_one.sql @@ -7,9 +7,10 @@ select * from ( - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - select count({{ column_name }}) as filler_column + select + {# In TSQL, subquery aggregate columns need aliases #} + {# thus: a filler col name, 'filler_column' #} + count({{ column_name }}) as filler_column from {{ model }} diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index ab140e85..df5efd2b 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -4,9 +4,10 @@ {% macro default__test_not_constant(model, column_name) %} -{# In TSQL, subquery aggregate columns need aliases #} -{# thus: a filler col name, 'filler_column' #} -select count(distinct {{ column_name }}) as filler_column +select + {# In TSQL, subquery aggregate columns need aliases #} + {# thus: a filler col name, 'filler_column' #} + count(distinct {{ column_name }}) as filler_column from {{ model }} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 1a1f91aa..c7dcecdd 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -154,8 +154,8 @@ from ({{ filtered_sql }}) -- `begin` happens here, so `commit` after it to finish the transaction {{ run_hooks(pre_hooks, inside_transaction=True) }} {% call statement() -%} -begin -- make extra sure we've closed out the transaction -; +begin +; -- make extra sure we've closed out the transaction commit ; {%- endcall %} @@ -219,8 +219,8 @@ commit {% set result = load_result("main-" ~ i) %} {% if "response" in result.keys() %} {# added in v0.19.0 #} {% set rows_inserted = result["response"]["rows_affected"] %} -{# older versions #} -{% else %} {% set rows_inserted = result["status"].split(" ")[2] | int %} +{% else %} {# older versions #} +{% set rows_inserted = result["status"].split(" ")[2] | int %} {% endif %} {%- set sum_rows_inserted = loop_vars["sum_rows_inserted"] + rows_inserted -%} @@ -257,8 +257,9 @@ commit {%- set status_string = "INSERT " ~ loop_vars["sum_rows_inserted"] -%} +{% call noop_statement("main", status_string) -%} -- no-op -{% call noop_statement("main", status_string) -%} {%- endcall %} +{%- endcall %} -- Return the relations created in this materialization {{ return({"relations": [target_relation]}) }} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index 7f42403f..97e5bd74 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -20,8 +20,10 @@ {%- for field in varargs %} {%- set _ = field_list_xf.append(field) -%} {%- endfor -%} +{%- else -%} + {# if using list, just set field_list_xf as field_list #} -{%- else -%} {%- set field_list_xf = field_list -%} +{%- set field_list_xf = field_list -%} {%- endif -%} From ae160c984bca0d4174d4bf597f9a076b9a2d600c Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Fri, 14 Apr 2023 14:45:05 -0600 Subject: [PATCH 20/22] chore: apply sqlfmt 54b8edd (#21) --- .../macros/assert_equal_values.sql | 18 +- integration_tests/macros/limit_zero.sql | 2 +- integration_tests/macros/tests.sql | 4 +- .../models/datetime/test_date_spine.sql | 10 +- .../models/generic_tests/test_recency.sql | 10 +- .../models/sql/test_get_column_values.sql | 32 +- .../test_get_filtered_columns_in_relation.sql | 4 +- .../models/sql/test_nullcheck_table.sql | 20 +- integration_tests/models/sql/test_pivot.sql | 2 +- .../models/sql/test_pivot_apostrophe.sql | 2 +- ...et_query_results_as_dict_objects_equal.sql | 70 +-- .../assert_pretty_output_msg_is_string.sql | 8 +- .../assert_pretty_time_is_string.sql | 8 +- .../tests/jinja_helpers/test_slugify.sql | 8 +- .../test_get_column_values_use_default.sql | 12 +- macros/cross_db_utils/_is_ephemeral.sql | 20 +- macros/cross_db_utils/_is_relation.sql | 19 +- macros/cross_db_utils/any_value.sql | 6 +- macros/cross_db_utils/bool_or.sql | 2 +- macros/cross_db_utils/cast_bool_to_text.sql | 10 +- macros/cross_db_utils/concat.sql | 2 +- macros/cross_db_utils/current_timestamp.sql | 18 +- macros/cross_db_utils/datatypes.sql | 12 +- macros/cross_db_utils/date_trunc.sql | 6 +- macros/cross_db_utils/dateadd.sql | 30 +- macros/cross_db_utils/datediff.sql | 158 +++---- .../cross_db_utils/escape_single_quotes.sql | 8 +- macros/cross_db_utils/except.sql | 8 +- macros/cross_db_utils/hash.sql | 6 +- macros/cross_db_utils/identifier.sql | 10 +- macros/cross_db_utils/intersect.sql | 8 +- macros/cross_db_utils/last_day.sql | 52 ++- macros/cross_db_utils/length.sql | 2 +- macros/cross_db_utils/listagg.sql | 127 +++--- macros/cross_db_utils/literal.sql | 2 +- macros/cross_db_utils/position.sql | 10 +- macros/cross_db_utils/replace.sql | 4 +- macros/cross_db_utils/right.sql | 28 +- macros/cross_db_utils/safe_cast.sql | 10 +- macros/cross_db_utils/split_part.sql | 16 +- macros/cross_db_utils/width_bucket.sql | 78 ++-- macros/generic_tests/accepted_range.sql | 56 +-- macros/generic_tests/at_least_one.sql | 26 +- macros/generic_tests/cardinality_equality.sql | 70 +-- macros/generic_tests/equal_rowcount.sql | 38 +- macros/generic_tests/equality.sql | 92 ++-- macros/generic_tests/expression_is_true.sql | 28 +- macros/generic_tests/fewer_rows_than.sql | 59 +-- .../mutually_exclusive_ranges.sql | 215 ++++----- macros/generic_tests/not_accepted_values.sql | 56 +-- macros/generic_tests/not_constant.sql | 18 +- macros/generic_tests/not_null_proportion.sql | 47 +- macros/generic_tests/recency.sql | 26 +- macros/generic_tests/relationships_where.sql | 54 +-- macros/generic_tests/sequential_values.sql | 80 ++-- macros/generic_tests/test_not_null_where.sql | 12 +- macros/generic_tests/test_unique_where.sql | 12 +- .../unique_combination_of_columns.sql | 72 +-- macros/jinja_helpers/log_info.sql | 4 +- macros/jinja_helpers/pretty_log_format.sql | 4 +- macros/jinja_helpers/pretty_time.sql | 4 +- macros/jinja_helpers/slugify.sql | 14 +- .../insert_by_period_materialization.sql | 413 +++++++++--------- macros/sql/date_spine.sql | 91 ++-- macros/sql/deduplicate.sql | 61 +-- macros/sql/generate_series.sql | 62 +-- macros/sql/get_column_values.sql | 99 ++--- .../sql/get_filtered_columns_in_relation.sql | 28 +- macros/sql/get_query_results_as_dict.sql | 26 +- macros/sql/get_relations_by_pattern.sql | 66 +-- macros/sql/get_relations_by_prefix.sql | 48 +- macros/sql/get_table_types_sql.sql | 62 +-- macros/sql/get_tables_by_pattern_sql.sql | 64 +-- macros/sql/get_tables_by_prefix_sql.sql | 28 +- macros/sql/groupby.sql | 6 +- macros/sql/haversine_distance.sql | 96 ++-- macros/sql/nullcheck.sql | 14 +- macros/sql/nullcheck_table.sql | 12 +- macros/sql/pivot.sql | 65 +-- macros/sql/safe_add.sql | 18 +- macros/sql/star.sql | 28 +- macros/sql/surrogate_key.sql | 54 ++- macros/sql/union.sql | 149 +++---- macros/sql/unpivot.sql | 135 +++--- macros/web/get_url_host.sql | 28 +- macros/web/get_url_parameter.sql | 16 +- macros/web/get_url_path.sql | 29 +- 87 files changed, 1828 insertions(+), 1719 deletions(-) diff --git a/integration_tests/macros/assert_equal_values.sql b/integration_tests/macros/assert_equal_values.sql index 2eee477d..e9eaa637 100644 --- a/integration_tests/macros/assert_equal_values.sql +++ b/integration_tests/macros/assert_equal_values.sql @@ -1,10 +1,10 @@ {% macro assert_equal_values(actual_object, expected_object) %} -{% if not execute %} + {% if not execute %} -{# pass #} -{% elif actual_object != expected_object %} + {# pass #} + {% elif actual_object != expected_object %} -{% set msg %} + {% set msg %} Expected did not match actual ----------- @@ -17,13 +17,13 @@ ----------- --->{{ expected_object }}<--- -{% endset %} + {% endset %} -{{ log(msg, info=True) }} + {{ log(msg, info=True) }} -select 'fail' + select 'fail' -{% else %} select 'ok' {{ limit_zero() }} + {% else %} select 'ok' {{ limit_zero() }} -{% endif %} + {% endif %} {% endmacro %} diff --git a/integration_tests/macros/limit_zero.sql b/integration_tests/macros/limit_zero.sql index 0752e234..f3dee37a 100644 --- a/integration_tests/macros/limit_zero.sql +++ b/integration_tests/macros/limit_zero.sql @@ -1,7 +1,7 @@ {% macro my_custom_macro() %} whatever {% endmacro %} {% macro limit_zero() %} -{{ return(adapter.dispatch("limit_zero", "dbt_utils")()) }} + {{ return(adapter.dispatch("limit_zero", "dbt_utils")()) }} {% endmacro %} {% macro default__limit_zero() %} {{ return("limit 0") }} {% endmacro %} diff --git a/integration_tests/macros/tests.sql b/integration_tests/macros/tests.sql index be561059..9ef0e19d 100644 --- a/integration_tests/macros/tests.sql +++ b/integration_tests/macros/tests.sql @@ -1,11 +1,11 @@ {% test assert_equal(model, actual, expected) %} -select * from {{ model }} where {{ actual }} != {{ expected }} + select * from {{ model }} where {{ actual }} != {{ expected }} {% endtest %} {% test not_empty_string(model, column_name) %} -select * from {{ model }} where {{ column_name }} = '' + select * from {{ model }} where {{ column_name }} = '' {% endtest %} diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index 172e1208..1bdf4dca 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -8,11 +8,15 @@ with date_spine as ( {% if target.type == "postgres" %} - {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} + {{ + dbt_utils.date_spine( + "day", "'2018-01-01'::date", "'2018-01-10'::date" + ) + }} {% elif target.type == "bigquery" %} - select cast(date_day as date) as date_day - from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) + select cast(date_day as date) as date_day + from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) {% else %}{{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} {% endif %} diff --git a/integration_tests/models/generic_tests/test_recency.sql b/integration_tests/models/generic_tests/test_recency.sql index fc91c3be..0b0d286a 100644 --- a/integration_tests/models/generic_tests/test_recency.sql +++ b/integration_tests/models/generic_tests/test_recency.sql @@ -1,12 +1,12 @@ {% if target.type == "postgres" %} -select {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as today + select {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as today {% else %} -select - cast( - {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as datetime - ) as today + select + cast( + {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as datetime + ) as today {% endif %} diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index 4445ecf8..b3c5ba8f 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -5,30 +5,30 @@ {% if target.type == "snowflake" %} -select - {% for val in column_values -%} + select + {% for val in column_values -%} - sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} - {%- if not loop.last %},{% endif -%} + sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} + {%- if not loop.last %},{% endif -%} - {%- endfor %} + {%- endfor %} -from {{ ref("data_get_column_values") }} + from {{ ref("data_get_column_values") }} {% else %} -select - {% for val in column_values -%} + select + {% for val in column_values -%} - {{ - dbt_utils.safe_cast( - "sum(case when field = '" ~ val ~ "' then 1 else 0 end)", - dbt_utils.type_string(), - ) - }} as count_{{ val }} {%- if not loop.last %},{% endif -%} + {{ + dbt_utils.safe_cast( + "sum(case when field = '" ~ val ~ "' then 1 else 0 end)", + dbt_utils.type_string(), + ) + }} as count_{{ val }} {%- if not loop.last %},{% endif -%} - {%- endfor %} + {%- endfor %} -from {{ ref("data_get_column_values") }} + from {{ ref("data_get_column_values") }} {% endif %} diff --git a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql index afc3ccaa..0690dd19 100644 --- a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql +++ b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql @@ -9,8 +9,8 @@ with select {% for column_name in column_names %} - max({{ column_name }}) as {{ column_name }} - {% if not loop.last %},{% endif %} + max({{ column_name }}) as {{ column_name }} + {% if not loop.last %},{% endif %} {% endfor %} from {{ ref("data_filtered_columns_in_relation") }} diff --git a/integration_tests/models/sql/test_nullcheck_table.sql b/integration_tests/models/sql/test_nullcheck_table.sql index 4eb60e4e..a2cec220 100644 --- a/integration_tests/models/sql/test_nullcheck_table.sql +++ b/integration_tests/models/sql/test_nullcheck_table.sql @@ -8,23 +8,23 @@ with nulled as ({{ dbt_utils.nullcheck_table(tbl) }}) {% if target.type == "snowflake" %} -select - field_1::varchar as field_1, - field_2::varchar as field_2, - field_3::varchar as field_3 + select + field_1::varchar as field_1, + field_2::varchar as field_2, + field_3::varchar as field_3 -from nulled + from nulled {% else %} -select + select - {{ dbt_utils.safe_cast("field_1", dbt_utils.type_string()) }} as field_1, + {{ dbt_utils.safe_cast("field_1", dbt_utils.type_string()) }} as field_1, - {{ dbt_utils.safe_cast("field_2", dbt_utils.type_string()) }} as field_2, + {{ dbt_utils.safe_cast("field_2", dbt_utils.type_string()) }} as field_2, - {{ dbt_utils.safe_cast("field_3", dbt_utils.type_string()) }} as field_3 + {{ dbt_utils.safe_cast("field_3", dbt_utils.type_string()) }} as field_3 -from nulled + from nulled {% endif %} diff --git a/integration_tests/models/sql/test_pivot.sql b/integration_tests/models/sql/test_pivot.sql index b84b7345..dcc0d9a1 100644 --- a/integration_tests/models/sql/test_pivot.sql +++ b/integration_tests/models/sql/test_pivot.sql @@ -1,6 +1,6 @@ -- TODO: How do we make this work nicely on Snowflake too? {% if target.type == "snowflake" %} -{% set column_values = ["RED", "BLUE"] %} {% set cmp = "ilike" %} + {% set column_values = ["RED", "BLUE"] %} {% set cmp = "ilike" %} {% else %} {% set column_values = ["red", "blue"] %} {% set cmp = "=" %} {% endif %} diff --git a/integration_tests/models/sql/test_pivot_apostrophe.sql b/integration_tests/models/sql/test_pivot_apostrophe.sql index b9f2274f..b1af6828 100644 --- a/integration_tests/models/sql/test_pivot_apostrophe.sql +++ b/integration_tests/models/sql/test_pivot_apostrophe.sql @@ -1,6 +1,6 @@ -- TODO: How do we make this work nicely on Snowflake too? {% if target.type == "snowflake" %} -{% set column_values = ["RED", "BLUE", "BLUE'S"] %} {% set cmp = "ilike" %} + {% set column_values = ["RED", "BLUE", "BLUE'S"] %} {% set cmp = "ilike" %} {% else %} {% set column_values = ["red", "blue", "blue's"] %} {% set cmp = "=" %} {% endif %} diff --git a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql index c7065506..db0f8dc4 100644 --- a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql +++ b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql @@ -7,11 +7,11 @@ {#- Handle snowflake casing silliness -#} {% if target.type == "snowflake" %} -{% set expected_dictionary = { - "COL_1": [1, 2, 3], - "COL_2": ["a", "b", "c"], - "COL_3": [True, False, none], -} %} + {% set expected_dictionary = { + "COL_1": [1, 2, 3], + "COL_2": ["a", "b", "c"], + "COL_3": [True, False, none], + } %} {% endif %} @@ -24,53 +24,53 @@ Instead, we'll manually check that the values of these dictionaries are equivale -#} {% set ns = namespace(pass=True, err_msg="") %} {% if execute %} -{#- Check that the dictionaries have the same keys -#} -{% set expected_keys = expected_dictionary.keys() | list | sort %} -{% set actual_keys = actual_dictionary.keys() | list | sort %} + {#- Check that the dictionaries have the same keys -#} + {% set expected_keys = expected_dictionary.keys() | list | sort %} + {% set actual_keys = actual_dictionary.keys() | list | sort %} -{% if expected_keys != actual_keys %} -{% set ns.pass = False %} -{% set ns.err_msg %} + {% if expected_keys != actual_keys %} + {% set ns.pass = False %} + {% set ns.err_msg %} The two dictionaries have different keys: expected_dictionary has keys: {{ expected_keys }} actual_dictionary has keys: {{ actual_keys }} -{% endset %} + {% endset %} -{% else %} + {% else %} -{% for key, value in expected_dictionary.items() %} -{% set expected_length = expected_dictionary[key] | length %} -{% set actual_length = actual_dictionary[key] | length %} + {% for key, value in expected_dictionary.items() %} + {% set expected_length = expected_dictionary[key] | length %} + {% set actual_length = actual_dictionary[key] | length %} -{% if expected_length != actual_length %} -{% set ns.pass = False %} -{% set ns.err_msg %} + {% if expected_length != actual_length %} + {% set ns.pass = False %} + {% set ns.err_msg %} The {{ key }} column has different lengths: expected_dictionary[{{ key }}] has length {{ expected_length }} actual_dictionary[{{ key }}] has length {{ actual_length }} -{% endset %} + {% endset %} -{% else %} + {% else %} -{% for i in range(value | length) %} -{% set expected_value = expected_dictionary[key][i] %} -{% set actual_value = actual_dictionary[key][i] %} -{% if expected_value != actual_value %} -{% set ns.pass = False %} -{% set ns.err_msg %} + {% for i in range(value | length) %} + {% set expected_value = expected_dictionary[key][i] %} + {% set actual_value = actual_dictionary[key][i] %} + {% if expected_value != actual_value %} + {% set ns.pass = False %} + {% set ns.err_msg %} The {{ key }} column has differing values: expected_dictionary[{{ key }}][{{ i }}] == {{ expected_value }} actual_dictionary[{{ key }}][{{ i }}] == {{ actual_value }} -{% endset %} + {% endset %} -{% endif %} -{% endfor %} -{% endif %} + {% endif %} + {% endfor %} + {% endif %} -{% endfor %} + {% endfor %} -{% endif %} + {% endif %} -{{ log(ns.err_msg, info=True) }} -select 1 as col_name {% if ns.pass %} {{ limit_zero() }} {% endif %} + {{ log(ns.err_msg, info=True) }} + select 1 as col_name {% if ns.pass %} {{ limit_zero() }} {% endif %} {% endif %} diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql index 85509c83..fea33594 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_output_msg_is_string.sql @@ -1,7 +1,7 @@ {% if dbt_utils.pretty_log_format() is string %} -{# Return 0 rows for the test to pass #} -select 1 as col_name {{ limit_zero() }} + {# Return 0 rows for the test to pass #} + select 1 as col_name {{ limit_zero() }} {% else %} -{# Return >0 rows for the test to fail #} -select 1 + {# Return >0 rows for the test to fail #} + select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql index ae151a86..a6347abd 100644 --- a/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql +++ b/integration_tests/tests/jinja_helpers/assert_pretty_time_is_string.sql @@ -1,7 +1,7 @@ {% if dbt_utils.pretty_time() is string %} -{# Return 0 rows for the test to pass #} -select 1 as col_name {{ limit_zero() }} + {# Return 0 rows for the test to pass #} + select 1 as col_name {{ limit_zero() }} {% else %} -{# Return >0 rows for the test to fail #} -select 1 + {# Return >0 rows for the test to fail #} + select 1 {% endif %} diff --git a/integration_tests/tests/jinja_helpers/test_slugify.sql b/integration_tests/tests/jinja_helpers/test_slugify.sql index 48ac16ea..9b72b797 100644 --- a/integration_tests/tests/jinja_helpers/test_slugify.sql +++ b/integration_tests/tests/jinja_helpers/test_slugify.sql @@ -1,7 +1,7 @@ {% if dbt_utils.slugify("!Hell0 world-hi") == "hell0_world_hi" %} -{# Return 0 rows for the test to pass #} -select 1 as col_name {{ limit_zero() }} + {# Return 0 rows for the test to pass #} + select 1 as col_name {{ limit_zero() }} {% else %} -{# Return >0 rows for the test to fail #} -select 1 as col_name + {# Return >0 rows for the test to fail #} + select 1 as col_name {% endif %} diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 4394fe39..e7d64be0 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -19,12 +19,12 @@ with actual as ( {% for val in column_values %} - select - {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} - as actual_column_value - {% if not loop.last %} - union all - {% endif %} + select + {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} + as actual_column_value + {% if not loop.last %} + union all + {% endif %} {% endfor %} ), diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index fc078e51..1bf39708 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -1,15 +1,15 @@ {% macro _is_ephemeral(obj, macro) %} -{%- if obj.is_cte -%} -{% set ephemeral_prefix = api.Relation.add_ephemeral_prefix("") %} -{% if obj.name.startswith(ephemeral_prefix) %} -{% set model_name = obj.name[(ephemeral_prefix | length) :] %} -{% else %} {% set model_name = obj.name %} -{%- endif -%} -{% set error_message %} + {%- if obj.is_cte -%} + {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix("") %} + {% if obj.name.startswith(ephemeral_prefix) %} + {% set model_name = obj.name[(ephemeral_prefix | length) :] %} + {% else %} {% set model_name = obj.name %} + {%- endif -%} + {% set error_message %} The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. `{{ model_name }}` is an ephemeral model. Consider making it a view or table instead. -{% endset %} -{%- do exceptions.raise_compiler_error(error_message) -%} -{%- endif -%} + {% endset %} + {%- do exceptions.raise_compiler_error(error_message) -%} + {%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/_is_relation.sql b/macros/cross_db_utils/_is_relation.sql index 2d46dfc5..d668c229 100644 --- a/macros/cross_db_utils/_is_relation.sql +++ b/macros/cross_db_utils/_is_relation.sql @@ -1,10 +1,13 @@ {% macro _is_relation(obj, macro) %} -{%- if not ( - obj is mapping - and obj.get("metadata", {}).get("type", "").endswith("Relation") -) -%} -{%- do exceptions.raise_compiler_error( - "Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj -) -%} -{%- endif -%} + {%- if not ( + obj is mapping + and obj.get("metadata", {}).get("type", "").endswith("Relation") + ) -%} + {%- do exceptions.raise_compiler_error( + "Macro " + ~ macro + ~ " expected a Relation but received the value: " + ~ obj + ) -%} + {%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/any_value.sql b/macros/cross_db_utils/any_value.sql index 6259e019..595d32d0 100644 --- a/macros/cross_db_utils/any_value.sql +++ b/macros/cross_db_utils/any_value.sql @@ -1,5 +1,5 @@ {% macro any_value(expression) -%} -{{ return(adapter.dispatch("any_value", "dbt_utils")(expression)) }} + {{ return(adapter.dispatch("any_value", "dbt_utils")(expression)) }} {% endmacro %} @@ -7,7 +7,7 @@ {% macro postgres__any_value(expression) -%} -{#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} -min({{ expression }}) + {#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} + min({{ expression }}) {%- endmacro %} diff --git a/macros/cross_db_utils/bool_or.sql b/macros/cross_db_utils/bool_or.sql index 10f4fbb6..04716878 100644 --- a/macros/cross_db_utils/bool_or.sql +++ b/macros/cross_db_utils/bool_or.sql @@ -1,5 +1,5 @@ {% macro bool_or(expression) -%} -{{ return(adapter.dispatch("bool_or", "dbt_utils")(expression)) }} + {{ return(adapter.dispatch("bool_or", "dbt_utils")(expression)) }} {% endmacro %} diff --git a/macros/cross_db_utils/cast_bool_to_text.sql b/macros/cross_db_utils/cast_bool_to_text.sql index eb8c768e..030d5681 100644 --- a/macros/cross_db_utils/cast_bool_to_text.sql +++ b/macros/cross_db_utils/cast_bool_to_text.sql @@ -1,14 +1,14 @@ {% macro cast_bool_to_text(field) %} -{{ adapter.dispatch("cast_bool_to_text", "dbt_utils")(field) }} + {{ adapter.dispatch("cast_bool_to_text", "dbt_utils")(field) }} {% endmacro %} {% macro default__cast_bool_to_text(field) %} -cast({{ field }} as {{ dbt_utils.type_string() }}) + cast({{ field }} as {{ dbt_utils.type_string() }}) {% endmacro %} {% macro redshift__cast_bool_to_text(field) %} -case - when {{ field }} is true then 'true' when {{ field }} is false then 'false' -end::text + case + when {{ field }} is true then 'true' when {{ field }} is false then 'false' + end::text {% endmacro %} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 140a7787..e2be3cee 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,5 +1,5 @@ {% macro concat(fields) -%} -{{ return(adapter.dispatch("concat", "dbt_utils")(fields)) }} + {{ return(adapter.dispatch("concat", "dbt_utils")(fields)) }} {%- endmacro %} {% macro default__concat(fields) -%} {{ fields | join(" || ") }} {%- endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index 266a5948..aa8e237c 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,9 +1,9 @@ {% macro current_timestamp() -%} -{{ return(adapter.dispatch("current_timestamp", "dbt_utils")()) }} + {{ return(adapter.dispatch("current_timestamp", "dbt_utils")()) }} {%- endmacro %} {% macro default__current_timestamp() %} -current_timestamp::{{ dbt_utils.type_timestamp() }} + current_timestamp::{{ dbt_utils.type_timestamp() }} {% endmacro %} {% macro redshift__current_timestamp() %} getdate() {% endmacro %} @@ -12,24 +12,24 @@ current_timestamp::{{ dbt_utils.type_timestamp() }} {% macro current_timestamp_in_utc() -%} -{{ return(adapter.dispatch("current_timestamp_in_utc", "dbt_utils")()) }} + {{ return(adapter.dispatch("current_timestamp_in_utc", "dbt_utils")()) }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} -{{ dbt_utils.current_timestamp() }} + {{ dbt_utils.current_timestamp() }} {% endmacro %} {% macro snowflake__current_timestamp_in_utc() %} -convert_timezone( - 'UTC', {{ dbt_utils.current_timestamp() }} -)::{{ dbt_utils.type_timestamp() }} + convert_timezone( + 'UTC', {{ dbt_utils.current_timestamp() }} + )::{{ dbt_utils.type_timestamp() }} {% endmacro %} {% macro postgres__current_timestamp_in_utc() %} -(current_timestamp at time zone 'utc')::{{ dbt_utils.type_timestamp() }} + (current_timestamp at time zone 'utc')::{{ dbt_utils.type_timestamp() }} {% endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__current_timestamp_in_utc() %} -{{ return(dbt_utils.default__current_timestamp_in_utc()) }} + {{ return(dbt_utils.default__current_timestamp_in_utc()) }} {% endmacro %} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index bfcc8baa..ba12737c 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,6 +1,6 @@ {# string ------------------------------------------------- #} {%- macro type_string() -%} -{{ return(adapter.dispatch("type_string", "dbt_utils")()) }} + {{ return(adapter.dispatch("type_string", "dbt_utils")()) }} {%- endmacro -%} {% macro default__type_string() %} string {% endmacro %} @@ -14,7 +14,7 @@ {# timestamp ------------------------------------------------- #} {%- macro type_timestamp() -%} -{{ return(adapter.dispatch("type_timestamp", "dbt_utils")()) }} + {{ return(adapter.dispatch("type_timestamp", "dbt_utils")()) }} {%- endmacro -%} {% macro default__type_timestamp() %} timestamp {% endmacro %} @@ -26,7 +26,7 @@ {# float ------------------------------------------------- #} {%- macro type_float() -%} -{{ return(adapter.dispatch("type_float", "dbt_utils")()) }} + {{ return(adapter.dispatch("type_float", "dbt_utils")()) }} {%- endmacro -%} {% macro default__type_float() %} float {% endmacro %} @@ -35,7 +35,7 @@ {# numeric ------------------------------------------------ #} {%- macro type_numeric() -%} -{{ return(adapter.dispatch("type_numeric", "dbt_utils")()) }} + {{ return(adapter.dispatch("type_numeric", "dbt_utils")()) }} {%- endmacro -%} {% macro default__type_numeric() %} numeric(28, 6) {% endmacro %} @@ -45,7 +45,7 @@ {# bigint ------------------------------------------------- #} {%- macro type_bigint() -%} -{{ return(adapter.dispatch("type_bigint", "dbt_utils")()) }} + {{ return(adapter.dispatch("type_bigint", "dbt_utils")()) }} {%- endmacro -%} {% macro default__type_bigint() %} bigint {% endmacro %} @@ -54,7 +54,7 @@ {# int ------------------------------------------------- #} {%- macro type_int() -%} -{{ return(adapter.dispatch("type_int", "dbt_utils")()) }} + {{ return(adapter.dispatch("type_int", "dbt_utils")()) }} {%- endmacro -%} {% macro default__type_int() %} int {% endmacro %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index 80c6d100..e173f6ad 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,12 +1,12 @@ {% macro date_trunc(datepart, date) -%} -{{ return(adapter.dispatch("date_trunc", "dbt_utils")(datepart, date)) }} + {{ return(adapter.dispatch("date_trunc", "dbt_utils")(datepart, date)) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) -%} -date_trunc('{{datepart}}', {{ date }}) + date_trunc('{{datepart}}', {{ date }}) {%- endmacro %} {% macro bigquery__date_trunc(datepart, date) -%} -timestamp_trunc(cast({{ date }} as timestamp), {{ datepart }}) + timestamp_trunc(cast({{ date }} as timestamp), {{ datepart }}) {%- endmacro %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 133846c6..fd8016cc 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,39 +1,43 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} -{{ - return( - adapter.dispatch("dateadd", "dbt_utils")( - datepart, interval, from_date_or_timestamp + {{ + return( + adapter.dispatch("dateadd", "dbt_utils")( + datepart, interval, from_date_or_timestamp + ) ) - ) -}} + }} {% endmacro %} {% macro default__dateadd(datepart, interval, from_date_or_timestamp) %} -dateadd({{ datepart }}, {{ interval }}, {{ from_date_or_timestamp }}) + dateadd({{ datepart }}, {{ interval }}, {{ from_date_or_timestamp }}) {% endmacro %} {% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %} -datetime_add( - cast({{ from_date_or_timestamp }} as datetime), - interval {{ interval }} {{ datepart }} -) + datetime_add( + cast({{ from_date_or_timestamp }} as datetime), + interval {{ interval }} {{ datepart }} + ) {% endmacro %} {% macro postgres__dateadd(datepart, interval, from_date_or_timestamp) %} -{{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) + {{ from_date_or_timestamp }} + ((interval '1 {{ datepart }}') * ({{ interval }})) {% endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %} -{{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }} + {{ + return( + dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp) + ) + }} {% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 9d3d17fa..3804ff88 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,96 +1,100 @@ {% macro datediff(first_date, second_date, datepart) %} -{{ - return( - adapter.dispatch("datediff", "dbt_utils")(first_date, second_date, datepart) - ) -}} + {{ + return( + adapter.dispatch("datediff", "dbt_utils")( + first_date, second_date, datepart + ) + ) + }} {% endmacro %} {% macro default__datediff(first_date, second_date, datepart) -%} -datediff({{ datepart }}, {{ first_date }}, {{ second_date }}) + datediff({{ datepart }}, {{ first_date }}, {{ second_date }}) {%- endmacro %} {% macro bigquery__datediff(first_date, second_date, datepart) -%} -datetime_diff( - cast({{ second_date }} as datetime), - cast({{ first_date }} as datetime), - {{ datepart }} -) + datetime_diff( + cast({{ second_date }} as datetime), + cast({{ first_date }} as datetime), + {{ datepart }} + ) {%- endmacro %} {% macro postgres__datediff(first_date, second_date, datepart) -%} -{% if datepart == "year" %} -( - date_part('year', ({{ second_date }})::date) - - date_part('year', ({{ first_date }})::date) -) -{% elif datepart == "quarter" %} -( - {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 - + date_part('quarter', ({{ second_date }})::date) - - date_part('quarter', ({{ first_date }})::date) -) -{% elif datepart == "month" %} -( - {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 - + date_part('month', ({{ second_date }})::date) - - date_part('month', ({{ first_date }})::date) -) -{% elif datepart == "day" %}(({{ second_date }})::date - ({{ first_date }})::date) -{% elif datepart == "week" %} -( - {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case - when - date_part('dow', ({{ first_date }})::timestamp) - <= date_part('dow', ({{ second_date }})::timestamp) - then case when {{ first_date }} <= {{ second_date }} then 0 else -1 end - else case when {{ first_date }} <= {{ second_date }} then 1 else 0 end - end -) -{% elif datepart == "hour" %} -( - {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 - + date_part('hour', ({{ second_date }})::timestamp) - - date_part('hour', ({{ first_date }})::timestamp) -) -{% elif datepart == "minute" %} -( - {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 - + date_part('minute', ({{ second_date }})::timestamp) - - date_part('minute', ({{ first_date }})::timestamp) -) -{% elif datepart == "second" %} -( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 - + floor(date_part('second', ({{ second_date }})::timestamp)) - - floor(date_part('second', ({{ first_date }})::timestamp)) -) -{% elif datepart == "millisecond" %} -( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 - + floor(date_part('millisecond', ({{ second_date }})::timestamp)) - - floor(date_part('millisecond', ({{ first_date }})::timestamp)) -) -{% elif datepart == "microsecond" %} -( - {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 - + floor(date_part('microsecond', ({{ second_date }})::timestamp)) - - floor(date_part('microsecond', ({{ first_date }})::timestamp)) -) -{% else %} -{{ - exceptions.raise_compiler_error( - "Unsupported datepart for macro datediff in postgres: {!r}".format(datepart) - ) -}} -{% endif %} + {% if datepart == "year" %} + ( + date_part('year', ({{ second_date }})::date) + - date_part('year', ({{ first_date }})::date) + ) + {% elif datepart == "quarter" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + + date_part('quarter', ({{ second_date }})::date) + - date_part('quarter', ({{ first_date }})::date) + ) + {% elif datepart == "month" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + + date_part('month', ({{ second_date }})::date) + - date_part('month', ({{ first_date }})::date) + ) + {% elif datepart == "day" %}(({{ second_date }})::date - ({{ first_date }})::date) + {% elif datepart == "week" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case + when + date_part('dow', ({{ first_date }})::timestamp) + <= date_part('dow', ({{ second_date }})::timestamp) + then case when {{ first_date }} <= {{ second_date }} then 0 else -1 end + else case when {{ first_date }} <= {{ second_date }} then 1 else 0 end + end + ) + {% elif datepart == "hour" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + + date_part('hour', ({{ second_date }})::timestamp) + - date_part('hour', ({{ first_date }})::timestamp) + ) + {% elif datepart == "minute" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + + date_part('minute', ({{ second_date }})::timestamp) + - date_part('minute', ({{ first_date }})::timestamp) + ) + {% elif datepart == "second" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + + floor(date_part('second', ({{ second_date }})::timestamp)) + - floor(date_part('second', ({{ first_date }})::timestamp)) + ) + {% elif datepart == "millisecond" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + + floor(date_part('millisecond', ({{ second_date }})::timestamp)) + - floor(date_part('millisecond', ({{ first_date }})::timestamp)) + ) + {% elif datepart == "microsecond" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + + floor(date_part('microsecond', ({{ second_date }})::timestamp)) + - floor(date_part('microsecond', ({{ first_date }})::timestamp)) + ) + {% else %} + {{ + exceptions.raise_compiler_error( + "Unsupported datepart for macro datediff in postgres: {!r}".format( + datepart + ) + ) + }} + {% endif %} {%- endmacro %} @@ -98,6 +102,6 @@ datetime_diff( {# redshift should use default instead of postgres #} {% macro redshift__datediff(first_date, second_date, datepart) -%} -{{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }} + {{ return(dbt_utils.default__datediff(first_date, second_date, datepart)) }} {%- endmacro %} diff --git a/macros/cross_db_utils/escape_single_quotes.sql b/macros/cross_db_utils/escape_single_quotes.sql index c42a3866..0a3c95a2 100644 --- a/macros/cross_db_utils/escape_single_quotes.sql +++ b/macros/cross_db_utils/escape_single_quotes.sql @@ -1,18 +1,18 @@ {% macro escape_single_quotes(expression) %} -{{ return(adapter.dispatch("escape_single_quotes", "dbt_utils")(expression)) }} + {{ return(adapter.dispatch("escape_single_quotes", "dbt_utils")(expression)) }} {% endmacro %} {# /*Default to replacing a single apostrophe with two apostrophes: they're -> they''re*/ #} {% macro default__escape_single_quotes(expression) -%} -{{ expression | replace("'", "''") }} + {{ expression | replace("'", "''") }} {%- endmacro %} {# /*Snowflake uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} {% macro snowflake__escape_single_quotes(expression) -%} -{{ expression | replace("'", "\\'") }} + {{ expression | replace("'", "\\'") }} {%- endmacro %} {# /*BigQuery uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} {% macro bigquery__escape_single_quotes(expression) -%} -{{ expression | replace("'", "\\'") }} + {{ expression | replace("'", "\\'") }} {%- endmacro %} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 1ddde746..2f208201 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,17 +1,17 @@ {% macro except() %} -{{ return(adapter.dispatch("except", "dbt_utils")()) }} + {{ return(adapter.dispatch("except", "dbt_utils")()) }} {% endmacro %} {% macro default__except() %} -except + except {% endmacro %} {% macro bigquery__except() %} -except -distinct + except + distinct {% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index aa1a166c..9674fefc 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,13 +1,13 @@ {% macro hash(field) -%} -{{ return(adapter.dispatch("hash", "dbt_utils")(field)) }} + {{ return(adapter.dispatch("hash", "dbt_utils")(field)) }} {%- endmacro %} {% macro default__hash(field) -%} -md5(cast({{ field }} as {{ dbt_utils.type_string() }})) + md5(cast({{ field }} as {{ dbt_utils.type_string() }})) {%- endmacro %} {% macro bigquery__hash(field) -%} -to_hex({{ dbt_utils.default__hash(field) }}) + to_hex({{ dbt_utils.default__hash(field) }}) {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index 8340e864..a4c1a73a 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -1,9 +1,9 @@ {% macro identifier(value) %} -{%- set error_message = " Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ ".format( - model.package_name, model.name -) -%} -{%- do exceptions.warn(error_message) -%} -{{ return(adapter.dispatch("identifier", "dbt_utils")(value)) }} + {%- set error_message = " Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name + ) -%} + {%- do exceptions.warn(error_message) -%} + {{ return(adapter.dispatch("identifier", "dbt_utils")(value)) }} {% endmacro %} {% macro default__identifier(value) -%} "{{ value }}" {%- endmacro %} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index d2eb752b..94f36cd9 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,17 +1,17 @@ {% macro intersect() %} -{{ return(adapter.dispatch("intersect", "dbt_utils")()) }} + {{ return(adapter.dispatch("intersect", "dbt_utils")()) }} {% endmacro %} {% macro default__intersect() %} -intersect + intersect {% endmacro %} {% macro bigquery__intersect() %} -intersect -distinct + intersect + distinct {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 0e9f6502..391cd4b5 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -3,49 +3,53 @@ This function has been tested with dateparts of month and quarters. Further testing is required to validate that it will work on other dateparts. */ {% macro last_day(date, datepart) %} -{{ return(adapter.dispatch("last_day", "dbt_utils")(date, datepart)) }} + {{ return(adapter.dispatch("last_day", "dbt_utils")(date, datepart)) }} {% endmacro %} {%- macro default_last_day(date, datepart) -%} -cast( - {{ - dbt_utils.dateadd( - "day", - "-1", - dbt_utils.dateadd(datepart, "1", dbt_utils.date_trunc(datepart, date)), - ) - }} as date -) + cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd( + datepart, "1", dbt_utils.date_trunc(datepart, date) + ), + ) + }} as date + ) {%- endmacro -%} {% macro default__last_day(date, datepart) -%} -{{ dbt_utils.default_last_day(date, datepart) }} + {{ dbt_utils.default_last_day(date, datepart) }} {%- endmacro %} {% macro postgres__last_day(date, datepart) -%} -{%- if datepart == "quarter" -%} --- postgres dateadd does not support quarter interval. -cast( - {{ - dbt_utils.dateadd( - "day", - "-1", - dbt_utils.dateadd("month", "3", dbt_utils.date_trunc(datepart, date)), + {%- if datepart == "quarter" -%} + -- postgres dateadd does not support quarter interval. + cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd( + "month", "3", dbt_utils.date_trunc(datepart, date) + ), + ) + }} as date ) - }} as date -) -{%- else -%} {{ dbt_utils.default_last_day(date, datepart) }} -{%- endif -%} + {%- else -%} {{ dbt_utils.default_last_day(date, datepart) }} + {%- endif -%} {%- endmacro %} {# redshift should use default instead of postgres #} {% macro redshift__last_day(date, datepart) %} -{{ return(dbt_utils.default__last_day(date, datepart)) }} + {{ return(dbt_utils.default__last_day(date, datepart)) }} {% endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index 2afdb89b..2e736fbd 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,5 +1,5 @@ {% macro length(expression) -%} -{{ return(adapter.dispatch("length", "dbt_utils")(expression)) }} + {{ return(adapter.dispatch("length", "dbt_utils")(expression)) }} {% endmacro %} diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 29ca9cff..18e58df9 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -1,90 +1,95 @@ {% macro listagg( measure, delimiter_text="','", order_by_clause=none, limit_num=none ) -%} -{{ - return( - adapter.dispatch("listagg", "dbt_utils")( - measure, delimiter_text, order_by_clause, limit_num + {{ + return( + adapter.dispatch("listagg", "dbt_utils")( + measure, delimiter_text, order_by_clause, limit_num + ) ) - ) -}} + }} {%- endmacro %} {% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} -{% if limit_num -%} -array_to_string( - array_slice( - array_agg({{ measure }}) - {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %}, - 0, - {{ limit_num }} - ), - {{ delimiter_text }} -) -{%- else %} -listagg({{ measure }}, {{ delimiter_text }}) -{% if order_by_clause -%} within group ({{ order_by_clause }}) -{%- endif %} -{%- endif %} + {% if limit_num -%} + array_to_string( + array_slice( + array_agg({{ measure }}) + {% if order_by_clause -%} + within group ({{ order_by_clause }}) + {%- endif %}, + 0, + {{ limit_num }} + ), + {{ delimiter_text }} + ) + {%- else %} + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group ({{ order_by_clause }}) + {%- endif %} + {%- endif %} {%- endmacro %} {% macro bigquery__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} -string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} - {% if limit_num -%} limit {{ limit_num }} {%- endif %} -) + string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + {% if limit_num -%} limit {{ limit_num }} {%- endif %} + ) {%- endmacro %} {% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} -{% if limit_num -%} -array_to_string( - ( - array_agg( - {{ measure }} {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + {% if limit_num -%} + array_to_string( + ( + array_agg( + {{ measure }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + ) + )[1:{{ limit_num }}], + {{ delimiter_text }} + ) + {%- else %} + string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} + {%- endif %} ) - )[1:{{ limit_num }}], - {{ delimiter_text }} -) -{%- else %} -string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} -) -{%- endif %} {%- endmacro %} {# if there are instances of delimiter_text within your measure, you cannot include a limit_num #} {% macro redshift__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} -{% if limit_num -%} -{% set ns = namespace() %} -{% set ns.delimiter_text_regex = delimiter_text | trim("'") %} -{% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} -{%- for char in special_chars.split(",") -%} -{% set escape_char %}\\{{ char }}{% endset %} -{% set ns.delimiter_text_regex = ns.delimiter_text_regex | replace(char, escape_char) %} -{%- endfor -%} + {% if limit_num -%} + {% set ns = namespace() %} + {% set ns.delimiter_text_regex = delimiter_text | trim("'") %} + {% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} + {%- for char in special_chars.split(",") -%} + {% set escape_char %}\\{{ char }}{% endset %} + {% set ns.delimiter_text_regex = ns.delimiter_text_regex | replace( + char, escape_char + ) %} + {%- endfor -%} -{% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} -regexp_substr( - listagg({{ measure }}, {{ delimiter_text }}) - {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %}, - {{ regex }} -) -{%- else %} -listagg({{ measure }}, {{ delimiter_text }}) -{% if order_by_clause -%} within group ({{ order_by_clause }}) -{%- endif %} -{%- endif %} + {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} + regexp_substr( + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %}, + {{ regex }} + ) + {%- else %} + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group ({{ order_by_clause }}) + {%- endif %} + {%- endif %} {%- endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index d1b066e8..26858c33 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,5 +1,5 @@ {%- macro string_literal(value) -%} -{{ return(adapter.dispatch("string_literal", "dbt_utils")(value)) }} + {{ return(adapter.dispatch("string_literal", "dbt_utils")(value)) }} {%- endmacro -%} {% macro default__string_literal(value) -%} '{{ value }}' {%- endmacro %} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index b8eac334..140d2680 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,16 +1,20 @@ {% macro position(substring_text, string_text) -%} -{{ return(adapter.dispatch("position", "dbt_utils")(substring_text, string_text)) }} + {{ + return( + adapter.dispatch("position", "dbt_utils")(substring_text, string_text) + ) + }} {% endmacro %} {% macro default__position(substring_text, string_text) %} -position({{ substring_text }} in {{ string_text }}) + position({{ substring_text }} in {{ string_text }}) {%- endmacro -%} {% macro bigquery__position(substring_text, string_text) %} -strpos({{ string_text }}, {{ substring_text }}) + strpos({{ string_text }}, {{ substring_text }}) {%- endmacro -%} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index 8a0e2375..66b859e7 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,10 +1,10 @@ {% macro replace(field, old_chars, new_chars) -%} -{{ return(adapter.dispatch("replace", "dbt_utils")(field, old_chars, new_chars)) }} + {{ return(adapter.dispatch("replace", "dbt_utils")(field, old_chars, new_chars)) }} {% endmacro %} {% macro default__replace(field, old_chars, new_chars) %} -replace({{ field }}, {{ old_chars }}, {{ new_chars }}) + replace({{ field }}, {{ old_chars }}, {{ new_chars }}) {% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 379012b7..841dc4f6 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,29 +1,33 @@ {% macro right(string_text, length_expression) -%} -{{ return(adapter.dispatch("right", "dbt_utils")(string_text, length_expression)) }} + {{ + return( + adapter.dispatch("right", "dbt_utils")(string_text, length_expression) + ) + }} {% endmacro %} {% macro default__right(string_text, length_expression) %} -right({{ string_text }}, {{ length_expression }}) + right({{ string_text }}, {{ length_expression }}) {%- endmacro -%} {% macro bigquery__right(string_text, length_expression) %} -case - when {{ length_expression }} = 0 - then '' - else substr({{ string_text }}, -1 * ({{ length_expression }})) -end + case + when {{ length_expression }} = 0 + then '' + else substr({{ string_text }}, -1 * ({{ length_expression }})) + end {%- endmacro -%} {% macro snowflake__right(string_text, length_expression) %} -case - when {{ length_expression }} = 0 - then '' - else right({{ string_text }}, {{ length_expression }}) -end + case + when {{ length_expression }} = 0 + then '' + else right({{ string_text }}, {{ length_expression }}) + end {%- endmacro -%} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index cd8dccda..3337beb2 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,20 +1,20 @@ {% macro safe_cast(field, type) %} -{{ return(adapter.dispatch("safe_cast", "dbt_utils")(field, type)) }} + {{ return(adapter.dispatch("safe_cast", "dbt_utils")(field, type)) }} {% endmacro %} {% macro default__safe_cast(field, type) %} -{# most databases don't support this function yet + {# most databases don't support this function yet so we just need to use cast #} -cast({{ field }} as {{ type }}) + cast({{ field }} as {{ type }}) {% endmacro %} {% macro snowflake__safe_cast(field, type) %} -try_cast({{ field }} as {{ type }}) + try_cast({{ field }} as {{ type }}) {% endmacro %} {% macro bigquery__safe_cast(field, type) %} -safe_cast({{ field }} as {{ type }}) + safe_cast({{ field }} as {{ type }}) {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index e5c204ae..5f10da88 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,23 +1,23 @@ {% macro split_part(string_text, delimiter_text, part_number) %} -{{ - return( - adapter.dispatch("split_part", "dbt_utils")( - string_text, delimiter_text, part_number + {{ + return( + adapter.dispatch("split_part", "dbt_utils")( + string_text, delimiter_text, part_number + ) ) - ) -}} + }} {% endmacro %} {% macro default__split_part(string_text, delimiter_text, part_number) %} -split_part({{ string_text }}, {{ delimiter_text }}, {{ part_number }}) + split_part({{ string_text }}, {{ delimiter_text }}, {{ part_number }}) {% endmacro %} {% macro bigquery__split_part(string_text, delimiter_text, part_number) %} -split({{ string_text }}, {{ delimiter_text }})[safe_offset({{ part_number - 1 }})] + split({{ string_text }}, {{ delimiter_text }})[safe_offset({{ part_number - 1 }})] {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index 39a232b4..452993de 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,56 +1,56 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} -{{ - return( - adapter.dispatch("width_bucket", "dbt_utils")( - expr, min_value, max_value, num_buckets + {{ + return( + adapter.dispatch("width_bucket", "dbt_utils")( + expr, min_value, max_value, num_buckets + ) ) - ) -}} + }} {% endmacro %} {% macro default__width_bucket(expr, min_value, max_value, num_buckets) -%} -{% set bin_size -%} + {% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) -{%- endset %} -( - -- to break ties when the amount is eaxtly at the bucket egde - case - when - mod( - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }}, - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} - ) - = 0 - then 1 - else 0 - end -) --- Anything over max_value goes the N+1 bucket -+ least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) + {%- endset %} + ( + -- to break ties when the amount is eaxtly at the bucket egde + case + when + mod( + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }}, + {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + ) + = 0 + then 1 + else 0 + end + ) + -- Anything over max_value goes the N+1 bucket + + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} -{% set bin_size -%} + {% set bin_size -%} (( {{ max_value }} - {{ min_value }} ) / {{ num_buckets }} ) -{%- endset %} -( - -- to break ties when the amount is exactly at the bucket edge - case - when - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }} - % {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} - = 0 - then 1 - else 0 - end -) --- Anything over max_value goes the N+1 bucket -+ least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) + {%- endset %} + ( + -- to break ties when the amount is exactly at the bucket edge + case + when + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }} + % {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + = 0 + then 1 + else 0 + end + ) + -- Anything over max_value goes the N+1 bucket + + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} -width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }}) + width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }}) {% endmacro %} diff --git a/macros/generic_tests/accepted_range.sql b/macros/generic_tests/accepted_range.sql index 68599ba6..aee6e7fb 100644 --- a/macros/generic_tests/accepted_range.sql +++ b/macros/generic_tests/accepted_range.sql @@ -1,44 +1,44 @@ {% test accepted_range( model, column_name, min_value=none, max_value=none, inclusive=true ) %} -{{ - return( - adapter.dispatch("test_accepted_range", "dbt_utils")( - model, column_name, min_value, max_value, inclusive + {{ + return( + adapter.dispatch("test_accepted_range", "dbt_utils")( + model, column_name, min_value, max_value, inclusive + ) ) - ) -}} + }} {% endtest %} {% macro default__test_accepted_range( model, column_name, min_value=none, max_value=none, inclusive=true ) %} -with - meet_condition as (select * from {{ model }}), + with + meet_condition as (select * from {{ model }}), - validation_errors as ( - select * - from meet_condition - where - -- never true, defaults to an empty result set. Exists to ensure any combo - -- of the `or` clauses below succeeds - 1 = 2 + validation_errors as ( + select * + from meet_condition + where + -- never true, defaults to an empty result set. Exists to ensure any + -- combo of the `or` clauses below succeeds + 1 = 2 - {%- if min_value is not none %} - -- records with a value >= min_value are permitted. The `not` flips this - -- to find records that don't meet the rule. - or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} - {%- endif %} + {%- if min_value is not none %} + -- records with a value >= min_value are permitted. The `not` + -- flips this to find records that don't meet the rule. + or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} + {%- endif %} - {%- if max_value is not none %} - -- records with a value <= max_value are permitted. The `not` flips this - -- to find records that don't meet the rule. - or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} - {%- endif %} - ) + {%- if max_value is not none %} + -- records with a value <= max_value are permitted. The `not` + -- flips this to find records that don't meet the rule. + or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} + {%- endif %} + ) -select * -from validation_errors + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/at_least_one.sql b/macros/generic_tests/at_least_one.sql index 5741c660..c13981af 100644 --- a/macros/generic_tests/at_least_one.sql +++ b/macros/generic_tests/at_least_one.sql @@ -1,21 +1,25 @@ {% test at_least_one(model, column_name) %} -{{ return(adapter.dispatch("test_at_least_one", "dbt_utils")(model, column_name)) }} + {{ + return( + adapter.dispatch("test_at_least_one", "dbt_utils")(model, column_name) + ) + }} {% endtest %} {% macro default__test_at_least_one(model, column_name) %} -select * -from - ( - select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count({{ column_name }}) as filler_column + select * + from + ( + select + {# In TSQL, subquery aggregate columns need aliases #} + {# thus: a filler col name, 'filler_column' #} + count({{ column_name }}) as filler_column - from {{ model }} + from {{ model }} - having count({{ column_name }}) = 0 + having count({{ column_name }}) = 0 - ) validation_errors + ) validation_errors {% endmacro %} diff --git a/macros/generic_tests/cardinality_equality.sql b/macros/generic_tests/cardinality_equality.sql index 06b30a6c..8ef1a901 100644 --- a/macros/generic_tests/cardinality_equality.sql +++ b/macros/generic_tests/cardinality_equality.sql @@ -1,41 +1,45 @@ {% test cardinality_equality(model, column_name, to, field) %} -{{ - return( - adapter.dispatch("test_cardinality_equality", "dbt_utils")( - model, column_name, to, field + {{ + return( + adapter.dispatch("test_cardinality_equality", "dbt_utils")( + model, column_name, to, field + ) ) - ) -}} + }} {% endtest %} {% macro default__test_cardinality_equality(model, column_name, to, field) %} -{# T-SQL does not let you use numbers as aliases for columns #} -{# Thus, no "GROUP BY 1" #} -with - table_a as ( - select {{ column_name }}, count(*) as num_rows - from {{ model }} - group by {{ column_name }} - ), - - table_b as ( - select {{ field }}, count(*) as num_rows from {{ to }} group by {{ field }} - ), - - except_a as (select * from table_a {{ dbt_utils.except() }} select * from table_b), - - except_b as (select * from table_b {{ dbt_utils.except() }} select * from table_a), - - unioned as ( - select * - from except_a - union all - select * - from except_b - ) - -select * -from unioned + {# T-SQL does not let you use numbers as aliases for columns #} + {# Thus, no "GROUP BY 1" #} + with + table_a as ( + select {{ column_name }}, count(*) as num_rows + from {{ model }} + group by {{ column_name }} + ), + + table_b as ( + select {{ field }}, count(*) as num_rows from {{ to }} group by {{ field }} + ), + + except_a as ( + select * from table_a {{ dbt_utils.except() }} select * from table_b + ), + + except_b as ( + select * from table_b {{ dbt_utils.except() }} select * from table_a + ), + + unioned as ( + select * + from except_a + union all + select * + from except_b + ) + + select * + from unioned {% endmacro %} diff --git a/macros/generic_tests/equal_rowcount.sql b/macros/generic_tests/equal_rowcount.sql index e0c638cb..d8404a6f 100644 --- a/macros/generic_tests/equal_rowcount.sql +++ b/macros/generic_tests/equal_rowcount.sql @@ -1,31 +1,31 @@ {% test equal_rowcount(model, compare_model) %} -{{ - return( - adapter.dispatch("test_equal_rowcount", "dbt_utils")(model, compare_model) - ) -}} + {{ + return( + adapter.dispatch("test_equal_rowcount", "dbt_utils")(model, compare_model) + ) + }} {% endtest %} {% macro default__test_equal_rowcount(model, compare_model) %} -{#- - Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc="coalesce(diff_count, 0)") }} + {#- - Needs to be set at parse time, before we return '' below --#} + {{ config(fail_calc="coalesce(diff_count, 0)") }} -{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} {{ return("") }} {% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("") }} {% endif %} -with - a as (select count(*) as count_a from {{ model }}), - b as (select count(*) as count_b from {{ compare_model }}), - final as ( + with + a as (select count(*) as count_a from {{ model }}), + b as (select count(*) as count_b from {{ compare_model }}), + final as ( - select count_a, count_b, abs(count_a - count_b) as diff_count - from a - cross join b + select count_a, count_b, abs(count_a - count_b) as diff_count + from a + cross join b - ) + ) -select * -from final + select * + from final {% endmacro %} diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index cd3c033f..28135583 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -1,78 +1,78 @@ {% test equality(model, compare_model, compare_columns=None) %} -{{ - return( - adapter.dispatch("test_equality", "dbt_utils")( - model, compare_model, compare_columns + {{ + return( + adapter.dispatch("test_equality", "dbt_utils")( + model, compare_model, compare_columns + ) ) - ) -}} + }} {% endtest %} {% macro default__test_equality(model, compare_model, compare_columns=None) %} -{% set set_diff %} + {% set set_diff %} count(*) + coalesce(abs( sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - sum(case when which_diff = 'b_minus_a' then 1 else 0 end) ), 0) -{% endset %} + {% endset %} -{#- - Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc=set_diff) }} + {#- - Needs to be set at parse time, before we return '' below --#} + {{ config(fail_calc=set_diff) }} -{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} {{ return("") }} {% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("") }} {% endif %} --- setup -{%- do dbt_utils._is_relation(model, "test_equality") -%} + -- setup + {%- do dbt_utils._is_relation(model, "test_equality") -%} -{#- + {#- If the compare_cols arg is provided, we can run this test without querying the information schema — this allows the model to be an ephemeral model -#} -{%- if not compare_columns -%} -{%- do dbt_utils._is_ephemeral(model, "test_equality") -%} -{%- set compare_columns = adapter.get_columns_in_relation(model) | map( - attribute="quoted" -) -%} -{%- endif -%} + {%- if not compare_columns -%} + {%- do dbt_utils._is_ephemeral(model, "test_equality") -%} + {%- set compare_columns = adapter.get_columns_in_relation(model) | map( + attribute="quoted" + ) -%} + {%- endif -%} -{% set compare_cols_csv = compare_columns | join(", ") %} + {% set compare_cols_csv = compare_columns | join(", ") %} -with - a as (select * from {{ model }}), + with + a as (select * from {{ model }}), - b as (select * from {{ compare_model }}), + b as (select * from {{ compare_model }}), - a_minus_b as ( + a_minus_b as ( - select {{ compare_cols_csv }} - from a {{ dbt_utils.except() }} - select {{ compare_cols_csv }} - from b + select {{ compare_cols_csv }} + from a {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from b - ), + ), - b_minus_a as ( + b_minus_a as ( - select {{ compare_cols_csv }} - from b {{ dbt_utils.except() }} - select {{ compare_cols_csv }} - from a + select {{ compare_cols_csv }} + from b {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from a - ), + ), - unioned as ( + unioned as ( - select 'a_minus_b' as which_diff, a_minus_b.* - from a_minus_b - union all - select 'b_minus_a' as which_diff, b_minus_a.* - from b_minus_a + select 'a_minus_b' as which_diff, a_minus_b.* + from a_minus_b + union all + select 'b_minus_a' as which_diff, b_minus_a.* + from b_minus_a - ) + ) -select * -from unioned + select * + from unioned {% endmacro %} diff --git a/macros/generic_tests/expression_is_true.sql b/macros/generic_tests/expression_is_true.sql index fdf854d1..a67e818c 100644 --- a/macros/generic_tests/expression_is_true.sql +++ b/macros/generic_tests/expression_is_true.sql @@ -1,23 +1,23 @@ {% test expression_is_true(model, expression, column_name=None, condition="1=1") %} -{# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} -{# ref https://stackoverflow.com/a/7170753/3842610 #} -{{ - return( - adapter.dispatch("test_expression_is_true", "dbt_utils")( - model, expression, column_name, condition + {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} + {# ref https://stackoverflow.com/a/7170753/3842610 #} + {{ + return( + adapter.dispatch("test_expression_is_true", "dbt_utils")( + model, expression, column_name, condition + ) ) - ) -}} + }} {% endtest %} {% macro default__test_expression_is_true(model, expression, column_name, condition) %} -with meet_condition as (select * from {{ model }} where {{ condition }}) + with meet_condition as (select * from {{ model }} where {{ condition }}) -select * -from meet_condition -{% if column_name is none %} where not ({{ expression }}) -{%- else %} where not ({{ column_name }} {{ expression }}) -{%- endif %} + select * + from meet_condition + {% if column_name is none %} where not ({{ expression }}) + {%- else %} where not ({{ column_name }} {{ expression }}) + {%- endif %} {% endmacro %} diff --git a/macros/generic_tests/fewer_rows_than.sql b/macros/generic_tests/fewer_rows_than.sql index acd8dd23..76818214 100644 --- a/macros/generic_tests/fewer_rows_than.sql +++ b/macros/generic_tests/fewer_rows_than.sql @@ -1,39 +1,42 @@ {% test fewer_rows_than(model, compare_model) %} -{{ - return( - adapter.dispatch("test_fewer_rows_than", "dbt_utils")(model, compare_model) - ) -}} + {{ + return( + adapter.dispatch("test_fewer_rows_than", "dbt_utils")( + model, compare_model + ) + ) + }} {% endtest %} {% macro default__test_fewer_rows_than(model, compare_model) %} -{{ config(fail_calc="coalesce(row_count_delta, 0)") }} + {{ config(fail_calc="coalesce(row_count_delta, 0)") }} -with - a as (select count(*) as count_our_model from {{ model }}), - b as (select count(*) as count_comparison_model from {{ compare_model }}), - counts as (select count_our_model, count_comparison_model from a cross join b), - final as ( + with + a as (select count(*) as count_our_model from {{ model }}), + b as (select count(*) as count_comparison_model from {{ compare_model }}), + counts as (select count_our_model, count_comparison_model from a cross join b), + final as ( - select - *, - case - -- fail the test if we have more rows than the reference model and - -- return the row count delta - when count_our_model > count_comparison_model - then (count_our_model - count_comparison_model) - -- fail the test if they are the same number - when count_our_model = count_comparison_model - then 1 - -- pass the test if the delta is positive (i.e. return the number 0) - else 0 - end as row_count_delta - from counts + select + *, + case + -- fail the test if we have more rows than the reference model and + -- return the row count delta + when count_our_model > count_comparison_model + then (count_our_model - count_comparison_model) + -- fail the test if they are the same number + when count_our_model = count_comparison_model + then 1 + -- pass the test if the delta is positive (i.e. return the number + -- 0) + else 0 + end as row_count_delta + from counts - ) + ) -select * -from final + select * + from final {% endmacro %} diff --git a/macros/generic_tests/mutually_exclusive_ranges.sql b/macros/generic_tests/mutually_exclusive_ranges.sql index f62cefb6..7c6020c5 100644 --- a/macros/generic_tests/mutually_exclusive_ranges.sql +++ b/macros/generic_tests/mutually_exclusive_ranges.sql @@ -6,18 +6,18 @@ gaps="allowed", zero_length_range_allowed=False ) %} -{{ - return( - adapter.dispatch("test_mutually_exclusive_ranges", "dbt_utils")( - model, - lower_bound_column, - upper_bound_column, - partition_by, - gaps, - zero_length_range_allowed, + {{ + return( + adapter.dispatch("test_mutually_exclusive_ranges", "dbt_utils")( + model, + lower_bound_column, + upper_bound_column, + partition_by, + gaps, + zero_length_range_allowed, + ) ) - ) -}} + }} {% endtest %} {% macro default__test_mutually_exclusive_ranges( @@ -28,101 +28,106 @@ gaps="allowed", zero_length_range_allowed=False ) %} -{% if gaps == "not_allowed" %} -{% set allow_gaps_operator = "=" %} {% set allow_gaps_operator_in_words = "equal_to" %} -{% elif gaps == "allowed" %} -{% set allow_gaps_operator = "<=" %} -{% set allow_gaps_operator_in_words = "less_than_or_equal_to" %} -{% elif gaps == "required" %} -{% set allow_gaps_operator = "<" %} {% set allow_gaps_operator_in_words = "less_than" %} -{% else %} -{{ - exceptions.raise_compiler_error( - "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" - ~ gaps - ~ "'.'" - ) -}} -{% endif %} -{% if not zero_length_range_allowed %} -{% set allow_zero_length_operator = "<" %} -{% set allow_zero_length_operator_in_words = "less_than" %} -{% elif zero_length_range_allowed %} -{% set allow_zero_length_operator = "<=" %} -{% set allow_zero_length_operator_in_words = "less_than_or_equal_to" %} -{% else %} -{{ - exceptions.raise_compiler_error( - "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" - ~ zero_length_range_allowed - ~ "'.'" - ) -}} -{% endif %} - -{% set partition_clause = "partition by " ~ partition_by if partition_by else "" %} - -with - window_functions as ( - - select - {% if partition_by %} {{ partition_by }} as partition_by_col, {% endif %} - {{ lower_bound_column }} as lower_bound, - {{ upper_bound_column }} as upper_bound, - - lead({{ lower_bound_column }}) over ( - {{ partition_clause }} order by {{ lower_bound_column }} - ) as next_lower_bound, - - row_number() over ( - {{ partition_clause }} order by {{ lower_bound_column }} desc + {% if gaps == "not_allowed" %} + {% set allow_gaps_operator = "=" %} + {% set allow_gaps_operator_in_words = "equal_to" %} + {% elif gaps == "allowed" %} + {% set allow_gaps_operator = "<=" %} + {% set allow_gaps_operator_in_words = "less_than_or_equal_to" %} + {% elif gaps == "required" %} + {% set allow_gaps_operator = "<" %} + {% set allow_gaps_operator_in_words = "less_than" %} + {% else %} + {{ + exceptions.raise_compiler_error( + "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" + ~ gaps + ~ "'.'" ) - = 1 as is_last_record - - from {{ model }} - - ), - - calc as ( - -- We want to return records where one of our assumptions fails, so we'll use - -- the `not` function with `and` statements so we can write our assumptions - -- nore cleanly - select - *, - - -- For each record: lower_bound should be < upper_bound. - -- Coalesce it to return an error on the null case (implicit assumption - -- these columns are not_null) - coalesce( - lower_bound {{ allow_zero_length_operator }} upper_bound, false - ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, - - -- For each record: upper_bound {{ allow_gaps_operator }} the next - -- lower_bound. - -- Coalesce it to handle null cases for the last record. - coalesce( - upper_bound {{ allow_gaps_operator }} next_lower_bound, - is_last_record, - false - ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound - - from window_functions - - ), - - validation_errors as ( - - select * - from calc - - where - not ( - -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound - and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + }} + {% endif %} + {% if not zero_length_range_allowed %} + {% set allow_zero_length_operator = "<" %} + {% set allow_zero_length_operator_in_words = "less_than" %} + {% elif zero_length_range_allowed %} + {% set allow_zero_length_operator = "<=" %} + {% set allow_zero_length_operator_in_words = "less_than_or_equal_to" %} + {% else %} + {{ + exceptions.raise_compiler_error( + "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" + ~ zero_length_range_allowed + ~ "'.'" ) - ) + }} + {% endif %} + + {% set partition_clause = "partition by " ~ partition_by if partition_by else "" %} + + with + window_functions as ( + + select + {% if partition_by %} + {{ partition_by }} as partition_by_col, + {% endif %} + {{ lower_bound_column }} as lower_bound, + {{ upper_bound_column }} as upper_bound, + + lead({{ lower_bound_column }}) over ( + {{ partition_clause }} order by {{ lower_bound_column }} + ) as next_lower_bound, + + row_number() over ( + {{ partition_clause }} order by {{ lower_bound_column }} desc + ) + = 1 as is_last_record + + from {{ model }} + + ), + + calc as ( + -- We want to return records where one of our assumptions fails, so we'll + -- use + -- the `not` function with `and` statements so we can write our + -- assumptions nore cleanly + select + *, + + -- For each record: lower_bound should be < upper_bound. + -- Coalesce it to return an error on the null case (implicit assumption + -- these columns are not_null) + coalesce( + lower_bound {{ allow_zero_length_operator }} upper_bound, false + ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, + + -- For each record: upper_bound {{ allow_gaps_operator }} the next + -- lower_bound. + -- Coalesce it to handle null cases for the last record. + coalesce( + upper_bound {{ allow_gaps_operator }} next_lower_bound, + is_last_record, + false + ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + + from window_functions + + ), + + validation_errors as ( + + select * + from calc + + where + not ( + -- THE FOLLOWING SHOULD BE TRUE -- + lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound + and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + ) + ) -select * -from validation_errors + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/not_accepted_values.sql b/macros/generic_tests/not_accepted_values.sql index 47d49e97..688efb5c 100644 --- a/macros/generic_tests/not_accepted_values.sql +++ b/macros/generic_tests/not_accepted_values.sql @@ -1,35 +1,39 @@ {% test not_accepted_values(model, column_name, values, quote=True) %} -{{ - return( - adapter.dispatch("test_not_accepted_values", "dbt_utils")( - model, column_name, values, quote + {{ + return( + adapter.dispatch("test_not_accepted_values", "dbt_utils")( + model, column_name, values, quote + ) ) - ) -}} + }} {% endtest %} {% macro default__test_not_accepted_values(model, column_name, values, quote=True) %} -with - all_values as (select distinct {{ column_name }} as value_field from {{ model }}), - - validation_errors as ( - - select value_field - - from all_values - where - value_field in ( - {% for value in values -%} - {% if quote -%}'{{ value }}' - {%- else -%}{{ value }} - {%- endif -%} - {%- if not loop.last -%},{%- endif %} - {%- endfor %} - ) + with + all_values as ( + + select distinct {{ column_name }} as value_field from {{ model }} + + ), - ) + validation_errors as ( + + select value_field + + from all_values + where + value_field in ( + {% for value in values -%} + {% if quote -%}'{{ value }}' + {%- else -%}{{ value }} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {%- endfor %} + ) + + ) -select * -from validation_errors + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index df5efd2b..48397170 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -1,16 +1,20 @@ {% test not_constant(model, column_name) %} -{{ return(adapter.dispatch("test_not_constant", "dbt_utils")(model, column_name)) }} + {{ + return( + adapter.dispatch("test_not_constant", "dbt_utils")(model, column_name) + ) + }} {% endtest %} {% macro default__test_not_constant(model, column_name) %} -select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count(distinct {{ column_name }}) as filler_column + select + {# In TSQL, subquery aggregate columns need aliases #} + {# thus: a filler col name, 'filler_column' #} + count(distinct {{ column_name }}) as filler_column -from {{ model }} + from {{ model }} -having count(distinct {{ column_name }}) = 1 + having count(distinct {{ column_name }}) = 1 {% endmacro %} diff --git a/macros/generic_tests/not_null_proportion.sql b/macros/generic_tests/not_null_proportion.sql index a9369808..b9e6da97 100644 --- a/macros/generic_tests/not_null_proportion.sql +++ b/macros/generic_tests/not_null_proportion.sql @@ -1,31 +1,32 @@ {% macro test_not_null_proportion(model) %} -{{ - return( - adapter.dispatch("test_not_null_proportion", "dbt_utils")(model, **kwargs) - ) -}} + {{ + return( + adapter.dispatch("test_not_null_proportion", "dbt_utils")(model, **kwargs) + ) + }} {% endmacro %} {% macro default__test_not_null_proportion(model) %} -{% set column_name = kwargs.get("column_name", kwargs.get("arg")) %} -{% set at_least = kwargs.get("at_least", kwargs.get("arg")) %} -{% set at_most = kwargs.get("at_most", kwargs.get("arg", 1)) %} + {% set column_name = kwargs.get("column_name", kwargs.get("arg")) %} + {% set at_least = kwargs.get("at_least", kwargs.get("arg")) %} + {% set at_most = kwargs.get("at_most", kwargs.get("arg", 1)) %} -with - validation as ( - select - sum(case when {{ column_name }} is null then 0 else 1 end) - / cast(count(*) as numeric) as not_null_proportion - from {{ model }} - ), - validation_errors as ( - select not_null_proportion - from validation - where - not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} - ) -select * -from validation_errors + with + validation as ( + select + sum(case when {{ column_name }} is null then 0 else 1 end) + / cast(count(*) as numeric) as not_null_proportion + from {{ model }} + ), + validation_errors as ( + select not_null_proportion + from validation + where + not_null_proportion < {{ at_least }} + or not_null_proportion > {{ at_most }} + ) + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/recency.sql b/macros/generic_tests/recency.sql index ed02969c..666f8e7a 100644 --- a/macros/generic_tests/recency.sql +++ b/macros/generic_tests/recency.sql @@ -1,24 +1,24 @@ {% test recency(model, field, datepart, interval) %} -{{ - return( - adapter.dispatch("test_recency", "dbt_utils")( - model, field, datepart, interval + {{ + return( + adapter.dispatch("test_recency", "dbt_utils")( + model, field, datepart, interval + ) ) - ) -}} + }} {% endtest %} {% macro default__test_recency(model, field, datepart, interval) %} -{% set threshold = dbt_utils.dateadd( - datepart, interval * -1, dbt_utils.current_timestamp() -) %} + {% set threshold = dbt_utils.dateadd( + datepart, interval * -1, dbt_utils.current_timestamp() + ) %} -with recency as (select max({{ field }}) as most_recent from {{ model }}) + with recency as (select max({{ field }}) as most_recent from {{ model }}) -select most_recent, {{ threshold }} as threshold + select most_recent, {{ threshold }} as threshold -from recency -where most_recent < {{ threshold }} + from recency + where most_recent < {{ threshold }} {% endmacro %} diff --git a/macros/generic_tests/relationships_where.sql b/macros/generic_tests/relationships_where.sql index 16fbaf7c..927e6706 100644 --- a/macros/generic_tests/relationships_where.sql +++ b/macros/generic_tests/relationships_where.sql @@ -1,55 +1,55 @@ {% test relationships_where( model, column_name, to, field, from_condition="1=1", to_condition="1=1" ) %} -{{ - return( - adapter.dispatch("test_relationships_where", "dbt_utils")( - model, column_name, to, field, from_condition, to_condition + {{ + return( + adapter.dispatch("test_relationships_where", "dbt_utils")( + model, column_name, to, field, from_condition, to_condition + ) ) - ) -}} + }} {% endtest %} {% macro default__test_relationships_where( model, column_name, to, field, from_condition="1=1", to_condition="1=1" ) %} -{# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} -{# ref https://stackoverflow.com/a/7170753/3842610 #} -with - left_table as ( + {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} + {# ref https://stackoverflow.com/a/7170753/3842610 #} + with + left_table as ( - select {{ column_name }} as id + select {{ column_name }} as id - from {{ model }} + from {{ model }} - where {{ column_name }} is not null and {{ from_condition }} + where {{ column_name }} is not null and {{ from_condition }} - ), + ), - right_table as ( + right_table as ( - select {{ field }} as id + select {{ field }} as id - from {{ to }} + from {{ to }} - where {{ field }} is not null and {{ to_condition }} + where {{ field }} is not null and {{ to_condition }} - ), + ), - exceptions as ( + exceptions as ( - select left_table.id, right_table.id as right_id + select left_table.id, right_table.id as right_id - from left_table + from left_table - left join right_table on left_table.id = right_table.id + left join right_table on left_table.id = right_table.id - where right_table.id is null + where right_table.id is null - ) + ) -select * -from exceptions + select * + from exceptions {% endmacro %} diff --git a/macros/generic_tests/sequential_values.sql b/macros/generic_tests/sequential_values.sql index 4693373c..71662749 100644 --- a/macros/generic_tests/sequential_values.sql +++ b/macros/generic_tests/sequential_values.sql @@ -1,12 +1,12 @@ {% test sequential_values(model, column_name, interval=1, datepart=None) %} -{{ - return( - adapter.dispatch("test_sequential_values", "dbt_utils")( - model, column_name, interval, datepart + {{ + return( + adapter.dispatch("test_sequential_values", "dbt_utils")( + model, column_name, interval, datepart + ) ) - ) -}} + }} {% endtest %} @@ -14,36 +14,44 @@ model, column_name, interval=1, datepart=None ) %} -{% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} - -with - windowed as ( - - select - {{ column_name }}, - lag({{ column_name }}) over ( - order by {{ column_name }} - ) as {{ previous_column_name }} - from {{ model }} - ), - - validation_errors as ( - select * - from windowed - {% if datepart %} - where - not ( - cast({{ column_name }} as {{ dbt_utils.type_timestamp() }}) = cast( - {{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} - as {{ dbt_utils.type_timestamp() }} - ) - ) - {% else %} - where not ({{ column_name }} = {{ previous_column_name }} + {{ interval }}) - {% endif %} - ) + {% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} + + with + windowed as ( + + select + {{ column_name }}, + lag({{ column_name }}) over ( + order by {{ column_name }} + ) as {{ previous_column_name }} + from {{ model }} + ), + + validation_errors as ( + select * + from windowed + {% if datepart %} + where + not ( + cast( + {{ column_name }} as {{ dbt_utils.type_timestamp() }} + ) = cast( + {{ + dbt_utils.dateadd( + datepart, interval, previous_column_name + ) + }} as {{ dbt_utils.type_timestamp() }} + ) + ) + {% else %} + where + not ( + {{ column_name }} = {{ previous_column_name }} + {{ interval }} + ) + {% endif %} + ) -select * -from validation_errors + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/test_not_null_where.sql b/macros/generic_tests/test_not_null_where.sql index 9b602a78..39b68b19 100644 --- a/macros/generic_tests/test_not_null_where.sql +++ b/macros/generic_tests/test_not_null_where.sql @@ -1,9 +1,13 @@ {% test not_null_where(model, column_name) %} -{%- set deprecation_warning = " Warning: `dbt_utils.not_null_where` is no longer supported. Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. " -%} -{%- do exceptions.warn(deprecation_warning) -%} -{{ return(adapter.dispatch("test_not_null_where", "dbt_utils")(model, column_name)) }} + {%- set deprecation_warning = " Warning: `dbt_utils.not_null_where` is no longer supported. Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. " -%} + {%- do exceptions.warn(deprecation_warning) -%} + {{ + return( + adapter.dispatch("test_not_null_where", "dbt_utils")(model, column_name) + ) + }} {% endtest %} {% macro default__test_not_null_where(model, column_name) %} -{{ return(test_not_null(model, column_name)) }} + {{ return(test_not_null(model, column_name)) }} {% endmacro %} diff --git a/macros/generic_tests/test_unique_where.sql b/macros/generic_tests/test_unique_where.sql index 73442631..f163cf8e 100644 --- a/macros/generic_tests/test_unique_where.sql +++ b/macros/generic_tests/test_unique_where.sql @@ -1,9 +1,13 @@ {% test unique_where(model, column_name) %} -{%- set deprecation_warning = " Warning: `dbt_utils.unique_where` is no longer supported. Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. " -%} -{%- do exceptions.warn(deprecation_warning) -%} -{{ return(adapter.dispatch("test_unique_where", "dbt_utils")(model, column_name)) }} + {%- set deprecation_warning = " Warning: `dbt_utils.unique_where` is no longer supported. Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. " -%} + {%- do exceptions.warn(deprecation_warning) -%} + {{ + return( + adapter.dispatch("test_unique_where", "dbt_utils")(model, column_name) + ) + }} {% endtest %} {% macro default__test_unique_where(model, column_name) %} -{{ return(test_unique(model, column_name)) }} + {{ return(test_unique(model, column_name)) }} {% endmacro %} diff --git a/macros/generic_tests/unique_combination_of_columns.sql b/macros/generic_tests/unique_combination_of_columns.sql index a0ab359b..83c28f12 100644 --- a/macros/generic_tests/unique_combination_of_columns.sql +++ b/macros/generic_tests/unique_combination_of_columns.sql @@ -1,48 +1,48 @@ {% test unique_combination_of_columns( model, combination_of_columns, quote_columns=false ) %} -{{ - return( - adapter.dispatch("test_unique_combination_of_columns", "dbt_utils")( - model, combination_of_columns, quote_columns + {{ + return( + adapter.dispatch("test_unique_combination_of_columns", "dbt_utils")( + model, combination_of_columns, quote_columns + ) ) - ) -}} + }} {% endtest %} {% macro default__test_unique_combination_of_columns( model, combination_of_columns, quote_columns=false ) %} -{% if not quote_columns %} {%- set column_list = combination_of_columns %} -{% elif quote_columns %} -{%- set column_list = [] %} -{% for column in combination_of_columns -%} -{% set column_list = column_list.append(adapter.quote(column)) %} -{%- endfor %} -{% else %} -{{ - exceptions.raise_compiler_error( - "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" - ~ quote - ~ "'.'" - ) -}} -{% endif %} - -{%- set columns_csv = column_list | join(", ") %} - -with - validation_errors as ( - - select {{ columns_csv }} - from {{ model }} - group by {{ columns_csv }} - having count(*) > 1 - - ) - -select * -from validation_errors + {% if not quote_columns %} {%- set column_list = combination_of_columns %} + {% elif quote_columns %} + {%- set column_list = [] %} + {% for column in combination_of_columns -%} + {% set column_list = column_list.append(adapter.quote(column)) %} + {%- endfor %} + {% else %} + {{ + exceptions.raise_compiler_error( + "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" + ~ quote + ~ "'.'" + ) + }} + {% endif %} + + {%- set columns_csv = column_list | join(", ") %} + + with + validation_errors as ( + + select {{ columns_csv }} + from {{ model }} + group by {{ columns_csv }} + having count(*) > 1 + + ) + + select * + from validation_errors {% endmacro %} diff --git a/macros/jinja_helpers/log_info.sql b/macros/jinja_helpers/log_info.sql index 44fe5d1c..0c8c7675 100644 --- a/macros/jinja_helpers/log_info.sql +++ b/macros/jinja_helpers/log_info.sql @@ -1,7 +1,7 @@ {% macro log_info(message) %} -{{ return(adapter.dispatch("log_info", "dbt_utils")(message)) }} + {{ return(adapter.dispatch("log_info", "dbt_utils")(message)) }} {% endmacro %} {% macro default__log_info(message) %} -{{ log(dbt_utils.pretty_log_format(message), info=True) }} + {{ log(dbt_utils.pretty_log_format(message), info=True) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_log_format.sql b/macros/jinja_helpers/pretty_log_format.sql index 67096f48..115b38b2 100644 --- a/macros/jinja_helpers/pretty_log_format.sql +++ b/macros/jinja_helpers/pretty_log_format.sql @@ -1,7 +1,7 @@ {% macro pretty_log_format(message) %} -{{ return(adapter.dispatch("pretty_log_format", "dbt_utils")(message)) }} + {{ return(adapter.dispatch("pretty_log_format", "dbt_utils")(message)) }} {% endmacro %} {% macro default__pretty_log_format(message) %} -{{ return(dbt_utils.pretty_time() ~ " + " ~ message) }} + {{ return(dbt_utils.pretty_time() ~ " + " ~ message) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_time.sql b/macros/jinja_helpers/pretty_time.sql index a342b9ce..285eda67 100644 --- a/macros/jinja_helpers/pretty_time.sql +++ b/macros/jinja_helpers/pretty_time.sql @@ -1,7 +1,7 @@ {% macro pretty_time(format="%H:%M:%S") %} -{{ return(adapter.dispatch("pretty_time", "dbt_utils")(format)) }} + {{ return(adapter.dispatch("pretty_time", "dbt_utils")(format)) }} {% endmacro %} {% macro default__pretty_time(format="%H:%M:%S") %} -{{ return(modules.datetime.datetime.now().strftime(format)) }} + {{ return(modules.datetime.datetime.now().strftime(format)) }} {% endmacro %} diff --git a/macros/jinja_helpers/slugify.sql b/macros/jinja_helpers/slugify.sql index 01cce92d..b5888adc 100644 --- a/macros/jinja_helpers/slugify.sql +++ b/macros/jinja_helpers/slugify.sql @@ -1,12 +1,12 @@ {% macro slugify(string) %} -{#- Lower case the string -#} -{% set string = string | lower %} -{#- Replace spaces and dashes with underscores -#} -{% set string = modules.re.sub("[ -]+", "_", string) %} -{#- Only take letters, numbers, and underscores -#} -{% set string = modules.re.sub("[^a-z0-9_]+", "", string) %} + {#- Lower case the string -#} + {% set string = string | lower %} + {#- Replace spaces and dashes with underscores -#} + {% set string = modules.re.sub("[ -]+", "_", string) %} + {#- Only take letters, numbers, and underscores -#} + {% set string = modules.re.sub("[^a-z0-9_]+", "", string) %} -{{ return(string) }} + {{ return(string) }} {% endmacro %} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index c7dcecdd..92942341 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -6,18 +6,18 @@ stop_date, period ) -%} -{{ - return( - adapter.dispatch("get_period_boundaries", "dbt_utils")( - target_schema, - target_table, - timestamp_field, - start_date, - stop_date, - period, + {{ + return( + adapter.dispatch("get_period_boundaries", "dbt_utils")( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period, + ) ) - ) -}} + }} {% endmacro %} {% macro default__get_period_boundaries( @@ -29,32 +29,32 @@ period ) -%} -{% call statement("period_boundaries", fetch_result=True) -%} -with - data as ( + {% call statement("period_boundaries", fetch_result=True) -%} + with + data as ( + select + coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp + as start_timestamp, + coalesce( + {{ + dbt_utils.dateadd( + "millisecond", + -1, + "nullif('" ~ stop_date ~ "','')::timestamp", + ) + }}, + {{ dbt_utils.current_timestamp() }} + ) as stop_timestamp + from "{{target_schema}}"."{{target_table}}" + ) + select - coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp - as start_timestamp, - coalesce( - {{ - dbt_utils.dateadd( - "millisecond", - -1, - "nullif('" ~ stop_date ~ "','')::timestamp", - ) - }}, - {{ dbt_utils.current_timestamp() }} - ) as stop_timestamp - from "{{target_schema}}"."{{target_table}}" - ) - -select - start_timestamp, - stop_timestamp, - {{ dbt_utils.datediff("start_timestamp", "stop_timestamp", period) }} - + 1 as num_periods -from data -{%- endcall %} + start_timestamp, + stop_timestamp, + {{ dbt_utils.datediff("start_timestamp", "stop_timestamp", period) }} + + 1 as num_periods + from data + {%- endcall %} {%- endmacro %} @@ -67,19 +67,19 @@ from data stop_timestamp, offset ) -%} -{{ - return( - adapter.dispatch("get_period_sql", "dbt_utils")( - target_cols_csv, - sql, - timestamp_field, - period, - start_timestamp, - stop_timestamp, - offset, + {{ + return( + adapter.dispatch("get_period_sql", "dbt_utils")( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset, + ) ) - ) -}} + }} {% endmacro %} {% macro default__get_period_sql( @@ -92,176 +92,185 @@ from data offset ) -%} -{%- set period_filter -%} + {%- set period_filter -%} ("{{timestamp_field}}" > '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' and "{{timestamp_field}}" <= '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' + interval '1 {{period}}' and "{{timestamp_field}}" < '{{stop_timestamp}}'::timestamp) -{%- endset -%} + {%- endset -%} -{%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} + {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} -select {{ target_cols_csv }} -from ({{ filtered_sql }}) + select {{ target_cols_csv }} + from ({{ filtered_sql }}) {%- endmacro %} {% materialization insert_by_period, default -%} -{%- set timestamp_field = config.require("timestamp_field") -%} -{%- set start_date = config.require("start_date") -%} -{%- set stop_date = config.get("stop_date") or "" -%} -{%- set period = config.get("period") or "week" -%} + {%- set timestamp_field = config.require("timestamp_field") -%} + {%- set start_date = config.require("start_date") -%} + {%- set stop_date = config.get("stop_date") or "" -%} + {%- set period = config.get("period") or "week" -%} -{%- if sql.find("__PERIOD_FILTER__") == -1 -%} -{%- set error_message -%} + {%- if sql.find("__PERIOD_FILTER__") == -1 -%} + {%- set error_message -%} Model '{{ model.unique_id }}' does not include the required string '__PERIOD_FILTER__' in its sql -{%- endset -%} -{{ exceptions.raise_compiler_error(error_message) }} -{%- endif -%} - -{%- set identifier = model["name"] -%} - -{%- set old_relation = adapter.get_relation( - database=database, schema=schema, identifier=identifier -) -%} -{%- set target_relation = api.Relation.create( - identifier=identifier, schema=schema, type="table" -) -%} - -{%- set non_destructive_mode = flags.NON_DESTRUCTIVE == True -%} -{%- set full_refresh_mode = flags.FULL_REFRESH == True -%} - -{%- set exists_as_table = old_relation is not none and old_relation.is_table -%} -{%- set exists_not_as_table = old_relation is not none and not old_relation.is_table -%} - -{%- set should_truncate = ( - non_destructive_mode and full_refresh_mode and exists_as_table -) -%} -{%- set should_drop = not should_truncate and ( - full_refresh_mode or exists_not_as_table -) -%} -{%- set force_create = flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE -%} - --- setup -{% if old_relation is none -%} --- noop -{%- elif should_truncate -%} {{ adapter.truncate_relation(old_relation) }} -{%- elif should_drop -%} -{{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} -{%- endif %} - -{{ run_hooks(pre_hooks, inside_transaction=False) }} - --- `begin` happens here, so `commit` after it to finish the transaction -{{ run_hooks(pre_hooks, inside_transaction=True) }} -{% call statement() -%} -begin -; -- make extra sure we've closed out the transaction -commit -; -{%- endcall %} - --- build model -{% if force_create or old_relation is none -%} -{# Create an empty target table -#} -{% call statement("main") -%} -{%- set empty_sql = sql | replace("__PERIOD_FILTER__", "false") -%} -{{ create_table_as(False, target_relation, empty_sql) }} -{%- endcall %} -{%- endif %} - -{% set _ = dbt_utils.get_period_boundaries( - schema, identifier, timestamp_field, start_date, stop_date, period -) %} -{%- set start_timestamp = load_result("period_boundaries")["data"][0][0] | string -%} -{%- set stop_timestamp = load_result("period_boundaries")["data"][0][1] | string -%} -{%- set num_periods = load_result("period_boundaries")["data"][0][2] | int -%} - -{% set target_columns = adapter.get_columns_in_relation(target_relation) %} -{%- set target_cols_csv = target_columns | map(attribute="quoted") | join(", ") -%} -{%- set loop_vars = {"sum_rows_inserted": 0} -%} - --- commit each period as a separate transaction -{% for i in range(num_periods) -%} -{%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} -{{ dbt_utils.log_info(msg) }} - -{%- set tmp_identifier = model["name"] ~ "__dbt_incremental_period" ~ i ~ "_tmp" -%} -{%- set tmp_relation = api.Relation.create( - identifier=tmp_identifier, schema=schema, type="table" -) -%} -{% call statement() -%} -{% set tmp_table_sql = dbt_utils.get_period_sql( - target_cols_csv, - sql, - timestamp_field, - period, - start_timestamp, - stop_timestamp, - i, -) %} -{{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} -{%- endcall %} - -{{ - adapter.expand_target_column_types( - from_relation=tmp_relation, to_relation=target_relation - ) -}} -{%- set name = "main-" ~ i -%} -{% call statement(name, fetch_result=True) -%} + {%- endset -%} + {{ exceptions.raise_compiler_error(error_message) }} + {%- endif -%} + + {%- set identifier = model["name"] -%} + + {%- set old_relation = adapter.get_relation( + database=database, schema=schema, identifier=identifier + ) -%} + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, type="table" + ) -%} + + {%- set non_destructive_mode = flags.NON_DESTRUCTIVE == True -%} + {%- set full_refresh_mode = flags.FULL_REFRESH == True -%} + + {%- set exists_as_table = old_relation is not none and old_relation.is_table -%} + {%- set exists_not_as_table = ( + old_relation is not none and not old_relation.is_table + ) -%} + + {%- set should_truncate = ( + non_destructive_mode and full_refresh_mode and exists_as_table + ) -%} + {%- set should_drop = not should_truncate and ( + full_refresh_mode or exists_not_as_table + ) -%} + {%- set force_create = flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE -%} + + -- setup + {% if old_relation is none -%} + -- noop + {%- elif should_truncate -%} {{ adapter.truncate_relation(old_relation) }} + {%- elif should_drop -%} + {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} + {%- endif %} + + {{ run_hooks(pre_hooks, inside_transaction=False) }} + + -- `begin` happens here, so `commit` after it to finish the transaction + {{ run_hooks(pre_hooks, inside_transaction=True) }} + {% call statement() -%} + begin -- make extra sure we've closed out the transaction + ; + commit + ; + {%- endcall %} + + -- build model + {% if force_create or old_relation is none -%} + {# Create an empty target table -#} + {% call statement("main") -%} + {%- set empty_sql = sql | replace("__PERIOD_FILTER__", "false") -%} + {{ create_table_as(False, target_relation, empty_sql) }} + {%- endcall %} + {%- endif %} + + {% set _ = dbt_utils.get_period_boundaries( + schema, identifier, timestamp_field, start_date, stop_date, period + ) %} + {%- set start_timestamp = ( + load_result("period_boundaries")["data"][0][0] | string + ) -%} + {%- set stop_timestamp = load_result("period_boundaries")["data"][0][1] | string -%} + {%- set num_periods = load_result("period_boundaries")["data"][0][2] | int -%} + + {% set target_columns = adapter.get_columns_in_relation(target_relation) %} + {%- set target_cols_csv = target_columns | map(attribute="quoted") | join(", ") -%} + {%- set loop_vars = {"sum_rows_inserted": 0} -%} + + -- commit each period as a separate transaction + {% for i in range(num_periods) -%} + {%- set msg = ( + "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) + ) -%} + {{ dbt_utils.log_info(msg) }} + + {%- set tmp_identifier = ( + model["name"] ~ "__dbt_incremental_period" ~ i ~ "_tmp" + ) -%} + {%- set tmp_relation = api.Relation.create( + identifier=tmp_identifier, schema=schema, type="table" + ) -%} + {% call statement() -%} + {% set tmp_table_sql = dbt_utils.get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + i, + ) %} + {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} + {%- endcall %} + + {{ + adapter.expand_target_column_types( + from_relation=tmp_relation, to_relation=target_relation + ) + }} + {%- set name = "main-" ~ i -%} + {% call statement(name, fetch_result=True) -%} insert into {{target_relation}} ({{target_cols_csv}}) ( select {{target_cols_csv}} from {{tmp_relation.include(schema=False)}} ); -{%- endcall %} -{% set result = load_result("main-" ~ i) %} -{% if "response" in result.keys() %} {# added in v0.19.0 #} -{% set rows_inserted = result["response"]["rows_affected"] %} -{% else %} {# older versions #} -{% set rows_inserted = result["status"].split(" ")[2] | int %} -{% endif %} - -{%- set sum_rows_inserted = loop_vars["sum_rows_inserted"] + rows_inserted -%} -{%- if loop_vars.update({"sum_rows_inserted": sum_rows_inserted}) %} {% endif -%} - -{%- set msg = ( - "Ran for " - ~ period - ~ " " - ~ (i + 1) - ~ " of " - ~ (num_periods) - ~ "; " - ~ rows_inserted - ~ " records inserted" -) -%} -{{ dbt_utils.log_info(msg) }} - -{%- endfor %} - -{% call statement() -%} -begin -; -{%- endcall %} - -{{ run_hooks(post_hooks, inside_transaction=True) }} - -{% call statement() -%} -commit -; -{%- endcall %} - -{{ run_hooks(post_hooks, inside_transaction=False) }} - -{%- set status_string = "INSERT " ~ loop_vars["sum_rows_inserted"] -%} - -{% call noop_statement("main", status_string) -%} --- no-op -{%- endcall %} - --- Return the relations created in this materialization -{{ return({"relations": [target_relation]}) }} + {%- endcall %} + {% set result = load_result("main-" ~ i) %} + {% if "response" in result.keys() %} {# added in v0.19.0 #} + {% set rows_inserted = result["response"]["rows_affected"] %} + {% else %} {# older versions #} + {% set rows_inserted = result["status"].split(" ")[2] | int %} + {% endif %} + + {%- set sum_rows_inserted = loop_vars["sum_rows_inserted"] + rows_inserted -%} + {%- if loop_vars.update({"sum_rows_inserted": sum_rows_inserted}) %} + {% endif -%} + + {%- set msg = ( + "Ran for " + ~ period + ~ " " + ~ (i + 1) + ~ " of " + ~ (num_periods) + ~ "; " + ~ rows_inserted + ~ " records inserted" + ) -%} + {{ dbt_utils.log_info(msg) }} + + {%- endfor %} + + {% call statement() -%} + begin + ; + {%- endcall %} + + {{ run_hooks(post_hooks, inside_transaction=True) }} + + {% call statement() -%} + commit + ; + {%- endcall %} + + {{ run_hooks(post_hooks, inside_transaction=False) }} + + {%- set status_string = "INSERT " ~ loop_vars["sum_rows_inserted"] -%} + + {% call noop_statement("main", status_string) -%} + -- no-op + {%- endcall %} + + -- Return the relations created in this materialization + {{ return({"relations": [target_relation]}) }} {%- endmaterialization %} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index 30064b79..3fc4e1c8 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -1,75 +1,82 @@ {% macro get_intervals_between(start_date, end_date, datepart) -%} -{{ - return( - adapter.dispatch("get_intervals_between", "dbt_utils")( - start_date, end_date, datepart + {{ + return( + adapter.dispatch("get_intervals_between", "dbt_utils")( + start_date, end_date, datepart + ) ) - ) -}} + }} {%- endmacro %} {% macro default__get_intervals_between(start_date, end_date, datepart) -%} -{%- call statement("get_intervals_between", fetch_result=True) %} + {%- call statement("get_intervals_between", fetch_result=True) %} -select {{ dbt_utils.datediff(start_date, end_date, datepart) }} + select {{ dbt_utils.datediff(start_date, end_date, datepart) }} -{%- endcall -%} + {%- endcall -%} -{%- set value_list = load_result("get_intervals_between") -%} + {%- set value_list = load_result("get_intervals_between") -%} -{%- if value_list and value_list["data"] -%} -{%- set values = value_list["data"] | map(attribute=0) | list %} {{ return(values[0]) }} -{%- else -%} {{ return(1) }} -{%- endif -%} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} + {{ return(values[0]) }} + {%- else -%} {{ return(1) }} + {%- endif -%} {%- endmacro %} {% macro date_spine(datepart, start_date, end_date) %} -{{ - return( - adapter.dispatch("date_spine", "dbt_utils")(datepart, start_date, end_date) - ) -}} + {{ + return( + adapter.dispatch("date_spine", "dbt_utils")( + datepart, start_date, end_date + ) + ) + }} {%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} -{# call as follows: + {# call as follows: date_spine( "day", "to_date('01/01/2016', 'mm/dd/yyyy')", "dateadd(week, 1, current_date)" ) #} -with - rawdata as ( + with + rawdata as ( - {{ - dbt_utils.generate_series( - dbt_utils.get_intervals_between(start_date, end_date, datepart) - ) - }} + {{ + dbt_utils.generate_series( + dbt_utils.get_intervals_between(start_date, end_date, datepart) + ) + }} - ), + ), - all_periods as ( + all_periods as ( - select - ( - {{ - dbt_utils.dateadd( - datepart, "row_number() over (order by 1) - 1", start_date - ) - }} - ) as date_{{ datepart }} - from rawdata + select + ( + {{ + dbt_utils.dateadd( + datepart, "row_number() over (order by 1) - 1", start_date + ) + }} + ) as date_{{ datepart }} + from rawdata - ), + ), - filtered as (select * from all_periods where date_{{ datepart }} <= {{ end_date }}) + filtered as ( + + select * from all_periods where date_{{ datepart }} <= {{ end_date }} + + ) -select * -from filtered + select * + from filtered {% endmacro %} diff --git a/macros/sql/deduplicate.sql b/macros/sql/deduplicate.sql index 51ebce05..bc48255f 100644 --- a/macros/sql/deduplicate.sql +++ b/macros/sql/deduplicate.sql @@ -1,29 +1,29 @@ {%- macro deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} -{{ - return( - adapter.dispatch("deduplicate", "dbt_utils")( - relation, group_by, order_by=order_by, relation_alias=relation_alias + {{ + return( + adapter.dispatch("deduplicate", "dbt_utils")( + relation, group_by, order_by=order_by, relation_alias=relation_alias + ) ) - ) -}} + }} {% endmacro %} {%- macro default__deduplicate( relation, group_by, order_by=none, relation_alias=none ) -%} -select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} -from - ( - select - _inner.*, - row_number() over ( - partition by {{ group_by }} - {% if order_by is not none -%} order by {{ order_by }} {%- endif %} - ) as rn - from {{ relation if relation_alias is none else relation_alias }} as _inner - ) as deduped -where deduped.rn = 1 + select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} + from + ( + select + _inner.*, + row_number() over ( + partition by {{ group_by }} + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} + ) as rn + from {{ relation if relation_alias is none else relation_alias }} as _inner + ) as deduped + where deduped.rn = 1 {%- endmacro -%} @@ -36,17 +36,18 @@ where deduped.rn = 1 relation, group_by, order_by=none, relation_alias=none ) -%} -select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} -from - ( - select - array_agg( - original - {% if order_by is not none -%} order by {{ order_by }} {%- endif %} - limit 1 - )[offset(0)] as deduped - from {{ relation if relation_alias is none else relation_alias }} as original - group by {{ group_by }} - ) + select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} + from + ( + select + array_agg( + original + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} + limit 1 + )[offset(0)] as deduped + from + {{ relation if relation_alias is none else relation_alias }} as original + group by {{ group_by }} + ) {%- endmacro -%} diff --git a/macros/sql/generate_series.sql b/macros/sql/generate_series.sql index f8ff6590..4aac864b 100644 --- a/macros/sql/generate_series.sql +++ b/macros/sql/generate_series.sql @@ -1,55 +1,55 @@ {% macro get_powers_of_two(upper_bound) %} -{{ return(adapter.dispatch("get_powers_of_two", "dbt_utils")(upper_bound)) }} + {{ return(adapter.dispatch("get_powers_of_two", "dbt_utils")(upper_bound)) }} {% endmacro %} {% macro default__get_powers_of_two(upper_bound) %} -{% if upper_bound <= 0 %} -{{ exceptions.raise_compiler_error("upper bound must be positive") }} -{% endif %} + {% if upper_bound <= 0 %} + {{ exceptions.raise_compiler_error("upper bound must be positive") }} + {% endif %} -{% for _ in range(1, 100) %} -{% if upper_bound <= 2**loop.index %} {{ return(loop.index) }}{% endif %} -{% endfor %} + {% for _ in range(1, 100) %} + {% if upper_bound <= 2**loop.index %} {{ return(loop.index) }}{% endif %} + {% endfor %} {% endmacro %} {% macro generate_series(upper_bound) %} -{{ return(adapter.dispatch("generate_series", "dbt_utils")(upper_bound)) }} + {{ return(adapter.dispatch("generate_series", "dbt_utils")(upper_bound)) }} {% endmacro %} {% macro default__generate_series(upper_bound) %} -{% set n = dbt_utils.get_powers_of_two(upper_bound) %} + {% set n = dbt_utils.get_powers_of_two(upper_bound) %} -with - p as ( - select 0 as generated_number - union all - select 1 - ), - unioned as ( + with + p as ( + select 0 as generated_number + union all + select 1 + ), + unioned as ( - select + select - {% for i in range(n) %} - p{{ i }}.generated_number * power(2, {{ i }}) - {% if not loop.last %} + {% endif %} - {% endfor %} - + 1 as generated_number + {% for i in range(n) %} + p{{ i }}.generated_number * power(2, {{ i }}) + {% if not loop.last %} + {% endif %} + {% endfor %} + + 1 as generated_number - from + from - {% for i in range(n) %} - p as p{{ i }} {% if not loop.last %} cross join {% endif %} - {% endfor %} + {% for i in range(n) %} + p as p{{ i }} {% if not loop.last %} cross join {% endif %} + {% endfor %} - ) + ) -select * -from unioned -where generated_number <= {{ upper_bound }} -order by generated_number + select * + from unioned + where generated_number <= {{ upper_bound }} + order by generated_number {% endmacro %} diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index 239478ed..a79c469f 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -1,77 +1,78 @@ {% macro get_column_values( table, column, order_by="count(*) desc", max_records=none, default=none ) -%} -{{ - return( - adapter.dispatch("get_column_values", "dbt_utils")( - table, column, order_by, max_records, default + {{ + return( + adapter.dispatch("get_column_values", "dbt_utils")( + table, column, order_by, max_records, default + ) ) - ) -}} + }} {% endmacro %} {% macro default__get_column_values( table, column, order_by="count(*) desc", max_records=none, default=none ) -%} -{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} -{% set default = [] if not default %} {{ return(default) }} -{% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} + {% set default = [] if not default %} {{ return(default) }} + {% endif %} -{%- do dbt_utils._is_ephemeral(table, "get_column_values") -%} + {%- do dbt_utils._is_ephemeral(table, "get_column_values") -%} -{# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} -{# TODO: Change the method signature in a future 0.x.0 release #} -{%- set target_relation = table -%} + {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} + {# TODO: Change the method signature in a future 0.x.0 release #} + {%- set target_relation = table -%} -{# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} -{% set relation_exists = (load_relation(target_relation)) is not none %} + {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} + {% set relation_exists = (load_relation(target_relation)) is not none %} -{%- call statement("get_column_values", fetch_result=true) %} + {%- call statement("get_column_values", fetch_result=true) %} -{%- if not relation_exists and default is none -%} + {%- if not relation_exists and default is none -%} -{{ - exceptions.raise_compiler_error( - "In get_column_values(): relation " - ~ target_relation - ~ " does not exist and no default value was provided." - ) -}} + {{ + exceptions.raise_compiler_error( + "In get_column_values(): relation " + ~ target_relation + ~ " does not exist and no default value was provided." + ) + }} -{%- elif not relation_exists and default is not none -%} + {%- elif not relation_exists and default is not none -%} -{{ - log( - "Relation " - ~ target_relation - ~ " does not exist. Returning the default value: " - ~ default - ) -}} + {{ + log( + "Relation " + ~ target_relation + ~ " does not exist. Returning the default value: " + ~ default + ) + }} -{{ return(default) }} + {{ return(default) }} -{%- else -%} + {%- else -%} -select {{ column }} as value + select {{ column }} as value -from {{ target_relation }} -group by {{ column }} -order by {{ order_by }} + from {{ target_relation }} + group by {{ column }} + order by {{ order_by }} -{% if max_records is not none %} limit {{ max_records }} -{% endif %} + {% if max_records is not none %} limit {{ max_records }} + {% endif %} -{% endif %} + {% endif %} -{%- endcall -%} + {%- endcall -%} -{%- set value_list = load_result("get_column_values") -%} + {%- set value_list = load_result("get_column_values") -%} -{%- if value_list and value_list["data"] -%} -{%- set values = value_list["data"] | map(attribute=0) | list %} {{ return(values) }} -{%- else -%} {{ return(default) }} -{%- endif -%} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} + {{ return(values) }} + {%- else -%} {{ return(default) }} + {%- endif -%} {%- endmacro %} diff --git a/macros/sql/get_filtered_columns_in_relation.sql b/macros/sql/get_filtered_columns_in_relation.sql index 454fd18c..8a14f6eb 100644 --- a/macros/sql/get_filtered_columns_in_relation.sql +++ b/macros/sql/get_filtered_columns_in_relation.sql @@ -1,23 +1,23 @@ {% macro get_filtered_columns_in_relation(from, except=[]) -%} -{{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} + {{ return(adapter.dispatch('get_filtered_columns_in_relation', 'dbt_utils')(from, except)) }} {% endmacro %} {% macro default__get_filtered_columns_in_relation(from, except=[]) -%} -{%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} -{%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} + {%- do dbt_utils._is_relation(from, 'get_filtered_columns_in_relation') -%} + {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} -{# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} {{ return("") }} {% endif %} + {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("") }} {% endif %} -{%- set include_cols = [] %} -{%- set cols = adapter.get_columns_in_relation(from) -%} -{%- set except = except | map("lower") | list %} -{%- for col in cols -%} -{%- if col.column|lower not in except -%} -{% do include_cols.append(col.column) %} -{%- endif %} -{%- endfor %} + {%- set include_cols = [] %} + {%- set cols = adapter.get_columns_in_relation(from) -%} + {%- set except = except | map("lower") | list %} + {%- for col in cols -%} + {%- if col.column|lower not in except -%} + {% do include_cols.append(col.column) %} + {%- endif %} + {%- endfor %} -{{ return(include_cols) }} + {{ return(include_cols) }} {%- endmacro %} diff --git a/macros/sql/get_query_results_as_dict.sql b/macros/sql/get_query_results_as_dict.sql index f2199767..6c68991f 100644 --- a/macros/sql/get_query_results_as_dict.sql +++ b/macros/sql/get_query_results_as_dict.sql @@ -1,25 +1,25 @@ {% macro get_query_results_as_dict(query) %} -{{ return(adapter.dispatch("get_query_results_as_dict", "dbt_utils")(query)) }} + {{ return(adapter.dispatch("get_query_results_as_dict", "dbt_utils")(query)) }} {% endmacro %} {% macro default__get_query_results_as_dict(query) %} -{# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} -{%- call statement("get_query_results", fetch_result=True, auto_begin=false) -%} + {# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} + {%- call statement("get_query_results", fetch_result=True, auto_begin=false) -%} -{{ query }} + {{ query }} -{%- endcall -%} + {%- endcall -%} -{% set sql_results = {} %} + {% set sql_results = {} %} -{%- if execute -%} -{% set sql_results_table = load_result("get_query_results").table.columns %} -{% for column_name, column in sql_results_table.items() %} -{% do sql_results.update({column_name: column.values()}) %} -{% endfor %} -{%- endif -%} + {%- if execute -%} + {% set sql_results_table = load_result("get_query_results").table.columns %} + {% for column_name, column in sql_results_table.items() %} + {% do sql_results.update({column_name: column.values()}) %} + {% endfor %} + {%- endif -%} -{{ return(sql_results) }} + {{ return(sql_results) }} {% endmacro %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 457a2fe4..b806bcc7 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -1,45 +1,45 @@ {% macro get_relations_by_pattern( schema_pattern, table_pattern, exclude="", database=target.database ) %} -{{ - return( - adapter.dispatch("get_relations_by_pattern", "dbt_utils")( - schema_pattern, table_pattern, exclude, database + {{ + return( + adapter.dispatch("get_relations_by_pattern", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) ) - ) -}} + }} {% endmacro %} {% macro default__get_relations_by_pattern( schema_pattern, table_pattern, exclude="", database=target.database ) %} -{%- call statement("get_tables", fetch_result=True) %} - -{{ - dbt_utils.get_tables_by_pattern_sql( - schema_pattern, table_pattern, exclude, database - ) -}} - -{%- endcall -%} - -{%- set table_list = load_result("get_tables") -%} - -{%- if table_list and table_list["table"] -%} -{%- set tbl_relations = [] -%} -{%- for row in table_list["table"] -%} -{%- set tbl_relation = api.Relation.create( - database=database, - schema=row.table_schema, - identifier=row.table_name, - type=row.table_type, -) -%} -{%- do tbl_relations.append(tbl_relation) -%} -{%- endfor -%} - -{{ return(tbl_relations) }} -{%- else -%} {{ return([]) }} -{%- endif -%} + {%- call statement("get_tables", fetch_result=True) %} + + {{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude, database + ) + }} + + {%- endcall -%} + + {%- set table_list = load_result("get_tables") -%} + + {%- if table_list and table_list["table"] -%} + {%- set tbl_relations = [] -%} + {%- for row in table_list["table"] -%} + {%- set tbl_relation = api.Relation.create( + database=database, + schema=row.table_schema, + identifier=row.table_name, + type=row.table_type, + ) -%} + {%- do tbl_relations.append(tbl_relation) -%} + {%- endfor -%} + + {{ return(tbl_relations) }} + {%- else -%} {{ return([]) }} + {%- endif -%} {% endmacro %} diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index 0f5aee98..59f1b90d 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -1,41 +1,41 @@ {% macro get_relations_by_prefix( schema, prefix, exclude="", database=target.database ) %} -{{ - return( - adapter.dispatch("get_relations_by_prefix", "dbt_utils")( - schema, prefix, exclude, database + {{ + return( + adapter.dispatch("get_relations_by_prefix", "dbt_utils")( + schema, prefix, exclude, database + ) ) - ) -}} + }} {% endmacro %} {% macro default__get_relations_by_prefix( schema, prefix, exclude="", database=target.database ) %} -{%- call statement("get_tables", fetch_result=True) %} + {%- call statement("get_tables", fetch_result=True) %} -{{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} + {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} -{%- endcall -%} + {%- endcall -%} -{%- set table_list = load_result("get_tables") -%} + {%- set table_list = load_result("get_tables") -%} -{%- if table_list and table_list["table"] -%} -{%- set tbl_relations = [] -%} -{%- for row in table_list["table"] -%} -{%- set tbl_relation = api.Relation.create( - database=database, - schema=row.table_schema, - identifier=row.table_name, - type=row.table_type, -) -%} -{%- do tbl_relations.append(tbl_relation) -%} -{%- endfor -%} + {%- if table_list and table_list["table"] -%} + {%- set tbl_relations = [] -%} + {%- for row in table_list["table"] -%} + {%- set tbl_relation = api.Relation.create( + database=database, + schema=row.table_schema, + identifier=row.table_name, + type=row.table_type, + ) -%} + {%- do tbl_relations.append(tbl_relation) -%} + {%- endfor -%} -{{ return(tbl_relations) }} -{%- else -%} {{ return([]) }} -{%- endif -%} + {{ return(tbl_relations) }} + {%- else -%} {{ return([]) }} + {%- endif -%} {% endmacro %} diff --git a/macros/sql/get_table_types_sql.sql b/macros/sql/get_table_types_sql.sql index 72c9a535..fe3eb4c1 100644 --- a/macros/sql/get_table_types_sql.sql +++ b/macros/sql/get_table_types_sql.sql @@ -1,43 +1,43 @@ {%- macro get_table_types_sql() -%} -{{ return(adapter.dispatch("get_table_types_sql", "dbt_utils")()) }} + {{ return(adapter.dispatch("get_table_types_sql", "dbt_utils")()) }} {%- endmacro -%} {% macro default__get_table_types_sql() %} -case - table_type - when 'BASE TABLE' - then 'table' - when 'EXTERNAL TABLE' - then 'external' - when 'MATERIALIZED VIEW' - then 'materializedview' - else lower(table_type) -end as "table_type" + case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) + end as "table_type" {% endmacro %} {% macro postgres__get_table_types_sql() %} -case - table_type - when 'BASE TABLE' - then 'table' - when 'FOREIGN' - then 'external' - when 'MATERIALIZED VIEW' - then 'materializedview' - else lower(table_type) -end as "table_type" + case + table_type + when 'BASE TABLE' + then 'table' + when 'FOREIGN' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) + end as "table_type" {% endmacro %} {% macro bigquery__get_table_types_sql() %} -case - table_type - when 'BASE TABLE' - then 'table' - when 'EXTERNAL TABLE' - then 'external' - when 'MATERIALIZED VIEW' - then 'materializedview' - else lower(table_type) -end as `table_type` + case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) + end as `table_type` {% endmacro %} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index c7f85513..40603f1d 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -1,28 +1,28 @@ {% macro get_tables_by_pattern_sql( schema_pattern, table_pattern, exclude="", database=target.database ) %} -{{ - return( - adapter.dispatch("get_tables_by_pattern_sql", "dbt_utils")( - schema_pattern, table_pattern, exclude, database + {{ + return( + adapter.dispatch("get_tables_by_pattern_sql", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) ) - ) -}} + }} {% endmacro %} {% macro default__get_tables_by_pattern_sql( schema_pattern, table_pattern, exclude="", database=target.database ) %} -select distinct - table_schema as "table_schema", - table_name as "table_name", - {{ dbt_utils.get_table_types_sql() }} -from {{ database }}.information_schema.tables -where - table_schema ilike '{{ schema_pattern }}' - and table_name ilike '{{ table_pattern }}' - and table_name not ilike '{{ exclude }}' + select distinct + table_schema as "table_schema", + table_name as "table_name", + {{ dbt_utils.get_table_types_sql() }} + from {{ database }}.information_schema.tables + where + table_schema ilike '{{ schema_pattern }}' + and table_name ilike '{{ table_pattern }}' + and table_name not ilike '{{ exclude }}' {% endmacro %} @@ -31,14 +31,14 @@ where schema_pattern, table_pattern, exclude="", database=target.database ) %} -{% if "%" in schema_pattern %} -{% set schemata = dbt_utils._bigquery__get_matching_schemata( - schema_pattern, database -) %} -{% else %} {% set schemata = [schema_pattern] %} -{% endif %} + {% if "%" in schema_pattern %} + {% set schemata = dbt_utils._bigquery__get_matching_schemata( + schema_pattern, database + ) %} + {% else %} {% set schemata = [schema_pattern] %} + {% endif %} -{% set sql %} + {% set sql %} {% for schema in schemata %} select distinct table_schema, @@ -52,29 +52,29 @@ where {% if not loop.last %} union all {% endif %} {% endfor %} -{% endset %} + {% endset %} -{{ return(sql) }} + {{ return(sql) }} {% endmacro %} {% macro _bigquery__get_matching_schemata(schema_pattern, database) %} -{% if execute %} + {% if execute %} -{% set sql %} + {% set sql %} select schema_name from {{ adapter.quote(database) }}.INFORMATION_SCHEMA.SCHEMATA where lower(schema_name) like lower('{{ schema_pattern }}') -{% endset %} + {% endset %} -{% set results = run_query(sql) %} + {% set results = run_query(sql) %} -{% set schemata = results.columns["schema_name"].values() %} + {% set schemata = results.columns["schema_name"].values() %} -{{ return(schemata) }} + {{ return(schemata) }} -{% else %} {{ return([]) }} + {% else %} {{ return([]) }} -{% endif %} + {% endif %} {% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index 68234b55..d4b6dda7 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,26 +1,26 @@ {% macro get_tables_by_prefix_sql( schema, prefix, exclude="", database=target.database ) %} -{{ - return( - adapter.dispatch("get_tables_by_prefix_sql", "dbt_utils")( - schema, prefix, exclude, database + {{ + return( + adapter.dispatch("get_tables_by_prefix_sql", "dbt_utils")( + schema, prefix, exclude, database + ) ) - ) -}} + }} {% endmacro %} {% macro default__get_tables_by_prefix_sql( schema, prefix, exclude="", database=target.database ) %} -{{ - dbt_utils.get_tables_by_pattern_sql( - schema_pattern=schema, - table_pattern=prefix ~ "%", - exclude=exclude, - database=database, - ) -}} + {{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern=schema, + table_pattern=prefix ~ "%", + exclude=exclude, + database=database, + ) + }} {% endmacro %} diff --git a/macros/sql/groupby.sql b/macros/sql/groupby.sql index b46201d1..b0ae3f70 100644 --- a/macros/sql/groupby.sql +++ b/macros/sql/groupby.sql @@ -1,10 +1,10 @@ {%- macro group_by(n) -%} -{{ return(adapter.dispatch("group_by", "dbt_utils")(n)) }} + {{ return(adapter.dispatch("group_by", "dbt_utils")(n)) }} {% endmacro %} {%- macro default__group_by(n) -%} -group by - {% for i in range(1, n + 1) -%} {{ i }}{{ ',' if not loop.last }} {%- endfor -%} + group by + {% for i in range(1, n + 1) -%} {{ i }}{{ ',' if not loop.last }} {%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index e2585c13..b3107c39 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -8,65 +8,65 @@ The arguments should be float type. {% macro degrees_to_radians(degrees) -%} acos(-1) * {{ degrees }} / 180 {%- endmacro %} {% macro haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} -{{ - return( - adapter.dispatch("haversine_distance", "dbt_utils")( - lat1, lon1, lat2, lon2, unit + {{ + return( + adapter.dispatch("haversine_distance", "dbt_utils")( + lat1, lon1, lat2, lon2, unit + ) ) - ) -}} + }} {% endmacro %} {% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} -{%- if unit == "mi" %} {% set conversion_rate = 1 %} -{% elif unit == "km" %} {% set conversion_rate = 1.60934 %} -{% else %} -{{ - exceptions.raise_compiler_error( - "unit input must be one of 'mi' or 'km'. Got " ~ unit - ) -}} -{% endif %} + {%- if unit == "mi" %} {% set conversion_rate = 1 %} + {% elif unit == "km" %} {% set conversion_rate = 1.60934 %} + {% else %} + {{ + exceptions.raise_compiler_error( + "unit input must be one of 'mi' or 'km'. Got " ~ unit + ) + }} + {% endif %} -2 -* 3961 -* asin( - sqrt( - power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) - + cos(radians({{ lat1 }})) - * cos(radians({{ lat2 }})) - * power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2) + 2 + * 3961 + * asin( + sqrt( + power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + + cos(radians({{ lat1 }})) + * cos(radians({{ lat2 }})) + * power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2) + ) ) -) -* {{ conversion_rate }} + * {{ conversion_rate }} {%- endmacro %} {% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} -{% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} -{% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} -{% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} -{% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} -{%- if unit == "mi" %} {% set conversion_rate = 1 %} -{% elif unit == "km" %} {% set conversion_rate = 1.60934 %} -{% else %} -{{ - exceptions.raise_compiler_error( - "unit input must be one of 'mi' or 'km'. Got " ~ unit - ) -}} -{% endif %} -2 -* 3961 -* asin( - sqrt( - power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) - + cos({{ radians_lat1 }}) - * cos({{ radians_lat2 }}) - * power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2) + {% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} + {% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} + {% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} + {% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} + {%- if unit == "mi" %} {% set conversion_rate = 1 %} + {% elif unit == "km" %} {% set conversion_rate = 1.60934 %} + {% else %} + {{ + exceptions.raise_compiler_error( + "unit input must be one of 'mi' or 'km'. Got " ~ unit + ) + }} + {% endif %} + 2 + * 3961 + * asin( + sqrt( + power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + + cos({{ radians_lat1 }}) + * cos({{ radians_lat2 }}) + * power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2) + ) ) -) -* {{ conversion_rate }} + * {{ conversion_rate }} {%- endmacro %} diff --git a/macros/sql/nullcheck.sql b/macros/sql/nullcheck.sql index 9e0aed1f..ba4b5a5b 100644 --- a/macros/sql/nullcheck.sql +++ b/macros/sql/nullcheck.sql @@ -1,17 +1,17 @@ {% macro nullcheck(cols) %} -{{ return(adapter.dispatch("nullcheck", "dbt_utils")(cols)) }} + {{ return(adapter.dispatch("nullcheck", "dbt_utils")(cols)) }} {% endmacro %} {% macro default__nullcheck(cols) %} -{%- for col in cols %} + {%- for col in cols %} -{% if col.is_string() -%} nullif({{ col.name }}, '') as {{ col.name }} + {% if col.is_string() -%} nullif({{ col.name }}, '') as {{ col.name }} -{%- else -%} {{ col.name }} + {%- else -%} {{ col.name }} -{%- endif -%} + {%- endif -%} -{%- if not loop.last -%}, {%- endif -%} + {%- if not loop.last -%}, {%- endif -%} -{%- endfor -%} + {%- endfor -%} {% endmacro %} diff --git a/macros/sql/nullcheck_table.sql b/macros/sql/nullcheck_table.sql index a01f2e86..ef521b4c 100644 --- a/macros/sql/nullcheck_table.sql +++ b/macros/sql/nullcheck_table.sql @@ -1,14 +1,14 @@ {% macro nullcheck_table(relation) %} -{{ return(adapter.dispatch("nullcheck_table", "dbt_utils")(relation)) }} + {{ return(adapter.dispatch("nullcheck_table", "dbt_utils")(relation)) }} {% endmacro %} {% macro default__nullcheck_table(relation) %} -{%- do dbt_utils._is_relation(relation, "nullcheck_table") -%} -{%- do dbt_utils._is_ephemeral(relation, "nullcheck_table") -%} -{% set cols = adapter.get_columns_in_relation(relation) %} + {%- do dbt_utils._is_relation(relation, "nullcheck_table") -%} + {%- do dbt_utils._is_ephemeral(relation, "nullcheck_table") -%} + {% set cols = adapter.get_columns_in_relation(relation) %} -select {{ dbt_utils.nullcheck(cols) }} -from {{ relation }} + select {{ dbt_utils.nullcheck(cols) }} + from {{ relation }} {% endmacro %} diff --git a/macros/sql/pivot.sql b/macros/sql/pivot.sql index ff20b17e..33118ef7 100644 --- a/macros/sql/pivot.sql +++ b/macros/sql/pivot.sql @@ -52,23 +52,23 @@ Arguments: quote_identifiers=True, distinct=False ) %} -{{ - return( - adapter.dispatch("pivot", "dbt_utils")( - column, - values, - alias, - agg, - cmp, - prefix, - suffix, - then_value, - else_value, - quote_identifiers, - distinct, + {{ + return( + adapter.dispatch("pivot", "dbt_utils")( + column, + values, + alias, + agg, + cmp, + prefix, + suffix, + then_value, + else_value, + quote_identifiers, + distinct, + ) ) - ) -}} + }} {% endmacro %} {% macro default__pivot( @@ -84,20 +84,21 @@ Arguments: quote_identifiers=True, distinct=False ) %} -{% for value in values %} -{{ agg }} ( - {% if distinct %}distinct {% endif %} - case - when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' - then {{ then_value }} - else {{ else_value }} - end -) -{% if alias %} -{% if quote_identifiers %} as {{ adapter.quote(prefix ~ value ~ suffix) }} -{% else %} as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} -{% endif %} -{% endif %} -{% if not loop.last %},{% endif %} -{% endfor %} + {% for value in values %} + {{ agg }} ( + {% if distinct %}distinct {% endif %} + case + when + {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' + then {{ then_value }} + else {{ else_value }} + end + ) + {% if alias %} + {% if quote_identifiers %} as {{ adapter.quote(prefix ~ value ~ suffix) }} + {% else %} as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} + {% endif %} + {% endif %} + {% if not loop.last %},{% endif %} + {% endfor %} {% endmacro %} diff --git a/macros/sql/safe_add.sql b/macros/sql/safe_add.sql index 62ff9d7f..82a5ce7f 100644 --- a/macros/sql/safe_add.sql +++ b/macros/sql/safe_add.sql @@ -1,20 +1,20 @@ {%- macro safe_add() -%} -{# needed for safe_add to allow for non-keyword arguments see SO post #} -{# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} -{% set frustrating_jinja_feature = varargs %} -{{ return(adapter.dispatch("safe_add", "dbt_utils")(*varargs)) }} + {# needed for safe_add to allow for non-keyword arguments see SO post #} + {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} + {% set frustrating_jinja_feature = varargs %} + {{ return(adapter.dispatch("safe_add", "dbt_utils")(*varargs)) }} {% endmacro %} {%- macro default__safe_add() -%} -{% set fields = [] %} + {% set fields = [] %} -{%- for field in varargs -%} + {%- for field in varargs -%} -{% do fields.append("coalesce(" ~ field ~ ", 0)") %} + {% do fields.append("coalesce(" ~ field ~ ", 0)") %} -{%- endfor -%} + {%- endfor -%} -{{ fields | join(" +\n ") }} + {{ fields | join(" +\n ") }} {%- endmacro -%} diff --git a/macros/sql/star.sql b/macros/sql/star.sql index 8d989d78..a92ac6a7 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -1,23 +1,23 @@ {% macro star(from, relation_alias=False, except=[], prefix='', suffix='') -%} -{{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} + {{ return(adapter.dispatch('star', 'dbt_utils')(from, relation_alias, except, prefix, suffix)) }} {% endmacro %} {% macro default__star(from, relation_alias=False, except=[], prefix='', suffix='') -%} -{%- do dbt_utils._is_relation(from, 'star') -%} -{%- do dbt_utils._is_ephemeral(from, 'star') -%} + {%- do dbt_utils._is_relation(from, 'star') -%} + {%- do dbt_utils._is_ephemeral(from, 'star') -%} -{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} {{ return("*") }} {% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("*") }} {% endif %} -{%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} + {%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} -{%- if relation_alias %} {{ relation_alias }}. -{% else %} -{%- endif -%} {{ adapter.quote(col) | trim }} -{%- if prefix != "" or suffix != "" %} -as {{ adapter.quote(prefix ~ col ~ suffix) | trim }} -{%- endif -%} -{%- if not loop.last %},{{ "\n " }}{% endif %} + {%- if relation_alias %} {{ relation_alias }}. + {% else %} + {%- endif -%} {{ adapter.quote(col) | trim }} + {%- if prefix != "" or suffix != "" %} + as {{ adapter.quote(prefix ~ col ~ suffix) | trim }} + {%- endif -%} + {%- if not loop.last %},{{ "\n " }}{% endif %} -{%- endfor -%} + {%- endfor -%} {%- endmacro %} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index 97e5bd74..137d4583 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -1,44 +1,50 @@ {%- macro surrogate_key(field_list) -%} -{# needed for safe_add to allow for non-keyword arguments see SO post #} -{# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} -{% set frustrating_jinja_feature = varargs %} -{{ return(adapter.dispatch("surrogate_key", "dbt_utils")(field_list, *varargs)) }} + {# needed for safe_add to allow for non-keyword arguments see SO post #} + {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} + {% set frustrating_jinja_feature = varargs %} + {{ return(adapter.dispatch("surrogate_key", "dbt_utils")(field_list, *varargs)) }} {% endmacro %} {%- macro default__surrogate_key(field_list) -%} -{%- if varargs | length >= 1 or field_list is string %} + {%- if varargs | length >= 1 or field_list is string %} -{%- set error_message = " Warning: the `surrogate_key` macro now takes a single list argument instead of \ multiple string arguments. Support for multiple string arguments will be \ deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ ".format( - model.package_name, model.name -) -%} + {%- set error_message = " Warning: the `surrogate_key` macro now takes a single list argument instead of \ multiple string arguments. Support for multiple string arguments will be \ deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name + ) -%} -{%- do exceptions.warn(error_message) -%} + {%- do exceptions.warn(error_message) -%} -{# first argument is not included in varargs, so add first element to field_list_xf #} -{%- set field_list_xf = [field_list] -%} + {# first argument is not included in varargs, so add first element to field_list_xf #} + {%- set field_list_xf = [field_list] -%} -{%- for field in varargs %} {%- set _ = field_list_xf.append(field) -%} {%- endfor -%} + {%- for field in varargs %} + {%- set _ = field_list_xf.append(field) -%} + {%- endfor -%} -{%- else -%} + {%- else -%} -{# if using list, just set field_list_xf as field_list #} -{%- set field_list_xf = field_list -%} + {# if using list, just set field_list_xf as field_list #} + {%- set field_list_xf = field_list -%} -{%- endif -%} + {%- endif -%} -{%- set fields = [] -%} + {%- set fields = [] -%} -{%- for field in field_list_xf -%} + {%- for field in field_list_xf -%} -{%- set _ = fields.append( - "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" -) -%} + {%- set _ = fields.append( + "coalesce(cast(" + ~ field + ~ " as " + ~ dbt_utils.type_string() + ~ "), '')" + ) -%} -{%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} + {%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} -{%- endfor -%} + {%- endfor -%} -{{ dbt_utils.hash(dbt_utils.concat(fields)) }} + {{ dbt_utils.hash(dbt_utils.concat(fields)) }} {%- endmacro -%} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index 413a63c9..3527fa49 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -5,13 +5,13 @@ exclude=[], source_column_name="_dbt_source_relation" ) -%} -{{ - return( - adapter.dispatch("union_relations", "dbt_utils")( - relations, column_override, include, exclude, source_column_name + {{ + return( + adapter.dispatch("union_relations", "dbt_utils")( + relations, column_override, include, exclude, source_column_name + ) ) - ) -}} + }} {% endmacro %} {%- macro default__union_relations( @@ -22,107 +22,110 @@ source_column_name="_dbt_source_relation" ) -%} -{%- if exclude and include -%} -{{ - exceptions.raise_compiler_error( - "Both an exclude and include list were provided to the `union` macro. Only one is allowed" - ) -}} -{%- endif -%} + {%- if exclude and include -%} + {{ + exceptions.raise_compiler_error( + "Both an exclude and include list were provided to the `union` macro. Only one is allowed" + ) + }} + {%- endif -%} -{#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} -{%- if not execute %} {{ return("") }} {% endif -%} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} + {%- if not execute %} {{ return("") }} {% endif -%} -{%- set column_override = column_override if column_override is not none else {} -%} + {%- set column_override = column_override if column_override is not none else {} -%} -{%- set relation_columns = {} -%} -{%- set column_superset = {} -%} + {%- set relation_columns = {} -%} + {%- set column_superset = {} -%} -{%- for relation in relations -%} + {%- for relation in relations -%} -{%- do relation_columns.update({relation: []}) -%} + {%- do relation_columns.update({relation: []}) -%} -{%- do dbt_utils._is_relation(relation, "union_relations") -%} -{%- do dbt_utils._is_ephemeral(relation, "union_relations") -%} -{%- set cols = adapter.get_columns_in_relation(relation) -%} -{%- for col in cols -%} + {%- do dbt_utils._is_relation(relation, "union_relations") -%} + {%- do dbt_utils._is_ephemeral(relation, "union_relations") -%} + {%- set cols = adapter.get_columns_in_relation(relation) -%} + {%- for col in cols -%} -{#- If an exclude list was provided and the column is in the list, do nothing -#} -{%- if exclude and col.column in exclude -%} + {#- If an exclude list was provided and the column is in the list, do nothing -#} + {%- if exclude and col.column in exclude -%} -{#- If an include list was provided and the column is not in the list, do nothing -#} -{%- elif include and col.column not in include -%} + {#- If an include list was provided and the column is not in the list, do nothing -#} + {%- elif include and col.column not in include -%} -{#- Otherwise add the column to the column superset -#} -{%- else -%} + {#- Otherwise add the column to the column superset -#} + {%- else -%} -{#- update the list of columns in this relation -#} -{%- do relation_columns[relation].append(col.column) -%} + {#- update the list of columns in this relation -#} + {%- do relation_columns[relation].append(col.column) -%} -{%- if col.column in column_superset -%} + {%- if col.column in column_superset -%} -{%- set stored = column_superset[col.column] -%} -{%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} + {%- set stored = column_superset[col.column] -%} + {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} -{%- do column_superset.update({col.column: col}) -%} + {%- do column_superset.update({col.column: col}) -%} -{%- endif %} + {%- endif %} -{%- else -%} {%- do column_superset.update({col.column: col}) -%} + {%- else -%} {%- do column_superset.update({col.column: col}) -%} -{%- endif -%} + {%- endif -%} -{%- endif -%} + {%- endif -%} -{%- endfor -%} -{%- endfor -%} + {%- endfor -%} + {%- endfor -%} -{%- set ordered_column_names = column_superset.keys() -%} + {%- set ordered_column_names = column_superset.keys() -%} -{% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} -{%- set relations_string -%} + {% if ( + include | length > 0 or exclude | length > 0 + ) and not column_superset.keys() %} + {%- set relations_string -%} {%- for relation in relations -%} {{ relation.name }} {%- if not loop.last %}, {% endif -%} {%- endfor -%} -{%- endset -%} + {%- endset -%} -{%- set error_message -%} + {%- set error_message -%} There were no columns found to union for relations {{ relations_string }} -{%- endset -%} + {%- endset -%} -{{ exceptions.raise_compiler_error(error_message) }} -{%- endif -%} + {{ exceptions.raise_compiler_error(error_message) }} + {%- endif -%} -{%- for relation in relations %} + {%- for relation in relations %} -( - select + ( + select - cast( - {{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }} - ) as {{ source_column_name }}, - {% for col_name in ordered_column_names -%} + cast( + {{ dbt_utils.string_literal(relation) }} + as {{ dbt_utils.type_string() }} + ) as {{ source_column_name }}, + {% for col_name in ordered_column_names -%} - {%- set col = column_superset[col_name] %} - {%- set col_type = column_override.get(col.column, col.data_type) %} - {%- set col_name = ( - adapter.quote(col_name) - if col_name in relation_columns[relation] - else "null" -) %} - cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} - {% if not loop.last %},{% endif -%} + {%- set col = column_superset[col_name] %} + {%- set col_type = column_override.get(col.column, col.data_type) %} + {%- set col_name = ( + adapter.quote(col_name) + if col_name in relation_columns[relation] + else "null" + ) %} + cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} + {% if not loop.last %},{% endif -%} - {%- endfor %} + {%- endfor %} - from {{ relation }} -) + from {{ relation }} + ) -{% if not loop.last -%} -union all -{% endif -%} + {% if not loop.last -%} + union all + {% endif -%} -{%- endfor -%} + {%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index 8c133a35..2a46c7f0 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -20,13 +20,13 @@ Arguments: value_name="value", table=none ) -%} -{{ - return( - adapter.dispatch("unpivot", "dbt_utils")( - relation, cast_to, exclude, remove, field_name, value_name, table + {{ + return( + adapter.dispatch("unpivot", "dbt_utils")( + relation, cast_to, exclude, remove, field_name, value_name, table + ) ) - ) -}} + }} {% endmacro %} {% macro default__unpivot( @@ -39,65 +39,68 @@ Arguments: table=none ) -%} -{% if table %} -{%- set error_message = " Warning: the `unpivot` macro no longer accepts a `table` parameter. \ This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ The {}.{} model triggered this warning. \ ".format( - model.package_name, model.name -) -%} -{%- do exceptions.warn(error_message) -%} -{% endif %} - -{% if relation and table %} -{{ - exceptions.raise_compiler_error( - "Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`)." - ) -}} -{% elif not relation and table %} {% set relation = table %} -{% elif not relation and not table %} -{{ - exceptions.raise_compiler_error( - "Error: argument `relation` is required for `unpivot` macro." - ) -}} -{% endif %} - -{%- set exclude = exclude if exclude is not none else [] %} -{%- set remove = remove if remove is not none else [] %} - -{%- set include_cols = [] %} - -{%- set table_columns = {} %} - -{%- do table_columns.update({relation: []}) %} - -{%- do dbt_utils._is_relation(relation, "unpivot") -%} -{%- do dbt_utils._is_ephemeral(relation, "unpivot") -%} -{%- set cols = adapter.get_columns_in_relation(relation) %} - -{%- for col in cols -%} -{%- if col.column.lower() not in remove | map( - "lower" -) and col.column.lower() not in exclude | map("lower") -%} -{% do include_cols.append(col) %} -{%- endif %} -{%- endfor %} - -{%- for col in include_cols -%} -select - {%- for exclude_col in exclude %} {{ exclude_col }}, {%- endfor %} - - cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, - cast( - {% if col.data_type == "boolean" %}{{ dbt_utils.cast_bool_to_text(col.column) }} - {% else %}{{ col.column }} - {% endif %} as {{ cast_to }} - ) as {{ value_name }} - -from {{ relation }} - -{% if not loop.last -%} -union all -{% endif -%} -{%- endfor -%} + {% if table %} + {%- set error_message = " Warning: the `unpivot` macro no longer accepts a `table` parameter. \ This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name + ) -%} + {%- do exceptions.warn(error_message) -%} + {% endif %} + + {% if relation and table %} + {{ + exceptions.raise_compiler_error( + "Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`)." + ) + }} + {% elif not relation and table %} {% set relation = table %} + {% elif not relation and not table %} + {{ + exceptions.raise_compiler_error( + "Error: argument `relation` is required for `unpivot` macro." + ) + }} + {% endif %} + + {%- set exclude = exclude if exclude is not none else [] %} + {%- set remove = remove if remove is not none else [] %} + + {%- set include_cols = [] %} + + {%- set table_columns = {} %} + + {%- do table_columns.update({relation: []}) %} + + {%- do dbt_utils._is_relation(relation, "unpivot") -%} + {%- do dbt_utils._is_ephemeral(relation, "unpivot") -%} + {%- set cols = adapter.get_columns_in_relation(relation) %} + + {%- for col in cols -%} + {%- if col.column.lower() not in remove | map( + "lower" + ) and col.column.lower() not in exclude | map("lower") -%} + {% do include_cols.append(col) %} + {%- endif %} + {%- endfor %} + + {%- for col in include_cols -%} + select + {%- for exclude_col in exclude %} {{ exclude_col }}, {%- endfor %} + + cast( + '{{ col.column }}' as {{ dbt_utils.type_string() }} + ) as {{ field_name }}, + cast( + {% if col.data_type == "boolean" %} + {{ dbt_utils.cast_bool_to_text(col.column) }} + {% else %}{{ col.column }} + {% endif %} as {{ cast_to }} + ) as {{ value_name }} + + from {{ relation }} + + {% if not loop.last -%} + union all + {% endif -%} + {%- endfor -%} {%- endmacro %} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index 1de744b6..faafff86 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -1,27 +1,27 @@ {% macro get_url_host(field) -%} -{{ return(adapter.dispatch("get_url_host", "dbt_utils")(field)) }} + {{ return(adapter.dispatch("get_url_host", "dbt_utils")(field)) }} {% endmacro %} {% macro default__get_url_host(field) -%} -{%- set parsed = dbt_utils.split_part( - dbt_utils.split_part( - dbt_utils.replace( + {%- set parsed = dbt_utils.split_part( + dbt_utils.split_part( dbt_utils.replace( - dbt_utils.replace(field, "'android-app://'", "''"), - "'http://'", + dbt_utils.replace( + dbt_utils.replace(field, "'android-app://'", "''"), + "'http://'", + "''", + ), + "'https://'", "''", ), - "'https://'", - "''", + "'/'", + 1, ), - "'/'", + "'?'", 1, - ), - "'?'", - 1, -) -%} + ) -%} -{{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} + {{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} {%- endmacro %} diff --git a/macros/web/get_url_parameter.sql b/macros/web/get_url_parameter.sql index 64b21ba4..9b3127ab 100644 --- a/macros/web/get_url_parameter.sql +++ b/macros/web/get_url_parameter.sql @@ -1,15 +1,19 @@ {% macro get_url_parameter(field, url_parameter) -%} -{{ return(adapter.dispatch("get_url_parameter", "dbt_utils")(field, url_parameter)) }} + {{ + return( + adapter.dispatch("get_url_parameter", "dbt_utils")(field, url_parameter) + ) + }} {% endmacro %} {% macro default__get_url_parameter(field, url_parameter) -%} -{%- set formatted_url_parameter = "'" + url_parameter + "='" -%} + {%- set formatted_url_parameter = "'" + url_parameter + "='" -%} -{%- set split = dbt_utils.split_part( - dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1 -) -%} + {%- set split = dbt_utils.split_part( + dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1 + ) -%} -nullif({{ split }}, '') + nullif({{ split }}, '') {%- endmacro %} diff --git a/macros/web/get_url_path.sql b/macros/web/get_url_path.sql index c295a13d..ef5c2f14 100644 --- a/macros/web/get_url_path.sql +++ b/macros/web/get_url_path.sql @@ -1,28 +1,29 @@ {% macro get_url_path(field) -%} -{{ return(adapter.dispatch("get_url_path", "dbt_utils")(field)) }} + {{ return(adapter.dispatch("get_url_path", "dbt_utils")(field)) }} {% endmacro %} {% macro default__get_url_path(field) -%} -{%- set stripped_url = dbt_utils.replace( - dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''" -) -%} + {%- set stripped_url = dbt_utils.replace( + dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''" + ) -%} -{%- set first_slash_pos -%} + {%- set first_slash_pos -%} coalesce( nullif({{dbt_utils.position("'/'", stripped_url)}}, 0), {{dbt_utils.position("'?'", stripped_url)}} - 1 ) -{%- endset -%} + {%- endset -%} -{%- set parsed_path = dbt_utils.split_part( - dbt_utils.right( - stripped_url, dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos - ), - "'?'", - 1, -) -%} + {%- set parsed_path = dbt_utils.split_part( + dbt_utils.right( + stripped_url, + dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos, + ), + "'?'", + 1, + ) -%} -{{ dbt_utils.safe_cast(parsed_path, dbt_utils.type_string()) }} + {{ dbt_utils.safe_cast(parsed_path, dbt_utils.type_string()) }} {%- endmacro %} From c62b99f46a8dd9ccd22abdabadff07a7d1899e3f Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Wed, 10 May 2023 16:19:55 -0600 Subject: [PATCH 21/22] chore: apply sqlfmt 6e9615c (#22) --- macros/cross_db_utils/except.sql | 3 +-- macros/cross_db_utils/intersect.sql | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 2f208201..888e1940 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -11,7 +11,6 @@ {% macro bigquery__except() %} - except - distinct + except distinct {% endmacro %} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index 94f36cd9..08abf793 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -11,7 +11,6 @@ {% macro bigquery__intersect() %} - intersect - distinct + intersect distinct {% endmacro %} From 72fe12f13279bdaec4dabd156fa3a3102ed9c4e7 Mon Sep 17 00:00:00 2001 From: Ted Conbeer Date: Fri, 22 Nov 2024 16:06:51 -0700 Subject: [PATCH 22/22] chore: apply sqlfmt 717530c --- .../macros/assert_equal_values.sql | 21 +- integration_tests/macros/limit_zero.sql | 10 +- integration_tests/macros/tests.sql | 5 +- .../models/cross_db_utils/test_any_value.sql | 41 +- .../models/cross_db_utils/test_bool_or.sql | 8 +- .../models/cross_db_utils/test_concat.sql | 11 +- .../cross_db_utils/test_current_timestamp.sql | 2 - .../test_current_timestamp_in_utc.sql | 3 +- .../models/cross_db_utils/test_date_trunc.sql | 11 +- .../models/cross_db_utils/test_dateadd.sql | 35 +- .../models/cross_db_utils/test_datediff.sql | 113 +++-- .../models/cross_db_utils/test_hash.sql | 11 +- .../models/cross_db_utils/test_last_day.sql | 16 +- .../models/cross_db_utils/test_length.sql | 13 +- .../models/cross_db_utils/test_listagg.sql | 130 +++--- .../models/cross_db_utils/test_position.sql | 10 +- .../models/cross_db_utils/test_replace.sql | 21 +- .../models/cross_db_utils/test_right.sql | 10 +- .../models/cross_db_utils/test_safe_cast.sql | 9 +- .../models/cross_db_utils/test_split_part.sql | 16 +- .../cross_db_utils/test_width_bucket.sql | 10 +- .../models/datetime/test_date_spine.sql | 31 +- .../test_equal_column_subset.sql | 10 +- .../generic_tests/test_equal_rowcount.sql | 11 +- .../generic_tests/test_fewer_rows_than.sql | 11 +- .../models/generic_tests/test_recency.sql | 14 +- .../models/geo/test_haversine_distance_km.sql | 39 +- .../models/geo/test_haversine_distance_mi.sql | 65 ++- .../expected_insert_by_period.sql | 9 +- .../test_insert_by_period.sql | 26 +- .../models/sql/test_deduplicate.sql | 29 +- .../models/sql/test_generate_series.sql | 13 +- .../models/sql/test_get_column_values.sql | 35 +- .../test_get_filtered_columns_in_relation.sql | 25 +- .../sql/test_get_relations_by_pattern.sql | 17 +- ...test_get_relations_by_prefix_and_union.sql | 4 +- integration_tests/models/sql/test_groupby.sql | 28 +- .../models/sql/test_nullcheck_table.sql | 41 +- integration_tests/models/sql/test_pivot.sql | 17 +- .../models/sql/test_pivot_apostrophe.sql | 15 +- .../models/sql/test_safe_add.sql | 11 +- integration_tests/models/sql/test_star.sql | 15 +- .../models/sql/test_star_aggregate.sql | 23 +- .../models/sql/test_star_prefix_suffix.sql | 22 +- .../models/sql/test_star_uppercase.sql | 15 +- .../models/sql/test_surrogate_key.sql | 15 +- integration_tests/models/sql/test_union.sql | 9 +- .../models/sql/test_union_base.sql | 6 +- integration_tests/models/sql/test_unpivot.sql | 36 +- .../models/sql/test_unpivot_bool.sql | 34 +- .../models/sql/test_unpivot_original_api.sql | 29 +- .../models/web/test_url_host.sql | 13 +- .../models/web/test_url_path.sql | 12 +- integration_tests/models/web/test_urls.sql | 13 +- ...et_query_results_as_dict_objects_equal.sql | 89 ++-- .../tests/jinja_helpers/test_slugify.sql | 2 +- .../test_get_column_values_use_default.sql | 44 +- macros/cross_db_utils/_is_ephemeral.sql | 7 +- macros/cross_db_utils/_is_relation.sql | 12 +- macros/cross_db_utils/any_value.sql | 12 +- macros/cross_db_utils/bool_or.sql | 20 +- macros/cross_db_utils/cast_bool_to_text.sql | 5 +- macros/cross_db_utils/concat.sql | 8 +- macros/cross_db_utils/current_timestamp.sql | 23 +- macros/cross_db_utils/datatypes.sql | 79 +--- macros/cross_db_utils/date_trunc.sql | 9 +- macros/cross_db_utils/dateadd.sql | 26 +- macros/cross_db_utils/datediff.sql | 111 +++-- .../cross_db_utils/escape_single_quotes.sql | 8 +- macros/cross_db_utils/except.sql | 6 +- macros/cross_db_utils/hash.sql | 6 +- macros/cross_db_utils/identifier.sql | 23 +- macros/cross_db_utils/intersect.sql | 2 +- macros/cross_db_utils/last_day.sql | 42 +- macros/cross_db_utils/length.sql | 18 +- macros/cross_db_utils/listagg.sql | 121 +++--- macros/cross_db_utils/literal.sql | 7 +- macros/cross_db_utils/position.sql | 20 +- macros/cross_db_utils/replace.sql | 11 +- macros/cross_db_utils/right.sql | 33 +- macros/cross_db_utils/safe_cast.sql | 8 +- macros/cross_db_utils/split_part.sql | 19 +- macros/cross_db_utils/width_bucket.sql | 41 +- macros/generic_tests/accepted_range.sql | 59 +-- macros/generic_tests/at_least_one.sql | 25 +- macros/generic_tests/cardinality_equality.sql | 84 ++-- macros/generic_tests/equal_rowcount.sql | 45 +- macros/generic_tests/equality.sql | 91 ++-- macros/generic_tests/expression_is_true.sql | 31 +- macros/generic_tests/fewer_rows_than.sql | 72 ++-- .../mutually_exclusive_ranges.sql | 223 +++++----- macros/generic_tests/not_accepted_values.sql | 48 +-- macros/generic_tests/not_constant.sql | 21 +- macros/generic_tests/not_null_proportion.sql | 42 +- macros/generic_tests/recency.sql | 28 +- macros/generic_tests/relationships_where.sql | 64 +-- macros/generic_tests/sequential_values.sql | 78 ++-- macros/generic_tests/test_not_null_where.sql | 25 +- macros/generic_tests/test_unique_where.sql | 25 +- .../unique_combination_of_columns.sql | 71 ++-- macros/jinja_helpers/log_info.sql | 2 +- macros/jinja_helpers/pretty_log_format.sql | 4 +- macros/jinja_helpers/pretty_time.sql | 6 +- macros/jinja_helpers/slugify.sql | 14 +- .../insert_by_period_materialization.sql | 387 +++++++++++------- macros/sql/date_spine.sql | 81 ++-- macros/sql/deduplicate.sql | 67 +-- macros/sql/generate_series.sql | 44 +- macros/sql/get_column_values.sql | 60 ++- .../sql/get_filtered_columns_in_relation.sql | 6 +- macros/sql/get_query_results_as_dict.sql | 11 +- macros/sql/get_relations_by_pattern.sql | 35 +- macros/sql/get_relations_by_prefix.sql | 31 +- macros/sql/get_table_types_sql.sql | 52 ++- macros/sql/get_tables_by_pattern_sql.sql | 54 ++- macros/sql/get_tables_by_prefix_sql.sql | 32 +- macros/sql/groupby.sql | 7 +- macros/sql/haversine_distance.sql | 89 ++-- macros/sql/nullcheck.sql | 18 +- macros/sql/nullcheck_table.sql | 12 +- macros/sql/pivot.sql | 102 +++-- macros/sql/safe_add.sql | 12 +- macros/sql/star.sql | 15 +- macros/sql/surrogate_key.sql | 55 ++- macros/sql/union.sql | 90 ++-- macros/sql/unpivot.sql | 115 ++++-- macros/web/get_url_host.sql | 32 +- macros/web/get_url_parameter.sql | 14 +- macros/web/get_url_path.sql | 33 +- 129 files changed, 2375 insertions(+), 2148 deletions(-) diff --git a/integration_tests/macros/assert_equal_values.sql b/integration_tests/macros/assert_equal_values.sql index d4f02618..e9eaa637 100644 --- a/integration_tests/macros/assert_equal_values.sql +++ b/integration_tests/macros/assert_equal_values.sql @@ -1,11 +1,10 @@ {% macro assert_equal_values(actual_object, expected_object) %} -{% if not execute %} + {% if not execute %} {# pass #} + {% elif actual_object != expected_object %} -{% elif actual_object != expected_object %} - - {% set msg %} + {% set msg %} Expected did not match actual ----------- @@ -18,15 +17,13 @@ ----------- --->{{ expected_object }}<--- - {% endset %} - - {{ log(msg, info=True) }} + {% endset %} - select 'fail' + {{ log(msg, info=True) }} -{% else %} + select 'fail' - select 'ok' {{ limit_zero() }} + {% else %} select 'ok' {{ limit_zero() }} -{% endif %} -{% endmacro %} \ No newline at end of file + {% endif %} +{% endmacro %} diff --git a/integration_tests/macros/limit_zero.sql b/integration_tests/macros/limit_zero.sql index 697849d1..f3dee37a 100644 --- a/integration_tests/macros/limit_zero.sql +++ b/integration_tests/macros/limit_zero.sql @@ -1,11 +1,7 @@ -{% macro my_custom_macro() %} - whatever -{% endmacro %} +{% macro my_custom_macro() %} whatever {% endmacro %} {% macro limit_zero() %} - {{ return(adapter.dispatch('limit_zero', 'dbt_utils')()) }} + {{ return(adapter.dispatch("limit_zero", "dbt_utils")()) }} {% endmacro %} -{% macro default__limit_zero() %} - {{ return('limit 0') }} -{% endmacro %} \ No newline at end of file +{% macro default__limit_zero() %} {{ return("limit 0") }} {% endmacro %} diff --git a/integration_tests/macros/tests.sql b/integration_tests/macros/tests.sql index 046fab3a..9ef0e19d 100644 --- a/integration_tests/macros/tests.sql +++ b/integration_tests/macros/tests.sql @@ -1,12 +1,11 @@ - {% test assert_equal(model, actual, expected) %} -select * from {{ model }} where {{ actual }} != {{ expected }} + select * from {{ model }} where {{ actual }} != {{ expected }} {% endtest %} {% test not_empty_string(model, column_name) %} -select * from {{ model }} where {{ column_name }} = '' + select * from {{ model }} where {{ column_name }} = '' {% endtest %} diff --git a/integration_tests/models/cross_db_utils/test_any_value.sql b/integration_tests/models/cross_db_utils/test_any_value.sql index 9b27ed1d..6781b41f 100644 --- a/integration_tests/models/cross_db_utils/test_any_value.sql +++ b/integration_tests/models/cross_db_utils/test_any_value.sql @@ -1,19 +1,26 @@ -with some_model as ( - select 1 as id, 'abc' as key_name, 'dbt' as static_col union all - select 2 as id, 'abc' as key_name, 'dbt' as static_col union all - select 3 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 4 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 5 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 6 as id, 'xyz' as key_name, 'test' as static_col -), +with + some_model as ( + select 1 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 2 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 3 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 4 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 5 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 6 as id, 'xyz' as key_name, 'test' as static_col + ), -final as ( - select - key_name, - {{ dbt_utils.any_value('static_col') }} as static_col, - count(id) as num_rows - from some_model - group by key_name -) + final as ( + select + key_name, + {{ dbt_utils.any_value("static_col") }} as static_col, + count(id) as num_rows + from some_model + group by key_name + ) -select * from final \ No newline at end of file +select * +from final diff --git a/integration_tests/models/cross_db_utils/test_bool_or.sql b/integration_tests/models/cross_db_utils/test_bool_or.sql index 7375d1e4..1d6f4ad5 100644 --- a/integration_tests/models/cross_db_utils/test_bool_or.sql +++ b/integration_tests/models/cross_db_utils/test_bool_or.sql @@ -1,5 +1,3 @@ -select - key, - {{ dbt_utils.bool_or('val1 = val2') }} as value -from {{ ref('data_bool_or' )}} -group by key \ No newline at end of file +select key, {{ dbt_utils.bool_or("val1 = val2") }} as value +from {{ ref("data_bool_or") }} +group by key diff --git a/integration_tests/models/cross_db_utils/test_concat.sql b/integration_tests/models/cross_db_utils/test_concat.sql index 9efeb748..c48697a2 100644 --- a/integration_tests/models/cross_db_utils/test_concat.sql +++ b/integration_tests/models/cross_db_utils/test_concat.sql @@ -1,12 +1,5 @@ +with data as (select * from {{ ref("data_concat") }}) -with data as ( - - select * from {{ ref('data_concat') }} - -) - -select - {{ dbt_utils.concat(['input_1', 'input_2']) }} as actual, - output as expected +select {{ dbt_utils.concat(["input_1", "input_2"]) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp.sql b/integration_tests/models/cross_db_utils/test_current_timestamp.sql index 9a777f28..8d5e99df 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp.sql @@ -1,6 +1,4 @@ - -- how can we test this better? select {{ dbt_utils.current_timestamp() }} as actual, {{ dbt_utils.current_timestamp() }} as expected - diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql index 55bc8e23..b3607bb9 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql @@ -1,5 +1,4 @@ - -- how can we test this better? select {{ dbt_utils.current_timestamp_in_utc() }} as actual, - {{ dbt_utils.current_timestamp_in_utc() }} as expected \ No newline at end of file + {{ dbt_utils.current_timestamp_in_utc() }} as expected diff --git a/integration_tests/models/cross_db_utils/test_date_trunc.sql b/integration_tests/models/cross_db_utils/test_date_trunc.sql index bac21fed..307495ee 100644 --- a/integration_tests/models/cross_db_utils/test_date_trunc.sql +++ b/integration_tests/models/cross_db_utils/test_date_trunc.sql @@ -1,12 +1,7 @@ - -with data as ( - - select * from {{ ref('data_date_trunc') }} - -) +with data as (select * from {{ ref("data_date_trunc") }}) select - cast({{dbt_utils.date_trunc('day', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("day", "updated_at") }} as date) as actual, day as expected from data @@ -14,7 +9,7 @@ from data union all select - cast({{ dbt_utils.date_trunc('month', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("month", "updated_at") }} as date) as actual, month as expected from data diff --git a/integration_tests/models/cross_db_utils/test_dateadd.sql b/integration_tests/models/cross_db_utils/test_dateadd.sql index 3e199d86..7adacfa8 100644 --- a/integration_tests/models/cross_db_utils/test_dateadd.sql +++ b/integration_tests/models/cross_db_utils/test_dateadd.sql @@ -1,16 +1,31 @@ - -with data as ( - - select * from {{ ref('data_dateadd') }} - -) +with data as (select * from {{ ref("data_dateadd") }}) select case - when datepart = 'hour' then cast({{ dbt_utils.dateadd('hour', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'day' then cast({{ dbt_utils.dateadd('day', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'month' then cast({{ dbt_utils.dateadd('month', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'year' then cast({{ dbt_utils.dateadd('year', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) + when datepart = 'hour' + then + cast( + {{ dbt_utils.dateadd("hour", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'day' + then + cast( + {{ dbt_utils.dateadd("day", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'month' + then + cast( + {{ dbt_utils.dateadd("month", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'year' + then + cast( + {{ dbt_utils.dateadd("year", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_datediff.sql b/integration_tests/models/cross_db_utils/test_datediff.sql index a2340456..bad3cbf1 100644 --- a/integration_tests/models/cross_db_utils/test_datediff.sql +++ b/integration_tests/models/cross_db_utils/test_datediff.sql @@ -1,20 +1,22 @@ - -with data as ( - - select * from {{ ref('data_datediff') }} - -) +with data as (select * from {{ ref("data_datediff") }}) select case - when datepart = 'second' then {{ dbt_utils.datediff('first_date', 'second_date', 'second') }} - when datepart = 'minute' then {{ dbt_utils.datediff('first_date', 'second_date', 'minute') }} - when datepart = 'hour' then {{ dbt_utils.datediff('first_date', 'second_date', 'hour') }} - when datepart = 'day' then {{ dbt_utils.datediff('first_date', 'second_date', 'day') }} - when datepart = 'week' then {{ dbt_utils.datediff('first_date', 'second_date', 'week') }} - when datepart = 'month' then {{ dbt_utils.datediff('first_date', 'second_date', 'month') }} - when datepart = 'year' then {{ dbt_utils.datediff('first_date', 'second_date', 'year') }} + when datepart = 'second' + then {{ dbt_utils.datediff("first_date", "second_date", "second") }} + when datepart = 'minute' + then {{ dbt_utils.datediff("first_date", "second_date", "minute") }} + when datepart = 'hour' + then {{ dbt_utils.datediff("first_date", "second_date", "hour") }} + when datepart = 'day' + then {{ dbt_utils.datediff("first_date", "second_date", "day") }} + when datepart = 'week' + then {{ dbt_utils.datediff("first_date", "second_date", "week") }} + when datepart = 'month' + then {{ dbt_utils.datediff("first_date", "second_date", "month") }} + when datepart = 'year' + then {{ dbt_utils.datediff("first_date", "second_date", "year") }} else null end as actual, result as expected @@ -22,14 +24,77 @@ select from data -- Also test correct casting of literal values. - -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "microsecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "millisecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "microsecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "millisecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year" + ) + }} as actual, 1 as expected diff --git a/integration_tests/models/cross_db_utils/test_hash.sql b/integration_tests/models/cross_db_utils/test_hash.sql index 8f017780..5f0cca83 100644 --- a/integration_tests/models/cross_db_utils/test_hash.sql +++ b/integration_tests/models/cross_db_utils/test_hash.sql @@ -1,12 +1,5 @@ +with data as (select * from {{ ref("data_hash") }}) -with data as ( - - select * from {{ ref('data_hash') }} - -) - -select - {{ dbt_utils.hash('input_1') }} as actual, - output as expected +select {{ dbt_utils.hash("input_1") }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_last_day.sql b/integration_tests/models/cross_db_utils/test_last_day.sql index fee3a823..080e286b 100644 --- a/integration_tests/models/cross_db_utils/test_last_day.sql +++ b/integration_tests/models/cross_db_utils/test_last_day.sql @@ -1,15 +1,13 @@ - -with data as ( - - select * from {{ ref('data_last_day') }} - -) +with data as (select * from {{ ref("data_last_day") }}) select case - when date_part = 'month' then {{ dbt_utils.last_day('date_day', 'month') }} - when date_part = 'quarter' then {{ dbt_utils.last_day('date_day', 'quarter') }} - when date_part = 'year' then {{ dbt_utils.last_day('date_day', 'year') }} + when date_part = 'month' + then {{ dbt_utils.last_day("date_day", "month") }} + when date_part = 'quarter' + then {{ dbt_utils.last_day("date_day", "quarter") }} + when date_part = 'year' + then {{ dbt_utils.last_day("date_day", "year") }} else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_length.sql b/integration_tests/models/cross_db_utils/test_length.sql index f4ef099c..d6352d05 100644 --- a/integration_tests/models/cross_db_utils/test_length.sql +++ b/integration_tests/models/cross_db_utils/test_length.sql @@ -1,12 +1,5 @@ -with data as ( +with data as (select * from {{ ref("data_length") }}) - select * from {{ ref('data_length') }} +select {{ dbt_utils.length("expression") }} as actual, output as expected -) - -select - - {{ dbt_utils.length('expression') }} as actual, - output as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_listagg.sql b/integration_tests/models/cross_db_utils/test_listagg.sql index 006948de..27d96745 100644 --- a/integration_tests/models/cross_db_utils/test_listagg.sql +++ b/integration_tests/models/cross_db_utils/test_listagg.sql @@ -1,69 +1,63 @@ -with data as ( - - select * from {{ ref('data_listagg') }} - -), - -data_output as ( - - select * from {{ ref('data_listagg_output') }} - -), - -calculate as ( - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col") }} as actual, - 'bottom_ordered' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col", 2) }} as actual, - 'bottom_ordered_limited' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "', '") }} as actual, - 'comma_whitespace_unordered' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('DISTINCT string_text', "','") }} as actual, - 'distinct_comma' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text') }} as actual, - 'no_params' as version - from data - where group_col = 3 - group by group_col - -) - -select - calculate.actual, - data_output.expected +with + data as (select * from {{ ref("data_listagg") }}), + + data_output as (select * from {{ ref("data_listagg_output") }}), + + calculate as ( + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col") }} + as actual, + 'bottom_ordered' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col", 2) }} + as actual, + 'bottom_ordered_limited' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "', '") }} as actual, + 'comma_whitespace_unordered' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("DISTINCT string_text", "','") }} as actual, + 'distinct_comma' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text") }} as actual, + 'no_params' as version + from data + where group_col = 3 + group by group_col + + ) + +select calculate.actual, data_output.expected from calculate -left join data_output -on calculate.group_col = data_output.group_col -and calculate.version = data_output.version \ No newline at end of file +left join + data_output + on calculate.group_col = data_output.group_col + and calculate.version = data_output.version diff --git a/integration_tests/models/cross_db_utils/test_position.sql b/integration_tests/models/cross_db_utils/test_position.sql index b09be66d..d1785f14 100644 --- a/integration_tests/models/cross_db_utils/test_position.sql +++ b/integration_tests/models/cross_db_utils/test_position.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_position') }} - -) +with data as (select * from {{ ref("data_position") }}) select - {{ dbt_utils.position('substring_text', 'string_text') }} as actual, + {{ dbt_utils.position("substring_text", "string_text") }} as actual, result as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_replace.sql b/integration_tests/models/cross_db_utils/test_replace.sql index 6c8a8aa1..0e413c8a 100644 --- a/integration_tests/models/cross_db_utils/test_replace.sql +++ b/integration_tests/models/cross_db_utils/test_replace.sql @@ -1,18 +1,19 @@ -with data as ( +with + data as ( - select - - *, - coalesce(search_chars, '') as old_chars, - coalesce(replace_chars, '') as new_chars - - from {{ ref('data_replace') }} + select -) + *, + coalesce(search_chars, '') as old_chars, + coalesce(replace_chars, '') as new_chars + + from {{ ref("data_replace") }} + + ) select - {{ dbt_utils.replace('string_text', 'old_chars', 'new_chars') }} as actual, + {{ dbt_utils.replace("string_text", "old_chars", "new_chars") }} as actual, result as expected from data diff --git a/integration_tests/models/cross_db_utils/test_right.sql b/integration_tests/models/cross_db_utils/test_right.sql index eaad4f4f..de2df7dc 100644 --- a/integration_tests/models/cross_db_utils/test_right.sql +++ b/integration_tests/models/cross_db_utils/test_right.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_right') }} - -) +with data as (select * from {{ ref("data_right") }}) select - {{ dbt_utils.right('string_text', 'length_expression') }} as actual, + {{ dbt_utils.right("string_text", "length_expression") }} as actual, coalesce(output, '') as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_safe_cast.sql b/integration_tests/models/cross_db_utils/test_safe_cast.sql index 8ed74fc3..95a3acde 100644 --- a/integration_tests/models/cross_db_utils/test_safe_cast.sql +++ b/integration_tests/models/cross_db_utils/test_safe_cast.sql @@ -1,12 +1,7 @@ - -with data as ( - - select * from {{ ref('data_safe_cast') }} - -) +with data as (select * from {{ ref("data_safe_cast") }}) select - {{ dbt_utils.safe_cast('field', dbt_utils.type_string()) }} as actual, + {{ dbt_utils.safe_cast("field", dbt_utils.type_string()) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_split_part.sql b/integration_tests/models/cross_db_utils/test_split_part.sql index 6a10327c..79b98da0 100644 --- a/integration_tests/models/cross_db_utils/test_split_part.sql +++ b/integration_tests/models/cross_db_utils/test_split_part.sql @@ -1,28 +1,20 @@ - -with data as ( - - select * from {{ ref('data_split_part') }} - -) +with data as (select * from {{ ref("data_split_part") }}) select - {{ dbt_utils.split_part('parts', 'split_on', 1) }} as actual, - result_1 as expected + {{ dbt_utils.split_part("parts", "split_on", 1) }} as actual, result_1 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 2) }} as actual, - result_2 as expected + {{ dbt_utils.split_part("parts", "split_on", 2) }} as actual, result_2 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 3) }} as actual, - result_3 as expected + {{ dbt_utils.split_part("parts", "split_on", 3) }} as actual, result_3 as expected from data diff --git a/integration_tests/models/cross_db_utils/test_width_bucket.sql b/integration_tests/models/cross_db_utils/test_width_bucket.sql index dc2b8609..ef445ed1 100644 --- a/integration_tests/models/cross_db_utils/test_width_bucket.sql +++ b/integration_tests/models/cross_db_utils/test_width_bucket.sql @@ -1,12 +1,8 @@ - -with data as ( - - select * from {{ ref('data_width_bucket') }} - -) +with data as (select * from {{ ref("data_width_bucket") }}) select - {{ dbt_utils.width_bucket('amount', 'min_value', 'max_value', 'num_buckets') }} as actual, + {{ dbt_utils.width_bucket("amount", "min_value", "max_value", "num_buckets") }} + as actual, bucket as expected from data diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index fa4ae52b..1bdf4dca 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -1,26 +1,27 @@ - -- snowflake doesn't like this as a view because the `generate_series` -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} + +with + date_spine as ( -{{ config(materialized='table') }} + {% if target.type == "postgres" %} + {{ + dbt_utils.date_spine( + "day", "'2018-01-01'::date", "'2018-01-10'::date" + ) + }} -with date_spine as ( + {% elif target.type == "bigquery" %} + select cast(date_day as date) as date_day + from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) - {% if target.type == 'postgres' %} - {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} - - {% elif target.type == 'bigquery' %} - select cast(date_day as date) as date_day - from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) - - {% else %} - {{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} - {% endif %} + {% else %}{{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} + {% endif %} -) + ) select date_day from date_spine - diff --git a/integration_tests/models/generic_tests/test_equal_column_subset.sql b/integration_tests/models/generic_tests/test_equal_column_subset.sql index b426bb63..7ecae303 100644 --- a/integration_tests/models/generic_tests/test_equal_column_subset.sql +++ b/integration_tests/models/generic_tests/test_equal_column_subset.sql @@ -1,9 +1,5 @@ -{{ config(materialized='ephemeral') }} +{{ config(materialized="ephemeral") }} -select +select first_name, last_name, email - first_name, - last_name, - email - -from {{ ref('data_people') }} +from {{ ref("data_people") }} diff --git a/integration_tests/models/generic_tests/test_equal_rowcount.sql b/integration_tests/models/generic_tests/test_equal_rowcount.sql index 01b1a549..621ccf61 100644 --- a/integration_tests/models/generic_tests/test_equal_rowcount.sql +++ b/integration_tests/models/generic_tests/test_equal_rowcount.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_equal_rowcount") }}) - select * from {{ ref('data_test_equal_rowcount') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_fewer_rows_than.sql b/integration_tests/models/generic_tests/test_fewer_rows_than.sql index c2ad4cbd..621fce6c 100644 --- a/integration_tests/models/generic_tests/test_fewer_rows_than.sql +++ b/integration_tests/models/generic_tests/test_fewer_rows_than.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_fewer_rows_than_table_1") }}) - select * from {{ ref('data_test_fewer_rows_than_table_1') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_recency.sql b/integration_tests/models/generic_tests/test_recency.sql index d300e572..0b0d286a 100644 --- a/integration_tests/models/generic_tests/test_recency.sql +++ b/integration_tests/models/generic_tests/test_recency.sql @@ -1,12 +1,12 @@ +{% if target.type == "postgres" %} -{% if target.type == 'postgres' %} - -select - {{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as today + select {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as today {% else %} -select - cast({{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as datetime) as today - + select + cast( + {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as datetime + ) as today + {% endif %} diff --git a/integration_tests/models/geo/test_haversine_distance_km.sql b/integration_tests/models/geo/test_haversine_distance_km.sql index e8ca4818..fce70682 100644 --- a/integration_tests/models/geo/test_haversine_distance_km.sql +++ b/integration_tests/models/geo/test_haversine_distance_km.sql @@ -1,23 +1,20 @@ -with data as ( - select * from {{ ref('data_haversine_km') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='km' +with + data as (select * from {{ ref("data_haversine_km") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="km", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/geo/test_haversine_distance_mi.sql b/integration_tests/models/geo/test_haversine_distance_mi.sql index 53f8172c..9ff4a1f7 100644 --- a/integration_tests/models/geo/test_haversine_distance_mi.sql +++ b/integration_tests/models/geo/test_haversine_distance_mi.sql @@ -1,39 +1,36 @@ -with data as ( - select * from {{ ref('data_haversine_mi') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='mi' +with + data as (select * from {{ ref("data_haversine_mi") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="mi", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data - union all + union all - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/materializations/expected_insert_by_period.sql b/integration_tests/models/materializations/expected_insert_by_period.sql index 0a198b6b..0d7d68c1 100644 --- a/integration_tests/models/materializations/expected_insert_by_period.sql +++ b/integration_tests/models/materializations/expected_insert_by_period.sql @@ -1,10 +1,5 @@ -{{ - config( - materialized = 'view', - enabled=(target.type == 'redshift') - ) -}} +{{ config(materialized="view", enabled=(target.type == "redshift")) }} select * -from {{ ref('data_insert_by_period') }} +from {{ ref("data_insert_by_period") }} where id in (2, 3, 4, 5, 6) diff --git a/integration_tests/models/materializations/test_insert_by_period.sql b/integration_tests/models/materializations/test_insert_by_period.sql index 8cc396d3..b48d22f6 100644 --- a/integration_tests/models/materializations/test_insert_by_period.sql +++ b/integration_tests/models/materializations/test_insert_by_period.sql @@ -1,18 +1,16 @@ {{ - config( - materialized = 'insert_by_period', - period = 'month', - timestamp_field = 'created_at', - start_date = '2018-01-01', - stop_date = '2018-06-01', - enabled=(target.type == 'redshift') - ) + config( + materialized="insert_by_period", + period="month", + timestamp_field="created_at", + start_date="2018-01-01", + stop_date="2018-06-01", + enabled=(target.type == "redshift"), + ) }} -with events as ( - select * - from {{ ref('data_insert_by_period') }} - where __PERIOD_FILTER__ -) +with + events as (select * from {{ ref("data_insert_by_period") }} where __period_filter__) -select * from events +select * +from events diff --git a/integration_tests/models/sql/test_deduplicate.sql b/integration_tests/models/sql/test_deduplicate.sql index 81fe81e7..47649a61 100644 --- a/integration_tests/models/sql/test_deduplicate.sql +++ b/integration_tests/models/sql/test_deduplicate.sql @@ -1,22 +1,19 @@ with -source as ( - select * - from {{ ref('data_deduplicate') }} - where user_id = 1 -), + source as (select * from {{ ref("data_deduplicate") }} where user_id = 1), -deduped as ( + deduped as ( - {{ - dbt_utils.deduplicate( - ref('data_deduplicate'), - group_by='user_id', - order_by='version desc', - relation_alias="source" - ) | indent - }} + {{ + dbt_utils.deduplicate( + ref("data_deduplicate"), + group_by="user_id", + order_by="version desc", + relation_alias="source", + ) | indent + }} -) + ) -select * from deduped +select * +from deduped diff --git a/integration_tests/models/sql/test_generate_series.sql b/integration_tests/models/sql/test_generate_series.sql index 11370b7b..fd104edf 100644 --- a/integration_tests/models/sql/test_generate_series.sql +++ b/integration_tests/models/sql/test_generate_series.sql @@ -1,15 +1,10 @@ - -- snowflake doesn't like this as a view because the `generate_series` -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} -{{ config(materialized='table') }} - -with data as ( - - {{ dbt_utils.generate_series(10) }} - -) +with data as ({{ dbt_utils.generate_series(10) }}) -select generated_number from data +select generated_number +from data diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index 36214984..b3c5ba8f 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -1,29 +1,34 @@ +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values"), "field", default=[], order_by="field" +) %} -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} +{% if target.type == "snowflake" %} -{% if target.type == 'snowflake' %} + select + {% for val in column_values -%} -select - {% for val in column_values -%} + sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} + {%- if not loop.last %},{% endif -%} - sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} - {%- if not loop.last %},{% endif -%} + {%- endfor %} - {%- endfor %} - -from {{ ref('data_get_column_values') }} + from {{ ref("data_get_column_values") }} {% else %} -select - {% for val in column_values -%} + select + {% for val in column_values -%} - {{dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} as count_{{ val }} - {%- if not loop.last %},{% endif -%} + {{ + dbt_utils.safe_cast( + "sum(case when field = '" ~ val ~ "' then 1 else 0 end)", + dbt_utils.type_string(), + ) + }} as count_{{ val }} {%- if not loop.last %},{% endif -%} - {%- endfor %} + {%- endfor %} -from {{ ref('data_get_column_values') }} + from {{ ref("data_get_column_values") }} {% endif %} diff --git a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql index 7b3ca72f..0690dd19 100644 --- a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql +++ b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql @@ -1,16 +1,21 @@ -{% set exclude_field = 'field_1' %} -{% set column_names = dbt_utils.get_filtered_columns_in_relation(from= ref('data_filtered_columns_in_relation'), except=[exclude_field]) %} +{% set exclude_field = "field_1" %} +{% set column_names = dbt_utils.get_filtered_columns_in_relation( + from=ref("data_filtered_columns_in_relation"), except=[exclude_field] +) %} -with data as ( +with + data as ( - select + select - {% for column_name in column_names %} - max({{ column_name }}) as {{ column_name }} {% if not loop.last %},{% endif %} - {% endfor %} + {% for column_name in column_names %} + max({{ column_name }}) as {{ column_name }} + {% if not loop.last %},{% endif %} + {% endfor %} - from {{ ref('data_filtered_columns_in_relation') }} + from {{ ref("data_filtered_columns_in_relation") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index 52ab07a2..653f3598 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,16 +1,11 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} -{% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %} +{% set relations = dbt_utils.get_relations_by_pattern( + target.schema ~ "%", "data_events_%" +) %} -with unioned as ( - - {{ dbt_utils.union_relations(relations) }} - -) +with unioned as ({{ dbt_utils.union_relations(relations) }}) -select - - user_id, - event +select user_id, event from unioned diff --git a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql index 481f233e..6e968e79 100644 --- a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql +++ b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql @@ -1,4 +1,4 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} -{% set relations = dbt_utils.get_relations_by_prefix(target.schema, 'data_events_') %} +{% set relations = dbt_utils.get_relations_by_prefix(target.schema, "data_events_") %} {{ dbt_utils.union_relations(relations) }} diff --git a/integration_tests/models/sql/test_groupby.sql b/integration_tests/models/sql/test_groupby.sql index 50a6ced7..a63f3a5c 100644 --- a/integration_tests/models/sql/test_groupby.sql +++ b/integration_tests/models/sql/test_groupby.sql @@ -1,24 +1,14 @@ -with test_data as ( - - select - - {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string() )}} as column_1, - {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string() )}} as column_2 - -), +with + test_data as ( -grouped as ( + select - select - *, - count(*) as total - - from test_data - {{ dbt_utils.group_by(2) }} - -) - -select * from grouped + {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string()) }} as column_1, + {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string()) }} as column_2 + ), + grouped as (select *, count(*) as total from test_data {{ dbt_utils.group_by(2) }}) +select * +from grouped diff --git a/integration_tests/models/sql/test_nullcheck_table.sql b/integration_tests/models/sql/test_nullcheck_table.sql index ea66c8ec..a2cec220 100644 --- a/integration_tests/models/sql/test_nullcheck_table.sql +++ b/integration_tests/models/sql/test_nullcheck_table.sql @@ -1,41 +1,30 @@ -{{ config( materialized = "table" ) }} - --- TO DO: remove if-statement - -{% set tbl = ref('data_nullcheck_table') %} - +{{ config(materialized="table") }} -with nulled as ( +-- TO DO: remove if-statement +{% set tbl = ref("data_nullcheck_table") %} - {{ dbt_utils.nullcheck_table(tbl) }} -) +with nulled as ({{ dbt_utils.nullcheck_table(tbl) }}) -{% if target.type == 'snowflake' %} +{% if target.type == "snowflake" %} -select - field_1::varchar as field_1, - field_2::varchar as field_2, - field_3::varchar as field_3 + select + field_1::varchar as field_1, + field_2::varchar as field_2, + field_3::varchar as field_3 -from nulled + from nulled {% else %} -select + select - {{ dbt_utils.safe_cast('field_1', - dbt_utils.type_string() - )}} as field_1, + {{ dbt_utils.safe_cast("field_1", dbt_utils.type_string()) }} as field_1, - {{ dbt_utils.safe_cast('field_2', - dbt_utils.type_string() - )}} as field_2, + {{ dbt_utils.safe_cast("field_2", dbt_utils.type_string()) }} as field_2, - {{ dbt_utils.safe_cast('field_3', - dbt_utils.type_string() - )}} as field_3 + {{ dbt_utils.safe_cast("field_3", dbt_utils.type_string()) }} as field_3 -from nulled + from nulled {% endif %} diff --git a/integration_tests/models/sql/test_pivot.sql b/integration_tests/models/sql/test_pivot.sql index a63e6e3b..dcc0d9a1 100644 --- a/integration_tests/models/sql/test_pivot.sql +++ b/integration_tests/models/sql/test_pivot.sql @@ -1,17 +1,10 @@ - -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE'] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue'] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} + {% set column_values = ["RED", "BLUE"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue"] %} {% set cmp = "=" %} {% endif %} -select - size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp) }} +select size, {{ dbt_utils.pivot("color", column_values, cmp=cmp) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_pivot_apostrophe.sql b/integration_tests/models/sql/test_pivot_apostrophe.sql index 792f4686..b1af6828 100644 --- a/integration_tests/models/sql/test_pivot_apostrophe.sql +++ b/integration_tests/models/sql/test_pivot_apostrophe.sql @@ -1,17 +1,12 @@ - -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE', "BLUE'S"] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue', "blue's"] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} + {% set column_values = ["RED", "BLUE", "BLUE'S"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue", "blue's"] %} {% set cmp = "=" %} {% endif %} select size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp, quote_identifiers=False) }} + {{ dbt_utils.pivot("color", column_values, cmp=cmp, quote_identifiers=False) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_safe_add.sql b/integration_tests/models/sql/test_safe_add.sql index af3894d0..2d6eb778 100644 --- a/integration_tests/models/sql/test_safe_add.sql +++ b/integration_tests/models/sql/test_safe_add.sql @@ -1,12 +1,5 @@ +with data as (select * from {{ ref("data_safe_add") }}) -with data as ( - - select * from {{ ref('data_safe_add') }} - -) - -select - {{ dbt_utils.safe_add('field_1', 'field_2', 'field_3') }} as actual, - expected +select {{ dbt_utils.safe_add("field_1", "field_2", "field_3") }} as actual, expected from data diff --git a/integration_tests/models/sql/test_star.sql b/integration_tests/models/sql/test_star.sql index 2092e16a..5932635c 100644 --- a/integration_tests/models/sql/test_star.sql +++ b/integration_tests/models/sql/test_star.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'field_3' %} +{% set exclude_field = "field_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref("data_star"), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_aggregate.sql b/integration_tests/models/sql/test_star_aggregate.sql index 9dcd7c2d..16b6c94b 100644 --- a/integration_tests/models/sql/test_star_aggregate.sql +++ b/integration_tests/models/sql/test_star_aggregate.sql @@ -1,16 +1,17 @@ -/*This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ +/* This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ +{% set selected_columns = dbt_utils.star( + from=ref("data_star_aggregate"), except=["value_field"] +) %} -{% set selected_columns = dbt_utils.star(from=ref('data_star_aggregate'), except=['value_field']) %} +with + data as ( -with data as ( + select {{ selected_columns }}, sum(value_field) as value_field_sum - select - {{ selected_columns }}, - sum(value_field) as value_field_sum + from {{ ref("data_star_aggregate") }} + group by {{ selected_columns }} - from {{ ref('data_star_aggregate') }} - group by {{ selected_columns }} + ) -) - -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_prefix_suffix.sql b/integration_tests/models/sql/test_star_prefix_suffix.sql index ae1c64fe..5d66ad06 100644 --- a/integration_tests/models/sql/test_star_prefix_suffix.sql +++ b/integration_tests/models/sql/test_star_prefix_suffix.sql @@ -1,13 +1,19 @@ -{% set prefix_with = 'prefix_' if target.type != 'snowflake' else 'PREFIX_' %} -{% set suffix_with = '_suffix' if target.type != 'snowflake' else '_SUFFIX' %} +{% set prefix_with = "prefix_" if target.type != "snowflake" else "PREFIX_" %} +{% set suffix_with = "_suffix" if target.type != "snowflake" else "_SUFFIX" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} + select + {{ + dbt_utils.star( + from=ref("data_star"), prefix=prefix_with, suffix=suffix_with + ) + }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data \ No newline at end of file +select * +from data diff --git a/integration_tests/models/sql/test_star_uppercase.sql b/integration_tests/models/sql/test_star_uppercase.sql index 6179e691..f8656abe 100644 --- a/integration_tests/models/sql/test_star_uppercase.sql +++ b/integration_tests/models/sql/test_star_uppercase.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'FIELD_3' %} +{% set exclude_field = "FIELD_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref("data_star"), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_surrogate_key.sql b/integration_tests/models/sql/test_surrogate_key.sql index fd90939f..8f694083 100644 --- a/integration_tests/models/sql/test_surrogate_key.sql +++ b/integration_tests/models/sql/test_surrogate_key.sql @@ -1,15 +1,12 @@ - -with data as ( - - select * from {{ ref('data_surrogate_key') }} - -) +with data as (select * from {{ ref("data_surrogate_key") }}) select - {{ dbt_utils.surrogate_key('column_1') }} as actual_column_1_only, + {{ dbt_utils.surrogate_key("column_1") }} as actual_column_1_only, expected_column_1_only, - {{ dbt_utils.surrogate_key('column_1', 'column_2', 'column_3') }} as actual_all_columns_arguments, - {{ dbt_utils.surrogate_key(['column_1', 'column_2', 'column_3']) }} as actual_all_columns_list, + {{ dbt_utils.surrogate_key("column_1", "column_2", "column_3") }} + as actual_all_columns_arguments, + {{ dbt_utils.surrogate_key(["column_1", "column_2", "column_3"]) }} + as actual_all_columns_list, expected_all_columns from data diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql index 69836833..e1b5e34c 100644 --- a/integration_tests/models/sql/test_union.sql +++ b/integration_tests/models/sql/test_union.sql @@ -1,8 +1 @@ - -select - id, - name, - favorite_color - -from {{ ref('test_union_base') }} - +select id, name, favorite_color from {{ ref("test_union_base") }} diff --git a/integration_tests/models/sql/test_union_base.sql b/integration_tests/models/sql/test_union_base.sql index a51bd2b8..869baecd 100644 --- a/integration_tests/models/sql/test_union_base.sql +++ b/integration_tests/models/sql/test_union_base.sql @@ -1,5 +1 @@ - -{{ dbt_utils.union_relations([ - ref('data_union_table_1'), - ref('data_union_table_2')] -) }} +{{ dbt_utils.union_relations([ref("data_union_table_1"), ref("data_union_table_2")]) }} diff --git a/integration_tests/models/sql/test_unpivot.sql b/integration_tests/models/sql/test_unpivot.sql index 8cf1ad6e..a9081a6c 100644 --- a/integration_tests/models/sql/test_unpivot.sql +++ b/integration_tests/models/sql/test_unpivot.sql @@ -1,33 +1,29 @@ - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - remove=['name'], - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + remove=["name"], + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_bool.sql b/integration_tests/models/sql/test_unpivot_bool.sql index 26842c01..2642afaf 100644 --- a/integration_tests/models/sql/test_unpivot_bool.sql +++ b/integration_tests/models/sql/test_unpivot_bool.sql @@ -1,32 +1,28 @@ - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot_bool'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot_bool"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_original_api.sql b/integration_tests/models/sql/test_unpivot_original_api.sql index f6b9395a..83f6ddf6 100644 --- a/integration_tests/models/sql/test_unpivot_original_api.sql +++ b/integration_tests/models/sql/test_unpivot_original_api.sql @@ -1,32 +1,29 @@ - -- unpivot() was enhanced with 3 new parameters -- This test targets the original API. - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, case - when '{{ target.name }}' = 'snowflake' then lower(FIELD_NAME) - else field_name + when '{{ target.name }}' = 'snowflake' then lower(field_name) else field_name end as field_name, value -from ( - {{ dbt_utils.unpivot( - table=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + table=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + ) + }} + ) as sbq diff --git a/integration_tests/models/web/test_url_host.sql b/integration_tests/models/web/test_url_host.sql index 06a9f69c..2e18e65d 100644 --- a/integration_tests/models/web/test_url_host.sql +++ b/integration_tests/models/web/test_url_host.sql @@ -1,12 +1,5 @@ -with data as ( - - select * from {{ref('data_url_host')}} - -) +with data as (select * from {{ ref("data_url_host") }}) -select +select {{ dbt_utils.get_url_host("original_url") }} as actual, parsed_url as expected - {{ dbt_utils.get_url_host('original_url') }} as actual, - parsed_url as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/web/test_url_path.sql b/integration_tests/models/web/test_url_path.sql index c03e5547..16f45ae0 100644 --- a/integration_tests/models/web/test_url_path.sql +++ b/integration_tests/models/web/test_url_path.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ref('data_url_path')}} - -) +with data as (select * from {{ ref("data_url_path") }}) select - coalesce({{ dbt_utils.get_url_path('original_url') }}, '') as actual, + coalesce({{ dbt_utils.get_url_path("original_url") }}, '') as actual, coalesce(parsed_path, '') as expected - -from data \ No newline at end of file + +from data diff --git a/integration_tests/models/web/test_urls.sql b/integration_tests/models/web/test_urls.sql index 60353dbb..321dce5c 100644 --- a/integration_tests/models/web/test_urls.sql +++ b/integration_tests/models/web/test_urls.sql @@ -1,20 +1,13 @@ - -with data as ( - - select * from {{ ref('data_urls') }} - -) +with data as (select * from {{ ref("data_urls") }}) select - {{ dbt_utils.get_url_parameter('url', 'utm_medium') }} as actual, - medium as expected + {{ dbt_utils.get_url_parameter("url", "utm_medium") }} as actual, medium as expected from data union all select - {{ dbt_utils.get_url_parameter('url', 'utm_source') }} as actual, - source as expected + {{ dbt_utils.get_url_parameter("url", "utm_source") }} as actual, source as expected from data diff --git a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql index 55d5400e..db0f8dc4 100644 --- a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql +++ b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql @@ -1,81 +1,76 @@ -- depends_on: {{ ref('data_get_query_results_as_dict') }} - -{% set expected_dictionary={ - 'col_1': [1, 2, 3], - 'col_2': ['a', 'b', 'c'], - 'col_3': [True, False, none] +{% set expected_dictionary = { + "col_1": [1, 2, 3], + "col_2": ["a", "b", "c"], + "col_3": [True, False, none], } %} {#- Handle snowflake casing silliness -#} -{% if target.type == 'snowflake' %} -{% set expected_dictionary={ - 'COL_1': [1, 2, 3], - 'COL_2': ['a', 'b', 'c'], - 'COL_3': [True, False, none] -} %} +{% if target.type == "snowflake" %} + {% set expected_dictionary = { + "COL_1": [1, 2, 3], + "COL_2": ["a", "b", "c"], + "COL_3": [True, False, none], + } %} {% endif %} -{% set actual_dictionary=dbt_utils.get_query_results_as_dict( - "select * from " ~ ref('data_get_query_results_as_dict') ~ " order by 1" +{% set actual_dictionary = dbt_utils.get_query_results_as_dict( + "select * from " ~ ref("data_get_query_results_as_dict") ~ " order by 1" ) %} {#- For reasons that remain unclear, Jinja won't return True for actual_dictionary == expected_dictionary. Instead, we'll manually check that the values of these dictionaries are equivalent. -#} - -{% set ns = namespace( - pass=True, - err_msg = "" -) %} +{% set ns = namespace(pass=True, err_msg="") %} {% if execute %} -{#- Check that the dictionaries have the same keys -#} -{% set expected_keys=expected_dictionary.keys() | list | sort %} -{% set actual_keys=actual_dictionary.keys() | list | sort %} + {#- Check that the dictionaries have the same keys -#} + {% set expected_keys = expected_dictionary.keys() | list | sort %} + {% set actual_keys = actual_dictionary.keys() | list | sort %} -{% if expected_keys != actual_keys %} - {% set ns.pass=False %} - {% set ns.err_msg %} + {% if expected_keys != actual_keys %} + {% set ns.pass = False %} + {% set ns.err_msg %} The two dictionaries have different keys: expected_dictionary has keys: {{ expected_keys }} actual_dictionary has keys: {{ actual_keys }} - {% endset %} + {% endset %} -{% else %} + {% else %} -{% for key, value in expected_dictionary.items() %} - {% set expected_length=expected_dictionary[key] | length %} - {% set actual_length=actual_dictionary[key] | length %} + {% for key, value in expected_dictionary.items() %} + {% set expected_length = expected_dictionary[key] | length %} + {% set actual_length = actual_dictionary[key] | length %} - {% if expected_length != actual_length %} - {% set ns.pass=False %} - {% set ns.err_msg %} + {% if expected_length != actual_length %} + {% set ns.pass = False %} + {% set ns.err_msg %} The {{ key }} column has different lengths: expected_dictionary[{{ key }}] has length {{ expected_length }} actual_dictionary[{{ key }}] has length {{ actual_length }} - {% endset %} + {% endset %} - {% else %} + {% else %} - {% for i in range(value | length) %} - {% set expected_value=expected_dictionary[key][i] %} - {% set actual_value=actual_dictionary[key][i] %} - {% if expected_value != actual_value %} - {% set ns.pass=False %} - {% set ns.err_msg %} + {% for i in range(value | length) %} + {% set expected_value = expected_dictionary[key][i] %} + {% set actual_value = actual_dictionary[key][i] %} + {% if expected_value != actual_value %} + {% set ns.pass = False %} + {% set ns.err_msg %} The {{ key }} column has differing values: expected_dictionary[{{ key }}][{{ i }}] == {{ expected_value }} actual_dictionary[{{ key }}][{{ i }}] == {{ actual_value }} - {% endset %} + {% endset %} + {% endif %} + {% endfor %} {% endif %} - {% endfor %} - {% endif %} -{% endfor %} + {% endfor %} -{% endif %} + {% endif %} -{{ log(ns.err_msg, info=True) }} -select 1 as col_name {% if ns.pass %} {{ limit_zero() }} {% endif %} + {{ log(ns.err_msg, info=True) }} + select 1 as col_name {% if ns.pass %} {{ limit_zero() }} {% endif %} {% endif %} diff --git a/integration_tests/tests/jinja_helpers/test_slugify.sql b/integration_tests/tests/jinja_helpers/test_slugify.sql index c0839f59..9b72b797 100644 --- a/integration_tests/tests/jinja_helpers/test_slugify.sql +++ b/integration_tests/tests/jinja_helpers/test_slugify.sql @@ -1,4 +1,4 @@ -{% if dbt_utils.slugify('!Hell0 world-hi') == 'hell0_world_hi' %} +{% if dbt_utils.slugify("!Hell0 world-hi") == "hell0_world_hi" %} {# Return 0 rows for the test to pass #} select 1 as col_name {{ limit_zero() }} {% else %} diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 7f9fa3a4..e7d64be0 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -1,26 +1,40 @@ +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values_dropped"), + "field", + default=["y", "z"], + order_by="field", +) %} -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} +with + expected as ( + select + {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} + as expected_column_value + union all + select + {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} + as expected_column_value + ), -with expected as ( - select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected_column_value union all - select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected_column_value -), + actual as ( -actual as ( - {% for val in column_values %} - select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual_column_value + select + {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} + as actual_column_value {% if not loop.last %} union all {% endif %} {% endfor %} -), + ), -failures as ( - select * from actual - where actual.actual_column_value not in ( - select expected.expected_column_value from expected + failures as ( + select * + from actual + where + actual.actual_column_value + not in (select expected.expected_column_value from expected) ) -) -select * from failures \ No newline at end of file +select * +from failures diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index 34c70db9..1bf39708 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -1,10 +1,9 @@ {% macro _is_ephemeral(obj, macro) %} {%- if obj.is_cte -%} - {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} + {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix("") %} {% if obj.name.startswith(ephemeral_prefix) %} - {% set model_name = obj.name[(ephemeral_prefix|length):] %} - {% else %} - {% set model_name = obj.name %} + {% set model_name = obj.name[(ephemeral_prefix | length) :] %} + {% else %} {% set model_name = obj.name %} {%- endif -%} {% set error_message %} The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. diff --git a/macros/cross_db_utils/_is_relation.sql b/macros/cross_db_utils/_is_relation.sql index bee6db97..d668c229 100644 --- a/macros/cross_db_utils/_is_relation.sql +++ b/macros/cross_db_utils/_is_relation.sql @@ -1,5 +1,13 @@ {% macro _is_relation(obj, macro) %} - {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%} - {%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} + {%- if not ( + obj is mapping + and obj.get("metadata", {}).get("type", "").endswith("Relation") + ) -%} + {%- do exceptions.raise_compiler_error( + "Macro " + ~ macro + ~ " expected a Relation but received the value: " + ~ obj + ) -%} {%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/any_value.sql b/macros/cross_db_utils/any_value.sql index 78cb75ba..595d32d0 100644 --- a/macros/cross_db_utils/any_value.sql +++ b/macros/cross_db_utils/any_value.sql @@ -1,17 +1,13 @@ {% macro any_value(expression) -%} - {{ return(adapter.dispatch('any_value', 'dbt_utils') (expression)) }} + {{ return(adapter.dispatch("any_value", "dbt_utils")(expression)) }} {% endmacro %} -{% macro default__any_value(expression) -%} - - any_value({{ expression }}) - -{%- endmacro %} +{% macro default__any_value(expression) -%} any_value({{ expression }}) {%- endmacro %} {% macro postgres__any_value(expression) -%} {#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} min({{ expression }}) - -{%- endmacro %} \ No newline at end of file + +{%- endmacro %} diff --git a/macros/cross_db_utils/bool_or.sql b/macros/cross_db_utils/bool_or.sql index ce0a6857..04716878 100644 --- a/macros/cross_db_utils/bool_or.sql +++ b/macros/cross_db_utils/bool_or.sql @@ -1,24 +1,12 @@ {% macro bool_or(expression) -%} - {{ return(adapter.dispatch('bool_or', 'dbt_utils') (expression)) }} + {{ return(adapter.dispatch("bool_or", "dbt_utils")(expression)) }} {% endmacro %} -{% macro default__bool_or(expression) -%} - - bool_or({{ expression }}) - -{%- endmacro %} +{% macro default__bool_or(expression) -%} bool_or({{ expression }}) {%- endmacro %} -{% macro snowflake__bool_or(expression) -%} - - boolor_agg({{ expression }}) - -{%- endmacro %} +{% macro snowflake__bool_or(expression) -%} boolor_agg({{ expression }}) {%- endmacro %} -{% macro bigquery__bool_or(expression) -%} - - logical_or({{ expression }}) - -{%- endmacro %} \ No newline at end of file +{% macro bigquery__bool_or(expression) -%} logical_or({{ expression }}) {%- endmacro %} diff --git a/macros/cross_db_utils/cast_bool_to_text.sql b/macros/cross_db_utils/cast_bool_to_text.sql index 5efa9d60..030d5681 100644 --- a/macros/cross_db_utils/cast_bool_to_text.sql +++ b/macros/cross_db_utils/cast_bool_to_text.sql @@ -1,5 +1,5 @@ {% macro cast_bool_to_text(field) %} - {{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }} + {{ adapter.dispatch("cast_bool_to_text", "dbt_utils")(field) }} {% endmacro %} @@ -9,7 +9,6 @@ {% macro redshift__cast_bool_to_text(field) %} case - when {{ field }} is true then 'true' - when {{ field }} is false then 'false' + when {{ field }} is true then 'true' when {{ field }} is false then 'false' end::text {% endmacro %} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 7e97e8f8..e2be3cee 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,9 +1,5 @@ - - {% macro concat(fields) -%} - {{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }} + {{ return(adapter.dispatch("concat", "dbt_utils")(fields)) }} {%- endmacro %} -{% macro default__concat(fields) -%} - {{ fields|join(' || ') }} -{%- endmacro %} +{% macro default__concat(fields) -%} {{ fields | join(" || ") }} {%- endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index 66ad8dc6..aa8e237c 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,35 +1,32 @@ {% macro current_timestamp() -%} - {{ return(adapter.dispatch('current_timestamp', 'dbt_utils')()) }} + {{ return(adapter.dispatch("current_timestamp", "dbt_utils")()) }} {%- endmacro %} {% macro default__current_timestamp() %} - current_timestamp::{{dbt_utils.type_timestamp()}} + current_timestamp::{{ dbt_utils.type_timestamp() }} {% endmacro %} -{% macro redshift__current_timestamp() %} - getdate() -{% endmacro %} - -{% macro bigquery__current_timestamp() %} - current_timestamp -{% endmacro %} +{% macro redshift__current_timestamp() %} getdate() {% endmacro %} +{% macro bigquery__current_timestamp() %} current_timestamp {% endmacro %} {% macro current_timestamp_in_utc() -%} - {{ return(adapter.dispatch('current_timestamp_in_utc', 'dbt_utils')()) }} + {{ return(adapter.dispatch("current_timestamp_in_utc", "dbt_utils")()) }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} - {{dbt_utils.current_timestamp()}} + {{ dbt_utils.current_timestamp() }} {% endmacro %} {% macro snowflake__current_timestamp_in_utc() %} - convert_timezone('UTC', {{dbt_utils.current_timestamp()}})::{{dbt_utils.type_timestamp()}} + convert_timezone( + 'UTC', {{ dbt_utils.current_timestamp() }} + )::{{ dbt_utils.type_timestamp() }} {% endmacro %} {% macro postgres__current_timestamp_in_utc() %} - (current_timestamp at time zone 'utc')::{{dbt_utils.type_timestamp()}} + (current_timestamp at time zone 'utc')::{{ dbt_utils.type_timestamp() }} {% endmacro %} {# redshift should use default instead of postgres #} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index f115b4e2..ba12737c 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,99 +1,62 @@ {# string ------------------------------------------------- #} - {%- macro type_string() -%} - {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_string", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_string() %} - string -{% endmacro %} - -{%- macro redshift__type_string() -%} - varchar -{%- endmacro -%} +{% macro default__type_string() %} string {% endmacro %} -{% macro postgres__type_string() %} - varchar -{% endmacro %} +{%- macro redshift__type_string() -%} varchar {%- endmacro -%} -{% macro snowflake__type_string() %} - varchar -{% endmacro %} +{% macro postgres__type_string() %} varchar {% endmacro %} +{% macro snowflake__type_string() %} varchar {% endmacro %} {# timestamp ------------------------------------------------- #} - {%- macro type_timestamp() -%} - {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_timestamp", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_timestamp() %} - timestamp -{% endmacro %} +{% macro default__type_timestamp() %} timestamp {% endmacro %} -{% macro postgres__type_timestamp() %} - timestamp without time zone -{% endmacro %} +{% macro postgres__type_timestamp() %} timestamp without time zone {% endmacro %} -{% macro snowflake__type_timestamp() %} - timestamp_ntz -{% endmacro %} +{% macro snowflake__type_timestamp() %} timestamp_ntz {% endmacro %} {# float ------------------------------------------------- #} - {%- macro type_float() -%} - {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_float", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_float() %} - float -{% endmacro %} +{% macro default__type_float() %} float {% endmacro %} -{% macro bigquery__type_float() %} - float64 -{% endmacro %} +{% macro bigquery__type_float() %} float64 {% endmacro %} {# numeric ------------------------------------------------ #} - {%- macro type_numeric() -%} - {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_numeric", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_numeric() %} - numeric(28, 6) -{% endmacro %} +{% macro default__type_numeric() %} numeric(28, 6) {% endmacro %} -{% macro bigquery__type_numeric() %} - numeric -{% endmacro %} +{% macro bigquery__type_numeric() %} numeric {% endmacro %} {# bigint ------------------------------------------------- #} - {%- macro type_bigint() -%} - {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_bigint", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_bigint() %} - bigint -{% endmacro %} +{% macro default__type_bigint() %} bigint {% endmacro %} -{% macro bigquery__type_bigint() %} - int64 -{% endmacro %} +{% macro bigquery__type_bigint() %} int64 {% endmacro %} {# int ------------------------------------------------- #} - {%- macro type_int() -%} - {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_int", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_int() %} - int -{% endmacro %} +{% macro default__type_int() %} int {% endmacro %} -{% macro bigquery__type_int() %} - int64 -{% endmacro %} +{% macro bigquery__type_int() %} int64 {% endmacro %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index f9d0364b..e173f6ad 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,15 +1,12 @@ {% macro date_trunc(datepart, date) -%} - {{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }} + {{ return(adapter.dispatch("date_trunc", "dbt_utils")(datepart, date)) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) -%} - date_trunc('{{datepart}}', {{date}}) + date_trunc('{{datepart}}', {{ date }}) {%- endmacro %} {% macro bigquery__date_trunc(datepart, date) -%} - timestamp_trunc( - cast({{date}} as timestamp), - {{datepart}} - ) + timestamp_trunc(cast({{ date }} as timestamp), {{ datepart }}) {%- endmacro %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 09c0f115..fd8016cc 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,25 +1,27 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }} + {{ + return( + adapter.dispatch("dateadd", "dbt_utils")( + datepart, interval, from_date_or_timestamp + ) + ) + }} {% endmacro %} {% macro default__dateadd(datepart, interval, from_date_or_timestamp) %} - dateadd( - {{ datepart }}, - {{ interval }}, - {{ from_date_or_timestamp }} - ) + dateadd({{ datepart }}, {{ interval }}, {{ from_date_or_timestamp }}) {% endmacro %} {% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %} - datetime_add( - cast( {{ from_date_or_timestamp }} as datetime), + datetime_add( + cast({{ from_date_or_timestamp }} as datetime), interval {{ interval }} {{ datepart }} - ) + ) {% endmacro %} @@ -32,6 +34,10 @@ {# redshift should use default instead of postgres #} {% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }} + {{ + return( + dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp) + ) + }} {% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 2b5d6613..3804ff88 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,15 +1,17 @@ {% macro datediff(first_date, second_date, datepart) %} - {{ return(adapter.dispatch('datediff', 'dbt_utils')(first_date, second_date, datepart)) }} + {{ + return( + adapter.dispatch("datediff", "dbt_utils")( + first_date, second_date, datepart + ) + ) + }} {% endmacro %} {% macro default__datediff(first_date, second_date, datepart) -%} - datediff( - {{ datepart }}, - {{ first_date }}, - {{ second_date }} - ) + datediff({{ datepart }}, {{ first_date }}, {{ second_date }}) {%- endmacro %} @@ -17,42 +19,81 @@ {% macro bigquery__datediff(first_date, second_date, datepart) -%} datetime_diff( - cast({{second_date}} as datetime), - cast({{first_date}} as datetime), - {{datepart}} + cast({{ second_date }} as datetime), + cast({{ first_date }} as datetime), + {{ datepart }} ) {%- endmacro %} {% macro postgres__datediff(first_date, second_date, datepart) -%} - {% if datepart == 'year' %} - (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) - {% elif datepart == 'quarter' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) - {% elif datepart == 'month' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) - {% elif datepart == 'day' %} - (({{second_date}})::date - ({{first_date}})::date) - {% elif datepart == 'week' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case - when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then - case when {{first_date}} <= {{second_date}} then 0 else -1 end - else - case when {{first_date}} <= {{second_date}} then 1 else 0 end - end) - {% elif datepart == 'hour' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) - {% elif datepart == 'minute' %} - ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) - {% elif datepart == 'second' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) - {% elif datepart == 'millisecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) - {% elif datepart == 'microsecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) + {% if datepart == "year" %} + ( + date_part('year', ({{ second_date }})::date) + - date_part('year', ({{ first_date }})::date) + ) + {% elif datepart == "quarter" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + + date_part('quarter', ({{ second_date }})::date) + - date_part('quarter', ({{ first_date }})::date) + ) + {% elif datepart == "month" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + + date_part('month', ({{ second_date }})::date) + - date_part('month', ({{ first_date }})::date) + ) + {% elif datepart == "day" %}(({{ second_date }})::date - ({{ first_date }})::date) + {% elif datepart == "week" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case + when + date_part('dow', ({{ first_date }})::timestamp) + <= date_part('dow', ({{ second_date }})::timestamp) + then case when {{ first_date }} <= {{ second_date }} then 0 else -1 end + else case when {{ first_date }} <= {{ second_date }} then 1 else 0 end + end + ) + {% elif datepart == "hour" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + + date_part('hour', ({{ second_date }})::timestamp) + - date_part('hour', ({{ first_date }})::timestamp) + ) + {% elif datepart == "minute" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + + date_part('minute', ({{ second_date }})::timestamp) + - date_part('minute', ({{ first_date }})::timestamp) + ) + {% elif datepart == "second" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + + floor(date_part('second', ({{ second_date }})::timestamp)) + - floor(date_part('second', ({{ first_date }})::timestamp)) + ) + {% elif datepart == "millisecond" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + + floor(date_part('millisecond', ({{ second_date }})::timestamp)) + - floor(date_part('millisecond', ({{ first_date }})::timestamp)) + ) + {% elif datepart == "microsecond" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + + floor(date_part('microsecond', ({{ second_date }})::timestamp)) + - floor(date_part('microsecond', ({{ first_date }})::timestamp)) + ) {% else %} - {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} + {{ + exceptions.raise_compiler_error( + "Unsupported datepart for macro datediff in postgres: {!r}".format( + datepart + ) + ) + }} {% endif %} {%- endmacro %} diff --git a/macros/cross_db_utils/escape_single_quotes.sql b/macros/cross_db_utils/escape_single_quotes.sql index d024f16f..0a3c95a2 100644 --- a/macros/cross_db_utils/escape_single_quotes.sql +++ b/macros/cross_db_utils/escape_single_quotes.sql @@ -1,18 +1,18 @@ {% macro escape_single_quotes(expression) %} - {{ return(adapter.dispatch('escape_single_quotes', 'dbt_utils') (expression)) }} + {{ return(adapter.dispatch("escape_single_quotes", "dbt_utils")(expression)) }} {% endmacro %} {# /*Default to replacing a single apostrophe with two apostrophes: they're -> they''re*/ #} {% macro default__escape_single_quotes(expression) -%} -{{ expression | replace("'","''") }} + {{ expression | replace("'", "''") }} {%- endmacro %} {# /*Snowflake uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} {% macro snowflake__escape_single_quotes(expression) -%} -{{ expression | replace("'", "\\'") }} + {{ expression | replace("'", "\\'") }} {%- endmacro %} {# /*BigQuery uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} {% macro bigquery__escape_single_quotes(expression) -%} -{{ expression | replace("'", "\\'") }} + {{ expression | replace("'", "\\'") }} {%- endmacro %} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 9bb75b86..888e1940 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,5 +1,5 @@ {% macro except() %} - {{ return(adapter.dispatch('except', 'dbt_utils')()) }} + {{ return(adapter.dispatch("except", "dbt_utils")()) }} {% endmacro %} @@ -8,9 +8,9 @@ except {% endmacro %} - + {% macro bigquery__except() %} except distinct -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index d086c2e6..9674fefc 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,13 +1,13 @@ {% macro hash(field) -%} - {{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }} + {{ return(adapter.dispatch("hash", "dbt_utils")(field)) }} {%- endmacro %} {% macro default__hash(field) -%} - md5(cast({{field}} as {{dbt_utils.type_string()}})) + md5(cast({{ field }} as {{ dbt_utils.type_string() }})) {%- endmacro %} {% macro bigquery__hash(field) -%} - to_hex({{dbt_utils.default__hash(field)}}) + to_hex({{ dbt_utils.default__hash(field) }}) {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index 619cb7cb..a4c1a73a 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -1,16 +1,11 @@ -{% macro identifier(value) %} - {%- set error_message = ' - Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ - Use `adapter.quote` instead. The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - {{ return(adapter.dispatch('identifier', 'dbt_utils') (value)) }} -{% endmacro %} +{% macro identifier(value) %} + {%- set error_message = " Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name + ) -%} + {%- do exceptions.warn(error_message) -%} + {{ return(adapter.dispatch("identifier", "dbt_utils")(value)) }} +{% endmacro %} -{% macro default__identifier(value) -%} - "{{ value }}" -{%- endmacro %} +{% macro default__identifier(value) -%} "{{ value }}" {%- endmacro %} -{% macro bigquery__identifier(value) -%} - `{{ value }}` -{%- endmacro %} +{% macro bigquery__identifier(value) -%} `{{ value }}` {%- endmacro %} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index b53c72f3..08abf793 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,5 +1,5 @@ {% macro intersect() %} - {{ return(adapter.dispatch('intersect', 'dbt_utils')()) }} + {{ return(adapter.dispatch("intersect", "dbt_utils")()) }} {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 18ca161c..391cd4b5 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -2,37 +2,47 @@ This function has been tested with dateparts of month and quarters. Further testing is required to validate that it will work on other dateparts. */ - {% macro last_day(date, datepart) %} - {{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }} + {{ return(adapter.dispatch("last_day", "dbt_utils")(date, datepart)) }} {% endmacro %} {%- macro default_last_day(date, datepart) -%} cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd(datepart, '1', dbt_utils.date_trunc(datepart, date)) - )}} - as date) + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd( + datepart, "1", dbt_utils.date_trunc(datepart, date) + ), + ) + }} as date + ) {%- endmacro -%} {% macro default__last_day(date, datepart) -%} - {{dbt_utils.default_last_day(date, datepart)}} + {{ dbt_utils.default_last_day(date, datepart) }} {%- endmacro %} {% macro postgres__last_day(date, datepart) -%} - {%- if datepart == 'quarter' -%} - -- postgres dateadd does not support quarter interval. - cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd('month', '3', dbt_utils.date_trunc(datepart, date)) - )}} - as date) - {%- else -%} - {{dbt_utils.default_last_day(date, datepart)}} + {%- if datepart == "quarter" -%} + -- postgres dateadd does not support quarter interval. + cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd( + "month", "3", dbt_utils.date_trunc(datepart, date) + ), + ) + }} as date + ) + {%- else -%} {{ dbt_utils.default_last_day(date, datepart) }} {%- endif -%} {%- endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index 8c2e265c..2e736fbd 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,21 +1,9 @@ {% macro length(expression) -%} - {{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }} + {{ return(adapter.dispatch("length", "dbt_utils")(expression)) }} {% endmacro %} -{% macro default__length(expression) %} - - length( - {{ expression }} - ) - -{%- endmacro -%} +{% macro default__length(expression) %} length({{ expression }}) {%- endmacro -%} -{% macro redshift__length(expression) %} - - len( - {{ expression }} - ) - -{%- endmacro -%} \ No newline at end of file +{% macro redshift__length(expression) %} len({{ expression }}) {%- endmacro -%} diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 1d19a54f..18e58df9 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -1,29 +1,32 @@ -{% macro listagg(measure, delimiter_text="','", order_by_clause=none, limit_num=none) -%} - {{ return(adapter.dispatch('listagg', 'dbt_utils') (measure, delimiter_text, order_by_clause, limit_num)) }} +{% macro listagg( + measure, delimiter_text="','", order_by_clause=none, limit_num=none +) -%} + {{ + return( + adapter.dispatch("listagg", "dbt_utils")( + measure, delimiter_text, order_by_clause, limit_num + ) + ) + }} {%- endmacro %} {% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} {% if limit_num -%} - array_to_string( - array_slice( - array_agg( - {{ measure }} - ){% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,0 - ,{{ limit_num }} - ), - {{ delimiter_text }} + array_to_string( + array_slice( + array_agg({{ measure }}) + {% if order_by_clause -%} + within group ({{ order_by_clause }}) + {%- endif %}, + 0, + {{ limit_num }} + ), + {{ delimiter_text }} ) {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %} {%- endif %} @@ -34,35 +37,30 @@ string_agg( {{ measure }}, {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - {% if limit_num -%} - limit {{ limit_num }} - {%- endif %} - ) + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + {% if limit_num -%} limit {{ limit_num }} {%- endif %} + ) {%- endmacro %} {% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - + {% if limit_num -%} - array_to_string( - (array_agg( - {{ measure }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - ))[1:{{ limit_num }}], - {{ delimiter_text }} + array_to_string( + ( + array_agg( + {{ measure }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + ) + )[1:{{ limit_num }}], + {{ delimiter_text }} ) {%- else %} - string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} + string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} + {%- endif %} ) {%- endif %} @@ -72,33 +70,26 @@ {% macro redshift__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} {% if limit_num -%} - {% set ns = namespace() %} - {% set ns.delimiter_text_regex = delimiter_text|trim("'") %} - {% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} - {%- for char in special_chars.split(',') -%} - {% set escape_char %}\\{{ char }}{% endset %} - {% set ns.delimiter_text_regex = ns.delimiter_text_regex|replace(char,escape_char) %} - {%- endfor -%} + {% set ns = namespace() %} + {% set ns.delimiter_text_regex = delimiter_text | trim("'") %} + {% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} + {%- for char in special_chars.split(",") -%} + {% set escape_char %}\\{{ char }}{% endset %} + {% set ns.delimiter_text_regex = ns.delimiter_text_regex | replace( + char, escape_char + ) %} + {%- endfor -%} - {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} - regexp_substr( - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,{{ regex }} + {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} + regexp_substr( + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %}, + {{ regex }} ) {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %} {%- endif %} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index c2291467..26858c33 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,8 +1,5 @@ - {%- macro string_literal(value) -%} - {{ return(adapter.dispatch('string_literal', 'dbt_utils') (value)) }} + {{ return(adapter.dispatch("string_literal", "dbt_utils")(value)) }} {%- endmacro -%} -{% macro default__string_literal(value) -%} - '{{ value }}' -{%- endmacro %} +{% macro default__string_literal(value) -%} '{{ value }}' {%- endmacro %} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index a67b7aa2..140d2680 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,22 +1,20 @@ {% macro position(substring_text, string_text) -%} - {{ return(adapter.dispatch('position', 'dbt_utils') (substring_text, string_text)) }} + {{ + return( + adapter.dispatch("position", "dbt_utils")(substring_text, string_text) + ) + }} {% endmacro %} {% macro default__position(substring_text, string_text) %} - position( - {{ substring_text }} in {{ string_text }} - ) - + position({{ substring_text }} in {{ string_text }}) + {%- endmacro -%} {% macro bigquery__position(substring_text, string_text) %} - strpos( - {{ string_text }}, - {{ substring_text }} - - ) - + strpos({{ string_text }}, {{ substring_text }}) + {%- endmacro -%} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index ef27d8fc..66b859e7 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,15 +1,10 @@ {% macro replace(field, old_chars, new_chars) -%} - {{ return(adapter.dispatch('replace', 'dbt_utils') (field, old_chars, new_chars)) }} + {{ return(adapter.dispatch("replace", "dbt_utils")(field, old_chars, new_chars)) }} {% endmacro %} {% macro default__replace(field, old_chars, new_chars) %} - replace( - {{ field }}, - {{ old_chars }}, - {{ new_chars }} - ) - + replace({{ field }}, {{ old_chars }}, {{ new_chars }}) -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 8ae1640f..841dc4f6 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,38 +1,33 @@ {% macro right(string_text, length_expression) -%} - {{ return(adapter.dispatch('right', 'dbt_utils') (string_text, length_expression)) }} + {{ + return( + adapter.dispatch("right", "dbt_utils")(string_text, length_expression) + ) + }} {% endmacro %} {% macro default__right(string_text, length_expression) %} - right( - {{ string_text }}, - {{ length_expression }} - ) - + right({{ string_text }}, {{ length_expression }}) + {%- endmacro -%} {% macro bigquery__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 + case + when {{ length_expression }} = 0 then '' - else - substr( - {{ string_text }}, - -1 * ({{ length_expression }}) - ) + else substr({{ string_text }}, -1 * ({{ length_expression }})) end {%- endmacro -%} {% macro snowflake__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 + case + when {{ length_expression }} = 0 then '' - else - right( - {{ string_text }}, - {{ length_expression }} - ) + else right({{ string_text }}, {{ length_expression }}) end -{%- endmacro -%} \ No newline at end of file +{%- endmacro -%} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index 8569eecd..3337beb2 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,20 +1,20 @@ {% macro safe_cast(field, type) %} - {{ return(adapter.dispatch('safe_cast', 'dbt_utils') (field, type)) }} + {{ return(adapter.dispatch("safe_cast", "dbt_utils")(field, type)) }} {% endmacro %} {% macro default__safe_cast(field, type) %} {# most databases don't support this function yet so we just need to use cast #} - cast({{field}} as {{type}}) + cast({{ field }} as {{ type }}) {% endmacro %} {% macro snowflake__safe_cast(field, type) %} - try_cast({{field}} as {{type}}) + try_cast({{ field }} as {{ type }}) {% endmacro %} {% macro bigquery__safe_cast(field, type) %} - safe_cast({{field}} as {{type}}) + safe_cast({{ field }} as {{ type }}) {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index 036f7d5a..5f10da88 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,24 +1,23 @@ {% macro split_part(string_text, delimiter_text, part_number) %} - {{ return(adapter.dispatch('split_part', 'dbt_utils') (string_text, delimiter_text, part_number)) }} + {{ + return( + adapter.dispatch("split_part", "dbt_utils")( + string_text, delimiter_text, part_number + ) + ) + }} {% endmacro %} {% macro default__split_part(string_text, delimiter_text, part_number) %} - split_part( - {{ string_text }}, - {{ delimiter_text }}, - {{ part_number }} - ) + split_part({{ string_text }}, {{ delimiter_text }}, {{ part_number }}) {% endmacro %} {% macro bigquery__split_part(string_text, delimiter_text, part_number) %} - split( - {{ string_text }}, - {{ delimiter_text }} - )[safe_offset({{ part_number - 1 }})] + split({{ string_text }}, {{ delimiter_text }})[safe_offset({{ part_number - 1 }})] {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index fc8ae81c..452993de 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,5 +1,11 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} - {{ return(adapter.dispatch('width_bucket', 'dbt_utils') (expr, min_value, max_value, num_buckets)) }} + {{ + return( + adapter.dispatch("width_bucket", "dbt_utils")( + expr, min_value, max_value, num_buckets + ) + ) + }} {% endmacro %} @@ -13,20 +19,16 @@ case when mod( - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }}, - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - ) = 0 + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }}, + {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + ) + = 0 then 1 else 0 end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 ) + -- Anything over max_value goes the N+1 bucket + + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} @@ -38,22 +40,17 @@ -- to break ties when the amount is exactly at the bucket edge case when - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }} % - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - = 0 + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }} + % {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + = 0 then 1 else 0 end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 ) + -- Anything over max_value goes the N+1 bucket + + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} - width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }} ) + width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }}) {% endmacro %} diff --git a/macros/generic_tests/accepted_range.sql b/macros/generic_tests/accepted_range.sql index 49ab00d9..aee6e7fb 100644 --- a/macros/generic_tests/accepted_range.sql +++ b/macros/generic_tests/accepted_range.sql @@ -1,33 +1,44 @@ -{% test accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} - {{ return(adapter.dispatch('test_accepted_range', 'dbt_utils')(model, column_name, min_value, max_value, inclusive)) }} +{% test accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} + {{ + return( + adapter.dispatch("test_accepted_range", "dbt_utils")( + model, column_name, min_value, max_value, inclusive + ) + ) + }} {% endtest %} -{% macro default__test_accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} +{% macro default__test_accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} -with meet_condition as( - select * - from {{ model }} -), + with + meet_condition as (select * from {{ model }}), -validation_errors as ( - select * - from meet_condition - where - -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds - 1 = 2 + validation_errors as ( + select * + from meet_condition + where + -- never true, defaults to an empty result set. Exists to ensure any + -- combo of the `or` clauses below succeeds + 1 = 2 - {%- if min_value is not none %} - -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} - {%- endif %} + {%- if min_value is not none %} + -- records with a value >= min_value are permitted. The `not` + -- flips this to find records that don't meet the rule. + or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} + {%- endif %} - {%- if max_value is not none %} - -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} - {%- endif %} -) + {%- if max_value is not none %} + -- records with a value <= max_value are permitted. The `not` + -- flips this to find records that don't meet the rule. + or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} + {%- endif %} + ) -select * -from validation_errors + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/at_least_one.sql b/macros/generic_tests/at_least_one.sql index ce02108f..c13981af 100644 --- a/macros/generic_tests/at_least_one.sql +++ b/macros/generic_tests/at_least_one.sql @@ -1,20 +1,25 @@ {% test at_least_one(model, column_name) %} - {{ return(adapter.dispatch('test_at_least_one', 'dbt_utils')(model, column_name)) }} + {{ + return( + adapter.dispatch("test_at_least_one", "dbt_utils")(model, column_name) + ) + }} {% endtest %} {% macro default__test_at_least_one(model, column_name) %} -select * -from ( - select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count({{ column_name }}) as filler_column + select * + from + ( + select + {# In TSQL, subquery aggregate columns need aliases #} + {# thus: a filler col name, 'filler_column' #} + count({{ column_name }}) as filler_column - from {{ model }} + from {{ model }} - having count({{ column_name }}) = 0 + having count({{ column_name }}) = 0 -) validation_errors + ) validation_errors {% endmacro %} diff --git a/macros/generic_tests/cardinality_equality.sql b/macros/generic_tests/cardinality_equality.sql index 4ec7d87f..8ef1a901 100644 --- a/macros/generic_tests/cardinality_equality.sql +++ b/macros/generic_tests/cardinality_equality.sql @@ -1,53 +1,45 @@ {% test cardinality_equality(model, column_name, to, field) %} - {{ return(adapter.dispatch('test_cardinality_equality', 'dbt_utils')(model, column_name, to, field)) }} + {{ + return( + adapter.dispatch("test_cardinality_equality", "dbt_utils")( + model, column_name, to, field + ) + ) + }} {% endtest %} {% macro default__test_cardinality_equality(model, column_name, to, field) %} -{# T-SQL does not let you use numbers as aliases for columns #} -{# Thus, no "GROUP BY 1" #} - -with table_a as ( -select - {{ column_name }}, - count(*) as num_rows -from {{ model }} -group by {{ column_name }} -), - -table_b as ( -select - {{ field }}, - count(*) as num_rows -from {{ to }} -group by {{ field }} -), - -except_a as ( - select * - from table_a - {{ dbt_utils.except() }} - select * - from table_b -), - -except_b as ( - select * - from table_b - {{ dbt_utils.except() }} - select * - from table_a -), - -unioned as ( - select * - from except_a - union all - select * - from except_b -) - -select * -from unioned + {# T-SQL does not let you use numbers as aliases for columns #} + {# Thus, no "GROUP BY 1" #} + with + table_a as ( + select {{ column_name }}, count(*) as num_rows + from {{ model }} + group by {{ column_name }} + ), + + table_b as ( + select {{ field }}, count(*) as num_rows from {{ to }} group by {{ field }} + ), + + except_a as ( + select * from table_a {{ dbt_utils.except() }} select * from table_b + ), + + except_b as ( + select * from table_b {{ dbt_utils.except() }} select * from table_a + ), + + unioned as ( + select * + from except_a + union all + select * + from except_b + ) + + select * + from unioned {% endmacro %} diff --git a/macros/generic_tests/equal_rowcount.sql b/macros/generic_tests/equal_rowcount.sql index 713cb12a..d8404a6f 100644 --- a/macros/generic_tests/equal_rowcount.sql +++ b/macros/generic_tests/equal_rowcount.sql @@ -1,38 +1,31 @@ {% test equal_rowcount(model, compare_model) %} - {{ return(adapter.dispatch('test_equal_rowcount', 'dbt_utils')(model, compare_model)) }} + {{ + return( + adapter.dispatch("test_equal_rowcount", "dbt_utils")(model, compare_model) + ) + }} {% endtest %} {% macro default__test_equal_rowcount(model, compare_model) %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = 'coalesce(diff_count, 0)') }} + {#- - Needs to be set at parse time, before we return '' below --#} + {{ config(fail_calc="coalesce(diff_count, 0)") }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("") }} {% endif %} -with a as ( + with + a as (select count(*) as count_a from {{ model }}), + b as (select count(*) as count_b from {{ compare_model }}), + final as ( - select count(*) as count_a from {{ model }} + select count_a, count_b, abs(count_a - count_b) as diff_count + from a + cross join b -), -b as ( + ) - select count(*) as count_b from {{ compare_model }} - -), -final as ( - - select - count_a, - count_b, - abs(count_a - count_b) as diff_count - from a - cross join b - -) - -select * from final + select * + from final {% endmacro %} diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index bc302c18..28135583 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -1,75 +1,78 @@ {% test equality(model, compare_model, compare_columns=None) %} - {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns)) }} + {{ + return( + adapter.dispatch("test_equality", "dbt_utils")( + model, compare_model, compare_columns + ) + ) + }} {% endtest %} {% macro default__test_equality(model, compare_model, compare_columns=None) %} -{% set set_diff %} + {% set set_diff %} count(*) + coalesce(abs( sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - sum(case when which_diff = 'b_minus_a' then 1 else 0 end) ), 0) -{% endset %} + {% endset %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = set_diff) }} + {#- - Needs to be set at parse time, before we return '' below --#} + {{ config(fail_calc=set_diff) }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("") }} {% endif %} --- setup -{%- do dbt_utils._is_relation(model, 'test_equality') -%} + -- setup + {%- do dbt_utils._is_relation(model, "test_equality") -%} -{#- + {#- If the compare_cols arg is provided, we can run this test without querying the information schema — this allows the model to be an ephemeral model -#} + {%- if not compare_columns -%} + {%- do dbt_utils._is_ephemeral(model, "test_equality") -%} + {%- set compare_columns = adapter.get_columns_in_relation(model) | map( + attribute="quoted" + ) -%} + {%- endif -%} -{%- if not compare_columns -%} - {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} -{%- endif -%} + {% set compare_cols_csv = compare_columns | join(", ") %} -{% set compare_cols_csv = compare_columns | join(', ') %} + with + a as (select * from {{ model }}), -with a as ( + b as (select * from {{ compare_model }}), - select * from {{ model }} + a_minus_b as ( -), + select {{ compare_cols_csv }} + from a {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from b -b as ( + ), - select * from {{ compare_model }} + b_minus_a as ( -), + select {{ compare_cols_csv }} + from b {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from a -a_minus_b as ( + ), - select {{compare_cols_csv}} from a - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from b + unioned as ( -), + select 'a_minus_b' as which_diff, a_minus_b.* + from a_minus_b + union all + select 'b_minus_a' as which_diff, b_minus_a.* + from b_minus_a -b_minus_a as ( + ) - select {{compare_cols_csv}} from b - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from a - -), - -unioned as ( - - select 'a_minus_b' as which_diff, a_minus_b.* from a_minus_b - union all - select 'b_minus_a' as which_diff, b_minus_a.* from b_minus_a - -) - -select * from unioned + select * + from unioned {% endmacro %} diff --git a/macros/generic_tests/expression_is_true.sql b/macros/generic_tests/expression_is_true.sql index 611685b4..a67e818c 100644 --- a/macros/generic_tests/expression_is_true.sql +++ b/macros/generic_tests/expression_is_true.sql @@ -1,22 +1,23 @@ -{% test expression_is_true(model, expression, column_name=None, condition='1=1') %} -{# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} -{# ref https://stackoverflow.com/a/7170753/3842610 #} - {{ return(adapter.dispatch('test_expression_is_true', 'dbt_utils')(model, expression, column_name, condition)) }} +{% test expression_is_true(model, expression, column_name=None, condition="1=1") %} + {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} + {# ref https://stackoverflow.com/a/7170753/3842610 #} + {{ + return( + adapter.dispatch("test_expression_is_true", "dbt_utils")( + model, expression, column_name, condition + ) + ) + }} {% endtest %} {% macro default__test_expression_is_true(model, expression, column_name, condition) %} -with meet_condition as ( - select * from {{ model }} where {{ condition }} -) + with meet_condition as (select * from {{ model }} where {{ condition }}) -select - * -from meet_condition -{% if column_name is none %} -where not({{ expression }}) -{%- else %} -where not({{ column_name }} {{ expression }}) -{%- endif %} + select * + from meet_condition + {% if column_name is none %} where not ({{ expression }}) + {%- else %} where not ({{ column_name }} {{ expression }}) + {%- endif %} {% endmacro %} diff --git a/macros/generic_tests/fewer_rows_than.sql b/macros/generic_tests/fewer_rows_than.sql index 450148b7..e083906f 100644 --- a/macros/generic_tests/fewer_rows_than.sql +++ b/macros/generic_tests/fewer_rows_than.sql @@ -1,45 +1,41 @@ {% test fewer_rows_than(model, compare_model) %} - {{ return(adapter.dispatch('test_fewer_rows_than', 'dbt_utils')(model, compare_model)) }} + {{ + return( + adapter.dispatch("test_fewer_rows_than", "dbt_utils")( + model, compare_model + ) + ) + }} {% endtest %} {% macro default__test_fewer_rows_than(model, compare_model) %} -{{ config(fail_calc = 'coalesce(row_count_delta, 0)') }} - -with a as ( - - select count(*) as count_our_model from {{ model }} - -), -b as ( - - select count(*) as count_comparison_model from {{ compare_model }} - -), -counts as ( - - select - count_our_model, - count_comparison_model - from a - cross join b - -), -final as ( - - select *, - case - -- fail the test if we have more rows than the reference model and return the row count delta - when count_our_model > count_comparison_model then (count_our_model - count_comparison_model) - -- fail the test if they are the same number - when count_our_model = count_comparison_model then 1 - -- pass the test if the delta is positive (i.e. return the number 0) - else 0 - end as row_count_delta - from counts - -) - -select * from final + {{ config(fail_calc="coalesce(row_count_delta, 0)") }} + + with + a as (select count(*) as count_our_model from {{ model }}), + b as (select count(*) as count_comparison_model from {{ compare_model }}), + counts as (select count_our_model, count_comparison_model from a cross join b), + final as ( + + select + *, + case + -- fail the test if we have more rows than the reference model and + -- return the row count delta + when count_our_model > count_comparison_model + then (count_our_model - count_comparison_model) + -- fail the test if they are the same number + when count_our_model = count_comparison_model + then 1 + -- pass the test if the delta is positive (i.e. return the number 0) + else 0 + end as row_count_delta + from counts + + ) + + select * + from final {% endmacro %} diff --git a/macros/generic_tests/mutually_exclusive_ranges.sql b/macros/generic_tests/mutually_exclusive_ranges.sql index f88efbe4..3826f74a 100644 --- a/macros/generic_tests/mutually_exclusive_ranges.sql +++ b/macros/generic_tests/mutually_exclusive_ranges.sql @@ -1,97 +1,132 @@ -{% test mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} - {{ return(adapter.dispatch('test_mutually_exclusive_ranges', 'dbt_utils')(model, lower_bound_column, upper_bound_column, partition_by, gaps, zero_length_range_allowed)) }} +{% test mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False +) %} + {{ + return( + adapter.dispatch("test_mutually_exclusive_ranges", "dbt_utils")( + model, + lower_bound_column, + upper_bound_column, + partition_by, + gaps, + zero_length_range_allowed, + ) + ) + }} {% endtest %} -{% macro default__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} -{% if gaps == 'not_allowed' %} - {% set allow_gaps_operator='=' %} - {% set allow_gaps_operator_in_words='equal_to' %} -{% elif gaps == 'allowed' %} - {% set allow_gaps_operator='<=' %} - {% set allow_gaps_operator_in_words='less_than_or_equal_to' %} -{% elif gaps == 'required' %} - {% set allow_gaps_operator='<' %} - {% set allow_gaps_operator_in_words='less_than' %} -{% else %} - {{ exceptions.raise_compiler_error( - "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" - ) }} -{% endif %} -{% if not zero_length_range_allowed %} - {% set allow_zero_length_operator='<' %} - {% set allow_zero_length_operator_in_words='less_than' %} -{% elif zero_length_range_allowed %} - {% set allow_zero_length_operator='<=' %} - {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %} -{% else %} - {{ exceptions.raise_compiler_error( - "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length_range_allowed ~"'.'" - ) }} -{% endif %} - -{% set partition_clause="partition by " ~ partition_by if partition_by else '' %} - -with window_functions as ( - - select - {% if partition_by %} - {{ partition_by }} as partition_by_col, - {% endif %} - {{ lower_bound_column }} as lower_bound, - {{ upper_bound_column }} as upper_bound, - - lead({{ lower_bound_column }}) over ( - {{ partition_clause }} - order by {{ lower_bound_column }} - ) as next_lower_bound, - - row_number() over ( - {{ partition_clause }} - order by {{ lower_bound_column }} desc - ) = 1 as is_last_record - - from {{ model }} - -), - -calc as ( - -- We want to return records where one of our assumptions fails, so we'll use - -- the `not` function with `and` statements so we can write our assumptions nore cleanly - select - *, - - -- For each record: lower_bound should be < upper_bound. - -- Coalesce it to return an error on the null case (implicit assumption - -- these columns are not_null) - coalesce( - lower_bound {{ allow_zero_length_operator }} upper_bound, - false - ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, - - -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. - -- Coalesce it to handle null cases for the last record. - coalesce( - upper_bound {{ allow_gaps_operator }} next_lower_bound, - is_last_record, - false - ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound - - from window_functions - -), - -validation_errors as ( - - select - * - from calc - - where not( - -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound - and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound - ) -) - -select * from validation_errors +{% macro default__test_mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False +) %} + {% if gaps == "not_allowed" %} + {% set allow_gaps_operator = "=" %} + {% set allow_gaps_operator_in_words = "equal_to" %} + {% elif gaps == "allowed" %} + {% set allow_gaps_operator = "<=" %} + {% set allow_gaps_operator_in_words = "less_than_or_equal_to" %} + {% elif gaps == "required" %} + {% set allow_gaps_operator = "<" %} + {% set allow_gaps_operator_in_words = "less_than" %} + {% else %} + {{ + exceptions.raise_compiler_error( + "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" + ~ gaps + ~ "'.'" + ) + }} + {% endif %} + {% if not zero_length_range_allowed %} + {% set allow_zero_length_operator = "<" %} + {% set allow_zero_length_operator_in_words = "less_than" %} + {% elif zero_length_range_allowed %} + {% set allow_zero_length_operator = "<=" %} + {% set allow_zero_length_operator_in_words = "less_than_or_equal_to" %} + {% else %} + {{ + exceptions.raise_compiler_error( + "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" + ~ zero_length_range_allowed + ~ "'.'" + ) + }} + {% endif %} + + {% set partition_clause = "partition by " ~ partition_by if partition_by else "" %} + + with + window_functions as ( + + select + {% if partition_by %} + {{ partition_by }} as partition_by_col, + {% endif %} + {{ lower_bound_column }} as lower_bound, + {{ upper_bound_column }} as upper_bound, + + lead({{ lower_bound_column }}) over ( + {{ partition_clause }} order by {{ lower_bound_column }} + ) as next_lower_bound, + + row_number() over ( + {{ partition_clause }} order by {{ lower_bound_column }} desc + ) + = 1 as is_last_record + + from {{ model }} + + ), + + calc as ( + -- We want to return records where one of our assumptions fails, so we'll + -- use + -- the `not` function with `and` statements so we can write our + -- assumptions nore cleanly + select + *, + + -- For each record: lower_bound should be < upper_bound. + -- Coalesce it to return an error on the null case (implicit assumption + -- these columns are not_null) + coalesce( + lower_bound {{ allow_zero_length_operator }} upper_bound, false + ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, + + -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. + -- Coalesce it to handle null cases for the last record. + coalesce( + upper_bound {{ allow_gaps_operator }} next_lower_bound, + is_last_record, + false + ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + + from window_functions + + ), + + validation_errors as ( + + select * + from calc + + where + not ( + -- THE FOLLOWING SHOULD BE TRUE -- + lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound + and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + ) + ) + + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/not_accepted_values.sql b/macros/generic_tests/not_accepted_values.sql index ab24188f..688efb5c 100644 --- a/macros/generic_tests/not_accepted_values.sql +++ b/macros/generic_tests/not_accepted_values.sql @@ -1,37 +1,39 @@ {% test not_accepted_values(model, column_name, values, quote=True) %} - {{ return(adapter.dispatch('test_not_accepted_values', 'dbt_utils')(model, column_name, values, quote)) }} + {{ + return( + adapter.dispatch("test_not_accepted_values", "dbt_utils")( + model, column_name, values, quote + ) + ) + }} {% endtest %} {% macro default__test_not_accepted_values(model, column_name, values, quote=True) %} -with all_values as ( + with + all_values as ( - select distinct - {{ column_name }} as value_field + select distinct {{ column_name }} as value_field from {{ model }} - from {{ model }} + ), -), + validation_errors as ( -validation_errors as ( + select value_field - select - value_field + from all_values + where + value_field in ( + {% for value in values -%} + {% if quote -%}'{{ value }}' + {%- else -%}{{ value }} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {%- endfor %} + ) - from all_values - where value_field in ( - {% for value in values -%} - {% if quote -%} - '{{ value }}' - {%- else -%} - {{ value }} - {%- endif -%} - {%- if not loop.last -%},{%- endif %} - {%- endfor %} ) -) - -select * -from validation_errors + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index 781ed7d9..48397170 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -1,19 +1,20 @@ - {% test not_constant(model, column_name) %} - {{ return(adapter.dispatch('test_not_constant', 'dbt_utils')(model, column_name)) }} + {{ + return( + adapter.dispatch("test_not_constant", "dbt_utils")(model, column_name) + ) + }} {% endtest %} {% macro default__test_not_constant(model, column_name) %} + select + {# In TSQL, subquery aggregate columns need aliases #} + {# thus: a filler col name, 'filler_column' #} + count(distinct {{ column_name }}) as filler_column -select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count(distinct {{ column_name }}) as filler_column - -from {{ model }} - -having count(distinct {{ column_name }}) = 1 + from {{ model }} + having count(distinct {{ column_name }}) = 1 {% endmacro %} diff --git a/macros/generic_tests/not_null_proportion.sql b/macros/generic_tests/not_null_proportion.sql index 45b9050c..b9e6da97 100644 --- a/macros/generic_tests/not_null_proportion.sql +++ b/macros/generic_tests/not_null_proportion.sql @@ -1,26 +1,32 @@ {% macro test_not_null_proportion(model) %} - {{ return(adapter.dispatch('test_not_null_proportion', 'dbt_utils')(model, **kwargs)) }} + {{ + return( + adapter.dispatch("test_not_null_proportion", "dbt_utils")(model, **kwargs) + ) + }} {% endmacro %} {% macro default__test_not_null_proportion(model) %} -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} -{% set at_least = kwargs.get('at_least', kwargs.get('arg')) %} -{% set at_most = kwargs.get('at_most', kwargs.get('arg', 1)) %} + {% set column_name = kwargs.get("column_name", kwargs.get("arg")) %} + {% set at_least = kwargs.get("at_least", kwargs.get("arg")) %} + {% set at_most = kwargs.get("at_most", kwargs.get("arg", 1)) %} -with validation as ( - select - sum(case when {{ column_name }} is null then 0 else 1 end) / cast(count(*) as numeric) as not_null_proportion - from {{ model }} -), -validation_errors as ( - select - not_null_proportion - from validation - where not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} -) -select - * -from validation_errors + with + validation as ( + select + sum(case when {{ column_name }} is null then 0 else 1 end) + / cast(count(*) as numeric) as not_null_proportion + from {{ model }} + ), + validation_errors as ( + select not_null_proportion + from validation + where + not_null_proportion < {{ at_least }} + or not_null_proportion > {{ at_most }} + ) + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/recency.sql b/macros/generic_tests/recency.sql index cb9a8de6..666f8e7a 100644 --- a/macros/generic_tests/recency.sql +++ b/macros/generic_tests/recency.sql @@ -1,24 +1,24 @@ {% test recency(model, field, datepart, interval) %} - {{ return(adapter.dispatch('test_recency', 'dbt_utils')(model, field, datepart, interval)) }} + {{ + return( + adapter.dispatch("test_recency", "dbt_utils")( + model, field, datepart, interval + ) + ) + }} {% endtest %} {% macro default__test_recency(model, field, datepart, interval) %} -{% set threshold = dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp()) %} + {% set threshold = dbt_utils.dateadd( + datepart, interval * -1, dbt_utils.current_timestamp() + ) %} -with recency as ( + with recency as (select max({{ field }}) as most_recent from {{ model }}) - select max({{field}}) as most_recent - from {{ model }} + select most_recent, {{ threshold }} as threshold -) - -select - - most_recent, - {{ threshold }} as threshold - -from recency -where most_recent < {{ threshold }} + from recency + where most_recent < {{ threshold }} {% endmacro %} diff --git a/macros/generic_tests/relationships_where.sql b/macros/generic_tests/relationships_where.sql index c35a380a..927e6706 100644 --- a/macros/generic_tests/relationships_where.sql +++ b/macros/generic_tests/relationships_where.sql @@ -1,51 +1,55 @@ -{% test relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} - {{ return(adapter.dispatch('test_relationships_where', 'dbt_utils')(model, column_name, to, field, from_condition, to_condition)) }} +{% test relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} + {{ + return( + adapter.dispatch("test_relationships_where", "dbt_utils")( + model, column_name, to, field, from_condition, to_condition + ) + ) + }} {% endtest %} -{% macro default__test_relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} +{% macro default__test_relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} -{# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} -{# ref https://stackoverflow.com/a/7170753/3842610 #} + {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} + {# ref https://stackoverflow.com/a/7170753/3842610 #} + with + left_table as ( -with left_table as ( + select {{ column_name }} as id - select - {{column_name}} as id + from {{ model }} - from {{model}} + where {{ column_name }} is not null and {{ from_condition }} - where {{column_name}} is not null - and {{from_condition}} + ), -), + right_table as ( -right_table as ( + select {{ field }} as id - select - {{field}} as id + from {{ to }} - from {{to}} + where {{ field }} is not null and {{ to_condition }} - where {{field}} is not null - and {{to_condition}} + ), -), + exceptions as ( -exceptions as ( + select left_table.id, right_table.id as right_id - select - left_table.id, - right_table.id as right_id + from left_table - from left_table + left join right_table on left_table.id = right_table.id - left join right_table - on left_table.id = right_table.id + where right_table.id is null - where right_table.id is null + ) -) - -select * from exceptions + select * + from exceptions {% endmacro %} diff --git a/macros/generic_tests/sequential_values.sql b/macros/generic_tests/sequential_values.sql index 8ddae707..71662749 100644 --- a/macros/generic_tests/sequential_values.sql +++ b/macros/generic_tests/sequential_values.sql @@ -1,35 +1,57 @@ {% test sequential_values(model, column_name, interval=1, datepart=None) %} - {{ return(adapter.dispatch('test_sequential_values', 'dbt_utils')(model, column_name, interval, datepart)) }} + {{ + return( + adapter.dispatch("test_sequential_values", "dbt_utils")( + model, column_name, interval, datepart + ) + ) + }} {% endtest %} -{% macro default__test_sequential_values(model, column_name, interval=1, datepart=None) %} - -{% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} - -with windowed as ( - - select - {{ column_name }}, - lag({{ column_name }}) over ( - order by {{ column_name }} - ) as {{ previous_column_name }} - from {{ model }} -), - -validation_errors as ( - select - * - from windowed - {% if datepart %} - where not(cast({{ column_name }} as {{ dbt_utils.type_timestamp() }})= cast({{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} as {{ dbt_utils.type_timestamp() }})) - {% else %} - where not({{ column_name }} = {{ previous_column_name }} + {{ interval }}) - {% endif %} -) - -select * -from validation_errors +{% macro default__test_sequential_values( + model, column_name, interval=1, datepart=None +) %} + + {% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} + + with + windowed as ( + + select + {{ column_name }}, + lag({{ column_name }}) over ( + order by {{ column_name }} + ) as {{ previous_column_name }} + from {{ model }} + ), + + validation_errors as ( + select * + from windowed + {% if datepart %} + where + not ( + cast( + {{ column_name }} as {{ dbt_utils.type_timestamp() }} + ) = cast( + {{ + dbt_utils.dateadd( + datepart, interval, previous_column_name + ) + }} as {{ dbt_utils.type_timestamp() }} + ) + ) + {% else %} + where + not ( + {{ column_name }} = {{ previous_column_name }} + {{ interval }} + ) + {% endif %} + ) + + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/test_not_null_where.sql b/macros/generic_tests/test_not_null_where.sql index d5dbf6c5..39b68b19 100644 --- a/macros/generic_tests/test_not_null_where.sql +++ b/macros/generic_tests/test_not_null_where.sql @@ -1,12 +1,13 @@ -{% test not_null_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.not_null_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_not_null_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_not_null_where(model, column_name) %} - {{ return(test_not_null(model, column_name)) }} -{% endmacro %} +{% test not_null_where(model, column_name) %} + {%- set deprecation_warning = " Warning: `dbt_utils.not_null_where` is no longer supported. Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. " -%} + {%- do exceptions.warn(deprecation_warning) -%} + {{ + return( + adapter.dispatch("test_not_null_where", "dbt_utils")(model, column_name) + ) + }} +{% endtest %} + +{% macro default__test_not_null_where(model, column_name) %} + {{ return(test_not_null(model, column_name)) }} +{% endmacro %} diff --git a/macros/generic_tests/test_unique_where.sql b/macros/generic_tests/test_unique_where.sql index e752d7b2..f163cf8e 100644 --- a/macros/generic_tests/test_unique_where.sql +++ b/macros/generic_tests/test_unique_where.sql @@ -1,12 +1,13 @@ -{% test unique_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.unique_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_unique_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_unique_where(model, column_name) %} - {{ return(test_unique(model, column_name)) }} -{% endmacro %} +{% test unique_where(model, column_name) %} + {%- set deprecation_warning = " Warning: `dbt_utils.unique_where` is no longer supported. Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. " -%} + {%- do exceptions.warn(deprecation_warning) -%} + {{ + return( + adapter.dispatch("test_unique_where", "dbt_utils")(model, column_name) + ) + }} +{% endtest %} + +{% macro default__test_unique_where(model, column_name) %} + {{ return(test_unique(model, column_name)) }} +{% endmacro %} diff --git a/macros/generic_tests/unique_combination_of_columns.sql b/macros/generic_tests/unique_combination_of_columns.sql index 74ccf5c6..83c28f12 100644 --- a/macros/generic_tests/unique_combination_of_columns.sql +++ b/macros/generic_tests/unique_combination_of_columns.sql @@ -1,37 +1,48 @@ -{% test unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} - {{ return(adapter.dispatch('test_unique_combination_of_columns', 'dbt_utils')(model, combination_of_columns, quote_columns)) }} +{% test unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} + {{ + return( + adapter.dispatch("test_unique_combination_of_columns", "dbt_utils")( + model, combination_of_columns, quote_columns + ) + ) + }} {% endtest %} -{% macro default__test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} +{% macro default__test_unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} -{% if not quote_columns %} - {%- set column_list=combination_of_columns %} -{% elif quote_columns %} - {%- set column_list=[] %} + {% if not quote_columns %} {%- set column_list = combination_of_columns %} + {% elif quote_columns %} + {%- set column_list = [] %} {% for column in combination_of_columns -%} - {% set column_list = column_list.append( adapter.quote(column) ) %} + {% set column_list = column_list.append(adapter.quote(column)) %} {%- endfor %} -{% else %} - {{ exceptions.raise_compiler_error( - "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" - ) }} -{% endif %} - -{%- set columns_csv=column_list | join(', ') %} - - -with validation_errors as ( - - select - {{ columns_csv }} - from {{ model }} - group by {{ columns_csv }} - having count(*) > 1 - -) - -select * -from validation_errors - + {% else %} + {{ + exceptions.raise_compiler_error( + "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" + ~ quote + ~ "'.'" + ) + }} + {% endif %} + + {%- set columns_csv = column_list | join(", ") %} + + with + validation_errors as ( + + select {{ columns_csv }} + from {{ model }} + group by {{ columns_csv }} + having count(*) > 1 + + ) + + select * + from validation_errors {% endmacro %} diff --git a/macros/jinja_helpers/log_info.sql b/macros/jinja_helpers/log_info.sql index 52b4b4a5..0c8c7675 100644 --- a/macros/jinja_helpers/log_info.sql +++ b/macros/jinja_helpers/log_info.sql @@ -1,5 +1,5 @@ {% macro log_info(message) %} - {{ return(adapter.dispatch('log_info', 'dbt_utils')(message)) }} + {{ return(adapter.dispatch("log_info", "dbt_utils")(message)) }} {% endmacro %} {% macro default__log_info(message) %} diff --git a/macros/jinja_helpers/pretty_log_format.sql b/macros/jinja_helpers/pretty_log_format.sql index fe580d13..115b38b2 100644 --- a/macros/jinja_helpers/pretty_log_format.sql +++ b/macros/jinja_helpers/pretty_log_format.sql @@ -1,7 +1,7 @@ {% macro pretty_log_format(message) %} - {{ return(adapter.dispatch('pretty_log_format', 'dbt_utils')(message)) }} + {{ return(adapter.dispatch("pretty_log_format", "dbt_utils")(message)) }} {% endmacro %} {% macro default__pretty_log_format(message) %} - {{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} + {{ return(dbt_utils.pretty_time() ~ " + " ~ message) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_time.sql b/macros/jinja_helpers/pretty_time.sql index bad37efe..285eda67 100644 --- a/macros/jinja_helpers/pretty_time.sql +++ b/macros/jinja_helpers/pretty_time.sql @@ -1,7 +1,7 @@ -{% macro pretty_time(format='%H:%M:%S') %} - {{ return(adapter.dispatch('pretty_time', 'dbt_utils')(format)) }} +{% macro pretty_time(format="%H:%M:%S") %} + {{ return(adapter.dispatch("pretty_time", "dbt_utils")(format)) }} {% endmacro %} -{% macro default__pretty_time(format='%H:%M:%S') %} +{% macro default__pretty_time(format="%H:%M:%S") %} {{ return(modules.datetime.datetime.now().strftime(format)) }} {% endmacro %} diff --git a/macros/jinja_helpers/slugify.sql b/macros/jinja_helpers/slugify.sql index 1b3c7272..b5888adc 100644 --- a/macros/jinja_helpers/slugify.sql +++ b/macros/jinja_helpers/slugify.sql @@ -1,12 +1,12 @@ {% macro slugify(string) %} -{#- Lower case the string -#} -{% set string = string | lower %} -{#- Replace spaces and dashes with underscores -#} -{% set string = modules.re.sub('[ -]+', '_', string) %} -{#- Only take letters, numbers, and underscores -#} -{% set string = modules.re.sub('[^a-z0-9_]+', '', string) %} + {#- Lower case the string -#} + {% set string = string | lower %} + {#- Replace spaces and dashes with underscores -#} + {% set string = modules.re.sub("[ -]+", "_", string) %} + {#- Only take letters, numbers, and underscores -#} + {% set string = modules.re.sub("[^a-z0-9_]+", "", string) %} -{{ return(string) }} + {{ return(string) }} {% endmacro %} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 5a15c815..a4cb0906 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -1,189 +1,276 @@ -{% macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - {{ return(adapter.dispatch('get_period_boundaries', 'dbt_utils')(target_schema, target_table, timestamp_field, start_date, stop_date, period)) }} +{% macro get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period +) -%} + {{ + return( + adapter.dispatch("get_period_boundaries", "dbt_utils")( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period, + ) + ) + }} {% endmacro %} -{% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - - {% call statement('period_boundaries', fetch_result=True) -%} - with data as ( - select - coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp as start_timestamp, - coalesce( - {{dbt_utils.dateadd('millisecond', +{% macro default__get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period +) -%} + + {% call statement("period_boundaries", fetch_result=True) -%} + with + data as ( + select + coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp + as start_timestamp, + coalesce( + {{ + dbt_utils.dateadd( + "millisecond", -1, - "nullif('" ~ stop_date ~ "','')::timestamp")}}, - {{dbt_utils.current_timestamp()}} - ) as stop_timestamp - from "{{target_schema}}"."{{target_table}}" - ) - - select - start_timestamp, - stop_timestamp, - {{dbt_utils.datediff('start_timestamp', - 'stop_timestamp', - period)}} + 1 as num_periods - from data - {%- endcall %} + "nullif('" ~ stop_date ~ "','')::timestamp", + ) + }}, + {{ dbt_utils.current_timestamp() }} + ) as stop_timestamp + from "{{target_schema}}"."{{target_table}}" + ) + + select + start_timestamp, + stop_timestamp, + {{ dbt_utils.datediff("start_timestamp", "stop_timestamp", period) }} + + 1 as num_periods + from data + {%- endcall %} {%- endmacro %} -{% macro get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} - {{ return(adapter.dispatch('get_period_sql', 'dbt_utils')(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset)) }} +{% macro get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset +) -%} + {{ + return( + adapter.dispatch("get_period_sql", "dbt_utils")( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset, + ) + ) + }} {% endmacro %} -{% macro default__get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} - - {%- set period_filter -%} +{% macro default__get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset +) -%} + + {%- set period_filter -%} ("{{timestamp_field}}" > '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' and "{{timestamp_field}}" <= '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' + interval '1 {{period}}' and "{{timestamp_field}}" < '{{stop_timestamp}}'::timestamp) - {%- endset -%} + {%- endset -%} - {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} + {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} - select - {{target_cols_csv}} - from ( - {{filtered_sql}} - ) + select {{ target_cols_csv }} + from ({{ filtered_sql }}) {%- endmacro %} {% materialization insert_by_period, default -%} - {%- set timestamp_field = config.require('timestamp_field') -%} - {%- set start_date = config.require('start_date') -%} - {%- set stop_date = config.get('stop_date') or '' -%} - {%- set period = config.get('period') or 'week' -%} + {%- set timestamp_field = config.require("timestamp_field") -%} + {%- set start_date = config.require("start_date") -%} + {%- set stop_date = config.get("stop_date") or "" -%} + {%- set period = config.get("period") or "week" -%} - {%- if sql.find('__PERIOD_FILTER__') == -1 -%} - {%- set error_message -%} + {%- if sql.find("__PERIOD_FILTER__") == -1 -%} + {%- set error_message -%} Model '{{ model.unique_id }}' does not include the required string '__PERIOD_FILTER__' in its sql - {%- endset -%} - {{ exceptions.raise_compiler_error(error_message) }} - {%- endif -%} - - {%- set identifier = model['name'] -%} - - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} - - {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - - {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} - {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} + {%- endset -%} + {{ exceptions.raise_compiler_error(error_message) }} + {%- endif -%} + + {%- set identifier = model["name"] -%} + + {%- set old_relation = adapter.get_relation( + database=database, schema=schema, identifier=identifier + ) -%} + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, type="table" + ) -%} + + {%- set non_destructive_mode = flags.NON_DESTRUCTIVE == True -%} + {%- set full_refresh_mode = flags.FULL_REFRESH == True -%} + + {%- set exists_as_table = old_relation is not none and old_relation.is_table -%} + {%- set exists_not_as_table = ( + old_relation is not none and not old_relation.is_table + ) -%} + + {%- set should_truncate = ( + non_destructive_mode and full_refresh_mode and exists_as_table + ) -%} + {%- set should_drop = not should_truncate and ( + full_refresh_mode or exists_not_as_table + ) -%} + {%- set force_create = flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE -%} + + -- setup + {% if old_relation is none -%} + -- noop + {%- elif should_truncate -%} {{ adapter.truncate_relation(old_relation) }} + {%- elif should_drop -%} + {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} + {%- endif %} - {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%} - {%- set should_drop = (not should_truncate and (full_refresh_mode or exists_not_as_table)) -%} - {%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%} + {{ run_hooks(pre_hooks, inside_transaction=False) }} - -- setup - {% if old_relation is none -%} - -- noop - {%- elif should_truncate -%} - {{adapter.truncate_relation(old_relation)}} - {%- elif should_drop -%} - {{adapter.drop_relation(old_relation)}} - {%- set old_relation = none -%} - {%- endif %} - - {{run_hooks(pre_hooks, inside_transaction=False)}} - - -- `begin` happens here, so `commit` after it to finish the transaction - {{run_hooks(pre_hooks, inside_transaction=True)}} - {% call statement() -%} - begin; -- make extra sure we've closed out the transaction - commit; - {%- endcall %} - - -- build model - {% if force_create or old_relation is none -%} - {# Create an empty target table -#} - {% call statement('main') -%} - {%- set empty_sql = sql | replace("__PERIOD_FILTER__", 'false') -%} - {{create_table_as(False, target_relation, empty_sql)}} - {%- endcall %} - {%- endif %} - - {% set _ = dbt_utils.get_period_boundaries(schema, - identifier, - timestamp_field, - start_date, - stop_date, - period) %} - {%- set start_timestamp = load_result('period_boundaries')['data'][0][0] | string -%} - {%- set stop_timestamp = load_result('period_boundaries')['data'][0][1] | string -%} - {%- set num_periods = load_result('period_boundaries')['data'][0][2] | int -%} - - {% set target_columns = adapter.get_columns_in_relation(target_relation) %} - {%- set target_cols_csv = target_columns | map(attribute='quoted') | join(', ') -%} - {%- set loop_vars = {'sum_rows_inserted': 0} -%} - - -- commit each period as a separate transaction - {% for i in range(num_periods) -%} - {%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} - {{ dbt_utils.log_info(msg) }} - - {%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%} - {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, - schema=schema, type='table') -%} + -- `begin` happens here, so `commit` after it to finish the transaction + {{ run_hooks(pre_hooks, inside_transaction=True) }} {% call statement() -%} - {% set tmp_table_sql = dbt_utils.get_period_sql(target_cols_csv, - sql, - timestamp_field, - period, - start_timestamp, - stop_timestamp, - i) %} - {{dbt.create_table_as(True, tmp_relation, tmp_table_sql)}} + begin -- make extra sure we've closed out the transaction + ; + commit + ; {%- endcall %} - {{adapter.expand_target_column_types(from_relation=tmp_relation, - to_relation=target_relation)}} - {%- set name = 'main-' ~ i -%} - {% call statement(name, fetch_result=True) -%} + -- build model + {% if force_create or old_relation is none -%} + {# Create an empty target table -#} + {% call statement("main") -%} + {%- set empty_sql = sql | replace("__PERIOD_FILTER__", "false") -%} + {{ create_table_as(False, target_relation, empty_sql) }} + {%- endcall %} + {%- endif %} + + {% set _ = dbt_utils.get_period_boundaries( + schema, identifier, timestamp_field, start_date, stop_date, period + ) %} + {%- set start_timestamp = ( + load_result("period_boundaries")["data"][0][0] | string + ) -%} + {%- set stop_timestamp = load_result("period_boundaries")["data"][0][1] | string -%} + {%- set num_periods = load_result("period_boundaries")["data"][0][2] | int -%} + + {% set target_columns = adapter.get_columns_in_relation(target_relation) %} + {%- set target_cols_csv = target_columns | map(attribute="quoted") | join(", ") -%} + {%- set loop_vars = {"sum_rows_inserted": 0} -%} + + -- commit each period as a separate transaction + {% for i in range(num_periods) -%} + {%- set msg = ( + "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) + ) -%} + {{ dbt_utils.log_info(msg) }} + + {%- set tmp_identifier = ( + model["name"] ~ "__dbt_incremental_period" ~ i ~ "_tmp" + ) -%} + {%- set tmp_relation = api.Relation.create( + identifier=tmp_identifier, schema=schema, type="table" + ) -%} + {% call statement() -%} + {% set tmp_table_sql = dbt_utils.get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + i, + ) %} + {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} + {%- endcall %} + + {{ + adapter.expand_target_column_types( + from_relation=tmp_relation, to_relation=target_relation + ) + }} + {%- set name = "main-" ~ i -%} + {% call statement(name, fetch_result=True) -%} insert into {{target_relation}} ({{target_cols_csv}}) ( select - {{target_cols_csv}} + {{ target_cols_csv }} from {{tmp_relation.include(schema=False)}} ); - {%- endcall %} - {% set result = load_result('main-' ~ i) %} - {% if 'response' in result.keys() %} {# added in v0.19.0 #} - {% set rows_inserted = result['response']['rows_affected'] %} - {% else %} {# older versions #} - {% set rows_inserted = result['status'].split(" ")[2] | int %} - {% endif %} - - {%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%} - {%- if loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %} {% endif -%} - - {%- set msg = "Ran for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) ~ "; " ~ rows_inserted ~ " records inserted" -%} - {{ dbt_utils.log_info(msg) }} - - {%- endfor %} + {%- endcall %} + {% set result = load_result("main-" ~ i) %} + {% if "response" in result.keys() %} {# added in v0.19.0 #} + {% set rows_inserted = result["response"]["rows_affected"] %} + {% else %} {# older versions #} + {% set rows_inserted = result["status"].split(" ")[2] | int %} + {% endif %} + + {%- set sum_rows_inserted = loop_vars["sum_rows_inserted"] + rows_inserted -%} + {%- if loop_vars.update({"sum_rows_inserted": sum_rows_inserted}) %} + {% endif -%} + + {%- set msg = ( + "Ran for " + ~ period + ~ " " + ~ (i + 1) + ~ " of " + ~ (num_periods) + ~ "; " + ~ rows_inserted + ~ " records inserted" + ) -%} + {{ dbt_utils.log_info(msg) }} + + {%- endfor %} - {% call statement() -%} - begin; - {%- endcall %} + {% call statement() -%} + begin + ; + {%- endcall %} - {{run_hooks(post_hooks, inside_transaction=True)}} + {{ run_hooks(post_hooks, inside_transaction=True) }} - {% call statement() -%} - commit; - {%- endcall %} + {% call statement() -%} + commit + ; + {%- endcall %} - {{run_hooks(post_hooks, inside_transaction=False)}} + {{ run_hooks(post_hooks, inside_transaction=False) }} - {%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} + {%- set status_string = "INSERT " ~ loop_vars["sum_rows_inserted"] -%} - {% call noop_statement('main', status_string) -%} + {% call noop_statement("main", status_string) -%} -- no-op - {%- endcall %} + {%- endcall %} - -- Return the relations created in this materialization - {{ return({'relations': [target_relation]}) }} + -- Return the relations created in this materialization + {{ return({"relations": [target_relation]}) }} {%- endmaterialization %} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index 759f8d59..3fc4e1c8 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -1,75 +1,82 @@ {% macro get_intervals_between(start_date, end_date, datepart) -%} - {{ return(adapter.dispatch('get_intervals_between', 'dbt_utils')(start_date, end_date, datepart)) }} + {{ + return( + adapter.dispatch("get_intervals_between", "dbt_utils")( + start_date, end_date, datepart + ) + ) + }} {%- endmacro %} {% macro default__get_intervals_between(start_date, end_date, datepart) -%} - {%- call statement('get_intervals_between', fetch_result=True) %} + {%- call statement("get_intervals_between", fetch_result=True) %} - select {{dbt_utils.datediff(start_date, end_date, datepart)}} + select {{ dbt_utils.datediff(start_date, end_date, datepart) }} {%- endcall -%} - {%- set value_list = load_result('get_intervals_between') -%} + {%- set value_list = load_result("get_intervals_between") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} {{ return(values[0]) }} - {%- else -%} - {{ return(1) }} + {%- else -%} {{ return(1) }} {%- endif -%} {%- endmacro %} - - {% macro date_spine(datepart, start_date, end_date) %} - {{ return(adapter.dispatch('date_spine', 'dbt_utils')(datepart, start_date, end_date)) }} + {{ + return( + adapter.dispatch("date_spine", "dbt_utils")( + datepart, start_date, end_date + ) + ) + }} {%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} - -{# call as follows: + {# call as follows: date_spine( "day", "to_date('01/01/2016', 'mm/dd/yyyy')", "dateadd(week, 1, current_date)" ) #} + with + rawdata as ( + {{ + dbt_utils.generate_series( + dbt_utils.get_intervals_between(start_date, end_date, datepart) + ) + }} -with rawdata as ( + ), - {{dbt_utils.generate_series( - dbt_utils.get_intervals_between(start_date, end_date, datepart) - )}} + all_periods as ( -), + select + ( + {{ + dbt_utils.dateadd( + datepart, "row_number() over (order by 1) - 1", start_date + ) + }} + ) as date_{{ datepart }} + from rawdata -all_periods as ( + ), - select ( - {{ - dbt_utils.dateadd( - datepart, - "row_number() over (order by 1) - 1", - start_date - ) - }} - ) as date_{{datepart}} - from rawdata + filtered as ( -), + select * from all_periods where date_{{ datepart }} <= {{ end_date }} -filtered as ( + ) select * - from all_periods - where date_{{datepart}} <= {{ end_date }} - -) - -select * from filtered + from filtered {% endmacro %} diff --git a/macros/sql/deduplicate.sql b/macros/sql/deduplicate.sql index 9a3571a2..bc48255f 100644 --- a/macros/sql/deduplicate.sql +++ b/macros/sql/deduplicate.sql @@ -1,22 +1,28 @@ {%- macro deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} - {{ return(adapter.dispatch('deduplicate', 'dbt_utils')(relation, group_by, order_by=order_by, relation_alias=relation_alias)) }} + {{ + return( + adapter.dispatch("deduplicate", "dbt_utils")( + relation, group_by, order_by=order_by, relation_alias=relation_alias + ) + ) + }} {% endmacro %} -{%- macro default__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro default__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( - select - _inner.*, - row_number() over ( - partition by {{ group_by }} - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} - ) as rn - from {{ relation if relation_alias is none else relation_alias }} as _inner - ) as deduped + select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} + from + ( + select + _inner.*, + row_number() over ( + partition by {{ group_by }} + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} + ) as rn + from {{ relation if relation_alias is none else relation_alias }} as _inner + ) as deduped where deduped.rn = 1 {%- endmacro -%} @@ -26,21 +32,22 @@ -- clause in BigQuery: -- https://github.com/dbt-labs/dbt-utils/issues/335#issuecomment-788157572 #} -{%- macro bigquery__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro bigquery__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( - select - array_agg ( - original - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} - limit 1 - )[offset(0)] as deduped - from {{ relation if relation_alias is none else relation_alias }} as original - group by {{ group_by }} - ) + select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} + from + ( + select + array_agg( + original + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} + limit 1 + )[offset(0)] as deduped + from + {{ relation if relation_alias is none else relation_alias }} as original + group by {{ group_by }} + ) {%- endmacro -%} diff --git a/macros/sql/generate_series.sql b/macros/sql/generate_series.sql index efcbd8ac..4aac864b 100644 --- a/macros/sql/generate_series.sql +++ b/macros/sql/generate_series.sql @@ -1,53 +1,55 @@ {% macro get_powers_of_two(upper_bound) %} - {{ return(adapter.dispatch('get_powers_of_two', 'dbt_utils')(upper_bound)) }} + {{ return(adapter.dispatch("get_powers_of_two", "dbt_utils")(upper_bound)) }} {% endmacro %} {% macro default__get_powers_of_two(upper_bound) %} {% if upper_bound <= 0 %} - {{ exceptions.raise_compiler_error("upper bound must be positive") }} + {{ exceptions.raise_compiler_error("upper bound must be positive") }} {% endif %} {% for _ in range(1, 100) %} - {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} + {% if upper_bound <= 2**loop.index %} {{ return(loop.index) }}{% endif %} {% endfor %} {% endmacro %} {% macro generate_series(upper_bound) %} - {{ return(adapter.dispatch('generate_series', 'dbt_utils')(upper_bound)) }} + {{ return(adapter.dispatch("generate_series", "dbt_utils")(upper_bound)) }} {% endmacro %} {% macro default__generate_series(upper_bound) %} {% set n = dbt_utils.get_powers_of_two(upper_bound) %} - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( + with + p as ( + select 0 as generated_number + union all + select 1 + ), + unioned as ( - select + select - {% for i in range(n) %} - p{{i}}.generated_number * power(2, {{i}}) - {% if not loop.last %} + {% endif %} - {% endfor %} - + 1 - as generated_number + {% for i in range(n) %} + p{{ i }}.generated_number * power(2, {{ i }}) + {% if not loop.last %} + {% endif %} + {% endfor %} + + 1 as generated_number - from + from - {% for i in range(n) %} - p as p{{i}} - {% if not loop.last %} cross join {% endif %} - {% endfor %} + {% for i in range(n) %} + p as p{{ i }} {% if not loop.last %} cross join {% endif %} + {% endfor %} - ) + ) select * from unioned - where generated_number <= {{upper_bound}} + where generated_number <= {{ upper_bound }} order by generated_number {% endmacro %} diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index f70890e2..a79c469f 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -1,15 +1,24 @@ -{% macro get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {{ return(adapter.dispatch('get_column_values', 'dbt_utils')(table, column, order_by, max_records, default)) }} +{% macro get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} + {{ + return( + adapter.dispatch("get_column_values", "dbt_utils")( + table, column, order_by, max_records, default + ) + ) + }} {% endmacro %} -{% macro default__get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{% macro default__get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} {%- if not execute -%} - {% set default = [] if not default %} - {{ return(default) }} + {% set default = [] if not default %} {{ return(default) }} {% endif %} - {%- do dbt_utils._is_ephemeral(table, 'get_column_values') -%} + {%- do dbt_utils._is_ephemeral(table, "get_column_values") -%} {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} {# TODO: Change the method signature in a future 0.x.0 release #} @@ -18,43 +27,52 @@ {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} {% set relation_exists = (load_relation(target_relation)) is not none %} - {%- call statement('get_column_values', fetch_result=true) %} + {%- call statement("get_column_values", fetch_result=true) %} {%- if not relation_exists and default is none -%} - {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} + {{ + exceptions.raise_compiler_error( + "In get_column_values(): relation " + ~ target_relation + ~ " does not exist and no default value was provided." + ) + }} {%- elif not relation_exists and default is not none -%} - {{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} + {{ + log( + "Relation " + ~ target_relation + ~ " does not exist. Returning the default value: " + ~ default + ) + }} - {{ return(default) }} + {{ return(default) }} {%- else -%} - - select - {{ column }} as value + select {{ column }} as value from {{ target_relation }} group by {{ column }} order by {{ order_by }} - {% if max_records is not none %} - limit {{ max_records }} + {% if max_records is not none %} limit {{ max_records }} {% endif %} {% endif %} {%- endcall -%} - {%- set value_list = load_result('get_column_values') -%} + {%- set value_list = load_result("get_column_values") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} {{ return(values) }} - {%- else -%} - {{ return(default) }} + {%- else -%} {{ return(default) }} {%- endif -%} {%- endmacro %} diff --git a/macros/sql/get_filtered_columns_in_relation.sql b/macros/sql/get_filtered_columns_in_relation.sql index 7f4af889..8a14f6eb 100644 --- a/macros/sql/get_filtered_columns_in_relation.sql +++ b/macros/sql/get_filtered_columns_in_relation.sql @@ -7,9 +7,7 @@ {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('') }} - {% endif %} + {%- if not execute -%} {{ return("") }} {% endif %} {%- set include_cols = [] %} {%- set cols = adapter.get_columns_in_relation(from) -%} @@ -22,4 +20,4 @@ {{ return(include_cols) }} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/sql/get_query_results_as_dict.sql b/macros/sql/get_query_results_as_dict.sql index 6548f2dd..6c68991f 100644 --- a/macros/sql/get_query_results_as_dict.sql +++ b/macros/sql/get_query_results_as_dict.sql @@ -1,21 +1,20 @@ {% macro get_query_results_as_dict(query) %} - {{ return(adapter.dispatch('get_query_results_as_dict', 'dbt_utils')(query)) }} + {{ return(adapter.dispatch("get_query_results_as_dict", "dbt_utils")(query)) }} {% endmacro %} {% macro default__get_query_results_as_dict(query) %} -{# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} - - {%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} + {# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} + {%- call statement("get_query_results", fetch_result=True, auto_begin=false) -%} {{ query }} {%- endcall -%} - {% set sql_results={} %} + {% set sql_results = {} %} {%- if execute -%} - {% set sql_results_table = load_result('get_query_results').table.columns %} + {% set sql_results_table = load_result("get_query_results").table.columns %} {% for column_name, column in sql_results_table.items() %} {% do sql_results.update({column_name: column.values()}) %} {% endfor %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 9325a883..b806bcc7 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -1,32 +1,45 @@ -{% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_pattern', 'dbt_utils')(schema_pattern, table_pattern, exclude, database)) }} +{% macro get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} + {{ + return( + adapter.dispatch("get_relations_by_pattern", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) + ) + }} {% endmacro %} -{% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro default__get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} + {%- call statement("get_tables", fetch_result=True) %} - {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} + {{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude, database + ) + }} {%- endcall -%} - {%- set table_list = load_result('get_tables') -%} + {%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} + {%- if table_list and table_list["table"] -%} {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} + {%- for row in table_list["table"] -%} {%- set tbl_relation = api.Relation.create( database=database, schema=row.table_schema, identifier=row.table_name, - type=row.table_type + type=row.table_type, ) -%} {%- do tbl_relations.append(tbl_relation) -%} {%- endfor -%} {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} + {%- else -%} {{ return([]) }} {%- endif -%} {% endmacro %} diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index b6733c4b..59f1b90d 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -1,32 +1,41 @@ -{% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_prefix', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} + {{ + return( + adapter.dispatch("get_relations_by_prefix", "dbt_utils")( + schema, prefix, exclude, database + ) + ) + }} {% endmacro %} -{% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} + {%- call statement("get_tables", fetch_result=True) %} - {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} + {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} {%- endcall -%} - {%- set table_list = load_result('get_tables') -%} + {%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} + {%- if table_list and table_list["table"] -%} {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} + {%- for row in table_list["table"] -%} {%- set tbl_relation = api.Relation.create( database=database, schema=row.table_schema, identifier=row.table_name, - type=row.table_type + type=row.table_type, ) -%} {%- do tbl_relations.append(tbl_relation) -%} {%- endfor -%} {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} + {%- else -%} {{ return([]) }} {%- endif -%} {% endmacro %} diff --git a/macros/sql/get_table_types_sql.sql b/macros/sql/get_table_types_sql.sql index 91573779..fe3eb4c1 100644 --- a/macros/sql/get_table_types_sql.sql +++ b/macros/sql/get_table_types_sql.sql @@ -1,31 +1,43 @@ {%- macro get_table_types_sql() -%} - {{ return(adapter.dispatch('get_table_types_sql', 'dbt_utils')()) }} + {{ return(adapter.dispatch("get_table_types_sql", "dbt_utils")()) }} {%- endmacro -%} {% macro default__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" + case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) + end as "table_type" {% endmacro %} {% macro postgres__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'FOREIGN' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" + case + table_type + when 'BASE TABLE' + then 'table' + when 'FOREIGN' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) + end as "table_type" {% endmacro %} {% macro bigquery__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as `table_type` -{% endmacro %} \ No newline at end of file + case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) + end as `table_type` +{% endmacro %} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index 4d5a8fc9..40603f1d 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -1,28 +1,41 @@ -{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_pattern_sql', 'dbt_utils') - (schema_pattern, table_pattern, exclude, database)) }} +{% macro get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} + {{ + return( + adapter.dispatch("get_tables_by_pattern_sql", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) + ) + }} {% endmacro %} -{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - - select distinct - table_schema as "table_schema", - table_name as "table_name", - {{ dbt_utils.get_table_types_sql() }} - from {{ database }}.information_schema.tables - where table_schema ilike '{{ schema_pattern }}' +{% macro default__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} + + select distinct + table_schema as "table_schema", + table_name as "table_name", + {{ dbt_utils.get_table_types_sql() }} + from {{ database }}.information_schema.tables + where + table_schema ilike '{{ schema_pattern }}' and table_name ilike '{{ table_pattern }}' and table_name not ilike '{{ exclude }}' {% endmacro %} -{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro bigquery__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {% if '%' in schema_pattern %} - {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %} - {% else %} - {% set schemata=[schema_pattern] %} + {% if "%" in schema_pattern %} + {% set schemata = dbt_utils._bigquery__get_matching_schemata( + schema_pattern, database + ) %} + {% else %} {% set schemata = [schema_pattern] %} {% endif %} {% set sql %} @@ -54,17 +67,14 @@ where lower(schema_name) like lower('{{ schema_pattern }}') {% endset %} - {% set results=run_query(sql) %} + {% set results = run_query(sql) %} - {% set schemata=results.columns['schema_name'].values() %} + {% set schemata = results.columns["schema_name"].values() %} {{ return(schemata) }} - {% else %} - - {{ return([]) }} + {% else %} {{ return([]) }} {% endif %} - {% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index f8fdfba6..d4b6dda7 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,14 +1,26 @@ -{% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_prefix_sql', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} + {{ + return( + adapter.dispatch("get_tables_by_prefix_sql", "dbt_utils")( + schema, prefix, exclude, database + ) + ) + }} {% endmacro %} -{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} + + {{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern=schema, + table_pattern=prefix ~ "%", + exclude=exclude, + database=database, + ) + }} - {{ dbt_utils.get_tables_by_pattern_sql( - schema_pattern = schema, - table_pattern = prefix ~ '%', - exclude = exclude, - database = database - ) }} - {% endmacro %} diff --git a/macros/sql/groupby.sql b/macros/sql/groupby.sql index 68a68cf2..b0ae3f70 100644 --- a/macros/sql/groupby.sql +++ b/macros/sql/groupby.sql @@ -1,11 +1,10 @@ {%- macro group_by(n) -%} - {{ return(adapter.dispatch('group_by', 'dbt_utils')(n)) }} + {{ return(adapter.dispatch("group_by", "dbt_utils")(n)) }} {% endmacro %} {%- macro default__group_by(n) -%} - group by {% for i in range(1, n + 1) -%} - {{ i }}{{ ',' if not loop.last }} - {%- endfor -%} + group by + {% for i in range(1, n + 1) -%} {{ i }}{{ ',' if not loop.last }} {%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 70f276ec..b3107c39 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -5,47 +5,68 @@ http://daynebatten.com/2015/09/latitude-longitude-distance-sql/ The arguments should be float type. #} +{% macro degrees_to_radians(degrees) -%} acos(-1) * {{ degrees }} / 180 {%- endmacro %} -{% macro degrees_to_radians(degrees) -%} - acos(-1) * {{degrees}} / 180 -{%- endmacro %} - -{% macro haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} - {{ return(adapter.dispatch('haversine_distance', 'dbt_utils')(lat1,lon1,lat2,lon2,unit)) }} +{% macro haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} + {{ + return( + adapter.dispatch("haversine_distance", "dbt_utils")( + lat1, lon1, lat2, lon2, unit + ) + ) + }} {% endmacro %} -{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} -{% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} -{% endif %} +{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} + {%- if unit == "mi" %} {% set conversion_rate = 1 %} + {% elif unit == "km" %} {% set conversion_rate = 1.60934 %} + {% else %} + {{ + exceptions.raise_compiler_error( + "unit input must be one of 'mi' or 'km'. Got " ~ unit + ) + }} + {% endif %} - 2 * 3961 * asin(sqrt(power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + - cos(radians({{lat1}})) * cos(radians({{lat2}})) * - power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2))) * {{ conversion_rate }} + 2 + * 3961 + * asin( + sqrt( + power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + + cos(radians({{ lat1 }})) + * cos(radians({{ lat2 }})) + * power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2) + ) + ) + * {{ conversion_rate }} {%- endmacro %} - -{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} -{% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} -{% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} -{% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} -{% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} -{% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} -{% endif %} - 2 * 3961 * asin(sqrt(power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + - cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * - power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2))) * {{ conversion_rate }} +{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} + {% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} + {% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} + {% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} + {% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} + {%- if unit == "mi" %} {% set conversion_rate = 1 %} + {% elif unit == "km" %} {% set conversion_rate = 1.60934 %} + {% else %} + {{ + exceptions.raise_compiler_error( + "unit input must be one of 'mi' or 'km'. Got " ~ unit + ) + }} + {% endif %} + 2 + * 3961 + * asin( + sqrt( + power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + + cos({{ radians_lat1 }}) + * cos({{ radians_lat2 }}) + * power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2) + ) + ) + * {{ conversion_rate }} {%- endmacro %} - diff --git a/macros/sql/nullcheck.sql b/macros/sql/nullcheck.sql index 509d24f7..ba4b5a5b 100644 --- a/macros/sql/nullcheck.sql +++ b/macros/sql/nullcheck.sql @@ -1,21 +1,17 @@ {% macro nullcheck(cols) %} - {{ return(adapter.dispatch('nullcheck', 'dbt_utils')(cols)) }} + {{ return(adapter.dispatch("nullcheck", "dbt_utils")(cols)) }} {% endmacro %} {% macro default__nullcheck(cols) %} -{%- for col in cols %} + {%- for col in cols %} - {% if col.is_string() -%} + {% if col.is_string() -%} nullif({{ col.name }}, '') as {{ col.name }} - nullif({{col.name}},'') as {{col.name}} + {%- else -%} {{ col.name }} - {%- else -%} + {%- endif -%} - {{col.name}} + {%- if not loop.last -%}, {%- endif -%} - {%- endif -%} - -{%- if not loop.last -%} , {%- endif -%} - -{%- endfor -%} + {%- endfor -%} {% endmacro %} diff --git a/macros/sql/nullcheck_table.sql b/macros/sql/nullcheck_table.sql index c9ab5838..ef521b4c 100644 --- a/macros/sql/nullcheck_table.sql +++ b/macros/sql/nullcheck_table.sql @@ -1,14 +1,14 @@ {% macro nullcheck_table(relation) %} - {{ return(adapter.dispatch('nullcheck_table', 'dbt_utils')(relation)) }} + {{ return(adapter.dispatch("nullcheck_table", "dbt_utils")(relation)) }} {% endmacro %} {% macro default__nullcheck_table(relation) %} - {%- do dbt_utils._is_relation(relation, 'nullcheck_table') -%} - {%- do dbt_utils._is_ephemeral(relation, 'nullcheck_table') -%} - {% set cols = adapter.get_columns_in_relation(relation) %} + {%- do dbt_utils._is_relation(relation, "nullcheck_table") -%} + {%- do dbt_utils._is_ephemeral(relation, "nullcheck_table") -%} + {% set cols = adapter.get_columns_in_relation(relation) %} - select {{ dbt_utils.nullcheck(cols) }} - from {{relation}} + select {{ dbt_utils.nullcheck(cols) }} + from {{ relation }} {% endmacro %} diff --git a/macros/sql/pivot.sql b/macros/sql/pivot.sql index 88751062..33118ef7 100644 --- a/macros/sql/pivot.sql +++ b/macros/sql/pivot.sql @@ -39,48 +39,66 @@ Arguments: quote_identifiers: Whether to surround column aliases with double quotes, default is true distinct: Whether to use distinct in the aggregation, default is False #} - -{% macro pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {{ return(adapter.dispatch('pivot', 'dbt_utils')(column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers, distinct)) }} +{% macro pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False +) %} + {{ + return( + adapter.dispatch("pivot", "dbt_utils")( + column, + values, + alias, + agg, + cmp, + prefix, + suffix, + then_value, + else_value, + quote_identifiers, + distinct, + ) + ) + }} {% endmacro %} -{% macro default__pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {% for value in values %} - {{ agg }}( - {% if distinct %} distinct {% endif %} - case - when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' - then {{ then_value }} - else {{ else_value }} - end - ) - {% if alias %} - {% if quote_identifiers %} - as {{ adapter.quote(prefix ~ value ~ suffix) }} - {% else %} - as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} - {% endif %} - {% endif %} - {% if not loop.last %},{% endif %} - {% endfor %} +{% macro default__pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False +) %} + {% for value in values %} + {{ agg }} ( + {% if distinct %}distinct {% endif %} + case + when + {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' + then {{ then_value }} + else {{ else_value }} + end + ) + {% if alias %} + {% if quote_identifiers %} as {{ adapter.quote(prefix ~ value ~ suffix) }} + {% else %} as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} + {% endif %} + {% endif %} + {% if not loop.last %},{% endif %} + {% endfor %} {% endmacro %} diff --git a/macros/sql/safe_add.sql b/macros/sql/safe_add.sql index 3b6195c8..82a5ce7f 100644 --- a/macros/sql/safe_add.sql +++ b/macros/sql/safe_add.sql @@ -2,19 +2,19 @@ {# needed for safe_add to allow for non-keyword arguments see SO post #} {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('safe_add', 'dbt_utils')(*varargs)) }} + {{ return(adapter.dispatch("safe_add", "dbt_utils")(*varargs)) }} {% endmacro %} {%- macro default__safe_add() -%} -{% set fields = [] %} + {% set fields = [] %} -{%- for field in varargs -%} + {%- for field in varargs -%} - {% do fields.append("coalesce(" ~ field ~ ", 0)") %} + {% do fields.append("coalesce(" ~ field ~ ", 0)") %} -{%- endfor -%} + {%- endfor -%} -{{ fields|join(' +\n ') }} + {{ fields | join(" +\n ") }} {%- endmacro -%} diff --git a/macros/sql/star.sql b/macros/sql/star.sql index 25e58969..a92ac6a7 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -6,15 +6,18 @@ {%- do dbt_utils._is_relation(from, 'star') -%} {%- do dbt_utils._is_ephemeral(from, 'star') -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('*') }} - {% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("*") }} {% endif %} {%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} - {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} {%- if prefix!='' or suffix!='' %} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- endif -%} - {%- if not loop.last %},{{ '\n ' }}{% endif %} + {%- if relation_alias %} {{ relation_alias }}. + {% else %} + {%- endif -%} {{ adapter.quote(col) | trim }} + {%- if prefix != "" or suffix != "" %} + as {{ adapter.quote(prefix ~ col ~ suffix) | trim }} + {%- endif -%} + {%- if not loop.last %},{{ "\n " }}{% endif %} {%- endfor -%} {%- endmacro %} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index ba5c3e44..137d4583 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -2,50 +2,49 @@ {# needed for safe_add to allow for non-keyword arguments see SO post #} {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('surrogate_key', 'dbt_utils')(field_list, *varargs)) }} + {{ return(adapter.dispatch("surrogate_key", "dbt_utils")(field_list, *varargs)) }} {% endmacro %} {%- macro default__surrogate_key(field_list) -%} -{%- if varargs|length >= 1 or field_list is string %} + {%- if varargs | length >= 1 or field_list is string %} -{%- set error_message = ' -Warning: the `surrogate_key` macro now takes a single list argument instead of \ -multiple string arguments. Support for multiple string arguments will be \ -deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ -'.format(model.package_name, model.name) -%} + {%- set error_message = " Warning: the `surrogate_key` macro now takes a single list argument instead of \ multiple string arguments. Support for multiple string arguments will be \ deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name + ) -%} -{%- do exceptions.warn(error_message) -%} + {%- do exceptions.warn(error_message) -%} -{# first argument is not included in varargs, so add first element to field_list_xf #} -{%- set field_list_xf = [field_list] -%} + {# first argument is not included in varargs, so add first element to field_list_xf #} + {%- set field_list_xf = [field_list] -%} -{%- for field in varargs %} -{%- set _ = field_list_xf.append(field) -%} -{%- endfor -%} + {%- for field in varargs %} + {%- set _ = field_list_xf.append(field) -%} + {%- endfor -%} -{%- else -%} + {%- else -%} -{# if using list, just set field_list_xf as field_list #} -{%- set field_list_xf = field_list -%} - -{%- endif -%} + {# if using list, just set field_list_xf as field_list #} + {%- set field_list_xf = field_list -%} + {%- endif -%} -{%- set fields = [] -%} + {%- set fields = [] -%} -{%- for field in field_list_xf -%} + {%- for field in field_list_xf -%} - {%- set _ = fields.append( - "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" - ) -%} + {%- set _ = fields.append( + "coalesce(cast(" + ~ field + ~ " as " + ~ dbt_utils.type_string() + ~ "), '')" + ) -%} - {%- if not loop.last %} - {%- set _ = fields.append("'-'") -%} - {%- endif -%} + {%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} -{%- endfor -%} + {%- endfor -%} -{{dbt_utils.hash(dbt_utils.concat(fields))}} + {{ dbt_utils.hash(dbt_utils.concat(fields)) }} {%- endmacro -%} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index a7bf1d95..3527fa49 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -1,17 +1,37 @@ -{%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} - {{ return(adapter.dispatch('union_relations', 'dbt_utils')(relations, column_override, include, exclude, source_column_name)) }} +{%- macro union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation" +) -%} + {{ + return( + adapter.dispatch("union_relations", "dbt_utils")( + relations, column_override, include, exclude, source_column_name + ) + ) + }} {% endmacro %} -{%- macro default__union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} +{%- macro default__union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation" +) -%} {%- if exclude and include -%} - {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} + {{ + exceptions.raise_compiler_error( + "Both an exclude and include list were provided to the `union` macro. Only one is allowed" + ) + }} {%- endif -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} - {%- if not execute %} - {{ return('') }} - {% endif -%} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} + {%- if not execute %} {{ return("") }} {% endif -%} {%- set column_override = column_override if column_override is not none else {} -%} @@ -22,46 +42,46 @@ {%- do relation_columns.update({relation: []}) -%} - {%- do dbt_utils._is_relation(relation, 'union_relations') -%} - {%- do dbt_utils._is_ephemeral(relation, 'union_relations') -%} + {%- do dbt_utils._is_relation(relation, "union_relations") -%} + {%- do dbt_utils._is_ephemeral(relation, "union_relations") -%} {%- set cols = adapter.get_columns_in_relation(relation) -%} {%- for col in cols -%} - {#- If an exclude list was provided and the column is in the list, do nothing -#} - {%- if exclude and col.column in exclude -%} + {#- If an exclude list was provided and the column is in the list, do nothing -#} + {%- if exclude and col.column in exclude -%} - {#- If an include list was provided and the column is not in the list, do nothing -#} - {%- elif include and col.column not in include -%} + {#- If an include list was provided and the column is not in the list, do nothing -#} + {%- elif include and col.column not in include -%} - {#- Otherwise add the column to the column superset -#} - {%- else -%} + {#- Otherwise add the column to the column superset -#} + {%- else -%} - {#- update the list of columns in this relation -#} - {%- do relation_columns[relation].append(col.column) -%} + {#- update the list of columns in this relation -#} + {%- do relation_columns[relation].append(col.column) -%} - {%- if col.column in column_superset -%} + {%- if col.column in column_superset -%} - {%- set stored = column_superset[col.column] -%} - {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} + {%- set stored = column_superset[col.column] -%} + {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} - {%- do column_superset.update({col.column: col}) -%} + {%- do column_superset.update({col.column: col}) -%} - {%- endif %} + {%- endif %} - {%- else -%} + {%- else -%} {%- do column_superset.update({col.column: col}) -%} - {%- do column_superset.update({col.column: col}) -%} + {%- endif -%} {%- endif -%} - {%- endif -%} - {%- endfor -%} {%- endfor -%} {%- set ordered_column_names = column_superset.keys() -%} - {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} + {% if ( + include | length > 0 or exclude | length > 0 + ) and not column_superset.keys() %} {%- set relations_string -%} {%- for relation in relations -%} {{ relation.name }} @@ -81,13 +101,21 @@ ( select - cast({{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }}) as {{ source_column_name }}, + cast( + {{ dbt_utils.string_literal(relation) }} + as {{ dbt_utils.type_string() }} + ) as {{ source_column_name }}, {% for col_name in ordered_column_names -%} {%- set col = column_superset[col_name] %} {%- set col_type = column_override.get(col.column, col.data_type) %} - {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %} - cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%} + {%- set col_name = ( + adapter.quote(col_name) + if col_name in relation_columns[relation] + else "null" + ) %} + cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} + {% if not loop.last %},{% endif -%} {%- endfor %} diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index 8821aad2..2a46c7f0 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -11,69 +11,96 @@ Arguments: field_name: Destination table column name for the source table column names. value_name: Destination table column name for the pivoted values #} - -{% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} - {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name, table)) }} +{% macro unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none +) -%} + {{ + return( + adapter.dispatch("unpivot", "dbt_utils")( + relation, cast_to, exclude, remove, field_name, value_name, table + ) + ) + }} {% endmacro %} -{% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} +{% macro default__unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none +) -%} {% if table %} - {%- set error_message = ' - Warning: the `unpivot` macro no longer accepts a `table` parameter. \ - This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ - The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} + {%- set error_message = " Warning: the `unpivot` macro no longer accepts a `table` parameter. \ This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name + ) -%} {%- do exceptions.warn(error_message) -%} {% endif %} {% if relation and table %} - {{ exceptions.raise_compiler_error("Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`).") }} - {% elif not relation and table %} - {% set relation=table %} + {{ + exceptions.raise_compiler_error( + "Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`)." + ) + }} + {% elif not relation and table %} {% set relation = table %} {% elif not relation and not table %} - {{ exceptions.raise_compiler_error("Error: argument `relation` is required for `unpivot` macro.") }} + {{ + exceptions.raise_compiler_error( + "Error: argument `relation` is required for `unpivot` macro." + ) + }} {% endif %} - {%- set exclude = exclude if exclude is not none else [] %} - {%- set remove = remove if remove is not none else [] %} - - {%- set include_cols = [] %} + {%- set exclude = exclude if exclude is not none else [] %} + {%- set remove = remove if remove is not none else [] %} - {%- set table_columns = {} %} + {%- set include_cols = [] %} - {%- do table_columns.update({relation: []}) %} + {%- set table_columns = {} %} - {%- do dbt_utils._is_relation(relation, 'unpivot') -%} - {%- do dbt_utils._is_ephemeral(relation, 'unpivot') -%} - {%- set cols = adapter.get_columns_in_relation(relation) %} + {%- do table_columns.update({relation: []}) %} - {%- for col in cols -%} - {%- if col.column.lower() not in remove|map('lower') and col.column.lower() not in exclude|map('lower') -%} - {% do include_cols.append(col) %} - {%- endif %} - {%- endfor %} + {%- do dbt_utils._is_relation(relation, "unpivot") -%} + {%- do dbt_utils._is_ephemeral(relation, "unpivot") -%} + {%- set cols = adapter.get_columns_in_relation(relation) %} + {%- for col in cols -%} + {%- if col.column.lower() not in remove | map( + "lower" + ) and col.column.lower() not in exclude | map("lower") -%} + {% do include_cols.append(col) %} + {%- endif %} + {%- endfor %} - {%- for col in include_cols -%} - select - {%- for exclude_col in exclude %} - {{ exclude_col }}, - {%- endfor %} + {%- for col in include_cols -%} + select + {%- for exclude_col in exclude %} {{ exclude_col }}, {%- endfor %} - cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, - cast( {% if col.data_type == 'boolean' %} - {{ dbt_utils.cast_bool_to_text(col.column) }} - {% else %} - {{ col.column }} - {% endif %} - as {{ cast_to }}) as {{ value_name }} + cast( + '{{ col.column }}' as {{ dbt_utils.type_string() }} + ) as {{ field_name }}, + cast( + {% if col.data_type == "boolean" %} + {{ dbt_utils.cast_bool_to_text(col.column) }} + {% else %}{{ col.column }} + {% endif %} as {{ cast_to }} + ) as {{ value_name }} - from {{ relation }} + from {{ relation }} - {% if not loop.last -%} - union all - {% endif -%} - {%- endfor -%} + {% if not loop.last -%} + union all + {% endif -%} + {%- endfor -%} {%- endmacro %} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index 43a58cc4..faafff86 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -1,27 +1,27 @@ {% macro get_url_host(field) -%} - {{ return(adapter.dispatch('get_url_host', 'dbt_utils')(field)) }} + {{ return(adapter.dispatch("get_url_host", "dbt_utils")(field)) }} {% endmacro %} {% macro default__get_url_host(field) -%} -{%- set parsed = - dbt_utils.split_part( + {%- set parsed = dbt_utils.split_part( dbt_utils.split_part( dbt_utils.replace( dbt_utils.replace( - dbt_utils.replace(field, "'android-app://'", "''" - ), "'http://'", "''" - ), "'https://'", "''" - ), "'/'", 1 - ), "'?'", 1 - ) + dbt_utils.replace(field, "'android-app://'", "''"), + "'http://'", + "''", + ), + "'https://'", + "''", + ), + "'/'", + 1, + ), + "'?'", + 1, + ) -%} --%} - - - {{ dbt_utils.safe_cast( - parsed, - dbt_utils.type_string() - )}} + {{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} {%- endmacro %} diff --git a/macros/web/get_url_parameter.sql b/macros/web/get_url_parameter.sql index f363fcbb..9b3127ab 100644 --- a/macros/web/get_url_parameter.sql +++ b/macros/web/get_url_parameter.sql @@ -1,13 +1,19 @@ {% macro get_url_parameter(field, url_parameter) -%} - {{ return(adapter.dispatch('get_url_parameter', 'dbt_utils')(field, url_parameter)) }} + {{ + return( + adapter.dispatch("get_url_parameter", "dbt_utils")(field, url_parameter) + ) + }} {% endmacro %} {% macro default__get_url_parameter(field, url_parameter) -%} -{%- set formatted_url_parameter = "'" + url_parameter + "='" -%} + {%- set formatted_url_parameter = "'" + url_parameter + "='" -%} -{%- set split = dbt_utils.split_part(dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1) -%} + {%- set split = dbt_utils.split_part( + dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1 + ) -%} -nullif({{ split }},'') + nullif({{ split }}, '') {%- endmacro %} diff --git a/macros/web/get_url_path.sql b/macros/web/get_url_path.sql index 4d224277..ef5c2f14 100644 --- a/macros/web/get_url_path.sql +++ b/macros/web/get_url_path.sql @@ -1,13 +1,12 @@ {% macro get_url_path(field) -%} - {{ return(adapter.dispatch('get_url_path', 'dbt_utils')(field)) }} + {{ return(adapter.dispatch("get_url_path", "dbt_utils")(field)) }} {% endmacro %} {% macro default__get_url_path(field) -%} - {%- set stripped_url = - dbt_utils.replace( - dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''") - -%} + {%- set stripped_url = dbt_utils.replace( + dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''" + ) -%} {%- set first_slash_pos -%} coalesce( @@ -16,19 +15,15 @@ ) {%- endset -%} - {%- set parsed_path = - dbt_utils.split_part( - dbt_utils.right( - stripped_url, - dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos - ), - "'?'", 1 - ) - -%} + {%- set parsed_path = dbt_utils.split_part( + dbt_utils.right( + stripped_url, + dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos, + ), + "'?'", + 1, + ) -%} + + {{ dbt_utils.safe_cast(parsed_path, dbt_utils.type_string()) }} - {{ dbt_utils.safe_cast( - parsed_path, - dbt_utils.type_string() - )}} - {%- endmacro %}