diff --git a/integration_tests/macros/assert_equal_values.sql b/integration_tests/macros/assert_equal_values.sql index d4f02618..e9eaa637 100644 --- a/integration_tests/macros/assert_equal_values.sql +++ b/integration_tests/macros/assert_equal_values.sql @@ -1,11 +1,10 @@ {% macro assert_equal_values(actual_object, expected_object) %} -{% if not execute %} + {% if not execute %} {# pass #} + {% elif actual_object != expected_object %} -{% elif actual_object != expected_object %} - - {% set msg %} + {% set msg %} Expected did not match actual ----------- @@ -18,15 +17,13 @@ ----------- --->{{ expected_object }}<--- - {% endset %} - - {{ log(msg, info=True) }} + {% endset %} - select 'fail' + {{ log(msg, info=True) }} -{% else %} + select 'fail' - select 'ok' {{ limit_zero() }} + {% else %} select 'ok' {{ limit_zero() }} -{% endif %} -{% endmacro %} \ No newline at end of file + {% endif %} +{% endmacro %} diff --git a/integration_tests/macros/limit_zero.sql b/integration_tests/macros/limit_zero.sql index 697849d1..f3dee37a 100644 --- a/integration_tests/macros/limit_zero.sql +++ b/integration_tests/macros/limit_zero.sql @@ -1,11 +1,7 @@ -{% macro my_custom_macro() %} - whatever -{% endmacro %} +{% macro my_custom_macro() %} whatever {% endmacro %} {% macro limit_zero() %} - {{ return(adapter.dispatch('limit_zero', 'dbt_utils')()) }} + {{ return(adapter.dispatch("limit_zero", "dbt_utils")()) }} {% endmacro %} -{% macro default__limit_zero() %} - {{ return('limit 0') }} -{% endmacro %} \ No newline at end of file +{% macro default__limit_zero() %} {{ return("limit 0") }} {% endmacro %} diff --git a/integration_tests/macros/tests.sql b/integration_tests/macros/tests.sql index 046fab3a..9ef0e19d 100644 --- a/integration_tests/macros/tests.sql +++ b/integration_tests/macros/tests.sql @@ -1,12 +1,11 @@ - {% test assert_equal(model, actual, expected) %} -select * from {{ model }} where {{ actual }} != {{ expected }} + select * from {{ model }} where {{ actual }} != {{ expected }} {% endtest %} {% test not_empty_string(model, column_name) %} -select * from {{ model }} where {{ column_name }} = '' + select * from {{ model }} where {{ column_name }} = '' {% endtest %} diff --git a/integration_tests/models/cross_db_utils/test_any_value.sql b/integration_tests/models/cross_db_utils/test_any_value.sql index 9b27ed1d..6781b41f 100644 --- a/integration_tests/models/cross_db_utils/test_any_value.sql +++ b/integration_tests/models/cross_db_utils/test_any_value.sql @@ -1,19 +1,26 @@ -with some_model as ( - select 1 as id, 'abc' as key_name, 'dbt' as static_col union all - select 2 as id, 'abc' as key_name, 'dbt' as static_col union all - select 3 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 4 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 5 as id, 'jkl' as key_name, 'dbt' as static_col union all - select 6 as id, 'xyz' as key_name, 'test' as static_col -), +with + some_model as ( + select 1 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 2 as id, 'abc' as key_name, 'dbt' as static_col + union all + select 3 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 4 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 5 as id, 'jkl' as key_name, 'dbt' as static_col + union all + select 6 as id, 'xyz' as key_name, 'test' as static_col + ), -final as ( - select - key_name, - {{ dbt_utils.any_value('static_col') }} as static_col, - count(id) as num_rows - from some_model - group by key_name -) + final as ( + select + key_name, + {{ dbt_utils.any_value("static_col") }} as static_col, + count(id) as num_rows + from some_model + group by key_name + ) -select * from final \ No newline at end of file +select * +from final diff --git a/integration_tests/models/cross_db_utils/test_bool_or.sql b/integration_tests/models/cross_db_utils/test_bool_or.sql index 7375d1e4..1d6f4ad5 100644 --- a/integration_tests/models/cross_db_utils/test_bool_or.sql +++ b/integration_tests/models/cross_db_utils/test_bool_or.sql @@ -1,5 +1,3 @@ -select - key, - {{ dbt_utils.bool_or('val1 = val2') }} as value -from {{ ref('data_bool_or' )}} -group by key \ No newline at end of file +select key, {{ dbt_utils.bool_or("val1 = val2") }} as value +from {{ ref("data_bool_or") }} +group by key diff --git a/integration_tests/models/cross_db_utils/test_concat.sql b/integration_tests/models/cross_db_utils/test_concat.sql index 9efeb748..c48697a2 100644 --- a/integration_tests/models/cross_db_utils/test_concat.sql +++ b/integration_tests/models/cross_db_utils/test_concat.sql @@ -1,12 +1,5 @@ +with data as (select * from {{ ref("data_concat") }}) -with data as ( - - select * from {{ ref('data_concat') }} - -) - -select - {{ dbt_utils.concat(['input_1', 'input_2']) }} as actual, - output as expected +select {{ dbt_utils.concat(["input_1", "input_2"]) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp.sql b/integration_tests/models/cross_db_utils/test_current_timestamp.sql index 9a777f28..8d5e99df 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp.sql @@ -1,6 +1,4 @@ - -- how can we test this better? select {{ dbt_utils.current_timestamp() }} as actual, {{ dbt_utils.current_timestamp() }} as expected - diff --git a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql index 55bc8e23..b3607bb9 100644 --- a/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql +++ b/integration_tests/models/cross_db_utils/test_current_timestamp_in_utc.sql @@ -1,5 +1,4 @@ - -- how can we test this better? select {{ dbt_utils.current_timestamp_in_utc() }} as actual, - {{ dbt_utils.current_timestamp_in_utc() }} as expected \ No newline at end of file + {{ dbt_utils.current_timestamp_in_utc() }} as expected diff --git a/integration_tests/models/cross_db_utils/test_date_trunc.sql b/integration_tests/models/cross_db_utils/test_date_trunc.sql index bac21fed..307495ee 100644 --- a/integration_tests/models/cross_db_utils/test_date_trunc.sql +++ b/integration_tests/models/cross_db_utils/test_date_trunc.sql @@ -1,12 +1,7 @@ - -with data as ( - - select * from {{ ref('data_date_trunc') }} - -) +with data as (select * from {{ ref("data_date_trunc") }}) select - cast({{dbt_utils.date_trunc('day', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("day", "updated_at") }} as date) as actual, day as expected from data @@ -14,7 +9,7 @@ from data union all select - cast({{ dbt_utils.date_trunc('month', 'updated_at') }} as date) as actual, + cast({{ dbt_utils.date_trunc("month", "updated_at") }} as date) as actual, month as expected from data diff --git a/integration_tests/models/cross_db_utils/test_dateadd.sql b/integration_tests/models/cross_db_utils/test_dateadd.sql index 3e199d86..7adacfa8 100644 --- a/integration_tests/models/cross_db_utils/test_dateadd.sql +++ b/integration_tests/models/cross_db_utils/test_dateadd.sql @@ -1,16 +1,31 @@ - -with data as ( - - select * from {{ ref('data_dateadd') }} - -) +with data as (select * from {{ ref("data_dateadd") }}) select case - when datepart = 'hour' then cast({{ dbt_utils.dateadd('hour', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'day' then cast({{ dbt_utils.dateadd('day', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'month' then cast({{ dbt_utils.dateadd('month', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) - when datepart = 'year' then cast({{ dbt_utils.dateadd('year', 'interval_length', 'from_time') }} as {{dbt_utils.type_timestamp()}}) + when datepart = 'hour' + then + cast( + {{ dbt_utils.dateadd("hour", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'day' + then + cast( + {{ dbt_utils.dateadd("day", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'month' + then + cast( + {{ dbt_utils.dateadd("month", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) + when datepart = 'year' + then + cast( + {{ dbt_utils.dateadd("year", "interval_length", "from_time") }} + as {{ dbt_utils.type_timestamp() }} + ) else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_datediff.sql b/integration_tests/models/cross_db_utils/test_datediff.sql index a2340456..bad3cbf1 100644 --- a/integration_tests/models/cross_db_utils/test_datediff.sql +++ b/integration_tests/models/cross_db_utils/test_datediff.sql @@ -1,20 +1,22 @@ - -with data as ( - - select * from {{ ref('data_datediff') }} - -) +with data as (select * from {{ ref("data_datediff") }}) select case - when datepart = 'second' then {{ dbt_utils.datediff('first_date', 'second_date', 'second') }} - when datepart = 'minute' then {{ dbt_utils.datediff('first_date', 'second_date', 'minute') }} - when datepart = 'hour' then {{ dbt_utils.datediff('first_date', 'second_date', 'hour') }} - when datepart = 'day' then {{ dbt_utils.datediff('first_date', 'second_date', 'day') }} - when datepart = 'week' then {{ dbt_utils.datediff('first_date', 'second_date', 'week') }} - when datepart = 'month' then {{ dbt_utils.datediff('first_date', 'second_date', 'month') }} - when datepart = 'year' then {{ dbt_utils.datediff('first_date', 'second_date', 'year') }} + when datepart = 'second' + then {{ dbt_utils.datediff("first_date", "second_date", "second") }} + when datepart = 'minute' + then {{ dbt_utils.datediff("first_date", "second_date", "minute") }} + when datepart = 'hour' + then {{ dbt_utils.datediff("first_date", "second_date", "hour") }} + when datepart = 'day' + then {{ dbt_utils.datediff("first_date", "second_date", "day") }} + when datepart = 'week' + then {{ dbt_utils.datediff("first_date", "second_date", "week") }} + when datepart = 'month' + then {{ dbt_utils.datediff("first_date", "second_date", "month") }} + when datepart = 'year' + then {{ dbt_utils.datediff("first_date", "second_date", "year") }} else null end as actual, result as expected @@ -22,14 +24,77 @@ select from data -- Also test correct casting of literal values. - -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "microsecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "millisecond") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter") }} as actual, 1 as expected -union all select {{ dbt_utils.datediff("'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year") }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "microsecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", + "'2000-01-01 00:00:00.000000'", + "millisecond", + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "second" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "minute" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "hour" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "day" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-03 00:00:00.000000'", "week" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "month" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "quarter" + ) + }} as actual, 1 as expected +union all +select + {{ + dbt_utils.datediff( + "'1999-12-31 23:59:59.999999'", "'2000-01-01 00:00:00.000000'", "year" + ) + }} as actual, 1 as expected diff --git a/integration_tests/models/cross_db_utils/test_hash.sql b/integration_tests/models/cross_db_utils/test_hash.sql index 8f017780..5f0cca83 100644 --- a/integration_tests/models/cross_db_utils/test_hash.sql +++ b/integration_tests/models/cross_db_utils/test_hash.sql @@ -1,12 +1,5 @@ +with data as (select * from {{ ref("data_hash") }}) -with data as ( - - select * from {{ ref('data_hash') }} - -) - -select - {{ dbt_utils.hash('input_1') }} as actual, - output as expected +select {{ dbt_utils.hash("input_1") }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_last_day.sql b/integration_tests/models/cross_db_utils/test_last_day.sql index fee3a823..080e286b 100644 --- a/integration_tests/models/cross_db_utils/test_last_day.sql +++ b/integration_tests/models/cross_db_utils/test_last_day.sql @@ -1,15 +1,13 @@ - -with data as ( - - select * from {{ ref('data_last_day') }} - -) +with data as (select * from {{ ref("data_last_day") }}) select case - when date_part = 'month' then {{ dbt_utils.last_day('date_day', 'month') }} - when date_part = 'quarter' then {{ dbt_utils.last_day('date_day', 'quarter') }} - when date_part = 'year' then {{ dbt_utils.last_day('date_day', 'year') }} + when date_part = 'month' + then {{ dbt_utils.last_day("date_day", "month") }} + when date_part = 'quarter' + then {{ dbt_utils.last_day("date_day", "quarter") }} + when date_part = 'year' + then {{ dbt_utils.last_day("date_day", "year") }} else null end as actual, result as expected diff --git a/integration_tests/models/cross_db_utils/test_length.sql b/integration_tests/models/cross_db_utils/test_length.sql index f4ef099c..d6352d05 100644 --- a/integration_tests/models/cross_db_utils/test_length.sql +++ b/integration_tests/models/cross_db_utils/test_length.sql @@ -1,12 +1,5 @@ -with data as ( +with data as (select * from {{ ref("data_length") }}) - select * from {{ ref('data_length') }} +select {{ dbt_utils.length("expression") }} as actual, output as expected -) - -select - - {{ dbt_utils.length('expression') }} as actual, - output as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_listagg.sql b/integration_tests/models/cross_db_utils/test_listagg.sql index 006948de..27d96745 100644 --- a/integration_tests/models/cross_db_utils/test_listagg.sql +++ b/integration_tests/models/cross_db_utils/test_listagg.sql @@ -1,69 +1,63 @@ -with data as ( - - select * from {{ ref('data_listagg') }} - -), - -data_output as ( - - select * from {{ ref('data_listagg_output') }} - -), - -calculate as ( - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col") }} as actual, - 'bottom_ordered' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "'_|_'", "order by order_col", 2) }} as actual, - 'bottom_ordered_limited' as version - from data - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text', "', '") }} as actual, - 'comma_whitespace_unordered' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('DISTINCT string_text', "','") }} as actual, - 'distinct_comma' as version - from data - where group_col = 3 - group by group_col - - union all - - select - group_col, - {{ dbt_utils.listagg('string_text') }} as actual, - 'no_params' as version - from data - where group_col = 3 - group by group_col - -) - -select - calculate.actual, - data_output.expected +with + data as (select * from {{ ref("data_listagg") }}), + + data_output as (select * from {{ ref("data_listagg_output") }}), + + calculate as ( + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col") }} + as actual, + 'bottom_ordered' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "'_|_'", "order by order_col", 2) }} + as actual, + 'bottom_ordered_limited' as version + from data + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text", "', '") }} as actual, + 'comma_whitespace_unordered' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("DISTINCT string_text", "','") }} as actual, + 'distinct_comma' as version + from data + where group_col = 3 + group by group_col + + union all + + select + group_col, + {{ dbt_utils.listagg("string_text") }} as actual, + 'no_params' as version + from data + where group_col = 3 + group by group_col + + ) + +select calculate.actual, data_output.expected from calculate -left join data_output -on calculate.group_col = data_output.group_col -and calculate.version = data_output.version \ No newline at end of file +left join + data_output + on calculate.group_col = data_output.group_col + and calculate.version = data_output.version diff --git a/integration_tests/models/cross_db_utils/test_position.sql b/integration_tests/models/cross_db_utils/test_position.sql index b09be66d..d1785f14 100644 --- a/integration_tests/models/cross_db_utils/test_position.sql +++ b/integration_tests/models/cross_db_utils/test_position.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_position') }} - -) +with data as (select * from {{ ref("data_position") }}) select - {{ dbt_utils.position('substring_text', 'string_text') }} as actual, + {{ dbt_utils.position("substring_text", "string_text") }} as actual, result as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_replace.sql b/integration_tests/models/cross_db_utils/test_replace.sql index 6c8a8aa1..0e413c8a 100644 --- a/integration_tests/models/cross_db_utils/test_replace.sql +++ b/integration_tests/models/cross_db_utils/test_replace.sql @@ -1,18 +1,19 @@ -with data as ( +with + data as ( - select - - *, - coalesce(search_chars, '') as old_chars, - coalesce(replace_chars, '') as new_chars - - from {{ ref('data_replace') }} + select -) + *, + coalesce(search_chars, '') as old_chars, + coalesce(replace_chars, '') as new_chars + + from {{ ref("data_replace") }} + + ) select - {{ dbt_utils.replace('string_text', 'old_chars', 'new_chars') }} as actual, + {{ dbt_utils.replace("string_text", "old_chars", "new_chars") }} as actual, result as expected from data diff --git a/integration_tests/models/cross_db_utils/test_right.sql b/integration_tests/models/cross_db_utils/test_right.sql index eaad4f4f..de2df7dc 100644 --- a/integration_tests/models/cross_db_utils/test_right.sql +++ b/integration_tests/models/cross_db_utils/test_right.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ ref('data_right') }} - -) +with data as (select * from {{ ref("data_right") }}) select - {{ dbt_utils.right('string_text', 'length_expression') }} as actual, + {{ dbt_utils.right("string_text", "length_expression") }} as actual, coalesce(output, '') as expected -from data \ No newline at end of file +from data diff --git a/integration_tests/models/cross_db_utils/test_safe_cast.sql b/integration_tests/models/cross_db_utils/test_safe_cast.sql index 8ed74fc3..95a3acde 100644 --- a/integration_tests/models/cross_db_utils/test_safe_cast.sql +++ b/integration_tests/models/cross_db_utils/test_safe_cast.sql @@ -1,12 +1,7 @@ - -with data as ( - - select * from {{ ref('data_safe_cast') }} - -) +with data as (select * from {{ ref("data_safe_cast") }}) select - {{ dbt_utils.safe_cast('field', dbt_utils.type_string()) }} as actual, + {{ dbt_utils.safe_cast("field", dbt_utils.type_string()) }} as actual, output as expected from data diff --git a/integration_tests/models/cross_db_utils/test_split_part.sql b/integration_tests/models/cross_db_utils/test_split_part.sql index 6a10327c..79b98da0 100644 --- a/integration_tests/models/cross_db_utils/test_split_part.sql +++ b/integration_tests/models/cross_db_utils/test_split_part.sql @@ -1,28 +1,20 @@ - -with data as ( - - select * from {{ ref('data_split_part') }} - -) +with data as (select * from {{ ref("data_split_part") }}) select - {{ dbt_utils.split_part('parts', 'split_on', 1) }} as actual, - result_1 as expected + {{ dbt_utils.split_part("parts", "split_on", 1) }} as actual, result_1 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 2) }} as actual, - result_2 as expected + {{ dbt_utils.split_part("parts", "split_on", 2) }} as actual, result_2 as expected from data union all select - {{ dbt_utils.split_part('parts', 'split_on', 3) }} as actual, - result_3 as expected + {{ dbt_utils.split_part("parts", "split_on", 3) }} as actual, result_3 as expected from data diff --git a/integration_tests/models/cross_db_utils/test_width_bucket.sql b/integration_tests/models/cross_db_utils/test_width_bucket.sql index dc2b8609..ef445ed1 100644 --- a/integration_tests/models/cross_db_utils/test_width_bucket.sql +++ b/integration_tests/models/cross_db_utils/test_width_bucket.sql @@ -1,12 +1,8 @@ - -with data as ( - - select * from {{ ref('data_width_bucket') }} - -) +with data as (select * from {{ ref("data_width_bucket") }}) select - {{ dbt_utils.width_bucket('amount', 'min_value', 'max_value', 'num_buckets') }} as actual, + {{ dbt_utils.width_bucket("amount", "min_value", "max_value", "num_buckets") }} + as actual, bucket as expected from data diff --git a/integration_tests/models/datetime/test_date_spine.sql b/integration_tests/models/datetime/test_date_spine.sql index fa4ae52b..1bdf4dca 100644 --- a/integration_tests/models/datetime/test_date_spine.sql +++ b/integration_tests/models/datetime/test_date_spine.sql @@ -1,26 +1,27 @@ - -- snowflake doesn't like this as a view because the `generate_series` -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} + +with + date_spine as ( -{{ config(materialized='table') }} + {% if target.type == "postgres" %} + {{ + dbt_utils.date_spine( + "day", "'2018-01-01'::date", "'2018-01-10'::date" + ) + }} -with date_spine as ( + {% elif target.type == "bigquery" %} + select cast(date_day as date) as date_day + from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) - {% if target.type == 'postgres' %} - {{ dbt_utils.date_spine("day", "'2018-01-01'::date", "'2018-01-10'::date") }} - - {% elif target.type == 'bigquery' %} - select cast(date_day as date) as date_day - from ({{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }}) - - {% else %} - {{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} - {% endif %} + {% else %}{{ dbt_utils.date_spine("day", "'2018-01-01'", "'2018-01-10'") }} + {% endif %} -) + ) select date_day from date_spine - diff --git a/integration_tests/models/generic_tests/test_equal_column_subset.sql b/integration_tests/models/generic_tests/test_equal_column_subset.sql index b426bb63..7ecae303 100644 --- a/integration_tests/models/generic_tests/test_equal_column_subset.sql +++ b/integration_tests/models/generic_tests/test_equal_column_subset.sql @@ -1,9 +1,5 @@ -{{ config(materialized='ephemeral') }} +{{ config(materialized="ephemeral") }} -select +select first_name, last_name, email - first_name, - last_name, - email - -from {{ ref('data_people') }} +from {{ ref("data_people") }} diff --git a/integration_tests/models/generic_tests/test_equal_rowcount.sql b/integration_tests/models/generic_tests/test_equal_rowcount.sql index 01b1a549..621ccf61 100644 --- a/integration_tests/models/generic_tests/test_equal_rowcount.sql +++ b/integration_tests/models/generic_tests/test_equal_rowcount.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_equal_rowcount") }}) - select * from {{ ref('data_test_equal_rowcount') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_fewer_rows_than.sql b/integration_tests/models/generic_tests/test_fewer_rows_than.sql index c2ad4cbd..621fce6c 100644 --- a/integration_tests/models/generic_tests/test_fewer_rows_than.sql +++ b/integration_tests/models/generic_tests/test_fewer_rows_than.sql @@ -1,9 +1,4 @@ -with data as ( +with data as (select * from {{ ref("data_test_fewer_rows_than_table_1") }}) - select * from {{ ref('data_test_fewer_rows_than_table_1') }} - -) - -select - field -from data \ No newline at end of file +select field +from data diff --git a/integration_tests/models/generic_tests/test_recency.sql b/integration_tests/models/generic_tests/test_recency.sql index d300e572..0b0d286a 100644 --- a/integration_tests/models/generic_tests/test_recency.sql +++ b/integration_tests/models/generic_tests/test_recency.sql @@ -1,12 +1,12 @@ +{% if target.type == "postgres" %} -{% if target.type == 'postgres' %} - -select - {{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as today + select {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as today {% else %} -select - cast({{ dbt_utils.date_trunc('day', dbt_utils.current_timestamp()) }} as datetime) as today - + select + cast( + {{ dbt_utils.date_trunc("day", dbt_utils.current_timestamp()) }} as datetime + ) as today + {% endif %} diff --git a/integration_tests/models/geo/test_haversine_distance_km.sql b/integration_tests/models/geo/test_haversine_distance_km.sql index e8ca4818..fce70682 100644 --- a/integration_tests/models/geo/test_haversine_distance_km.sql +++ b/integration_tests/models/geo/test_haversine_distance_km.sql @@ -1,23 +1,20 @@ -with data as ( - select * from {{ ref('data_haversine_km') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='km' +with + data as (select * from {{ ref("data_haversine_km") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="km", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/geo/test_haversine_distance_mi.sql b/integration_tests/models/geo/test_haversine_distance_mi.sql index 53f8172c..9ff4a1f7 100644 --- a/integration_tests/models/geo/test_haversine_distance_mi.sql +++ b/integration_tests/models/geo/test_haversine_distance_mi.sql @@ -1,39 +1,36 @@ -with data as ( - select * from {{ ref('data_haversine_mi') }} -), -final as ( - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', - unit='mi' +with + data as (select * from {{ ref("data_haversine_mi") }}), + final as ( + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", + unit="mi", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data - union all + union all - select - output as expected, - cast( - {{ - dbt_utils.haversine_distance( - lat1='lat_1', - lon1='lon_1', - lat2='lat_2', - lon2='lon_2', + select + output as expected, + cast( + {{ + dbt_utils.haversine_distance( + lat1="lat_1", + lon1="lon_1", + lat2="lat_2", + lon2="lon_2", ) - }} as {{ dbt_utils.type_numeric() }} - ) as actual - from data -) -select - expected, - round(actual,0) as actual + }} as {{ dbt_utils.type_numeric() }} + ) as actual + from data + ) +select expected, round(actual, 0) as actual from final diff --git a/integration_tests/models/materializations/expected_insert_by_period.sql b/integration_tests/models/materializations/expected_insert_by_period.sql index 0a198b6b..0d7d68c1 100644 --- a/integration_tests/models/materializations/expected_insert_by_period.sql +++ b/integration_tests/models/materializations/expected_insert_by_period.sql @@ -1,10 +1,5 @@ -{{ - config( - materialized = 'view', - enabled=(target.type == 'redshift') - ) -}} +{{ config(materialized="view", enabled=(target.type == "redshift")) }} select * -from {{ ref('data_insert_by_period') }} +from {{ ref("data_insert_by_period") }} where id in (2, 3, 4, 5, 6) diff --git a/integration_tests/models/materializations/test_insert_by_period.sql b/integration_tests/models/materializations/test_insert_by_period.sql index 8cc396d3..b48d22f6 100644 --- a/integration_tests/models/materializations/test_insert_by_period.sql +++ b/integration_tests/models/materializations/test_insert_by_period.sql @@ -1,18 +1,16 @@ {{ - config( - materialized = 'insert_by_period', - period = 'month', - timestamp_field = 'created_at', - start_date = '2018-01-01', - stop_date = '2018-06-01', - enabled=(target.type == 'redshift') - ) + config( + materialized="insert_by_period", + period="month", + timestamp_field="created_at", + start_date="2018-01-01", + stop_date="2018-06-01", + enabled=(target.type == "redshift"), + ) }} -with events as ( - select * - from {{ ref('data_insert_by_period') }} - where __PERIOD_FILTER__ -) +with + events as (select * from {{ ref("data_insert_by_period") }} where __period_filter__) -select * from events +select * +from events diff --git a/integration_tests/models/sql/test_deduplicate.sql b/integration_tests/models/sql/test_deduplicate.sql index 81fe81e7..47649a61 100644 --- a/integration_tests/models/sql/test_deduplicate.sql +++ b/integration_tests/models/sql/test_deduplicate.sql @@ -1,22 +1,19 @@ with -source as ( - select * - from {{ ref('data_deduplicate') }} - where user_id = 1 -), + source as (select * from {{ ref("data_deduplicate") }} where user_id = 1), -deduped as ( + deduped as ( - {{ - dbt_utils.deduplicate( - ref('data_deduplicate'), - group_by='user_id', - order_by='version desc', - relation_alias="source" - ) | indent - }} + {{ + dbt_utils.deduplicate( + ref("data_deduplicate"), + group_by="user_id", + order_by="version desc", + relation_alias="source", + ) | indent + }} -) + ) -select * from deduped +select * +from deduped diff --git a/integration_tests/models/sql/test_generate_series.sql b/integration_tests/models/sql/test_generate_series.sql index 11370b7b..fd104edf 100644 --- a/integration_tests/models/sql/test_generate_series.sql +++ b/integration_tests/models/sql/test_generate_series.sql @@ -1,15 +1,10 @@ - -- snowflake doesn't like this as a view because the `generate_series` -- call creates a CTE called `unioned`, as does the `equality` generic test. -- Ideally, Snowflake would be smart enough to know that these CTE names are -- different, as they live in different relations. TODO: use a less common cte name +{{ config(materialized="table") }} -{{ config(materialized='table') }} - -with data as ( - - {{ dbt_utils.generate_series(10) }} - -) +with data as ({{ dbt_utils.generate_series(10) }}) -select generated_number from data +select generated_number +from data diff --git a/integration_tests/models/sql/test_get_column_values.sql b/integration_tests/models/sql/test_get_column_values.sql index 36214984..b3c5ba8f 100644 --- a/integration_tests/models/sql/test_get_column_values.sql +++ b/integration_tests/models/sql/test_get_column_values.sql @@ -1,29 +1,34 @@ +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values"), "field", default=[], order_by="field" +) %} -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values'), 'field', default=[], order_by="field") %} +{% if target.type == "snowflake" %} -{% if target.type == 'snowflake' %} + select + {% for val in column_values -%} -select - {% for val in column_values -%} + sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} + {%- if not loop.last %},{% endif -%} - sum(case when field = '{{ val }}' then 1 else 0 end) as count_{{ val }} - {%- if not loop.last %},{% endif -%} + {%- endfor %} - {%- endfor %} - -from {{ ref('data_get_column_values') }} + from {{ ref("data_get_column_values") }} {% else %} -select - {% for val in column_values -%} + select + {% for val in column_values -%} - {{dbt_utils.safe_cast("sum(case when field = '" ~ val ~ "' then 1 else 0 end)", dbt_utils.type_string()) }} as count_{{ val }} - {%- if not loop.last %},{% endif -%} + {{ + dbt_utils.safe_cast( + "sum(case when field = '" ~ val ~ "' then 1 else 0 end)", + dbt_utils.type_string(), + ) + }} as count_{{ val }} {%- if not loop.last %},{% endif -%} - {%- endfor %} + {%- endfor %} -from {{ ref('data_get_column_values') }} + from {{ ref("data_get_column_values") }} {% endif %} diff --git a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql index 7b3ca72f..0690dd19 100644 --- a/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql +++ b/integration_tests/models/sql/test_get_filtered_columns_in_relation.sql @@ -1,16 +1,21 @@ -{% set exclude_field = 'field_1' %} -{% set column_names = dbt_utils.get_filtered_columns_in_relation(from= ref('data_filtered_columns_in_relation'), except=[exclude_field]) %} +{% set exclude_field = "field_1" %} +{% set column_names = dbt_utils.get_filtered_columns_in_relation( + from=ref("data_filtered_columns_in_relation"), except=[exclude_field] +) %} -with data as ( +with + data as ( - select + select - {% for column_name in column_names %} - max({{ column_name }}) as {{ column_name }} {% if not loop.last %},{% endif %} - {% endfor %} + {% for column_name in column_names %} + max({{ column_name }}) as {{ column_name }} + {% if not loop.last %},{% endif %} + {% endfor %} - from {{ ref('data_filtered_columns_in_relation') }} + from {{ ref("data_filtered_columns_in_relation") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_get_relations_by_pattern.sql b/integration_tests/models/sql/test_get_relations_by_pattern.sql index 52ab07a2..653f3598 100644 --- a/integration_tests/models/sql/test_get_relations_by_pattern.sql +++ b/integration_tests/models/sql/test_get_relations_by_pattern.sql @@ -1,16 +1,11 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} -{% set relations = dbt_utils.get_relations_by_pattern(target.schema ~ '%', 'data_events_%') %} +{% set relations = dbt_utils.get_relations_by_pattern( + target.schema ~ "%", "data_events_%" +) %} -with unioned as ( - - {{ dbt_utils.union_relations(relations) }} - -) +with unioned as ({{ dbt_utils.union_relations(relations) }}) -select - - user_id, - event +select user_id, event from unioned diff --git a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql index 481f233e..6e968e79 100644 --- a/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql +++ b/integration_tests/models/sql/test_get_relations_by_prefix_and_union.sql @@ -1,4 +1,4 @@ -{{ config(materialized = 'table') }} +{{ config(materialized="table") }} -{% set relations = dbt_utils.get_relations_by_prefix(target.schema, 'data_events_') %} +{% set relations = dbt_utils.get_relations_by_prefix(target.schema, "data_events_") %} {{ dbt_utils.union_relations(relations) }} diff --git a/integration_tests/models/sql/test_groupby.sql b/integration_tests/models/sql/test_groupby.sql index 50a6ced7..a63f3a5c 100644 --- a/integration_tests/models/sql/test_groupby.sql +++ b/integration_tests/models/sql/test_groupby.sql @@ -1,24 +1,14 @@ -with test_data as ( - - select - - {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string() )}} as column_1, - {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string() )}} as column_2 - -), +with + test_data as ( -grouped as ( + select - select - *, - count(*) as total - - from test_data - {{ dbt_utils.group_by(2) }} - -) - -select * from grouped + {{ dbt_utils.safe_cast("'a'", dbt_utils.type_string()) }} as column_1, + {{ dbt_utils.safe_cast("'b'", dbt_utils.type_string()) }} as column_2 + ), + grouped as (select *, count(*) as total from test_data {{ dbt_utils.group_by(2) }}) +select * +from grouped diff --git a/integration_tests/models/sql/test_nullcheck_table.sql b/integration_tests/models/sql/test_nullcheck_table.sql index ea66c8ec..a2cec220 100644 --- a/integration_tests/models/sql/test_nullcheck_table.sql +++ b/integration_tests/models/sql/test_nullcheck_table.sql @@ -1,41 +1,30 @@ -{{ config( materialized = "table" ) }} - --- TO DO: remove if-statement - -{% set tbl = ref('data_nullcheck_table') %} - +{{ config(materialized="table") }} -with nulled as ( +-- TO DO: remove if-statement +{% set tbl = ref("data_nullcheck_table") %} - {{ dbt_utils.nullcheck_table(tbl) }} -) +with nulled as ({{ dbt_utils.nullcheck_table(tbl) }}) -{% if target.type == 'snowflake' %} +{% if target.type == "snowflake" %} -select - field_1::varchar as field_1, - field_2::varchar as field_2, - field_3::varchar as field_3 + select + field_1::varchar as field_1, + field_2::varchar as field_2, + field_3::varchar as field_3 -from nulled + from nulled {% else %} -select + select - {{ dbt_utils.safe_cast('field_1', - dbt_utils.type_string() - )}} as field_1, + {{ dbt_utils.safe_cast("field_1", dbt_utils.type_string()) }} as field_1, - {{ dbt_utils.safe_cast('field_2', - dbt_utils.type_string() - )}} as field_2, + {{ dbt_utils.safe_cast("field_2", dbt_utils.type_string()) }} as field_2, - {{ dbt_utils.safe_cast('field_3', - dbt_utils.type_string() - )}} as field_3 + {{ dbt_utils.safe_cast("field_3", dbt_utils.type_string()) }} as field_3 -from nulled + from nulled {% endif %} diff --git a/integration_tests/models/sql/test_pivot.sql b/integration_tests/models/sql/test_pivot.sql index a63e6e3b..dcc0d9a1 100644 --- a/integration_tests/models/sql/test_pivot.sql +++ b/integration_tests/models/sql/test_pivot.sql @@ -1,17 +1,10 @@ - -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE'] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue'] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} + {% set column_values = ["RED", "BLUE"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue"] %} {% set cmp = "=" %} {% endif %} -select - size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp) }} +select size, {{ dbt_utils.pivot("color", column_values, cmp=cmp) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_pivot_apostrophe.sql b/integration_tests/models/sql/test_pivot_apostrophe.sql index 792f4686..b1af6828 100644 --- a/integration_tests/models/sql/test_pivot_apostrophe.sql +++ b/integration_tests/models/sql/test_pivot_apostrophe.sql @@ -1,17 +1,12 @@ - -- TODO: How do we make this work nicely on Snowflake too? - -{% if target.type == 'snowflake' %} - {% set column_values = ['RED', 'BLUE', "BLUE'S"] %} - {% set cmp = 'ilike' %} -{% else %} - {% set column_values = ['red', 'blue', "blue's"] %} - {% set cmp = '=' %} +{% if target.type == "snowflake" %} + {% set column_values = ["RED", "BLUE", "BLUE'S"] %} {% set cmp = "ilike" %} +{% else %} {% set column_values = ["red", "blue", "blue's"] %} {% set cmp = "=" %} {% endif %} select size, - {{ dbt_utils.pivot('color', column_values, cmp=cmp, quote_identifiers=False) }} + {{ dbt_utils.pivot("color", column_values, cmp=cmp, quote_identifiers=False) }} -from {{ ref('data_pivot') }} +from {{ ref("data_pivot") }} group by size diff --git a/integration_tests/models/sql/test_safe_add.sql b/integration_tests/models/sql/test_safe_add.sql index af3894d0..2d6eb778 100644 --- a/integration_tests/models/sql/test_safe_add.sql +++ b/integration_tests/models/sql/test_safe_add.sql @@ -1,12 +1,5 @@ +with data as (select * from {{ ref("data_safe_add") }}) -with data as ( - - select * from {{ ref('data_safe_add') }} - -) - -select - {{ dbt_utils.safe_add('field_1', 'field_2', 'field_3') }} as actual, - expected +select {{ dbt_utils.safe_add("field_1", "field_2", "field_3") }} as actual, expected from data diff --git a/integration_tests/models/sql/test_star.sql b/integration_tests/models/sql/test_star.sql index 2092e16a..5932635c 100644 --- a/integration_tests/models/sql/test_star.sql +++ b/integration_tests/models/sql/test_star.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'field_3' %} +{% set exclude_field = "field_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref("data_star"), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_aggregate.sql b/integration_tests/models/sql/test_star_aggregate.sql index 9dcd7c2d..16b6c94b 100644 --- a/integration_tests/models/sql/test_star_aggregate.sql +++ b/integration_tests/models/sql/test_star_aggregate.sql @@ -1,16 +1,17 @@ -/*This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ +/* This test checks that column aliases aren't applied unless there's a prefix/suffix necessary, to ensure that GROUP BYs keep working*/ +{% set selected_columns = dbt_utils.star( + from=ref("data_star_aggregate"), except=["value_field"] +) %} -{% set selected_columns = dbt_utils.star(from=ref('data_star_aggregate'), except=['value_field']) %} +with + data as ( -with data as ( + select {{ selected_columns }}, sum(value_field) as value_field_sum - select - {{ selected_columns }}, - sum(value_field) as value_field_sum + from {{ ref("data_star_aggregate") }} + group by {{ selected_columns }} - from {{ ref('data_star_aggregate') }} - group by {{ selected_columns }} + ) -) - -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_star_prefix_suffix.sql b/integration_tests/models/sql/test_star_prefix_suffix.sql index ae1c64fe..5d66ad06 100644 --- a/integration_tests/models/sql/test_star_prefix_suffix.sql +++ b/integration_tests/models/sql/test_star_prefix_suffix.sql @@ -1,13 +1,19 @@ -{% set prefix_with = 'prefix_' if target.type != 'snowflake' else 'PREFIX_' %} -{% set suffix_with = '_suffix' if target.type != 'snowflake' else '_SUFFIX' %} +{% set prefix_with = "prefix_" if target.type != "snowflake" else "PREFIX_" %} +{% set suffix_with = "_suffix" if target.type != "snowflake" else "_SUFFIX" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), prefix=prefix_with, suffix=suffix_with) }} + select + {{ + dbt_utils.star( + from=ref("data_star"), prefix=prefix_with, suffix=suffix_with + ) + }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data \ No newline at end of file +select * +from data diff --git a/integration_tests/models/sql/test_star_uppercase.sql b/integration_tests/models/sql/test_star_uppercase.sql index 6179e691..f8656abe 100644 --- a/integration_tests/models/sql/test_star_uppercase.sql +++ b/integration_tests/models/sql/test_star_uppercase.sql @@ -1,13 +1,14 @@ -{% set exclude_field = 'FIELD_3' %} +{% set exclude_field = "FIELD_3" %} -with data as ( +with + data as ( - select - {{ dbt_utils.star(from=ref('data_star'), except=[exclude_field]) }} + select {{ dbt_utils.star(from=ref("data_star"), except=[exclude_field]) }} - from {{ ref('data_star') }} + from {{ ref("data_star") }} -) + ) -select * from data +select * +from data diff --git a/integration_tests/models/sql/test_surrogate_key.sql b/integration_tests/models/sql/test_surrogate_key.sql index fd90939f..8f694083 100644 --- a/integration_tests/models/sql/test_surrogate_key.sql +++ b/integration_tests/models/sql/test_surrogate_key.sql @@ -1,15 +1,12 @@ - -with data as ( - - select * from {{ ref('data_surrogate_key') }} - -) +with data as (select * from {{ ref("data_surrogate_key") }}) select - {{ dbt_utils.surrogate_key('column_1') }} as actual_column_1_only, + {{ dbt_utils.surrogate_key("column_1") }} as actual_column_1_only, expected_column_1_only, - {{ dbt_utils.surrogate_key('column_1', 'column_2', 'column_3') }} as actual_all_columns_arguments, - {{ dbt_utils.surrogate_key(['column_1', 'column_2', 'column_3']) }} as actual_all_columns_list, + {{ dbt_utils.surrogate_key("column_1", "column_2", "column_3") }} + as actual_all_columns_arguments, + {{ dbt_utils.surrogate_key(["column_1", "column_2", "column_3"]) }} + as actual_all_columns_list, expected_all_columns from data diff --git a/integration_tests/models/sql/test_union.sql b/integration_tests/models/sql/test_union.sql index 69836833..e1b5e34c 100644 --- a/integration_tests/models/sql/test_union.sql +++ b/integration_tests/models/sql/test_union.sql @@ -1,8 +1 @@ - -select - id, - name, - favorite_color - -from {{ ref('test_union_base') }} - +select id, name, favorite_color from {{ ref("test_union_base") }} diff --git a/integration_tests/models/sql/test_union_base.sql b/integration_tests/models/sql/test_union_base.sql index a51bd2b8..869baecd 100644 --- a/integration_tests/models/sql/test_union_base.sql +++ b/integration_tests/models/sql/test_union_base.sql @@ -1,5 +1 @@ - -{{ dbt_utils.union_relations([ - ref('data_union_table_1'), - ref('data_union_table_2')] -) }} +{{ dbt_utils.union_relations([ref("data_union_table_1"), ref("data_union_table_2")]) }} diff --git a/integration_tests/models/sql/test_unpivot.sql b/integration_tests/models/sql/test_unpivot.sql index 8cf1ad6e..a9081a6c 100644 --- a/integration_tests/models/sql/test_unpivot.sql +++ b/integration_tests/models/sql/test_unpivot.sql @@ -1,33 +1,29 @@ - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - remove=['name'], - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + remove=["name"], + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_bool.sql b/integration_tests/models/sql/test_unpivot_bool.sql index 26842c01..2642afaf 100644 --- a/integration_tests/models/sql/test_unpivot_bool.sql +++ b/integration_tests/models/sql/test_unpivot_bool.sql @@ -1,32 +1,28 @@ - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, - case - when '{{ target.name }}' = 'snowflake' then lower(prop) - else prop - end as prop, + case when '{{ target.name }}' = 'snowflake' then lower(prop) else prop end as prop, val -from ( - {{ dbt_utils.unpivot( - relation=ref('data_unpivot_bool'), - cast_to=dbt_utils.type_string(), - exclude=exclude, - field_name='prop', - value_name='val' - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + relation=ref("data_unpivot_bool"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + field_name="prop", + value_name="val", + ) + }} + ) as sbq diff --git a/integration_tests/models/sql/test_unpivot_original_api.sql b/integration_tests/models/sql/test_unpivot_original_api.sql index f6b9395a..83f6ddf6 100644 --- a/integration_tests/models/sql/test_unpivot_original_api.sql +++ b/integration_tests/models/sql/test_unpivot_original_api.sql @@ -1,32 +1,29 @@ - -- unpivot() was enhanced with 3 new parameters -- This test targets the original API. - -- snowflake messes with these tests pretty badly since the -- output of the macro considers the casing of the source -- table columns. Using some hacks here to get this to work, -- but we should consider lowercasing the unpivot macro output -- at some point in the future for consistency - -{% if target.name == 'snowflake' %} - {% set exclude = ['CUSTOMER_ID', 'CREATED_AT'] %} -{% else %} - {% set exclude = ['customer_id', 'created_at'] %} +{% if target.name == "snowflake" %} {% set exclude = ["CUSTOMER_ID", "CREATED_AT"] %} +{% else %} {% set exclude = ["customer_id", "created_at"] %} {% endif %} select customer_id, created_at, case - when '{{ target.name }}' = 'snowflake' then lower(FIELD_NAME) - else field_name + when '{{ target.name }}' = 'snowflake' then lower(field_name) else field_name end as field_name, value -from ( - {{ dbt_utils.unpivot( - table=ref('data_unpivot'), - cast_to=dbt_utils.type_string(), - exclude=exclude - ) }} -) as sbq +from + ( + {{ + dbt_utils.unpivot( + table=ref("data_unpivot"), + cast_to=dbt_utils.type_string(), + exclude=exclude, + ) + }} + ) as sbq diff --git a/integration_tests/models/web/test_url_host.sql b/integration_tests/models/web/test_url_host.sql index 06a9f69c..2e18e65d 100644 --- a/integration_tests/models/web/test_url_host.sql +++ b/integration_tests/models/web/test_url_host.sql @@ -1,12 +1,5 @@ -with data as ( - - select * from {{ref('data_url_host')}} - -) +with data as (select * from {{ ref("data_url_host") }}) -select +select {{ dbt_utils.get_url_host("original_url") }} as actual, parsed_url as expected - {{ dbt_utils.get_url_host('original_url') }} as actual, - parsed_url as expected - -from data \ No newline at end of file +from data diff --git a/integration_tests/models/web/test_url_path.sql b/integration_tests/models/web/test_url_path.sql index c03e5547..16f45ae0 100644 --- a/integration_tests/models/web/test_url_path.sql +++ b/integration_tests/models/web/test_url_path.sql @@ -1,12 +1,8 @@ -with data as ( - - select * from {{ref('data_url_path')}} - -) +with data as (select * from {{ ref("data_url_path") }}) select - coalesce({{ dbt_utils.get_url_path('original_url') }}, '') as actual, + coalesce({{ dbt_utils.get_url_path("original_url") }}, '') as actual, coalesce(parsed_path, '') as expected - -from data \ No newline at end of file + +from data diff --git a/integration_tests/models/web/test_urls.sql b/integration_tests/models/web/test_urls.sql index 60353dbb..321dce5c 100644 --- a/integration_tests/models/web/test_urls.sql +++ b/integration_tests/models/web/test_urls.sql @@ -1,20 +1,13 @@ - -with data as ( - - select * from {{ ref('data_urls') }} - -) +with data as (select * from {{ ref("data_urls") }}) select - {{ dbt_utils.get_url_parameter('url', 'utm_medium') }} as actual, - medium as expected + {{ dbt_utils.get_url_parameter("url", "utm_medium") }} as actual, medium as expected from data union all select - {{ dbt_utils.get_url_parameter('url', 'utm_source') }} as actual, - source as expected + {{ dbt_utils.get_url_parameter("url", "utm_source") }} as actual, source as expected from data diff --git a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql index 55d5400e..db0f8dc4 100644 --- a/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql +++ b/integration_tests/tests/assert_get_query_results_as_dict_objects_equal.sql @@ -1,81 +1,76 @@ -- depends_on: {{ ref('data_get_query_results_as_dict') }} - -{% set expected_dictionary={ - 'col_1': [1, 2, 3], - 'col_2': ['a', 'b', 'c'], - 'col_3': [True, False, none] +{% set expected_dictionary = { + "col_1": [1, 2, 3], + "col_2": ["a", "b", "c"], + "col_3": [True, False, none], } %} {#- Handle snowflake casing silliness -#} -{% if target.type == 'snowflake' %} -{% set expected_dictionary={ - 'COL_1': [1, 2, 3], - 'COL_2': ['a', 'b', 'c'], - 'COL_3': [True, False, none] -} %} +{% if target.type == "snowflake" %} + {% set expected_dictionary = { + "COL_1": [1, 2, 3], + "COL_2": ["a", "b", "c"], + "COL_3": [True, False, none], + } %} {% endif %} -{% set actual_dictionary=dbt_utils.get_query_results_as_dict( - "select * from " ~ ref('data_get_query_results_as_dict') ~ " order by 1" +{% set actual_dictionary = dbt_utils.get_query_results_as_dict( + "select * from " ~ ref("data_get_query_results_as_dict") ~ " order by 1" ) %} {#- For reasons that remain unclear, Jinja won't return True for actual_dictionary == expected_dictionary. Instead, we'll manually check that the values of these dictionaries are equivalent. -#} - -{% set ns = namespace( - pass=True, - err_msg = "" -) %} +{% set ns = namespace(pass=True, err_msg="") %} {% if execute %} -{#- Check that the dictionaries have the same keys -#} -{% set expected_keys=expected_dictionary.keys() | list | sort %} -{% set actual_keys=actual_dictionary.keys() | list | sort %} + {#- Check that the dictionaries have the same keys -#} + {% set expected_keys = expected_dictionary.keys() | list | sort %} + {% set actual_keys = actual_dictionary.keys() | list | sort %} -{% if expected_keys != actual_keys %} - {% set ns.pass=False %} - {% set ns.err_msg %} + {% if expected_keys != actual_keys %} + {% set ns.pass = False %} + {% set ns.err_msg %} The two dictionaries have different keys: expected_dictionary has keys: {{ expected_keys }} actual_dictionary has keys: {{ actual_keys }} - {% endset %} + {% endset %} -{% else %} + {% else %} -{% for key, value in expected_dictionary.items() %} - {% set expected_length=expected_dictionary[key] | length %} - {% set actual_length=actual_dictionary[key] | length %} + {% for key, value in expected_dictionary.items() %} + {% set expected_length = expected_dictionary[key] | length %} + {% set actual_length = actual_dictionary[key] | length %} - {% if expected_length != actual_length %} - {% set ns.pass=False %} - {% set ns.err_msg %} + {% if expected_length != actual_length %} + {% set ns.pass = False %} + {% set ns.err_msg %} The {{ key }} column has different lengths: expected_dictionary[{{ key }}] has length {{ expected_length }} actual_dictionary[{{ key }}] has length {{ actual_length }} - {% endset %} + {% endset %} - {% else %} + {% else %} - {% for i in range(value | length) %} - {% set expected_value=expected_dictionary[key][i] %} - {% set actual_value=actual_dictionary[key][i] %} - {% if expected_value != actual_value %} - {% set ns.pass=False %} - {% set ns.err_msg %} + {% for i in range(value | length) %} + {% set expected_value = expected_dictionary[key][i] %} + {% set actual_value = actual_dictionary[key][i] %} + {% if expected_value != actual_value %} + {% set ns.pass = False %} + {% set ns.err_msg %} The {{ key }} column has differing values: expected_dictionary[{{ key }}][{{ i }}] == {{ expected_value }} actual_dictionary[{{ key }}][{{ i }}] == {{ actual_value }} - {% endset %} + {% endset %} + {% endif %} + {% endfor %} {% endif %} - {% endfor %} - {% endif %} -{% endfor %} + {% endfor %} -{% endif %} + {% endif %} -{{ log(ns.err_msg, info=True) }} -select 1 as col_name {% if ns.pass %} {{ limit_zero() }} {% endif %} + {{ log(ns.err_msg, info=True) }} + select 1 as col_name {% if ns.pass %} {{ limit_zero() }} {% endif %} {% endif %} diff --git a/integration_tests/tests/jinja_helpers/test_slugify.sql b/integration_tests/tests/jinja_helpers/test_slugify.sql index c0839f59..9b72b797 100644 --- a/integration_tests/tests/jinja_helpers/test_slugify.sql +++ b/integration_tests/tests/jinja_helpers/test_slugify.sql @@ -1,4 +1,4 @@ -{% if dbt_utils.slugify('!Hell0 world-hi') == 'hell0_world_hi' %} +{% if dbt_utils.slugify("!Hell0 world-hi") == "hell0_world_hi" %} {# Return 0 rows for the test to pass #} select 1 as col_name {{ limit_zero() }} {% else %} diff --git a/integration_tests/tests/sql/test_get_column_values_use_default.sql b/integration_tests/tests/sql/test_get_column_values_use_default.sql index 7f9fa3a4..e7d64be0 100644 --- a/integration_tests/tests/sql/test_get_column_values_use_default.sql +++ b/integration_tests/tests/sql/test_get_column_values_use_default.sql @@ -1,26 +1,40 @@ +{% set column_values = dbt_utils.get_column_values( + ref("data_get_column_values_dropped"), + "field", + default=["y", "z"], + order_by="field", +) %} -{% set column_values = dbt_utils.get_column_values(ref('data_get_column_values_dropped'), 'field', default=['y', 'z'], order_by="field") %} +with + expected as ( + select + {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} + as expected_column_value + union all + select + {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} + as expected_column_value + ), -with expected as ( - select {{ dbt_utils.safe_cast("'y'", dbt_utils.type_string()) }} as expected_column_value union all - select {{ dbt_utils.safe_cast("'z'", dbt_utils.type_string()) }} as expected_column_value -), + actual as ( -actual as ( - {% for val in column_values %} - select {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} as actual_column_value + select + {{ dbt_utils.safe_cast("'" ~ val ~ "'", dbt_utils.type_string()) }} + as actual_column_value {% if not loop.last %} union all {% endif %} {% endfor %} -), + ), -failures as ( - select * from actual - where actual.actual_column_value not in ( - select expected.expected_column_value from expected + failures as ( + select * + from actual + where + actual.actual_column_value + not in (select expected.expected_column_value from expected) ) -) -select * from failures \ No newline at end of file +select * +from failures diff --git a/macros/cross_db_utils/_is_ephemeral.sql b/macros/cross_db_utils/_is_ephemeral.sql index 34c70db9..1bf39708 100644 --- a/macros/cross_db_utils/_is_ephemeral.sql +++ b/macros/cross_db_utils/_is_ephemeral.sql @@ -1,10 +1,9 @@ {% macro _is_ephemeral(obj, macro) %} {%- if obj.is_cte -%} - {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix('') %} + {% set ephemeral_prefix = api.Relation.add_ephemeral_prefix("") %} {% if obj.name.startswith(ephemeral_prefix) %} - {% set model_name = obj.name[(ephemeral_prefix|length):] %} - {% else %} - {% set model_name = obj.name %} + {% set model_name = obj.name[(ephemeral_prefix | length) :] %} + {% else %} {% set model_name = obj.name %} {%- endif -%} {% set error_message %} The `{{ macro }}` macro cannot be used with ephemeral models, as it relies on the information schema. diff --git a/macros/cross_db_utils/_is_relation.sql b/macros/cross_db_utils/_is_relation.sql index bee6db97..d668c229 100644 --- a/macros/cross_db_utils/_is_relation.sql +++ b/macros/cross_db_utils/_is_relation.sql @@ -1,5 +1,13 @@ {% macro _is_relation(obj, macro) %} - {%- if not (obj is mapping and obj.get('metadata', {}).get('type', '').endswith('Relation')) -%} - {%- do exceptions.raise_compiler_error("Macro " ~ macro ~ " expected a Relation but received the value: " ~ obj) -%} + {%- if not ( + obj is mapping + and obj.get("metadata", {}).get("type", "").endswith("Relation") + ) -%} + {%- do exceptions.raise_compiler_error( + "Macro " + ~ macro + ~ " expected a Relation but received the value: " + ~ obj + ) -%} {%- endif -%} {% endmacro %} diff --git a/macros/cross_db_utils/any_value.sql b/macros/cross_db_utils/any_value.sql index 78cb75ba..595d32d0 100644 --- a/macros/cross_db_utils/any_value.sql +++ b/macros/cross_db_utils/any_value.sql @@ -1,17 +1,13 @@ {% macro any_value(expression) -%} - {{ return(adapter.dispatch('any_value', 'dbt_utils') (expression)) }} + {{ return(adapter.dispatch("any_value", "dbt_utils")(expression)) }} {% endmacro %} -{% macro default__any_value(expression) -%} - - any_value({{ expression }}) - -{%- endmacro %} +{% macro default__any_value(expression) -%} any_value({{ expression }}) {%- endmacro %} {% macro postgres__any_value(expression) -%} {#- /*Postgres doesn't support any_value, so we're using min() to get the same result*/ -#} min({{ expression }}) - -{%- endmacro %} \ No newline at end of file + +{%- endmacro %} diff --git a/macros/cross_db_utils/bool_or.sql b/macros/cross_db_utils/bool_or.sql index ce0a6857..04716878 100644 --- a/macros/cross_db_utils/bool_or.sql +++ b/macros/cross_db_utils/bool_or.sql @@ -1,24 +1,12 @@ {% macro bool_or(expression) -%} - {{ return(adapter.dispatch('bool_or', 'dbt_utils') (expression)) }} + {{ return(adapter.dispatch("bool_or", "dbt_utils")(expression)) }} {% endmacro %} -{% macro default__bool_or(expression) -%} - - bool_or({{ expression }}) - -{%- endmacro %} +{% macro default__bool_or(expression) -%} bool_or({{ expression }}) {%- endmacro %} -{% macro snowflake__bool_or(expression) -%} - - boolor_agg({{ expression }}) - -{%- endmacro %} +{% macro snowflake__bool_or(expression) -%} boolor_agg({{ expression }}) {%- endmacro %} -{% macro bigquery__bool_or(expression) -%} - - logical_or({{ expression }}) - -{%- endmacro %} \ No newline at end of file +{% macro bigquery__bool_or(expression) -%} logical_or({{ expression }}) {%- endmacro %} diff --git a/macros/cross_db_utils/cast_bool_to_text.sql b/macros/cross_db_utils/cast_bool_to_text.sql index 5efa9d60..030d5681 100644 --- a/macros/cross_db_utils/cast_bool_to_text.sql +++ b/macros/cross_db_utils/cast_bool_to_text.sql @@ -1,5 +1,5 @@ {% macro cast_bool_to_text(field) %} - {{ adapter.dispatch('cast_bool_to_text', 'dbt_utils') (field) }} + {{ adapter.dispatch("cast_bool_to_text", "dbt_utils")(field) }} {% endmacro %} @@ -9,7 +9,6 @@ {% macro redshift__cast_bool_to_text(field) %} case - when {{ field }} is true then 'true' - when {{ field }} is false then 'false' + when {{ field }} is true then 'true' when {{ field }} is false then 'false' end::text {% endmacro %} diff --git a/macros/cross_db_utils/concat.sql b/macros/cross_db_utils/concat.sql index 7e97e8f8..e2be3cee 100644 --- a/macros/cross_db_utils/concat.sql +++ b/macros/cross_db_utils/concat.sql @@ -1,9 +1,5 @@ - - {% macro concat(fields) -%} - {{ return(adapter.dispatch('concat', 'dbt_utils')(fields)) }} + {{ return(adapter.dispatch("concat", "dbt_utils")(fields)) }} {%- endmacro %} -{% macro default__concat(fields) -%} - {{ fields|join(' || ') }} -{%- endmacro %} +{% macro default__concat(fields) -%} {{ fields | join(" || ") }} {%- endmacro %} diff --git a/macros/cross_db_utils/current_timestamp.sql b/macros/cross_db_utils/current_timestamp.sql index 66ad8dc6..aa8e237c 100644 --- a/macros/cross_db_utils/current_timestamp.sql +++ b/macros/cross_db_utils/current_timestamp.sql @@ -1,35 +1,32 @@ {% macro current_timestamp() -%} - {{ return(adapter.dispatch('current_timestamp', 'dbt_utils')()) }} + {{ return(adapter.dispatch("current_timestamp", "dbt_utils")()) }} {%- endmacro %} {% macro default__current_timestamp() %} - current_timestamp::{{dbt_utils.type_timestamp()}} + current_timestamp::{{ dbt_utils.type_timestamp() }} {% endmacro %} -{% macro redshift__current_timestamp() %} - getdate() -{% endmacro %} - -{% macro bigquery__current_timestamp() %} - current_timestamp -{% endmacro %} +{% macro redshift__current_timestamp() %} getdate() {% endmacro %} +{% macro bigquery__current_timestamp() %} current_timestamp {% endmacro %} {% macro current_timestamp_in_utc() -%} - {{ return(adapter.dispatch('current_timestamp_in_utc', 'dbt_utils')()) }} + {{ return(adapter.dispatch("current_timestamp_in_utc", "dbt_utils")()) }} {%- endmacro %} {% macro default__current_timestamp_in_utc() %} - {{dbt_utils.current_timestamp()}} + {{ dbt_utils.current_timestamp() }} {% endmacro %} {% macro snowflake__current_timestamp_in_utc() %} - convert_timezone('UTC', {{dbt_utils.current_timestamp()}})::{{dbt_utils.type_timestamp()}} + convert_timezone( + 'UTC', {{ dbt_utils.current_timestamp() }} + )::{{ dbt_utils.type_timestamp() }} {% endmacro %} {% macro postgres__current_timestamp_in_utc() %} - (current_timestamp at time zone 'utc')::{{dbt_utils.type_timestamp()}} + (current_timestamp at time zone 'utc')::{{ dbt_utils.type_timestamp() }} {% endmacro %} {# redshift should use default instead of postgres #} diff --git a/macros/cross_db_utils/datatypes.sql b/macros/cross_db_utils/datatypes.sql index f115b4e2..ba12737c 100644 --- a/macros/cross_db_utils/datatypes.sql +++ b/macros/cross_db_utils/datatypes.sql @@ -1,99 +1,62 @@ {# string ------------------------------------------------- #} - {%- macro type_string() -%} - {{ return(adapter.dispatch('type_string', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_string", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_string() %} - string -{% endmacro %} - -{%- macro redshift__type_string() -%} - varchar -{%- endmacro -%} +{% macro default__type_string() %} string {% endmacro %} -{% macro postgres__type_string() %} - varchar -{% endmacro %} +{%- macro redshift__type_string() -%} varchar {%- endmacro -%} -{% macro snowflake__type_string() %} - varchar -{% endmacro %} +{% macro postgres__type_string() %} varchar {% endmacro %} +{% macro snowflake__type_string() %} varchar {% endmacro %} {# timestamp ------------------------------------------------- #} - {%- macro type_timestamp() -%} - {{ return(adapter.dispatch('type_timestamp', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_timestamp", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_timestamp() %} - timestamp -{% endmacro %} +{% macro default__type_timestamp() %} timestamp {% endmacro %} -{% macro postgres__type_timestamp() %} - timestamp without time zone -{% endmacro %} +{% macro postgres__type_timestamp() %} timestamp without time zone {% endmacro %} -{% macro snowflake__type_timestamp() %} - timestamp_ntz -{% endmacro %} +{% macro snowflake__type_timestamp() %} timestamp_ntz {% endmacro %} {# float ------------------------------------------------- #} - {%- macro type_float() -%} - {{ return(adapter.dispatch('type_float', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_float", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_float() %} - float -{% endmacro %} +{% macro default__type_float() %} float {% endmacro %} -{% macro bigquery__type_float() %} - float64 -{% endmacro %} +{% macro bigquery__type_float() %} float64 {% endmacro %} {# numeric ------------------------------------------------ #} - {%- macro type_numeric() -%} - {{ return(adapter.dispatch('type_numeric', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_numeric", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_numeric() %} - numeric(28, 6) -{% endmacro %} +{% macro default__type_numeric() %} numeric(28, 6) {% endmacro %} -{% macro bigquery__type_numeric() %} - numeric -{% endmacro %} +{% macro bigquery__type_numeric() %} numeric {% endmacro %} {# bigint ------------------------------------------------- #} - {%- macro type_bigint() -%} - {{ return(adapter.dispatch('type_bigint', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_bigint", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_bigint() %} - bigint -{% endmacro %} +{% macro default__type_bigint() %} bigint {% endmacro %} -{% macro bigquery__type_bigint() %} - int64 -{% endmacro %} +{% macro bigquery__type_bigint() %} int64 {% endmacro %} {# int ------------------------------------------------- #} - {%- macro type_int() -%} - {{ return(adapter.dispatch('type_int', 'dbt_utils')()) }} + {{ return(adapter.dispatch("type_int", "dbt_utils")()) }} {%- endmacro -%} -{% macro default__type_int() %} - int -{% endmacro %} +{% macro default__type_int() %} int {% endmacro %} -{% macro bigquery__type_int() %} - int64 -{% endmacro %} +{% macro bigquery__type_int() %} int64 {% endmacro %} diff --git a/macros/cross_db_utils/date_trunc.sql b/macros/cross_db_utils/date_trunc.sql index f9d0364b..e173f6ad 100644 --- a/macros/cross_db_utils/date_trunc.sql +++ b/macros/cross_db_utils/date_trunc.sql @@ -1,15 +1,12 @@ {% macro date_trunc(datepart, date) -%} - {{ return(adapter.dispatch('date_trunc', 'dbt_utils') (datepart, date)) }} + {{ return(adapter.dispatch("date_trunc", "dbt_utils")(datepart, date)) }} {%- endmacro %} {% macro default__date_trunc(datepart, date) -%} - date_trunc('{{datepart}}', {{date}}) + date_trunc('{{datepart}}', {{ date }}) {%- endmacro %} {% macro bigquery__date_trunc(datepart, date) -%} - timestamp_trunc( - cast({{date}} as timestamp), - {{datepart}} - ) + timestamp_trunc(cast({{ date }} as timestamp), {{ datepart }}) {%- endmacro %} diff --git a/macros/cross_db_utils/dateadd.sql b/macros/cross_db_utils/dateadd.sql index 09c0f115..fd8016cc 100644 --- a/macros/cross_db_utils/dateadd.sql +++ b/macros/cross_db_utils/dateadd.sql @@ -1,25 +1,27 @@ {% macro dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(adapter.dispatch('dateadd', 'dbt_utils')(datepart, interval, from_date_or_timestamp)) }} + {{ + return( + adapter.dispatch("dateadd", "dbt_utils")( + datepart, interval, from_date_or_timestamp + ) + ) + }} {% endmacro %} {% macro default__dateadd(datepart, interval, from_date_or_timestamp) %} - dateadd( - {{ datepart }}, - {{ interval }}, - {{ from_date_or_timestamp }} - ) + dateadd({{ datepart }}, {{ interval }}, {{ from_date_or_timestamp }}) {% endmacro %} {% macro bigquery__dateadd(datepart, interval, from_date_or_timestamp) %} - datetime_add( - cast( {{ from_date_or_timestamp }} as datetime), + datetime_add( + cast({{ from_date_or_timestamp }} as datetime), interval {{ interval }} {{ datepart }} - ) + ) {% endmacro %} @@ -32,6 +34,10 @@ {# redshift should use default instead of postgres #} {% macro redshift__dateadd(datepart, interval, from_date_or_timestamp) %} - {{ return(dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp)) }} + {{ + return( + dbt_utils.default__dateadd(datepart, interval, from_date_or_timestamp) + ) + }} {% endmacro %} diff --git a/macros/cross_db_utils/datediff.sql b/macros/cross_db_utils/datediff.sql index 2b5d6613..3804ff88 100644 --- a/macros/cross_db_utils/datediff.sql +++ b/macros/cross_db_utils/datediff.sql @@ -1,15 +1,17 @@ {% macro datediff(first_date, second_date, datepart) %} - {{ return(adapter.dispatch('datediff', 'dbt_utils')(first_date, second_date, datepart)) }} + {{ + return( + adapter.dispatch("datediff", "dbt_utils")( + first_date, second_date, datepart + ) + ) + }} {% endmacro %} {% macro default__datediff(first_date, second_date, datepart) -%} - datediff( - {{ datepart }}, - {{ first_date }}, - {{ second_date }} - ) + datediff({{ datepart }}, {{ first_date }}, {{ second_date }}) {%- endmacro %} @@ -17,42 +19,81 @@ {% macro bigquery__datediff(first_date, second_date, datepart) -%} datetime_diff( - cast({{second_date}} as datetime), - cast({{first_date}} as datetime), - {{datepart}} + cast({{ second_date }} as datetime), + cast({{ first_date }} as datetime), + {{ datepart }} ) {%- endmacro %} {% macro postgres__datediff(first_date, second_date, datepart) -%} - {% if datepart == 'year' %} - (date_part('year', ({{second_date}})::date) - date_part('year', ({{first_date}})::date)) - {% elif datepart == 'quarter' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 4 + date_part('quarter', ({{second_date}})::date) - date_part('quarter', ({{first_date}})::date)) - {% elif datepart == 'month' %} - ({{ dbt_utils.datediff(first_date, second_date, 'year') }} * 12 + date_part('month', ({{second_date}})::date) - date_part('month', ({{first_date}})::date)) - {% elif datepart == 'day' %} - (({{second_date}})::date - ({{first_date}})::date) - {% elif datepart == 'week' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} / 7 + case - when date_part('dow', ({{first_date}})::timestamp) <= date_part('dow', ({{second_date}})::timestamp) then - case when {{first_date}} <= {{second_date}} then 0 else -1 end - else - case when {{first_date}} <= {{second_date}} then 1 else 0 end - end) - {% elif datepart == 'hour' %} - ({{ dbt_utils.datediff(first_date, second_date, 'day') }} * 24 + date_part('hour', ({{second_date}})::timestamp) - date_part('hour', ({{first_date}})::timestamp)) - {% elif datepart == 'minute' %} - ({{ dbt_utils.datediff(first_date, second_date, 'hour') }} * 60 + date_part('minute', ({{second_date}})::timestamp) - date_part('minute', ({{first_date}})::timestamp)) - {% elif datepart == 'second' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60 + floor(date_part('second', ({{second_date}})::timestamp)) - floor(date_part('second', ({{first_date}})::timestamp))) - {% elif datepart == 'millisecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000 + floor(date_part('millisecond', ({{second_date}})::timestamp)) - floor(date_part('millisecond', ({{first_date}})::timestamp))) - {% elif datepart == 'microsecond' %} - ({{ dbt_utils.datediff(first_date, second_date, 'minute') }} * 60000000 + floor(date_part('microsecond', ({{second_date}})::timestamp)) - floor(date_part('microsecond', ({{first_date}})::timestamp))) + {% if datepart == "year" %} + ( + date_part('year', ({{ second_date }})::date) + - date_part('year', ({{ first_date }})::date) + ) + {% elif datepart == "quarter" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 4 + + date_part('quarter', ({{ second_date }})::date) + - date_part('quarter', ({{ first_date }})::date) + ) + {% elif datepart == "month" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "year") }} * 12 + + date_part('month', ({{ second_date }})::date) + - date_part('month', ({{ first_date }})::date) + ) + {% elif datepart == "day" %}(({{ second_date }})::date - ({{ first_date }})::date) + {% elif datepart == "week" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "day") }} / 7 + case + when + date_part('dow', ({{ first_date }})::timestamp) + <= date_part('dow', ({{ second_date }})::timestamp) + then case when {{ first_date }} <= {{ second_date }} then 0 else -1 end + else case when {{ first_date }} <= {{ second_date }} then 1 else 0 end + end + ) + {% elif datepart == "hour" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "day") }} * 24 + + date_part('hour', ({{ second_date }})::timestamp) + - date_part('hour', ({{ first_date }})::timestamp) + ) + {% elif datepart == "minute" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "hour") }} * 60 + + date_part('minute', ({{ second_date }})::timestamp) + - date_part('minute', ({{ first_date }})::timestamp) + ) + {% elif datepart == "second" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60 + + floor(date_part('second', ({{ second_date }})::timestamp)) + - floor(date_part('second', ({{ first_date }})::timestamp)) + ) + {% elif datepart == "millisecond" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000 + + floor(date_part('millisecond', ({{ second_date }})::timestamp)) + - floor(date_part('millisecond', ({{ first_date }})::timestamp)) + ) + {% elif datepart == "microsecond" %} + ( + {{ dbt_utils.datediff(first_date, second_date, "minute") }} * 60000000 + + floor(date_part('microsecond', ({{ second_date }})::timestamp)) + - floor(date_part('microsecond', ({{ first_date }})::timestamp)) + ) {% else %} - {{ exceptions.raise_compiler_error("Unsupported datepart for macro datediff in postgres: {!r}".format(datepart)) }} + {{ + exceptions.raise_compiler_error( + "Unsupported datepart for macro datediff in postgres: {!r}".format( + datepart + ) + ) + }} {% endif %} {%- endmacro %} diff --git a/macros/cross_db_utils/escape_single_quotes.sql b/macros/cross_db_utils/escape_single_quotes.sql index d024f16f..0a3c95a2 100644 --- a/macros/cross_db_utils/escape_single_quotes.sql +++ b/macros/cross_db_utils/escape_single_quotes.sql @@ -1,18 +1,18 @@ {% macro escape_single_quotes(expression) %} - {{ return(adapter.dispatch('escape_single_quotes', 'dbt_utils') (expression)) }} + {{ return(adapter.dispatch("escape_single_quotes", "dbt_utils")(expression)) }} {% endmacro %} {# /*Default to replacing a single apostrophe with two apostrophes: they're -> they''re*/ #} {% macro default__escape_single_quotes(expression) -%} -{{ expression | replace("'","''") }} + {{ expression | replace("'", "''") }} {%- endmacro %} {# /*Snowflake uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} {% macro snowflake__escape_single_quotes(expression) -%} -{{ expression | replace("'", "\\'") }} + {{ expression | replace("'", "\\'") }} {%- endmacro %} {# /*BigQuery uses a single backslash: they're -> they\'re. The second backslash is to escape it from Jinja */ #} {% macro bigquery__escape_single_quotes(expression) -%} -{{ expression | replace("'", "\\'") }} + {{ expression | replace("'", "\\'") }} {%- endmacro %} diff --git a/macros/cross_db_utils/except.sql b/macros/cross_db_utils/except.sql index 9bb75b86..888e1940 100644 --- a/macros/cross_db_utils/except.sql +++ b/macros/cross_db_utils/except.sql @@ -1,5 +1,5 @@ {% macro except() %} - {{ return(adapter.dispatch('except', 'dbt_utils')()) }} + {{ return(adapter.dispatch("except", "dbt_utils")()) }} {% endmacro %} @@ -8,9 +8,9 @@ except {% endmacro %} - + {% macro bigquery__except() %} except distinct -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/cross_db_utils/hash.sql b/macros/cross_db_utils/hash.sql index d086c2e6..9674fefc 100644 --- a/macros/cross_db_utils/hash.sql +++ b/macros/cross_db_utils/hash.sql @@ -1,13 +1,13 @@ {% macro hash(field) -%} - {{ return(adapter.dispatch('hash', 'dbt_utils') (field)) }} + {{ return(adapter.dispatch("hash", "dbt_utils")(field)) }} {%- endmacro %} {% macro default__hash(field) -%} - md5(cast({{field}} as {{dbt_utils.type_string()}})) + md5(cast({{ field }} as {{ dbt_utils.type_string() }})) {%- endmacro %} {% macro bigquery__hash(field) -%} - to_hex({{dbt_utils.default__hash(field)}}) + to_hex({{ dbt_utils.default__hash(field) }}) {%- endmacro %} diff --git a/macros/cross_db_utils/identifier.sql b/macros/cross_db_utils/identifier.sql index 619cb7cb..a4c1a73a 100644 --- a/macros/cross_db_utils/identifier.sql +++ b/macros/cross_db_utils/identifier.sql @@ -1,16 +1,11 @@ -{% macro identifier(value) %} - {%- set error_message = ' - Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ - Use `adapter.quote` instead. The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} - {%- do exceptions.warn(error_message) -%} - {{ return(adapter.dispatch('identifier', 'dbt_utils') (value)) }} -{% endmacro %} +{% macro identifier(value) %} + {%- set error_message = " Warning: the `identifier` macro is no longer supported and will be deprecated in a future release of dbt-utils. \ Use `adapter.quote` instead. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name + ) -%} + {%- do exceptions.warn(error_message) -%} + {{ return(adapter.dispatch("identifier", "dbt_utils")(value)) }} +{% endmacro %} -{% macro default__identifier(value) -%} - "{{ value }}" -{%- endmacro %} +{% macro default__identifier(value) -%} "{{ value }}" {%- endmacro %} -{% macro bigquery__identifier(value) -%} - `{{ value }}` -{%- endmacro %} +{% macro bigquery__identifier(value) -%} `{{ value }}` {%- endmacro %} diff --git a/macros/cross_db_utils/intersect.sql b/macros/cross_db_utils/intersect.sql index b53c72f3..08abf793 100644 --- a/macros/cross_db_utils/intersect.sql +++ b/macros/cross_db_utils/intersect.sql @@ -1,5 +1,5 @@ {% macro intersect() %} - {{ return(adapter.dispatch('intersect', 'dbt_utils')()) }} + {{ return(adapter.dispatch("intersect", "dbt_utils")()) }} {% endmacro %} diff --git a/macros/cross_db_utils/last_day.sql b/macros/cross_db_utils/last_day.sql index 18ca161c..391cd4b5 100644 --- a/macros/cross_db_utils/last_day.sql +++ b/macros/cross_db_utils/last_day.sql @@ -2,37 +2,47 @@ This function has been tested with dateparts of month and quarters. Further testing is required to validate that it will work on other dateparts. */ - {% macro last_day(date, datepart) %} - {{ return(adapter.dispatch('last_day', 'dbt_utils') (date, datepart)) }} + {{ return(adapter.dispatch("last_day", "dbt_utils")(date, datepart)) }} {% endmacro %} {%- macro default_last_day(date, datepart) -%} cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd(datepart, '1', dbt_utils.date_trunc(datepart, date)) - )}} - as date) + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd( + datepart, "1", dbt_utils.date_trunc(datepart, date) + ), + ) + }} as date + ) {%- endmacro -%} {% macro default__last_day(date, datepart) -%} - {{dbt_utils.default_last_day(date, datepart)}} + {{ dbt_utils.default_last_day(date, datepart) }} {%- endmacro %} {% macro postgres__last_day(date, datepart) -%} - {%- if datepart == 'quarter' -%} - -- postgres dateadd does not support quarter interval. - cast( - {{dbt_utils.dateadd('day', '-1', - dbt_utils.dateadd('month', '3', dbt_utils.date_trunc(datepart, date)) - )}} - as date) - {%- else -%} - {{dbt_utils.default_last_day(date, datepart)}} + {%- if datepart == "quarter" -%} + -- postgres dateadd does not support quarter interval. + cast( + {{ + dbt_utils.dateadd( + "day", + "-1", + dbt_utils.dateadd( + "month", "3", dbt_utils.date_trunc(datepart, date) + ), + ) + }} as date + ) + {%- else -%} {{ dbt_utils.default_last_day(date, datepart) }} {%- endif -%} {%- endmacro %} diff --git a/macros/cross_db_utils/length.sql b/macros/cross_db_utils/length.sql index 8c2e265c..2e736fbd 100644 --- a/macros/cross_db_utils/length.sql +++ b/macros/cross_db_utils/length.sql @@ -1,21 +1,9 @@ {% macro length(expression) -%} - {{ return(adapter.dispatch('length', 'dbt_utils') (expression)) }} + {{ return(adapter.dispatch("length", "dbt_utils")(expression)) }} {% endmacro %} -{% macro default__length(expression) %} - - length( - {{ expression }} - ) - -{%- endmacro -%} +{% macro default__length(expression) %} length({{ expression }}) {%- endmacro -%} -{% macro redshift__length(expression) %} - - len( - {{ expression }} - ) - -{%- endmacro -%} \ No newline at end of file +{% macro redshift__length(expression) %} len({{ expression }}) {%- endmacro -%} diff --git a/macros/cross_db_utils/listagg.sql b/macros/cross_db_utils/listagg.sql index 1d19a54f..18e58df9 100644 --- a/macros/cross_db_utils/listagg.sql +++ b/macros/cross_db_utils/listagg.sql @@ -1,29 +1,32 @@ -{% macro listagg(measure, delimiter_text="','", order_by_clause=none, limit_num=none) -%} - {{ return(adapter.dispatch('listagg', 'dbt_utils') (measure, delimiter_text, order_by_clause, limit_num)) }} +{% macro listagg( + measure, delimiter_text="','", order_by_clause=none, limit_num=none +) -%} + {{ + return( + adapter.dispatch("listagg", "dbt_utils")( + measure, delimiter_text, order_by_clause, limit_num + ) + ) + }} {%- endmacro %} {% macro default__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} {% if limit_num -%} - array_to_string( - array_slice( - array_agg( - {{ measure }} - ){% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,0 - ,{{ limit_num }} - ), - {{ delimiter_text }} + array_to_string( + array_slice( + array_agg({{ measure }}) + {% if order_by_clause -%} + within group ({{ order_by_clause }}) + {%- endif %}, + 0, + {{ limit_num }} + ), + {{ delimiter_text }} ) {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %} {%- endif %} @@ -34,35 +37,30 @@ string_agg( {{ measure }}, {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - {% if limit_num -%} - limit {{ limit_num }} - {%- endif %} - ) + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + {% if limit_num -%} limit {{ limit_num }} {%- endif %} + ) {%- endmacro %} {% macro postgres__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} - + {% if limit_num -%} - array_to_string( - (array_agg( - {{ measure }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} - ))[1:{{ limit_num }}], - {{ delimiter_text }} + array_to_string( + ( + array_agg( + {{ measure }} + {% if order_by_clause -%} {{ order_by_clause }} {%- endif %} + ) + )[1:{{ limit_num }}], + {{ delimiter_text }} ) {%- else %} - string_agg( - {{ measure }}, - {{ delimiter_text }} - {% if order_by_clause -%} - {{ order_by_clause }} - {%- endif %} + string_agg( + {{ measure }}, + {{ delimiter_text }} + {% if order_by_clause -%} {{ order_by_clause }} + {%- endif %} ) {%- endif %} @@ -72,33 +70,26 @@ {% macro redshift__listagg(measure, delimiter_text, order_by_clause, limit_num) -%} {% if limit_num -%} - {% set ns = namespace() %} - {% set ns.delimiter_text_regex = delimiter_text|trim("'") %} - {% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} - {%- for char in special_chars.split(',') -%} - {% set escape_char %}\\{{ char }}{% endset %} - {% set ns.delimiter_text_regex = ns.delimiter_text_regex|replace(char,escape_char) %} - {%- endfor -%} + {% set ns = namespace() %} + {% set ns.delimiter_text_regex = delimiter_text | trim("'") %} + {% set special_chars %}\,^,$,.,|,?,*,+,(,),[,],{,}{% endset %} + {%- for char in special_chars.split(",") -%} + {% set escape_char %}\\{{ char }}{% endset %} + {% set ns.delimiter_text_regex = ns.delimiter_text_regex | replace( + char, escape_char + ) %} + {%- endfor -%} - {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} - regexp_substr( - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) - {%- endif %} - ,{{ regex }} + {% set regex %}'([^{{ ns.delimiter_text_regex }}]+{{ ns.delimiter_text_regex }}){1,{{ limit_num - 1}}}[^{{ ns.delimiter_text_regex }}]+'{% endset %} + regexp_substr( + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %}, + {{ regex }} ) {%- else %} - listagg( - {{ measure }}, - {{ delimiter_text }} - ) - {% if order_by_clause -%} - within group ({{ order_by_clause }}) + listagg({{ measure }}, {{ delimiter_text }}) + {% if order_by_clause -%} within group ({{ order_by_clause }}) {%- endif %} {%- endif %} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/cross_db_utils/literal.sql b/macros/cross_db_utils/literal.sql index c2291467..26858c33 100644 --- a/macros/cross_db_utils/literal.sql +++ b/macros/cross_db_utils/literal.sql @@ -1,8 +1,5 @@ - {%- macro string_literal(value) -%} - {{ return(adapter.dispatch('string_literal', 'dbt_utils') (value)) }} + {{ return(adapter.dispatch("string_literal", "dbt_utils")(value)) }} {%- endmacro -%} -{% macro default__string_literal(value) -%} - '{{ value }}' -{%- endmacro %} +{% macro default__string_literal(value) -%} '{{ value }}' {%- endmacro %} diff --git a/macros/cross_db_utils/position.sql b/macros/cross_db_utils/position.sql index a67b7aa2..140d2680 100644 --- a/macros/cross_db_utils/position.sql +++ b/macros/cross_db_utils/position.sql @@ -1,22 +1,20 @@ {% macro position(substring_text, string_text) -%} - {{ return(adapter.dispatch('position', 'dbt_utils') (substring_text, string_text)) }} + {{ + return( + adapter.dispatch("position", "dbt_utils")(substring_text, string_text) + ) + }} {% endmacro %} {% macro default__position(substring_text, string_text) %} - position( - {{ substring_text }} in {{ string_text }} - ) - + position({{ substring_text }} in {{ string_text }}) + {%- endmacro -%} {% macro bigquery__position(substring_text, string_text) %} - strpos( - {{ string_text }}, - {{ substring_text }} - - ) - + strpos({{ string_text }}, {{ substring_text }}) + {%- endmacro -%} diff --git a/macros/cross_db_utils/replace.sql b/macros/cross_db_utils/replace.sql index ef27d8fc..66b859e7 100644 --- a/macros/cross_db_utils/replace.sql +++ b/macros/cross_db_utils/replace.sql @@ -1,15 +1,10 @@ {% macro replace(field, old_chars, new_chars) -%} - {{ return(adapter.dispatch('replace', 'dbt_utils') (field, old_chars, new_chars)) }} + {{ return(adapter.dispatch("replace", "dbt_utils")(field, old_chars, new_chars)) }} {% endmacro %} {% macro default__replace(field, old_chars, new_chars) %} - replace( - {{ field }}, - {{ old_chars }}, - {{ new_chars }} - ) - + replace({{ field }}, {{ old_chars }}, {{ new_chars }}) -{% endmacro %} \ No newline at end of file +{% endmacro %} diff --git a/macros/cross_db_utils/right.sql b/macros/cross_db_utils/right.sql index 8ae1640f..841dc4f6 100644 --- a/macros/cross_db_utils/right.sql +++ b/macros/cross_db_utils/right.sql @@ -1,38 +1,33 @@ {% macro right(string_text, length_expression) -%} - {{ return(adapter.dispatch('right', 'dbt_utils') (string_text, length_expression)) }} + {{ + return( + adapter.dispatch("right", "dbt_utils")(string_text, length_expression) + ) + }} {% endmacro %} {% macro default__right(string_text, length_expression) %} - right( - {{ string_text }}, - {{ length_expression }} - ) - + right({{ string_text }}, {{ length_expression }}) + {%- endmacro -%} {% macro bigquery__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 + case + when {{ length_expression }} = 0 then '' - else - substr( - {{ string_text }}, - -1 * ({{ length_expression }}) - ) + else substr({{ string_text }}, -1 * ({{ length_expression }})) end {%- endmacro -%} {% macro snowflake__right(string_text, length_expression) %} - case when {{ length_expression }} = 0 + case + when {{ length_expression }} = 0 then '' - else - right( - {{ string_text }}, - {{ length_expression }} - ) + else right({{ string_text }}, {{ length_expression }}) end -{%- endmacro -%} \ No newline at end of file +{%- endmacro -%} diff --git a/macros/cross_db_utils/safe_cast.sql b/macros/cross_db_utils/safe_cast.sql index 8569eecd..3337beb2 100644 --- a/macros/cross_db_utils/safe_cast.sql +++ b/macros/cross_db_utils/safe_cast.sql @@ -1,20 +1,20 @@ {% macro safe_cast(field, type) %} - {{ return(adapter.dispatch('safe_cast', 'dbt_utils') (field, type)) }} + {{ return(adapter.dispatch("safe_cast", "dbt_utils")(field, type)) }} {% endmacro %} {% macro default__safe_cast(field, type) %} {# most databases don't support this function yet so we just need to use cast #} - cast({{field}} as {{type}}) + cast({{ field }} as {{ type }}) {% endmacro %} {% macro snowflake__safe_cast(field, type) %} - try_cast({{field}} as {{type}}) + try_cast({{ field }} as {{ type }}) {% endmacro %} {% macro bigquery__safe_cast(field, type) %} - safe_cast({{field}} as {{type}}) + safe_cast({{ field }} as {{ type }}) {% endmacro %} diff --git a/macros/cross_db_utils/split_part.sql b/macros/cross_db_utils/split_part.sql index 036f7d5a..5f10da88 100644 --- a/macros/cross_db_utils/split_part.sql +++ b/macros/cross_db_utils/split_part.sql @@ -1,24 +1,23 @@ {% macro split_part(string_text, delimiter_text, part_number) %} - {{ return(adapter.dispatch('split_part', 'dbt_utils') (string_text, delimiter_text, part_number)) }} + {{ + return( + adapter.dispatch("split_part", "dbt_utils")( + string_text, delimiter_text, part_number + ) + ) + }} {% endmacro %} {% macro default__split_part(string_text, delimiter_text, part_number) %} - split_part( - {{ string_text }}, - {{ delimiter_text }}, - {{ part_number }} - ) + split_part({{ string_text }}, {{ delimiter_text }}, {{ part_number }}) {% endmacro %} {% macro bigquery__split_part(string_text, delimiter_text, part_number) %} - split( - {{ string_text }}, - {{ delimiter_text }} - )[safe_offset({{ part_number - 1 }})] + split({{ string_text }}, {{ delimiter_text }})[safe_offset({{ part_number - 1 }})] {% endmacro %} diff --git a/macros/cross_db_utils/width_bucket.sql b/macros/cross_db_utils/width_bucket.sql index fc8ae81c..452993de 100644 --- a/macros/cross_db_utils/width_bucket.sql +++ b/macros/cross_db_utils/width_bucket.sql @@ -1,5 +1,11 @@ {% macro width_bucket(expr, min_value, max_value, num_buckets) %} - {{ return(adapter.dispatch('width_bucket', 'dbt_utils') (expr, min_value, max_value, num_buckets)) }} + {{ + return( + adapter.dispatch("width_bucket", "dbt_utils")( + expr, min_value, max_value, num_buckets + ) + ) + }} {% endmacro %} @@ -13,20 +19,16 @@ case when mod( - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }}, - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - ) = 0 + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }}, + {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + ) + = 0 then 1 else 0 end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 ) + -- Anything over max_value goes the N+1 bucket + + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro redshift__width_bucket(expr, min_value, max_value, num_buckets) -%} @@ -38,22 +40,17 @@ -- to break ties when the amount is exactly at the bucket edge case when - {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric() ) }} % - {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric() ) }} - = 0 + {{ dbt_utils.safe_cast(expr, dbt_utils.type_numeric()) }} + % {{ dbt_utils.safe_cast(bin_size, dbt_utils.type_numeric()) }} + = 0 then 1 else 0 end - ) + - -- Anything over max_value goes the N+1 bucket - least( - ceil( - ({{ expr }} - {{ min_value }})/{{ bin_size }} - ), - {{ num_buckets }} + 1 ) + -- Anything over max_value goes the N+1 bucket + + least(ceil(({{ expr }} - {{ min_value }}) /{{ bin_size }}), {{ num_buckets }} + 1) {%- endmacro %} {% macro snowflake__width_bucket(expr, min_value, max_value, num_buckets) %} - width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }} ) + width_bucket({{ expr }}, {{ min_value }}, {{ max_value }}, {{ num_buckets }}) {% endmacro %} diff --git a/macros/generic_tests/accepted_range.sql b/macros/generic_tests/accepted_range.sql index 49ab00d9..aee6e7fb 100644 --- a/macros/generic_tests/accepted_range.sql +++ b/macros/generic_tests/accepted_range.sql @@ -1,33 +1,44 @@ -{% test accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} - {{ return(adapter.dispatch('test_accepted_range', 'dbt_utils')(model, column_name, min_value, max_value, inclusive)) }} +{% test accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} + {{ + return( + adapter.dispatch("test_accepted_range", "dbt_utils")( + model, column_name, min_value, max_value, inclusive + ) + ) + }} {% endtest %} -{% macro default__test_accepted_range(model, column_name, min_value=none, max_value=none, inclusive=true) %} +{% macro default__test_accepted_range( + model, column_name, min_value=none, max_value=none, inclusive=true +) %} -with meet_condition as( - select * - from {{ model }} -), + with + meet_condition as (select * from {{ model }}), -validation_errors as ( - select * - from meet_condition - where - -- never true, defaults to an empty result set. Exists to ensure any combo of the `or` clauses below succeeds - 1 = 2 + validation_errors as ( + select * + from meet_condition + where + -- never true, defaults to an empty result set. Exists to ensure any + -- combo of the `or` clauses below succeeds + 1 = 2 - {%- if min_value is not none %} - -- records with a value >= min_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} - {%- endif %} + {%- if min_value is not none %} + -- records with a value >= min_value are permitted. The `not` + -- flips this to find records that don't meet the rule. + or not {{ column_name }} > {{- "=" if inclusive }} {{ min_value }} + {%- endif %} - {%- if max_value is not none %} - -- records with a value <= max_value are permitted. The `not` flips this to find records that don't meet the rule. - or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} - {%- endif %} -) + {%- if max_value is not none %} + -- records with a value <= max_value are permitted. The `not` + -- flips this to find records that don't meet the rule. + or not {{ column_name }} < {{- "=" if inclusive }} {{ max_value }} + {%- endif %} + ) -select * -from validation_errors + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/at_least_one.sql b/macros/generic_tests/at_least_one.sql index ce02108f..c13981af 100644 --- a/macros/generic_tests/at_least_one.sql +++ b/macros/generic_tests/at_least_one.sql @@ -1,20 +1,25 @@ {% test at_least_one(model, column_name) %} - {{ return(adapter.dispatch('test_at_least_one', 'dbt_utils')(model, column_name)) }} + {{ + return( + adapter.dispatch("test_at_least_one", "dbt_utils")(model, column_name) + ) + }} {% endtest %} {% macro default__test_at_least_one(model, column_name) %} -select * -from ( - select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count({{ column_name }}) as filler_column + select * + from + ( + select + {# In TSQL, subquery aggregate columns need aliases #} + {# thus: a filler col name, 'filler_column' #} + count({{ column_name }}) as filler_column - from {{ model }} + from {{ model }} - having count({{ column_name }}) = 0 + having count({{ column_name }}) = 0 -) validation_errors + ) validation_errors {% endmacro %} diff --git a/macros/generic_tests/cardinality_equality.sql b/macros/generic_tests/cardinality_equality.sql index 4ec7d87f..8ef1a901 100644 --- a/macros/generic_tests/cardinality_equality.sql +++ b/macros/generic_tests/cardinality_equality.sql @@ -1,53 +1,45 @@ {% test cardinality_equality(model, column_name, to, field) %} - {{ return(adapter.dispatch('test_cardinality_equality', 'dbt_utils')(model, column_name, to, field)) }} + {{ + return( + adapter.dispatch("test_cardinality_equality", "dbt_utils")( + model, column_name, to, field + ) + ) + }} {% endtest %} {% macro default__test_cardinality_equality(model, column_name, to, field) %} -{# T-SQL does not let you use numbers as aliases for columns #} -{# Thus, no "GROUP BY 1" #} - -with table_a as ( -select - {{ column_name }}, - count(*) as num_rows -from {{ model }} -group by {{ column_name }} -), - -table_b as ( -select - {{ field }}, - count(*) as num_rows -from {{ to }} -group by {{ field }} -), - -except_a as ( - select * - from table_a - {{ dbt_utils.except() }} - select * - from table_b -), - -except_b as ( - select * - from table_b - {{ dbt_utils.except() }} - select * - from table_a -), - -unioned as ( - select * - from except_a - union all - select * - from except_b -) - -select * -from unioned + {# T-SQL does not let you use numbers as aliases for columns #} + {# Thus, no "GROUP BY 1" #} + with + table_a as ( + select {{ column_name }}, count(*) as num_rows + from {{ model }} + group by {{ column_name }} + ), + + table_b as ( + select {{ field }}, count(*) as num_rows from {{ to }} group by {{ field }} + ), + + except_a as ( + select * from table_a {{ dbt_utils.except() }} select * from table_b + ), + + except_b as ( + select * from table_b {{ dbt_utils.except() }} select * from table_a + ), + + unioned as ( + select * + from except_a + union all + select * + from except_b + ) + + select * + from unioned {% endmacro %} diff --git a/macros/generic_tests/equal_rowcount.sql b/macros/generic_tests/equal_rowcount.sql index 713cb12a..d8404a6f 100644 --- a/macros/generic_tests/equal_rowcount.sql +++ b/macros/generic_tests/equal_rowcount.sql @@ -1,38 +1,31 @@ {% test equal_rowcount(model, compare_model) %} - {{ return(adapter.dispatch('test_equal_rowcount', 'dbt_utils')(model, compare_model)) }} + {{ + return( + adapter.dispatch("test_equal_rowcount", "dbt_utils")(model, compare_model) + ) + }} {% endtest %} {% macro default__test_equal_rowcount(model, compare_model) %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = 'coalesce(diff_count, 0)') }} + {#- - Needs to be set at parse time, before we return '' below --#} + {{ config(fail_calc="coalesce(diff_count, 0)") }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("") }} {% endif %} -with a as ( + with + a as (select count(*) as count_a from {{ model }}), + b as (select count(*) as count_b from {{ compare_model }}), + final as ( - select count(*) as count_a from {{ model }} + select count_a, count_b, abs(count_a - count_b) as diff_count + from a + cross join b -), -b as ( + ) - select count(*) as count_b from {{ compare_model }} - -), -final as ( - - select - count_a, - count_b, - abs(count_a - count_b) as diff_count - from a - cross join b - -) - -select * from final + select * + from final {% endmacro %} diff --git a/macros/generic_tests/equality.sql b/macros/generic_tests/equality.sql index bc302c18..28135583 100644 --- a/macros/generic_tests/equality.sql +++ b/macros/generic_tests/equality.sql @@ -1,75 +1,78 @@ {% test equality(model, compare_model, compare_columns=None) %} - {{ return(adapter.dispatch('test_equality', 'dbt_utils')(model, compare_model, compare_columns)) }} + {{ + return( + adapter.dispatch("test_equality", "dbt_utils")( + model, compare_model, compare_columns + ) + ) + }} {% endtest %} {% macro default__test_equality(model, compare_model, compare_columns=None) %} -{% set set_diff %} + {% set set_diff %} count(*) + coalesce(abs( sum(case when which_diff = 'a_minus_b' then 1 else 0 end) - sum(case when which_diff = 'b_minus_a' then 1 else 0 end) ), 0) -{% endset %} + {% endset %} -{#-- Needs to be set at parse time, before we return '' below --#} -{{ config(fail_calc = set_diff) }} + {#- - Needs to be set at parse time, before we return '' below --#} + {{ config(fail_calc=set_diff) }} -{#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} -{%- if not execute -%} - {{ return('') }} -{% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("") }} {% endif %} --- setup -{%- do dbt_utils._is_relation(model, 'test_equality') -%} + -- setup + {%- do dbt_utils._is_relation(model, "test_equality") -%} -{#- + {#- If the compare_cols arg is provided, we can run this test without querying the information schema — this allows the model to be an ephemeral model -#} + {%- if not compare_columns -%} + {%- do dbt_utils._is_ephemeral(model, "test_equality") -%} + {%- set compare_columns = adapter.get_columns_in_relation(model) | map( + attribute="quoted" + ) -%} + {%- endif -%} -{%- if not compare_columns -%} - {%- do dbt_utils._is_ephemeral(model, 'test_equality') -%} - {%- set compare_columns = adapter.get_columns_in_relation(model) | map(attribute='quoted') -%} -{%- endif -%} + {% set compare_cols_csv = compare_columns | join(", ") %} -{% set compare_cols_csv = compare_columns | join(', ') %} + with + a as (select * from {{ model }}), -with a as ( + b as (select * from {{ compare_model }}), - select * from {{ model }} + a_minus_b as ( -), + select {{ compare_cols_csv }} + from a {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from b -b as ( + ), - select * from {{ compare_model }} + b_minus_a as ( -), + select {{ compare_cols_csv }} + from b {{ dbt_utils.except() }} + select {{ compare_cols_csv }} + from a -a_minus_b as ( + ), - select {{compare_cols_csv}} from a - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from b + unioned as ( -), + select 'a_minus_b' as which_diff, a_minus_b.* + from a_minus_b + union all + select 'b_minus_a' as which_diff, b_minus_a.* + from b_minus_a -b_minus_a as ( + ) - select {{compare_cols_csv}} from b - {{ dbt_utils.except() }} - select {{compare_cols_csv}} from a - -), - -unioned as ( - - select 'a_minus_b' as which_diff, a_minus_b.* from a_minus_b - union all - select 'b_minus_a' as which_diff, b_minus_a.* from b_minus_a - -) - -select * from unioned + select * + from unioned {% endmacro %} diff --git a/macros/generic_tests/expression_is_true.sql b/macros/generic_tests/expression_is_true.sql index 611685b4..a67e818c 100644 --- a/macros/generic_tests/expression_is_true.sql +++ b/macros/generic_tests/expression_is_true.sql @@ -1,22 +1,23 @@ -{% test expression_is_true(model, expression, column_name=None, condition='1=1') %} -{# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} -{# ref https://stackoverflow.com/a/7170753/3842610 #} - {{ return(adapter.dispatch('test_expression_is_true', 'dbt_utils')(model, expression, column_name, condition)) }} +{% test expression_is_true(model, expression, column_name=None, condition="1=1") %} + {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} + {# ref https://stackoverflow.com/a/7170753/3842610 #} + {{ + return( + adapter.dispatch("test_expression_is_true", "dbt_utils")( + model, expression, column_name, condition + ) + ) + }} {% endtest %} {% macro default__test_expression_is_true(model, expression, column_name, condition) %} -with meet_condition as ( - select * from {{ model }} where {{ condition }} -) + with meet_condition as (select * from {{ model }} where {{ condition }}) -select - * -from meet_condition -{% if column_name is none %} -where not({{ expression }}) -{%- else %} -where not({{ column_name }} {{ expression }}) -{%- endif %} + select * + from meet_condition + {% if column_name is none %} where not ({{ expression }}) + {%- else %} where not ({{ column_name }} {{ expression }}) + {%- endif %} {% endmacro %} diff --git a/macros/generic_tests/fewer_rows_than.sql b/macros/generic_tests/fewer_rows_than.sql index 450148b7..e083906f 100644 --- a/macros/generic_tests/fewer_rows_than.sql +++ b/macros/generic_tests/fewer_rows_than.sql @@ -1,45 +1,41 @@ {% test fewer_rows_than(model, compare_model) %} - {{ return(adapter.dispatch('test_fewer_rows_than', 'dbt_utils')(model, compare_model)) }} + {{ + return( + adapter.dispatch("test_fewer_rows_than", "dbt_utils")( + model, compare_model + ) + ) + }} {% endtest %} {% macro default__test_fewer_rows_than(model, compare_model) %} -{{ config(fail_calc = 'coalesce(row_count_delta, 0)') }} - -with a as ( - - select count(*) as count_our_model from {{ model }} - -), -b as ( - - select count(*) as count_comparison_model from {{ compare_model }} - -), -counts as ( - - select - count_our_model, - count_comparison_model - from a - cross join b - -), -final as ( - - select *, - case - -- fail the test if we have more rows than the reference model and return the row count delta - when count_our_model > count_comparison_model then (count_our_model - count_comparison_model) - -- fail the test if they are the same number - when count_our_model = count_comparison_model then 1 - -- pass the test if the delta is positive (i.e. return the number 0) - else 0 - end as row_count_delta - from counts - -) - -select * from final + {{ config(fail_calc="coalesce(row_count_delta, 0)") }} + + with + a as (select count(*) as count_our_model from {{ model }}), + b as (select count(*) as count_comparison_model from {{ compare_model }}), + counts as (select count_our_model, count_comparison_model from a cross join b), + final as ( + + select + *, + case + -- fail the test if we have more rows than the reference model and + -- return the row count delta + when count_our_model > count_comparison_model + then (count_our_model - count_comparison_model) + -- fail the test if they are the same number + when count_our_model = count_comparison_model + then 1 + -- pass the test if the delta is positive (i.e. return the number 0) + else 0 + end as row_count_delta + from counts + + ) + + select * + from final {% endmacro %} diff --git a/macros/generic_tests/mutually_exclusive_ranges.sql b/macros/generic_tests/mutually_exclusive_ranges.sql index f88efbe4..3826f74a 100644 --- a/macros/generic_tests/mutually_exclusive_ranges.sql +++ b/macros/generic_tests/mutually_exclusive_ranges.sql @@ -1,97 +1,132 @@ -{% test mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} - {{ return(adapter.dispatch('test_mutually_exclusive_ranges', 'dbt_utils')(model, lower_bound_column, upper_bound_column, partition_by, gaps, zero_length_range_allowed)) }} +{% test mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False +) %} + {{ + return( + adapter.dispatch("test_mutually_exclusive_ranges", "dbt_utils")( + model, + lower_bound_column, + upper_bound_column, + partition_by, + gaps, + zero_length_range_allowed, + ) + ) + }} {% endtest %} -{% macro default__test_mutually_exclusive_ranges(model, lower_bound_column, upper_bound_column, partition_by=None, gaps='allowed', zero_length_range_allowed=False) %} -{% if gaps == 'not_allowed' %} - {% set allow_gaps_operator='=' %} - {% set allow_gaps_operator_in_words='equal_to' %} -{% elif gaps == 'allowed' %} - {% set allow_gaps_operator='<=' %} - {% set allow_gaps_operator_in_words='less_than_or_equal_to' %} -{% elif gaps == 'required' %} - {% set allow_gaps_operator='<' %} - {% set allow_gaps_operator_in_words='less_than' %} -{% else %} - {{ exceptions.raise_compiler_error( - "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" ~ gaps ~"'.'" - ) }} -{% endif %} -{% if not zero_length_range_allowed %} - {% set allow_zero_length_operator='<' %} - {% set allow_zero_length_operator_in_words='less_than' %} -{% elif zero_length_range_allowed %} - {% set allow_zero_length_operator='<=' %} - {% set allow_zero_length_operator_in_words='less_than_or_equal_to' %} -{% else %} - {{ exceptions.raise_compiler_error( - "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" ~ zero_length_range_allowed ~"'.'" - ) }} -{% endif %} - -{% set partition_clause="partition by " ~ partition_by if partition_by else '' %} - -with window_functions as ( - - select - {% if partition_by %} - {{ partition_by }} as partition_by_col, - {% endif %} - {{ lower_bound_column }} as lower_bound, - {{ upper_bound_column }} as upper_bound, - - lead({{ lower_bound_column }}) over ( - {{ partition_clause }} - order by {{ lower_bound_column }} - ) as next_lower_bound, - - row_number() over ( - {{ partition_clause }} - order by {{ lower_bound_column }} desc - ) = 1 as is_last_record - - from {{ model }} - -), - -calc as ( - -- We want to return records where one of our assumptions fails, so we'll use - -- the `not` function with `and` statements so we can write our assumptions nore cleanly - select - *, - - -- For each record: lower_bound should be < upper_bound. - -- Coalesce it to return an error on the null case (implicit assumption - -- these columns are not_null) - coalesce( - lower_bound {{ allow_zero_length_operator }} upper_bound, - false - ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, - - -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. - -- Coalesce it to handle null cases for the last record. - coalesce( - upper_bound {{ allow_gaps_operator }} next_lower_bound, - is_last_record, - false - ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound - - from window_functions - -), - -validation_errors as ( - - select - * - from calc - - where not( - -- THE FOLLOWING SHOULD BE TRUE -- - lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound - and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound - ) -) - -select * from validation_errors +{% macro default__test_mutually_exclusive_ranges( + model, + lower_bound_column, + upper_bound_column, + partition_by=None, + gaps="allowed", + zero_length_range_allowed=False +) %} + {% if gaps == "not_allowed" %} + {% set allow_gaps_operator = "=" %} + {% set allow_gaps_operator_in_words = "equal_to" %} + {% elif gaps == "allowed" %} + {% set allow_gaps_operator = "<=" %} + {% set allow_gaps_operator_in_words = "less_than_or_equal_to" %} + {% elif gaps == "required" %} + {% set allow_gaps_operator = "<" %} + {% set allow_gaps_operator_in_words = "less_than" %} + {% else %} + {{ + exceptions.raise_compiler_error( + "`gaps` argument for mutually_exclusive_ranges test must be one of ['not_allowed', 'allowed', 'required'] Got: '" + ~ gaps + ~ "'.'" + ) + }} + {% endif %} + {% if not zero_length_range_allowed %} + {% set allow_zero_length_operator = "<" %} + {% set allow_zero_length_operator_in_words = "less_than" %} + {% elif zero_length_range_allowed %} + {% set allow_zero_length_operator = "<=" %} + {% set allow_zero_length_operator_in_words = "less_than_or_equal_to" %} + {% else %} + {{ + exceptions.raise_compiler_error( + "`zero_length_range_allowed` argument for mutually_exclusive_ranges test must be one of [true, false] Got: '" + ~ zero_length_range_allowed + ~ "'.'" + ) + }} + {% endif %} + + {% set partition_clause = "partition by " ~ partition_by if partition_by else "" %} + + with + window_functions as ( + + select + {% if partition_by %} + {{ partition_by }} as partition_by_col, + {% endif %} + {{ lower_bound_column }} as lower_bound, + {{ upper_bound_column }} as upper_bound, + + lead({{ lower_bound_column }}) over ( + {{ partition_clause }} order by {{ lower_bound_column }} + ) as next_lower_bound, + + row_number() over ( + {{ partition_clause }} order by {{ lower_bound_column }} desc + ) + = 1 as is_last_record + + from {{ model }} + + ), + + calc as ( + -- We want to return records where one of our assumptions fails, so we'll + -- use + -- the `not` function with `and` statements so we can write our + -- assumptions nore cleanly + select + *, + + -- For each record: lower_bound should be < upper_bound. + -- Coalesce it to return an error on the null case (implicit assumption + -- these columns are not_null) + coalesce( + lower_bound {{ allow_zero_length_operator }} upper_bound, false + ) as lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound, + + -- For each record: upper_bound {{ allow_gaps_operator }} the next lower_bound. + -- Coalesce it to handle null cases for the last record. + coalesce( + upper_bound {{ allow_gaps_operator }} next_lower_bound, + is_last_record, + false + ) as upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + + from window_functions + + ), + + validation_errors as ( + + select * + from calc + + where + not ( + -- THE FOLLOWING SHOULD BE TRUE -- + lower_bound_{{ allow_zero_length_operator_in_words }}_upper_bound + and upper_bound_{{ allow_gaps_operator_in_words }}_next_lower_bound + ) + ) + + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/not_accepted_values.sql b/macros/generic_tests/not_accepted_values.sql index ab24188f..688efb5c 100644 --- a/macros/generic_tests/not_accepted_values.sql +++ b/macros/generic_tests/not_accepted_values.sql @@ -1,37 +1,39 @@ {% test not_accepted_values(model, column_name, values, quote=True) %} - {{ return(adapter.dispatch('test_not_accepted_values', 'dbt_utils')(model, column_name, values, quote)) }} + {{ + return( + adapter.dispatch("test_not_accepted_values", "dbt_utils")( + model, column_name, values, quote + ) + ) + }} {% endtest %} {% macro default__test_not_accepted_values(model, column_name, values, quote=True) %} -with all_values as ( + with + all_values as ( - select distinct - {{ column_name }} as value_field + select distinct {{ column_name }} as value_field from {{ model }} - from {{ model }} + ), -), + validation_errors as ( -validation_errors as ( + select value_field - select - value_field + from all_values + where + value_field in ( + {% for value in values -%} + {% if quote -%}'{{ value }}' + {%- else -%}{{ value }} + {%- endif -%} + {%- if not loop.last -%},{%- endif %} + {%- endfor %} + ) - from all_values - where value_field in ( - {% for value in values -%} - {% if quote -%} - '{{ value }}' - {%- else -%} - {{ value }} - {%- endif -%} - {%- if not loop.last -%},{%- endif %} - {%- endfor %} ) -) - -select * -from validation_errors + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/not_constant.sql b/macros/generic_tests/not_constant.sql index 781ed7d9..48397170 100644 --- a/macros/generic_tests/not_constant.sql +++ b/macros/generic_tests/not_constant.sql @@ -1,19 +1,20 @@ - {% test not_constant(model, column_name) %} - {{ return(adapter.dispatch('test_not_constant', 'dbt_utils')(model, column_name)) }} + {{ + return( + adapter.dispatch("test_not_constant", "dbt_utils")(model, column_name) + ) + }} {% endtest %} {% macro default__test_not_constant(model, column_name) %} + select + {# In TSQL, subquery aggregate columns need aliases #} + {# thus: a filler col name, 'filler_column' #} + count(distinct {{ column_name }}) as filler_column -select - {# In TSQL, subquery aggregate columns need aliases #} - {# thus: a filler col name, 'filler_column' #} - count(distinct {{ column_name }}) as filler_column - -from {{ model }} - -having count(distinct {{ column_name }}) = 1 + from {{ model }} + having count(distinct {{ column_name }}) = 1 {% endmacro %} diff --git a/macros/generic_tests/not_null_proportion.sql b/macros/generic_tests/not_null_proportion.sql index 45b9050c..b9e6da97 100644 --- a/macros/generic_tests/not_null_proportion.sql +++ b/macros/generic_tests/not_null_proportion.sql @@ -1,26 +1,32 @@ {% macro test_not_null_proportion(model) %} - {{ return(adapter.dispatch('test_not_null_proportion', 'dbt_utils')(model, **kwargs)) }} + {{ + return( + adapter.dispatch("test_not_null_proportion", "dbt_utils")(model, **kwargs) + ) + }} {% endmacro %} {% macro default__test_not_null_proportion(model) %} -{% set column_name = kwargs.get('column_name', kwargs.get('arg')) %} -{% set at_least = kwargs.get('at_least', kwargs.get('arg')) %} -{% set at_most = kwargs.get('at_most', kwargs.get('arg', 1)) %} + {% set column_name = kwargs.get("column_name", kwargs.get("arg")) %} + {% set at_least = kwargs.get("at_least", kwargs.get("arg")) %} + {% set at_most = kwargs.get("at_most", kwargs.get("arg", 1)) %} -with validation as ( - select - sum(case when {{ column_name }} is null then 0 else 1 end) / cast(count(*) as numeric) as not_null_proportion - from {{ model }} -), -validation_errors as ( - select - not_null_proportion - from validation - where not_null_proportion < {{ at_least }} or not_null_proportion > {{ at_most }} -) -select - * -from validation_errors + with + validation as ( + select + sum(case when {{ column_name }} is null then 0 else 1 end) + / cast(count(*) as numeric) as not_null_proportion + from {{ model }} + ), + validation_errors as ( + select not_null_proportion + from validation + where + not_null_proportion < {{ at_least }} + or not_null_proportion > {{ at_most }} + ) + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/recency.sql b/macros/generic_tests/recency.sql index cb9a8de6..666f8e7a 100644 --- a/macros/generic_tests/recency.sql +++ b/macros/generic_tests/recency.sql @@ -1,24 +1,24 @@ {% test recency(model, field, datepart, interval) %} - {{ return(adapter.dispatch('test_recency', 'dbt_utils')(model, field, datepart, interval)) }} + {{ + return( + adapter.dispatch("test_recency", "dbt_utils")( + model, field, datepart, interval + ) + ) + }} {% endtest %} {% macro default__test_recency(model, field, datepart, interval) %} -{% set threshold = dbt_utils.dateadd(datepart, interval * -1, dbt_utils.current_timestamp()) %} + {% set threshold = dbt_utils.dateadd( + datepart, interval * -1, dbt_utils.current_timestamp() + ) %} -with recency as ( + with recency as (select max({{ field }}) as most_recent from {{ model }}) - select max({{field}}) as most_recent - from {{ model }} + select most_recent, {{ threshold }} as threshold -) - -select - - most_recent, - {{ threshold }} as threshold - -from recency -where most_recent < {{ threshold }} + from recency + where most_recent < {{ threshold }} {% endmacro %} diff --git a/macros/generic_tests/relationships_where.sql b/macros/generic_tests/relationships_where.sql index c35a380a..927e6706 100644 --- a/macros/generic_tests/relationships_where.sql +++ b/macros/generic_tests/relationships_where.sql @@ -1,51 +1,55 @@ -{% test relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} - {{ return(adapter.dispatch('test_relationships_where', 'dbt_utils')(model, column_name, to, field, from_condition, to_condition)) }} +{% test relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} + {{ + return( + adapter.dispatch("test_relationships_where", "dbt_utils")( + model, column_name, to, field, from_condition, to_condition + ) + ) + }} {% endtest %} -{% macro default__test_relationships_where(model, column_name, to, field, from_condition="1=1", to_condition="1=1") %} +{% macro default__test_relationships_where( + model, column_name, to, field, from_condition="1=1", to_condition="1=1" +) %} -{# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} -{# ref https://stackoverflow.com/a/7170753/3842610 #} + {# T-SQL has no boolean data type so we use 1=1 which returns TRUE #} + {# ref https://stackoverflow.com/a/7170753/3842610 #} + with + left_table as ( -with left_table as ( + select {{ column_name }} as id - select - {{column_name}} as id + from {{ model }} - from {{model}} + where {{ column_name }} is not null and {{ from_condition }} - where {{column_name}} is not null - and {{from_condition}} + ), -), + right_table as ( -right_table as ( + select {{ field }} as id - select - {{field}} as id + from {{ to }} - from {{to}} + where {{ field }} is not null and {{ to_condition }} - where {{field}} is not null - and {{to_condition}} + ), -), + exceptions as ( -exceptions as ( + select left_table.id, right_table.id as right_id - select - left_table.id, - right_table.id as right_id + from left_table - from left_table + left join right_table on left_table.id = right_table.id - left join right_table - on left_table.id = right_table.id + where right_table.id is null - where right_table.id is null + ) -) - -select * from exceptions + select * + from exceptions {% endmacro %} diff --git a/macros/generic_tests/sequential_values.sql b/macros/generic_tests/sequential_values.sql index 8ddae707..71662749 100644 --- a/macros/generic_tests/sequential_values.sql +++ b/macros/generic_tests/sequential_values.sql @@ -1,35 +1,57 @@ {% test sequential_values(model, column_name, interval=1, datepart=None) %} - {{ return(adapter.dispatch('test_sequential_values', 'dbt_utils')(model, column_name, interval, datepart)) }} + {{ + return( + adapter.dispatch("test_sequential_values", "dbt_utils")( + model, column_name, interval, datepart + ) + ) + }} {% endtest %} -{% macro default__test_sequential_values(model, column_name, interval=1, datepart=None) %} - -{% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} - -with windowed as ( - - select - {{ column_name }}, - lag({{ column_name }}) over ( - order by {{ column_name }} - ) as {{ previous_column_name }} - from {{ model }} -), - -validation_errors as ( - select - * - from windowed - {% if datepart %} - where not(cast({{ column_name }} as {{ dbt_utils.type_timestamp() }})= cast({{ dbt_utils.dateadd(datepart, interval, previous_column_name) }} as {{ dbt_utils.type_timestamp() }})) - {% else %} - where not({{ column_name }} = {{ previous_column_name }} + {{ interval }}) - {% endif %} -) - -select * -from validation_errors +{% macro default__test_sequential_values( + model, column_name, interval=1, datepart=None +) %} + + {% set previous_column_name = "previous_" ~ dbt_utils.slugify(column_name) %} + + with + windowed as ( + + select + {{ column_name }}, + lag({{ column_name }}) over ( + order by {{ column_name }} + ) as {{ previous_column_name }} + from {{ model }} + ), + + validation_errors as ( + select * + from windowed + {% if datepart %} + where + not ( + cast( + {{ column_name }} as {{ dbt_utils.type_timestamp() }} + ) = cast( + {{ + dbt_utils.dateadd( + datepart, interval, previous_column_name + ) + }} as {{ dbt_utils.type_timestamp() }} + ) + ) + {% else %} + where + not ( + {{ column_name }} = {{ previous_column_name }} + {{ interval }} + ) + {% endif %} + ) + + select * + from validation_errors {% endmacro %} diff --git a/macros/generic_tests/test_not_null_where.sql b/macros/generic_tests/test_not_null_where.sql index d5dbf6c5..39b68b19 100644 --- a/macros/generic_tests/test_not_null_where.sql +++ b/macros/generic_tests/test_not_null_where.sql @@ -1,12 +1,13 @@ -{% test not_null_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.not_null_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_not_null_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_not_null_where(model, column_name) %} - {{ return(test_not_null(model, column_name)) }} -{% endmacro %} +{% test not_null_where(model, column_name) %} + {%- set deprecation_warning = " Warning: `dbt_utils.not_null_where` is no longer supported. Starting in dbt v0.20.0, the built-in `not_null` test supports a `where` config. " -%} + {%- do exceptions.warn(deprecation_warning) -%} + {{ + return( + adapter.dispatch("test_not_null_where", "dbt_utils")(model, column_name) + ) + }} +{% endtest %} + +{% macro default__test_not_null_where(model, column_name) %} + {{ return(test_not_null(model, column_name)) }} +{% endmacro %} diff --git a/macros/generic_tests/test_unique_where.sql b/macros/generic_tests/test_unique_where.sql index e752d7b2..f163cf8e 100644 --- a/macros/generic_tests/test_unique_where.sql +++ b/macros/generic_tests/test_unique_where.sql @@ -1,12 +1,13 @@ -{% test unique_where(model, column_name) %} - {%- set deprecation_warning = ' - Warning: `dbt_utils.unique_where` is no longer supported. - Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. - ' -%} - {%- do exceptions.warn(deprecation_warning) -%} - {{ return(adapter.dispatch('test_unique_where', 'dbt_utils')(model, column_name)) }} -{% endtest %} - -{% macro default__test_unique_where(model, column_name) %} - {{ return(test_unique(model, column_name)) }} -{% endmacro %} +{% test unique_where(model, column_name) %} + {%- set deprecation_warning = " Warning: `dbt_utils.unique_where` is no longer supported. Starting in dbt v0.20.0, the built-in `unique` test supports a `where` config. " -%} + {%- do exceptions.warn(deprecation_warning) -%} + {{ + return( + adapter.dispatch("test_unique_where", "dbt_utils")(model, column_name) + ) + }} +{% endtest %} + +{% macro default__test_unique_where(model, column_name) %} + {{ return(test_unique(model, column_name)) }} +{% endmacro %} diff --git a/macros/generic_tests/unique_combination_of_columns.sql b/macros/generic_tests/unique_combination_of_columns.sql index 74ccf5c6..83c28f12 100644 --- a/macros/generic_tests/unique_combination_of_columns.sql +++ b/macros/generic_tests/unique_combination_of_columns.sql @@ -1,37 +1,48 @@ -{% test unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} - {{ return(adapter.dispatch('test_unique_combination_of_columns', 'dbt_utils')(model, combination_of_columns, quote_columns)) }} +{% test unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} + {{ + return( + adapter.dispatch("test_unique_combination_of_columns", "dbt_utils")( + model, combination_of_columns, quote_columns + ) + ) + }} {% endtest %} -{% macro default__test_unique_combination_of_columns(model, combination_of_columns, quote_columns=false) %} +{% macro default__test_unique_combination_of_columns( + model, combination_of_columns, quote_columns=false +) %} -{% if not quote_columns %} - {%- set column_list=combination_of_columns %} -{% elif quote_columns %} - {%- set column_list=[] %} + {% if not quote_columns %} {%- set column_list = combination_of_columns %} + {% elif quote_columns %} + {%- set column_list = [] %} {% for column in combination_of_columns -%} - {% set column_list = column_list.append( adapter.quote(column) ) %} + {% set column_list = column_list.append(adapter.quote(column)) %} {%- endfor %} -{% else %} - {{ exceptions.raise_compiler_error( - "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" ~ quote ~"'.'" - ) }} -{% endif %} - -{%- set columns_csv=column_list | join(', ') %} - - -with validation_errors as ( - - select - {{ columns_csv }} - from {{ model }} - group by {{ columns_csv }} - having count(*) > 1 - -) - -select * -from validation_errors - + {% else %} + {{ + exceptions.raise_compiler_error( + "`quote_columns` argument for unique_combination_of_columns test must be one of [True, False] Got: '" + ~ quote + ~ "'.'" + ) + }} + {% endif %} + + {%- set columns_csv = column_list | join(", ") %} + + with + validation_errors as ( + + select {{ columns_csv }} + from {{ model }} + group by {{ columns_csv }} + having count(*) > 1 + + ) + + select * + from validation_errors {% endmacro %} diff --git a/macros/jinja_helpers/log_info.sql b/macros/jinja_helpers/log_info.sql index 52b4b4a5..0c8c7675 100644 --- a/macros/jinja_helpers/log_info.sql +++ b/macros/jinja_helpers/log_info.sql @@ -1,5 +1,5 @@ {% macro log_info(message) %} - {{ return(adapter.dispatch('log_info', 'dbt_utils')(message)) }} + {{ return(adapter.dispatch("log_info", "dbt_utils")(message)) }} {% endmacro %} {% macro default__log_info(message) %} diff --git a/macros/jinja_helpers/pretty_log_format.sql b/macros/jinja_helpers/pretty_log_format.sql index fe580d13..115b38b2 100644 --- a/macros/jinja_helpers/pretty_log_format.sql +++ b/macros/jinja_helpers/pretty_log_format.sql @@ -1,7 +1,7 @@ {% macro pretty_log_format(message) %} - {{ return(adapter.dispatch('pretty_log_format', 'dbt_utils')(message)) }} + {{ return(adapter.dispatch("pretty_log_format", "dbt_utils")(message)) }} {% endmacro %} {% macro default__pretty_log_format(message) %} - {{ return( dbt_utils.pretty_time() ~ ' + ' ~ message) }} + {{ return(dbt_utils.pretty_time() ~ " + " ~ message) }} {% endmacro %} diff --git a/macros/jinja_helpers/pretty_time.sql b/macros/jinja_helpers/pretty_time.sql index bad37efe..285eda67 100644 --- a/macros/jinja_helpers/pretty_time.sql +++ b/macros/jinja_helpers/pretty_time.sql @@ -1,7 +1,7 @@ -{% macro pretty_time(format='%H:%M:%S') %} - {{ return(adapter.dispatch('pretty_time', 'dbt_utils')(format)) }} +{% macro pretty_time(format="%H:%M:%S") %} + {{ return(adapter.dispatch("pretty_time", "dbt_utils")(format)) }} {% endmacro %} -{% macro default__pretty_time(format='%H:%M:%S') %} +{% macro default__pretty_time(format="%H:%M:%S") %} {{ return(modules.datetime.datetime.now().strftime(format)) }} {% endmacro %} diff --git a/macros/jinja_helpers/slugify.sql b/macros/jinja_helpers/slugify.sql index 1b3c7272..b5888adc 100644 --- a/macros/jinja_helpers/slugify.sql +++ b/macros/jinja_helpers/slugify.sql @@ -1,12 +1,12 @@ {% macro slugify(string) %} -{#- Lower case the string -#} -{% set string = string | lower %} -{#- Replace spaces and dashes with underscores -#} -{% set string = modules.re.sub('[ -]+', '_', string) %} -{#- Only take letters, numbers, and underscores -#} -{% set string = modules.re.sub('[^a-z0-9_]+', '', string) %} + {#- Lower case the string -#} + {% set string = string | lower %} + {#- Replace spaces and dashes with underscores -#} + {% set string = modules.re.sub("[ -]+", "_", string) %} + {#- Only take letters, numbers, and underscores -#} + {% set string = modules.re.sub("[^a-z0-9_]+", "", string) %} -{{ return(string) }} + {{ return(string) }} {% endmacro %} diff --git a/macros/materializations/insert_by_period_materialization.sql b/macros/materializations/insert_by_period_materialization.sql index 5a15c815..a4cb0906 100644 --- a/macros/materializations/insert_by_period_materialization.sql +++ b/macros/materializations/insert_by_period_materialization.sql @@ -1,189 +1,276 @@ -{% macro get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - {{ return(adapter.dispatch('get_period_boundaries', 'dbt_utils')(target_schema, target_table, timestamp_field, start_date, stop_date, period)) }} +{% macro get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period +) -%} + {{ + return( + adapter.dispatch("get_period_boundaries", "dbt_utils")( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period, + ) + ) + }} {% endmacro %} -{% macro default__get_period_boundaries(target_schema, target_table, timestamp_field, start_date, stop_date, period) -%} - - {% call statement('period_boundaries', fetch_result=True) -%} - with data as ( - select - coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp as start_timestamp, - coalesce( - {{dbt_utils.dateadd('millisecond', +{% macro default__get_period_boundaries( + target_schema, + target_table, + timestamp_field, + start_date, + stop_date, + period +) -%} + + {% call statement("period_boundaries", fetch_result=True) -%} + with + data as ( + select + coalesce(max("{{timestamp_field}}"), '{{start_date}}')::timestamp + as start_timestamp, + coalesce( + {{ + dbt_utils.dateadd( + "millisecond", -1, - "nullif('" ~ stop_date ~ "','')::timestamp")}}, - {{dbt_utils.current_timestamp()}} - ) as stop_timestamp - from "{{target_schema}}"."{{target_table}}" - ) - - select - start_timestamp, - stop_timestamp, - {{dbt_utils.datediff('start_timestamp', - 'stop_timestamp', - period)}} + 1 as num_periods - from data - {%- endcall %} + "nullif('" ~ stop_date ~ "','')::timestamp", + ) + }}, + {{ dbt_utils.current_timestamp() }} + ) as stop_timestamp + from "{{target_schema}}"."{{target_table}}" + ) + + select + start_timestamp, + stop_timestamp, + {{ dbt_utils.datediff("start_timestamp", "stop_timestamp", period) }} + + 1 as num_periods + from data + {%- endcall %} {%- endmacro %} -{% macro get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} - {{ return(adapter.dispatch('get_period_sql', 'dbt_utils')(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset)) }} +{% macro get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset +) -%} + {{ + return( + adapter.dispatch("get_period_sql", "dbt_utils")( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset, + ) + ) + }} {% endmacro %} -{% macro default__get_period_sql(target_cols_csv, sql, timestamp_field, period, start_timestamp, stop_timestamp, offset) -%} - - {%- set period_filter -%} +{% macro default__get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + offset +) -%} + + {%- set period_filter -%} ("{{timestamp_field}}" > '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' and "{{timestamp_field}}" <= '{{start_timestamp}}'::timestamp + interval '{{offset}} {{period}}' + interval '1 {{period}}' and "{{timestamp_field}}" < '{{stop_timestamp}}'::timestamp) - {%- endset -%} + {%- endset -%} - {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} + {%- set filtered_sql = sql | replace("__PERIOD_FILTER__", period_filter) -%} - select - {{target_cols_csv}} - from ( - {{filtered_sql}} - ) + select {{ target_cols_csv }} + from ({{ filtered_sql }}) {%- endmacro %} {% materialization insert_by_period, default -%} - {%- set timestamp_field = config.require('timestamp_field') -%} - {%- set start_date = config.require('start_date') -%} - {%- set stop_date = config.get('stop_date') or '' -%} - {%- set period = config.get('period') or 'week' -%} + {%- set timestamp_field = config.require("timestamp_field") -%} + {%- set start_date = config.require("start_date") -%} + {%- set stop_date = config.get("stop_date") or "" -%} + {%- set period = config.get("period") or "week" -%} - {%- if sql.find('__PERIOD_FILTER__') == -1 -%} - {%- set error_message -%} + {%- if sql.find("__PERIOD_FILTER__") == -1 -%} + {%- set error_message -%} Model '{{ model.unique_id }}' does not include the required string '__PERIOD_FILTER__' in its sql - {%- endset -%} - {{ exceptions.raise_compiler_error(error_message) }} - {%- endif -%} - - {%- set identifier = model['name'] -%} - - {%- set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) -%} - {%- set target_relation = api.Relation.create(identifier=identifier, schema=schema, type='table') -%} - - {%- set non_destructive_mode = (flags.NON_DESTRUCTIVE == True) -%} - {%- set full_refresh_mode = (flags.FULL_REFRESH == True) -%} - - {%- set exists_as_table = (old_relation is not none and old_relation.is_table) -%} - {%- set exists_not_as_table = (old_relation is not none and not old_relation.is_table) -%} + {%- endset -%} + {{ exceptions.raise_compiler_error(error_message) }} + {%- endif -%} + + {%- set identifier = model["name"] -%} + + {%- set old_relation = adapter.get_relation( + database=database, schema=schema, identifier=identifier + ) -%} + {%- set target_relation = api.Relation.create( + identifier=identifier, schema=schema, type="table" + ) -%} + + {%- set non_destructive_mode = flags.NON_DESTRUCTIVE == True -%} + {%- set full_refresh_mode = flags.FULL_REFRESH == True -%} + + {%- set exists_as_table = old_relation is not none and old_relation.is_table -%} + {%- set exists_not_as_table = ( + old_relation is not none and not old_relation.is_table + ) -%} + + {%- set should_truncate = ( + non_destructive_mode and full_refresh_mode and exists_as_table + ) -%} + {%- set should_drop = not should_truncate and ( + full_refresh_mode or exists_not_as_table + ) -%} + {%- set force_create = flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE -%} + + -- setup + {% if old_relation is none -%} + -- noop + {%- elif should_truncate -%} {{ adapter.truncate_relation(old_relation) }} + {%- elif should_drop -%} + {{ adapter.drop_relation(old_relation) }} {%- set old_relation = none -%} + {%- endif %} - {%- set should_truncate = (non_destructive_mode and full_refresh_mode and exists_as_table) -%} - {%- set should_drop = (not should_truncate and (full_refresh_mode or exists_not_as_table)) -%} - {%- set force_create = (flags.FULL_REFRESH and not flags.NON_DESTRUCTIVE) -%} + {{ run_hooks(pre_hooks, inside_transaction=False) }} - -- setup - {% if old_relation is none -%} - -- noop - {%- elif should_truncate -%} - {{adapter.truncate_relation(old_relation)}} - {%- elif should_drop -%} - {{adapter.drop_relation(old_relation)}} - {%- set old_relation = none -%} - {%- endif %} - - {{run_hooks(pre_hooks, inside_transaction=False)}} - - -- `begin` happens here, so `commit` after it to finish the transaction - {{run_hooks(pre_hooks, inside_transaction=True)}} - {% call statement() -%} - begin; -- make extra sure we've closed out the transaction - commit; - {%- endcall %} - - -- build model - {% if force_create or old_relation is none -%} - {# Create an empty target table -#} - {% call statement('main') -%} - {%- set empty_sql = sql | replace("__PERIOD_FILTER__", 'false') -%} - {{create_table_as(False, target_relation, empty_sql)}} - {%- endcall %} - {%- endif %} - - {% set _ = dbt_utils.get_period_boundaries(schema, - identifier, - timestamp_field, - start_date, - stop_date, - period) %} - {%- set start_timestamp = load_result('period_boundaries')['data'][0][0] | string -%} - {%- set stop_timestamp = load_result('period_boundaries')['data'][0][1] | string -%} - {%- set num_periods = load_result('period_boundaries')['data'][0][2] | int -%} - - {% set target_columns = adapter.get_columns_in_relation(target_relation) %} - {%- set target_cols_csv = target_columns | map(attribute='quoted') | join(', ') -%} - {%- set loop_vars = {'sum_rows_inserted': 0} -%} - - -- commit each period as a separate transaction - {% for i in range(num_periods) -%} - {%- set msg = "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) -%} - {{ dbt_utils.log_info(msg) }} - - {%- set tmp_identifier = model['name'] ~ '__dbt_incremental_period' ~ i ~ '_tmp' -%} - {%- set tmp_relation = api.Relation.create(identifier=tmp_identifier, - schema=schema, type='table') -%} + -- `begin` happens here, so `commit` after it to finish the transaction + {{ run_hooks(pre_hooks, inside_transaction=True) }} {% call statement() -%} - {% set tmp_table_sql = dbt_utils.get_period_sql(target_cols_csv, - sql, - timestamp_field, - period, - start_timestamp, - stop_timestamp, - i) %} - {{dbt.create_table_as(True, tmp_relation, tmp_table_sql)}} + begin -- make extra sure we've closed out the transaction + ; + commit + ; {%- endcall %} - {{adapter.expand_target_column_types(from_relation=tmp_relation, - to_relation=target_relation)}} - {%- set name = 'main-' ~ i -%} - {% call statement(name, fetch_result=True) -%} + -- build model + {% if force_create or old_relation is none -%} + {# Create an empty target table -#} + {% call statement("main") -%} + {%- set empty_sql = sql | replace("__PERIOD_FILTER__", "false") -%} + {{ create_table_as(False, target_relation, empty_sql) }} + {%- endcall %} + {%- endif %} + + {% set _ = dbt_utils.get_period_boundaries( + schema, identifier, timestamp_field, start_date, stop_date, period + ) %} + {%- set start_timestamp = ( + load_result("period_boundaries")["data"][0][0] | string + ) -%} + {%- set stop_timestamp = load_result("period_boundaries")["data"][0][1] | string -%} + {%- set num_periods = load_result("period_boundaries")["data"][0][2] | int -%} + + {% set target_columns = adapter.get_columns_in_relation(target_relation) %} + {%- set target_cols_csv = target_columns | map(attribute="quoted") | join(", ") -%} + {%- set loop_vars = {"sum_rows_inserted": 0} -%} + + -- commit each period as a separate transaction + {% for i in range(num_periods) -%} + {%- set msg = ( + "Running for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) + ) -%} + {{ dbt_utils.log_info(msg) }} + + {%- set tmp_identifier = ( + model["name"] ~ "__dbt_incremental_period" ~ i ~ "_tmp" + ) -%} + {%- set tmp_relation = api.Relation.create( + identifier=tmp_identifier, schema=schema, type="table" + ) -%} + {% call statement() -%} + {% set tmp_table_sql = dbt_utils.get_period_sql( + target_cols_csv, + sql, + timestamp_field, + period, + start_timestamp, + stop_timestamp, + i, + ) %} + {{ dbt.create_table_as(True, tmp_relation, tmp_table_sql) }} + {%- endcall %} + + {{ + adapter.expand_target_column_types( + from_relation=tmp_relation, to_relation=target_relation + ) + }} + {%- set name = "main-" ~ i -%} + {% call statement(name, fetch_result=True) -%} insert into {{target_relation}} ({{target_cols_csv}}) ( select - {{target_cols_csv}} + {{ target_cols_csv }} from {{tmp_relation.include(schema=False)}} ); - {%- endcall %} - {% set result = load_result('main-' ~ i) %} - {% if 'response' in result.keys() %} {# added in v0.19.0 #} - {% set rows_inserted = result['response']['rows_affected'] %} - {% else %} {# older versions #} - {% set rows_inserted = result['status'].split(" ")[2] | int %} - {% endif %} - - {%- set sum_rows_inserted = loop_vars['sum_rows_inserted'] + rows_inserted -%} - {%- if loop_vars.update({'sum_rows_inserted': sum_rows_inserted}) %} {% endif -%} - - {%- set msg = "Ran for " ~ period ~ " " ~ (i + 1) ~ " of " ~ (num_periods) ~ "; " ~ rows_inserted ~ " records inserted" -%} - {{ dbt_utils.log_info(msg) }} - - {%- endfor %} + {%- endcall %} + {% set result = load_result("main-" ~ i) %} + {% if "response" in result.keys() %} {# added in v0.19.0 #} + {% set rows_inserted = result["response"]["rows_affected"] %} + {% else %} {# older versions #} + {% set rows_inserted = result["status"].split(" ")[2] | int %} + {% endif %} + + {%- set sum_rows_inserted = loop_vars["sum_rows_inserted"] + rows_inserted -%} + {%- if loop_vars.update({"sum_rows_inserted": sum_rows_inserted}) %} + {% endif -%} + + {%- set msg = ( + "Ran for " + ~ period + ~ " " + ~ (i + 1) + ~ " of " + ~ (num_periods) + ~ "; " + ~ rows_inserted + ~ " records inserted" + ) -%} + {{ dbt_utils.log_info(msg) }} + + {%- endfor %} - {% call statement() -%} - begin; - {%- endcall %} + {% call statement() -%} + begin + ; + {%- endcall %} - {{run_hooks(post_hooks, inside_transaction=True)}} + {{ run_hooks(post_hooks, inside_transaction=True) }} - {% call statement() -%} - commit; - {%- endcall %} + {% call statement() -%} + commit + ; + {%- endcall %} - {{run_hooks(post_hooks, inside_transaction=False)}} + {{ run_hooks(post_hooks, inside_transaction=False) }} - {%- set status_string = "INSERT " ~ loop_vars['sum_rows_inserted'] -%} + {%- set status_string = "INSERT " ~ loop_vars["sum_rows_inserted"] -%} - {% call noop_statement('main', status_string) -%} + {% call noop_statement("main", status_string) -%} -- no-op - {%- endcall %} + {%- endcall %} - -- Return the relations created in this materialization - {{ return({'relations': [target_relation]}) }} + -- Return the relations created in this materialization + {{ return({"relations": [target_relation]}) }} {%- endmaterialization %} diff --git a/macros/sql/date_spine.sql b/macros/sql/date_spine.sql index 759f8d59..3fc4e1c8 100644 --- a/macros/sql/date_spine.sql +++ b/macros/sql/date_spine.sql @@ -1,75 +1,82 @@ {% macro get_intervals_between(start_date, end_date, datepart) -%} - {{ return(adapter.dispatch('get_intervals_between', 'dbt_utils')(start_date, end_date, datepart)) }} + {{ + return( + adapter.dispatch("get_intervals_between", "dbt_utils")( + start_date, end_date, datepart + ) + ) + }} {%- endmacro %} {% macro default__get_intervals_between(start_date, end_date, datepart) -%} - {%- call statement('get_intervals_between', fetch_result=True) %} + {%- call statement("get_intervals_between", fetch_result=True) %} - select {{dbt_utils.datediff(start_date, end_date, datepart)}} + select {{ dbt_utils.datediff(start_date, end_date, datepart) }} {%- endcall -%} - {%- set value_list = load_result('get_intervals_between') -%} + {%- set value_list = load_result("get_intervals_between") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} {{ return(values[0]) }} - {%- else -%} - {{ return(1) }} + {%- else -%} {{ return(1) }} {%- endif -%} {%- endmacro %} - - {% macro date_spine(datepart, start_date, end_date) %} - {{ return(adapter.dispatch('date_spine', 'dbt_utils')(datepart, start_date, end_date)) }} + {{ + return( + adapter.dispatch("date_spine", "dbt_utils")( + datepart, start_date, end_date + ) + ) + }} {%- endmacro %} {% macro default__date_spine(datepart, start_date, end_date) %} - -{# call as follows: + {# call as follows: date_spine( "day", "to_date('01/01/2016', 'mm/dd/yyyy')", "dateadd(week, 1, current_date)" ) #} + with + rawdata as ( + {{ + dbt_utils.generate_series( + dbt_utils.get_intervals_between(start_date, end_date, datepart) + ) + }} -with rawdata as ( + ), - {{dbt_utils.generate_series( - dbt_utils.get_intervals_between(start_date, end_date, datepart) - )}} + all_periods as ( -), + select + ( + {{ + dbt_utils.dateadd( + datepart, "row_number() over (order by 1) - 1", start_date + ) + }} + ) as date_{{ datepart }} + from rawdata -all_periods as ( + ), - select ( - {{ - dbt_utils.dateadd( - datepart, - "row_number() over (order by 1) - 1", - start_date - ) - }} - ) as date_{{datepart}} - from rawdata + filtered as ( -), + select * from all_periods where date_{{ datepart }} <= {{ end_date }} -filtered as ( + ) select * - from all_periods - where date_{{datepart}} <= {{ end_date }} - -) - -select * from filtered + from filtered {% endmacro %} diff --git a/macros/sql/deduplicate.sql b/macros/sql/deduplicate.sql index 9a3571a2..bc48255f 100644 --- a/macros/sql/deduplicate.sql +++ b/macros/sql/deduplicate.sql @@ -1,22 +1,28 @@ {%- macro deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} - {{ return(adapter.dispatch('deduplicate', 'dbt_utils')(relation, group_by, order_by=order_by, relation_alias=relation_alias)) }} + {{ + return( + adapter.dispatch("deduplicate", "dbt_utils")( + relation, group_by, order_by=order_by, relation_alias=relation_alias + ) + ) + }} {% endmacro %} -{%- macro default__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro default__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( - select - _inner.*, - row_number() over ( - partition by {{ group_by }} - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} - ) as rn - from {{ relation if relation_alias is none else relation_alias }} as _inner - ) as deduped + select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} + from + ( + select + _inner.*, + row_number() over ( + partition by {{ group_by }} + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} + ) as rn + from {{ relation if relation_alias is none else relation_alias }} as _inner + ) as deduped where deduped.rn = 1 {%- endmacro -%} @@ -26,21 +32,22 @@ -- clause in BigQuery: -- https://github.com/dbt-labs/dbt-utils/issues/335#issuecomment-788157572 #} -{%- macro bigquery__deduplicate(relation, group_by, order_by=none, relation_alias=none) -%} +{%- macro bigquery__deduplicate( + relation, group_by, order_by=none, relation_alias=none +) -%} - select - {{ dbt_utils.star(relation, relation_alias='deduped') | indent }} - from ( - select - array_agg ( - original - {% if order_by is not none -%} - order by {{ order_by }} - {%- endif %} - limit 1 - )[offset(0)] as deduped - from {{ relation if relation_alias is none else relation_alias }} as original - group by {{ group_by }} - ) + select {{ dbt_utils.star(relation, relation_alias="deduped") | indent }} + from + ( + select + array_agg( + original + {% if order_by is not none -%} order by {{ order_by }} {%- endif %} + limit 1 + )[offset(0)] as deduped + from + {{ relation if relation_alias is none else relation_alias }} as original + group by {{ group_by }} + ) {%- endmacro -%} diff --git a/macros/sql/generate_series.sql b/macros/sql/generate_series.sql index efcbd8ac..4aac864b 100644 --- a/macros/sql/generate_series.sql +++ b/macros/sql/generate_series.sql @@ -1,53 +1,55 @@ {% macro get_powers_of_two(upper_bound) %} - {{ return(adapter.dispatch('get_powers_of_two', 'dbt_utils')(upper_bound)) }} + {{ return(adapter.dispatch("get_powers_of_two", "dbt_utils")(upper_bound)) }} {% endmacro %} {% macro default__get_powers_of_two(upper_bound) %} {% if upper_bound <= 0 %} - {{ exceptions.raise_compiler_error("upper bound must be positive") }} + {{ exceptions.raise_compiler_error("upper bound must be positive") }} {% endif %} {% for _ in range(1, 100) %} - {% if upper_bound <= 2 ** loop.index %}{{ return(loop.index) }}{% endif %} + {% if upper_bound <= 2**loop.index %} {{ return(loop.index) }}{% endif %} {% endfor %} {% endmacro %} {% macro generate_series(upper_bound) %} - {{ return(adapter.dispatch('generate_series', 'dbt_utils')(upper_bound)) }} + {{ return(adapter.dispatch("generate_series", "dbt_utils")(upper_bound)) }} {% endmacro %} {% macro default__generate_series(upper_bound) %} {% set n = dbt_utils.get_powers_of_two(upper_bound) %} - with p as ( - select 0 as generated_number union all select 1 - ), unioned as ( + with + p as ( + select 0 as generated_number + union all + select 1 + ), + unioned as ( - select + select - {% for i in range(n) %} - p{{i}}.generated_number * power(2, {{i}}) - {% if not loop.last %} + {% endif %} - {% endfor %} - + 1 - as generated_number + {% for i in range(n) %} + p{{ i }}.generated_number * power(2, {{ i }}) + {% if not loop.last %} + {% endif %} + {% endfor %} + + 1 as generated_number - from + from - {% for i in range(n) %} - p as p{{i}} - {% if not loop.last %} cross join {% endif %} - {% endfor %} + {% for i in range(n) %} + p as p{{ i }} {% if not loop.last %} cross join {% endif %} + {% endfor %} - ) + ) select * from unioned - where generated_number <= {{upper_bound}} + where generated_number <= {{ upper_bound }} order by generated_number {% endmacro %} diff --git a/macros/sql/get_column_values.sql b/macros/sql/get_column_values.sql index f70890e2..a79c469f 100644 --- a/macros/sql/get_column_values.sql +++ b/macros/sql/get_column_values.sql @@ -1,15 +1,24 @@ -{% macro get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {{ return(adapter.dispatch('get_column_values', 'dbt_utils')(table, column, order_by, max_records, default)) }} +{% macro get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} + {{ + return( + adapter.dispatch("get_column_values", "dbt_utils")( + table, column, order_by, max_records, default + ) + ) + }} {% endmacro %} -{% macro default__get_column_values(table, column, order_by='count(*) desc', max_records=none, default=none) -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} +{% macro default__get_column_values( + table, column, order_by="count(*) desc", max_records=none, default=none +) -%} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} {%- if not execute -%} - {% set default = [] if not default %} - {{ return(default) }} + {% set default = [] if not default %} {{ return(default) }} {% endif %} - {%- do dbt_utils._is_ephemeral(table, 'get_column_values') -%} + {%- do dbt_utils._is_ephemeral(table, "get_column_values") -%} {# Not all relations are tables. Renaming for internal clarity without breaking functionality for anyone using named arguments #} {# TODO: Change the method signature in a future 0.x.0 release #} @@ -18,43 +27,52 @@ {# adapter.load_relation is a convenience wrapper to avoid building a Relation when we already have one #} {% set relation_exists = (load_relation(target_relation)) is not none %} - {%- call statement('get_column_values', fetch_result=true) %} + {%- call statement("get_column_values", fetch_result=true) %} {%- if not relation_exists and default is none -%} - {{ exceptions.raise_compiler_error("In get_column_values(): relation " ~ target_relation ~ " does not exist and no default value was provided.") }} + {{ + exceptions.raise_compiler_error( + "In get_column_values(): relation " + ~ target_relation + ~ " does not exist and no default value was provided." + ) + }} {%- elif not relation_exists and default is not none -%} - {{ log("Relation " ~ target_relation ~ " does not exist. Returning the default value: " ~ default) }} + {{ + log( + "Relation " + ~ target_relation + ~ " does not exist. Returning the default value: " + ~ default + ) + }} - {{ return(default) }} + {{ return(default) }} {%- else -%} - - select - {{ column }} as value + select {{ column }} as value from {{ target_relation }} group by {{ column }} order by {{ order_by }} - {% if max_records is not none %} - limit {{ max_records }} + {% if max_records is not none %} limit {{ max_records }} {% endif %} {% endif %} {%- endcall -%} - {%- set value_list = load_result('get_column_values') -%} + {%- set value_list = load_result("get_column_values") -%} - {%- if value_list and value_list['data'] -%} - {%- set values = value_list['data'] | map(attribute=0) | list %} + {%- if value_list and value_list["data"] -%} + {%- set values = value_list["data"] | map(attribute=0) | list %} {{ return(values) }} - {%- else -%} - {{ return(default) }} + {%- else -%} {{ return(default) }} {%- endif -%} {%- endmacro %} diff --git a/macros/sql/get_filtered_columns_in_relation.sql b/macros/sql/get_filtered_columns_in_relation.sql index 7f4af889..8a14f6eb 100644 --- a/macros/sql/get_filtered_columns_in_relation.sql +++ b/macros/sql/get_filtered_columns_in_relation.sql @@ -7,9 +7,7 @@ {%- do dbt_utils._is_ephemeral(from, 'get_filtered_columns_in_relation') -%} {# -- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('') }} - {% endif %} + {%- if not execute -%} {{ return("") }} {% endif %} {%- set include_cols = [] %} {%- set cols = adapter.get_columns_in_relation(from) -%} @@ -22,4 +20,4 @@ {{ return(include_cols) }} -{%- endmacro %} \ No newline at end of file +{%- endmacro %} diff --git a/macros/sql/get_query_results_as_dict.sql b/macros/sql/get_query_results_as_dict.sql index 6548f2dd..6c68991f 100644 --- a/macros/sql/get_query_results_as_dict.sql +++ b/macros/sql/get_query_results_as_dict.sql @@ -1,21 +1,20 @@ {% macro get_query_results_as_dict(query) %} - {{ return(adapter.dispatch('get_query_results_as_dict', 'dbt_utils')(query)) }} + {{ return(adapter.dispatch("get_query_results_as_dict", "dbt_utils")(query)) }} {% endmacro %} {% macro default__get_query_results_as_dict(query) %} -{# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} - - {%- call statement('get_query_results', fetch_result=True,auto_begin=false) -%} + {# This macro returns a dictionary of the form {column_name: (tuple_of_results)} #} + {%- call statement("get_query_results", fetch_result=True, auto_begin=false) -%} {{ query }} {%- endcall -%} - {% set sql_results={} %} + {% set sql_results = {} %} {%- if execute -%} - {% set sql_results_table = load_result('get_query_results').table.columns %} + {% set sql_results_table = load_result("get_query_results").table.columns %} {% for column_name, column in sql_results_table.items() %} {% do sql_results.update({column_name: column.values()}) %} {% endfor %} diff --git a/macros/sql/get_relations_by_pattern.sql b/macros/sql/get_relations_by_pattern.sql index 9325a883..b806bcc7 100644 --- a/macros/sql/get_relations_by_pattern.sql +++ b/macros/sql/get_relations_by_pattern.sql @@ -1,32 +1,45 @@ -{% macro get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_pattern', 'dbt_utils')(schema_pattern, table_pattern, exclude, database)) }} +{% macro get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} + {{ + return( + adapter.dispatch("get_relations_by_pattern", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) + ) + }} {% endmacro %} -{% macro default__get_relations_by_pattern(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro default__get_relations_by_pattern( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} + {%- call statement("get_tables", fetch_result=True) %} - {{ dbt_utils.get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude, database) }} + {{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude, database + ) + }} {%- endcall -%} - {%- set table_list = load_result('get_tables') -%} + {%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} + {%- if table_list and table_list["table"] -%} {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} + {%- for row in table_list["table"] -%} {%- set tbl_relation = api.Relation.create( database=database, schema=row.table_schema, identifier=row.table_name, - type=row.table_type + type=row.table_type, ) -%} {%- do tbl_relations.append(tbl_relation) -%} {%- endfor -%} {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} + {%- else -%} {{ return([]) }} {%- endif -%} {% endmacro %} diff --git a/macros/sql/get_relations_by_prefix.sql b/macros/sql/get_relations_by_prefix.sql index b6733c4b..59f1b90d 100644 --- a/macros/sql/get_relations_by_prefix.sql +++ b/macros/sql/get_relations_by_prefix.sql @@ -1,32 +1,41 @@ -{% macro get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_relations_by_prefix', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} + {{ + return( + adapter.dispatch("get_relations_by_prefix", "dbt_utils")( + schema, prefix, exclude, database + ) + ) + }} {% endmacro %} -{% macro default__get_relations_by_prefix(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_relations_by_prefix( + schema, prefix, exclude="", database=target.database +) %} - {%- call statement('get_tables', fetch_result=True) %} + {%- call statement("get_tables", fetch_result=True) %} - {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} + {{ dbt_utils.get_tables_by_prefix_sql(schema, prefix, exclude, database) }} {%- endcall -%} - {%- set table_list = load_result('get_tables') -%} + {%- set table_list = load_result("get_tables") -%} - {%- if table_list and table_list['table'] -%} + {%- if table_list and table_list["table"] -%} {%- set tbl_relations = [] -%} - {%- for row in table_list['table'] -%} + {%- for row in table_list["table"] -%} {%- set tbl_relation = api.Relation.create( database=database, schema=row.table_schema, identifier=row.table_name, - type=row.table_type + type=row.table_type, ) -%} {%- do tbl_relations.append(tbl_relation) -%} {%- endfor -%} {{ return(tbl_relations) }} - {%- else -%} - {{ return([]) }} + {%- else -%} {{ return([]) }} {%- endif -%} {% endmacro %} diff --git a/macros/sql/get_table_types_sql.sql b/macros/sql/get_table_types_sql.sql index 91573779..fe3eb4c1 100644 --- a/macros/sql/get_table_types_sql.sql +++ b/macros/sql/get_table_types_sql.sql @@ -1,31 +1,43 @@ {%- macro get_table_types_sql() -%} - {{ return(adapter.dispatch('get_table_types_sql', 'dbt_utils')()) }} + {{ return(adapter.dispatch("get_table_types_sql", "dbt_utils")()) }} {%- endmacro -%} {% macro default__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" + case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) + end as "table_type" {% endmacro %} {% macro postgres__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'FOREIGN' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as "table_type" + case + table_type + when 'BASE TABLE' + then 'table' + when 'FOREIGN' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) + end as "table_type" {% endmacro %} {% macro bigquery__get_table_types_sql() %} - case table_type - when 'BASE TABLE' then 'table' - when 'EXTERNAL TABLE' then 'external' - when 'MATERIALIZED VIEW' then 'materializedview' - else lower(table_type) - end as `table_type` -{% endmacro %} \ No newline at end of file + case + table_type + when 'BASE TABLE' + then 'table' + when 'EXTERNAL TABLE' + then 'external' + when 'MATERIALIZED VIEW' + then 'materializedview' + else lower(table_type) + end as `table_type` +{% endmacro %} diff --git a/macros/sql/get_tables_by_pattern_sql.sql b/macros/sql/get_tables_by_pattern_sql.sql index 4d5a8fc9..40603f1d 100644 --- a/macros/sql/get_tables_by_pattern_sql.sql +++ b/macros/sql/get_tables_by_pattern_sql.sql @@ -1,28 +1,41 @@ -{% macro get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_pattern_sql', 'dbt_utils') - (schema_pattern, table_pattern, exclude, database)) }} +{% macro get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} + {{ + return( + adapter.dispatch("get_tables_by_pattern_sql", "dbt_utils")( + schema_pattern, table_pattern, exclude, database + ) + ) + }} {% endmacro %} -{% macro default__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} - - select distinct - table_schema as "table_schema", - table_name as "table_name", - {{ dbt_utils.get_table_types_sql() }} - from {{ database }}.information_schema.tables - where table_schema ilike '{{ schema_pattern }}' +{% macro default__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} + + select distinct + table_schema as "table_schema", + table_name as "table_name", + {{ dbt_utils.get_table_types_sql() }} + from {{ database }}.information_schema.tables + where + table_schema ilike '{{ schema_pattern }}' and table_name ilike '{{ table_pattern }}' and table_name not ilike '{{ exclude }}' {% endmacro %} -{% macro bigquery__get_tables_by_pattern_sql(schema_pattern, table_pattern, exclude='', database=target.database) %} +{% macro bigquery__get_tables_by_pattern_sql( + schema_pattern, table_pattern, exclude="", database=target.database +) %} - {% if '%' in schema_pattern %} - {% set schemata=dbt_utils._bigquery__get_matching_schemata(schema_pattern, database) %} - {% else %} - {% set schemata=[schema_pattern] %} + {% if "%" in schema_pattern %} + {% set schemata = dbt_utils._bigquery__get_matching_schemata( + schema_pattern, database + ) %} + {% else %} {% set schemata = [schema_pattern] %} {% endif %} {% set sql %} @@ -54,17 +67,14 @@ where lower(schema_name) like lower('{{ schema_pattern }}') {% endset %} - {% set results=run_query(sql) %} + {% set results = run_query(sql) %} - {% set schemata=results.columns['schema_name'].values() %} + {% set schemata = results.columns["schema_name"].values() %} {{ return(schemata) }} - {% else %} - - {{ return([]) }} + {% else %} {{ return([]) }} {% endif %} - {% endmacro %} diff --git a/macros/sql/get_tables_by_prefix_sql.sql b/macros/sql/get_tables_by_prefix_sql.sql index f8fdfba6..d4b6dda7 100644 --- a/macros/sql/get_tables_by_prefix_sql.sql +++ b/macros/sql/get_tables_by_prefix_sql.sql @@ -1,14 +1,26 @@ -{% macro get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} - {{ return(adapter.dispatch('get_tables_by_prefix_sql', 'dbt_utils')(schema, prefix, exclude, database)) }} +{% macro get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} + {{ + return( + adapter.dispatch("get_tables_by_prefix_sql", "dbt_utils")( + schema, prefix, exclude, database + ) + ) + }} {% endmacro %} -{% macro default__get_tables_by_prefix_sql(schema, prefix, exclude='', database=target.database) %} +{% macro default__get_tables_by_prefix_sql( + schema, prefix, exclude="", database=target.database +) %} + + {{ + dbt_utils.get_tables_by_pattern_sql( + schema_pattern=schema, + table_pattern=prefix ~ "%", + exclude=exclude, + database=database, + ) + }} - {{ dbt_utils.get_tables_by_pattern_sql( - schema_pattern = schema, - table_pattern = prefix ~ '%', - exclude = exclude, - database = database - ) }} - {% endmacro %} diff --git a/macros/sql/groupby.sql b/macros/sql/groupby.sql index 68a68cf2..b0ae3f70 100644 --- a/macros/sql/groupby.sql +++ b/macros/sql/groupby.sql @@ -1,11 +1,10 @@ {%- macro group_by(n) -%} - {{ return(adapter.dispatch('group_by', 'dbt_utils')(n)) }} + {{ return(adapter.dispatch("group_by", "dbt_utils")(n)) }} {% endmacro %} {%- macro default__group_by(n) -%} - group by {% for i in range(1, n + 1) -%} - {{ i }}{{ ',' if not loop.last }} - {%- endfor -%} + group by + {% for i in range(1, n + 1) -%} {{ i }}{{ ',' if not loop.last }} {%- endfor -%} {%- endmacro -%} diff --git a/macros/sql/haversine_distance.sql b/macros/sql/haversine_distance.sql index 70f276ec..b3107c39 100644 --- a/macros/sql/haversine_distance.sql +++ b/macros/sql/haversine_distance.sql @@ -5,47 +5,68 @@ http://daynebatten.com/2015/09/latitude-longitude-distance-sql/ The arguments should be float type. #} +{% macro degrees_to_radians(degrees) -%} acos(-1) * {{ degrees }} / 180 {%- endmacro %} -{% macro degrees_to_radians(degrees) -%} - acos(-1) * {{degrees}} / 180 -{%- endmacro %} - -{% macro haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} - {{ return(adapter.dispatch('haversine_distance', 'dbt_utils')(lat1,lon1,lat2,lon2,unit)) }} +{% macro haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} + {{ + return( + adapter.dispatch("haversine_distance", "dbt_utils")( + lat1, lon1, lat2, lon2, unit + ) + ) + }} {% endmacro %} -{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} -{% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} -{% endif %} +{% macro default__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} + {%- if unit == "mi" %} {% set conversion_rate = 1 %} + {% elif unit == "km" %} {% set conversion_rate = 1.60934 %} + {% else %} + {{ + exceptions.raise_compiler_error( + "unit input must be one of 'mi' or 'km'. Got " ~ unit + ) + }} + {% endif %} - 2 * 3961 * asin(sqrt(power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + - cos(radians({{lat1}})) * cos(radians({{lat2}})) * - power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2))) * {{ conversion_rate }} + 2 + * 3961 + * asin( + sqrt( + power((sin(radians(({{ lat2 }} - {{ lat1 }}) / 2))), 2) + + cos(radians({{ lat1 }})) + * cos(radians({{ lat2 }})) + * power((sin(radians(({{ lon2 }} - {{ lon1 }}) / 2))), 2) + ) + ) + * {{ conversion_rate }} {%- endmacro %} - -{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit='mi') -%} -{% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} -{% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} -{% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} -{% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} -{%- if unit == 'mi' %} - {% set conversion_rate = 1 %} -{% elif unit == 'km' %} - {% set conversion_rate = 1.60934 %} -{% else %} - {{ exceptions.raise_compiler_error("unit input must be one of 'mi' or 'km'. Got " ~ unit) }} -{% endif %} - 2 * 3961 * asin(sqrt(power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + - cos({{ radians_lat1 }}) * cos({{ radians_lat2 }}) * - power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2))) * {{ conversion_rate }} +{% macro bigquery__haversine_distance(lat1, lon1, lat2, lon2, unit="mi") -%} + {% set radians_lat1 = dbt_utils.degrees_to_radians(lat1) %} + {% set radians_lat2 = dbt_utils.degrees_to_radians(lat2) %} + {% set radians_lon1 = dbt_utils.degrees_to_radians(lon1) %} + {% set radians_lon2 = dbt_utils.degrees_to_radians(lon2) %} + {%- if unit == "mi" %} {% set conversion_rate = 1 %} + {% elif unit == "km" %} {% set conversion_rate = 1.60934 %} + {% else %} + {{ + exceptions.raise_compiler_error( + "unit input must be one of 'mi' or 'km'. Got " ~ unit + ) + }} + {% endif %} + 2 + * 3961 + * asin( + sqrt( + power(sin(({{ radians_lat2 }} - {{ radians_lat1 }}) / 2), 2) + + cos({{ radians_lat1 }}) + * cos({{ radians_lat2 }}) + * power(sin(({{ radians_lon2 }} - {{ radians_lon1 }}) / 2), 2) + ) + ) + * {{ conversion_rate }} {%- endmacro %} - diff --git a/macros/sql/nullcheck.sql b/macros/sql/nullcheck.sql index 509d24f7..ba4b5a5b 100644 --- a/macros/sql/nullcheck.sql +++ b/macros/sql/nullcheck.sql @@ -1,21 +1,17 @@ {% macro nullcheck(cols) %} - {{ return(adapter.dispatch('nullcheck', 'dbt_utils')(cols)) }} + {{ return(adapter.dispatch("nullcheck", "dbt_utils")(cols)) }} {% endmacro %} {% macro default__nullcheck(cols) %} -{%- for col in cols %} + {%- for col in cols %} - {% if col.is_string() -%} + {% if col.is_string() -%} nullif({{ col.name }}, '') as {{ col.name }} - nullif({{col.name}},'') as {{col.name}} + {%- else -%} {{ col.name }} - {%- else -%} + {%- endif -%} - {{col.name}} + {%- if not loop.last -%}, {%- endif -%} - {%- endif -%} - -{%- if not loop.last -%} , {%- endif -%} - -{%- endfor -%} + {%- endfor -%} {% endmacro %} diff --git a/macros/sql/nullcheck_table.sql b/macros/sql/nullcheck_table.sql index c9ab5838..ef521b4c 100644 --- a/macros/sql/nullcheck_table.sql +++ b/macros/sql/nullcheck_table.sql @@ -1,14 +1,14 @@ {% macro nullcheck_table(relation) %} - {{ return(adapter.dispatch('nullcheck_table', 'dbt_utils')(relation)) }} + {{ return(adapter.dispatch("nullcheck_table", "dbt_utils")(relation)) }} {% endmacro %} {% macro default__nullcheck_table(relation) %} - {%- do dbt_utils._is_relation(relation, 'nullcheck_table') -%} - {%- do dbt_utils._is_ephemeral(relation, 'nullcheck_table') -%} - {% set cols = adapter.get_columns_in_relation(relation) %} + {%- do dbt_utils._is_relation(relation, "nullcheck_table") -%} + {%- do dbt_utils._is_ephemeral(relation, "nullcheck_table") -%} + {% set cols = adapter.get_columns_in_relation(relation) %} - select {{ dbt_utils.nullcheck(cols) }} - from {{relation}} + select {{ dbt_utils.nullcheck(cols) }} + from {{ relation }} {% endmacro %} diff --git a/macros/sql/pivot.sql b/macros/sql/pivot.sql index 88751062..33118ef7 100644 --- a/macros/sql/pivot.sql +++ b/macros/sql/pivot.sql @@ -39,48 +39,66 @@ Arguments: quote_identifiers: Whether to surround column aliases with double quotes, default is true distinct: Whether to use distinct in the aggregation, default is False #} - -{% macro pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {{ return(adapter.dispatch('pivot', 'dbt_utils')(column, values, alias, agg, cmp, prefix, suffix, then_value, else_value, quote_identifiers, distinct)) }} +{% macro pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False +) %} + {{ + return( + adapter.dispatch("pivot", "dbt_utils")( + column, + values, + alias, + agg, + cmp, + prefix, + suffix, + then_value, + else_value, + quote_identifiers, + distinct, + ) + ) + }} {% endmacro %} -{% macro default__pivot(column, - values, - alias=True, - agg='sum', - cmp='=', - prefix='', - suffix='', - then_value=1, - else_value=0, - quote_identifiers=True, - distinct=False) %} - {% for value in values %} - {{ agg }}( - {% if distinct %} distinct {% endif %} - case - when {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' - then {{ then_value }} - else {{ else_value }} - end - ) - {% if alias %} - {% if quote_identifiers %} - as {{ adapter.quote(prefix ~ value ~ suffix) }} - {% else %} - as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} - {% endif %} - {% endif %} - {% if not loop.last %},{% endif %} - {% endfor %} +{% macro default__pivot( + column, + values, + alias=True, + agg="sum", + cmp="=", + prefix="", + suffix="", + then_value=1, + else_value=0, + quote_identifiers=True, + distinct=False +) %} + {% for value in values %} + {{ agg }} ( + {% if distinct %}distinct {% endif %} + case + when + {{ column }} {{ cmp }} '{{ dbt_utils.escape_single_quotes(value) }}' + then {{ then_value }} + else {{ else_value }} + end + ) + {% if alias %} + {% if quote_identifiers %} as {{ adapter.quote(prefix ~ value ~ suffix) }} + {% else %} as {{ dbt_utils.slugify(prefix ~ value ~ suffix) }} + {% endif %} + {% endif %} + {% if not loop.last %},{% endif %} + {% endfor %} {% endmacro %} diff --git a/macros/sql/safe_add.sql b/macros/sql/safe_add.sql index 3b6195c8..82a5ce7f 100644 --- a/macros/sql/safe_add.sql +++ b/macros/sql/safe_add.sql @@ -2,19 +2,19 @@ {# needed for safe_add to allow for non-keyword arguments see SO post #} {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('safe_add', 'dbt_utils')(*varargs)) }} + {{ return(adapter.dispatch("safe_add", "dbt_utils")(*varargs)) }} {% endmacro %} {%- macro default__safe_add() -%} -{% set fields = [] %} + {% set fields = [] %} -{%- for field in varargs -%} + {%- for field in varargs -%} - {% do fields.append("coalesce(" ~ field ~ ", 0)") %} + {% do fields.append("coalesce(" ~ field ~ ", 0)") %} -{%- endfor -%} + {%- endfor -%} -{{ fields|join(' +\n ') }} + {{ fields | join(" +\n ") }} {%- endmacro -%} diff --git a/macros/sql/star.sql b/macros/sql/star.sql index 25e58969..a92ac6a7 100644 --- a/macros/sql/star.sql +++ b/macros/sql/star.sql @@ -6,15 +6,18 @@ {%- do dbt_utils._is_relation(from, 'star') -%} {%- do dbt_utils._is_ephemeral(from, 'star') -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} - {%- if not execute -%} - {{ return('*') }} - {% endif %} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. #} + {%- if not execute -%} {{ return("*") }} {% endif %} {%- for col in dbt_utils.get_filtered_columns_in_relation(from, except) %} - {%- if relation_alias %}{{ relation_alias }}.{% else %}{%- endif -%}{{ adapter.quote(col)|trim }} {%- if prefix!='' or suffix!='' %} as {{ adapter.quote(prefix ~ col ~ suffix)|trim }} {%- endif -%} - {%- if not loop.last %},{{ '\n ' }}{% endif %} + {%- if relation_alias %} {{ relation_alias }}. + {% else %} + {%- endif -%} {{ adapter.quote(col) | trim }} + {%- if prefix != "" or suffix != "" %} + as {{ adapter.quote(prefix ~ col ~ suffix) | trim }} + {%- endif -%} + {%- if not loop.last %},{{ "\n " }}{% endif %} {%- endfor -%} {%- endmacro %} diff --git a/macros/sql/surrogate_key.sql b/macros/sql/surrogate_key.sql index ba5c3e44..137d4583 100644 --- a/macros/sql/surrogate_key.sql +++ b/macros/sql/surrogate_key.sql @@ -2,50 +2,49 @@ {# needed for safe_add to allow for non-keyword arguments see SO post #} {# https://stackoverflow.com/questions/13944751/args-kwargs-in-jinja2-macros #} {% set frustrating_jinja_feature = varargs %} - {{ return(adapter.dispatch('surrogate_key', 'dbt_utils')(field_list, *varargs)) }} + {{ return(adapter.dispatch("surrogate_key", "dbt_utils")(field_list, *varargs)) }} {% endmacro %} {%- macro default__surrogate_key(field_list) -%} -{%- if varargs|length >= 1 or field_list is string %} + {%- if varargs | length >= 1 or field_list is string %} -{%- set error_message = ' -Warning: the `surrogate_key` macro now takes a single list argument instead of \ -multiple string arguments. Support for multiple string arguments will be \ -deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ -'.format(model.package_name, model.name) -%} + {%- set error_message = " Warning: the `surrogate_key` macro now takes a single list argument instead of \ multiple string arguments. Support for multiple string arguments will be \ deprecated in a future release of dbt-utils. The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name + ) -%} -{%- do exceptions.warn(error_message) -%} + {%- do exceptions.warn(error_message) -%} -{# first argument is not included in varargs, so add first element to field_list_xf #} -{%- set field_list_xf = [field_list] -%} + {# first argument is not included in varargs, so add first element to field_list_xf #} + {%- set field_list_xf = [field_list] -%} -{%- for field in varargs %} -{%- set _ = field_list_xf.append(field) -%} -{%- endfor -%} + {%- for field in varargs %} + {%- set _ = field_list_xf.append(field) -%} + {%- endfor -%} -{%- else -%} + {%- else -%} -{# if using list, just set field_list_xf as field_list #} -{%- set field_list_xf = field_list -%} - -{%- endif -%} + {# if using list, just set field_list_xf as field_list #} + {%- set field_list_xf = field_list -%} + {%- endif -%} -{%- set fields = [] -%} + {%- set fields = [] -%} -{%- for field in field_list_xf -%} + {%- for field in field_list_xf -%} - {%- set _ = fields.append( - "coalesce(cast(" ~ field ~ " as " ~ dbt_utils.type_string() ~ "), '')" - ) -%} + {%- set _ = fields.append( + "coalesce(cast(" + ~ field + ~ " as " + ~ dbt_utils.type_string() + ~ "), '')" + ) -%} - {%- if not loop.last %} - {%- set _ = fields.append("'-'") -%} - {%- endif -%} + {%- if not loop.last %} {%- set _ = fields.append("'-'") -%} {%- endif -%} -{%- endfor -%} + {%- endfor -%} -{{dbt_utils.hash(dbt_utils.concat(fields))}} + {{ dbt_utils.hash(dbt_utils.concat(fields)) }} {%- endmacro -%} diff --git a/macros/sql/union.sql b/macros/sql/union.sql index a7bf1d95..3527fa49 100644 --- a/macros/sql/union.sql +++ b/macros/sql/union.sql @@ -1,17 +1,37 @@ -{%- macro union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} - {{ return(adapter.dispatch('union_relations', 'dbt_utils')(relations, column_override, include, exclude, source_column_name)) }} +{%- macro union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation" +) -%} + {{ + return( + adapter.dispatch("union_relations", "dbt_utils")( + relations, column_override, include, exclude, source_column_name + ) + ) + }} {% endmacro %} -{%- macro default__union_relations(relations, column_override=none, include=[], exclude=[], source_column_name='_dbt_source_relation') -%} +{%- macro default__union_relations( + relations, + column_override=none, + include=[], + exclude=[], + source_column_name="_dbt_source_relation" +) -%} {%- if exclude and include -%} - {{ exceptions.raise_compiler_error("Both an exclude and include list were provided to the `union` macro. Only one is allowed") }} + {{ + exceptions.raise_compiler_error( + "Both an exclude and include list were provided to the `union` macro. Only one is allowed" + ) + }} {%- endif -%} - {#-- Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} - {%- if not execute %} - {{ return('') }} - {% endif -%} + {#- - Prevent querying of db in parsing mode. This works because this macro does not create any new refs. -#} + {%- if not execute %} {{ return("") }} {% endif -%} {%- set column_override = column_override if column_override is not none else {} -%} @@ -22,46 +42,46 @@ {%- do relation_columns.update({relation: []}) -%} - {%- do dbt_utils._is_relation(relation, 'union_relations') -%} - {%- do dbt_utils._is_ephemeral(relation, 'union_relations') -%} + {%- do dbt_utils._is_relation(relation, "union_relations") -%} + {%- do dbt_utils._is_ephemeral(relation, "union_relations") -%} {%- set cols = adapter.get_columns_in_relation(relation) -%} {%- for col in cols -%} - {#- If an exclude list was provided and the column is in the list, do nothing -#} - {%- if exclude and col.column in exclude -%} + {#- If an exclude list was provided and the column is in the list, do nothing -#} + {%- if exclude and col.column in exclude -%} - {#- If an include list was provided and the column is not in the list, do nothing -#} - {%- elif include and col.column not in include -%} + {#- If an include list was provided and the column is not in the list, do nothing -#} + {%- elif include and col.column not in include -%} - {#- Otherwise add the column to the column superset -#} - {%- else -%} + {#- Otherwise add the column to the column superset -#} + {%- else -%} - {#- update the list of columns in this relation -#} - {%- do relation_columns[relation].append(col.column) -%} + {#- update the list of columns in this relation -#} + {%- do relation_columns[relation].append(col.column) -%} - {%- if col.column in column_superset -%} + {%- if col.column in column_superset -%} - {%- set stored = column_superset[col.column] -%} - {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} + {%- set stored = column_superset[col.column] -%} + {%- if col.is_string() and stored.is_string() and col.string_size() > stored.string_size() -%} - {%- do column_superset.update({col.column: col}) -%} + {%- do column_superset.update({col.column: col}) -%} - {%- endif %} + {%- endif %} - {%- else -%} + {%- else -%} {%- do column_superset.update({col.column: col}) -%} - {%- do column_superset.update({col.column: col}) -%} + {%- endif -%} {%- endif -%} - {%- endif -%} - {%- endfor -%} {%- endfor -%} {%- set ordered_column_names = column_superset.keys() -%} - {% if (include | length > 0 or exclude | length > 0) and not column_superset.keys() %} + {% if ( + include | length > 0 or exclude | length > 0 + ) and not column_superset.keys() %} {%- set relations_string -%} {%- for relation in relations -%} {{ relation.name }} @@ -81,13 +101,21 @@ ( select - cast({{ dbt_utils.string_literal(relation) }} as {{ dbt_utils.type_string() }}) as {{ source_column_name }}, + cast( + {{ dbt_utils.string_literal(relation) }} + as {{ dbt_utils.type_string() }} + ) as {{ source_column_name }}, {% for col_name in ordered_column_names -%} {%- set col = column_superset[col_name] %} {%- set col_type = column_override.get(col.column, col.data_type) %} - {%- set col_name = adapter.quote(col_name) if col_name in relation_columns[relation] else 'null' %} - cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} {% if not loop.last %},{% endif -%} + {%- set col_name = ( + adapter.quote(col_name) + if col_name in relation_columns[relation] + else "null" + ) %} + cast({{ col_name }} as {{ col_type }}) as {{ col.quoted }} + {% if not loop.last %},{% endif -%} {%- endfor %} diff --git a/macros/sql/unpivot.sql b/macros/sql/unpivot.sql index 8821aad2..2a46c7f0 100644 --- a/macros/sql/unpivot.sql +++ b/macros/sql/unpivot.sql @@ -11,69 +11,96 @@ Arguments: field_name: Destination table column name for the source table column names. value_name: Destination table column name for the pivoted values #} - -{% macro unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} - {{ return(adapter.dispatch('unpivot', 'dbt_utils')(relation, cast_to, exclude, remove, field_name, value_name, table)) }} +{% macro unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none +) -%} + {{ + return( + adapter.dispatch("unpivot", "dbt_utils")( + relation, cast_to, exclude, remove, field_name, value_name, table + ) + ) + }} {% endmacro %} -{% macro default__unpivot(relation=none, cast_to='varchar', exclude=none, remove=none, field_name='field_name', value_name='value', table=none) -%} +{% macro default__unpivot( + relation=none, + cast_to="varchar", + exclude=none, + remove=none, + field_name="field_name", + value_name="value", + table=none +) -%} {% if table %} - {%- set error_message = ' - Warning: the `unpivot` macro no longer accepts a `table` parameter. \ - This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ - The {}.{} model triggered this warning. \ - '.format(model.package_name, model.name) -%} + {%- set error_message = " Warning: the `unpivot` macro no longer accepts a `table` parameter. \ This parameter will be deprecated in a future release of dbt-utils. Use the `relation` parameter instead. \ The {}.{} model triggered this warning. \ ".format( + model.package_name, model.name + ) -%} {%- do exceptions.warn(error_message) -%} {% endif %} {% if relation and table %} - {{ exceptions.raise_compiler_error("Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`).") }} - {% elif not relation and table %} - {% set relation=table %} + {{ + exceptions.raise_compiler_error( + "Error: both the `relation` and `table` parameters were provided to `unpivot` macro. Choose one only (we recommend `relation`)." + ) + }} + {% elif not relation and table %} {% set relation = table %} {% elif not relation and not table %} - {{ exceptions.raise_compiler_error("Error: argument `relation` is required for `unpivot` macro.") }} + {{ + exceptions.raise_compiler_error( + "Error: argument `relation` is required for `unpivot` macro." + ) + }} {% endif %} - {%- set exclude = exclude if exclude is not none else [] %} - {%- set remove = remove if remove is not none else [] %} - - {%- set include_cols = [] %} + {%- set exclude = exclude if exclude is not none else [] %} + {%- set remove = remove if remove is not none else [] %} - {%- set table_columns = {} %} + {%- set include_cols = [] %} - {%- do table_columns.update({relation: []}) %} + {%- set table_columns = {} %} - {%- do dbt_utils._is_relation(relation, 'unpivot') -%} - {%- do dbt_utils._is_ephemeral(relation, 'unpivot') -%} - {%- set cols = adapter.get_columns_in_relation(relation) %} + {%- do table_columns.update({relation: []}) %} - {%- for col in cols -%} - {%- if col.column.lower() not in remove|map('lower') and col.column.lower() not in exclude|map('lower') -%} - {% do include_cols.append(col) %} - {%- endif %} - {%- endfor %} + {%- do dbt_utils._is_relation(relation, "unpivot") -%} + {%- do dbt_utils._is_ephemeral(relation, "unpivot") -%} + {%- set cols = adapter.get_columns_in_relation(relation) %} + {%- for col in cols -%} + {%- if col.column.lower() not in remove | map( + "lower" + ) and col.column.lower() not in exclude | map("lower") -%} + {% do include_cols.append(col) %} + {%- endif %} + {%- endfor %} - {%- for col in include_cols -%} - select - {%- for exclude_col in exclude %} - {{ exclude_col }}, - {%- endfor %} + {%- for col in include_cols -%} + select + {%- for exclude_col in exclude %} {{ exclude_col }}, {%- endfor %} - cast('{{ col.column }}' as {{ dbt_utils.type_string() }}) as {{ field_name }}, - cast( {% if col.data_type == 'boolean' %} - {{ dbt_utils.cast_bool_to_text(col.column) }} - {% else %} - {{ col.column }} - {% endif %} - as {{ cast_to }}) as {{ value_name }} + cast( + '{{ col.column }}' as {{ dbt_utils.type_string() }} + ) as {{ field_name }}, + cast( + {% if col.data_type == "boolean" %} + {{ dbt_utils.cast_bool_to_text(col.column) }} + {% else %}{{ col.column }} + {% endif %} as {{ cast_to }} + ) as {{ value_name }} - from {{ relation }} + from {{ relation }} - {% if not loop.last -%} - union all - {% endif -%} - {%- endfor -%} + {% if not loop.last -%} + union all + {% endif -%} + {%- endfor -%} {%- endmacro %} diff --git a/macros/web/get_url_host.sql b/macros/web/get_url_host.sql index 43a58cc4..faafff86 100644 --- a/macros/web/get_url_host.sql +++ b/macros/web/get_url_host.sql @@ -1,27 +1,27 @@ {% macro get_url_host(field) -%} - {{ return(adapter.dispatch('get_url_host', 'dbt_utils')(field)) }} + {{ return(adapter.dispatch("get_url_host", "dbt_utils")(field)) }} {% endmacro %} {% macro default__get_url_host(field) -%} -{%- set parsed = - dbt_utils.split_part( + {%- set parsed = dbt_utils.split_part( dbt_utils.split_part( dbt_utils.replace( dbt_utils.replace( - dbt_utils.replace(field, "'android-app://'", "''" - ), "'http://'", "''" - ), "'https://'", "''" - ), "'/'", 1 - ), "'?'", 1 - ) + dbt_utils.replace(field, "'android-app://'", "''"), + "'http://'", + "''", + ), + "'https://'", + "''", + ), + "'/'", + 1, + ), + "'?'", + 1, + ) -%} --%} - - - {{ dbt_utils.safe_cast( - parsed, - dbt_utils.type_string() - )}} + {{ dbt_utils.safe_cast(parsed, dbt_utils.type_string()) }} {%- endmacro %} diff --git a/macros/web/get_url_parameter.sql b/macros/web/get_url_parameter.sql index f363fcbb..9b3127ab 100644 --- a/macros/web/get_url_parameter.sql +++ b/macros/web/get_url_parameter.sql @@ -1,13 +1,19 @@ {% macro get_url_parameter(field, url_parameter) -%} - {{ return(adapter.dispatch('get_url_parameter', 'dbt_utils')(field, url_parameter)) }} + {{ + return( + adapter.dispatch("get_url_parameter", "dbt_utils")(field, url_parameter) + ) + }} {% endmacro %} {% macro default__get_url_parameter(field, url_parameter) -%} -{%- set formatted_url_parameter = "'" + url_parameter + "='" -%} + {%- set formatted_url_parameter = "'" + url_parameter + "='" -%} -{%- set split = dbt_utils.split_part(dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1) -%} + {%- set split = dbt_utils.split_part( + dbt_utils.split_part(field, formatted_url_parameter, 2), "'&'", 1 + ) -%} -nullif({{ split }},'') + nullif({{ split }}, '') {%- endmacro %} diff --git a/macros/web/get_url_path.sql b/macros/web/get_url_path.sql index 4d224277..ef5c2f14 100644 --- a/macros/web/get_url_path.sql +++ b/macros/web/get_url_path.sql @@ -1,13 +1,12 @@ {% macro get_url_path(field) -%} - {{ return(adapter.dispatch('get_url_path', 'dbt_utils')(field)) }} + {{ return(adapter.dispatch("get_url_path", "dbt_utils")(field)) }} {% endmacro %} {% macro default__get_url_path(field) -%} - {%- set stripped_url = - dbt_utils.replace( - dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''") - -%} + {%- set stripped_url = dbt_utils.replace( + dbt_utils.replace(field, "'http://'", "''"), "'https://'", "''" + ) -%} {%- set first_slash_pos -%} coalesce( @@ -16,19 +15,15 @@ ) {%- endset -%} - {%- set parsed_path = - dbt_utils.split_part( - dbt_utils.right( - stripped_url, - dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos - ), - "'?'", 1 - ) - -%} + {%- set parsed_path = dbt_utils.split_part( + dbt_utils.right( + stripped_url, + dbt_utils.length(stripped_url) ~ "-" ~ first_slash_pos, + ), + "'?'", + 1, + ) -%} + + {{ dbt_utils.safe_cast(parsed_path, dbt_utils.type_string()) }} - {{ dbt_utils.safe_cast( - parsed_path, - dbt_utils.type_string() - )}} - {%- endmacro %}