From 6a65d4ecb14d6bed512b3acdc48d43f779a794fb Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Sat, 14 Nov 2020 17:46:30 -0500 Subject: [PATCH 1/5] Commit transactions on Snowflake --- macros/helpers/{redshift => }/transaction.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) rename macros/helpers/{redshift => }/transaction.sql (71%) diff --git a/macros/helpers/redshift/transaction.sql b/macros/helpers/transaction.sql similarity index 71% rename from macros/helpers/redshift/transaction.sql rename to macros/helpers/transaction.sql index b41550a9..dee91f8d 100644 --- a/macros/helpers/redshift/transaction.sql +++ b/macros/helpers/transaction.sql @@ -3,9 +3,13 @@ {% endmacro %} {% macro default__exit_transaction() %} + {{ return('begin; commit;') }} +{% endmacro %} + +{% macro bigquery__exit_transaction() %} {{ return('') }} {% endmacro %} -{% macro redshift__exit_transaction() %} - {{ return('begin; commit;') }} +{% macro spark__exit_transaction() %} + {{ return('') }} {% endmacro %} From 0d01b30c83013b3b9d2decd1d387772aec41672f Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 16 Nov 2020 01:21:53 -0500 Subject: [PATCH 2/5] Try replicating via autocommit --- .circleci/config.yml | 2 ++ integration_tests/Makefile | 2 ++ integration_tests/macros/cleanup_external.sql | 19 +++++++++++++++++++ integration_tests/macros/prep_external.sql | 12 ++++++++++++ 4 files changed, 35 insertions(+) create mode 100644 integration_tests/macros/cleanup_external.sql diff --git a/.circleci/config.yml b/.circleci/config.yml index 6939e568..2a88b4a3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,6 +35,7 @@ jobs: dbt --warn-error run-operation stage_external_sources --var 'ext_full_refresh: true' --target redshift dbt --warn-error run-operation stage_external_sources --target redshift dbt --warn-error test --target redshift + dbt run-operation cleanup_external --target redshift - run: name: "Run Tests - Snowflake" @@ -48,6 +49,7 @@ jobs: dbt --warn-error run-operation stage_external_sources --var 'ext_full_refresh: true' --target snowflake dbt --warn-error run-operation stage_external_sources --target snowflake dbt --warn-error test --target snowflake + dbt run-operation cleanup_external --target snowflake - save_cache: key: deps1-{{ .Branch }} diff --git a/integration_tests/Makefile b/integration_tests/Makefile index 7cf524cd..54a6a870 100644 --- a/integration_tests/Makefile +++ b/integration_tests/Makefile @@ -5,6 +5,7 @@ test-redshift: dbt run-operation stage_external_sources --var 'ext_full_refresh: true' --target redshift dbt run-operation stage_external_sources --target redshift dbt test --target redshift + dbt run-operation cleanup_external --target redshift test-snowflake: dbt deps --target snowflake @@ -13,3 +14,4 @@ test-snowflake: dbt run-operation stage_external_sources --var 'ext_full_refresh: true' --target snowflake dbt run-operation stage_external_sources --target snowflake dbt test --target snowflake + dbt run-operation cleanup_external --target snowflake diff --git a/integration_tests/macros/cleanup_external.sql b/integration_tests/macros/cleanup_external.sql new file mode 100644 index 00000000..c1d4d842 --- /dev/null +++ b/integration_tests/macros/cleanup_external.sql @@ -0,0 +1,19 @@ +{% macro cleanup_external() %} + {{ return(adapter.dispatch('cleanup_external', dbt_external_tables._get_dbt_external_tables_namespaces())()) }} +{% endmacro %} + +{% macro default__cleanup_external() %} + {% do log('No cleanup necessary, skipping', info = true) %} + {# noop #} +{% endmacro %} + +{% macro snowflake__cleanup_external() %} + + {% set unset_autocommit %} + alter user {{ target.user }} unset autocommit; + {% endset %} + + {% do log('Unsetting autocommit parameter for user ' ~ target.user, info = true) %} + {% do run_query(unset_autocommit) %} + +{% endmacro %} diff --git a/integration_tests/macros/prep_external.sql b/integration_tests/macros/prep_external.sql index 9a9a3dfa..834e4acc 100644 --- a/integration_tests/macros/prep_external.sql +++ b/integration_tests/macros/prep_external.sql @@ -2,6 +2,11 @@ {{ return(adapter.dispatch('prep_external', dbt_external_tables._get_dbt_external_tables_namespaces())()) }} {% endmacro %} +{% macro default__prep_external() %} + {% do log('No prep necessary, skipping', info = true) %} + {# noop #} +{% endmacro %} + {% macro redshift__prep_external() %} {% set external_schema = target.schema ~ '_spectrum' %} @@ -37,4 +42,11 @@ {% do log('Creating external stage ' ~ external_stage, info = true) %} {% do run_query(create_external_stage) %} + {% set set_autocommit_false %} + alter user {{ target.user }} set autocommit = false; + {% endset %} + + {% do log('Turning off autocommit for user ' ~ target.user, info = true) %} + {% do run_query(set_autocommit_false) %} + {% endmacro %} From de571e5d6ccb872eeb1ce9ab97a8555e2cef9622 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 16 Nov 2020 01:23:33 -0500 Subject: [PATCH 3/5] Circle hates tabs --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2a88b4a3..31ec6ec5 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,7 +35,7 @@ jobs: dbt --warn-error run-operation stage_external_sources --var 'ext_full_refresh: true' --target redshift dbt --warn-error run-operation stage_external_sources --target redshift dbt --warn-error test --target redshift - dbt run-operation cleanup_external --target redshift + dbt run-operation cleanup_external --target redshift - run: name: "Run Tests - Snowflake" @@ -49,7 +49,7 @@ jobs: dbt --warn-error run-operation stage_external_sources --var 'ext_full_refresh: true' --target snowflake dbt --warn-error run-operation stage_external_sources --target snowflake dbt --warn-error test --target snowflake - dbt run-operation cleanup_external --target snowflake + dbt run-operation cleanup_external --target snowflake - save_cache: key: deps1-{{ .Branch }} From 557bf47c292515e57925b74376f5a2c3828d2954 Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 25 Jan 2021 11:51:10 +0100 Subject: [PATCH 4/5] Add back code lost in merge --- .../macros/{ => common}/cleanup_external.sql | 11 ----------- .../macros/plugins/snowflake/cleanup_external.sql | 10 ++++++++++ .../macros/plugins/snowflake/prep_external.sql | 7 +++++++ 3 files changed, 17 insertions(+), 11 deletions(-) rename integration_tests/macros/{ => common}/cleanup_external.sql (50%) create mode 100644 integration_tests/macros/plugins/snowflake/cleanup_external.sql diff --git a/integration_tests/macros/cleanup_external.sql b/integration_tests/macros/common/cleanup_external.sql similarity index 50% rename from integration_tests/macros/cleanup_external.sql rename to integration_tests/macros/common/cleanup_external.sql index c1d4d842..5ddcaf19 100644 --- a/integration_tests/macros/cleanup_external.sql +++ b/integration_tests/macros/common/cleanup_external.sql @@ -6,14 +6,3 @@ {% do log('No cleanup necessary, skipping', info = true) %} {# noop #} {% endmacro %} - -{% macro snowflake__cleanup_external() %} - - {% set unset_autocommit %} - alter user {{ target.user }} unset autocommit; - {% endset %} - - {% do log('Unsetting autocommit parameter for user ' ~ target.user, info = true) %} - {% do run_query(unset_autocommit) %} - -{% endmacro %} diff --git a/integration_tests/macros/plugins/snowflake/cleanup_external.sql b/integration_tests/macros/plugins/snowflake/cleanup_external.sql new file mode 100644 index 00000000..305b9f48 --- /dev/null +++ b/integration_tests/macros/plugins/snowflake/cleanup_external.sql @@ -0,0 +1,10 @@ +{% macro snowflake__cleanup_external() %} + + {% set unset_autocommit %} + alter user {{ target.user }} unset autocommit; + {% endset %} + + {% do log('Unsetting autocommit parameter for user ' ~ target.user, info = true) %} + {% do run_query(unset_autocommit) %} + +{% endmacro %} diff --git a/integration_tests/macros/plugins/snowflake/prep_external.sql b/integration_tests/macros/plugins/snowflake/prep_external.sql index 5dd4dca9..6d33ca0b 100644 --- a/integration_tests/macros/plugins/snowflake/prep_external.sql +++ b/integration_tests/macros/plugins/snowflake/prep_external.sql @@ -13,4 +13,11 @@ {% do log('Creating external stage ' ~ external_stage, info = true) %} {% do run_query(create_external_stage) %} + {% set set_autocommit_false %} + alter user {{ target.user }} set autocommit = false; + {% endset %} + + {% do log('Turning off autocommit for user ' ~ target.user, info = true) %} + {% do run_query(set_autocommit_false) %} + {% endmacro %} From 970452b04322bd66d02935e50809053456925fcd Mon Sep 17 00:00:00 2001 From: Jeremy Cohen Date: Mon, 25 Jan 2021 11:51:48 +0100 Subject: [PATCH 5/5] Readd cleanup step --- run_test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run_test.sh b/run_test.sh index db5b196b..7f040477 100755 --- a/run_test.sh +++ b/run_test.sh @@ -33,3 +33,4 @@ dbt run-operation prep_external --target $1 dbt run-operation stage_external_sources --var 'ext_full_refresh: true' --target $1 dbt run-operation stage_external_sources --target $1 dbt test --target $1 +dbt run-operation cleanup_external --target $1