Merge pull request #22 from fishtown-analytics/020/end-to-end-testing

Readier for v0.2.0
dbt-labs · May 7, 2020 · 740f5f1 · 740f5f1
2 parents d7c69cb + 0aacf80
commit 740f5f1
Show file tree

Hide file tree

Showing 6 changed files with 25 additions and 15 deletions.
diff --git a/README.md b/README.md
@@ -1,9 +1,11 @@
-### External tables in dbt
+# External sources in dbt
 
 * Source config extension for metadata about external file structure
 * Adapter macros to create external tables and refresh external table partitions
 * Snowflake-specific macros to create, backfill, and refresh snowpipes
 
+## Syntax
+
 ```bash
 # iterate through all source nodes, create if missing + refresh if appropriate
 $ dbt run-operation stage_external_sources
@@ -15,7 +17,7 @@ $ dbt run-operation stage_external_sources --vars 'ext_full_refresh: true'
 ![sample docs](etc/sample_docs.png)
 
 The macros assume that you have already created an external stage (Snowflake)
-or external schema (Spectrum), and that you have permissions to select from it
+or external schema (Redshift/Spectrum), and that you have permissions to select from it
 and create tables in it.
 
 The `stage_external_sources` macro accepts a similar node selection syntax to
@@ -36,7 +38,7 @@ $ dbt source stage-external --full-refresh
 $ dbt source stage-external --select snowplow.event logs
 ```
 
-### Spec
+## Spec
 
 ```yml
 version: 2
@@ -107,10 +109,14 @@ sources:
         ...
 ```
 
-See [`sample_sources`](sample_sources) for full valid YML config that establishes Snowplow events
+## Resources
+
+* [`sample_sources`](sample_sources) for full valid YML config that establishes Snowplow events
 as a dbt source and stage-ready external table in Snowflake and Spectrum.
+* [`sample_analysis`](sample_analysis) for a "dry run" version of the DDL/DML that
+`stage_external_sources` will run as an operation
 
-### Supported databases
+## Supported databases
 
 * Redshift (Spectrum)
 * Snowflake

diff --git a/macros/external/create_snowpipe.sql b/macros/external/create_snowpipe.sql
@@ -71,7 +71,6 @@
 
     {% if auto_ingest is true %}
 
-        {{ dbt_utils.log_info('PASS') }}
         {% do return([]) %}
 
     {% else %}

diff --git a/macros/external/refresh_external_table.sql b/macros/external/refresh_external_table.sql
@@ -72,7 +72,6 @@
 
     {% else %}
 
-        {{ dbt_utils.log_info('PASS') }}
         {% do return([]) %}
 
     {% endif %}
@@ -99,7 +98,6 @@
 
     {% else %}
 
-        {{ dbt_utils.log_info('PASS') }}
         {% do return([]) %}
 
     {% endif %}

diff --git a/macros/external/stage_external_sources.sql b/macros/external/stage_external_sources.sql
@@ -112,7 +112,9 @@
 
         {% set run_queue = get_external_build_plan(node) %}
 
-        {% do exit_transaction() %}
+        {% do dbt_utils.log_info(loop_label ~ ' SKIP') if run_queue == [] %}
+
+        {% do dbt_external_tables.exit_transaction() %}
 
         {% for q in run_queue %}
 

diff --git a/macros/helpers/common.sql b/macros/helpers/common.sql
@@ -30,9 +30,3 @@
     {{return(ddl)}}
 
 {% endmacro %}
-
-{% macro exit_transaction() %}
-
-    {% do run_query('begin; commit;') %}
-
-{% endmacro %}
diff --git a/macros/helpers/redshift/transaction.sql b/macros/helpers/redshift/transaction.sql
@@ -0,0 +1,11 @@
+{% macro exit_transaction() %}
+    {{ return(adapter_macro('dbt_external_tables.exit_transaction')) }}
+{% endmacro %}
+
+{% macro default__exit_transaction() %}
+    {# noop #}
+{% endmacro %}
+
+{% macro redshift__exit_transaction() %}
+    {% do run_query('begin; commit;') %}
+{% endmacro %}