Merge pull request #12 from Divergent-Insights/ftests

Streamlining logic to load internal stages and keeping the stage clean
Divergent-Insights · May 5, 2022 · 3893816 · 3893816
2 parents 47f9a8e + 9261581
commit 3893816
Show file tree

Hide file tree

Showing 4 changed files with 16 additions and 17 deletions.
diff --git a/macros/load_log_manifest.sql b/macros/load_log_manifest.sql
@@ -1,12 +1,8 @@
-{% macro load_log_manifest(load_from_internal_stage=True) %}
+{% macro load_log_manifest() %}
 
   {% set config = _get_config() %}
   {% set log_file = config["dbt_target_path"] ~ '/manifest.json' %}
 
-  {% if load_from_internal_stage %}
-      {{ load_internal_stage(file=log_file) }}
-  {% endif %}
-
-  {{ load_src_table() }}
+  {{ load_internal_stage(file=log_file) }}
 
 {% endmacro %}
diff --git a/macros/load_log_sources.sql b/macros/load_log_sources.sql
@@ -1,11 +1,15 @@
-{% macro load_log_sources(load_from_internal_stage=True) %}
+{% macro load_log_sources(load_from_internal_stage=true, clean_stage=true) %}
 
-  {{ load_log_manifest(load_from_internal_stage) }}
+  -- Removing all files from the internal stage
+  {% if clean_stage %}
+      {{ clean_internal_stage() }}
+  {% endif %}  
 
   {% set config = _get_config() %}
   {% set log_file = config["dbt_target_path"] ~ '/sources.json' %}
 
   {% if load_from_internal_stage %}
+      {{ load_log_manifest() }}
       {{ load_internal_stage(file=log_file) }}
   {% endif %}  
 

diff --git a/macros/load_log_tests.sql b/macros/load_log_tests.sql
@@ -1,13 +1,17 @@
-{% macro load_log_tests(load_from_internal_stage=True) %}
+{% macro load_log_tests(load_from_internal_stage=true, clean_stage=true) %}
 
-  {{ load_log_manifest(load_from_internal_stage) }}
+  -- Removing all files from the internal stage
+  {% if clean_stage %}
+      {{ clean_internal_stage() }}
+  {% endif %}  
 
   {% set config = _get_config() %}
   {% set log_file = config["dbt_target_path"] ~ '/run_results.json' %}
 
   {% if load_from_internal_stage %}
+      {{ load_log_manifest() }}
       {{ load_internal_stage(file=log_file) }}
-  {% endif %}
+  {% endif %}  
 
   {{ load_src_table() }}
 

diff --git a/macros/stage/load_internal_stage.sql b/macros/stage/load_internal_stage.sql
@@ -1,13 +1,8 @@
-{% macro load_internal_stage(file, clean_stage=true) %}
+{% macro load_internal_stage(file) %}
 
     {% do log("load_internal_stage started", info=True) %}
     {% set config = _get_config() %}
 
-    -- Removing all files from the internal stage
-    {% if clean_stage == true %}
-        {{ clean_internal_stage() }}
-    {% endif %}
-
     -- Populating internal stage
     {% set sql %}
         put file://{{ file }} @{{ config["database"] }}.{{ config["schema"] }}.{{ config["stage"] }} auto_compress=true;