diff --git a/macros/load_log_manifest.sql b/macros/load_log_manifest.sql index 98ba1e0..2b6313b 100644 --- a/macros/load_log_manifest.sql +++ b/macros/load_log_manifest.sql @@ -1,12 +1,8 @@ -{% macro load_log_manifest(load_from_internal_stage=True) %} +{% macro load_log_manifest() %} {% set config = _get_config() %} {% set log_file = config["dbt_target_path"] ~ '/manifest.json' %} - {% if load_from_internal_stage %} - {{ load_internal_stage(file=log_file) }} - {% endif %} - - {{ load_src_table() }} + {{ load_internal_stage(file=log_file) }} {% endmacro %} diff --git a/macros/load_log_sources.sql b/macros/load_log_sources.sql index 1c78248..b031645 100644 --- a/macros/load_log_sources.sql +++ b/macros/load_log_sources.sql @@ -1,11 +1,15 @@ -{% macro load_log_sources(load_from_internal_stage=True) %} +{% macro load_log_sources(load_from_internal_stage=true, clean_stage=true) %} - {{ load_log_manifest(load_from_internal_stage) }} + -- Removing all files from the internal stage + {% if clean_stage %} + {{ clean_internal_stage() }} + {% endif %} {% set config = _get_config() %} {% set log_file = config["dbt_target_path"] ~ '/sources.json' %} {% if load_from_internal_stage %} + {{ load_log_manifest() }} {{ load_internal_stage(file=log_file) }} {% endif %} diff --git a/macros/load_log_tests.sql b/macros/load_log_tests.sql index 40a82f4..995ba9f 100644 --- a/macros/load_log_tests.sql +++ b/macros/load_log_tests.sql @@ -1,13 +1,17 @@ -{% macro load_log_tests(load_from_internal_stage=True) %} +{% macro load_log_tests(load_from_internal_stage=true, clean_stage=true) %} - {{ load_log_manifest(load_from_internal_stage) }} + -- Removing all files from the internal stage + {% if clean_stage %} + {{ clean_internal_stage() }} + {% endif %} {% set config = _get_config() %} {% set log_file = config["dbt_target_path"] ~ '/run_results.json' %} {% if load_from_internal_stage %} + {{ load_log_manifest() }} {{ load_internal_stage(file=log_file) }} - {% endif %} + {% endif %} {{ load_src_table() }} diff --git a/macros/stage/load_internal_stage.sql b/macros/stage/load_internal_stage.sql index bca574f..0b15c7e 100644 --- a/macros/stage/load_internal_stage.sql +++ b/macros/stage/load_internal_stage.sql @@ -1,13 +1,8 @@ -{% macro load_internal_stage(file, clean_stage=true) %} +{% macro load_internal_stage(file) %} {% do log("load_internal_stage started", info=True) %} {% set config = _get_config() %} - -- Removing all files from the internal stage - {% if clean_stage == true %} - {{ clean_internal_stage() }} - {% endif %} - -- Populating internal stage {% set sql %} put file://{{ file }} @{{ config["database"] }}.{{ config["schema"] }}.{{ config["stage"] }} auto_compress=true;