Merge branch 'master' of github.com:fishtown-analytics/dbt-external-t…

…ables into fix/snowpipe-copy-commit
dbt-labs · May 25, 2021 · 5fdca89 · 5fdca89
2 parents 970452b + a3b5619
commit 5fdca89
Show file tree

Hide file tree

Showing 23 changed files with 178 additions and 111 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,11 +1,14 @@
 
 version: 2.1
 
+orbs:
+  azure-cli: circleci/[email protected]
+
 jobs:
 
   integration-redshift:
     docker:
-      - image: circleci/python:3.6.3-stretch
+      - image: circleci/python:3.6.13-stretch
     steps:
       - checkout
       - run:
@@ -16,7 +19,7 @@ jobs:
 
   integration-snowflake:
     docker:
-      - image: circleci/python:3.6.3-stretch
+      - image: circleci/python:3.6.13-stretch
     steps:
       - checkout
       - run:
@@ -29,7 +32,7 @@ jobs:
     environment:
       BIGQUERY_SERVICE_KEY_PATH: "/home/circleci/bigquery-service-key.json"
     docker:
-      - image: circleci/python:3.6.3-stretch
+      - image: circleci/python:3.6.13-stretch
     steps:
       - checkout
       - run:
@@ -63,9 +66,23 @@ jobs:
       - image: dataders/pyodbc:1.2
     steps:
       - checkout
+      - run: &gnupg2
+          name: az cli dep
+          command: apt-get install gnupg2 -y
+      - azure-cli/install
+      - azure-cli/login-with-service-principal: &azure-creds
+          azure-sp: DBT_AZURE_SP_NAME
+          azure-sp-password: DBT_AZURE_SP_SECRET
+          azure-sp-tenant: DBT_AZURE_TENANT
+      - run:
+          name: resume Synapse pool/db
+          command: az synapse sql pool resume --name $DBT_SYNAPSE_DB --workspace-name $DBT_SYNAPSE_SERVER --resource-group dbt-msft
       - run:
           name: "Run Tests - synapse"
           command: ./run_test.sh synapse
+      - run:
+          name: pause Synapse pool/db
+          command: az synapse sql pool resume --name $DBT_SYNAPSE_DB --workspace-name $DBT_SYNAPSE_SERVER --resource-group dbt-msft
       - store_artifacts:
           path: ./logs
 
@@ -74,9 +91,9 @@ jobs:
       - image: dataders/pyodbc:1.2
     steps:
       - checkout
-      - run:
-          name: "wait for Synapse tests to finish"
-          command: sleep 60
+      - run: *gnupg2
+      - azure-cli/install
+      - azure-cli/login-with-service-principal: *azure-creds
       - run:
           name: "Run Tests - azuresql"
           command: ./run_test.sh azuresql
@@ -93,4 +110,6 @@ workflows:
       - integration-bigquery
       - integration-databricks
       - integration-synapse
-      - integration-azuresql
+      - integration-azuresql:
+          requires:
+            - integration-synapse
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -6,4 +6,4 @@ Describe your changes, and why you're making them.
 ## Checklist
 - [ ] I have verified that these changes work locally
 - [ ] I have updated the README.md (if applicable)
-- [ ] I have added tests & descriptions to my models (and macros if applicable)
+- [ ] I have added an integration test for my fix/feature (if applicable)
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,6 @@
+
+**/target/
+**/dbt_modules/
+**/logs/
+**/env/
+**/venv/
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # External sources in dbt
 
-dbt v0.15.0 added support for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties.
+dbt v0.15.0 [added support](https://github.com/fishtown-analytics/dbt/pull/1784) for an `external` property within `sources` that can include information about `location`, `partitions`, and other database-specific properties.
 
 This package provides:
 * Macros to create/replace external tables and refresh their partitions, using the metadata provided in your `.yml` file source definitions
@@ -17,6 +17,10 @@ This package provides:
 
 ![sample docs](etc/sample_docs.png)
 
+## Installation
+
+Follow the instructions at [hub.getdbt.com](https://hub.getdbt.com/fishtown-analytics/dbt_external_tables/latest/) on how to modify your `packages.yml` and run `dbt deps`.
+
 ## Syntax
 
 The `stage_external_sources` macro is the primary point of entry when using this package. It has two operational modes: standard and "full refresh."
@@ -26,18 +30,18 @@ The `stage_external_sources` macro is the primary point of entry when using this
 $ dbt run-operation stage_external_sources
 
 # iterate through all source nodes, create or replace (+ refresh if necessary)
-$ dbt run-operation stage_external_sources --vars 'ext_full_refresh: true'
+$ dbt run-operation stage_external_sources --vars "ext_full_refresh: true"
 ```
 
 The `stage_external_sources` macro accepts a limited node selection syntax similar to
 [snapshotting source freshness](https://docs.getdbt.com/docs/running-a-dbt-project/command-line-interface/source/#specifying-sources-to-snapshot):
 
 ```bash
 # stage all Snowplow and Logs external sources:
-$ dbt run-operation stage_external_sources --args 'select: snowplow logs'
+$ dbt run-operation stage_external_sources --args "select: snowplow logs"
 
 # stage a particular external source table:
-$ dbt run-operation stage_external_sources --args 'select: snowplow.event'
+$ dbt run-operation stage_external_sources --args "select: snowplow.event"
 ```
 
 ## Setup

diff --git a/integration_tests/.gitignore b/integration_tests/.gitignore
diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml
@@ -50,16 +50,13 @@ integration_tests:
       schema: dbt_external_tables_integration_tests_databricks
 
     synapse:
-      type: sqlserver
+      type: synapse
       driver: "ODBC Driver 17 for SQL Server"
       port: 1433
-      host: "{{ env_var('DBT_SYNAPSE_SERVER') }}"
+      host: "{{ env_var('DBT_SYNAPSE_SERVER') }}.sql.azuresynapse.net"
       database: "{{ env_var('DBT_SYNAPSE_DB') }}"
-      username: "{{ env_var('DBT_SYNAPSE_UID') }}"
-      password: "{{ env_var('DBT_SYNAPSE_PWD') }}"
+      authentication: CLI
       schema: dbt_external_tables_integration_tests_synapse
-      encrypt: 'yes'
-      trust_cert: 'yes'
       threads: 1
 
     azuresql:
@@ -68,9 +65,6 @@ integration_tests:
       port: 1433
       host: "{{ env_var('DBT_AZURESQL_SERVER') }}"
       database: "{{ env_var('DBT_AZURESQL_DB') }}"
-      username: "{{ env_var('DBT_AZURESQL_UID') }}"
-      password: "{{ env_var('DBT_AZURESQL_PWD') }}"
+      authentication: CLI
       schema: dbt_external_tables_integration_tests_azuresql
-      encrypt: yes
-      trust_cert: yes
       threads: 1
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -4,8 +4,6 @@ version: '1.0'
 
 profile: 'integration_tests'
 
-# require-dbt-version: inherit this from package
-
 config-version: 2
 
 source-paths: ["models"]
@@ -14,14 +12,17 @@ test-paths: ["tests"]
 data-paths: ["data"]
 macro-paths: ["macros"]
 
-target-path: "target"  # directory which will store compiled SQL files
-clean-targets:         # directories to be removed by `dbt clean`
+target-path: "target"
+clean-targets:
     - "target"
     - "dbt_modules"
 
 vars:
   dbt_external_tables_dispatch_list: ['dbt_external_tables_integration_tests']
 
+seeds:
+  +quote_columns: false
+
 sources:
   dbt_external_tables_integration_tests:
     plugins:
@@ -34,9 +35,6 @@ sources:
       spark_external:
         +enabled: "{{ target.type == 'spark' }}"
       synapse_external:
-        +enabled: "{{ target.name == 'synapse' }}"
+        +enabled: "{{ target.type == 'synapse' }}"
       azuresql_external:
-        +enabled: "{{ target.name == 'azuresql' }}"
-
-seeds:
-  quote_columns: false
+        +enabled: "{{ target.type == 'sqlserver' }}"
diff --git a/integration_tests/macros/plugins/sqlserver/prep_external.sql b/integration_tests/macros/plugins/sqlserver/prep_external.sql
@@ -2,7 +2,7 @@
 
     {% set external_data_source = target.schema ~ '.dbt_external_tables_testing' %}
 
-    {% if target.name == "synapse"%}
+    {% if target.type == "synapse"%}
 
         {% set create_external_data_source %}
             IF NOT EXISTS ( SELECT * FROM sys.external_data_sources WHERE name = '{{external_data_source}}' )
@@ -29,7 +29,7 @@
             )
         {% endset %}
 
-    {% elif target.name == "azuresql" %}
+    {% elif target.type == "sqlserver" %}
 
         {% set cred_name = 'synapse_reader' %}
 
@@ -55,17 +55,21 @@
     {%- endif %}
 
 
-    {% if target.name == "azuresql" -%}
+    {% if target.type == "sqlserver" -%}
         {% do log('Creating database scoped credential ' ~ cred_name, info = true) %}
         {% do run_query(create_database_scoped_credential) %}
     {%- endif %}
 
     {% do log('Creating external data source ' ~ external_data_source, info = true) %}
     {% do run_query(create_external_data_source) %}
 
-    {% if target.name == "synapse" -%}
+    {% if target.type == "synapse" -%}
         {% do log('Creating external file format ' ~ external_file_format, info = true) %}
         {% do run_query(create_external_file_format) %}
     {%- endif %}
 
 {% endmacro %}
+
+{% macro synapse__prep_external() %}
+    {% do return( dbt_external_tables_integration_tests.sqlserver__prep_external()) %}
+{% endmacro %}
diff --git a/...ration_tests/models/plugins/sqlserver.yml → ...ests/models/plugins/azuresql_external.yml b/...ration_tests/models/plugins/sqlserver.yml → ...ests/models/plugins/azuresql_external.yml
@@ -1,21 +1,15 @@
 version: 2
 
 sources:
-  - name: synapse_external
+  - name: azuresql_external
     schema: "{{ target.schema }}"
-    loader: ADLSblob
-
+    loader: RDBMS cross database query
     tables:
-
       - name: people_csv_unpartitioned
-        external: &csv-people
-          location: '/csv'
-          file_format: "{{ target.schema ~ '.dbt_external_ff_testing' }}"
+        external:
           data_source: "{{ target.schema ~ '.dbt_external_tables_testing' }}"
-          reject_type: VALUE
-          reject_value: 0
-          ansi_nulls: true
-          quoted_identifier: true
+          schema_name: 'dbt_external_tables_integration_tests_synapse'
+          object_name: 'people_csv_unpartitioned'
         columns: &cols-of-the-people
           - name: id
             data_type: int
@@ -34,28 +28,6 @@ sources:
                 - last_name
                 - email
 
-      - name: people_csv_partitioned
-        external:
-          <<: *csv-people
-          # TODO: SYNAPSE DOES NOT DO PARTITIONS
-          # (BUT WE COULD MAKE A WORKAROUND !!!)
-          # partitions: &parts-of-the-people
-          #   - name: section
-          #     data_type: varchar
-        columns: *cols-of-the-people
-        tests: *equal-to-the-people
-  - name: azuresql_external
-    schema: "{{ target.schema }}"
-    loader: RDBMS cross database query
-    tables:
-      - name: people_csv_unpartitioned
-        external:
-          data_source: "{{ target.schema ~ '.dbt_external_tables_testing' }}"
-          schema_name: 'dbt_external_tables_integration_tests_synapse'
-          object_name: 'people_csv_unpartitioned'
-        columns: *cols-of-the-people
-        tests: *equal-to-the-people
-
       # TODO: JSON IS NOT SUPPORTED BY SYNAPSE ATM
 
       # - name: people_json_unpartitioned

diff --git a/...gration_tests/models/plugins/bigquery.yml → ...ests/models/plugins/bigquery_external.yml b/...gration_tests/models/plugins/bigquery.yml → ...ests/models/plugins/bigquery_external.yml
diff --git a/...gration_tests/models/plugins/redshift.yml → ...ests/models/plugins/redshift_external.yml b/...gration_tests/models/plugins/redshift.yml → ...ests/models/plugins/redshift_external.yml
diff --git a/...ration_tests/models/plugins/snowflake.yml → ...sts/models/plugins/snowflake_external.yml b/...ration_tests/models/plugins/snowflake.yml → ...sts/models/plugins/snowflake_external.yml
@@ -71,12 +71,20 @@ sources:
 
       - name: people_csv_partitioned_no_columns
         external:
-          <<: *json-people
+          <<: *csv-people
           partitions: *parts-of-the-people
         tests: *same-rowcount
+
+      - name: people_csv_with_keyword_colname
+        external: *csv-people
+        columns:
+          - name: UNION
+            quote: true
+            data_type: varchar(64)
+        tests: *same-rowcount
 
       - name: people_json_unpartitioned_no_columns
-        external: *csv-people
+        external: *json-people
         tests: *same-rowcount
 
       - name: people_json_partitioned_no_columns

diff --git a/integration_tests/models/plugins/spark.yml → ...n_tests/models/plugins/spark_external.yml b/integration_tests/models/plugins/spark.yml → ...n_tests/models/plugins/spark_external.yml
diff --git a/integration_tests/models/plugins/synapse_external.yml b/integration_tests/models/plugins/synapse_external.yml
@@ -0,0 +1,46 @@
+version: 2
+
+sources:
+  - name: synapse_external
+    schema: "{{ target.schema }}"
+    loader: ADLSblob
+
+    tables:
+
+      - name: people_csv_unpartitioned
+        external: &csv-people
+          location: '/csv'
+          file_format: "{{ target.schema ~ '.dbt_external_ff_testing' }}"
+          data_source: "{{ target.schema ~ '.dbt_external_tables_testing' }}"
+          reject_type: VALUE
+          reject_value: 0
+          ansi_nulls: true
+          quoted_identifier: true
+        columns: &cols-of-the-people
+          - name: id
+            data_type: int
+          - name: first_name
+            data_type: varchar(64)
+          - name: last_name
+            data_type: varchar(64)
+          - name: email
+            data_type: varchar(64)
+        tests: &equal-to-the-people
+          - dbt_external_tables_integration_tests.tsql_equality:
+              compare_model: ref('people')
+              compare_columns:
+                - id
+                - first_name
+                - last_name
+                - email
+
+      - name: people_csv_partitioned
+        external:
+          <<: *csv-people
+          # TODO: SYNAPSE DOES NOT DO PARTITIONS
+          # (BUT WE COULD MAKE A WORKAROUND !!!)
+          # partitions: &parts-of-the-people
+          #   - name: section
+          #     data_type: varchar
+        columns: *cols-of-the-people
+        tests: *equal-to-the-people
diff --git a/macros/common/stage_external_sources.sql b/macros/common/stage_external_sources.sql
@@ -32,6 +32,10 @@
         {% endif %}
 
     {% endfor %}
+
+    {% if sources_to_stage|length == 0 %}
+        {% do dbt_utils.log_info('No external sources selected') %}
+    {% endif %}
 
     {% for node in sources_to_stage %}