fix: Call to webhook on elastic dags

Signed-off-by: Arturo Volpe <[email protected]>
InstIDEA · Dec 22, 2020 · ede4e45 · ede4e45
1 parent 4ee86bb
commit ede4e45
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 7 deletions.
diff --git a/scripts/python/airflow/dags/elastic_fts_auth_ddjj.py b/scripts/python/airflow/dags/elastic_fts_auth_ddjj.py
@@ -26,8 +26,8 @@
 with dag:
     do_curl = BashOperator(
         task_id=f'call_webhook',
-        bash_command=f"""
-            curl {{ var.value.ELASTIC_IDX_AUTH_DDJJ }}
+        bash_command="""
+            curl "{{ var.value.ELASTIC_IDX_AUTH_DDJJ }}"
             """,
         retries=10
     )

diff --git a/scripts/python/airflow/dags/elastic_fts_full_data_index.py b/scripts/python/airflow/dags/elastic_fts_full_data_index.py
@@ -4,7 +4,6 @@
 from airflow.operators.bash_operator import BashOperator
 from airflow.operators.postgres_operator import PostgresOperator
 
-
 default_args = {
     'owner': 'airflow',
     'depends_on_past': False,
@@ -27,16 +26,20 @@
 with dag:
     do_curl = BashOperator(
         task_id=f'call_webhook',
-        bash_command=f"""
-            curl {{ var.value.ELASTIC_IDX_FULL_DATA_HOOK }}
+        bash_command="""
+            curl "{{ var.value.ELASTIC_IDX_FULL_DATA_HOOK }}"
             """,
         retries=10
     )
 
+    clean_db = PostgresOperator(task_id='clean_table',
+                                sql="DROP TABLE analysis.full_data")
+
     do_query = PostgresOperator(task_id='do_query',
+
                                 sql="sql/elastic_index_full_data.sql")
 
-    do_query >> do_curl
+    clean_db >> do_query >> do_curl
 
 if __name__ == '__main__':
     dag.clear(reset_dag_runs=True)

diff --git a/scripts/python/airflow/dags/sql/elastic_index_full_data.sql b/scripts/python/airflow/dags/sql/elastic_index_full_data.sql
@@ -1,4 +1,3 @@
-DROP TABLE analysis.full_data;
 CREATE TABLE analysis.full_data AS (
     WITH sfp_documents AS (
         select documento                       as document,