- fix docker image version

- add common settings for journey_activation and journey_summary
treasure-data · Jan 31, 2024 · 5a8ad79 · 5a8ad79
1 parent ea7b1c7
commit 5a8ad79
Show file tree

Hide file tree

Showing 3 changed files with 102 additions and 5 deletions.
diff --git a/scenarios/monitoring/cdp_monitoring/common/settings.yaml b/scenarios/monitoring/cdp_monitoring/common/settings.yaml
@@ -7,6 +7,10 @@ td:
     entities_history: entities_history
     journey_statistics: journey_statistics
     journey_statistics_history: journey_statistics_history
+    journey_summary: journey_summary
+    journey_summary_history: journey_summary_history
+    journey_activation: journey_activation
+    journey_activation_history: journey_activation_history
     activations: activations
     activations_history: activations_history
   api_endpoint: api.treasuredata.com

diff --git a/scenarios/monitoring/cdp_monitoring/incremental_ingest.dig b/scenarios/monitoring/cdp_monitoring/incremental_ingest.dig
@@ -19,7 +19,7 @@ schedule:
         api_endpoint: ${td.api_endpoint}
         cdp_api_endpoint: ${td.cdp_api_endpoint}
         docker:
-            image: "digdag/digdag-python:3.9"
+            image: "digdag/digdag-python:3.10.1"
         _env:
             TD_API_KEY: ${secret:td.apikey}
 
@@ -44,7 +44,7 @@ schedule:
         api_endpoint: ${td.api_endpoint}
         cdp_api_endpoint: ${td.cdp_api_endpoint}
         docker: 
-            image: "digdag/digdag-python:3.9"
+            image: "digdag/digdag-python:3.10.1"
         _env:
             TD_API_KEY: ${secret:td.apikey}
 
@@ -69,7 +69,7 @@ schedule:
         api_endpoint: ${td.api_endpoint}
         cdp_api_endpoint: ${td.cdp_api_endpoint}
         docker: 
-            image: "digdag/digdag-python:3.9"
+            image: "digdag/digdag-python:3.10.1"
         _env:
             TD_API_KEY: ${secret:td.apikey}
 
@@ -95,6 +95,59 @@ schedule:
         api_endpoint: ${td.api_endpoint}
         cdp_api_endpoint: ${td.cdp_api_endpoint}
         docker:
-            image: "digdag/digdag-python:3.9"
+            image: "digdag/digdag-python:3.10.1"
         _env:
             TD_API_KEY: ${secret:td.apikey}
+
+
++incremental_ingest_journey_summary:
+    +append_journey_summary_history:
+        td>:
+        query: select * from ${td.tables.journey_summary}
+        insert_into: ${td.tables.journey_summary_history}
+        database: ${td.database}
+
+    +get_current_journey_list:
+        td>:
+        query: select ARRAY_JOIN(array_agg(id), ',') as ids from ${td.tables.entities} where type = 'journey'
+        store_last_results: true
+        database: ${td.database}
+
+    +ingest_journey_summary:
+        py>: scripts.ingest_journey_summary.run
+        session_unixtime: ${session_unixtime}
+        dest_db: ${td.database}
+        dest_table: ${td.tables.journey_summary}
+        journey_ids: ${td.last_results.ids}
+        api_endpoint: ${td.api_endpoint}
+        cdp_api_endpoint: ${td.cdp_api_endpoint}
+        docker:
+            image: "digdag/digdag-python:3.10.1"
+        _env:
+            TD_API_KEY: ${secret:td.apikey}
+
++incremental_ingest_journey_activation:
+    +append_journey_activation_history:
+        td>:
+        query: select * from ${td.tables.journey_activation}
+        insert_into: ${td.tables.journey_activation_history}
+        database: ${td.database}
+
+    +get_current_journey_list:
+        td>:
+        query: select ARRAY_JOIN(array_agg(id), ',') as ids from ${td.tables.entities} where type = 'journey'
+        store_last_results: true
+        database: ${td.database}
+
+    +ingest_journey_activation:
+        py>: scripts.ingest_journey_activation.run
+        session_unixtime: ${session_unixtime}
+        dest_db: ${td.database}
+        dest_table: ${td.tables.journey_activation}
+        journey_ids: ${td.last_results.ids}
+        api_endpoint: ${td.api_endpoint}
+        cdp_api_endpoint: ${td.cdp_api_endpoint}
+        docker:
+            image: "digdag/digdag-python:3.10.1"
+        _env:
+            TD_API_KEY: ${secret:td.apikey}
diff --git a/scenarios/monitoring/cdp_monitoring/initial_ingest.dig b/scenarios/monitoring/cdp_monitoring/initial_ingest.dig
@@ -43,7 +43,7 @@ _export:
 +initial_ingest_activations:
     +get_current_parent_segment_list:
         td>:
-        query: select ARRAY_AGG(id) as ids from ${td.tables.parent_segments}
+        query: select ARRAY_JOIN(ARRAY_AGG(id), ',') as ids from ${td.tables.parent_segments}
         store_last_results: true
         database: ${td.database}
     +ingest_activations:
@@ -75,6 +75,46 @@ _export:
         journey_ids: ${td.last_results.ids}
         api_endpoint: ${td.api_endpoint}
         cdp_api_endpoint: ${td.cdp_api_endpoint}
+        docker:
+            image: "digdag/digdag-python:3.9"
+        _env:
+            TD_API_KEY: ${secret:td.apikey}
+
++initial_ingest_journey_summary:
+    +get_current_journey_list:
+        td>:
+        query: select ARRAY_JOIN(array_agg(id), ',') as ids from ${td.tables.entities} where type = 'journey'
+        store_last_results: true
+        database: ${td.database}
+
+    +ingest_journey_summary:
+        py>: scripts.ingest_journey_summary.run
+        session_unixtime: ${session_unixtime}
+        dest_db: ${td.database}
+        dest_table: ${td.tables.journey_summary}
+        journey_ids: ${td.last_results.ids}
+        api_endpoint: ${td.api_endpoint}
+        cdp_api_endpoint: ${td.cdp_api_endpoint}
+        docker:
+            image: "digdag/digdag-python:3.9"
+        _env:
+            TD_API_KEY: ${secret:td.apikey}
+
++initial_ingest_journey_activation:
+    +get_current_journey_list:
+        td>:
+        query: select ARRAY_JOIN(array_agg(id), ',') as ids from ${td.tables.entities} where type = 'journey'
+        store_last_results: true
+        database: ${td.database}
+
+    +ingest_journey_activation:
+        py>: scripts.ingest_journey_activation.run
+        session_unixtime: ${session_unixtime}
+        dest_db: ${td.database}
+        dest_table: ${td.tables.journey_activation}
+        journey_ids: ${td.last_results.ids}
+        api_endpoint: ${td.api_endpoint}
+        cdp_api_endpoint: ${td.cdp_api_endpoint}
         docker:
             image: "digdag/digdag-python:3.9"
         _env: