From 5a8ad791e7244ccaf451a32fa224427a2069083a Mon Sep 17 00:00:00 2001 From: Yuu Ohmura Date: Wed, 31 Jan 2024 14:08:09 +0900 Subject: [PATCH] - fix docker image version - add common settings for journey_activation and journey_summary --- .../cdp_monitoring/common/settings.yaml | 4 ++ .../cdp_monitoring/incremental_ingest.dig | 61 +++++++++++++++++-- .../cdp_monitoring/initial_ingest.dig | 42 ++++++++++++- 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/scenarios/monitoring/cdp_monitoring/common/settings.yaml b/scenarios/monitoring/cdp_monitoring/common/settings.yaml index b78087e6..299c8153 100644 --- a/scenarios/monitoring/cdp_monitoring/common/settings.yaml +++ b/scenarios/monitoring/cdp_monitoring/common/settings.yaml @@ -7,6 +7,10 @@ td: entities_history: entities_history journey_statistics: journey_statistics journey_statistics_history: journey_statistics_history + journey_summary: journey_summary + journey_summary_history: journey_summary_history + journey_activation: journey_activation + journey_activation_history: journey_activation_history activations: activations activations_history: activations_history api_endpoint: api.treasuredata.com diff --git a/scenarios/monitoring/cdp_monitoring/incremental_ingest.dig b/scenarios/monitoring/cdp_monitoring/incremental_ingest.dig index 12c65233..bc8a314d 100644 --- a/scenarios/monitoring/cdp_monitoring/incremental_ingest.dig +++ b/scenarios/monitoring/cdp_monitoring/incremental_ingest.dig @@ -19,7 +19,7 @@ schedule: api_endpoint: ${td.api_endpoint} cdp_api_endpoint: ${td.cdp_api_endpoint} docker: - image: "digdag/digdag-python:3.9" + image: "digdag/digdag-python:3.10.1" _env: TD_API_KEY: ${secret:td.apikey} @@ -44,7 +44,7 @@ schedule: api_endpoint: ${td.api_endpoint} cdp_api_endpoint: ${td.cdp_api_endpoint} docker: - image: "digdag/digdag-python:3.9" + image: "digdag/digdag-python:3.10.1" _env: TD_API_KEY: ${secret:td.apikey} @@ -69,7 +69,7 @@ schedule: api_endpoint: ${td.api_endpoint} cdp_api_endpoint: ${td.cdp_api_endpoint} docker: - image: "digdag/digdag-python:3.9" + image: "digdag/digdag-python:3.10.1" _env: TD_API_KEY: ${secret:td.apikey} @@ -95,6 +95,59 @@ schedule: api_endpoint: ${td.api_endpoint} cdp_api_endpoint: ${td.cdp_api_endpoint} docker: - image: "digdag/digdag-python:3.9" + image: "digdag/digdag-python:3.10.1" _env: TD_API_KEY: ${secret:td.apikey} + + ++incremental_ingest_journey_summary: + +append_journey_summary_history: + td>: + query: select * from ${td.tables.journey_summary} + insert_into: ${td.tables.journey_summary_history} + database: ${td.database} + + +get_current_journey_list: + td>: + query: select ARRAY_JOIN(array_agg(id), ',') as ids from ${td.tables.entities} where type = 'journey' + store_last_results: true + database: ${td.database} + + +ingest_journey_summary: + py>: scripts.ingest_journey_summary.run + session_unixtime: ${session_unixtime} + dest_db: ${td.database} + dest_table: ${td.tables.journey_summary} + journey_ids: ${td.last_results.ids} + api_endpoint: ${td.api_endpoint} + cdp_api_endpoint: ${td.cdp_api_endpoint} + docker: + image: "digdag/digdag-python:3.10.1" + _env: + TD_API_KEY: ${secret:td.apikey} + ++incremental_ingest_journey_activation: + +append_journey_activation_history: + td>: + query: select * from ${td.tables.journey_activation} + insert_into: ${td.tables.journey_activation_history} + database: ${td.database} + + +get_current_journey_list: + td>: + query: select ARRAY_JOIN(array_agg(id), ',') as ids from ${td.tables.entities} where type = 'journey' + store_last_results: true + database: ${td.database} + + +ingest_journey_activation: + py>: scripts.ingest_journey_activation.run + session_unixtime: ${session_unixtime} + dest_db: ${td.database} + dest_table: ${td.tables.journey_activation} + journey_ids: ${td.last_results.ids} + api_endpoint: ${td.api_endpoint} + cdp_api_endpoint: ${td.cdp_api_endpoint} + docker: + image: "digdag/digdag-python:3.10.1" + _env: + TD_API_KEY: ${secret:td.apikey} \ No newline at end of file diff --git a/scenarios/monitoring/cdp_monitoring/initial_ingest.dig b/scenarios/monitoring/cdp_monitoring/initial_ingest.dig index 2529dbdd..2eee105d 100644 --- a/scenarios/monitoring/cdp_monitoring/initial_ingest.dig +++ b/scenarios/monitoring/cdp_monitoring/initial_ingest.dig @@ -43,7 +43,7 @@ _export: +initial_ingest_activations: +get_current_parent_segment_list: td>: - query: select ARRAY_AGG(id) as ids from ${td.tables.parent_segments} + query: select ARRAY_JOIN(ARRAY_AGG(id), ',') as ids from ${td.tables.parent_segments} store_last_results: true database: ${td.database} +ingest_activations: @@ -75,6 +75,46 @@ _export: journey_ids: ${td.last_results.ids} api_endpoint: ${td.api_endpoint} cdp_api_endpoint: ${td.cdp_api_endpoint} + docker: + image: "digdag/digdag-python:3.9" + _env: + TD_API_KEY: ${secret:td.apikey} + ++initial_ingest_journey_summary: + +get_current_journey_list: + td>: + query: select ARRAY_JOIN(array_agg(id), ',') as ids from ${td.tables.entities} where type = 'journey' + store_last_results: true + database: ${td.database} + + +ingest_journey_summary: + py>: scripts.ingest_journey_summary.run + session_unixtime: ${session_unixtime} + dest_db: ${td.database} + dest_table: ${td.tables.journey_summary} + journey_ids: ${td.last_results.ids} + api_endpoint: ${td.api_endpoint} + cdp_api_endpoint: ${td.cdp_api_endpoint} + docker: + image: "digdag/digdag-python:3.9" + _env: + TD_API_KEY: ${secret:td.apikey} + ++initial_ingest_journey_activation: + +get_current_journey_list: + td>: + query: select ARRAY_JOIN(array_agg(id), ',') as ids from ${td.tables.entities} where type = 'journey' + store_last_results: true + database: ${td.database} + + +ingest_journey_activation: + py>: scripts.ingest_journey_activation.run + session_unixtime: ${session_unixtime} + dest_db: ${td.database} + dest_table: ${td.tables.journey_activation} + journey_ids: ${td.last_results.ids} + api_endpoint: ${td.api_endpoint} + cdp_api_endpoint: ${td.cdp_api_endpoint} docker: image: "digdag/digdag-python:3.9" _env: