Add endpoints as arguments (#394)

- add endpoints as arguments (workflow_monitoring) - add all table initialization process (cdp_monitoring)
treasure-data · May 30, 2024 · d435d4f · d435d4f
1 parent 24f5dfe
commit d435d4f
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 3 deletions.
diff --git a/scenarios/monitoring/cdp_monitoring/initial_ingest.dig b/scenarios/monitoring/cdp_monitoring/initial_ingest.dig
@@ -5,9 +5,9 @@ _export:
     +create_db:
         td_ddl>: 
         create_databases: [ "${td.database}" ]
-    +cerate_tables:
-        td>: 
-        query: create table ${td.tables.parent_segments} (id varchar)
+    +create_tables:
+        td_ddl>:
+        create_tables: ${Object.keys(td.tables)}
 
 +initial_ingest_ps:
     py>: scripts.ingest_ps.run

diff --git a/scenarios/monitoring/workflow_monitoring/incremental_ingest.dig b/scenarios/monitoring/workflow_monitoring/incremental_ingest.dig
@@ -17,6 +17,8 @@ schedule:
     session_unixtime: ${session_unixtime}
     dest_db: ${td.database}
     dest_table: ${td.tables.projects}
+    api_endpoint: ${td.api_endpoint}
+    workflow_endpoint: ${td.workflow_endpoint}
     docker:
       image: "digdag/digdag-python:3.10.1"
     _env:
@@ -34,6 +36,8 @@ schedule:
     session_unixtime: ${session_unixtime}
     dest_db: ${td.database}
     dest_table: ${td.tables.workflows}
+    api_endpoint: ${td.api_endpoint}
+    workflow_endpoint: ${td.workflow_endpoint}
     docker:
       image: "digdag/digdag-python:3.10.1"
     _env:
@@ -51,6 +55,8 @@ schedule:
     session_unixtime: ${session_unixtime}
     dest_db: ${td.database}
     dest_table: ${td.tables.schedules}
+    api_endpoint: ${td.api_endpoint}
+    workflow_endpoint: ${td.workflow_endpoint}
     docker:
       image: "digdag/digdag-python:3.10.1"
     _env:

diff --git a/scenarios/monitoring/workflow_monitoring/initial_ingest_session_attempt.dig b/scenarios/monitoring/workflow_monitoring/initial_ingest_session_attempt.dig
@@ -11,6 +11,8 @@ _export:
   session_unixtime: ${session_unixtime}
   dest_db: ${td.database}
   dest_table: ${td.tables.projects}
+  api_endpoint: ${td.api_endpoint}
+  workflow_endpoint: ${td.workflow_endpoint}
   docker:
     image: "digdag/digdag-python:3.10.1"
   _env:
@@ -21,6 +23,8 @@ _export:
   session_unixtime: ${session_unixtime}
   dest_db: ${td.database}
   dest_table: ${td.tables.workflows}
+  api_endpoint: ${td.api_endpoint}
+  workflow_endpoint: ${td.workflow_endpoint}
   docker:
     image: "digdag/digdag-python:3.10.1"
   _env:
@@ -31,6 +35,8 @@ _export:
   session_unixtime: ${session_unixtime}
   dest_db: ${td.database}
   dest_table: ${td.tables.schedules}
+  api_endpoint: ${td.api_endpoint}
+  workflow_endpoint: ${td.workflow_endpoint}
   docker:
     image: "digdag/digdag-python:3.10.1"
   _env:

diff --git a/scenarios/monitoring/workflow_monitoring/manual_ingest.dig b/scenarios/monitoring/workflow_monitoring/manual_ingest.dig
@@ -13,6 +13,8 @@ _export:
   dest_db: ${td.database}
   dest_table: ${td.tables.tasks}
   attempt_ids: ${attempt_ids}
+  api_endpoint: ${td.api_endpoint}
+  workflow_endpoint: ${td.workflow_endpoint}
   docker:
     image: "digdag/digdag-python:3.10.1"
   _env:
@@ -24,6 +26,8 @@ _export:
   dest_db: ${td.database}
   dest_table: ${td.tables.revisions}
   project_ids: ${project_ids}
+  api_endpoint: ${td.api_endpoint}
+  workflow_endpoint: ${td.workflow_endpoint}
   docker:
     image: "digdag/digdag-python:3.10.1"
   _env: