diff --git a/packages/kbn-apm-synthtrace/README.md b/packages/kbn-apm-synthtrace/README.md index a9704b1905d1b..43b32bd97d0d0 100644 --- a/packages/kbn-apm-synthtrace/README.md +++ b/packages/kbn-apm-synthtrace/README.md @@ -119,7 +119,7 @@ Scenario files accept 3 arguments, 2 of them optional and 1 mandatory |-------------|:----------|------------------------------------------------------------------------------------------------------------------------------------------------------| | `generate` | mandatory | This is the main function responsible for returning the events which will be indexed | | `bootstrap` | optional | In case some setup needs to be done, before the data is generated, this function provides access to all available ES Clients to play with | -| `setClient` | optional | By default the apmEsClient used to generate data. If anyother client like logsEsClient needs to be used instead, this is where it should be returned | +| `teardown` | optional | In case some setup needs to be done, after all data is generated, this function provides access to all available ES Clients to play with | The following options are supported: diff --git a/packages/kbn-apm-synthtrace/src/cli/scenario.ts b/packages/kbn-apm-synthtrace/src/cli/scenario.ts index 271b39db8c024..6b25f7873514c 100644 --- a/packages/kbn-apm-synthtrace/src/cli/scenario.ts +++ b/packages/kbn-apm-synthtrace/src/cli/scenario.ts @@ -42,4 +42,5 @@ type Generate = (options: { export type Scenario = (options: RunOptions & { logger: Logger }) => Promise<{ bootstrap?: (options: EsClients & KibanaClients) => Promise; generate: Generate; + teardown?: (options: EsClients & KibanaClients) => Promise; }>; diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts b/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts index 9478ae8f26af2..7166bef523f4d 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts @@ -38,7 +38,11 @@ export async function startLiveDataUpload({ } = await bootstrap(runOptions); const scenario = await getScenario({ file, logger }); - const { generate, bootstrap: scenarioBootsrap } = await scenario({ ...runOptions, logger }); + const { + generate, + bootstrap: scenarioBootsrap, + teardown: scenarioTearDown, + } = await scenario({ ...runOptions, logger }); if (scenarioBootsrap) { await scenarioBootsrap({ @@ -59,11 +63,27 @@ export async function startLiveDataUpload({ // @ts-expect-error upgrade typescript v4.9.5 const cachedStreams: WeakMap = new WeakMap(); - process.on('SIGINT', () => closeStreams()); - process.on('SIGTERM', () => closeStreams()); - process.on('SIGQUIT', () => closeStreams()); + process.on('SIGINT', () => closeStreamsAndTeardown()); + process.on('SIGTERM', () => closeStreamsAndTeardown()); + process.on('SIGQUIT', () => closeStreamsAndTeardown()); + + async function closeStreamsAndTeardown() { + if (scenarioTearDown) { + try { + await scenarioTearDown({ + apmEsClient, + logsEsClient, + infraEsClient, + otelEsClient, + syntheticsEsClient, + entitiesEsClient, + entitiesKibanaClient, + }); + } catch (error) { + logger.error('Error during scenario teardown', error); + } + } - function closeStreams() { currentStreams.forEach((stream) => { stream.end(() => { process.exit(0); diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts index 0d5398f473cb2..4a4d0af10e47f 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts @@ -84,7 +84,7 @@ async function start() { logger.info(`Running scenario from ${bucketFrom.toISOString()} to ${bucketTo.toISOString()}`); - const { generate, bootstrap } = await scenario({ ...runOptions, logger }); + const { generate, bootstrap, teardown } = await scenario({ ...runOptions, logger }); if (bootstrap) { await bootstrap({ @@ -142,6 +142,18 @@ async function start() { await Promise.all(promises); }); + + if (teardown) { + await teardown({ + apmEsClient, + logsEsClient, + infraEsClient, + syntheticsEsClient, + otelEsClient, + entitiesEsClient, + entitiesKibanaClient, + }); + } } parentPort!.on('message', (message) => { diff --git a/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts index e6afdd381c51b..337b614b8e5e3 100644 --- a/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts +++ b/packages/kbn-apm-synthtrace/src/lib/logs/logs_synthtrace_es_client.ts @@ -52,6 +52,15 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient { } } + async deleteIndexTemplate(name: IndexTemplateName) { + try { + await this.client.indices.deleteIndexTemplate({ name }); + this.logger.info(`Index template successfully deleted: ${name}`); + } catch (err) { + this.logger.error(`Index template deletion failed: ${name} - ${err.message}`); + } + } + async createComponentTemplate({ name, mappings, @@ -150,6 +159,17 @@ export class LogsSynthtraceEsClient extends SynthtraceEsClient { } } + async deleteCustomPipeline(id = LogsCustom) { + try { + this.client.ingest.deletePipeline({ + id, + }); + this.logger.info(`Custom pipeline deleted: ${id}`); + } catch (err) { + this.logger.error(`Custom pipeline deletion failed: ${id} - ${err.message}`); + } + } + getDefaultPipeline({ includeSerialization }: Pipeline = { includeSerialization: true }) { return logsPipeline({ includeSerialization }); } diff --git a/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts index b3e41bbdd4e28..520dd2b0d6422 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/degraded_logs.ts @@ -36,6 +36,9 @@ const scenario: Scenario = async (runOptions) => { bootstrap: async ({ logsEsClient }) => { if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb); }, + teardown: async ({ logsEsClient }) => { + await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb); + }, generate: ({ range, clients: { logsEsClient } }) => { const { logger } = runOptions; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts index 94a33f9130233..a484ae7d2962b 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/failed_logs.ts @@ -75,6 +75,11 @@ const scenario: Scenario = async (runOptions) => { }, }); }, + teardown: async ({ logsEsClient }) => { + await logsEsClient.deleteComponentTemplate(LogsCustom); + await logsEsClient.deleteCustomPipeline(); + if (isLogsDb) await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb); + }, generate: ({ range, clients: { logsEsClient } }) => { const { logger } = runOptions; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/logs_and_metrics.ts b/packages/kbn-apm-synthtrace/src/scenarios/logs_and_metrics.ts index 96f7220a7dcdc..638b5f9853437 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/logs_and_metrics.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/logs_and_metrics.ts @@ -38,6 +38,9 @@ const scenario: Scenario = async (runOptions) => { bootstrap: async ({ logsEsClient }) => { if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb); }, + teardown: async ({ logsEsClient }) => { + await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb); + }, generate: ({ range, clients: { logsEsClient, apmEsClient } }) => { const { numServices = 3 } = runOptions.scenarioOpts || {}; const { logger } = runOptions; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts b/packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts index 6dac3fc9f3226..b7effec5d9513 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/logs_traces_hosts.ts @@ -52,6 +52,9 @@ const scenario: Scenario = async (runOp bootstrap: async ({ logsEsClient }) => { if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb); }, + teardown: async ({ logsEsClient }) => { + await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb); + }, generate: ({ range, clients: { logsEsClient, infraEsClient, apmEsClient } }) => { const { numSpaces, diff --git a/packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts index 1e3f742937a48..2472b3c30982b 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/simple_logs.ts @@ -70,6 +70,9 @@ const scenario: Scenario = async (runOptions) => { bootstrap: async ({ logsEsClient }) => { if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb); }, + teardown: async ({ logsEsClient }) => { + await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb); + }, generate: ({ range, clients: { logsEsClient } }) => { const { logger } = runOptions; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/simple_non_ecs_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/simple_non_ecs_logs.ts index 8c965cec5b2b0..56667809da80a 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/simple_non_ecs_logs.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/simple_non_ecs_logs.ts @@ -78,6 +78,9 @@ const scenario: Scenario = async (runOptions) => { await logsEsClient.createIndex('cloud-logs-synth.2-default'); if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb); }, + teardown: async ({ logsEsClient }) => { + await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb); + }, generate: ({ range, clients: { logsEsClient } }) => { const { logger } = runOptions; diff --git a/packages/kbn-apm-synthtrace/src/scenarios/unstructured_logs.ts b/packages/kbn-apm-synthtrace/src/scenarios/unstructured_logs.ts index b87fd7038a7d3..fcf3567d676e2 100644 --- a/packages/kbn-apm-synthtrace/src/scenarios/unstructured_logs.ts +++ b/packages/kbn-apm-synthtrace/src/scenarios/unstructured_logs.ts @@ -20,6 +20,9 @@ const scenario: Scenario = async (runOptions) => { bootstrap: async ({ logsEsClient }) => { if (isLogsDb) await logsEsClient.createIndexTemplate(IndexTemplateName.LogsDb); }, + teardown: async ({ logsEsClient }) => { + await logsEsClient.deleteIndexTemplate(IndexTemplateName.LogsDb); + }, generate: ({ range, clients: { logsEsClient } }) => { const { logger } = runOptions;