From 09ed988b9846d54cf973c6050595ae4486a0d4e4 Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Thu, 17 Oct 2024 14:47:36 -0500 Subject: [PATCH 1/2] Clean up expression error messages (#5079) Signed-off-by: Taylor Gray --- .../plugins/processor/aggregate/AggregateProcessor.java | 4 +++- .../dataprepper/plugins/processor/date/DateProcessor.java | 4 +++- .../plugins/processor/dissect/DissectProcessor.java | 4 +++- .../plugins/processor/flatten/FlattenProcessor.java | 4 +++- .../dataprepper/plugins/geoip/processor/GeoIPProcessor.java | 4 +++- .../dataprepper/plugins/processor/grok/GrokProcessor.java | 4 +++- .../plugins/sink/opensearch/index/IndexConfiguration.java | 3 ++- 7 files changed, 20 insertions(+), 7 deletions(-) diff --git a/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessor.java b/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessor.java index 544928c242..2b19e98516 100644 --- a/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessor.java +++ b/data-prepper-plugins/aggregate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/aggregate/AggregateProcessor.java @@ -81,7 +81,9 @@ public AggregateProcessor(final AggregateProcessorConfig aggregateProcessorConfi pluginMetrics.gauge(CURRENT_AGGREGATE_GROUPS, aggregateGroupManager, AggregateGroupManager::getAllGroupsSize); if (aggregateProcessorConfig.getWhenCondition() != null && (!expressionEvaluator.isValidExpressionStatement(aggregateProcessorConfig.getWhenCondition()))) { - throw new InvalidPluginConfigurationException("aggregate_when {} is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax"); + throw new InvalidPluginConfigurationException( + String.format("aggregate_when \"%s\" is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax", + aggregateProcessorConfig.getWhenCondition())); } } diff --git a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java index 8c85250cb8..f2bf8cf187 100644 --- a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java +++ b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessor.java @@ -67,7 +67,9 @@ public DateProcessor(PluginMetrics pluginMetrics, final DateProcessorConfig date extractKeyAndFormatters(); if (dateProcessorConfig.getDateWhen() != null && (!expressionEvaluator.isValidExpressionStatement(dateProcessorConfig.getDateWhen()))) { - throw new InvalidPluginConfigurationException("date_when {} is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax"); + throw new InvalidPluginConfigurationException( + String.format("date_when \"%s\" is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax", + dateProcessorConfig.getDateWhen())); } } diff --git a/data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/DissectProcessor.java b/data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/DissectProcessor.java index 50453c6131..d1ea5cfbd2 100644 --- a/data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/DissectProcessor.java +++ b/data-prepper-plugins/dissect-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/dissect/DissectProcessor.java @@ -52,7 +52,9 @@ public DissectProcessor(PluginMetrics pluginMetrics, final DissectProcessorConfi if (dissectConfig.getDissectWhen() != null && (!expressionEvaluator.isValidExpressionStatement(dissectConfig.getDissectWhen()))) { - throw new InvalidPluginConfigurationException("dissect_when {} is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax"); + throw new InvalidPluginConfigurationException( + String.format("dissect_when \"%s\" is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax", + dissectConfig.getDissectWhen())); } } diff --git a/data-prepper-plugins/flatten-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessor.java b/data-prepper-plugins/flatten-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessor.java index 24a8dbf2bc..e59ee7e7a5 100644 --- a/data-prepper-plugins/flatten-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessor.java +++ b/data-prepper-plugins/flatten-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/flatten/FlattenProcessor.java @@ -45,7 +45,9 @@ public FlattenProcessor(final PluginMetrics pluginMetrics, final FlattenProcesso if (config.getFlattenWhen() != null && (!expressionEvaluator.isValidExpressionStatement(config.getFlattenWhen()))) { - throw new InvalidPluginConfigurationException("flatten_when {} is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax"); + throw new InvalidPluginConfigurationException( + String.format("flatten_when \"%s\" is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax", + config.getFlattenWhen())); } } diff --git a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessor.java b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessor.java index 748ef2847a..afb0ed23d3 100644 --- a/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessor.java +++ b/data-prepper-plugins/geoip-processor/src/main/java/org/opensearch/dataprepper/plugins/geoip/processor/GeoIPProcessor.java @@ -78,7 +78,9 @@ public GeoIPProcessor(final PluginMetrics pluginMetrics, if (geoIPProcessorConfig.getWhenCondition() != null && (!expressionEvaluator.isValidExpressionStatement(geoIPProcessorConfig.getWhenCondition()))) { - throw new InvalidPluginConfigurationException("geoip_when {} is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax"); + throw new InvalidPluginConfigurationException( + String.format("geoip_when \"%s\" is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax", + geoIPProcessorConfig.getWhenCondition())); } this.geoIPProcessorService = geoIpConfigSupplier.getGeoIPProcessorService().orElseThrow(() -> diff --git a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java index 5605fbee45..ee6578f051 100644 --- a/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java +++ b/data-prepper-plugins/grok-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/grok/GrokProcessor.java @@ -123,7 +123,9 @@ public GrokProcessor(final PluginMetrics pluginMetrics, if (grokProcessorConfig.getGrokWhen() != null && (!expressionEvaluator.isValidExpressionStatement(grokProcessorConfig.getGrokWhen()))) { - throw new InvalidPluginConfigurationException("grok_when {} is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax"); + throw new InvalidPluginConfigurationException( + String.format("grok_when \"%s\" is not a valid expression statement. See https://opensearch.org/docs/latest/data-prepper/pipelines/expression-syntax/ for valid expression syntax", + grokProcessorConfig.getGrokWhen())); } } diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java index 21b178ea9c..392b3bf556 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/index/IndexConfiguration.java @@ -244,7 +244,8 @@ public static IndexConfiguration readIndexConfig(final PluginSetting pluginSetti builder = builder.withVersionExpression(versionExpression); if (versionExpression != null && (!expressionEvaluator.isValidFormatExpression(versionExpression))) { - throw new InvalidPluginConfigurationException("document_version {} is not a valid format expression."); + throw new InvalidPluginConfigurationException( + String.format("document_version \"%s\" is not a valid format expression.", versionExpression)); } builder = builder.withVersionType(versionType); From 2575db341812c50ce00a187734b3033999a0e126 Mon Sep 17 00:00:00 2001 From: Souvik Bose Date: Thu, 17 Oct 2024 16:01:03 -0700 Subject: [PATCH 2/2] Fix AWS Sdk dependency conflict (#5082) Signed-off-by: Souvik Bose Co-authored-by: Souvik Bose --- build.gradle | 2 +- .../dataprepper/plugins/kinesis/source/KinesisService.java | 4 +++- data-prepper-plugins/s3-sink/build.gradle | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index be3b3dcc49..7f78c2424f 100644 --- a/build.gradle +++ b/build.gradle @@ -269,7 +269,7 @@ subprojects { configure(subprojects.findAll {it.name != 'data-prepper-api'}) { dependencies { - implementation platform('software.amazon.awssdk:bom:2.21.23') + implementation platform('software.amazon.awssdk:bom:2.25.11') implementation 'jakarta.validation:jakarta.validation-api:3.0.2' } } diff --git a/data-prepper-plugins/kinesis-source/src/main/java/org/opensearch/dataprepper/plugins/kinesis/source/KinesisService.java b/data-prepper-plugins/kinesis-source/src/main/java/org/opensearch/dataprepper/plugins/kinesis/source/KinesisService.java index 8919999c2a..7f712d9e73 100644 --- a/data-prepper-plugins/kinesis-source/src/main/java/org/opensearch/dataprepper/plugins/kinesis/source/KinesisService.java +++ b/data-prepper-plugins/kinesis-source/src/main/java/org/opensearch/dataprepper/plugins/kinesis/source/KinesisService.java @@ -45,6 +45,8 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; + public class KinesisService { private static final Logger LOG = LoggerFactory.getLogger(KinesisService.class); private static final int GRACEFUL_SHUTDOWN_WAIT_INTERVAL_SECONDS = 20; @@ -138,7 +140,7 @@ public Scheduler getScheduler(final Buffer> buffer) { try { scheduler = createScheduler(buffer); } catch (Exception ex) { - LOG.error("Caught exception when initializing KCL Scheduler. Will retry"); + LOG.error(NOISY, "Caught exception when initializing KCL Scheduler. Will retry", ex); } if (scheduler == null) { diff --git a/data-prepper-plugins/s3-sink/build.gradle b/data-prepper-plugins/s3-sink/build.gradle index 722f784300..8dd6b5783d 100644 --- a/data-prepper-plugins/s3-sink/build.gradle +++ b/data-prepper-plugins/s3-sink/build.gradle @@ -17,7 +17,7 @@ dependencies { implementation 'software.amazon.awssdk:netty-nio-client' implementation 'software.amazon.awssdk:s3' implementation 'software.amazon.awssdk:sts' - implementation 'software.amazon.awssdk:securitylake:2.28.20' + implementation 'software.amazon.awssdk:securitylake' implementation 'org.jetbrains.kotlin:kotlin-stdlib:1.9.22' implementation project(':data-prepper-plugins:avro-codecs') implementation libs.avro.core