From 791e355794bed42e39490063a504bc592c4b1093 Mon Sep 17 00:00:00 2001 From: Katherine Shen <40495707+shenkw1@users.noreply.github.com> Date: Tue, 7 Jan 2025 10:22:21 -0600 Subject: [PATCH 01/36] Remove code tags from processor names, fill in default values (#5278) * remove code tags from processor names in descriptions, add missing default values Signed-off-by: Katherine Shen --- .../AnomalyDetectorProcessorConfig.java | 10 +++++----- .../modes/RandomCutForestModeConfig.java | 17 +++++++++-------- .../lambda/processor/LambdaProcessorConfig.java | 2 +- .../processor/date/DateProcessorConfig.java | 2 +- .../mutateevent/CopyValueProcessorConfig.java | 12 ++++++++---- .../obfuscation/ObfuscationProcessorConfig.java | 2 +- 6 files changed, 25 insertions(+), 20 deletions(-) diff --git a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java index 69d4f8c8a4..578e22a351 100644 --- a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java +++ b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/AnomalyDetectorProcessorConfig.java @@ -23,17 +23,17 @@ @JsonClassDescription("The anomaly detector processor takes structured data and runs anomaly detection algorithms " + "on fields that you can configure in that data.") public class AnomalyDetectorProcessorConfig { + @JsonPropertyDescription("A non-ordered List that is used as input to the ML algorithm to detect anomalies in the values of the keys in the list. At least one key is required.") + @JsonProperty("keys") + @NotEmpty + private List keys; + @JsonPropertyDescription("The ML algorithm (or model) used to detect anomalies. You must provide a mode. See random_cut_forest mode.") @JsonProperty("mode") @NotNull @UsesDataPrepperPlugin(pluginType = AnomalyDetectorMode.class) private PluginModel detectorMode; - @JsonPropertyDescription("A non-ordered List that is used as input to the ML algorithm to detect anomalies in the values of the keys in the list. At least one key is required.") - @JsonProperty("keys") - @NotEmpty - private List keys; - @JsonPropertyDescription("If provided, anomalies will be detected within each unique instance of these keys. For example, if you provide the ip field, anomalies will be detected separately for each unique IP address.") @JsonProperty("identification_keys") @ExampleValues({ diff --git a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/modes/RandomCutForestModeConfig.java b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/modes/RandomCutForestModeConfig.java index c477746253..74c90fca50 100644 --- a/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/modes/RandomCutForestModeConfig.java +++ b/data-prepper-plugins/anomaly-detector-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/anomalydetector/modes/RandomCutForestModeConfig.java @@ -13,6 +13,7 @@ import java.util.HashSet; public class RandomCutForestModeConfig { + public static final String DEFAULT_TYPE = "metrics"; public static final int DEFAULT_SHINGLE_SIZE = 4; private static final int MIN_SHINGLE_SIZE = 1; public static final int MAX_SHINGLE_SIZE = 60; @@ -27,31 +28,31 @@ public class RandomCutForestModeConfig { public static final String VERSION_1_0 = "1.0"; @JsonPropertyDescription("The algorithm version number. Default is 1.0.") - @JsonProperty("version") + @JsonProperty(value = "version", defaultValue = VERSION_1_0) private String version = VERSION_1_0; public static final Set validVersions = new HashSet<>(Set.of(VERSION_1_0)); @JsonPropertyDescription("The type of data sent to the algorithm. Default is metrics type") - @JsonProperty("type") + @JsonProperty(value = "type", defaultValue = DEFAULT_TYPE) private String type = RandomCutForestType.METRICS.toString(); public static final Set validTypes = new HashSet<>(Set.of(RandomCutForestType.METRICS.toString())); - @JsonPropertyDescription("The shingle size used in the ML algorithm. Default is 60.") - @JsonProperty("shingle_size") + @JsonPropertyDescription("The shingle size used in the ML algorithm. Default is 4.") + @JsonProperty(value = "shingle_size", defaultValue = "" + DEFAULT_SHINGLE_SIZE) private int shingleSize = DEFAULT_SHINGLE_SIZE; @JsonPropertyDescription("The sample size used in the ML algorithm. Default is 256.") - @JsonProperty("sample_size") + @JsonProperty(value = "sample_size", defaultValue = "" + DEFAULT_SAMPLE_SIZE) private int sampleSize = DEFAULT_SAMPLE_SIZE; @JsonPropertyDescription("The time decay value used in the ML algorithm. Used as the mathematical expression timeDecay divided by SampleSize in the ML algorithm. Default is 0.1") - @JsonProperty("time_decay") + @JsonProperty(value = "time_decay", defaultValue = "" + DEFAULT_TIME_DECAY) private double timeDecay = DEFAULT_TIME_DECAY; - @JsonPropertyDescription("Output after indicates the number of events to consume before outputting anamolies. Default is 32.") - @JsonProperty("output_after") + @JsonPropertyDescription("Output after indicates the number of events to consume before outputting anomalies. Default is 32.") + @JsonProperty(value = "output_after", defaultValue = "" + DEFAULT_OUTPUT_AFTER) private int outputAfter = DEFAULT_OUTPUT_AFTER; @AssertTrue(message = "Value of output_after must be less than or equal to the value of sample_size") diff --git a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorConfig.java b/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorConfig.java index cfa7417896..c9675ed931 100644 --- a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorConfig.java +++ b/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorConfig.java @@ -17,7 +17,7 @@ import org.opensearch.dataprepper.plugins.lambda.common.config.LambdaCommonConfig; @JsonPropertyOrder -@JsonClassDescription("The aws_lambda processor enables invocation of an AWS Lambda function within your Data Prepper pipeline in order to process events." + +@JsonClassDescription("The aws_lambda processor enables invocation of an AWS Lambda function within your Data Prepper pipeline in order to process events." + "It supports both synchronous and asynchronous invocations based on your use case.") public class LambdaProcessorConfig extends LambdaCommonConfig { static final String DEFAULT_INVOCATION_TYPE = "request-response"; diff --git a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java index f813a8fa1f..2e3114d979 100644 --- a/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java +++ b/data-prepper-plugins/date-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/date/DateProcessorConfig.java @@ -142,7 +142,7 @@ public static boolean isValidPattern(final String pattern) { }) private List match; - @JsonProperty("destination") + @JsonProperty(value = "destination", defaultValue = DEFAULT_DESTINATION) @JsonPropertyDescription("The field used to store the timestamp parsed by the date processor. " + "Can be used with both match and from_time_received. Default is @timestamp.") private String destination = DEFAULT_DESTINATION; diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java index c7ec7683ef..e44c1da74e 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/CopyValueProcessorConfig.java @@ -30,13 +30,15 @@ public static class Entry { @NotEmpty @NotNull @JsonProperty("from_key") - @JsonPropertyDescription("The key of the entry to be copied.") + @JsonPropertyDescription("The key of the entry to be copied. Either from_key and " + + "to_key or from_list and to_list must be defined.") private String fromKey; @NotEmpty @NotNull @JsonProperty("to_key") - @JsonPropertyDescription("The key of the new entry to be added.") + @JsonPropertyDescription("The key of the new entry to be added. Either from_key and " + + "to_key or from_list and to_list must be defined.") private String toKey; @JsonProperty("overwrite_if_to_key_exists") @@ -86,14 +88,16 @@ public Entry() { private List entries; @JsonProperty(FROM_LIST_KEY) - @JsonPropertyDescription("The key of the list of objects to be copied. to_list must also be defined.") + @JsonPropertyDescription("The key of the list of objects to be copied. Either from_key and " + + "to_key or from_list and to_list must be defined.") @AlsoRequired(values = { @AlsoRequired.Required(name = TO_LIST_KEY) }) private String fromList; @JsonProperty(TO_LIST_KEY) - @JsonPropertyDescription("The key of the new list to be added. from_list must also be defined.") + @JsonPropertyDescription("The key of the new list to be added. Either from_key and " + + "to_key or from_list and to_list must be defined.") @AlsoRequired(values = { @AlsoRequired.Required(name = FROM_LIST_KEY) }) diff --git a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java index c8a5ef4e07..fa0ddf355c 100644 --- a/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java +++ b/data-prepper-plugins/obfuscate-processor/src/main/java/org/opensearch/dataprepper/plugins/processor/obfuscation/ObfuscationProcessorConfig.java @@ -22,7 +22,7 @@ import java.util.List; @JsonPropertyOrder -@JsonClassDescription("The obfuscate process enables obfuscation of fields inside your documents in order to " + +@JsonClassDescription("The obfuscate processor enables obfuscation of fields inside your documents in order to " + "protect sensitive data.") public class ObfuscationProcessorConfig { From bad172c3f10b277ea888e8dc910f8baf961baf8b Mon Sep 17 00:00:00 2001 From: Maxwell Brown <55033421+Galactus22625@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:38:23 -0800 Subject: [PATCH 02/36] Jira Source Configuration and Filter Changes and add license headers (#5306) * jira yaml changes and tests, authentication and hosts Signed-off-by: Maxwell Brown * filter config file changes Signed-off-by: Maxwell Brown * jira filter config changes and implementation Signed-off-by: Maxwell Brown * filter change test corrections Signed-off-by: Maxwell Brown * project name -> key for now Signed-off-by: Maxwell Brown * removed unfishished function Signed-off-by: Maxwell Brown * address pr comments Signed-off-by: Maxwell Brown * Add license headers to all jira source files Signed-off-by: Maxwell Brown --------- Signed-off-by: Maxwell Brown --- .../plugins/source/jira/JiraClient.java | 10 ++ .../plugins/source/jira/JiraItemInfo.java | 10 ++ .../plugins/source/jira/JiraIterator.java | 10 ++ .../plugins/source/jira/JiraService.java | 68 +++++++++-- .../plugins/source/jira/JiraSource.java | 10 ++ .../plugins/source/jira/JiraSourceConfig.java | 101 ++++------------ .../configuration/AuthenticationConfig.java | 45 +++++++ .../jira/configuration/BasicConfig.java | 29 +++++ .../jira/configuration/FilterConfig.java | 26 ++++ .../jira/configuration/IssueTypeConfig.java | 29 +++++ .../source/jira/configuration/NameConfig.java | 30 +++++ .../jira/configuration/Oauth2Config.java | 35 ++++++ .../jira/configuration/ProjectConfig.java | 20 +++ .../jira/configuration/StatusConfig.java | 29 +++++ .../jira/exception/BadRequestException.java | 10 ++ .../jira/exception/UnAuthorizedException.java | 10 ++ .../plugins/source/jira/models/IssueBean.java | 10 ++ .../source/jira/models/SearchResults.java | 11 ++ .../jira/rest/BasicAuthInterceptor.java | 14 ++- .../jira/rest/CustomRestTemplateConfig.java | 10 ++ .../source/jira/rest/JiraRestClient.java | 15 +++ .../jira/rest/OAuth2RequestInterceptor.java | 10 ++ .../source/jira/rest/auth/JiraAuthConfig.java | 10 ++ .../jira/rest/auth/JiraAuthFactory.java | 10 ++ .../jira/rest/auth/JiraBasicAuthConfig.java | 10 ++ .../jira/rest/auth/JiraOauthConfig.java | 18 ++- .../source/jira/utils/AddressValidation.java | 10 ++ .../plugins/source/jira/utils/Constants.java | 10 ++ .../source/jira/utils/JiraConfigHelper.java | 66 ++++++++-- .../source/jira/utils/JiraContentType.java | 10 ++ .../source/jira/utils/JqlConstants.java | 13 ++ .../plugins/source/jira/JiraClientTest.java | 10 ++ .../source/jira/JiraConfigHelperTest.java | 104 +++++++++++++--- .../plugins/source/jira/JiraItemInfoTest.java | 10 ++ .../plugins/source/jira/JiraIteratorTest.java | 10 ++ .../plugins/source/jira/JiraServiceTest.java | 54 +++++++-- .../source/jira/JiraSourceConfigTest.java | 114 ++++++++++-------- .../plugins/source/jira/JiraSourceTest.java | 31 ++++- .../exception/BadRequestExceptionTest.java | 10 ++ .../exception/UnAuthorizedExceptionTest.java | 10 ++ .../source/jira/models/IssueBeanTest.java | 10 ++ .../source/jira/models/SearchResultsTest.java | 10 ++ .../jira/rest/BasicAuthInterceptorTest.java | 24 +++- .../rest/CustomRestTemplateConfigTest.java | 33 +++++ .../source/jira/rest/JiraRestClientTest.java | 10 ++ .../rest/OAuth2RequestInterceptorTest.java | 10 ++ .../jira/rest/auth/JiraAuthFactoryTest.java | 20 +++ .../rest/auth/JiraBasicAuthConfigTest.java | 10 ++ .../jira/rest/auth/JiraOauthConfigTest.java | 19 ++- .../jira/utils/AddressValidationTest.java | 10 ++ .../jira/utils/JiraContentTypeTest.java | 10 ++ .../resources/basic-auth-jira-pipeline.yaml | 10 +- .../resources/oauth2-auth-jira-pipeline.yaml | 15 ++- 53 files changed, 1042 insertions(+), 201 deletions(-) create mode 100644 data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/AuthenticationConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/BasicConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/FilterConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/IssueTypeConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/NameConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/Oauth2Config.java create mode 100644 data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/ProjectConfig.java create mode 100644 data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/StatusConfig.java diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java index 7e463f4534..3c809fa798 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraClient.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import com.fasterxml.jackson.core.JsonProcessingException; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java index 0952f20a2a..349ab7bccb 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfo.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import lombok.Getter; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java index 3027b9ec99..5ae040cc7f 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraIterator.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java index 38eb9eed40..4600d1bdeb 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraService.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import io.micrometer.core.instrument.Counter; @@ -14,8 +24,10 @@ import javax.inject.Named; import java.time.Instant; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Queue; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -25,9 +37,12 @@ import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.DELIMITER; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.GREATER_THAN_EQUALS; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.ISSUE_TYPE_IN; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.ISSUE_TYPE_NOT_IN; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PREFIX; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PROJECT_IN; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.PROJECT_NOT_IN; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.STATUS_IN; +import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.STATUS_NOT_IN; import static org.opensearch.dataprepper.plugins.source.jira.utils.JqlConstants.SUFFIX; @@ -117,26 +132,41 @@ private void addItemsToQueue(List issueList, Queue itemInfo private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration, Instant ts) { log.info("Creating issue filter criteria"); - if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectKeyFilter(configuration))) { + if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameIncludeFilter(configuration)) || !CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameExcludeFilter(configuration)) ) { validateProjectFilters(configuration); } StringBuilder jiraQl = new StringBuilder(UPDATED + GREATER_THAN_EQUALS + ts.toEpochMilli()); - if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectKeyFilter(configuration))) { - jiraQl.append(PROJECT_IN).append(JiraConfigHelper.getProjectKeyFilter(configuration).stream() + if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameIncludeFilter(configuration))) { + jiraQl.append(PROJECT_IN).append(JiraConfigHelper.getProjectNameIncludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(JiraConfigHelper.getProjectNameExcludeFilter(configuration))) { + jiraQl.append(PROJECT_NOT_IN).append(JiraConfigHelper.getProjectNameExcludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeIncludeFilter(configuration))) { + jiraQl.append(ISSUE_TYPE_IN).append(JiraConfigHelper.getIssueTypeIncludeFilter(configuration).stream() + .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) + .append(CLOSING_ROUND_BRACKET); + } + if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeExcludeFilter(configuration))) { + jiraQl.append(ISSUE_TYPE_NOT_IN).append(JiraConfigHelper.getIssueTypeExcludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueTypeFilter(configuration))) { - jiraQl.append(ISSUE_TYPE_IN).append(JiraConfigHelper.getIssueTypeFilter(configuration).stream() + if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusIncludeFilter(configuration))) { + jiraQl.append(STATUS_IN).append(JiraConfigHelper.getIssueStatusIncludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusFilter(configuration))) { - jiraQl.append(STATUS_IN).append(JiraConfigHelper.getIssueStatusFilter(configuration).stream() + if (!CollectionUtils.isEmpty(JiraConfigHelper.getIssueStatusExcludeFilter(configuration))) { + jiraQl.append(STATUS_NOT_IN).append(JiraConfigHelper.getIssueStatusExcludeFilter(configuration).stream() .collect(Collectors.joining(DELIMITER, PREFIX, SUFFIX))) .append(CLOSING_ROUND_BRACKET); } - log.trace("Created issue filter criteria JiraQl query: {}", jiraQl); + log.error("Created issue filter criteria JiraQl query: {}", jiraQl); return jiraQl; } @@ -148,9 +178,21 @@ private StringBuilder createIssueFilterCriteria(JiraSourceConfig configuration, private void validateProjectFilters(JiraSourceConfig configuration) { log.trace("Validating project filters"); List badFilters = new ArrayList<>(); + Set includedProjects = new HashSet<>(); + List includedAndExcludedProjects = new ArrayList<>(); Pattern regex = Pattern.compile("[^A-Z0-9]"); - JiraConfigHelper.getProjectKeyFilter(configuration).forEach(projectFilter -> { + JiraConfigHelper.getProjectNameIncludeFilter(configuration).forEach(projectFilter -> { Matcher matcher = regex.matcher(projectFilter); + includedProjects.add(projectFilter); + if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { + badFilters.add(projectFilter); + } + }); + JiraConfigHelper.getProjectNameExcludeFilter(configuration).forEach(projectFilter -> { + Matcher matcher = regex.matcher(projectFilter); + if (includedProjects.contains(projectFilter)) { + includedAndExcludedProjects.add(projectFilter); + } if (matcher.find() || projectFilter.length() <= 1 || projectFilter.length() > 10) { badFilters.add(projectFilter); } @@ -162,6 +204,14 @@ private void validateProjectFilters(JiraSourceConfig configuration) { "Invalid project key found in filter configuration for " + filters); } + if (!includedAndExcludedProjects.isEmpty()) { + String filters = String.join("\"" + includedAndExcludedProjects + "\"", ", "); + log.error("One or more project keys found in both include and exclude: {}", includedAndExcludedProjects); + throw new BadRequestException("Bad request exception occurred " + + "Project filters is invalid because the following projects are listed in both include and exclude" + + filters); + } + } } \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java index 2641ab60ad..2ffc7b3b53 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSource.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java index f37434cf9d..df5cd70f0b 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfig.java @@ -1,16 +1,24 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.Size; +import jakarta.validation.Valid; import lombok.Getter; +import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.FilterConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig; import java.time.Duration; -import java.util.ArrayList; import java.util.List; -import java.util.Map; - -import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; @Getter public class JiraSourceConfig implements CrawlerSourceConfig { @@ -20,50 +28,23 @@ public class JiraSourceConfig implements CrawlerSourceConfig { /** * Jira account url */ - @JsonProperty("account_url") - private String accountUrl; + @JsonProperty("hosts") + private List hosts; /** - * A map of connector credentials specific to this source + * Authentication Config to Access Jira */ - @JsonProperty("connector_credentials") - private Map connectorCredentials; + @JsonProperty("authentication") + @Valid + private AuthenticationConfig authenticationConfig; - /** - * List of projects to ingest - */ - @JsonProperty("projects") - @Size(max = 1000, message = "Project type filter should not be more than 1000") - private List project = new ArrayList<>(); /** - * List of specific issue types to ingest. - * Ex: Story, Epic, Task etc + * Filter Config to filter what tickets get ingested */ - @JsonProperty("issue_types") - @Size(max = 1000, message = "Issue type filter should be less than 1000") - private List issueType = new ArrayList<>(); + @JsonProperty("filter") + private FilterConfig filterConfig; - /** - * Optional Inclusion patterns for filtering some tickets - */ - @JsonProperty("inclusion_patterns") - @Size(max = 100, message = "inclusion pattern filters should not be more than 1000") - private List inclusionPatterns; - - /** - * Optional Exclusion patterns for excluding some tickets - */ - @JsonProperty("exclusion_patterns") - @Size(max = 1000, message = "exclusion pattern filter should be less than 1000") - private List exclusionPatterns; - - /** - * Optional Status filter to ingest the tickets - */ - @JsonProperty("statuses") - @Size(max = 1000, message = "Status filter should be less than 1000") - private List status = new ArrayList<>(); /** * Number of worker threads to spawn to parallel source fetching @@ -78,43 +59,11 @@ public class JiraSourceConfig implements CrawlerSourceConfig { @JsonProperty("backoff_time") private Duration backOff = DEFAULT_BACKOFF_MILLIS; - public String getJiraId() { - return this.getConnectorCredentials().get("jira_id"); - } - - public String getJiraCredential() { - return this.getConnectorCredentials().get("jira_credential"); + public String getAccountUrl() { + return this.getHosts().get(0); } public String getAuthType() { - return this.getConnectorCredentials().get("auth_type"); - } - - public String getAccessToken() { - return fetchGivenOAuthAttribute("access_token"); - } - - public String getRefreshToken() { - return fetchGivenOAuthAttribute("refresh_token"); - } - - public String getClientId() { - return fetchGivenOAuthAttribute("client_id"); - } - - public String getClientSecret() { - return fetchGivenOAuthAttribute("client_secret"); + return this.getAuthenticationConfig().getAuthType(); } - - private String fetchGivenOAuthAttribute(String givenAttribute) { - if (!OAUTH2.equals(getAuthType())) { - throw new RuntimeException("Authentication Type is not OAuth2."); - } - String attributeValue = this.getConnectorCredentials().get(givenAttribute); - if (attributeValue == null || attributeValue.isEmpty()) { - throw new RuntimeException(String.format("%s is required for OAuth2 AuthType", givenAttribute)); - } - return attributeValue; - } - } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/AuthenticationConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/AuthenticationConfig.java new file mode 100644 index 0000000000..25cfd4185a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/AuthenticationConfig.java @@ -0,0 +1,45 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.jira.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; + +@Getter +public class AuthenticationConfig { + @JsonProperty("basic") + @Valid + private BasicConfig basicConfig; + + @JsonProperty("oauth2") + @Valid + private Oauth2Config oauth2Config; + + @AssertTrue(message = "Authentication config should have either basic or oauth2") + private boolean isValidAuthenticationConfig() { + boolean hasBasic = basicConfig != null; + boolean hasOauth = oauth2Config != null; + return hasBasic ^ hasOauth; + } + + public String getAuthType() { + if (basicConfig != null) { + return BASIC; + } else { + return OAUTH2; + } + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/BasicConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/BasicConfig.java new file mode 100644 index 0000000000..b3a261f13a --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/BasicConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.jira.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +@Getter +public class BasicConfig { + @JsonProperty("username") + private String username; + + @JsonProperty("password") + private String password; + + @AssertTrue(message = "Username and Password are both required for Basic Auth") + private boolean isBasicConfigValid() { + return username != null && password != null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/FilterConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/FilterConfig.java new file mode 100644 index 0000000000..987fa545e7 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/FilterConfig.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.jira.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +@Getter +public class FilterConfig { + @JsonProperty("project") + private ProjectConfig projectConfig; + + @JsonProperty("status") + private StatusConfig statusConfig; + + @JsonProperty("issue_type") + private IssueTypeConfig issueTypeConfig; +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/IssueTypeConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/IssueTypeConfig.java new file mode 100644 index 0000000000..977d9a3a9e --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/IssueTypeConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.jira.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class IssueTypeConfig { + @JsonProperty("include") + @Size(max = 1000, message = "Issue type filter should not be more than 1000") + private List include = new ArrayList<>(); + + @JsonProperty("exclude") + @Size(max = 1000, message = "Issue type filter should not be more than 1000") + private List exclude = new ArrayList<>(); +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/NameConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/NameConfig.java new file mode 100644 index 0000000000..3df833d072 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/NameConfig.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + + +package org.opensearch.dataprepper.plugins.source.jira.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class NameConfig { + @JsonProperty("include") + @Size(max = 1000, message = "Project name type filter should not be more than 1000") + private List include = new ArrayList<>(); + + @JsonProperty("exclude") + @Size(max = 1000, message = "Project name type filter should not be more than 1000") + private List exclude = new ArrayList<>(); +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/Oauth2Config.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/Oauth2Config.java new file mode 100644 index 0000000000..3282e7b38f --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/Oauth2Config.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.jira.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.AssertTrue; +import lombok.Getter; + +@Getter +public class Oauth2Config { + @JsonProperty("client_id") + private String clientId; + + @JsonProperty("client_secret") + private String clientSecret; + + @JsonProperty("access_token") + private String accessToken; + + @JsonProperty("refresh_token") + private String refreshToken; + + @AssertTrue(message = "Client ID, Client Secret, Access Token, and Refresh Token are both required for Oauth2") + private boolean isOauth2ConfigValid() { + return clientId != null && clientSecret != null && accessToken != null && refreshToken != null; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/ProjectConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/ProjectConfig.java new file mode 100644 index 0000000000..19bec8ad05 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/ProjectConfig.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.jira.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; + +@Getter +public class ProjectConfig { + @JsonProperty("key") + private NameConfig nameConfig; +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/StatusConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/StatusConfig.java new file mode 100644 index 0000000000..288d138f31 --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/StatusConfig.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.jira.configuration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.constraints.Size; +import lombok.Getter; + +import java.util.ArrayList; +import java.util.List; + +@Getter +public class StatusConfig { + @JsonProperty("include") + @Size(max = 1000, message = "status type filter should not be more than 1000") + private List include = new ArrayList<>(); + + @JsonProperty("exclude") + @Size(max = 1000, message = "status type filter should not be more than 1000") + private List exclude = new ArrayList<>(); +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestException.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestException.java index 589e8626ab..69c0cca638 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestException.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestException.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.exception; /** diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedException.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedException.java index 1efa30e032..441f966bd8 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedException.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedException.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.exception; /** diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBean.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBean.java index ec437ac25d..5f2769883f 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBean.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBean.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.models; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java index 96bc445cdb..677967a340 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResults.java @@ -1,3 +1,14 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + + package org.opensearch.dataprepper.plugins.source.jira.models; import com.fasterxml.jackson.annotation.JsonProperty; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptor.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptor.java index 57dbb65f94..e3a1b992f0 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptor.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptor.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest; import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; @@ -17,8 +27,8 @@ public class BasicAuthInterceptor implements ClientHttpRequestInterceptor { private final String password; public BasicAuthInterceptor(JiraSourceConfig config) { - this.username = config.getJiraId(); - this.password = config.getJiraCredential(); + this.username = config.getAuthenticationConfig().getBasicConfig().getUsername(); + this.password = config.getAuthenticationConfig().getBasicConfig().getPassword();; } @Override diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfig.java index 231351bcee..70cd189267 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfig.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index 6b71a032b8..caf5d84ee7 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest; import com.google.common.annotations.VisibleForTesting; @@ -34,17 +44,20 @@ public class JiraRestClient { public static final String REST_API_SEARCH = "rest/api/3/search"; public static final String REST_API_FETCH_ISSUE = "rest/api/3/issue"; + public static final String REST_API_PROJECTS = "/rest/api/3/project/search"; public static final String FIFTY = "50"; public static final String START_AT = "startAt"; public static final String MAX_RESULT = "maxResults"; public static final List RETRY_ATTEMPT_SLEEP_TIME = List.of(1, 2, 5, 10, 20, 40); private static final String TICKET_FETCH_LATENCY_TIMER = "ticketFetchLatency"; private static final String SEARCH_CALL_LATENCY_TIMER = "searchCallLatency"; + private static final String PROJECTS_FETCH_LATENCY_TIMER = "projectFetchLatency"; private static final String ISSUES_REQUESTED = "issuesRequested"; private final RestTemplate restTemplate; private final JiraAuthConfig authConfig; private final Timer ticketFetchLatencyTimer; private final Timer searchCallLatencyTimer; + private final Timer projectFetchLatencyTimer; private final Counter issuesRequestedCounter; private final PluginMetrics jiraPluginMetrics = PluginMetrics.fromNames("jiraRestClient", "aws"); private int sleepTimeMultiplier = 1000; @@ -55,6 +68,8 @@ public JiraRestClient(RestTemplate restTemplate, JiraAuthConfig authConfig) { ticketFetchLatencyTimer = jiraPluginMetrics.timer(TICKET_FETCH_LATENCY_TIMER); searchCallLatencyTimer = jiraPluginMetrics.timer(SEARCH_CALL_LATENCY_TIMER); + projectFetchLatencyTimer = jiraPluginMetrics.timer(PROJECTS_FETCH_LATENCY_TIMER); + issuesRequestedCounter = jiraPluginMetrics.counter(ISSUES_REQUESTED); } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptor.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptor.java index bf748ceb26..be71df7ac7 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptor.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptor.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest; import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthConfig.java index 274ce8b5d7..cae78da954 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthConfig.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest.auth; /** diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactory.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactory.java index 1ba1d9717f..f178d56812 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactory.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactory.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest.auth; import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfig.java index 233cbf9f49..32509a2229 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfig.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest.auth; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java index f7f4e8493f..791c9e6ff4 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest.auth; import lombok.Getter; @@ -55,10 +65,10 @@ public class JiraOauthConfig implements JiraAuthConfig { public JiraOauthConfig(JiraSourceConfig jiraSourceConfig) { this.jiraSourceConfig = jiraSourceConfig; - this.accessToken = jiraSourceConfig.getAccessToken(); - this.refreshToken = jiraSourceConfig.getRefreshToken(); - this.clientId = jiraSourceConfig.getClientId(); - this.clientSecret = jiraSourceConfig.getClientSecret(); + this.accessToken = jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken(); + this.refreshToken = jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getRefreshToken(); + this.clientId = jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId(); + this.clientSecret = jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret(); } public String getJiraAccountCloudId() { diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java index 0fc1c379f5..d6cc166226 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidation.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.utils; import lombok.NonNull; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/Constants.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/Constants.java index ff6a780bfd..067380def6 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/Constants.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/Constants.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.utils; /** diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraConfigHelper.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraConfigHelper.java index 07b1e1a213..dd815bd607 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraConfigHelper.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraConfigHelper.java @@ -1,9 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.utils; import lombok.extern.slf4j.Slf4j; import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import java.util.ArrayList; import java.util.List; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; @@ -14,18 +25,23 @@ */ @Slf4j public class JiraConfigHelper { - - public static final String ISSUE_STATUS_FILTER = "status"; - public static final String ISSUE_TYPE_FILTER = "issuetype"; - - /** * Get Issue Status Filter from repository configuration. * * @return List Issue Status Filter. */ - public static List getIssueStatusFilter(JiraSourceConfig repositoryConfiguration) { - return repositoryConfiguration.getStatus(); + public static List getIssueStatusIncludeFilter(JiraSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getStatusConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getStatusConfig().getInclude(); + } + + public static List getIssueStatusExcludeFilter(JiraSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getStatusConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getStatusConfig().getExclude(); } /** @@ -33,8 +49,18 @@ public static List getIssueStatusFilter(JiraSourceConfig repositoryConfi * * @return List Issue Type Filter. */ - public static List getIssueTypeFilter(JiraSourceConfig repositoryConfiguration) { - return repositoryConfiguration.getIssueType(); + public static List getIssueTypeIncludeFilter(JiraSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getIssueTypeConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getIssueTypeConfig().getInclude(); + } + + public static List getIssueTypeExcludeFilter(JiraSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || repositoryConfiguration.getFilterConfig().getIssueTypeConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getIssueTypeConfig().getExclude(); } /** @@ -43,8 +69,22 @@ public static List getIssueTypeFilter(JiraSourceConfig repositoryConfigu * * @return List Project Filter. */ - public static List getProjectKeyFilter(JiraSourceConfig repositoryConfiguration) { - return repositoryConfiguration.getProject(); + public static List getProjectNameIncludeFilter(JiraSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || + repositoryConfiguration.getFilterConfig().getProjectConfig() == null || + repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig().getInclude(); + } + + public static List getProjectNameExcludeFilter(JiraSourceConfig repositoryConfiguration) { + if (repositoryConfiguration.getFilterConfig() == null || + repositoryConfiguration.getFilterConfig().getProjectConfig() == null || + repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig() == null) { + return new ArrayList<>(); + } + return repositoryConfiguration.getFilterConfig().getProjectConfig().getNameConfig().getExclude(); } @@ -62,13 +102,13 @@ public static boolean validateConfig(JiraSourceConfig config) { } if (BASIC.equals(authType)) { - if (config.getJiraId() == null || config.getJiraCredential() == null) { + if (config.getAuthenticationConfig().getBasicConfig().getUsername() == null || config.getAuthenticationConfig().getBasicConfig().getPassword() == null) { throw new RuntimeException("Jira ID or Credential are required for Basic AuthType"); } } if (OAUTH2.equals(authType)) { - if (config.getAccessToken() == null || config.getRefreshToken() == null) { + if (config.getAuthenticationConfig().getOauth2Config().getAccessToken() == null || config.getAuthenticationConfig().getOauth2Config().getRefreshToken() == null) { throw new RuntimeException("Access Token or Refresh Token are required for OAuth2 AuthType"); } } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentType.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentType.java index 9d37a6e8ca..3f43c0e01e 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentType.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentType.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.utils; import lombok.AllArgsConstructor; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java index 5b88208f74..99474024e1 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/utils/JqlConstants.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.utils; public class JqlConstants { @@ -7,10 +17,13 @@ public class JqlConstants { public static final String SLASH = "/"; public static final String PROJECT_IN = " AND project in ("; public static final String STATUS_IN = " AND status in ("; + public static final String PROJECT_NOT_IN = " AND project not in ("; + public static final String STATUS_NOT_IN = " AND status not in ("; public static final String DELIMITER = "\",\""; public static final String PREFIX = "\""; public static final String SUFFIX = "\""; public static final String ISSUE_TYPE_IN = " AND issueType in ("; + public static final String ISSUE_TYPE_NOT_IN = " AND issueType not in ("; public static final String JQL_FIELD = "jql"; public static final String EXPAND_FIELD = "expand"; public static final String EXPAND_VALUE = "all"; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraClientTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraClientTest.java index 4720ffc89b..78531afd61 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraClientTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import com.fasterxml.jackson.core.JsonProcessingException; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java index ac26864a25..eeb9c03941 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java @@ -1,9 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.FilterConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.IssueTypeConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.NameConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.jira.configuration.ProjectConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.StatusConfig; import org.opensearch.dataprepper.plugins.source.jira.utils.JiraConfigHelper; import java.util.List; @@ -23,6 +41,30 @@ public class JiraConfigHelperTest { @Mock JiraSourceConfig jiraSourceConfig; + @Mock + FilterConfig filterConfig; + + @Mock + StatusConfig statusConfig; + + @Mock + IssueTypeConfig issueTypeConfig; + + @Mock + ProjectConfig projectConfig; + + @Mock + NameConfig nameConfig; + + @Mock + AuthenticationConfig authenticationConfig; + + @Mock + BasicConfig basicConfig; + + @Mock + Oauth2Config oauth2Config; + @Test void testInitialization() { JiraConfigHelper jiraConfigHelper = new JiraConfigHelper(); @@ -31,28 +73,48 @@ void testInitialization() { @Test void testGetIssueStatusFilter() { - assertTrue(JiraConfigHelper.getIssueStatusFilter(jiraSourceConfig).isEmpty()); + when(jiraSourceConfig.getFilterConfig()).thenReturn(filterConfig); + when(filterConfig.getStatusConfig()).thenReturn(statusConfig); + assertTrue(JiraConfigHelper.getIssueStatusIncludeFilter(jiraSourceConfig).isEmpty()); + assertTrue(JiraConfigHelper.getIssueStatusExcludeFilter(jiraSourceConfig).isEmpty()); List issueStatusFilter = List.of("Done", "In Progress"); - when(jiraSourceConfig.getProject()).thenReturn(issueStatusFilter); - assertEquals(issueStatusFilter, JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig)); + List issueStatusExcludeFilter = List.of("Done2", "In Progress2"); + when(statusConfig.getInclude()).thenReturn(issueStatusFilter); + when(statusConfig.getExclude()).thenReturn(issueStatusExcludeFilter); + assertEquals(issueStatusFilter, JiraConfigHelper.getIssueStatusIncludeFilter(jiraSourceConfig)); + assertEquals(issueStatusExcludeFilter, JiraConfigHelper.getIssueStatusExcludeFilter(jiraSourceConfig)); } @Test void testGetIssueTypeFilter() { - assertTrue(JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig).isEmpty()); + when(jiraSourceConfig.getFilterConfig()).thenReturn(filterConfig); + when(filterConfig.getIssueTypeConfig()).thenReturn(issueTypeConfig); + assertTrue(JiraConfigHelper.getIssueTypeIncludeFilter(jiraSourceConfig).isEmpty()); + assertTrue(JiraConfigHelper.getIssueTypeExcludeFilter(jiraSourceConfig).isEmpty()); List issueTypeFilter = List.of("Bug", "Story"); - when(jiraSourceConfig.getProject()).thenReturn(issueTypeFilter); - assertEquals(issueTypeFilter, JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig)); + List issueTypeExcludeFilter = List.of("Bug2", "Story2"); + when(issueTypeConfig.getInclude()).thenReturn(issueTypeFilter); + when(issueTypeConfig.getExclude()).thenReturn(issueTypeExcludeFilter); + assertEquals(issueTypeFilter, JiraConfigHelper.getIssueTypeIncludeFilter(jiraSourceConfig)); + assertEquals(issueTypeExcludeFilter, JiraConfigHelper.getIssueTypeExcludeFilter(jiraSourceConfig)); } @Test - void testGetProjectKeyFilter() { - assertTrue(JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig).isEmpty()); - List projectKeyFilter = List.of("TEST", "TEST2"); - when(jiraSourceConfig.getProject()).thenReturn(projectKeyFilter); - assertEquals(projectKeyFilter, JiraConfigHelper.getProjectKeyFilter(jiraSourceConfig)); + void testGetProjectNameFilter() { + when(jiraSourceConfig.getFilterConfig()).thenReturn(filterConfig); + when(filterConfig.getProjectConfig()).thenReturn(projectConfig); + when(projectConfig.getNameConfig()).thenReturn(nameConfig); + assertTrue(JiraConfigHelper.getProjectNameIncludeFilter(jiraSourceConfig).isEmpty()); + assertTrue(JiraConfigHelper.getProjectNameExcludeFilter(jiraSourceConfig).isEmpty()); + List projectNameFilter = List.of("TEST", "TEST2"); + List projectNameExcludeFilter = List.of("TEST3", "TEST4"); + when(nameConfig.getInclude()).thenReturn(projectNameFilter); + when(nameConfig.getExclude()).thenReturn(projectNameExcludeFilter); + assertEquals(projectNameFilter, JiraConfigHelper.getProjectNameIncludeFilter(jiraSourceConfig)); + assertEquals(projectNameExcludeFilter, JiraConfigHelper.getProjectNameExcludeFilter(jiraSourceConfig)); } + @Test void testValidateConfig() { assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); @@ -68,16 +130,18 @@ void testValidateConfig() { void testValidateConfigBasic() { when(jiraSourceConfig.getAccountUrl()).thenReturn("https://test.com"); when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); + when(jiraSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(jiraSourceConfig.getJiraId()).thenReturn("id"); + when(basicConfig.getUsername()).thenReturn("id"); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(jiraSourceConfig.getJiraCredential()).thenReturn("credential"); - when(jiraSourceConfig.getJiraId()).thenReturn(null); + when(basicConfig.getPassword()).thenReturn("credential"); + when(basicConfig.getUsername()).thenReturn(null); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(jiraSourceConfig.getJiraId()).thenReturn("id"); + when(basicConfig.getUsername()).thenReturn("id"); assertDoesNotThrow(() -> JiraConfigHelper.validateConfig(jiraSourceConfig)); } @@ -85,16 +149,18 @@ void testValidateConfigBasic() { void testValidateConfigOauth2() { when(jiraSourceConfig.getAccountUrl()).thenReturn("https://test.com"); when(jiraSourceConfig.getAuthType()).thenReturn(OAUTH2); + when(jiraSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(jiraSourceConfig.getAccessToken()).thenReturn("id"); + when(oauth2Config.getAccessToken()).thenReturn("id"); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(jiraSourceConfig.getRefreshToken()).thenReturn("credential"); - when(jiraSourceConfig.getAccessToken()).thenReturn(null); + when(oauth2Config.getRefreshToken()).thenReturn("credential"); + when(oauth2Config.getAccessToken()).thenReturn(null); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(jiraSourceConfig.getAccessToken()).thenReturn("id"); + when(oauth2Config.getAccessToken()).thenReturn("id"); assertDoesNotThrow(() -> JiraConfigHelper.validateConfig(jiraSourceConfig)); } } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java index 49dc6873bd..f5e6b0906b 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraItemInfoTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import org.junit.jupiter.api.BeforeEach; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java index 8d9dc85869..73e184330a 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraIteratorTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import org.junit.jupiter.api.BeforeEach; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java index af07ab6f0d..0de0ea47dc 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import com.fasterxml.jackson.core.JsonProcessingException; @@ -45,6 +55,7 @@ import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.CREATED; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.KEY; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.NAME; +import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.PROJECT; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.UPDATED; @@ -85,16 +96,45 @@ public static JiraSourceConfig createJiraConfiguration(String auth_type, List issueStatus, List projectKey) throws JsonProcessingException { ObjectMapper objectMapper = new ObjectMapper(); - Map connectorCredentialsMap = new HashMap<>(); - connectorCredentialsMap.put("auth_type", auth_type); + Map authenticationMap = new HashMap<>(); + Map basicMap = new HashMap<>(); + Map oauth2Map = new HashMap<>(); + if (auth_type.equals(BASIC)) { + basicMap.put("username", "test_username"); + basicMap.put("password", "test_password"); + authenticationMap.put("basic", basicMap); + } else if (auth_type.equals(OAUTH2)) { + oauth2Map.put("client_id", "test-client-id"); + oauth2Map.put("client_secret", "test-client-secret"); + oauth2Map.put("access_token", "test-access-token"); + oauth2Map.put("refresh_token", "test-refresh-token"); + authenticationMap.put("oauth2", oauth2Map); + } Map jiraSourceConfigMap = new HashMap<>(); - jiraSourceConfigMap.put("account_url", ACCESSIBLE_RESOURCES); - jiraSourceConfigMap.put("connector_credentials", connectorCredentialsMap); - jiraSourceConfigMap.put("issue_types", issueType); - jiraSourceConfigMap.put("statuses", issueStatus); - jiraSourceConfigMap.put("projects", projectKey); + List hosts = new ArrayList<>(); + hosts.add(ACCESSIBLE_RESOURCES); + + Map filterMap = new HashMap<>(); + Map projectMap = new HashMap<>(); + Map issueTypeMap = new HashMap<>(); + Map statusMap = new HashMap<>(); + + issueTypeMap.put("include", issueType); + filterMap.put("issue_type", issueTypeMap); + + statusMap.put("include", issueStatus); + filterMap.put("status", statusMap); + + Map nameMap = new HashMap<>(); + nameMap.put("include", projectKey); + projectMap.put("key", nameMap); + filterMap.put("project", projectMap); + + jiraSourceConfigMap.put("hosts", hosts); + jiraSourceConfigMap.put("authentication", authenticationMap); + jiraSourceConfigMap.put("filter", filterMap); String jiraSourceConfigJsonString = objectMapper.writeValueAsString(jiraSourceConfigMap); return objectMapper.readValue(jiraSourceConfigJsonString, JiraSourceConfig.class); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java index 9da7011d0d..35a2450fdb 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import com.fasterxml.jackson.core.JsonProcessingException; @@ -21,54 +31,69 @@ public class JiraSourceConfigTest { private final String refreshToken = "refresh token test"; private final String clientId = "client id test"; private final String clientSecret = "client secret test"; - private final String jiraCredential = "test Jira Credential"; - private final String jiraId = "test Jira Id"; + private final String password = "test Jira Credential"; + private final String username = "test Jira Id"; private final String accountUrl = "https://example.atlassian.net"; private List projectList = new ArrayList<>(); private List issueTypeList = new ArrayList<>(); - private List inclusionPatternList = new ArrayList<>(); - private List exclusionPatternList = new ArrayList<>(); private List statusList = new ArrayList<>(); - private Map connectorCredentialMap = new HashMap<>(); private JiraSourceConfig jiraSourceConfig; private JiraSourceConfig createJiraSourceConfig(String authtype, boolean hasToken) throws JsonProcessingException { Map configMap = new HashMap<>(); - configMap.put("account_url", accountUrl); - - connectorCredentialMap.put("auth_type", authtype); - if (hasToken) { - connectorCredentialMap.put("access_token", accessToken); - connectorCredentialMap.put("refresh_token", refreshToken); - } else { - connectorCredentialMap.put("refresh_token", ""); + List hosts = new ArrayList<>(); + hosts.add(accountUrl); + + configMap.put("hosts", hosts); + + Map authenticationMap = new HashMap<>(); + Map basicMap = new HashMap<>(); + Map oauth2Map = new HashMap<>(); + if (authtype.equals(BASIC)) { + basicMap.put("username", username); + basicMap.put("password", password); + authenticationMap.put("basic", basicMap); + } else if (authtype.equals(OAUTH2)) { + if (hasToken) { + oauth2Map.put("access_token", accessToken); + oauth2Map.put("refresh_token", refreshToken); + } else { + oauth2Map.put("access_token", null); + oauth2Map.put("refresh_token", null); + } + oauth2Map.put("client_id", clientId); + oauth2Map.put("client_secret", clientSecret); + authenticationMap.put("oauth2", oauth2Map); } - connectorCredentialMap.put("jira_id", jiraId); - connectorCredentialMap.put("jira_credential", jiraCredential); - connectorCredentialMap.put("client_id", clientId); - connectorCredentialMap.put("client_secret", clientSecret); - configMap.put("connector_credentials", connectorCredentialMap); + configMap.put("authentication", authenticationMap); projectList.add("project1"); projectList.add("project2"); - configMap.put("projects", projectList); issueTypeList.add("issue type 1"); issueTypeList.add("issue type 2"); - configMap.put("issue_types", issueTypeList); - - inclusionPatternList.add("pattern 1"); - inclusionPatternList.add("pattern 2"); - configMap.put("inclusion_patterns", inclusionPatternList); - - exclusionPatternList.add("pattern 3"); - exclusionPatternList.add("pattern 4"); - configMap.put("exclusion_patterns", exclusionPatternList); statusList.add("status 1"); statusList.add("status 2"); - configMap.put("statuses", statusList); + + Map filterMap = new HashMap<>(); + Map projectMap = new HashMap<>(); + Map issueTypeMap = new HashMap<>(); + Map statusMap = new HashMap<>(); + + issueTypeMap.put("include", issueTypeList); + filterMap.put("issue_type", issueTypeMap); + + statusMap.put("include", statusList); + filterMap.put("status", statusMap); + + Map nameMap = new HashMap<>(); + nameMap.put("include", projectList); + projectMap.put("key", nameMap); + filterMap.put("project", projectMap); + + configMap.put("filter", filterMap); ObjectMapper objectMapper = new ObjectMapper(); String jsonConfig = objectMapper.writeValueAsString(configMap); @@ -79,40 +104,29 @@ private JiraSourceConfig createJiraSourceConfig(String authtype, boolean hasToke @Test void testGetters() throws JsonProcessingException { jiraSourceConfig = createJiraSourceConfig(BASIC, false); - assertEquals(jiraSourceConfig.getInclusionPatterns(), inclusionPatternList); - assertEquals(jiraSourceConfig.getIssueType(), issueTypeList); - assertEquals(jiraSourceConfig.getExclusionPatterns(), exclusionPatternList); + assertEquals(jiraSourceConfig.getFilterConfig().getIssueTypeConfig().getInclude(), issueTypeList); assertEquals(jiraSourceConfig.getNumWorkers(), DEFAULT_NUMBER_OF_WORKERS); - assertEquals(jiraSourceConfig.getProject(), projectList); - assertEquals(jiraSourceConfig.getStatus(), statusList); - assertEquals(jiraSourceConfig.getConnectorCredentials(), connectorCredentialMap); + assertEquals(jiraSourceConfig.getFilterConfig().getProjectConfig().getNameConfig().getInclude(), projectList); + assertEquals(jiraSourceConfig.getFilterConfig().getStatusConfig().getInclude(), statusList); assertEquals(jiraSourceConfig.getAccountUrl(), accountUrl); assertNotNull(jiraSourceConfig.getBackOff()); - assertEquals(jiraSourceConfig.getJiraCredential(), jiraCredential); - assertEquals(jiraSourceConfig.getJiraId(), jiraId); + assertEquals(jiraSourceConfig.getAuthenticationConfig().getBasicConfig().getPassword(), password); + assertEquals(jiraSourceConfig.getAuthenticationConfig().getBasicConfig().getUsername(), username); } @Test void testFetchGivenOauthAttributeWrongAuthType() throws JsonProcessingException { jiraSourceConfig = createJiraSourceConfig(BASIC, true); - assertThrows(RuntimeException.class, () -> jiraSourceConfig.getAccessToken()); + assertThrows(RuntimeException.class, () -> jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); } @Test void testFetchGivenOauthAtrribute() throws JsonProcessingException { jiraSourceConfig = createJiraSourceConfig(OAUTH2, true); - assertEquals(accessToken, jiraSourceConfig.getAccessToken()); - assertEquals(refreshToken, jiraSourceConfig.getRefreshToken()); - assertEquals(clientId, jiraSourceConfig.getClientId()); - assertEquals(clientSecret, jiraSourceConfig.getClientSecret()); - } - - @Test - void testFetchGivenOauthAtrributeMissing() throws JsonProcessingException { - jiraSourceConfig = createJiraSourceConfig(OAUTH2, false); - assertThrows(RuntimeException.class, () -> jiraSourceConfig.getAccessToken()); - assertThrows(RuntimeException.class, () -> jiraSourceConfig.getRefreshToken()); - + assertEquals(accessToken, jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); + assertEquals(refreshToken, jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getRefreshToken()); + assertEquals(clientId, jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId()); + assertEquals(clientSecret, jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret()); } } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java index 46cf58b7a9..725c91b66e 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira; import org.junit.jupiter.api.Test; @@ -11,6 +21,8 @@ import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; +import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.BasicConfig; import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; import org.opensearch.dataprepper.plugins.source.source_crawler.base.Crawler; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; @@ -58,7 +70,12 @@ public class JiraSourceTest { @Mock private ExecutorService executorService; -// = new PluginExecutorServiceProvider(); + + @Mock + AuthenticationConfig authenticationConfig; + + @Mock + BasicConfig basicConfig; @Test void initialization() { @@ -73,8 +90,10 @@ void testStart() { JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); when(jiraSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); - when(jiraSourceConfig.getJiraId()).thenReturn("Test Id"); - when(jiraSourceConfig.getJiraCredential()).thenReturn("Test Credential"); + when(jiraSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + when(basicConfig.getUsername()).thenReturn("Test Id"); + when(basicConfig.getPassword()).thenReturn("Test Credential"); source.setEnhancedSourceCoordinator(sourceCooridinator); source.start(buffer); @@ -87,8 +106,10 @@ void testStop() { JiraSource source = new JiraSource(pluginMetrics, jiraSourceConfig, jiraOauthConfig, pluginFactory, acknowledgementSetManager, crawler, executorServiceProvider); when(jiraSourceConfig.getAccountUrl()).thenReturn(ACCESSIBLE_RESOURCES); when(jiraSourceConfig.getAuthType()).thenReturn(BASIC); - when(jiraSourceConfig.getJiraId()).thenReturn("Test Id"); - when(jiraSourceConfig.getJiraCredential()).thenReturn("Test Credential"); + when(jiraSourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + when(basicConfig.getUsername()).thenReturn("Test Id"); + when(basicConfig.getPassword()).thenReturn("Test Credential"); source.setEnhancedSourceCoordinator(sourceCooridinator); source.start(buffer); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestExceptionTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestExceptionTest.java index 74f5873af2..91578ec810 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestExceptionTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/BadRequestExceptionTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.exception; import org.junit.jupiter.api.BeforeEach; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedExceptionTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedExceptionTest.java index ecedff2d60..87c06051e1 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedExceptionTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/exception/UnAuthorizedExceptionTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.exception; import org.junit.jupiter.api.BeforeEach; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBeanTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBeanTest.java index 0f99ddccbf..812c59c1f7 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBeanTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/IssueBeanTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.models; import org.junit.jupiter.api.BeforeEach; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java index 3a569ca39d..bd0d54e2bf 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/models/SearchResultsTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.models; import com.fasterxml.jackson.core.JsonProcessingException; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptorTest.java index 18c94cb426..a57b879d6d 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptorTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/BasicAuthInterceptorTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest; import org.junit.jupiter.api.BeforeEach; @@ -6,6 +16,8 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.BasicConfig; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpRequest; import org.springframework.http.client.ClientHttpRequestExecution; @@ -36,6 +48,12 @@ public class BasicAuthInterceptorTest { @Mock private JiraSourceConfig mockConfig; + @Mock + AuthenticationConfig authenticationConfig; + + @Mock + BasicConfig basicConfig; + @Mock private HttpHeaders mockHeaders; @@ -43,8 +61,10 @@ public class BasicAuthInterceptorTest { @BeforeEach void setUp() { - when(mockConfig.getJiraId()).thenReturn("testUser"); - when(mockConfig.getJiraCredential()).thenReturn("testPassword"); + when(mockConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getBasicConfig()).thenReturn(basicConfig); + when(basicConfig.getUsername()).thenReturn("testUser"); + when(basicConfig.getPassword()).thenReturn("testPassword"); when(mockRequest.getHeaders()).thenReturn(mockHeaders); interceptor = new BasicAuthInterceptor(mockConfig); } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java index ce0675f22a..11d29c78c0 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest; import org.junit.jupiter.api.BeforeEach; @@ -8,6 +18,9 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.BasicConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.jira.rest.auth.JiraAuthConfig; import org.springframework.http.client.ClientHttpRequestInterceptor; import org.springframework.http.client.InterceptingClientHttpRequestFactory; @@ -19,6 +32,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.BASIC; import static org.opensearch.dataprepper.plugins.source.jira.utils.Constants.OAUTH2; @@ -34,6 +48,15 @@ class CustomRestTemplateConfigTest { @Mock private JiraAuthConfig mockAuthConfig; + @Mock + private BasicConfig mockBasicConfig; + + @Mock + private Oauth2Config mockOauth2Config; + + @Mock + private AuthenticationConfig mockAuthenticationConfig; + private static Stream provideAuthTypeAndExpectedInterceptorType() { return Stream.of( Arguments.of(OAUTH2, OAuth2RequestInterceptor.class), @@ -52,6 +75,16 @@ void setUp() { @MethodSource("provideAuthTypeAndExpectedInterceptorType") void testBasicAuthRestTemplateWithOAuth2(String authType, Class interceptorClassType) { when(mockSourceConfig.getAuthType()).thenReturn(authType); + lenient().when(mockSourceConfig.getAuthenticationConfig()).thenReturn(mockAuthenticationConfig); + lenient().when(mockAuthenticationConfig.getOauth2Config()).thenReturn(mockOauth2Config); + lenient().when(mockOauth2Config.getAccessToken()).thenReturn("accessToken"); + lenient().when(mockOauth2Config.getRefreshToken()).thenReturn("refreshToken"); + lenient().when(mockOauth2Config.getClientId()).thenReturn("clientId"); + lenient().when(mockOauth2Config.getClientSecret()).thenReturn("clientSecret"); + lenient().when(mockAuthenticationConfig.getBasicConfig()).thenReturn(mockBasicConfig); + lenient().when(mockBasicConfig.getUsername()).thenReturn("username"); + lenient().when(mockBasicConfig.getPassword()).thenReturn("password"); + RestTemplate restTemplate = config.basicAuthRestTemplate(mockSourceConfig, mockAuthConfig); assertNotNull(restTemplate); assertInstanceOf(InterceptingClientHttpRequestFactory.class, restTemplate.getRequestFactory()); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java index d294b4f599..3fe0640700 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClientTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest; import com.fasterxml.jackson.core.JsonProcessingException; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptorTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptorTest.java index 277c42ca2e..01e850fec9 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptorTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/OAuth2RequestInterceptorTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest; import org.junit.jupiter.api.BeforeEach; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java index 566b3ae8ed..5106bd8ad1 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest.auth; import org.junit.jupiter.api.BeforeEach; @@ -6,6 +16,8 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; +import org.opensearch.dataprepper.plugins.source.jira.configuration.Oauth2Config; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertInstanceOf; @@ -18,6 +30,12 @@ public class JiraAuthFactoryTest { @Mock private JiraSourceConfig sourceConfig; + @Mock + private AuthenticationConfig authenticationConfig; + + @Mock + private Oauth2Config oauth2Config; + private JiraAuthFactory jiraAuthFactory; @BeforeEach @@ -28,6 +46,8 @@ void setUp() { @Test void testGetObjectOauth2() { when(sourceConfig.getAuthType()).thenReturn(OAUTH2); + when(sourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); + when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); assertInstanceOf(JiraOauthConfig.class, jiraAuthFactory.getObject()); } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfigTest.java index 60dad25507..8ab10eb0bc 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraBasicAuthConfigTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest.auth; import org.junit.jupiter.api.BeforeEach; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfigTest.java index 1b648ab404..bafe995801 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfigTest.java @@ -1,5 +1,16 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.rest.auth; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; @@ -38,7 +49,13 @@ public class JiraOauthConfigTest { @Mock RestTemplate restTemplateMock; - JiraSourceConfig jiraSourceConfig = createJiraConfigurationFromYaml("oauth2-auth-jira-pipeline.yaml"); + + JiraSourceConfig jiraSourceConfig; + + @BeforeEach + void setUp() { + jiraSourceConfig = createJiraConfigurationFromYaml("oauth2-auth-jira-pipeline.yaml"); + } @Test void testRenewToken() throws InterruptedException { diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidationTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidationTest.java index b6f56a11ee..3c346c8e0e 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidationTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/AddressValidationTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.utils; import org.junit.jupiter.api.Test; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentTypeTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentTypeTest.java index c342e9a3cd..3437ab9319 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentTypeTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/JiraContentTypeTest.java @@ -1,3 +1,13 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + package org.opensearch.dataprepper.plugins.source.jira.utils; import org.junit.jupiter.api.Test; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/basic-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/basic-auth-jira-pipeline.yaml index 09b15d40d6..0bfa6384e8 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/basic-auth-jira-pipeline.yaml +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/basic-auth-jira-pipeline.yaml @@ -1,6 +1,6 @@ -account_url: "https://jira.com/" -connector_credentials: - auth_type: "Basic" - jira_id: "jiraId" - jira_credential: "jiraApiKey" +hosts: ["https://jira.com/"] +authentication: + basic: + username: "jiraId" + password: "jiraApiKey" diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml index ae1f0b508d..09c9e9f2c5 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml @@ -1,9 +1,8 @@ -account_url: "https://jira.com/" -connector_credentials: - auth_type: "OAuth2" - jira_id: "jira_id" - client_id: "client_id" - client_secret: "client_secret" - access_token: "access_token" - refresh_token: "refresh_token" +hosts: ["https://jira.com/"] +authentication: + oauth2: + client_id: "client_id" + client_secret: "client_secret" + access_token: "access_token" + refresh_token: "refresh_token" From 70899e7e284ada1fb8c1c27f28f750b628e1771f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:27:39 -0800 Subject: [PATCH 03/36] Bump org.apache.commons:commons-text (#5303) Bumps org.apache.commons:commons-text from 1.11.0 to 1.13.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-text dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/parquet-codecs/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/parquet-codecs/build.gradle b/data-prepper-plugins/parquet-codecs/build.gradle index c402fb6741..4c3b252fb9 100644 --- a/data-prepper-plugins/parquet-codecs/build.gradle +++ b/data-prepper-plugins/parquet-codecs/build.gradle @@ -7,7 +7,7 @@ dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:common') implementation libs.avro.core - implementation 'org.apache.commons:commons-text:1.11.0' + implementation 'org.apache.commons:commons-text:1.13.0' implementation libs.parquet.avro implementation libs.parquet.column implementation libs.parquet.common From 1bcbfe9b1e009ee0b1c5db42f8c9995f76838823 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:28:18 -0800 Subject: [PATCH 04/36] Bump org.assertj:assertj-core (#5302) Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.25.3 to 3.27.0. - [Release notes](https://github.com/assertj/assertj/releases) - [Commits](https://github.com/assertj/assertj/compare/assertj-build-3.25.3...assertj-build-3.27.0) --- updated-dependencies: - dependency-name: org.assertj:assertj-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/otel-metrics-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-metrics-source/build.gradle b/data-prepper-plugins/otel-metrics-source/build.gradle index 96d250d67d..6c5a95d795 100644 --- a/data-prepper-plugins/otel-metrics-source/build.gradle +++ b/data-prepper-plugins/otel-metrics-source/build.gradle @@ -30,7 +30,7 @@ dependencies { implementation libs.commons.lang3 implementation libs.bouncycastle.bcprov implementation libs.bouncycastle.bcpkix - testImplementation 'org.assertj:assertj-core:3.25.3' + testImplementation 'org.assertj:assertj-core:3.27.0' testImplementation libs.commons.io } From 8608da1dc72a06384977c44203af7b22b803267e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:28:47 -0800 Subject: [PATCH 05/36] Bump org.hibernate.validator:hibernate-validator (#5301) Bumps [org.hibernate.validator:hibernate-validator](https://github.com/hibernate/hibernate-validator) from 8.0.1.Final to 8.0.2.Final. - [Changelog](https://github.com/hibernate/hibernate-validator/blob/8.0.2.Final/changelog.txt) - [Commits](https://github.com/hibernate/hibernate-validator/compare/8.0.1.Final...8.0.2.Final) --- updated-dependencies: - dependency-name: org.hibernate.validator:hibernate-validator dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/aws-plugin/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/aws-plugin/build.gradle b/data-prepper-plugins/aws-plugin/build.gradle index 710ce5b7b4..6915278023 100644 --- a/data-prepper-plugins/aws-plugin/build.gradle +++ b/data-prepper-plugins/aws-plugin/build.gradle @@ -8,7 +8,7 @@ dependencies { implementation 'software.amazon.awssdk:secretsmanager' implementation 'software.amazon.awssdk:sts' implementation 'software.amazon.awssdk:arns' - implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final' + implementation 'org.hibernate.validator:hibernate-validator:8.0.2.Final' testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' } From 8720c4acf2409844077ef79eb4369f68e535e497 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:29:10 -0800 Subject: [PATCH 06/36] Bump org.hibernate.validator:hibernate-validator (#5300) Bumps [org.hibernate.validator:hibernate-validator](https://github.com/hibernate/hibernate-validator) from 8.0.1.Final to 8.0.2.Final. - [Changelog](https://github.com/hibernate/hibernate-validator/blob/8.0.2.Final/changelog.txt) - [Commits](https://github.com/hibernate/hibernate-validator/compare/8.0.1.Final...8.0.2.Final) --- updated-dependencies: - dependency-name: org.hibernate.validator:hibernate-validator dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/aws-plugin-api/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/aws-plugin-api/build.gradle b/data-prepper-plugins/aws-plugin-api/build.gradle index 9383c8c9f7..1042ffadc9 100644 --- a/data-prepper-plugins/aws-plugin-api/build.gradle +++ b/data-prepper-plugins/aws-plugin-api/build.gradle @@ -3,7 +3,7 @@ dependencies { implementation 'software.amazon.awssdk:auth' implementation 'software.amazon.awssdk:apache-client' implementation 'org.apache.httpcomponents.client5:httpclient5:5.3.1' - testImplementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final' + testImplementation 'org.hibernate.validator:hibernate-validator:8.0.2.Final' } test { From f264c09967d58b85054cc24f73f49cba744f59e4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:31:10 -0800 Subject: [PATCH 07/36] Bump org.assertj:assertj-core in /data-prepper-plugins/http-source (#5298) Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.25.3 to 3.27.0. - [Release notes](https://github.com/assertj/assertj/releases) - [Commits](https://github.com/assertj/assertj/compare/assertj-build-3.25.3...assertj-build-3.27.0) --- updated-dependencies: - dependency-name: org.assertj:assertj-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/http-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/http-source/build.gradle b/data-prepper-plugins/http-source/build.gradle index 2d5c5ceceb..fcfb4ad67f 100644 --- a/data-prepper-plugins/http-source/build.gradle +++ b/data-prepper-plugins/http-source/build.gradle @@ -19,7 +19,7 @@ dependencies { implementation 'software.amazon.awssdk:acm' implementation 'software.amazon.awssdk:s3' implementation 'software.amazon.awssdk:apache-client' - testImplementation 'org.assertj:assertj-core:3.25.3' + testImplementation 'org.assertj:assertj-core:3.27.0' testImplementation project(':data-prepper-api').sourceSets.test.output testImplementation project(':data-prepper-test-common') } From 055b2ed7742f469ded6d6e4c4b78187e6b713ee2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:44:59 -0800 Subject: [PATCH 08/36] Bump werkzeug in /examples/trace-analytics-sample-app/sample-app (#5115) Bumps [werkzeug](https://github.com/pallets/werkzeug) from 3.0.3 to 3.0.6. - [Release notes](https://github.com/pallets/werkzeug/releases) - [Changelog](https://github.com/pallets/werkzeug/blob/main/CHANGES.rst) - [Commits](https://github.com/pallets/werkzeug/compare/3.0.3...3.0.6) --- updated-dependencies: - dependency-name: werkzeug dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> From c449eab87b5317ee28547277276addf9e18e082f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:45:29 -0800 Subject: [PATCH 09/36] Bump joda-time:joda-time in /data-prepper-plugins/s3-source (#5006) Bumps [joda-time:joda-time](https://github.com/JodaOrg/joda-time) from 2.12.7 to 2.13.0. - [Release notes](https://github.com/JodaOrg/joda-time/releases) - [Changelog](https://github.com/JodaOrg/joda-time/blob/main/RELEASE-NOTES.txt) - [Commits](https://github.com/JodaOrg/joda-time/compare/v2.12.7...v2.13.0) --- updated-dependencies: - dependency-name: joda-time:joda-time dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/s3-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/s3-source/build.gradle b/data-prepper-plugins/s3-source/build.gradle index 1a3f213237..a11c34b49c 100644 --- a/data-prepper-plugins/s3-source/build.gradle +++ b/data-prepper-plugins/s3-source/build.gradle @@ -22,7 +22,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation libs.commons.io implementation libs.commons.compress - implementation 'joda-time:joda-time:2.12.7' + implementation 'joda-time:joda-time:2.13.0' implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' From 9855522c005fd08478561fc953ec941781b59e3e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:45:42 -0800 Subject: [PATCH 10/36] Bump joda-time:joda-time in /data-prepper-plugins/s3-sink (#5001) Bumps [joda-time:joda-time](https://github.com/JodaOrg/joda-time) from 2.12.7 to 2.13.0. - [Release notes](https://github.com/JodaOrg/joda-time/releases) - [Changelog](https://github.com/JodaOrg/joda-time/blob/main/RELEASE-NOTES.txt) - [Commits](https://github.com/JodaOrg/joda-time/compare/v2.12.7...v2.13.0) --- updated-dependencies: - dependency-name: joda-time:joda-time dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/s3-sink/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/s3-sink/build.gradle b/data-prepper-plugins/s3-sink/build.gradle index 8dd6b5783d..e298b6cbd7 100644 --- a/data-prepper-plugins/s3-sink/build.gradle +++ b/data-prepper-plugins/s3-sink/build.gradle @@ -11,7 +11,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation libs.commons.compress - implementation 'joda-time:joda-time:2.12.7' + implementation 'joda-time:joda-time:2.13.0' implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-csv' implementation 'software.amazon.awssdk:netty-nio-client' From eabeae6ef6ed97cea1bd75dc6b9b1df89bf871bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:47:11 -0800 Subject: [PATCH 11/36] Bump org.apache.curator:curator-test (#5163) Bumps [org.apache.curator:curator-test](https://github.com/apache/curator) from 5.5.0 to 5.7.1. - [Commits](https://github.com/apache/curator/compare/apache-curator-5.5.0...apache-curator-5.7.1) --- updated-dependencies: - dependency-name: org.apache.curator:curator-test dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/kafka-plugins/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/kafka-plugins/build.gradle b/data-prepper-plugins/kafka-plugins/build.gradle index 0ccb88b395..60c0472387 100644 --- a/data-prepper-plugins/kafka-plugins/build.gradle +++ b/data-prepper-plugins/kafka-plugins/build.gradle @@ -64,7 +64,7 @@ dependencies { testImplementation project(':data-prepper-pipeline-parser') testImplementation 'org.apache.kafka:kafka_2.13:3.6.1' testImplementation 'org.apache.kafka:kafka_2.13:3.6.1:test' - testImplementation 'org.apache.curator:curator-test:5.5.0' + testImplementation 'org.apache.curator:curator-test:5.7.1' testImplementation('com.kjetland:mbknor-jackson-jsonschema_2.13:1.0.39') testImplementation project(':data-prepper-plugins:otel-metrics-source') testImplementation project(':data-prepper-plugins:otel-proto-common') From 35dabf3f4fd479918077e59e3c145aae7d2f49da Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:51:57 -0800 Subject: [PATCH 12/36] Bump net.bytebuddy:byte-buddy in /data-prepper-plugins/opensearch (#5295) Bumps [net.bytebuddy:byte-buddy](https://github.com/raphw/byte-buddy) from 1.14.17 to 1.15.11. - [Release notes](https://github.com/raphw/byte-buddy/releases) - [Changelog](https://github.com/raphw/byte-buddy/blob/master/release-notes.md) - [Commits](https://github.com/raphw/byte-buddy/compare/byte-buddy-1.14.17...byte-buddy-1.15.11) --- updated-dependencies: - dependency-name: net.bytebuddy:byte-buddy dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/opensearch/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index 5e7879d8d1..e9f5b19f34 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -41,8 +41,8 @@ dependencies { } testImplementation testLibs.junit.vintage testImplementation libs.commons.io - testImplementation 'net.bytebuddy:byte-buddy:1.14.17' - testImplementation 'net.bytebuddy:byte-buddy-agent:1.14.17' + testImplementation 'net.bytebuddy:byte-buddy:1.15.11' + testImplementation 'net.bytebuddy:byte-buddy-agent:1.15.11' testImplementation testLibs.slf4j.simple } From 6fe2141ed8fa7ed7a3c17476c28efc21758ea945 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:53:51 -0800 Subject: [PATCH 13/36] Bump org.assertj:assertj-core in /data-prepper-plugins/otel-proto-common (#5286) Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.25.3 to 3.27.0. - [Release notes](https://github.com/assertj/assertj/releases) - [Commits](https://github.com/assertj/assertj/compare/assertj-build-3.25.3...assertj-build-3.27.0) --- updated-dependencies: - dependency-name: org.assertj:assertj-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/otel-proto-common/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-proto-common/build.gradle b/data-prepper-plugins/otel-proto-common/build.gradle index 03bafff809..657d9a8bd9 100644 --- a/data-prepper-plugins/otel-proto-common/build.gradle +++ b/data-prepper-plugins/otel-proto-common/build.gradle @@ -15,5 +15,5 @@ dependencies { implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation libs.commons.lang3 implementation libs.commons.codec - testImplementation 'org.assertj:assertj-core:3.25.3' + testImplementation 'org.assertj:assertj-core:3.27.0' } From aa4171c524c47d37e206ba27c8cef7ab99a28654 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 10:54:30 -0800 Subject: [PATCH 14/36] Bump org.wiremock:wiremock in /data-prepper-plugins/s3-source (#5236) Bumps [org.wiremock:wiremock](https://github.com/wiremock/wiremock) from 3.8.0 to 3.10.0. - [Release notes](https://github.com/wiremock/wiremock/releases) - [Commits](https://github.com/wiremock/wiremock/compare/3.8.0...3.10.0) --- updated-dependencies: - dependency-name: org.wiremock:wiremock dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/s3-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/s3-source/build.gradle b/data-prepper-plugins/s3-source/build.gradle index a11c34b49c..f4afbfbfe3 100644 --- a/data-prepper-plugins/s3-source/build.gradle +++ b/data-prepper-plugins/s3-source/build.gradle @@ -31,7 +31,7 @@ dependencies { implementation 'dev.failsafe:failsafe:3.3.2' implementation 'org.apache.httpcomponents:httpcore:4.4.16' testImplementation libs.commons.lang3 - testImplementation 'org.wiremock:wiremock:3.8.0' + testImplementation 'org.wiremock:wiremock:3.10.0' testImplementation 'org.eclipse.jetty:jetty-bom:11.0.20' testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' testImplementation testLibs.junit.vintage From be95d87dc0d65a6468e2b59a6068c838fe4f55d4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:34:35 -0800 Subject: [PATCH 15/36] Bump org.apache.logging.log4j:log4j-bom in /data-prepper-core (#5294) Bumps [org.apache.logging.log4j:log4j-bom](https://github.com/apache/logging-log4j2) from 2.23.1 to 2.24.3. - [Release notes](https://github.com/apache/logging-log4j2/releases) - [Changelog](https://github.com/apache/logging-log4j2/blob/2.x/RELEASE-NOTES.adoc) - [Commits](https://github.com/apache/logging-log4j2/compare/rel/2.23.1...rel/2.24.3) --- updated-dependencies: - dependency-name: org.apache.logging.log4j:log4j-bom dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-core/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-core/build.gradle b/data-prepper-core/build.gradle index c939129a1c..6ad15ff472 100644 --- a/data-prepper-core/build.gradle +++ b/data-prepper-core/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation 'io.micrometer:micrometer-registry-cloudwatch2' implementation 'javax.ws.rs:javax.ws.rs-api:2.1.1' implementation 'software.amazon.awssdk:cloudwatch' - implementation platform('org.apache.logging.log4j:log4j-bom:2.23.1') + implementation platform('org.apache.logging.log4j:log4j-bom:2.24.3') implementation 'org.apache.logging.log4j:log4j-core' implementation 'org.apache.logging.log4j:log4j-slf4j2-impl' implementation 'javax.inject:javax.inject:1' From ba168be0cf2fa3730766981257c09422287ba8bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:35:42 -0800 Subject: [PATCH 16/36] Bump org.assertj:assertj-core in /data-prepper-plugins/otel-logs-source (#5289) Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.25.3 to 3.27.0. - [Release notes](https://github.com/assertj/assertj/releases) - [Commits](https://github.com/assertj/assertj/compare/assertj-build-3.25.3...assertj-build-3.27.0) --- updated-dependencies: - dependency-name: org.assertj:assertj-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/otel-logs-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-logs-source/build.gradle b/data-prepper-plugins/otel-logs-source/build.gradle index 822e945ba9..b1bf2857e9 100644 --- a/data-prepper-plugins/otel-logs-source/build.gradle +++ b/data-prepper-plugins/otel-logs-source/build.gradle @@ -30,7 +30,7 @@ dependencies { implementation libs.commons.lang3 implementation libs.bouncycastle.bcprov implementation libs.bouncycastle.bcpkix - testImplementation 'org.assertj:assertj-core:3.25.3' + testImplementation 'org.assertj:assertj-core:3.27.0' testImplementation libs.commons.io } From 600f6432cbb4662b39c5a3f62e5de35bdac443ad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:36:17 -0800 Subject: [PATCH 17/36] Bump org.assertj:assertj-core in /data-prepper-plugins/otel-trace-source (#5287) Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.25.3 to 3.27.0. - [Release notes](https://github.com/assertj/assertj/releases) - [Commits](https://github.com/assertj/assertj/compare/assertj-build-3.25.3...assertj-build-3.27.0) --- updated-dependencies: - dependency-name: org.assertj:assertj-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/otel-trace-source/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-trace-source/build.gradle b/data-prepper-plugins/otel-trace-source/build.gradle index d1dcdfa12a..1791e0a4de 100644 --- a/data-prepper-plugins/otel-trace-source/build.gradle +++ b/data-prepper-plugins/otel-trace-source/build.gradle @@ -28,7 +28,7 @@ dependencies { implementation libs.commons.lang3 implementation libs.bouncycastle.bcprov implementation libs.bouncycastle.bcpkix - testImplementation 'org.assertj:assertj-core:3.25.3' + testImplementation 'org.assertj:assertj-core:3.27.0' testImplementation testLibs.slf4j.simple testImplementation libs.commons.io } From 00cc4aa756feb6308d08737eeda0f71536d9f915 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 11:36:33 -0800 Subject: [PATCH 18/36] Bump org.assertj:assertj-core (#5285) Bumps [org.assertj:assertj-core](https://github.com/assertj/assertj) from 3.25.3 to 3.27.0. - [Release notes](https://github.com/assertj/assertj/releases) - [Commits](https://github.com/assertj/assertj/compare/assertj-build-3.25.3...assertj-build-3.27.0) --- updated-dependencies: - dependency-name: org.assertj:assertj-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/otel-metrics-raw-processor/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/otel-metrics-raw-processor/build.gradle b/data-prepper-plugins/otel-metrics-raw-processor/build.gradle index a4316fca16..261b8b29b4 100644 --- a/data-prepper-plugins/otel-metrics-raw-processor/build.gradle +++ b/data-prepper-plugins/otel-metrics-raw-processor/build.gradle @@ -21,7 +21,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' implementation libs.guava.core - testImplementation 'org.assertj:assertj-core:3.25.3' + testImplementation 'org.assertj:assertj-core:3.27.0' } jacocoTestCoverageVerification { From 0d03cc9b276fddc39f0d3ca3e19f6e51a32506f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:35:07 -0800 Subject: [PATCH 19/36] Bump org.apache.maven:maven-artifact in /data-prepper-plugins/opensearch (#4894) Bumps [org.apache.maven:maven-artifact](https://github.com/apache/maven) from 3.9.8 to 3.9.9. - [Release notes](https://github.com/apache/maven/releases) - [Commits](https://github.com/apache/maven/compare/maven-3.9.8...maven-3.9.9) --- updated-dependencies: - dependency-name: org.apache.maven:maven-artifact dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- data-prepper-plugins/opensearch/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-prepper-plugins/opensearch/build.gradle b/data-prepper-plugins/opensearch/build.gradle index e9f5b19f34..1cd2f54335 100644 --- a/data-prepper-plugins/opensearch/build.gradle +++ b/data-prepper-plugins/opensearch/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation 'software.amazon.awssdk:apache-client' implementation 'software.amazon.awssdk:netty-nio-client' implementation 'co.elastic.clients:elasticsearch-java:7.17.0' - implementation('org.apache.maven:maven-artifact:3.9.8') { + implementation('org.apache.maven:maven-artifact:3.9.9') { exclude group: 'org.codehaus.plexus' } testImplementation testLibs.junit.vintage From 2aa376e3be1261a6d56dd20ac19640156e114cbc Mon Sep 17 00:00:00 2001 From: Jeremy Michael <60355474+jmsusanto@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:21:30 -0800 Subject: [PATCH 20/36] SQS source plugin implementation (#5274) * initial poc of new sqs-source plugin Signed-off-by: Jeremy Michael * modified README Signed-off-by: Jeremy Michael * Changed parser to format described in issue Signed-off-by: Jeremy Michael * Renamed sqs-source-new to sqs-source and removed deprecated sqs-source implementation Signed-off-by: Jeremy Michael * Update README.md Signed-off-by: Jeremy Michael <60355474+jmsusanto@users.noreply.github.com> * Update README.md Signed-off-by: Jeremy Michael <60355474+jmsusanto@users.noreply.github.com> * Add queueUrl and sentTimestamp metadata to SQS events Signed-off-by: Jeremy Michael * Assign individual BufferAccumulators to each SQS worker Signed-off-by: Jeremy Michael * added unit tests and addressed comments in design doc Signed-off-by: Jeremy Michael * added additional unit tests Signed-off-by: Jeremy Michael * Fix region loading check Signed-off-by: Jeremy Michael * addressed PR comments Signed-off-by: Jeremy Michael * addressed PR comments - modified Buffer and parser implementation Signed-off-by: Jeremy Michael * added copyright headers to tests and updated metadata naming Signed-off-by: Jeremy Michael * some cleanup and converted all metadata to lowerCamelCase Signed-off-by: Jeremy Michael --------- Signed-off-by: Jeremy Michael Signed-off-by: Jeremy Michael <60355474+jmsusanto@users.noreply.github.com> Co-authored-by: Jeremy Michael --- data-prepper-plugins/sqs-source/README.md | 78 +--- data-prepper-plugins/sqs-source/build.gradle | 45 +-- .../source/sqssource/RecordsGenerator.java | 11 - .../source/sqssource/SqsRecordsGenerator.java | 47 --- .../source/sqssource/SqsSourceTaskIT.java | 147 ------- .../source/sqs/AwsAuthenticationAdapter.java | 36 ++ .../source/sqs/AwsAuthenticationOptions.java | 69 ++++ .../plugins/source/sqs/QueueConfig.java | 97 +++++ .../source/sqs/RawSqsMessageHandler.java | 65 +++ .../plugins/source/sqs/SqsEventProcessor.java | 29 ++ .../plugins/source/sqs/SqsMessageHandler.java | 20 + .../sqs/SqsRetriesExhaustedException.java | 18 + .../plugins/source/sqs/SqsService.java | 128 ++++++ .../plugins/source/sqs/SqsSource.java | 69 ++++ .../plugins/source/sqs/SqsSourceConfig.java | 50 +++ .../plugins/source/sqs/SqsWorker.java | 326 +++++++++++++++ .../plugins/source/sqssource/SqsSource.java | 115 ------ .../source/sqssource/SqsSourceTask.java | 112 ------ .../config/AwsAuthenticationOptions.java | 70 ---- .../sqssource/config/SqsSourceConfig.java | 77 ---- .../handler/RawSqsMessageHandler.java | 67 ---- .../sqs/AwsAuthenticationAdapterTest.java | 90 +++++ .../sqs/AwsAuthenticationOptionsTest.java | 136 +++++++ .../plugins/source/sqs/QueueConfigTest.java | 31 ++ .../source/sqs/RawSqsMessageHandlerTest.java | 44 ++ .../source/sqs/SqsEventProcessorTest.java | 60 +++ .../plugins/source/sqs/SqsServiceTest.java | 81 ++++ .../source/sqs/SqsSourceConfigTest.java | 27 ++ .../plugins/source/sqs/SqsSourceTest.java | 65 +++ .../plugins/source/sqs/SqsWorkerTest.java | 378 ++++++++++++++++++ .../source/sqssource/SqsSourceTaskTest.java | 170 -------- .../source/sqssource/SqsSourceTest.java | 57 --- .../sqssource/config/SqsSourceConfigTest.java | 47 --- .../handler/RawSqsMessageHandlerTest.java | 132 ------ .../src/test/resources/pipeline.yaml | 10 - settings.gradle | 2 +- 36 files changed, 1847 insertions(+), 1159 deletions(-) delete mode 100644 data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/RecordsGenerator.java delete mode 100644 data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsRecordsGenerator.java delete mode 100644 data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTaskIT.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapter.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptions.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfig.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandler.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessor.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsMessageHandler.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsRetriesExhaustedException.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsService.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSource.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfig.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorker.java delete mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSource.java delete mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTask.java delete mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/config/AwsAuthenticationOptions.java delete mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/config/SqsSourceConfig.java delete mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/handler/RawSqsMessageHandler.java create mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapterTest.java create mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptionsTest.java create mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfigTest.java create mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandlerTest.java create mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessorTest.java create mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsServiceTest.java create mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfigTest.java create mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceTest.java create mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorkerTest.java delete mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTaskTest.java delete mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTest.java delete mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/config/SqsSourceConfigTest.java delete mode 100644 data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/handler/RawSqsMessageHandlerTest.java delete mode 100644 data-prepper-plugins/sqs-source/src/test/resources/pipeline.yaml diff --git a/data-prepper-plugins/sqs-source/README.md b/data-prepper-plugins/sqs-source/README.md index 708389f769..ff4313605f 100644 --- a/data-prepper-plugins/sqs-source/README.md +++ b/data-prepper-plugins/sqs-source/README.md @@ -1,70 +1,22 @@ -# SQS Source +# SQS Source -This source allows Data Prepper to use SQS as a source. It uses SQS for notifications -of which data are new and loads those messages to push out events. +This source allows Data Prepper to use SQS as a source. It reads messages from specified SQS queues and processes them into events. -### Example: +## Example Configuration -The following configuration shows a minimum configuration for reading and Sqs messages and push out events. - -``` +```yaml sqs-pipeline: source: sqs: - acknowledgments: true - queue_urls: - - https://sqs.us-east-1.amazonaws.com/895099421235/MyQueue-1 - - https://sqs.us-east-1.amazonaws.com/895099421235/MyQueue-2 - - https://sqs.us-east-1.amazonaws.com/895099421235/MyQueue-3 - number_of_threads : 1 - batch_size : 10 - visibility_timeout: PT30S - wait_time : PT20S + queues: + - url: + batch_size: 10 + workers: 1 + - url: + batch_size: 10 + workers: 1 aws: - sts_region: us-east-1 - sts_role_arn: arn:aws:iam::895099421235:role/test-arn -``` - -## Configuration Options - -All Duration values are a string that represents a duration. They support ISO_8601 notation string ("PT20.345S", "PT15M", etc.) as well as simple notation Strings for seconds ("60s") and milliseconds ("1500ms"). - -* `queue_url or queue_urls` (Required) : The SQS configuration. See [SQS Configuration](#sqs_configuration) for details. - -* `aws` (Optional) : AWS configurations. See [AWS Configuration](#aws_configuration) for details. - -* `acknowledgments` (Optional) : Enables End-to-end acknowledgments. If set to `true`, sqs message is deleted only after all events from the sqs message are successfully acknowledged by all sinks. Default value `false`. - -### SQS Configuration - -* `number_of_threads` (Optional) : define no of threads for sqs queue processing. default to 1. -* `batch_size` (Optional) : define batch size for sqs messages processing. default to 10. -* `polling_frequency` (Optional) : Duration - A delay to place between reading and processing a batch of SQS messages and making a subsequent request. Defaults to 0 seconds. -* `visibility_timeout` (Optional) : Duration - The visibility timeout to apply to messages read from the SQS queue. Defaults to null -* `wait_time` (Optional) : Duration - The time to wait for long-polling on the SQS API. Defaults to null. - -### AWS Configuration - -The AWS configuration is the same for both SQS. - -* `sts_region` (Optional) : The AWS region to use for credentials. Defaults to [standard SDK behavior to determine the region](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/region-selection.html). -* `sts_role_arn` (Optional) : The AWS STS role to assume for requests to SQS. Defaults to null, which will use the [standard SDK behavior for credentials](https://docs.aws.amazon.com/sdk-for-java/latest/developer-guide/credentials.html). -* `sts_header_overrides` (Optional): A map of header overrides to make when assuming the IAM role for the sink plugin. - -## Metrics - -* `sqsMessagesReceived` - The number of SQS messages received from the queue by the SQS Source. -* `sqsMessagesDeleted` - The number of SQS messages deleted from the queue by the SQS Source. -* `sqsMessagesFailed` - The number of SQS messages that the SQS Source failed to parse. -* `sqsMessagesDeleteFailed` - The number of SQS messages that the SQS Source failed to delete from the SQS queue. -* `acknowledgementSetCallbackCounter` - The number of SQS messages processed by SQS Source and successfully acknowledge by sink. - -## Developer Guide - -The integration tests for this plugin do not run as part of the Data Prepper build. - -The following command runs the integration tests: - -``` -./gradlew :data-prepper-plugins:sqs-source:integrationTest -Dtests.sqs.source.aws.region= -Dtests.sqs.source.queue.url= -``` + region: + sts_role_arn: + sink: + - stdout: diff --git a/data-prepper-plugins/sqs-source/build.gradle b/data-prepper-plugins/sqs-source/build.gradle index 722d5afb7f..b4ffbc8e5e 100644 --- a/data-prepper-plugins/sqs-source/build.gradle +++ b/data-prepper-plugins/sqs-source/build.gradle @@ -1,8 +1,17 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +plugins { + id 'java' +} + dependencies { implementation project(':data-prepper-api') - implementation libs.armeria.core - implementation project(':data-prepper-plugins:aws-sqs-common') implementation project(':data-prepper-plugins:buffer-common') + implementation project(':data-prepper-plugins:common') + implementation libs.armeria.core implementation project(':data-prepper-plugins:aws-plugin-api') implementation 'software.amazon.awssdk:sqs' implementation 'software.amazon.awssdk:arns' @@ -10,40 +19,10 @@ dependencies { implementation 'io.micrometer:micrometer-core' implementation 'com.fasterxml.jackson.core:jackson-annotations' implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + implementation 'org.hibernate.validator:hibernate-validator:8.0.1.Final' testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' testImplementation project(':data-prepper-plugins:blocking-buffer') } test { useJUnitPlatform() } - -sourceSets { - integrationTest { - java { - compileClasspath += main.output + test.output - runtimeClasspath += main.output + test.output - srcDir file('src/integrationTest/java') - } - resources.srcDir file('src/integrationTest/resources') - } -} - -configurations { - integrationTestImplementation.extendsFrom testImplementation - integrationTestRuntime.extendsFrom testRuntime -} - -task integrationTest(type: Test) { - group = 'verification' - testClassesDirs = sourceSets.integrationTest.output.classesDirs - - useJUnitPlatform() - - classpath = sourceSets.integrationTest.runtimeClasspath - systemProperty 'tests.sqs.source.aws.region', System.getProperty('tests.sqs.source.aws.region') - systemProperty 'tests.sqs.source.queue.url', System.getProperty('tests.sqs.source.queue.url') - - filter { - includeTestsMatching '*IT' - } -} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/RecordsGenerator.java b/data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/RecordsGenerator.java deleted file mode 100644 index a0d4a24d68..0000000000 --- a/data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/RecordsGenerator.java +++ /dev/null @@ -1,11 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.source.sqssource; - -import java.util.List; - -public interface RecordsGenerator { - void pushMessages(final List messages, final String queueUrl); -} diff --git a/data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsRecordsGenerator.java b/data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsRecordsGenerator.java deleted file mode 100644 index 18e4ea2980..0000000000 --- a/data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsRecordsGenerator.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.source.sqssource; - -import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequest; -import software.amazon.awssdk.services.sqs.model.SendMessageBatchRequestEntry; - -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -public class SqsRecordsGenerator implements RecordsGenerator { - - private final SqsClient sqsClient; - - public SqsRecordsGenerator(final SqsClient sqsClient){ - this.sqsClient = sqsClient; - } - - @Override - public void pushMessages(final List messages, String queueUrl) { - final List> batches = splitIntoBatches(messages, 10); - batches.forEach(batch -> { - List entries = new ArrayList<>(); - batch.forEach(msg -> entries.add(SendMessageBatchRequestEntry.builder() - .id(UUID.randomUUID() + "-" + UUID.randomUUID()).messageBody(msg).build())); - sqsClient.sendMessageBatch(SendMessageBatchRequest.builder().queueUrl(queueUrl).entries(entries).build()); - }); - } - - private static List> splitIntoBatches(List messages, int batchSize) { - List> batches = new ArrayList<>(); - int totalRecords = messages.size(); - int numBatches = (int) Math.ceil((double) totalRecords / batchSize); - - for (int i = 0; i < numBatches; i++) { - int startIndex = i * batchSize; - int endIndex = Math.min(startIndex + batchSize, totalRecords); - List batch = messages.subList(startIndex, endIndex); - batches.add(batch); - } - return batches; - } -} diff --git a/data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTaskIT.java b/data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTaskIT.java deleted file mode 100644 index 31e12db422..0000000000 --- a/data-prepper-plugins/sqs-source/src/integrationTest/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTaskIT.java +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.source.sqssource; - -import com.linecorp.armeria.client.retry.Backoff; -import io.micrometer.core.instrument.Counter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.CsvSource; -import org.mockito.Mock; -import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; -import org.opensearch.dataprepper.model.configuration.PluginSetting; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.aws.sqs.common.SqsService; -import org.opensearch.dataprepper.plugins.aws.sqs.common.handler.SqsMessageHandler; -import org.opensearch.dataprepper.plugins.aws.sqs.common.metrics.SqsMetrics; -import org.opensearch.dataprepper.plugins.aws.sqs.common.model.SqsOptions; -import org.opensearch.dataprepper.plugins.buffer.blockingbuffer.BlockingBuffer; -import org.opensearch.dataprepper.plugins.source.sqssource.handler.RawSqsMessageHandler; -import software.amazon.awssdk.regions.Region; -import software.amazon.awssdk.services.sqs.SqsClient; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class SqsSourceTaskIT { - - static final long INITIAL_DELAY = Duration.ofSeconds(20).toMillis(); - - static final long MAXIMUM_DELAY = Duration.ofMinutes(5).toMillis(); - - static final double JITTER_RATE = 0.20; - - private static final String TEST_PIPELINE_NAME = "pipeline"; - - private static final String MESSAGE = "message"; - - private static final String JSON_MESSAGE = "{\"array\":[{\"name\":\"abc\",\"test\":[{\"company\":\"xyz\"}]},{\"number\":1}]}"; - - private static final String LOG_MESSAGE = "2023-06-14T11:59:54,350 [main] INFO Test - Application started Successfully\n"; - - private static final String AWS_SQS_QUEUE_URL = "tests.sqs.source.queue.url"; - - private static final String AWS_REGION = "tests.sqs.source.aws.region"; - public static final Duration BUFFER_TIMEOUT = Duration.ofSeconds(10); - public static final int RECORDS_TO_ACCUMULATE = 100; - - @Mock - private AcknowledgementSetManager acknowledgementSetManager; - - private BlockingBuffer> buffer; - - private Counter messageReceivedCounter; - - private Counter messageDeletedCounter; - - private Backoff backoff; - - private SqsClient sqsClient; - - private SqsMetrics sqsMetrics; - - private ScheduledExecutorService executorService; - - @ParameterizedTest - @CsvSource({"2,1","10,2","50,4","100,5","200,7","500,10","1000,15","2000,24"}) - public void process_sqs_messages(int messageLoad,int threadSleepTime){ - final SqsRecordsGenerator sqsRecordsGenerator = new SqsRecordsGenerator(sqsClient); - final String queueUrl = System.getProperty(AWS_SQS_QUEUE_URL); - - List inputDataList = pushMessagesToQueue(sqsRecordsGenerator, queueUrl,messageLoad); - this.buffer = getBuffer(inputDataList.size()); - - SqsOptions sqsOptions = new SqsOptions.Builder().setSqsUrl(queueUrl).setMaximumMessages(10).build(); - executorService.scheduleAtFixedRate(createObjectUnderTest(sqsOptions),0,1, TimeUnit.MILLISECONDS); - try { - Thread.sleep(Duration.ofSeconds(threadSleepTime).toMillis()); - } catch (InterruptedException e) { - } - executorService.shutdown(); - final List> bufferEvents = new ArrayList<>(buffer.read((int) Duration.ofSeconds(10).toMillis()).getKey()); - final List bufferData = bufferEvents.stream().map(obj -> obj.getData().get(MESSAGE, String.class)).collect(Collectors.toList()); - assertThat(bufferData, containsInAnyOrder(inputDataList.toArray())); - assertThat(bufferData.size(),equalTo(inputDataList.size())); - } - - @BeforeEach - public void setup(){ - this.acknowledgementSetManager = mock(AcknowledgementSetManager.class); - this.messageReceivedCounter = mock(Counter.class); - this.messageDeletedCounter = mock(Counter.class); - this.sqsMetrics = mock(SqsMetrics.class); - when(sqsMetrics.getSqsMessagesReceivedCounter()).thenReturn(messageReceivedCounter); - when(sqsMetrics.getSqsMessagesDeletedCounter()).thenReturn(messageDeletedCounter); - this.backoff = Backoff.exponential(INITIAL_DELAY, MAXIMUM_DELAY).withJitter(JITTER_RATE) - .withMaxAttempts(Integer.MAX_VALUE); - this.sqsClient = SqsClient.builder().region(Region.of(System.getProperty(AWS_REGION))).build(); - executorService = Executors.newSingleThreadScheduledExecutor(); - } - - public SqsSourceTask createObjectUnderTest(final SqsOptions sqsOptions){ - SqsService sqsService = new SqsService(sqsMetrics,sqsClient,backoff); - SqsMessageHandler sqsHandler = new RawSqsMessageHandler(sqsService); - return new SqsSourceTask(buffer, RECORDS_TO_ACCUMULATE, BUFFER_TIMEOUT - ,sqsService,sqsOptions,sqsMetrics, - acknowledgementSetManager,Boolean.FALSE,sqsHandler); - } - - private static List pushMessagesToQueue(SqsRecordsGenerator sqsRecordsGenerator, String queueUrl,final int load) { - List inputDataList = new ArrayList<>(); - for(int msgCount = 0; msgCount < load/2; msgCount++) - generateMessagesForSqsPush().forEach(obj -> inputDataList.add(obj)); - sqsRecordsGenerator.pushMessages(inputDataList, queueUrl); - return inputDataList; - } - - private static List generateMessagesForSqsPush(){ - List messages = new ArrayList<>(2); - messages.add(JSON_MESSAGE); - messages.add(LOG_MESSAGE); - return messages; - } - - private BlockingBuffer> getBuffer(final int bufferSize) { - final HashMap integerHashMap = new HashMap<>(); - integerHashMap.put("buffer_size", bufferSize); - integerHashMap.put("batch_size", bufferSize); - final PluginSetting pluginSetting = new PluginSetting("blocking_buffer", integerHashMap); - pluginSetting.setPipelineName(TEST_PIPELINE_NAME); - return new BlockingBuffer<>(pluginSetting); - } -} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapter.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapter.java new file mode 100644 index 0000000000..08600cba13 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapter.java @@ -0,0 +1,36 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; +import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + +class AwsAuthenticationAdapter { + private final AwsCredentialsSupplier awsCredentialsSupplier; + private final SqsSourceConfig sqsSourceConfig; + + + AwsAuthenticationAdapter( + final AwsCredentialsSupplier awsCredentialsSupplier, + final SqsSourceConfig sqsSourceConfig) { + this.awsCredentialsSupplier = awsCredentialsSupplier; + this.sqsSourceConfig = sqsSourceConfig; + } + + AwsCredentialsProvider getCredentialsProvider() { + final AwsAuthenticationOptions awsAuthenticationOptions = sqsSourceConfig.getAwsAuthenticationOptions(); + + final AwsCredentialsOptions options = AwsCredentialsOptions.builder() + .withStsRoleArn(awsAuthenticationOptions.getAwsStsRoleArn()) + .withRegion(awsAuthenticationOptions.getAwsRegion()) + .withStsHeaderOverrides(awsAuthenticationOptions.getAwsStsHeaderOverrides()) + .withStsExternalId(awsAuthenticationOptions.getAwsStsExternalId()) + .build(); + + return awsCredentialsSupplier.getProvider(options); + } +} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptions.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptions.java new file mode 100644 index 0000000000..99da366de3 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptions.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + package org.opensearch.dataprepper.plugins.source.sqs; + + import com.fasterxml.jackson.annotation.JsonProperty; + import jakarta.validation.constraints.Size; + import software.amazon.awssdk.arns.Arn; + import software.amazon.awssdk.regions.Region; + + import java.util.Map; + import java.util.Optional; + + public class AwsAuthenticationOptions { + private static final String AWS_IAM_ROLE = "role"; + private static final String AWS_IAM = "iam"; + + @JsonProperty("region") + @Size(min = 1, message = "Region cannot be empty string") + private String awsRegion; + + @JsonProperty("sts_role_arn") + @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") + private String awsStsRoleArn; + + @JsonProperty("sts_external_id") + @Size(min = 2, max = 1224, message = "awsStsExternalId length should be between 2 and 1224 characters") + private String awsStsExternalId; + + @JsonProperty("sts_header_overrides") + @Size(max = 5, message = "sts_header_overrides supports a maximum of 5 headers to override") + private Map awsStsHeaderOverrides; + + void validateStsRoleArn() { + final Arn arn = getArn(); + if (!AWS_IAM.equals(arn.service())) { + throw new IllegalArgumentException("sts_role_arn must be an IAM Role"); + } + final Optional resourceType = arn.resource().resourceType(); + if (resourceType.isEmpty() || !resourceType.get().equals(AWS_IAM_ROLE)) { + throw new IllegalArgumentException("sts_role_arn must be an IAM Role"); + } + } + + private Arn getArn() { + try { + return Arn.fromString(awsStsRoleArn); + } catch (final Exception e) { + throw new IllegalArgumentException(String.format("The value provided for sts_role_arn is not a valid AWS ARN. Provided value: %s", awsStsRoleArn)); } + } + + public String getAwsStsRoleArn() { + return awsStsRoleArn; + } + + public String getAwsStsExternalId() { + return awsStsExternalId; + } + + public Region getAwsRegion() { + return awsRegion != null ? Region.of(awsRegion) : null; + } + + public Map getAwsStsHeaderOverrides() { + return awsStsHeaderOverrides; + } + } \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfig.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfig.java new file mode 100644 index 0000000000..ca5566d6cd --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfig.java @@ -0,0 +1,97 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import com.fasterxml.jackson.annotation.JsonProperty; + +import jakarta.validation.Valid; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; +import java.time.Duration; + +import org.hibernate.validator.constraints.time.DurationMax; +import org.hibernate.validator.constraints.time.DurationMin; + +public class QueueConfig { + + private static final Integer DEFAULT_MAXIMUM_MESSAGES = null; + private static final boolean DEFAULT_VISIBILITY_DUPLICATE_PROTECTION = false; + private static final Duration DEFAULT_VISIBILITY_TIMEOUT_SECONDS = null; + private static final Duration DEFAULT_VISIBILITY_DUPLICATE_PROTECTION_TIMEOUT = Duration.ofHours(2); + private static final Duration DEFAULT_WAIT_TIME_SECONDS = null; + private static final Duration DEFAULT_POLL_DELAY_SECONDS = Duration.ofSeconds(0); + static final int DEFAULT_NUMBER_OF_WORKERS = 1; + + @JsonProperty("url") + @NotNull + private String url; + + @JsonProperty("workers") + @Valid + private int numWorkers = DEFAULT_NUMBER_OF_WORKERS; + + @JsonProperty("maximum_messages") + @Min(1) + @Max(10) + private Integer maximumMessages = DEFAULT_MAXIMUM_MESSAGES; + + @JsonProperty("poll_delay") + @DurationMin(seconds = 0) + private Duration pollDelay = DEFAULT_POLL_DELAY_SECONDS; + + @JsonProperty("visibility_timeout") + @DurationMin(seconds = 0) + @DurationMax(seconds = 43200) + private Duration visibilityTimeout = DEFAULT_VISIBILITY_TIMEOUT_SECONDS; + + @JsonProperty("visibility_duplication_protection") + @NotNull + private boolean visibilityDuplicateProtection = DEFAULT_VISIBILITY_DUPLICATE_PROTECTION; + + @JsonProperty("visibility_duplicate_protection_timeout") + @DurationMin(seconds = 30) + @DurationMax(hours = 24) + private Duration visibilityDuplicateProtectionTimeout = DEFAULT_VISIBILITY_DUPLICATE_PROTECTION_TIMEOUT; + + @JsonProperty("wait_time") + @DurationMin(seconds = 0) + @DurationMax(seconds = 20) + private Duration waitTime = DEFAULT_WAIT_TIME_SECONDS; + + public String getUrl() { + return url; + } + + public Integer getMaximumMessages() { + return maximumMessages; + } + + public int getNumWorkers() { + return numWorkers; + } + + public Duration getVisibilityTimeout() { + return visibilityTimeout; + } + + public boolean getVisibilityDuplicateProtection() { + return visibilityDuplicateProtection; + } + + public Duration getVisibilityDuplicateProtectionTimeout() { + return visibilityDuplicateProtectionTimeout; + } + + public Duration getWaitTime() { + return waitTime; + } + + public Duration getPollDelay() { + return pollDelay; + } +} + diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandler.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandler.java new file mode 100644 index 0000000000..493b7ab7d7 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandler.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventMetadata; +import org.opensearch.dataprepper.model.event.JacksonEvent; +import org.opensearch.dataprepper.model.record.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageAttributeValue; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; + +import java.util.Collections; +import java.util.Map; + +public class RawSqsMessageHandler implements SqsMessageHandler { + + private static final Logger LOG = LoggerFactory.getLogger(RawSqsMessageHandler.class); + + @Override + public void handleMessage(final Message message, + final String url, + final Buffer> buffer, + final int bufferTimeoutMillis, + final AcknowledgementSet acknowledgementSet) { + try { + final Map systemAttributes = message.attributes(); + final Map customAttributes = message.messageAttributes(); + final Event event = JacksonEvent.builder() + .withEventType("DOCUMENT") + .withData(Collections.singletonMap("message", message.body())) + .build(); + + final EventMetadata eventMetadata = event.getMetadata(); + eventMetadata.setAttribute("queueUrl", url); + + for (Map.Entry entry : systemAttributes.entrySet()) { + String originalKey = entry.getKey().toString(); + String lowerCamelCaseKey = originalKey.substring(0, 1).toLowerCase() + originalKey.substring(1); + eventMetadata.setAttribute(lowerCamelCaseKey, entry.getValue()); + } + + for (Map.Entry entry : customAttributes.entrySet()) { + String originalKey = entry.getKey(); + String lowerCamelCaseKey = originalKey.substring(0, 1).toLowerCase() + originalKey.substring(1); + eventMetadata.setAttribute(lowerCamelCaseKey, entry.getValue().stringValue()); + } + + if (acknowledgementSet != null) { + acknowledgementSet.add(event); + } + buffer.write(new Record<>(event), bufferTimeoutMillis); + } catch (Exception e) { + LOG.error("Error processing SQS message: {}", e.getMessage(), e); + throw new RuntimeException(e); + } + } +} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessor.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessor.java new file mode 100644 index 0000000000..a03c485c37 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessor.java @@ -0,0 +1,29 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import software.amazon.awssdk.services.sqs.model.Message; +import java.io.IOException; + +public class SqsEventProcessor { + private final SqsMessageHandler sqsMessageHandler; + SqsEventProcessor(final SqsMessageHandler sqsMessageHandler) { + this.sqsMessageHandler= sqsMessageHandler; + } + + void addSqsObject(final Message message, + final String url, + final Buffer> buffer, + final int bufferTimeoutmillis, + final AcknowledgementSet acknowledgementSet) throws IOException { + sqsMessageHandler.handleMessage(message, url, buffer, bufferTimeoutmillis, acknowledgementSet); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsMessageHandler.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsMessageHandler.java new file mode 100644 index 0000000000..79012b5e00 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsMessageHandler.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import software.amazon.awssdk.services.sqs.model.Message; +import java.io.IOException; + +public interface SqsMessageHandler { + void handleMessage(final Message message, + final String url, + final Buffer> buffer, + final int bufferTimeoutMillis, + final AcknowledgementSet acknowledgementSet) throws IOException ; +} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsRetriesExhaustedException.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsRetriesExhaustedException.java new file mode 100644 index 0000000000..4e1f9507e6 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsRetriesExhaustedException.java @@ -0,0 +1,18 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +/** + * This exception is thrown when SQS retries are exhausted + * + * @since 2.1 + */ +public class SqsRetriesExhaustedException extends RuntimeException { + + public SqsRetriesExhaustedException(final String errorMessage) { + super(errorMessage); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsService.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsService.java new file mode 100644 index 0000000000..d53f269323 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsService.java @@ -0,0 +1,128 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + + package org.opensearch.dataprepper.plugins.source.sqs; + + import com.linecorp.armeria.client.retry.Backoff; + import org.opensearch.dataprepper.common.concurrent.BackgroundThreadFactory; + import org.opensearch.dataprepper.metrics.PluginMetrics; + import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; + import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; + import software.amazon.awssdk.core.retry.RetryPolicy; + import software.amazon.awssdk.services.sqs.SqsClient; + import org.opensearch.dataprepper.model.buffer.Buffer; + import org.opensearch.dataprepper.model.event.Event; + import org.opensearch.dataprepper.model.record.Record; + import java.time.Duration; + import java.util.ArrayList; + import java.util.List; + import java.util.concurrent.TimeUnit; + import java.util.concurrent.Executors; + import java.util.concurrent.ExecutorService; + import java.util.stream.Collectors; + import java.util.stream.IntStream; + + public class SqsService { + private static final Logger LOG = LoggerFactory.getLogger(SqsService.class); + static final long SHUTDOWN_TIMEOUT = 30L; + static final long INITIAL_DELAY = Duration.ofSeconds(20).toMillis(); + static final long MAXIMUM_DELAY = Duration.ofMinutes(5).toMillis(); + static final double JITTER_RATE = 0.20; + + private final SqsSourceConfig sqsSourceConfig; + private final SqsEventProcessor sqsEventProcessor; + private final SqsClient sqsClient; + private final PluginMetrics pluginMetrics; + private final AcknowledgementSetManager acknowledgementSetManager; + private final List allSqsUrlExecutorServices; + private final List sqsWorkers; + private final Buffer> buffer; + + public SqsService(final Buffer> buffer, + final AcknowledgementSetManager acknowledgementSetManager, + final SqsSourceConfig sqsSourceConfig, + final SqsEventProcessor sqsEventProcessor, + final PluginMetrics pluginMetrics, + final AwsCredentialsProvider credentialsProvider) { + + this.sqsSourceConfig = sqsSourceConfig; + this.sqsEventProcessor = sqsEventProcessor; + this.pluginMetrics = pluginMetrics; + this.acknowledgementSetManager = acknowledgementSetManager; + this.allSqsUrlExecutorServices = new ArrayList<>(); + this.sqsWorkers = new ArrayList<>(); + this.sqsClient = createSqsClient(credentialsProvider); + this.buffer = buffer; + } + + + public void start() { + final Backoff backoff = Backoff.exponential(INITIAL_DELAY, MAXIMUM_DELAY).withJitter(JITTER_RATE) + .withMaxAttempts(Integer.MAX_VALUE); + + LOG.info("Starting SqsService"); + + sqsSourceConfig.getQueues().forEach(queueConfig -> { + String queueUrl = queueConfig.getUrl(); + String queueName = queueUrl.substring(queueUrl.lastIndexOf('/') + 1); + + int numWorkers = queueConfig.getNumWorkers(); + ExecutorService executorService = Executors.newFixedThreadPool( + numWorkers, BackgroundThreadFactory.defaultExecutorThreadFactory("sqs-source" + queueName)); + allSqsUrlExecutorServices.add(executorService); + List workers = IntStream.range(0, numWorkers) + .mapToObj(i -> new SqsWorker( + buffer, + acknowledgementSetManager, + sqsClient, + sqsEventProcessor, + sqsSourceConfig, + queueConfig, + pluginMetrics, + backoff)) + .collect(Collectors.toList()); + + sqsWorkers.addAll(workers); + workers.forEach(executorService::submit); + LOG.info("Started SQS workers for queue {} with {} workers", queueUrl, numWorkers); + }); + } + + SqsClient createSqsClient(final AwsCredentialsProvider credentialsProvider) { + LOG.debug("Creating SQS client"); + return SqsClient.builder() + .region(sqsSourceConfig.getAwsAuthenticationOptions().getAwsRegion()) + .credentialsProvider(credentialsProvider) + .overrideConfiguration(ClientOverrideConfiguration.builder() + .retryPolicy(RetryPolicy.builder().numRetries(5).build()) + .build()) + .build(); + } + + public void stop() { + allSqsUrlExecutorServices.forEach(ExecutorService::shutdown); + sqsWorkers.forEach(SqsWorker::stop); + allSqsUrlExecutorServices.forEach(executorService -> { + try { + if (!executorService.awaitTermination(SHUTDOWN_TIMEOUT, TimeUnit.SECONDS)) { + LOG.warn("Failed to terminate SqsWorkers"); + executorService.shutdownNow(); + } + } catch (InterruptedException e) { + LOG.error("Interrupted during shutdown, exiting uncleanly...", e); + executorService.shutdownNow(); + Thread.currentThread().interrupt(); + } + }); + + sqsClient.close(); + LOG.info("SqsService shutdown completed."); + } + + } + \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSource.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSource.java new file mode 100644 index 0000000000..980e59048b --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSource.java @@ -0,0 +1,69 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import org.opensearch.dataprepper.model.source.Source; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import java.util.Objects; + +@DataPrepperPlugin(name = "sqs", pluginType = Source.class,pluginConfigurationType = SqsSourceConfig.class) +public class SqsSource implements Source> { + + private final PluginMetrics pluginMetrics; + private final SqsSourceConfig sqsSourceConfig; + private SqsService sqsService; + private final AcknowledgementSetManager acknowledgementSetManager; + private final AwsCredentialsSupplier awsCredentialsSupplier; + private final boolean acknowledgementsEnabled; + + + @DataPrepperPluginConstructor + public SqsSource(final PluginMetrics pluginMetrics, + final SqsSourceConfig sqsSourceConfig, + final AcknowledgementSetManager acknowledgementSetManager, + final AwsCredentialsSupplier awsCredentialsSupplier) { + + this.pluginMetrics = pluginMetrics; + this.sqsSourceConfig = sqsSourceConfig; + this.acknowledgementsEnabled = sqsSourceConfig.getAcknowledgements(); + this.acknowledgementSetManager = acknowledgementSetManager; + this.awsCredentialsSupplier = awsCredentialsSupplier; + + } + + @Override + public void start(Buffer> buffer) { + if (buffer == null) { + throw new IllegalStateException("Buffer is null"); + } + final AwsAuthenticationAdapter awsAuthenticationAdapter = new AwsAuthenticationAdapter(awsCredentialsSupplier, sqsSourceConfig); + final AwsCredentialsProvider credentialsProvider = awsAuthenticationAdapter.getCredentialsProvider(); + final SqsMessageHandler rawSqsMessageHandler = new RawSqsMessageHandler(); + final SqsEventProcessor sqsEventProcessor = new SqsEventProcessor(rawSqsMessageHandler); + sqsService = new SqsService(buffer, acknowledgementSetManager, sqsSourceConfig, sqsEventProcessor, pluginMetrics, credentialsProvider); + sqsService.start(); + } + + @Override + public boolean areAcknowledgementsEnabled() { + return acknowledgementsEnabled; + } + + @Override + public void stop() { + if (Objects.nonNull(sqsService)) { + sqsService.stop(); + } + } +} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfig.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfig.java new file mode 100644 index 0000000000..c84a3a3d69 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfig.java @@ -0,0 +1,50 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import com.fasterxml.jackson.annotation.JsonProperty; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotNull; +import java.time.Duration; +import java.util.List; + +public class SqsSourceConfig { + + static final Duration DEFAULT_BUFFER_TIMEOUT = Duration.ofSeconds(10); + static final int DEFAULT_NUMBER_OF_RECORDS_TO_ACCUMULATE = 100; + + @JsonProperty("aws") + @NotNull + @Valid + private AwsAuthenticationOptions awsAuthenticationOptions; + + @JsonProperty("acknowledgments") + private boolean acknowledgments = false; + + @JsonProperty("buffer_timeout") + private Duration bufferTimeout = DEFAULT_BUFFER_TIMEOUT; + + @JsonProperty("queues") + @NotNull + @Valid + private List queues; + + public AwsAuthenticationOptions getAwsAuthenticationOptions() { + return awsAuthenticationOptions; + } + + public boolean getAcknowledgements() { + return acknowledgments; + } + + public Duration getBufferTimeout() { + return bufferTimeout; + } + + public List getQueues() { + return queues; + } +} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorker.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorker.java new file mode 100644 index 0000000000..3f58906b33 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorker.java @@ -0,0 +1,326 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import com.linecorp.armeria.client.retry.Backoff; +import io.micrometer.core.instrument.Counter; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import software.amazon.awssdk.core.exception.SdkException; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityRequest; +import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequest; +import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequestEntry; +import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchResponse; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest; +import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse; +import software.amazon.awssdk.services.sqs.model.SqsException; +import software.amazon.awssdk.services.sts.model.StsException; +import org.opensearch.dataprepper.model.buffer.Buffer; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + + +public class SqsWorker implements Runnable { + private static final Logger LOG = LoggerFactory.getLogger(SqsWorker.class); + static final String SQS_MESSAGES_RECEIVED_METRIC_NAME = "sqsMessagesReceived"; + static final String SQS_MESSAGES_DELETED_METRIC_NAME = "sqsMessagesDeleted"; + static final String SQS_MESSAGES_FAILED_METRIC_NAME = "sqsMessagesFailed"; + static final String SQS_MESSAGES_DELETE_FAILED_METRIC_NAME = "sqsMessagesDeleteFailed"; + static final String SQS_VISIBILITY_TIMEOUT_CHANGED_COUNT_METRIC_NAME = "sqsVisibilityTimeoutChangedCount"; + static final String SQS_VISIBILITY_TIMEOUT_CHANGE_FAILED_COUNT_METRIC_NAME = "sqsVisibilityTimeoutChangeFailedCount"; + static final String ACKNOWLEDGEMENT_SET_CALLACK_METRIC_NAME = "acknowledgementSetCallbackCounter"; + + private final SqsClient sqsClient; + private final SqsEventProcessor sqsEventProcessor; + private final Counter sqsMessagesReceivedCounter; + private final Counter sqsMessagesDeletedCounter; + private final Counter sqsMessagesFailedCounter; + private final Counter sqsMessagesDeleteFailedCounter; + private final Counter acknowledgementSetCallbackCounter; + private final Counter sqsVisibilityTimeoutChangedCount; + private final Counter sqsVisibilityTimeoutChangeFailedCount; + private final Backoff standardBackoff; + private final QueueConfig queueConfig; + private int failedAttemptCount; + private final boolean endToEndAcknowledgementsEnabled; + private final AcknowledgementSetManager acknowledgementSetManager; + private volatile boolean isStopped = false; + private final Buffer> buffer; + private final int bufferTimeoutMillis; + private Map messageVisibilityTimesMap; + + public SqsWorker(final Buffer> buffer, + final AcknowledgementSetManager acknowledgementSetManager, + final SqsClient sqsClient, + final SqsEventProcessor sqsEventProcessor, + final SqsSourceConfig sqsSourceConfig, + final QueueConfig queueConfig, + final PluginMetrics pluginMetrics, + final Backoff backoff) { + + this.sqsClient = sqsClient; + this.sqsEventProcessor = sqsEventProcessor; + this.queueConfig = queueConfig; + this.acknowledgementSetManager = acknowledgementSetManager; + this.standardBackoff = backoff; + this.endToEndAcknowledgementsEnabled = sqsSourceConfig.getAcknowledgements(); + this.buffer = buffer; + this.bufferTimeoutMillis = (int) sqsSourceConfig.getBufferTimeout().toMillis(); + + messageVisibilityTimesMap = new HashMap<>(); + failedAttemptCount = 0; + sqsMessagesReceivedCounter = pluginMetrics.counter(SQS_MESSAGES_RECEIVED_METRIC_NAME); + sqsMessagesDeletedCounter = pluginMetrics.counter(SQS_MESSAGES_DELETED_METRIC_NAME); + sqsMessagesFailedCounter = pluginMetrics.counter(SQS_MESSAGES_FAILED_METRIC_NAME); + sqsMessagesDeleteFailedCounter = pluginMetrics.counter(SQS_MESSAGES_DELETE_FAILED_METRIC_NAME); + acknowledgementSetCallbackCounter = pluginMetrics.counter(ACKNOWLEDGEMENT_SET_CALLACK_METRIC_NAME); + sqsVisibilityTimeoutChangedCount = pluginMetrics.counter(SQS_VISIBILITY_TIMEOUT_CHANGED_COUNT_METRIC_NAME); + sqsVisibilityTimeoutChangeFailedCount = pluginMetrics.counter(SQS_VISIBILITY_TIMEOUT_CHANGE_FAILED_COUNT_METRIC_NAME); + } + + @Override + public void run() { + while (!isStopped) { + int messagesProcessed = 0; + try { + messagesProcessed = processSqsMessages(); + + } catch (final Exception e) { + LOG.error("Unable to process SQS messages. Processing error due to: {}", e.getMessage()); + applyBackoff(); + } + + if (messagesProcessed > 0 && queueConfig.getPollDelay().toMillis() > 0) { + try { + Thread.sleep(queueConfig.getPollDelay().toMillis()); + } catch (final InterruptedException e) { + LOG.error("Thread is interrupted while polling SQS.", e); + } + } + } + } + + int processSqsMessages() { + final List messages = getMessagesFromSqs(); + if (!messages.isEmpty()) { + sqsMessagesReceivedCounter.increment(messages.size()); + final List deleteMessageBatchRequestEntries = processSqsEvents(messages); + if (!deleteMessageBatchRequestEntries.isEmpty()) { + deleteSqsMessages(deleteMessageBatchRequestEntries); + } + } + return messages.size(); + } + + private List getMessagesFromSqs() { + try { + final ReceiveMessageRequest request = createReceiveMessageRequest(); + final ReceiveMessageResponse response = sqsClient.receiveMessage(request); + List messages = response.messages(); + failedAttemptCount = 0; + return messages; + + } catch (final SqsException | StsException e) { + LOG.error("Error reading from SQS: {}. Retrying with exponential backoff.", e.getMessage()); + applyBackoff(); + return Collections.emptyList(); + } + } + + private void applyBackoff() { + final long delayMillis = standardBackoff.nextDelayMillis(++failedAttemptCount); + if (delayMillis < 0) { + Thread.currentThread().interrupt(); + throw new SqsRetriesExhaustedException("SQS retries exhausted. Make sure that SQS configuration is valid, SQS queue exists, and IAM role has required permissions."); + } + final Duration delayDuration = Duration.ofMillis(delayMillis); + LOG.info("Pausing SQS processing for {}.{} seconds due to an error in processing.", + delayDuration.getSeconds(), delayDuration.toMillisPart()); + try { + Thread.sleep(delayMillis); + } catch (final InterruptedException e){ + LOG.error("Thread is interrupted while polling SQS with retry.", e); + } + } + + private ReceiveMessageRequest createReceiveMessageRequest() { + ReceiveMessageRequest.Builder requestBuilder = ReceiveMessageRequest.builder() + .queueUrl(queueConfig.getUrl()) + .attributeNamesWithStrings("All") + .messageAttributeNames("All"); + + if (queueConfig.getWaitTime() != null) { + requestBuilder.waitTimeSeconds((int) queueConfig.getWaitTime().getSeconds()); + } + if (queueConfig.getMaximumMessages() != null) { + requestBuilder.maxNumberOfMessages(queueConfig.getMaximumMessages()); + } + if (queueConfig.getVisibilityTimeout() != null) { + requestBuilder.visibilityTimeout((int) queueConfig.getVisibilityTimeout().getSeconds()); + } + return requestBuilder.build(); + } + + private List processSqsEvents(final List messages) { + final List deleteMessageBatchRequestEntryCollection = new ArrayList<>(); + final Map messageAcknowledgementSetMap = new HashMap<>(); + final Map> messageWaitingForAcknowledgementsMap = new HashMap<>(); + + for (Message message : messages) { + List waitingForAcknowledgements = new ArrayList<>(); + AcknowledgementSet acknowledgementSet = null; + + final int visibilityTimeout; + if (queueConfig.getVisibilityTimeout() != null) { + visibilityTimeout = (int) queueConfig.getVisibilityTimeout().getSeconds(); + } else { + visibilityTimeout = (int) Duration.ofSeconds(30).getSeconds(); + + } + + final int maxVisibilityTimeout = (int)queueConfig.getVisibilityDuplicateProtectionTimeout().getSeconds(); + final int progressCheckInterval = visibilityTimeout/2 - 1; + if (endToEndAcknowledgementsEnabled) { + int expiryTimeout = visibilityTimeout - 2; + final boolean visibilityDuplicateProtectionEnabled = queueConfig.getVisibilityDuplicateProtection(); + if (visibilityDuplicateProtectionEnabled) { + expiryTimeout = maxVisibilityTimeout; + } + acknowledgementSet = acknowledgementSetManager.create( + (result) -> { + acknowledgementSetCallbackCounter.increment(); + // Delete only if this is positive acknowledgement + if (visibilityDuplicateProtectionEnabled) { + messageVisibilityTimesMap.remove(message); + } + if (result) { + deleteSqsMessages(waitingForAcknowledgements); + } + }, + Duration.ofSeconds(expiryTimeout)); + if (visibilityDuplicateProtectionEnabled) { + acknowledgementSet.addProgressCheck( + (ratio) -> { + int newValue = messageVisibilityTimesMap.getOrDefault(message, visibilityTimeout) + progressCheckInterval; + if (newValue >= maxVisibilityTimeout) { + return; + } + messageVisibilityTimesMap.put(message, newValue); + final int newVisibilityTimeoutSeconds = visibilityTimeout; + increaseVisibilityTimeout(message, newVisibilityTimeoutSeconds); + }, + Duration.ofSeconds(progressCheckInterval)); + } + messageAcknowledgementSetMap.put(message, acknowledgementSet); + messageWaitingForAcknowledgementsMap.put(message, waitingForAcknowledgements); + } + } + + if (endToEndAcknowledgementsEnabled) { + LOG.debug("Created acknowledgement sets for {} messages.", messages.size()); + } + + for (Message message : messages) { + final AcknowledgementSet acknowledgementSet = messageAcknowledgementSetMap.get(message); + final List waitingForAcknowledgements = messageWaitingForAcknowledgementsMap.get(message); + final Optional deleteMessageBatchRequestEntry = processSqsObject(message, acknowledgementSet); + if (endToEndAcknowledgementsEnabled) { + deleteMessageBatchRequestEntry.ifPresent(waitingForAcknowledgements::add); + acknowledgementSet.complete(); + } else { + deleteMessageBatchRequestEntry.ifPresent(deleteMessageBatchRequestEntryCollection::add); + } + } + + return deleteMessageBatchRequestEntryCollection; + } + + + private Optional processSqsObject( + final Message message, + final AcknowledgementSet acknowledgementSet) { + try { + sqsEventProcessor.addSqsObject(message, queueConfig.getUrl(), buffer, bufferTimeoutMillis, acknowledgementSet); + return Optional.of(buildDeleteMessageBatchRequestEntry(message)); + } catch (final Exception e) { + sqsMessagesFailedCounter.increment(); + LOG.error("Error processing from SQS: {}. Retrying with exponential backoff.", e.getMessage()); + applyBackoff(); + return Optional.empty(); + } + } + + private void increaseVisibilityTimeout(final Message message, final int newVisibilityTimeoutSeconds) { + if(isStopped) { + LOG.info("Some messages are pending completion of acknowledgments. Data Prepper will not increase the visibility timeout because it is shutting down. {}", message); + return; + } + final ChangeMessageVisibilityRequest changeMessageVisibilityRequest = ChangeMessageVisibilityRequest.builder() + .visibilityTimeout(newVisibilityTimeoutSeconds) + .queueUrl(queueConfig.getUrl()) + .receiptHandle(message.receiptHandle()) + .build(); + + try { + sqsClient.changeMessageVisibility(changeMessageVisibilityRequest); + sqsVisibilityTimeoutChangedCount.increment(); + LOG.debug("Set visibility timeout for message {} to {}", message.messageId(), newVisibilityTimeoutSeconds); + } catch (Exception e) { + LOG.error("Failed to set visibility timeout for message {} to {}", message.messageId(), newVisibilityTimeoutSeconds, e); + sqsVisibilityTimeoutChangeFailedCount.increment(); + } + } + + + private DeleteMessageBatchRequestEntry buildDeleteMessageBatchRequestEntry(Message message) { + return DeleteMessageBatchRequestEntry.builder() + .id(message.messageId()) + .receiptHandle(message.receiptHandle()) + .build(); + } + + private void deleteSqsMessages(final List deleteEntries) { + if (deleteEntries.isEmpty()) return; + + try { + DeleteMessageBatchRequest deleteRequest = DeleteMessageBatchRequest.builder() + .queueUrl(queueConfig.getUrl()) + .entries(deleteEntries) + .build(); + DeleteMessageBatchResponse response = sqsClient.deleteMessageBatch(deleteRequest); + + if (response.hasSuccessful()) { + int successfulDeletes = response.successful().size(); + sqsMessagesDeletedCounter.increment(successfulDeletes); + } + if (response.hasFailed()) { + int failedDeletes = response.failed().size(); + sqsMessagesDeleteFailedCounter.increment(failedDeletes); + LOG.error("Failed to delete {} messages from SQS.", failedDeletes); + } + } catch (SdkException e) { + LOG.error("Failed to delete messages from SQS: {}", e.getMessage()); + sqsMessagesDeleteFailedCounter.increment(deleteEntries.size()); + } + } + + void stop() { + isStopped = true; + } +} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSource.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSource.java deleted file mode 100644 index 21888e756b..0000000000 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSource.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.source.sqssource; - -import com.linecorp.armeria.client.retry.Backoff; -import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; -import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; -import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; -import org.opensearch.dataprepper.model.buffer.Buffer; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.model.source.Source; -import org.opensearch.dataprepper.plugins.aws.sqs.common.ClientFactory; -import org.opensearch.dataprepper.plugins.aws.sqs.common.SqsService; -import org.opensearch.dataprepper.plugins.aws.sqs.common.handler.SqsMessageHandler; -import org.opensearch.dataprepper.plugins.aws.sqs.common.metrics.SqsMetrics; -import org.opensearch.dataprepper.plugins.aws.sqs.common.model.SqsOptions; -import org.opensearch.dataprepper.plugins.source.sqssource.config.SqsSourceConfig; -import org.opensearch.dataprepper.plugins.source.sqssource.handler.RawSqsMessageHandler; -import software.amazon.awssdk.services.sqs.SqsClient; - -import java.time.Duration; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; - -@DataPrepperPlugin(name = "sqs", pluginType = Source.class,pluginConfigurationType = SqsSourceConfig.class) -public class SqsSource implements Source> { - - static final long INITIAL_DELAY = Duration.ofSeconds(20).toMillis(); - - static final long MAXIMUM_DELAY = Duration.ofMinutes(5).toMillis(); - - static final Duration BUFFER_TIMEOUT = Duration.ofSeconds(10); - - static final int NO_OF_RECORDS_TO_ACCUMULATE = 100; - - static final double JITTER_RATE = 0.20; - - private final SqsSourceConfig sqsSourceConfig; - - private final AcknowledgementSetManager acknowledgementSetManager; - - private final PluginMetrics pluginMetrics; - - private final boolean acknowledgementsEnabled; - - private final AwsCredentialsSupplier awsCredentialsSupplier; - - private final ScheduledExecutorService scheduledExecutorService; - - @DataPrepperPluginConstructor - public SqsSource(final PluginMetrics pluginMetrics, - final SqsSourceConfig sqsSourceConfig, - final AcknowledgementSetManager acknowledgementSetManager, - final AwsCredentialsSupplier awsCredentialsSupplier) { - this.sqsSourceConfig = sqsSourceConfig; - this.acknowledgementSetManager = acknowledgementSetManager; - this.acknowledgementsEnabled = sqsSourceConfig.getAcknowledgements(); - this.pluginMetrics = pluginMetrics; - this.awsCredentialsSupplier = awsCredentialsSupplier; - this.scheduledExecutorService = Executors.newScheduledThreadPool(sqsSourceConfig.getNumberOfThreads()); - } - - @Override - public void start(Buffer> buffer) { - if (buffer == null) { - throw new IllegalStateException("Buffer is null"); - } - - final SqsMetrics sqsMetrics = new SqsMetrics(pluginMetrics); - - final SqsClient sqsClient = ClientFactory.createSqsClient(sqsSourceConfig.getAws().getAwsRegion(), - sqsSourceConfig.getAws().getAwsStsRoleArn(), - sqsSourceConfig.getAws().getAwsStsHeaderOverrides(), - awsCredentialsSupplier); - - final Backoff backoff = Backoff.exponential(INITIAL_DELAY, MAXIMUM_DELAY).withJitter(JITTER_RATE) - .withMaxAttempts(Integer.MAX_VALUE); - final SqsService sqsService = new SqsService(sqsMetrics,sqsClient,backoff); - - final SqsMessageHandler sqsHandler = new RawSqsMessageHandler(sqsService); - final SqsOptions.Builder sqsOptionsBuilder = new SqsOptions.Builder() - .setPollDelay(sqsSourceConfig.getPollingFrequency()) - .setVisibilityTimeout(sqsSourceConfig.getVisibilityTimeout()) - .setWaitTime(sqsSourceConfig.getWaitTime()) - .setMaximumMessages(sqsSourceConfig.getBatchSize()); - final long pollingFrequencyInMillis = sqsSourceConfig.getPollingFrequency().toMillis(); - sqsSourceConfig.getUrls().forEach(url -> { - scheduledExecutorService.scheduleAtFixedRate(new SqsSourceTask(buffer,NO_OF_RECORDS_TO_ACCUMULATE,BUFFER_TIMEOUT, - sqsService, - sqsOptionsBuilder.setSqsUrl(url).build(), - sqsMetrics, - acknowledgementSetManager, - sqsSourceConfig.getAcknowledgements(), - sqsHandler) - ,0, pollingFrequencyInMillis == 0 ? 1 : pollingFrequencyInMillis, - TimeUnit.MILLISECONDS); - }); - } - - @Override - public boolean areAcknowledgementsEnabled() { - return acknowledgementsEnabled; - } - - @Override - public void stop() { - scheduledExecutorService.shutdown(); - } -} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTask.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTask.java deleted file mode 100644 index 0d23b69172..0000000000 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTask.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.source.sqssource; - -import org.opensearch.dataprepper.buffer.common.BufferAccumulator; -import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; -import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; -import org.opensearch.dataprepper.model.buffer.Buffer; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.aws.sqs.common.SqsService; -import org.opensearch.dataprepper.plugins.aws.sqs.common.handler.SqsMessageHandler; -import org.opensearch.dataprepper.plugins.aws.sqs.common.metrics.SqsMetrics; -import org.opensearch.dataprepper.plugins.aws.sqs.common.model.SqsOptions; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequestEntry; -import software.amazon.awssdk.services.sqs.model.Message; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; - -/** - * Class responsible for processing the sqs message with the help of SqsMessageHandler - * - */ -public class SqsSourceTask implements Runnable{ - - private static final Logger LOG = LoggerFactory.getLogger(SqsSourceTask.class); - - private final SqsService sqsService; - - private final SqsOptions sqsOptions; - - private final SqsMetrics sqsMetrics; - - private final AcknowledgementSetManager acknowledgementSetManager; - - private final boolean endToEndAcknowledgementsEnabled; - - private final SqsMessageHandler sqsHandler; - - private final BufferAccumulator> bufferAccumulator; - - public SqsSourceTask(final Buffer> buffer, - final int noOfRecordsToAccumulate, - final Duration bufferTimeout, - final SqsService sqsService, - final SqsOptions sqsOptions, - final SqsMetrics sqsMetrics, - final AcknowledgementSetManager acknowledgementSetManager, - final boolean endToEndAcknowledgementsEnabled, - final SqsMessageHandler sqsHandler) { - this.sqsService = sqsService; - this.sqsOptions = sqsOptions; - this.sqsMetrics = sqsMetrics; - this.acknowledgementSetManager = acknowledgementSetManager; - this.endToEndAcknowledgementsEnabled = endToEndAcknowledgementsEnabled; - this.sqsHandler = sqsHandler; - this.bufferAccumulator = BufferAccumulator.create(buffer,noOfRecordsToAccumulate,bufferTimeout); - } - - /** - * read the messages from sqs queue and push the message into buffer in a loop. - */ - @Override - public void run() { - try { - processSqsMessages(); - } catch (final Exception e) { - LOG.error("Unable to process SQS messages. Processing error due to: {}", e.getMessage()); - sqsService.applyBackoff(); - } - } - - /** - * read the messages from sqs queue and push the message into buffer and finally will delete - * the sqs message from queue after successful buffer push. - */ - void processSqsMessages() { - AcknowledgementSet acknowledgementSet = null; - List deleteMessageBatchRequestEntries = null; - final List waitingForAcknowledgements = new ArrayList<>(); - - if(endToEndAcknowledgementsEnabled) - acknowledgementSet = sqsService.createAcknowledgementSet(sqsOptions.getSqsUrl(), - acknowledgementSetManager, - waitingForAcknowledgements); - - final List messages = sqsService.getMessagesFromSqs(sqsOptions); - - if(!messages.isEmpty()) { - LOG.info("Thread Name : {} , messages processed: {}",Thread.currentThread().getName(),messages.size()); - sqsMetrics.getSqsMessagesReceivedCounter().increment(); - try { - deleteMessageBatchRequestEntries = sqsHandler.handleMessages(messages, bufferAccumulator, acknowledgementSet); - } catch(final Exception e) { - LOG.error("Error while processing handleMessages : ",e); - sqsService.applyBackoff(); - } - if(deleteMessageBatchRequestEntries != null) { - if (endToEndAcknowledgementsEnabled) - waitingForAcknowledgements.addAll(deleteMessageBatchRequestEntries); - else - sqsService.deleteMessagesFromQueue(deleteMessageBatchRequestEntries, sqsOptions.getSqsUrl()); - } - } - } -} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/config/AwsAuthenticationOptions.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/config/AwsAuthenticationOptions.java deleted file mode 100644 index c5c71f585d..0000000000 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/config/AwsAuthenticationOptions.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.source.sqssource.config; - -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.constraints.AssertTrue; -import jakarta.validation.constraints.Size; -import software.amazon.awssdk.arns.Arn; -import software.amazon.awssdk.regions.Region; - -import java.util.Map; -import java.util.Optional; - - -public class AwsAuthenticationOptions { - - private static final String AWS_IAM_ROLE = "role"; - - private static final String AWS_IAM = "iam"; - - @JsonProperty("sts_region") - @Size(min = 1, message = "Region cannot be empty string") - private String awsRegion; - - @JsonProperty("sts_role_arn") - @Size(min = 20, max = 2048, message = "awsStsRoleArn length should be between 1 and 2048 characters") - private String awsStsRoleArn; - - @JsonProperty("sts_header_overrides") - @Size(max = 5, message = "sts_header_overrides supports a maximum of 5 headers to override") - private Map awsStsHeaderOverrides; - - - - @AssertTrue(message = "sts_role_arn must be an IAM Role") - boolean isValidStsRoleArn() { - final Arn arn = getArn(); - boolean status = true; - if (!AWS_IAM.equals(arn.service())) { - status = false; - } - final Optional resourceType = arn.resource().resourceType(); - if (resourceType.isEmpty() || !resourceType.get().equals(AWS_IAM_ROLE)) { - status = false; - } - return status; - } - - private Arn getArn() { - try { - return Arn.fromString(awsStsRoleArn); - } catch (final Exception e) { - throw new IllegalArgumentException(String.format("Invalid ARN format for awsStsRoleArn. Check the format of %s", awsStsRoleArn)); - } - } - - public String getAwsStsRoleArn() { - return awsStsRoleArn; - } - - public Region getAwsRegion() { - return awsRegion != null ? Region.of(awsRegion) : null; - } - - public Map getAwsStsHeaderOverrides() { - return awsStsHeaderOverrides; - } -} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/config/SqsSourceConfig.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/config/SqsSourceConfig.java deleted file mode 100644 index 8ff8e531f7..0000000000 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/config/SqsSourceConfig.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.source.sqssource.config; - -import com.fasterxml.jackson.annotation.JsonAlias; -import com.fasterxml.jackson.annotation.JsonProperty; -import jakarta.validation.Valid; -import jakarta.validation.constraints.Max; -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Positive; - -import java.time.Duration; -import java.util.List; - -public class SqsSourceConfig { - @Valid - @JsonProperty("aws") - private AwsAuthenticationOptions aws; - - @JsonProperty("acknowledgments") - private boolean acknowledgments = false; - - @JsonProperty("visibility_timeout") - private Duration visibilityTimeout; - - @JsonProperty("wait_time") - private Duration waitTime; - - @JsonAlias({"queue_url", "queue_urls"}) - @NotNull - private List urls; - - @JsonProperty("polling_frequency") - private Duration pollingFrequency = Duration.ZERO; - - @JsonProperty("batch_size") - @Max(10) - private Integer batchSize = 10; - - @JsonProperty("number_of_threads") - @Positive(message = "number_of_threads should be unsigned value") - private Integer numberOfThreads = 1; - - public List getUrls() { - return urls; - } - - public Duration getPollingFrequency() { - return pollingFrequency; - } - - public Integer getBatchSize() { - return batchSize; - } - - public Integer getNumberOfThreads() { - return numberOfThreads; - } - - public AwsAuthenticationOptions getAws() { - return aws; - } - - public boolean getAcknowledgements() { - return acknowledgments; - } - - public Duration getVisibilityTimeout() { - return visibilityTimeout; - } - - public Duration getWaitTime() { - return waitTime; - } -} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/handler/RawSqsMessageHandler.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/handler/RawSqsMessageHandler.java deleted file mode 100644 index 844eecb97b..0000000000 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqssource/handler/RawSqsMessageHandler.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.source.sqssource.handler; - -import org.opensearch.dataprepper.buffer.common.BufferAccumulator; -import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.JacksonEvent; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.aws.sqs.common.SqsService; -import org.opensearch.dataprepper.plugins.aws.sqs.common.handler.SqsMessageHandler; -import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequestEntry; -import software.amazon.awssdk.services.sqs.model.Message; - -import java.util.List; -import java.util.Objects; - -/** - * implements the SqsMessageHandler to read and parse the sqs message and push to buffer. - * - */ -public class RawSqsMessageHandler implements SqsMessageHandler { - - private final SqsService sqsService; - - public RawSqsMessageHandler(final SqsService sqsService){ - this.sqsService = sqsService; - } - - /** - * helps to send end to end acknowledgements after successful processing. - * - * @param messages - list of sqs messages for processing - * @return AcknowledgementSet - will generate the AcknowledgementSet if endToEndAcknowledgementsEnabled is true else null - */ - @Override - public List handleMessages(final List messages, - final BufferAccumulator> bufferAccumulator, - final AcknowledgementSet acknowledgementSet) { - messages.forEach(message -> { - final Record eventRecord = new Record(JacksonEvent.fromMessage(message.body())); - try { - // Always add record to acknowledgementSet before adding to - // buffer because another thread may take and process - // buffer contents before the event record is added - // to acknowledgement set - if(Objects.nonNull(acknowledgementSet)){ - acknowledgementSet.add(eventRecord.getData()); - } - bufferAccumulator.add(eventRecord); - } catch (Exception e) { - // Exception may occur when we failed to flush. In which - // case, not sending acknowledgement would be correct because - // we need to retry - throw new RuntimeException(e); - } - }); - try { - bufferAccumulator.flush(); - } catch (final Exception e) { - throw new RuntimeException(e); - } - return sqsService.getDeleteMessageBatchRequestEntryList(messages); - } -} diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapterTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapterTest.java new file mode 100644 index 0000000000..04806ff4d3 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapterTest.java @@ -0,0 +1,90 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.aws.api.AwsCredentialsOptions; +import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.regions.Region; + +import java.util.Collections; +import java.util.Map; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.mock; + + +@ExtendWith(MockitoExtension.class) +class AwsAuthenticationAdapterTest { + @Mock + private AwsCredentialsSupplier awsCredentialsSupplier; + @Mock + private SqsSourceConfig sqsSourceConfig; + + @Mock + private AwsAuthenticationOptions awsAuthenticationOptions; + private String stsRoleArn; + + @BeforeEach + void setUp() { + when(sqsSourceConfig.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions); + + stsRoleArn = UUID.randomUUID().toString(); + when(awsAuthenticationOptions.getAwsStsRoleArn()).thenReturn(stsRoleArn); + } + + private AwsAuthenticationAdapter createObjectUnderTest() { + return new AwsAuthenticationAdapter(awsCredentialsSupplier, sqsSourceConfig); + } + + @Test + void getCredentialsProvider_returns_AwsCredentialsProvider_from_AwsCredentialsSupplier() { + final AwsCredentialsProvider expectedProvider = mock(AwsCredentialsProvider.class); + when(awsCredentialsSupplier.getProvider(any(AwsCredentialsOptions.class))) + .thenReturn(expectedProvider); + + assertThat(createObjectUnderTest().getCredentialsProvider(), equalTo(expectedProvider)); + } + + @ParameterizedTest + @ValueSource(strings = {"us-east-1", "eu-west-1"}) + void getCredentialsProvider_creates_expected_AwsCredentialsOptions(final String regionString) { + final String externalId = UUID.randomUUID().toString(); + final Region region = Region.of(regionString); + + final Map headerOverrides = Collections.singletonMap(UUID.randomUUID().toString(), UUID.randomUUID().toString()); + when(awsAuthenticationOptions.getAwsStsExternalId()).thenReturn(externalId); + when(awsAuthenticationOptions.getAwsRegion()).thenReturn(region); + when(awsAuthenticationOptions.getAwsStsHeaderOverrides()).thenReturn(headerOverrides); + + createObjectUnderTest().getCredentialsProvider(); + + final ArgumentCaptor credentialsOptionsArgumentCaptor = ArgumentCaptor.forClass(AwsCredentialsOptions.class); + verify(awsCredentialsSupplier).getProvider(credentialsOptionsArgumentCaptor.capture()); + + final AwsCredentialsOptions actualOptions = credentialsOptionsArgumentCaptor.getValue(); + + assertThat(actualOptions, notNullValue()); + assertThat(actualOptions.getStsRoleArn(), equalTo(stsRoleArn)); + assertThat(actualOptions.getStsExternalId(), equalTo(externalId)); + assertThat(actualOptions.getRegion(), equalTo(region)); + assertThat(actualOptions.getStsHeaderOverrides(), equalTo(headerOverrides)); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptionsTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptionsTest.java new file mode 100644 index 0000000000..77eeeb519a --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptionsTest.java @@ -0,0 +1,136 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import software.amazon.awssdk.arns.Arn; +import software.amazon.awssdk.regions.Region; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.Map; +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.nullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; + +class AwsAuthenticationOptionsTest { + + private AwsAuthenticationOptions awsAuthenticationOptions; + + @BeforeEach + void setUp() { + awsAuthenticationOptions = new AwsAuthenticationOptions(); + } + + @Test + void getAwsRegion_returns_Region_of() throws NoSuchFieldException, IllegalAccessException { + final String regionString = UUID.randomUUID().toString(); + final Region expectedRegionObject = mock(Region.class); + reflectivelySetField(awsAuthenticationOptions, "awsRegion", regionString); + final Region actualRegion; + try (final MockedStatic regionMockedStatic = mockStatic(Region.class)) { + regionMockedStatic.when(() -> Region.of(regionString)).thenReturn(expectedRegionObject); + actualRegion = awsAuthenticationOptions.getAwsRegion(); + } + assertThat(actualRegion, equalTo(expectedRegionObject)); + } + + @Test + void getAwsRegion_returns_null_when_region_is_null() throws NoSuchFieldException, IllegalAccessException { + reflectivelySetField(awsAuthenticationOptions, "awsRegion", null); + assertThat(awsAuthenticationOptions.getAwsRegion(), nullValue()); + } + + @Test + void getStsExternalId_notNull() throws NoSuchFieldException, IllegalAccessException { + final String externalId = UUID.randomUUID().toString(); + reflectivelySetField(awsAuthenticationOptions, "awsStsExternalId", externalId); + assertThat(awsAuthenticationOptions.getAwsStsExternalId(), equalTo(externalId)); + } + + @Test + void getStsExternalId_Null() throws NoSuchFieldException, IllegalAccessException { + reflectivelySetField(awsAuthenticationOptions, "awsStsExternalId", null); + assertThat(awsAuthenticationOptions.getAwsStsExternalId(), nullValue()); + } + + @Test + void getAwsStsRoleArn_returns_correct_arn() throws NoSuchFieldException, IllegalAccessException { + final String stsRoleArn = "arn:aws:iam::123456789012:role/SampleRole"; + reflectivelySetField(awsAuthenticationOptions, "awsStsRoleArn", stsRoleArn); + assertThat(awsAuthenticationOptions.getAwsStsRoleArn(), equalTo(stsRoleArn)); + } + + @Test + void getAwsStsHeaderOverrides_returns_correct_map() throws NoSuchFieldException, IllegalAccessException { + Map headerOverrides = new HashMap<>(); + headerOverrides.put("header1", "value1"); + headerOverrides.put("header2", "value2"); + reflectivelySetField(awsAuthenticationOptions, "awsStsHeaderOverrides", headerOverrides); + assertThat(awsAuthenticationOptions.getAwsStsHeaderOverrides(), equalTo(headerOverrides)); + } + + @Test + void validateStsRoleArn_with_invalid_format_throws_exception() throws NoSuchFieldException, IllegalAccessException { + final String invalidFormatArn = "invalid-arn-format"; + reflectivelySetField(awsAuthenticationOptions, "awsStsRoleArn", invalidFormatArn); + + try (final MockedStatic arnMockedStatic = mockStatic(Arn.class)) { + arnMockedStatic.when(() -> Arn.fromString(invalidFormatArn)) + .thenThrow(new IllegalArgumentException("The value provided for sts_role_arn is not a valid AWS ARN. Provided value: " + invalidFormatArn)); + + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> { + awsAuthenticationOptions.validateStsRoleArn(); + }); + assertThat(exception.getMessage(), equalTo("The value provided for sts_role_arn is not a valid AWS ARN. Provided value: " + invalidFormatArn)); + } + } + + @Test + void validateStsRoleArn_does_not_throw_for_valid_role_Arn() throws NoSuchFieldException, IllegalAccessException { + final String validRoleArn = "arn:aws:iam::123456789012:role/SampleRole"; + reflectivelySetField(awsAuthenticationOptions, "awsStsRoleArn", validRoleArn); + try { + awsAuthenticationOptions.validateStsRoleArn(); + } catch (Exception e) { + throw new AssertionError("Exception should not be thrown for a valid role ARN", e); + } + } + + @Test + void validateStsRoleArn_throws_exception_for_non_role_resource() throws NoSuchFieldException, IllegalAccessException { + final String nonRoleResourceArn = "arn:aws:iam::123456789012:group/MyGroup"; + reflectivelySetField(awsAuthenticationOptions, "awsStsRoleArn", nonRoleResourceArn); + Exception exception = assertThrows(IllegalArgumentException.class, () -> awsAuthenticationOptions.validateStsRoleArn()); + assertThat(exception.getMessage(), equalTo("sts_role_arn must be an IAM Role")); + } + + @Test + void validateStsRoleArn_throws_exception_when_service_is_not_iam() throws NoSuchFieldException, IllegalAccessException { + final String invalidServiceArn = "arn:aws:s3::123456789012:role/SampleRole"; + reflectivelySetField(awsAuthenticationOptions, "awsStsRoleArn", invalidServiceArn); + Exception exception = assertThrows(IllegalArgumentException.class, () -> awsAuthenticationOptions.validateStsRoleArn()); + assertThat(exception.getMessage(), equalTo("sts_role_arn must be an IAM Role")); + } + + + private void reflectivelySetField(final AwsAuthenticationOptions awsAuthenticationOptions, final String fieldName, final Object value) throws NoSuchFieldException, IllegalAccessException { + final Field field = AwsAuthenticationOptions.class.getDeclaredField(fieldName); + try { + field.setAccessible(true); + field.set(awsAuthenticationOptions, value); + } finally { + field.setAccessible(false); + } + } +} diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfigTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfigTest.java new file mode 100644 index 0000000000..f312d8abc6 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfigTest.java @@ -0,0 +1,31 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.junit.jupiter.api.Test; +import java.time.Duration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class QueueConfigTest { + + @Test + void testDefaultValues() { + final QueueConfig queueConfig = new QueueConfig(); + + assertNull(queueConfig.getUrl(), "URL should be null by default"); + assertEquals(1, queueConfig.getNumWorkers(), "Number of workers should default to 1"); + assertNull(queueConfig.getMaximumMessages(), "Maximum messages should be null by default"); + assertEquals(Duration.ofSeconds(0), queueConfig.getPollDelay(), "Poll delay should default to 0 seconds"); + assertNull(queueConfig.getVisibilityTimeout(), "Visibility timeout should be null by default"); + assertFalse(queueConfig.getVisibilityDuplicateProtection(), "Visibility duplicate protection should default to false"); + assertEquals(Duration.ofHours(2), queueConfig.getVisibilityDuplicateProtectionTimeout(), + "Visibility duplicate protection timeout should default to 2 hours"); + assertNull(queueConfig.getWaitTime(), "Wait time should default to null"); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandlerTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandlerTest.java new file mode 100644 index 0000000000..4606df45c6 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandlerTest.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import software.amazon.awssdk.services.sqs.model.Message; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +class RawSqsMessageHandlerTest { + + private final RawSqsMessageHandler rawSqsMessageHandler = new RawSqsMessageHandler(); + private Buffer> mockBuffer; + private int mockBufferTimeoutMillis; + + @BeforeEach + void setUp() { + mockBuffer = mock(Buffer.class); + mockBufferTimeoutMillis = 10000; + } + + @Test + void handleMessage_callsBufferWriteOnce() throws Exception { + Message message = Message.builder().body("{\"key\":\"value\"}").build(); + String queueUrl = "https://sqs.us-east-1.amazonaws.com/123456789012/test-queue"; + rawSqsMessageHandler.handleMessage(message, queueUrl, mockBuffer, mockBufferTimeoutMillis, null); + ArgumentCaptor> argumentCaptor = ArgumentCaptor.forClass(Record.class); + verify(mockBuffer, times(1)).write(argumentCaptor.capture(), eq(mockBufferTimeoutMillis)); + Record capturedRecord = argumentCaptor.getValue(); + assertEquals("DOCUMENT", capturedRecord.getData().getMetadata().getEventType(), "Event type should be 'DOCUMENT'"); + } +} diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessorTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessorTest.java new file mode 100644 index 0000000000..e10b8f471f --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessorTest.java @@ -0,0 +1,60 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import software.amazon.awssdk.services.sqs.model.Message; + +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +class SqsEventProcessorTest { + + private SqsMessageHandler mockSqsMessageHandler; + private SqsEventProcessor sqsEventProcessor; + private Buffer> mockBuffer; + private int mockBufferTimeoutMillis; + private AcknowledgementSet mockAcknowledgementSet; + + @BeforeEach + void setUp() { + mockSqsMessageHandler = Mockito.mock(SqsMessageHandler.class); + mockBuffer = Mockito.mock(Buffer.class); + mockAcknowledgementSet = Mockito.mock(AcknowledgementSet.class); + mockBufferTimeoutMillis = 10000; + sqsEventProcessor = new SqsEventProcessor(mockSqsMessageHandler); + } + + @Test + void addSqsObject_callsHandleMessageWithCorrectParameters() throws IOException { + Message message = Message.builder().body("Test Message Body").build(); + String queueUrl = "https://sqs.us-east-1.amazonaws.com/123456789012/test-queue"; + sqsEventProcessor.addSqsObject(message, queueUrl, mockBuffer, mockBufferTimeoutMillis, mockAcknowledgementSet); + verify(mockSqsMessageHandler, times(1)).handleMessage(message, queueUrl, mockBuffer, mockBufferTimeoutMillis, mockAcknowledgementSet); + } + + @Test + void addSqsObject_propagatesIOExceptionThrownByHandleMessage() throws IOException { + Message message = Message.builder().body("Test Message Body").build(); + String queueUrl = "https://sqs.us-east-1.amazonaws.com/123456789012/test-queue"; + doThrow(new IOException("Handle message failed")).when(mockSqsMessageHandler).handleMessage(message, queueUrl, mockBuffer, mockBufferTimeoutMillis, mockAcknowledgementSet); + IOException thrownException = assertThrows(IOException.class, () -> + sqsEventProcessor.addSqsObject(message, queueUrl, mockBuffer, mockBufferTimeoutMillis, mockAcknowledgementSet) + ); + assert(thrownException.getMessage().equals("Handle message failed")); + verify(mockSqsMessageHandler, times(1)).handleMessage(message, queueUrl, mockBuffer, mockBufferTimeoutMillis, mockAcknowledgementSet); + } +} diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsServiceTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsServiceTest.java new file mode 100644 index 0000000000..3bbc44bbe6 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsServiceTest.java @@ -0,0 +1,81 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.sqs.SqsClient; +import java.util.List; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + + +class SqsServiceTest { + private SqsSourceConfig sqsSourceConfig; + private SqsEventProcessor sqsEventProcessor; + private SqsClient sqsClient; + private PluginMetrics pluginMetrics; + private AcknowledgementSetManager acknowledgementSetManager; + private Buffer> buffer; + private AwsCredentialsProvider credentialsProvider; + + @BeforeEach + void setUp() { + sqsSourceConfig = mock(SqsSourceConfig.class); + sqsEventProcessor = mock(SqsEventProcessor.class); + sqsClient = mock(SqsClient.class, withSettings()); + pluginMetrics = mock(PluginMetrics.class); + acknowledgementSetManager = mock(AcknowledgementSetManager.class); + buffer = mock(Buffer.class); + credentialsProvider = mock(AwsCredentialsProvider.class); + + AwsAuthenticationOptions awsAuthenticationOptions = mock(AwsAuthenticationOptions.class); + when(awsAuthenticationOptions.getAwsRegion()).thenReturn(Region.US_EAST_1); + when(sqsSourceConfig.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions); + } + + @Test + void start_with_single_queue_starts_workers() { + QueueConfig queueConfig = mock(QueueConfig.class); + when(queueConfig.getUrl()).thenReturn("https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue"); + when(queueConfig.getNumWorkers()).thenReturn(2); + when(sqsSourceConfig.getQueues()).thenReturn(List.of(queueConfig)); + SqsService sqsService = spy(new SqsService(buffer, acknowledgementSetManager, sqsSourceConfig, sqsEventProcessor, pluginMetrics, credentialsProvider)); + doReturn(sqsClient).when(sqsService).createSqsClient(credentialsProvider); + sqsService.start(); // if no exception is thrown here, then workers have been started + } + + @Test + void stop_should_shutdown_executors_and_workers_and_close_client() throws InterruptedException { + QueueConfig queueConfig = mock(QueueConfig.class); + when(queueConfig.getUrl()).thenReturn("MyQueue"); + when(queueConfig.getNumWorkers()).thenReturn(1); + when(sqsSourceConfig.getQueues()).thenReturn(List.of(queueConfig)); + SqsClient sqsClient = mock(SqsClient.class); + SqsService sqsService = new SqsService(buffer, acknowledgementSetManager, sqsSourceConfig, sqsEventProcessor, pluginMetrics, credentialsProvider) { + @Override + SqsClient createSqsClient(final AwsCredentialsProvider credentialsProvider) { + return sqsClient; + } + }; + sqsService.start(); + sqsService.stop(); + verify(sqsClient, times(1)).close(); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfigTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfigTest.java new file mode 100644 index 0000000000..29f0443670 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfigTest.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; + +public class SqsSourceConfigTest { + + private final ObjectMapper objectMapper = new ObjectMapper(); + + @Test + void testDefaultValues() { + final SqsSourceConfig config = new SqsSourceConfig(); + assertNull(config.getAwsAuthenticationOptions(), "AWS Authentication Options should be null by default"); + assertFalse(config.getAcknowledgements(), "Acknowledgments should be false by default"); + assertEquals(SqsSourceConfig.DEFAULT_BUFFER_TIMEOUT, config.getBufferTimeout(), "Buffer timeout should default to 10 seconds"); + assertNull(config.getQueues(), "Queues should be null by default"); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceTest.java new file mode 100644 index 0000000000..cf130c102e --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceTest.java @@ -0,0 +1,65 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.regions.Region; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class SqsSourceTest { + private final String PLUGIN_NAME = "sqs"; + private final String TEST_PIPELINE_NAME = "test_pipeline"; + private SqsSource sqsSource; + private PluginMetrics pluginMetrics; + private SqsSourceConfig sqsSourceConfig; + private AcknowledgementSetManager acknowledgementSetManager; + private AwsCredentialsSupplier awsCredentialsSupplier; + private Buffer> buffer; + + + @BeforeEach + void setUp() { + pluginMetrics = PluginMetrics.fromNames(PLUGIN_NAME, TEST_PIPELINE_NAME); + sqsSourceConfig = mock(SqsSourceConfig.class); + acknowledgementSetManager = mock(AcknowledgementSetManager.class); + awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); + sqsSource = new SqsSource(pluginMetrics, sqsSourceConfig, acknowledgementSetManager, awsCredentialsSupplier); + buffer = mock(Buffer.class); + } + + @Test + void start_should_throw_IllegalStateException_when_buffer_is_null() { + assertThrows(IllegalStateException.class, () -> sqsSource.start(null)); + } + + @Test + void start_should_not_throw_when_buffer_is_not_null() { + AwsAuthenticationOptions awsAuthenticationOptions = mock(AwsAuthenticationOptions.class); + when(awsAuthenticationOptions.getAwsStsRoleArn()).thenReturn("arn:aws:iam::123456789012:role/example-role"); + when(sqsSourceConfig.getAwsAuthenticationOptions()).thenReturn(awsAuthenticationOptions); + when(awsCredentialsSupplier.getProvider(any())).thenReturn(mock(AwsCredentialsProvider.class)); + when(awsAuthenticationOptions.getAwsRegion()).thenReturn(Region.of("us-east-2")); + assertDoesNotThrow(() -> sqsSource.start(buffer)); + } + + @Test + void stop_should_not_throw_when_sqsService_is_null() { + sqsSource.stop(); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorkerTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorkerTest.java new file mode 100644 index 0000000000..7bb8e082cc --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorkerTest.java @@ -0,0 +1,378 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import com.linecorp.armeria.client.retry.Backoff; +import io.micrometer.core.instrument.Counter; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.metrics.PluginMetrics; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; +import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; +import org.opensearch.dataprepper.model.acknowledgements.ProgressCheck; +import org.opensearch.dataprepper.model.buffer.Buffer; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import software.amazon.awssdk.services.sqs.SqsClient; +import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityRequest; +import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequest; +import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchResponse; +import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchResultEntry; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest; +import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse; +import software.amazon.awssdk.services.sqs.model.SqsException; +import software.amazon.awssdk.services.sqs.model.BatchResultErrorEntry; + +import java.io.IOException; +import java.time.Duration; +import java.util.Collections; +import java.util.UUID; +import java.util.function.Consumer; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyDouble; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.ArgumentMatchers.isNull; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class SqsWorkerTest { + + @Mock + private Buffer> buffer; + @Mock + private AcknowledgementSetManager acknowledgementSetManager; + @Mock + private SqsClient sqsClient; + @Mock + private SqsEventProcessor sqsEventProcessor; + @Mock + private SqsSourceConfig sqsSourceConfig; + @Mock + private QueueConfig queueConfig; + @Mock + private PluginMetrics pluginMetrics; + @Mock + private Backoff backoff; + @Mock + private Counter sqsMessagesReceivedCounter; + @Mock + private Counter sqsMessagesDeletedCounter; + @Mock + private Counter sqsMessagesFailedCounter; + @Mock + private Counter sqsMessagesDeleteFailedCounter; + @Mock + private Counter acknowledgementSetCallbackCounter; + @Mock + private Counter sqsVisibilityTimeoutChangedCount; + @Mock + private Counter sqsVisibilityTimeoutChangeFailedCount; + private int mockBufferTimeoutMillis = 10000; + + private SqsWorker createObjectUnderTest() { + return new SqsWorker( + buffer, + acknowledgementSetManager, + sqsClient, + sqsEventProcessor, + sqsSourceConfig, + queueConfig, + pluginMetrics, + backoff); + } + + @BeforeEach + void setUp() { + when(pluginMetrics.counter(SqsWorker.SQS_MESSAGES_RECEIVED_METRIC_NAME)).thenReturn(sqsMessagesReceivedCounter); + when(pluginMetrics.counter(SqsWorker.SQS_MESSAGES_DELETED_METRIC_NAME)).thenReturn(sqsMessagesDeletedCounter); + when(pluginMetrics.counter(SqsWorker.SQS_MESSAGES_FAILED_METRIC_NAME)).thenReturn(sqsMessagesFailedCounter); + when(pluginMetrics.counter(SqsWorker.SQS_MESSAGES_DELETE_FAILED_METRIC_NAME)).thenReturn(sqsMessagesDeleteFailedCounter); + when(pluginMetrics.counter(SqsWorker.ACKNOWLEDGEMENT_SET_CALLACK_METRIC_NAME)).thenReturn(acknowledgementSetCallbackCounter); + when(pluginMetrics.counter(SqsWorker.SQS_VISIBILITY_TIMEOUT_CHANGED_COUNT_METRIC_NAME)).thenReturn(sqsVisibilityTimeoutChangedCount); + when(pluginMetrics.counter(SqsWorker.SQS_VISIBILITY_TIMEOUT_CHANGE_FAILED_COUNT_METRIC_NAME)).thenReturn(sqsVisibilityTimeoutChangeFailedCount); + when(sqsSourceConfig.getAcknowledgements()).thenReturn(false); + when(sqsSourceConfig.getBufferTimeout()).thenReturn(Duration.ofSeconds(10)); + when(queueConfig.getUrl()).thenReturn("https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue"); + when(queueConfig.getWaitTime()).thenReturn(Duration.ofSeconds(1)); + } + + @Test + void processSqsMessages_should_return_number_of_messages_processed_and_increment_counters() throws IOException { + final Message message = Message.builder() + .messageId(UUID.randomUUID().toString()) + .receiptHandle(UUID.randomUUID().toString()) + .body("{\"Records\":[{\"eventSource\":\"custom\",\"message\":\"Hello World\"}]}") + .build(); + + final ReceiveMessageResponse response = ReceiveMessageResponse.builder().messages(message).build(); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(response); + + final DeleteMessageBatchResultEntry successfulDelete = DeleteMessageBatchResultEntry.builder().id(message.messageId()).build(); + final DeleteMessageBatchResponse deleteResponse = DeleteMessageBatchResponse.builder().successful(successfulDelete).build(); + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))).thenReturn(deleteResponse); + + int messagesProcessed = createObjectUnderTest().processSqsMessages(); + assertThat(messagesProcessed, equalTo(1)); + + verify(sqsMessagesReceivedCounter).increment(1); + verify(sqsMessagesDeletedCounter).increment(1); + verify(sqsMessagesDeleteFailedCounter, never()).increment(anyDouble()); + } + + @Test + void processSqsMessages_should_invoke_processSqsEvent_and_deleteSqsMessages_when_entries_non_empty() throws IOException { + final Message message = Message.builder() + .messageId(UUID.randomUUID().toString()) + .receiptHandle(UUID.randomUUID().toString()) + .body("{\"Records\":[{\"eventSource\":\"custom\",\"message\":\"Hello World\"}]}") + .build(); + + final ReceiveMessageResponse response = ReceiveMessageResponse.builder() + .messages(message) + .build(); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(response); + + final DeleteMessageBatchResultEntry successfulDelete = DeleteMessageBatchResultEntry.builder() + .id(message.messageId()) + .build(); + final DeleteMessageBatchResponse deleteResponse = DeleteMessageBatchResponse.builder() + .successful(successfulDelete) + .build(); + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))).thenReturn(deleteResponse); + + SqsWorker sqsWorker = createObjectUnderTest(); + int messagesProcessed = sqsWorker.processSqsMessages(); + + assertThat(messagesProcessed, equalTo(1)); + verify(sqsEventProcessor, times(1)).addSqsObject(eq(message), eq("https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue"), eq(buffer), eq(mockBufferTimeoutMillis), isNull()); + verify(sqsClient, times(1)).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); + verify(sqsMessagesReceivedCounter).increment(1); + verify(sqsMessagesDeletedCounter).increment(1); + verify(sqsMessagesDeleteFailedCounter, never()).increment(anyDouble()); + } + + @Test + void processSqsMessages_should_not_invoke_processSqsEvent_and_deleteSqsMessages_when_entries_are_empty() throws IOException { + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))) + .thenReturn(ReceiveMessageResponse.builder().messages(Collections.emptyList()).build()); + SqsWorker sqsWorker = createObjectUnderTest(); + int messagesProcessed = sqsWorker.processSqsMessages(); + assertThat(messagesProcessed, equalTo(0)); + verify(sqsEventProcessor, never()).addSqsObject(any(), anyString(), any(), anyInt(), any()); + verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); + verify(sqsMessagesReceivedCounter, never()).increment(anyDouble()); + verify(sqsMessagesDeletedCounter, never()).increment(anyDouble()); + } + + + @Test + void processSqsMessages_should_not_delete_messages_if_acknowledgements_enabled_until_acknowledged() throws IOException { + when(sqsSourceConfig.getAcknowledgements()).thenReturn(true); + AcknowledgementSet acknowledgementSet = mock(AcknowledgementSet.class); + when(acknowledgementSetManager.create(any(), any())).thenReturn(acknowledgementSet); + when(queueConfig.getUrl()).thenReturn("https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue"); + + final Message message = Message.builder() + .messageId("msg-1") + .receiptHandle("rh-1") + .body("{\"Records\":[{\"eventSource\":\"custom\",\"message\":\"Hello World\"}]}") + .build(); + + final ReceiveMessageResponse response = ReceiveMessageResponse.builder().messages(message).build(); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(response); + int messagesProcessed = createObjectUnderTest().processSqsMessages(); + assertThat(messagesProcessed, equalTo(1)); + verify(sqsEventProcessor, times(1)).addSqsObject(eq(message), + eq("https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue"), + eq(buffer), + eq(mockBufferTimeoutMillis), + eq(acknowledgementSet)); + verify(sqsMessagesReceivedCounter).increment(1); + verifyNoInteractions(sqsMessagesDeletedCounter); + } + + @Test + void acknowledgementsEnabled_and_visibilityDuplicateProtectionEnabled_should_create_ack_sets_and_progress_check() { + when(sqsSourceConfig.getAcknowledgements()).thenReturn(true); + when(queueConfig.getVisibilityDuplicateProtection()).thenReturn(true); + + SqsWorker worker = new SqsWorker(buffer, acknowledgementSetManager, sqsClient, sqsEventProcessor, sqsSourceConfig, queueConfig, pluginMetrics, backoff); + Message message = Message.builder().messageId("msg-dup").receiptHandle("handle-dup").build(); + ReceiveMessageResponse response = ReceiveMessageResponse.builder().messages(message).build(); + when(sqsClient.receiveMessage((ReceiveMessageRequest) any())).thenReturn(response); + + AcknowledgementSet acknowledgementSet = mock(AcknowledgementSet.class); + when(acknowledgementSetManager.create(any(), any())).thenReturn(acknowledgementSet); + + int processed = worker.processSqsMessages(); + assertThat(processed, equalTo(1)); + + verify(acknowledgementSetManager).create(any(), any()); + verify(acknowledgementSet).addProgressCheck(any(), any()); + } + + @Test + void processSqsMessages_should_return_zero_messages_with_backoff_when_a_SqsException_is_thrown() { + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenThrow(SqsException.class); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); + verify(backoff).nextDelayMillis(1); + assertThat(messagesProcessed, equalTo(0)); + } + + @Test + void processSqsMessages_should_throw_when_a_SqsException_is_thrown_with_max_retries() { + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenThrow(SqsException.class); + when(backoff.nextDelayMillis(anyInt())).thenReturn((long) -1); + SqsWorker objectUnderTest = createObjectUnderTest(); + assertThrows(SqsRetriesExhaustedException.class, objectUnderTest::processSqsMessages); + } + + @Test + void processSqsMessages_should_update_visibility_timeout_when_progress_changes() throws IOException { + AcknowledgementSet acknowledgementSet = mock(AcknowledgementSet.class); + when(queueConfig.getVisibilityDuplicateProtection()).thenReturn(true); + when(queueConfig.getVisibilityTimeout()).thenReturn(Duration.ofMillis(1)); + when(acknowledgementSetManager.create(any(), any(Duration.class))).thenReturn(acknowledgementSet); + when(sqsSourceConfig.getAcknowledgements()).thenReturn(true); + final Message message = mock(Message.class); + final String testReceiptHandle = UUID.randomUUID().toString(); + when(message.messageId()).thenReturn(testReceiptHandle); + when(message.receiptHandle()).thenReturn(testReceiptHandle); + + final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); + when(receiveMessageResponse.messages()).thenReturn(Collections.singletonList(message)); + + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); + + assertThat(messagesProcessed, equalTo(1)); + verify(sqsEventProcessor).addSqsObject(any(), anyString(), any(), anyInt(), any()); + verify(acknowledgementSetManager).create(any(), any(Duration.class)); + + ArgumentCaptor> progressConsumerArgumentCaptor = ArgumentCaptor.forClass(Consumer.class); + verify(acknowledgementSet).addProgressCheck(progressConsumerArgumentCaptor.capture(), any(Duration.class)); + final Consumer actualConsumer = progressConsumerArgumentCaptor.getValue(); + final ProgressCheck progressCheck = mock(ProgressCheck.class); + actualConsumer.accept(progressCheck); + + ArgumentCaptor changeMessageVisibilityRequestArgumentCaptor = ArgumentCaptor.forClass(ChangeMessageVisibilityRequest.class); + verify(sqsClient).changeMessageVisibility(changeMessageVisibilityRequestArgumentCaptor.capture()); + ChangeMessageVisibilityRequest actualChangeVisibilityRequest = changeMessageVisibilityRequestArgumentCaptor.getValue(); + assertThat(actualChangeVisibilityRequest.queueUrl(), equalTo(queueConfig.getUrl())); + assertThat(actualChangeVisibilityRequest.receiptHandle(), equalTo(testReceiptHandle)); + verify(sqsMessagesReceivedCounter).increment(1); + } + @Test + void increaseVisibilityTimeout_doesNothing_whenIsStopped() throws IOException { + when(sqsSourceConfig.getAcknowledgements()).thenReturn(true); + when(queueConfig.getVisibilityDuplicateProtection()).thenReturn(false); + when(queueConfig.getVisibilityTimeout()).thenReturn(Duration.ofSeconds(30)); + AcknowledgementSet mockAcknowledgementSet = mock(AcknowledgementSet.class); + when(acknowledgementSetManager.create(any(), any())).thenReturn(mockAcknowledgementSet); + Message message = Message.builder() + .messageId(UUID.randomUUID().toString()) + .receiptHandle(UUID.randomUUID().toString()) + .body("{\"Records\":[{\"eventSource\":\"custom\",\"message\":\"Hello World\"}]}") + .build(); + ReceiveMessageResponse response = ReceiveMessageResponse.builder() + .messages(message) + .build(); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(response); + SqsWorker sqsWorker = createObjectUnderTest(); + sqsWorker.stop(); + int messagesProcessed = sqsWorker.processSqsMessages(); + assertThat(messagesProcessed, equalTo(1)); + verify(sqsEventProcessor, times(1)).addSqsObject(eq(message), + eq("https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue"), + eq(buffer), + eq(mockBufferTimeoutMillis), + eq(mockAcknowledgementSet)); + verify(sqsClient, never()).changeMessageVisibility(any(ChangeMessageVisibilityRequest.class)); + verify(sqsVisibilityTimeoutChangeFailedCount, never()).increment(); + } + + @Test + void deleteSqsMessages_incrementsFailedCounter_whenDeleteResponseHasFailedDeletes() throws IOException { + final Message message1 = Message.builder() + .messageId(UUID.randomUUID().toString()) + .receiptHandle(UUID.randomUUID().toString()) + .body("{\"Records\":[{\"eventSource\":\"custom\",\"message\":\"Hello World 1\"}]}") + .build(); + final Message message2 = Message.builder() + .messageId(UUID.randomUUID().toString()) + .receiptHandle(UUID.randomUUID().toString()) + .body("{\"Records\":[{\"eventSource\":\"custom\",\"message\":\"Hello World 2\"}]}") + .build(); + + final ReceiveMessageResponse receiveResponse = ReceiveMessageResponse.builder() + .messages(message1, message2) + .build(); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveResponse); + + DeleteMessageBatchResultEntry successfulDelete = DeleteMessageBatchResultEntry.builder() + .id(message1.messageId()) + .build(); + + BatchResultErrorEntry failedDelete = BatchResultErrorEntry.builder() + .id(message2.messageId()) + .code("ReceiptHandleIsInvalid") + .senderFault(true) + .message("Failed to delete message due to invalid receipt handle.") + .build(); + + DeleteMessageBatchResponse deleteResponse = DeleteMessageBatchResponse.builder() + .successful(successfulDelete) + .failed(failedDelete) + .build(); + + when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))).thenReturn(deleteResponse); + SqsWorker sqsWorker = createObjectUnderTest(); + int messagesProcessed = sqsWorker.processSqsMessages(); + assertThat(messagesProcessed, equalTo(2)); + verify(sqsMessagesReceivedCounter).increment(2); + verify(sqsMessagesDeletedCounter).increment(1); + verify(sqsMessagesDeleteFailedCounter).increment(1); + } + @Test + void processSqsMessages_handlesException_correctly_when_addSqsObject_throwsException() throws IOException { + final Message message = Message.builder() + .messageId(UUID.randomUUID().toString()) + .receiptHandle(UUID.randomUUID().toString()) + .body("{\"Records\":[{\"eventSource\":\"custom\",\"message\":\"Hello World\"}]}") + .build(); + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn( + ReceiveMessageResponse.builder().messages(message).build() + ); + doThrow(new RuntimeException("Processing failed")).when(sqsEventProcessor) + .addSqsObject(eq(message), eq("https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue"), + any(), anyInt(), any()); + SqsWorker sqsWorker = createObjectUnderTest(); + int messagesProcessed = sqsWorker.processSqsMessages(); + assertThat(messagesProcessed, equalTo(1)); + verify(sqsMessagesReceivedCounter).increment(1); + verify(sqsMessagesFailedCounter).increment(); + verify(backoff).nextDelayMillis(anyInt()); + verify(sqsClient, never()).deleteMessageBatch(any(DeleteMessageBatchRequest.class)); + verify(sqsMessagesDeletedCounter, never()).increment(anyInt()); + } +} \ No newline at end of file diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTaskTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTaskTest.java deleted file mode 100644 index c4dac097a1..0000000000 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTaskTest.java +++ /dev/null @@ -1,170 +0,0 @@ -package org.opensearch.dataprepper.plugins.source.sqssource; - -import com.linecorp.armeria.client.retry.Backoff; -import io.micrometer.core.instrument.Counter; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; -import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; -import org.opensearch.dataprepper.model.buffer.Buffer; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.aws.sqs.common.SqsService; -import org.opensearch.dataprepper.plugins.aws.sqs.common.handler.SqsMessageHandler; -import org.opensearch.dataprepper.plugins.aws.sqs.common.metrics.SqsMetrics; -import org.opensearch.dataprepper.plugins.aws.sqs.common.model.SqsOptions; -import software.amazon.awssdk.services.sqs.SqsClient; -import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequest; -import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchResponse; -import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.ReceiveMessageRequest; -import software.amazon.awssdk.services.sqs.model.ReceiveMessageResponse; -import software.amazon.awssdk.services.sqs.model.SqsException; -import software.amazon.awssdk.services.sts.model.StsException; - -import java.time.Duration; -import java.util.List; -import java.util.UUID; -import java.util.function.Consumer; - -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoInteractions; -import static org.mockito.Mockito.when; - - -@ExtendWith(MockitoExtension.class) -class SqsSourceTaskTest { - - private SqsService sqsService; - - private SqsOptions sqsOptions; - - @Mock - private SqsMetrics sqsMetrics; - - @Mock - private AcknowledgementSetManager acknowledgementSetManager; - - private boolean endToEndAcknowledgementsEnabled = false; - - @Mock - private SqsMessageHandler sqsHandler; - - @Mock - private SqsClient sqsClient; - - @Mock - private Buffer> buffer; - - @Mock - private Backoff backoff; - - private Counter messageReceivedCounter; - - private Counter messageDeletedCounter; - - private Counter sqsMessagesFailedCounter; - - private AcknowledgementSet acknowledgementSet; - - @BeforeEach - public void setup(){ - messageReceivedCounter = mock(Counter.class); - messageDeletedCounter = mock(Counter.class); - sqsMessagesFailedCounter = mock(Counter.class); - acknowledgementSet = mock(AcknowledgementSet.class); - } - - private SqsSourceTask createObjectUnderTest() { - sqsService = new SqsService(sqsMetrics,sqsClient,backoff); - sqsOptions = new SqsOptions.Builder() - .setSqsUrl("https://sqs.us-east-2.amazonaws.com/123456789012/MyQueue") - .setVisibilityTimeout(Duration.ofSeconds(30)) - .setWaitTime(Duration.ofSeconds(20)).build(); - return new SqsSourceTask(buffer,1,Duration.ofSeconds(10),sqsService, - sqsOptions, - sqsMetrics, - acknowledgementSetManager, - endToEndAcknowledgementsEnabled, - sqsHandler); - } - - @ParameterizedTest - @ValueSource(strings = { - "'{\"S.No\":\"1\",\"name\":\"data-prep\",\"country\":\"USA\"}'", - "Test Message", - "'2023-05-30T13:25:11,889 [main] INFO org.opensearch.dataprepper.pipeline.server.DataPrepperServer - Data Prepper server running at :4900'"}) - void processSqsMessages_test_with_different_types_of_messages(final String message) throws Exception { - when(sqsMetrics.getSqsMessagesReceivedCounter()).thenReturn(messageReceivedCounter); - when(sqsMetrics.getSqsMessagesDeletedCounter()).thenReturn(messageDeletedCounter); - - List messageList = List.of(Message.builder().body(message).messageId(UUID.randomUUID().toString()).receiptHandle(UUID.randomUUID().toString()).build()); - when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(ReceiveMessageResponse.builder().messages(messageList).build()); - when(sqsMetrics.getSqsMessagesReceivedCounter()).thenReturn(messageReceivedCounter); - when(sqsMetrics.getSqsMessagesDeletedCounter()).thenReturn(messageDeletedCounter); - - when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))). - thenReturn(DeleteMessageBatchResponse.builder().successful(builder -> builder.id(UUID.randomUUID().toString()).build()).build()); - final SqsSourceTask sqsSourceTask = createObjectUnderTest(); - sqsSourceTask.processSqsMessages(); - - verify(sqsHandler).handleMessages(eq(messageList), any(), isNull()); - - verify(sqsMetrics.getSqsMessagesReceivedCounter()).increment(); - verify(sqsMetrics.getSqsMessagesDeletedCounter()).increment(1); - } - - @Test - void processSqsMessages_should_return_zero_messages_with_backoff() { - when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenThrow(SqsException.class); - when(sqsMetrics.getSqsReceiveMessagesFailedCounter()).thenReturn(sqsMessagesFailedCounter); - createObjectUnderTest().processSqsMessages(); - verify(backoff).nextDelayMillis(1); - verify(sqsMessagesFailedCounter).increment(); - } - - @Test - void processSqsMessages_should_return_one_message_with_buffer_write_fail_with_backoff() { - String message ="'{\"S.No\":\"1\",\"name\":\"data-prep\",\"country\":\"USA\"}'"; - List messageList = List.of(Message.builder().body(message).messageId(UUID.randomUUID().toString()).receiptHandle(UUID.randomUUID().toString()).build()); - when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(ReceiveMessageResponse.builder().messages(messageList).build()); - when(sqsClient.deleteMessageBatch(any(DeleteMessageBatchRequest.class))).thenThrow(mock(StsException.class)); - when(sqsMetrics.getSqsMessagesReceivedCounter()).thenReturn(messageReceivedCounter); - when(sqsMetrics.getSqsMessagesDeleteFailedCounter()).thenReturn(messageDeletedCounter); - createObjectUnderTest().processSqsMessages(); - verify(backoff).nextDelayMillis(1); - verify(messageReceivedCounter).increment(); - } - - @Test - void processSqsMessages_test_with_different_types_of_messages_with_end_to_end_ack() throws Exception { - when(sqsMetrics.getSqsMessagesReceivedCounter()).thenReturn(messageReceivedCounter); - when(sqsMetrics.getSqsMessagesDeletedCounter()).thenReturn(messageDeletedCounter); - - endToEndAcknowledgementsEnabled = true; - - String message = "'{\"S.No\":\"1\",\"name\":\"data-prep\",\"country\":\"USA\"}'"; - when(acknowledgementSetManager.create(any( Consumer.class), any(Duration.class))).thenReturn(acknowledgementSet); - List messageList = List.of(Message.builder().body(message).messageId(UUID.randomUUID().toString()).receiptHandle(UUID.randomUUID().toString()).build()); - when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(ReceiveMessageResponse.builder().messages(messageList).build()); - - createObjectUnderTest().processSqsMessages(); - - verify(sqsHandler).handleMessages(eq(messageList), any(), eq(acknowledgementSet)); - - verify(sqsMetrics.getSqsMessagesReceivedCounter()).increment(); - verify(acknowledgementSetManager).create(any(), any(Duration.class)); - verifyNoInteractions(sqsMetrics.getSqsMessagesDeletedCounter()); - } - -} - diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTest.java deleted file mode 100644 index 2d3400b7cc..0000000000 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/SqsSourceTest.java +++ /dev/null @@ -1,57 +0,0 @@ -package org.opensearch.dataprepper.plugins.source.sqssource; - -import io.micrometer.core.instrument.Counter; -import io.micrometer.core.instrument.Timer; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.Mock; -import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -import org.opensearch.dataprepper.metrics.PluginMetrics; -import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; -import org.opensearch.dataprepper.plugins.aws.sqs.common.metrics.SqsMetrics; -import org.opensearch.dataprepper.plugins.source.sqssource.config.SqsSourceConfig; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -class SqsSourceTest { - - @Mock - private PluginMetrics pluginMetrics; - - @Mock - private AcknowledgementSetManager acknowledgementSetManager; - - @Mock - private AwsCredentialsSupplier awsCredentialsSupplier; - - private SqsSource sqsSource; - - @BeforeEach - public void setup() { - SqsSourceConfig sqsSourceConfig = mock(SqsSourceConfig.class); - pluginMetrics = mock(PluginMetrics.class); - Timer timer = mock(Timer.class); - Counter counter = mock(Counter.class); - - awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); - when(pluginMetrics.timer(SqsMetrics.SQS_MESSAGE_DELAY_METRIC_NAME)).thenReturn(timer); - when(pluginMetrics.counter(SqsMetrics.SQS_MESSAGES_RECEIVED_METRIC_NAME)).thenReturn(counter); - when(pluginMetrics.counter(SqsMetrics.SQS_MESSAGES_DELETED_METRIC_NAME)).thenReturn(counter); - when(pluginMetrics.counter(SqsMetrics.SQS_RECEIVE_MESSAGES_FAILED_METRIC_NAME)).thenReturn(counter); - when(pluginMetrics.counter(SqsMetrics.SQS_MESSAGES_DELETE_FAILED_METRIC_NAME)).thenReturn(counter); - when(pluginMetrics.counter(SqsMetrics.ACKNOWLEDGEMENT_SET_CALLBACK_METRIC_NAME)).thenReturn(counter); - when(sqsSourceConfig.getUrls()).thenReturn(List.of("https://sqs.us-east-1.amazonaws.com/123099425585/dp")); - this. sqsSource = - new SqsSource(pluginMetrics,sqsSourceConfig,acknowledgementSetManager,awsCredentialsSupplier); - } - - @Test - void start_should_throw_IllegalStateException_when_buffer_is_null(){ - assertThrows(IllegalStateException.class, () -> sqsSource.start(null)); - } - -} diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/config/SqsSourceConfigTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/config/SqsSourceConfigTest.java deleted file mode 100644 index ddecd8a346..0000000000 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/config/SqsSourceConfigTest.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ -package org.opensearch.dataprepper.plugins.source.sqssource.config; - -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; -import org.junit.jupiter.api.Test; -import software.amazon.awssdk.regions.Region; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.nullValue; - -class SqsSourceConfigTest { - - private static final String SQS_CONFIGURATION_YAML = "/src/test/resources/pipeline.yaml"; - - private final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory().enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS)); - - @Test - void sqs_source_configuration_test() throws IOException { - final byte[] bytes = Files.readAllBytes(Path.of(Paths.get("").toAbsolutePath() + SQS_CONFIGURATION_YAML)); - final SqsSourceConfig sqsSourceConfig = objectMapper.readValue(bytes, SqsSourceConfig.class); - final AwsAuthenticationOptions aws = sqsSourceConfig.getAws(); - - assertThat(sqsSourceConfig.getAcknowledgements(),equalTo(false)); - assertThat(sqsSourceConfig.getPollingFrequency(),equalTo(Duration.ZERO)); - assertThat(sqsSourceConfig.getBatchSize(),equalTo(10)); - assertThat(sqsSourceConfig.getNumberOfThreads(),equalTo(5)); - assertThat(sqsSourceConfig.getVisibilityTimeout(),nullValue()); - assertThat(sqsSourceConfig.getWaitTime(),nullValue()); - assertThat(sqsSourceConfig.getUrls().get(0),equalTo("https://sqs.us-east-1.amazonaws.com/123099425585/dp")); - - assertThat(aws.getAwsRegion(),equalTo(Region.US_EAST_1)); - assertThat(aws.getAwsStsRoleArn(),equalTo("arn:aws:iam::278936200144:role/aos-role")); - assertThat(aws.getAwsStsHeaderOverrides().get("test"),equalTo("test")); - } -} diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/handler/RawSqsMessageHandlerTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/handler/RawSqsMessageHandlerTest.java deleted file mode 100644 index c71ae04204..0000000000 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqssource/handler/RawSqsMessageHandlerTest.java +++ /dev/null @@ -1,132 +0,0 @@ -package org.opensearch.dataprepper.plugins.source.sqssource.handler; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Nested; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.mockito.ArgumentCaptor; -import org.mockito.InOrder; -import org.mockito.Mock; -import org.mockito.junit.jupiter.MockitoExtension; -import org.opensearch.dataprepper.buffer.common.BufferAccumulator; -import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; -import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.record.Record; -import org.opensearch.dataprepper.plugins.aws.sqs.common.SqsService; -import software.amazon.awssdk.services.sqs.model.DeleteMessageBatchRequestEntry; -import software.amazon.awssdk.services.sqs.model.Message; - -import java.util.List; -import java.util.UUID; -import java.util.stream.Collectors; -import java.util.stream.IntStream; - -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.instanceOf; -import static org.mockito.Mockito.inOrder; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -@ExtendWith(MockitoExtension.class) -class RawSqsMessageHandlerTest { - @Mock - private SqsService sqsService; - - @Mock - private BufferAccumulator bufferAccumulator; - - private AcknowledgementSet acknowledgementSet; - private List messageBodies; - private List messages; - - @BeforeEach - void setUp() { - messageBodies = IntStream.range(0, 3).mapToObj(i -> UUID.randomUUID().toString()) - .collect(Collectors.toList()); - - messages = messageBodies.stream() - .map(body -> { - Message message = mock(Message.class); - when(message.body()).thenReturn(body); - return message; - }) - .collect(Collectors.toList()); - - - acknowledgementSet = null; - } - - private RawSqsMessageHandler createObjectUnderTest() { - return new RawSqsMessageHandler(sqsService); - } - - @Test - void handleMessages_writes_to_buffer_and_flushes() throws Exception { - createObjectUnderTest().handleMessages(messages, bufferAccumulator, acknowledgementSet); - - InOrder inOrder = inOrder(bufferAccumulator); - - ArgumentCaptor> recordArgumentCaptor = ArgumentCaptor.forClass(Record.class); - - inOrder.verify(bufferAccumulator, times(messages.size())).add(recordArgumentCaptor.capture()); - inOrder.verify(bufferAccumulator).flush(); - - List actualEventData = recordArgumentCaptor.getAllValues() - .stream() - .map(Record::getData) - .map(e -> e.get("message", Object.class)) - .collect(Collectors.toList()); - - assertThat(actualEventData.size(), equalTo(messages.size())); - - for (int i = 0; i < actualEventData.size(); i++){ - Object messageData = actualEventData.get(i); - assertThat(messageData, instanceOf(String.class)); - assertThat(messageData, equalTo(messageBodies.get(i))); - } - } - - @Test - void handleMessages_returns_deleteList() throws Exception { - List stubbedDeleteList = List.of(mock(DeleteMessageBatchRequestEntry.class)); - when(sqsService.getDeleteMessageBatchRequestEntryList(messages)) - .thenReturn(stubbedDeleteList); - - List actualList = createObjectUnderTest().handleMessages(messages, bufferAccumulator, acknowledgementSet); - - assertThat(actualList, equalTo(stubbedDeleteList)); - } - - @Nested - class WithAcknowledgementSet { - @BeforeEach - void setUp() { - acknowledgementSet = mock(AcknowledgementSet.class); - } - - @Test - void handleMessages_with_acknowledgementSet_adds_events() throws Exception { - createObjectUnderTest().handleMessages(messages, bufferAccumulator, acknowledgementSet); - - ArgumentCaptor eventArgumentCaptor = ArgumentCaptor.forClass(Event.class); - - verify(acknowledgementSet, times(messages.size())).add(eventArgumentCaptor.capture()); - - List actualEventData = eventArgumentCaptor.getAllValues() - .stream() - .map(e -> e.get("message", Object.class)) - .collect(Collectors.toList()); - - assertThat(actualEventData.size(), equalTo(messages.size())); - - for (int i = 0; i < actualEventData.size(); i++) { - Object messageData = actualEventData.get(i); - assertThat(messageData, instanceOf(String.class)); - assertThat(messageData, equalTo(messageBodies.get(i))); - } - } - } -} diff --git a/data-prepper-plugins/sqs-source/src/test/resources/pipeline.yaml b/data-prepper-plugins/sqs-source/src/test/resources/pipeline.yaml deleted file mode 100644 index 2e1e6c5500..0000000000 --- a/data-prepper-plugins/sqs-source/src/test/resources/pipeline.yaml +++ /dev/null @@ -1,10 +0,0 @@ -acknowledgments: false -queue_urls: - - https://sqs.us-east-1.amazonaws.com/123099425585/dp - - https://sqs.us-east-1.amazonaws.com/123099425585/dp1 -batch_size: 10 -number_of_threads: 5 -aws: - sts_region: us-east-1 - sts_role_arn: arn:aws:iam::278936200144:role/aos-role - sts_header_overrides: {"test":"test"} diff --git a/settings.gradle b/settings.gradle index f8fc52a4d3..d86bc7e1da 100644 --- a/settings.gradle +++ b/settings.gradle @@ -166,7 +166,7 @@ include 'data-prepper-plugins:obfuscate-processor' include 'data-prepper-plugins:parquet-codecs' include 'data-prepper-plugins:aws-sqs-common' include 'data-prepper-plugins:buffer-common' -//include 'data-prepper-plugins:sqs-source' +include 'data-prepper-plugins:sqs-source' //include 'data-prepper-plugins:cloudwatch-logs' //include 'data-prepper-plugins:http-sink' //include 'data-prepper-plugins:sns-sink' From 3f7f5d2d76118c1495c50b0d9aae494d2f5074f3 Mon Sep 17 00:00:00 2001 From: Dinu John <86094133+dinujoh@users.noreply.github.com> Date: Thu, 9 Jan 2025 19:03:29 -0600 Subject: [PATCH 21/36] Remove NOISY log from Mongo StreamAcknowledgementManager (#5321) Remove NOISY log from Mongo StreamAcknowledgementManager. We have metric to capture when this condition occurs, so removing logs as they are noisy Signed-off-by: Dinu John <86094133+dinujoh@users.noreply.github.com> --- .../plugins/mongo/stream/StreamAcknowledgementManager.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/data-prepper-plugins/mongodb/src/main/java/org/opensearch/dataprepper/plugins/mongo/stream/StreamAcknowledgementManager.java b/data-prepper-plugins/mongodb/src/main/java/org/opensearch/dataprepper/plugins/mongo/stream/StreamAcknowledgementManager.java index 38d4fc9794..b8e3026fee 100644 --- a/data-prepper-plugins/mongodb/src/main/java/org/opensearch/dataprepper/plugins/mongo/stream/StreamAcknowledgementManager.java +++ b/data-prepper-plugins/mongodb/src/main/java/org/opensearch/dataprepper/plugins/mongo/stream/StreamAcknowledgementManager.java @@ -18,9 +18,6 @@ import java.util.concurrent.Executors; import java.util.function.Consumer; -import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; - - public class StreamAcknowledgementManager { private static final Logger LOG = LoggerFactory.getLogger(StreamAcknowledgementManager.class); private static final int CHECKPOINT_RECORD_INTERVAL = 50; @@ -111,7 +108,6 @@ private void monitorAcknowledgment(final ExecutorService executorService, final } } else { if (System.currentTimeMillis() - lastCheckpointTime >= checkPointIntervalInMs) { - LOG.info(NOISY, "No records processed. Extend the lease of the partition worker."); partitionCheckpoint.extendLease(); this.noDataExtendLeaseCount.increment(); lastCheckpointTime = System.currentTimeMillis(); From c614f956c2732e52ddae6433559168869e14243a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Fri, 10 Jan 2025 15:32:27 +0100 Subject: [PATCH 22/36] Setting 'insecure' overrides 'certPath' (#5268) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan Høydahl --- data-prepper-plugins/opensearch/README.md | 8 ++++---- .../sink/opensearch/ConnectionConfiguration.java | 10 +++++----- .../opensearch/ConnectionConfigurationTests.java | 15 ++++++++++++++- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/data-prepper-plugins/opensearch/README.md b/data-prepper-plugins/opensearch/README.md index 141fa8c001..99e6f0f4dc 100644 --- a/data-prepper-plugins/opensearch/README.md +++ b/data-prepper-plugins/opensearch/README.md @@ -105,7 +105,7 @@ pipeline: - `hosts`: A list of IP addresses of OpenSearch nodes. -- `cert`(optional): CA certificate that is pem encoded. Accepts both .pem or .crt. This enables the client to trust the CA that has signed the certificate that the OpenSearch cluster is using. +- `cert`(optional): CA certificate that is pem encoded. Accepts both `.pem` or `.crt`. This enables the client to trust the CA that has signed the certificate that the OpenSearch cluster is using. This setting has no effect if `insecure` is set to `true`. Default is null. - `aws_sigv4`: A boolean flag to sign the HTTP request with AWS credentials. Only applies to Amazon OpenSearch Service. See [security](security.md) for details. Default to `false`. @@ -118,7 +118,7 @@ Default is null. - `aws_sts_header_overrides`: An optional map of header overrides to make when assuming the IAM role for the sink plugin. -- `insecure`: A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Default to `false`. +- `insecure`: A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Setting this will override any `cert` configured. Default to `false`. - `aws` (Optional) : AWS configurations. See [AWS Configuration](#aws_configuration) for details. SigV4 is enabled by default when this option is used. If this option is present, `aws_` options are not expected to be present. If any of `aws_` options are present along with this, error is thrown. @@ -674,10 +674,10 @@ and then every hour after the first time the index is processed. ### Connection Configuration -* `insecure` (Optional): A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Default to false. +* `insecure` (Optional): A boolean flag to turn off SSL certificate verification. If set to true, CA certificate verification will be turned off and insecure HTTP requests will be sent. Setting this will override any `cert` configured. Default to false. -* `cert` (Optional) : CA certificate that is pem encoded. Accepts both .pem or .crt. This enables the client to trust the CA that has signed the certificate that the OpenSearch cluster is using. Default is null. +* `cert` (Optional) : CA certificate that is pem encoded. Accepts both `.pem` or `.crt`. This enables the client to trust the CA that has signed the certificate that the OpenSearch cluster is using. This setting has no effect if `insecure` is set to `true`. Default is null. * `socket_timeout` (Optional) : A String that indicates the timeout duration for waiting for data. Supports ISO_8601 notation Strings ("PT20.345S", "PT15M", etc.) as well as simple notation Strings for seconds ("60s") and milliseconds ("1500ms"). If this timeout value not set, the underlying Apache HttpClient would rely on operating system settings for managing socket timeouts. diff --git a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java index 7557c0390d..4af2ef70a8 100644 --- a/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java +++ b/data-prepper-plugins/opensearch/src/main/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfiguration.java @@ -275,11 +275,11 @@ public static ConnectionConfiguration readConnectionConfiguration(final PluginSe final String certPath = pluginSetting.getStringOrDefault(CERT_PATH, null); final boolean insecure = pluginSetting.getBooleanOrDefault(INSECURE, false); - if (certPath != null) { - builder = builder.withCert(certPath); - } else { - //We will set insecure flag only if certPath is null - builder = builder.withInsecure(insecure); + // Insecure == true will override configured certPath + if (insecure) { + builder.withInsecure(insecure); + } else if (certPath != null) { + builder.withCert(certPath); } final String proxy = pluginSetting.getStringOrDefault(PROXY, null); builder = builder.withProxy(proxy); diff --git a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java index 382afeb869..eb12ec5fc5 100644 --- a/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java +++ b/data-prepper-plugins/opensearch/src/test/java/org/opensearch/dataprepper/plugins/sink/opensearch/ConnectionConfigurationTests.java @@ -283,7 +283,20 @@ void testCreateClientWithCertPath() throws IOException { client.close(); } - @Test + @Test + void testCreateClientWithInsecureAndCertPath() throws IOException { + // Insecure should take precedence over cert path when both are set + final PluginSetting pluginSetting = generatePluginSetting( + TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, true); + final ConnectionConfiguration connectionConfiguration = + ConnectionConfiguration.readConnectionConfiguration(pluginSetting); + assertNull(connectionConfiguration.getCertPath()); + final RestHighLevelClient client = connectionConfiguration.createClient(awsCredentialsSupplier); + assertNotNull(client); + client.close(); + } + + @Test void testCreateOpenSearchClientWithCertPath() throws IOException { final PluginSetting pluginSetting = generatePluginSetting( TEST_HOSTS, TEST_USERNAME, TEST_PASSWORD, TEST_CONNECT_TIMEOUT, TEST_SOCKET_TIMEOUT, false, null, null, TEST_CERT_PATH, false); From 81be7e36f62a481f105c4b23ca6a12a17e1f6cab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20H=C3=B8ydahl?= Date: Tue, 14 Jan 2025 17:26:37 +0100 Subject: [PATCH 23/36] [Docker] Do not run container as root (#5314) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [Docker] Do not run container as root Fixes #5311 Signed-off-by: Jan Høydahl --- build.gradle | 7 ++++--- release/docker/Dockerfile | 8 +++++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 7f78c2424f..7c957ade81 100644 --- a/build.gradle +++ b/build.gradle @@ -350,7 +350,8 @@ coreProjects.each { coreProject -> def assembleTasks = collectTasksRecursively(coreProject, 'assemble') def publishTasks = collectTasksRecursively(coreProject, 'publish') - // Add these tasks as dependencies of the release task - release.dependsOn assembleTasks - release.dependsOn publishTasks + // Explicitly declare release task for better gradle compatibility + def releaseTask = tasks.named('release').get() + releaseTask.dependsOn assembleTasks + releaseTask.dependsOn publishTasks } \ No newline at end of file diff --git a/release/docker/Dockerfile b/release/docker/Dockerfile index dcc586dc52..4052d59909 100644 --- a/release/docker/Dockerfile +++ b/release/docker/Dockerfile @@ -11,9 +11,12 @@ ENV ENV_PIPELINE_FILEPATH=$PIPELINE_FILEPATH # Update all packages RUN dnf -y update -RUN dnf -y install bash bc +RUN dnf -y install bash bc shadow-utils RUN dnf -y upgrade +# Create a dedicated user and group with specific UID/GID +RUN useradd -u 1000 -M -U -d / -s /sbin/nologin -c "Data Prepper" data_prepper + # Setup the Adoptium package repo and install Temurin Java ADD adoptium.repo /etc/yum.repos.d/adoptium.repo RUN dnf -y install temurin-17-jdk @@ -25,5 +28,8 @@ RUN mv /usr/share/$ARCHIVE_FILE_UNPACKED /usr/share/data-prepper COPY default-data-prepper-config.yaml $ENV_CONFIG_FILEPATH COPY default-keystore.p12 /usr/share/data-prepper/keystore.p12 +RUN chown -R 1000:1000 $DATA_PREPPER_PATH /var/log/data-prepper +USER data_prepper + WORKDIR $DATA_PREPPER_PATH CMD ["bin/data-prepper"] From 9b07d4041e047d002a2dc961522405a54e740926 Mon Sep 17 00:00:00 2001 From: David Venable Date: Tue, 14 Jan 2025 10:35:50 -0800 Subject: [PATCH 24/36] Support cross account DynamoDB tables for streams. (#4776) Update requests to DynamoDB to provide the table ARN instead of the table name. This allows Data Prepper to use the new cross-account and resource policy changes available in DynamoDB. Resolves #4424 Signed-off-by: David Venable --- .../dynamodb/leader/LeaderScheduler.java | 5 ++--- .../source/dynamodb/utils/TableUtil.java | 10 ---------- .../dynamodb/leader/LeaderSchedulerTest.java | 18 +++++++++++++++--- .../source/dynamodb/utils/TableUtilTest.java | 6 ------ 4 files changed, 17 insertions(+), 22 deletions(-) diff --git a/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/leader/LeaderScheduler.java b/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/leader/LeaderScheduler.java index 005d518388..464c20d0ec 100644 --- a/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/leader/LeaderScheduler.java +++ b/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/leader/LeaderScheduler.java @@ -14,7 +14,6 @@ import org.opensearch.dataprepper.plugins.source.dynamodb.coordination.state.StreamProgressState; import org.opensearch.dataprepper.plugins.source.dynamodb.model.TableInfo; import org.opensearch.dataprepper.plugins.source.dynamodb.model.TableMetadata; -import org.opensearch.dataprepper.plugins.source.dynamodb.utils.TableUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.dynamodb.DynamoDbClient; @@ -234,8 +233,8 @@ private void compareAndCreateChildrenPartitions(List so * Conduct Metadata info for table and also perform validation on configuration. * Once created, the info should not be changed. */ - private TableInfo getTableInfo(TableConfig tableConfig) { - String tableName = TableUtil.getTableNameFromArn(tableConfig.getTableArn()); + private TableInfo getTableInfo(final TableConfig tableConfig) { + final String tableName = tableConfig.getTableArn(); DescribeTableResponse describeTableResult; try { // Need to call describe table to get the Key schema for table diff --git a/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/utils/TableUtil.java b/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/utils/TableUtil.java index 55a7b46a81..55d36ada97 100644 --- a/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/utils/TableUtil.java +++ b/data-prepper-plugins/dynamodb-source/src/main/java/org/opensearch/dataprepper/plugins/source/dynamodb/utils/TableUtil.java @@ -1,15 +1,7 @@ package org.opensearch.dataprepper.plugins.source.dynamodb.utils; -import software.amazon.awssdk.arns.Arn; - public class TableUtil { - public static String getTableNameFromArn(String tableArn) { - Arn arn = Arn.fromString(tableArn); - // resourceAsString is table/xxx - return arn.resourceAsString().substring("table/".length()); - } - public static String getTableArnFromStreamArn(String streamArn) { // e.g. Given a stream arn: arn:aws:dynamodb:us-west-2:xxx:table/test-table/stream/2023-07-31T04:59:58.190 // Returns arn:aws:dynamodb:us-west-2:xxx:table/test-table @@ -21,6 +13,4 @@ public static String getTableArnFromExportArn(String exportArn) { // returns: arn:aws:dynamodb:us-west-2:123456789012:table/Thread return exportArn.substring(0, exportArn.lastIndexOf("/export/")); } - - } diff --git a/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/leader/LeaderSchedulerTest.java b/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/leader/LeaderSchedulerTest.java index 245c107fec..ac523a29d9 100644 --- a/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/leader/LeaderSchedulerTest.java +++ b/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/leader/LeaderSchedulerTest.java @@ -3,6 +3,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.source.coordinator.enhanced.EnhancedSourceCoordinator; @@ -223,7 +224,10 @@ void test_should_init() throws InterruptedException { // Should call describe table to get basic table info - verify(dynamoDbClient).describeTable(any(DescribeTableRequest.class)); + ArgumentCaptor describeTableRequestArgumentCaptor = ArgumentCaptor.forClass(DescribeTableRequest.class); + verify(dynamoDbClient).describeTable(describeTableRequestArgumentCaptor.capture()); + DescribeTableRequest actualDescribeTableRequest = describeTableRequestArgumentCaptor.getValue(); + assertThat(actualDescribeTableRequest.tableName(), equalTo(tableArn)); // Should check PITR enabled or not verify(dynamoDbClient).describeContinuousBackups(any(DescribeContinuousBackupsRequest.class)); // Acquire the init partition @@ -252,7 +256,11 @@ void test_PITR_not_enabled_init_should_failed() throws InterruptedException { executorService.shutdownNow(); // Should call describe table to get basic table info - verify(dynamoDbClient).describeTable(any(DescribeTableRequest.class)); + ArgumentCaptor describeTableRequestArgumentCaptor = ArgumentCaptor.forClass(DescribeTableRequest.class); + verify(dynamoDbClient).describeTable(describeTableRequestArgumentCaptor.capture()); + DescribeTableRequest actualDescribeTableRequest = describeTableRequestArgumentCaptor.getValue(); + assertThat(actualDescribeTableRequest.tableName(), equalTo(tableArn)); + // Should check PITR enabled or not verify(dynamoDbClient).describeContinuousBackups(any(DescribeContinuousBackupsRequest.class)); @@ -277,7 +285,11 @@ void test_streaming_not_enabled_init_should_failed() throws InterruptedException executorService.shutdownNow(); // Should call describe table to get basic table info - verify(dynamoDbClient).describeTable(any(DescribeTableRequest.class)); + ArgumentCaptor describeTableRequestArgumentCaptor = ArgumentCaptor.forClass(DescribeTableRequest.class); + verify(dynamoDbClient).describeTable(describeTableRequestArgumentCaptor.capture()); + DescribeTableRequest actualDescribeTableRequest = describeTableRequestArgumentCaptor.getValue(); + assertThat(actualDescribeTableRequest.tableName(), equalTo(tableArn)); + // Should check PITR enabled or not verify(dynamoDbClient).describeContinuousBackups(any(DescribeContinuousBackupsRequest.class)); diff --git a/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/utils/TableUtilTest.java b/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/utils/TableUtilTest.java index ff1991b54f..c1718ab606 100644 --- a/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/utils/TableUtilTest.java +++ b/data-prepper-plugins/dynamodb-source/src/test/java/org/opensearch/dataprepper/plugins/source/dynamodb/utils/TableUtilTest.java @@ -15,12 +15,6 @@ class TableUtilTest { private final String exportArn = tableArn + "/export/01693291918297-bfeccbea"; private final String streamArn = tableArn + "/stream/2023-09-14T05:46:45.367"; - @Test - void test_getTableNameFromArn_should_return_tableName() { - String result = TableUtil.getTableNameFromArn(tableArn); - assertThat(result, equalTo(tableName)); - } - @Test void test_getTableArnFromStreamArn_should_return_tableArn() { String result = TableUtil.getTableArnFromStreamArn(streamArn); From 5f51c2bf091761b5fb55046e6aec8625f4c53be8 Mon Sep 17 00:00:00 2001 From: Qi Chen Date: Tue, 14 Jan 2025 15:51:57 -0500 Subject: [PATCH 25/36] FIX: include route validation error into collector (#5332) * MAINT: include route validation error into collector Signed-off-by: George Chen --- .../dataprepper/core/parser/PipelineTransformer.java | 12 ++++++------ .../core/parser/PipelineTransformerTests.java | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/PipelineTransformer.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/PipelineTransformer.java index 77cf649535..1b2a9f7d04 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/PipelineTransformer.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/PipelineTransformer.java @@ -176,10 +176,6 @@ private void buildPipelineFromConfiguration( .map(this::buildRoutedSinkOrConnector) .collect(Collectors.toList()); - final List subPipelinePluginErrors = pluginErrorCollector.getPluginErrors() - .stream().filter(pluginError -> pipelineName.equals(pluginError.getPipelineName())) - .collect(Collectors.toList()); - final List invalidRouteExpressions = pipelineConfiguration.getRoutes() .stream().filter(route -> !expressionEvaluator.isValidExpressionStatement(route.getCondition())) .map(route -> PluginError.builder() @@ -190,8 +186,12 @@ private void buildPipelineFromConfiguration( .build()) .collect(Collectors.toList()); - if (!subPipelinePluginErrors.isEmpty() || !invalidRouteExpressions.isEmpty()) { - subPipelinePluginErrors.addAll(invalidRouteExpressions); + invalidRouteExpressions.forEach(pluginErrorCollector::collectPluginError); + final List subPipelinePluginErrors = pluginErrorCollector.getPluginErrors() + .stream().filter(pluginError -> pipelineName.equals(pluginError.getPipelineName())) + .collect(Collectors.toList()); + + if (!subPipelinePluginErrors.isEmpty()) { pluginErrorsHandler.handleErrors(subPipelinePluginErrors); throw new InvalidPluginConfigurationException( String.format("One or more plugins are not configured correctly in the pipeline: %s.\n", diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/core/parser/PipelineTransformerTests.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/core/parser/PipelineTransformerTests.java index 9caa18820f..300850d9c9 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/core/parser/PipelineTransformerTests.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/core/parser/PipelineTransformerTests.java @@ -396,6 +396,7 @@ void parseConfiguration_with_invalid_route_expressions_handles_errors_and_return final Collection pluginErrorCollection = pluginErrorArgumentCaptor.getValue(); assertThat(pluginErrorCollection, notNullValue()); assertThat(pluginErrorCollection.size(), equalTo(1)); + assertThat(pluginErrorCollector.getPluginErrors(), equalTo(pluginErrorCollection)); final PluginError pluginError = pluginErrorCollection.stream().findAny().orElseThrow(); final String expectedErrorMessage = String.format(CONDITIONAL_ROUTE_INVALID_EXPRESSION_FORMAT, "service", "/value == service"); From c398e5551b671c3cafd8e2ab24d23db459b3306d Mon Sep 17 00:00:00 2001 From: Hai Yan <8153134+oeyh@users.noreply.github.com> Date: Tue, 14 Jan 2025 23:33:34 -0600 Subject: [PATCH 26/36] Add sleep time before checking (#5323) Signed-off-by: Hai Yan --- .../plugins/source/rds/export/DataFileSchedulerTest.java | 3 ++- .../plugins/source/rds/export/ExportSchedulerTest.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileSchedulerTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileSchedulerTest.java index 150ad209b2..7f021dd7be 100644 --- a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileSchedulerTest.java +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/DataFileSchedulerTest.java @@ -187,12 +187,13 @@ void test_data_file_loader_throws_exception_then_give_up_partition() { } @Test - void test_shutdown() { + void test_shutdown() throws InterruptedException { DataFileScheduler objectUnderTest = createObjectUnderTest(); final ExecutorService executorService = Executors.newSingleThreadExecutor(); executorService.submit(objectUnderTest); objectUnderTest.shutdown(); + Thread.sleep(100); verifyNoMoreInteractions(sourceCoordinator); executorService.shutdownNow(); diff --git a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportSchedulerTest.java b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportSchedulerTest.java index 43f08ff3fc..780078e490 100644 --- a/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportSchedulerTest.java +++ b/data-prepper-plugins/rds-source/src/test/java/org/opensearch/dataprepper/plugins/source/rds/export/ExportSchedulerTest.java @@ -262,12 +262,13 @@ void test_given_export_partition_and_null_export_task_id_then_close_partition_wi } @Test - void test_shutDown() { + void test_shutDown() throws InterruptedException { lenient().when(sourceCoordinator.acquireAvailablePartition(ExportPartition.PARTITION_TYPE)).thenReturn(Optional.empty()); final ExecutorService executorService = Executors.newSingleThreadExecutor(); executorService.submit(exportScheduler); exportScheduler.shutdown(); + Thread.sleep(100); verifyNoMoreInteractions(sourceCoordinator, snapshotManager, exportTaskManager, s3Client, exportJobSuccessCounter, exportJobFailureCounter, exportS3ObjectsTotalCounter); executorService.shutdownNow(); From f0480430ad60601150fe5093a3fd0282ab537d0c Mon Sep 17 00:00:00 2001 From: David Venable Date: Wed, 15 Jan 2025 07:31:20 -0800 Subject: [PATCH 27/36] Adds a merge function on the Event interface. This will merge in all the fields from another Event into the current event. Contributes toward #5312. (#5316) Signed-off-by: David Venable --- .../dataprepper/model/event/Event.java | 12 +++++ .../dataprepper/model/event/JacksonEvent.java | 19 +++++++- .../model/event/JacksonEventTest.java | 46 +++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java index e0e36d9237..fa83819a4d 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java @@ -110,6 +110,18 @@ public interface Event extends Serializable { */ void clear(); + /** + * Merges another Event into the current Event. + * The values from the other Event will overwrite the values in the current Event for all keys in the current Event. + * If the other Event has keys that are not in the current Event, they will be unmodified. + * + * @param other the other Event to merge into this Event + * @throws IllegalArgumentException if the input event is not compatible to merge. + * @throws UnsupportedOperationException if the current Event does not support merging. + * @since 2.11 + */ + void merge(Event other); + /** * Generates a serialized Json string of the entire Event * diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java index 09a0705e0e..741216becc 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java @@ -123,7 +123,6 @@ public static Event fromMessage(String message) { } private JsonNode getInitialJsonNode(final Object data) { - if (data == null) { return mapper.valueToTree(new HashMap<>()); } else if (data instanceof String) { @@ -348,6 +347,23 @@ public void clear() { } } + @Override + public void merge(final Event other) { + if(!(other instanceof JacksonEvent)) + throw new IllegalArgumentException("Unable to merge the Event. The input Event must be a JacksonEvent."); + final JacksonEvent otherJacksonEvent = (JacksonEvent) other; + if(!(otherJacksonEvent.jsonNode instanceof ObjectNode)) { + throw new IllegalArgumentException("Unable to merge the Event. The input Event must be a JacksonEvent with object data."); + } + final ObjectNode otherObjectNode = (ObjectNode) otherJacksonEvent.jsonNode; + + if(!(jsonNode instanceof ObjectNode)) { + throw new UnsupportedOperationException("Unable to merge the Event. The current Event must have object data."); + } + + ((ObjectNode) jsonNode).setAll(otherObjectNode); + } + @Override public String toJsonString() { return jsonNode.toString(); @@ -355,7 +371,6 @@ public String toJsonString() { @Override public String getAsJsonString(EventKey key) { - JacksonEventKey jacksonEventKey = asJacksonEventKey(key); final JsonNode node = getNode(jacksonEventKey); if (node.isMissingNode()) { diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java index d16bc345c8..a1fd74b1e1 100644 --- a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java @@ -446,6 +446,52 @@ public void testClear() { assertThat(event.toMap().size(), equalTo(0)); } + @Test + void merge_with_non_JacksonEvent_throws() { + final Event otherEvent = mock(Event.class); + assertThrows(IllegalArgumentException.class, () -> event.merge(otherEvent)); + } + + @Test + void merge_with_array_JsonNode_throws() { + final JacksonEvent otherEvent = (JacksonEvent) event; + event = JacksonEvent.builder().withEventType(EventType.DOCUMENT.toString()).withData(List.of(UUID.randomUUID().toString())).build(); + assertThrows(UnsupportedOperationException.class, () -> event.merge(otherEvent)); + } + + @Test + void merge_with_array_JsonNode_in_other_throws() { + Event otherEvent = JacksonEvent.builder().withEventType(EventType.DOCUMENT.toString()).withData(List.of(UUID.randomUUID().toString())).build(); + assertThrows(IllegalArgumentException.class, () -> event.merge(otherEvent)); + } + + @Test + void merge_sets_all_values() { + final String jsonString = "{\"a\": \"alpha\", \"info\": {\"ids\": {\"id\":\"idx\"}}}"; + event.put("b", "original"); + Event otherEvent = JacksonEvent.builder().withEventType(EventType.DOCUMENT.toString()).withData(jsonString).build(); + event.merge(otherEvent); + + assertThat(event.get("b", Object.class), equalTo("original")); + assertThat(event.get("a", Object.class), equalTo("alpha")); + assertThat(event.containsKey("info"), equalTo(true)); + assertThat(event.get("info/ids/id", String.class), equalTo("idx")); + } + + @Test + void merge_overrides_existing_values() { + final String jsonString = "{\"a\": \"alpha\", \"info\": {\"ids\": {\"id\":\"idx\"}}}"; + event.put("a", "original"); + event.put("b", "original"); + Event otherEvent = JacksonEvent.builder().withEventType(EventType.DOCUMENT.toString()).withData(jsonString).build(); + event.merge(otherEvent); + + assertThat(event.get("b", Object.class), equalTo("original")); + assertThat(event.get("a", Object.class), equalTo("alpha")); + assertThat(event.containsKey("info"), equalTo(true)); + assertThat(event.get("info/ids/id", String.class), equalTo("idx")); + } + @ParameterizedTest @ValueSource(strings = {"/", "foo", "/foo", "/foo/bar", "foo/bar", "foo/bar/", "/foo/bar/leaf/key"}) public void testDelete_withNonexistentKey(final String key) { From 80dc36df3709946fe84723fdc97e9e5b80c52bf6 Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Wed, 15 Jan 2025 10:18:11 -0600 Subject: [PATCH 28/36] Do not filter out objects based on last modified timestamp when delete_objects_on_read is enabled (#5319) Signed-off-by: Taylor Gray --- .../s3/S3ScanPartitionCreationSupplier.java | 8 +- .../plugins/source/s3/ScanObjectWorker.java | 2 +- .../S3ScanPartitionCreationSupplierTest.java | 133 +++++++++++++++++- 3 files changed, 139 insertions(+), 4 deletions(-) diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplier.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplier.java index 1d4bf6ea81..00422ef7d9 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplier.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplier.java @@ -50,17 +50,21 @@ public class S3ScanPartitionCreationSupplier implements Function scanOptionsList, final S3ScanSchedulingOptions schedulingOptions, - final FolderPartitioningOptions folderPartitioningOptions) { + final FolderPartitioningOptions folderPartitioningOptions, + final boolean deleteS3ObjectsOnRead) { this.s3Client = s3Client; this.bucketOwnerProvider = bucketOwnerProvider; this.scanOptionsList = scanOptionsList; this.schedulingOptions = schedulingOptions; this.folderPartitioningOptions = folderPartitioningOptions; + this.deleteS3ObjectsOnRead = deleteS3ObjectsOnRead; } @Override @@ -120,7 +124,7 @@ private List listFilteredS3ObjectsForBucket(final List isLastModifiedTimeAfterMostRecentScanForBucket(previousScanTime, s3Object)) + .filter(s3Object -> deleteS3ObjectsOnRead || isLastModifiedTimeAfterMostRecentScanForBucket(previousScanTime, s3Object)) .map(s3Object -> Pair.of(s3Object.key(), instantToLocalDateTime(s3Object.lastModified()))) .filter(keyTimestampPair -> !keyTimestampPair.left().endsWith("/")) .filter(keyTimestampPair -> excludeKeyPaths.stream() diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ScanObjectWorker.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ScanObjectWorker.java index 4ecd17c584..6de0f6e274 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ScanObjectWorker.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/ScanObjectWorker.java @@ -129,7 +129,7 @@ public ScanObjectWorker(final S3Client s3Client, this.folderPartitioningOptions = s3SourceConfig.getS3ScanScanOptions().getPartitioningOptions(); this.acknowledgmentSetTimeout = s3SourceConfig.getS3ScanScanOptions().getAcknowledgmentTimeout(); - this.partitionCreationSupplier = new S3ScanPartitionCreationSupplier(s3Client, bucketOwnerProvider, scanOptionsBuilderList, s3ScanSchedulingOptions, s3SourceConfig.getS3ScanScanOptions().getPartitioningOptions()); + this.partitionCreationSupplier = new S3ScanPartitionCreationSupplier(s3Client, bucketOwnerProvider, scanOptionsBuilderList, s3ScanSchedulingOptions, s3SourceConfig.getS3ScanScanOptions().getPartitioningOptions(), s3SourceConfig.isDeleteS3ObjectsOnRead()); this.acknowledgmentsRemainingForPartitions = new ConcurrentHashMap<>(); this.objectsToDeleteForAcknowledgmentSets = new ConcurrentHashMap<>(); } diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplierTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplierTest.java index 867bcb1dc5..2183f69d02 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplierTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/S3ScanPartitionCreationSupplierTest.java @@ -69,15 +69,18 @@ public class S3ScanPartitionCreationSupplierTest { private FolderPartitioningOptions folderPartitioningOptions; + private boolean isDeleteS3ObjectsOnRead; + @BeforeEach void setup() { scanOptionsList = new ArrayList<>(); folderPartitioningOptions = null; + isDeleteS3ObjectsOnRead = false; } private Function, List> createObjectUnderTest() { - return new S3ScanPartitionCreationSupplier(s3Client, bucketOwnerProvider, scanOptionsList, schedulingOptions, folderPartitioningOptions); + return new S3ScanPartitionCreationSupplier(s3Client, bucketOwnerProvider, scanOptionsList, schedulingOptions, folderPartitioningOptions, isDeleteS3ObjectsOnRead); } @Test @@ -452,4 +455,132 @@ void getNextPartition_with_folder_partitioning_enabled_returns_the_expected_part assertThat(resultingPartitions.stream().map(PartitionIdentifier::getPartitionKey).collect(Collectors.toList()), containsInAnyOrder(expectedPartitionIdentifiers.stream().map(PartitionIdentifier::getPartitionKey).map(Matchers::equalTo).collect(Collectors.toList()))); } + + @Test + void object_is_not_filtered_out_based_on_last_modified_timestamp_when_delete_objects_on_read_is_enabled() { + schedulingOptions = mock(S3ScanSchedulingOptions.class); + given(schedulingOptions.getInterval()).willReturn(Duration.ofMillis(0)); + given(schedulingOptions.getCount()).willReturn(2); + isDeleteS3ObjectsOnRead = true; + + final String firstBucket = "bucket-one"; + final String secondBucket = "bucket-two"; + + final ScanOptions firstBucketScanOptions = mock(ScanOptions.class); + final S3ScanBucketOption firstBucketScanBucketOption = mock(S3ScanBucketOption.class); + given(firstBucketScanOptions.getBucketOption()).willReturn(firstBucketScanBucketOption); + given(firstBucketScanBucketOption.getName()).willReturn(firstBucket); + given(firstBucketScanOptions.getUseStartDateTime()).willReturn(null); + given(firstBucketScanOptions.getUseEndDateTime()).willReturn(null); + final S3ScanKeyPathOption firstBucketScanKeyPath = mock(S3ScanKeyPathOption.class); + given(firstBucketScanBucketOption.getS3ScanFilter()).willReturn(firstBucketScanKeyPath); + given(firstBucketScanKeyPath.getS3scanIncludePrefixOptions()).willReturn(List.of(UUID.randomUUID().toString())); + given(firstBucketScanKeyPath.getS3ScanExcludeSuffixOptions()).willReturn(List.of(".invalid")); + scanOptionsList.add(firstBucketScanOptions); + + final ScanOptions secondBucketScanOptions = mock(ScanOptions.class); + final S3ScanBucketOption secondBucketScanBucketOption = mock(S3ScanBucketOption.class); + given(secondBucketScanOptions.getBucketOption()).willReturn(secondBucketScanBucketOption); + given(secondBucketScanBucketOption.getName()).willReturn(secondBucket); + given(secondBucketScanOptions.getUseStartDateTime()).willReturn(null); + given(secondBucketScanOptions.getUseEndDateTime()).willReturn(null); + final S3ScanKeyPathOption secondBucketScanKeyPath = mock(S3ScanKeyPathOption.class); + given(secondBucketScanBucketOption.getS3ScanFilter()).willReturn(secondBucketScanKeyPath); + given(secondBucketScanKeyPath.getS3scanIncludePrefixOptions()).willReturn(null); + given(secondBucketScanKeyPath.getS3ScanExcludeSuffixOptions()).willReturn(null); + scanOptionsList.add(secondBucketScanOptions); + + final Function, List> partitionCreationSupplier = createObjectUnderTest(); + + final List expectedPartitionIdentifiers = new ArrayList<>(); + + final ListObjectsV2Response listObjectsResponse = mock(ListObjectsV2Response.class); + final List s3ObjectsList = new ArrayList<>(); + + final S3Object invalidFolderObject = mock(S3Object.class); + given(invalidFolderObject.key()).willReturn("folder-key/"); + given(invalidFolderObject.lastModified()).willReturn(Instant.now()); + s3ObjectsList.add(invalidFolderObject); + + final S3Object invalidForFirstBucketSuffixObject = mock(S3Object.class); + given(invalidForFirstBucketSuffixObject.key()).willReturn("test.invalid"); + given(invalidForFirstBucketSuffixObject.lastModified()).willReturn(Instant.now().minusSeconds(2)); + s3ObjectsList.add(invalidForFirstBucketSuffixObject); + expectedPartitionIdentifiers.add(PartitionIdentifier.builder().withPartitionKey(secondBucket + "|" + invalidForFirstBucketSuffixObject.key()).build()); + + final Instant mostRecentFirstScan = Instant.now().plusSeconds(2); + final S3Object validObject = mock(S3Object.class); + given(validObject.key()).willReturn("valid"); + given(validObject.lastModified()).willReturn(mostRecentFirstScan); + s3ObjectsList.add(validObject); + expectedPartitionIdentifiers.add(PartitionIdentifier.builder().withPartitionKey(firstBucket + "|" + validObject.key()).build()); + expectedPartitionIdentifiers.add(PartitionIdentifier.builder().withPartitionKey(secondBucket + "|" + validObject.key()).build()); + + final S3Object secondScanObject = mock(S3Object.class); + final Instant mostRecentSecondScan = Instant.now().plusSeconds(10); + given(secondScanObject.key()).willReturn("second-scan"); + given(secondScanObject.lastModified()).willReturn(mostRecentSecondScan); + + final List expectedPartitionIdentifiersSecondScan = new ArrayList<>(); + expectedPartitionIdentifiersSecondScan.add(PartitionIdentifier.builder().withPartitionKey(firstBucket + "|" + secondScanObject.key()).build()); + expectedPartitionIdentifiersSecondScan.add(PartitionIdentifier.builder().withPartitionKey(firstBucket + "|" + validObject.key()).build()); + expectedPartitionIdentifiersSecondScan.add(PartitionIdentifier.builder().withPartitionKey(secondBucket + "|" + secondScanObject.key()).build()); + expectedPartitionIdentifiersSecondScan.add(PartitionIdentifier.builder().withPartitionKey(secondBucket + "|" + validObject.key()).build()); + + // Since delete objects on read is enabled, the second scan will pick up the same object from the first scan + expectedPartitionIdentifiersSecondScan.add(PartitionIdentifier.builder().withPartitionKey(secondBucket + "|" + invalidForFirstBucketSuffixObject.key()).build()); + + final List secondScanObjects = new ArrayList<>(s3ObjectsList); + secondScanObjects.add(secondScanObject); + given(listObjectsResponse.contents()) + .willReturn(s3ObjectsList) + .willReturn(s3ObjectsList) + .willReturn(secondScanObjects) + .willReturn(secondScanObjects); + + final ArgumentCaptor listObjectsV2RequestArgumentCaptor = ArgumentCaptor.forClass(ListObjectsV2Request.class); + given(s3Client.listObjectsV2(listObjectsV2RequestArgumentCaptor.capture())).willReturn(listObjectsResponse); + + final Map globalStateMap = new HashMap<>(); + + final Instant beforeFirstScan = Instant.now(); + final List resultingPartitions = partitionCreationSupplier.apply(globalStateMap); + + assertThat(resultingPartitions, notNullValue()); + assertThat(resultingPartitions.size(), equalTo(expectedPartitionIdentifiers.size())); + assertThat(resultingPartitions.stream().map(PartitionIdentifier::getPartitionKey).collect(Collectors.toList()), + containsInAnyOrder(expectedPartitionIdentifiers.stream().map(PartitionIdentifier::getPartitionKey) + .map(Matchers::equalTo).collect(Collectors.toList()))); + + assertThat(globalStateMap, notNullValue()); + assertThat(globalStateMap.containsKey(SCAN_COUNT), equalTo(true)); + assertThat(globalStateMap.get(SCAN_COUNT), equalTo(1)); + assertThat(globalStateMap.containsKey(firstBucket), equalTo(true)); + assertThat(Instant.parse((CharSequence) globalStateMap.get(firstBucket)), lessThanOrEqualTo(mostRecentFirstScan)); + assertThat(Instant.parse((CharSequence) globalStateMap.get(firstBucket)), greaterThanOrEqualTo(beforeFirstScan)); + assertThat(globalStateMap.containsKey(secondBucket), equalTo(true)); + assertThat(Instant.parse((CharSequence) globalStateMap.get(secondBucket)), lessThanOrEqualTo(mostRecentFirstScan)); + assertThat(Instant.parse((CharSequence) globalStateMap.get(secondBucket)), greaterThanOrEqualTo(beforeFirstScan)); + + final Instant beforeSecondScan = Instant.now(); + final List secondScanPartitions = partitionCreationSupplier.apply(globalStateMap); + assertThat(secondScanPartitions.size(), equalTo(expectedPartitionIdentifiersSecondScan.size())); + assertThat(secondScanPartitions.stream().map(PartitionIdentifier::getPartitionKey).collect(Collectors.toList()), + containsInAnyOrder(expectedPartitionIdentifiersSecondScan.stream().map(PartitionIdentifier::getPartitionKey).map(Matchers::equalTo).collect(Collectors.toList()))); + + assertThat(globalStateMap, notNullValue()); + assertThat(globalStateMap.containsKey(SCAN_COUNT), equalTo(true)); + assertThat(globalStateMap.get(SCAN_COUNT), equalTo(2)); + assertThat(globalStateMap.containsKey(firstBucket), equalTo(true)); + assertThat(Instant.parse((CharSequence) globalStateMap.get(firstBucket)), lessThanOrEqualTo(mostRecentSecondScan)); + assertThat(Instant.parse((CharSequence) globalStateMap.get(firstBucket)), greaterThanOrEqualTo(beforeSecondScan)); + assertThat(globalStateMap.containsKey(secondBucket), equalTo(true)); + assertThat(Instant.parse((CharSequence) globalStateMap.get(secondBucket)), lessThanOrEqualTo(mostRecentSecondScan)); + assertThat(Instant.parse((CharSequence) globalStateMap.get(secondBucket)), greaterThan(beforeSecondScan)); + assertThat(Instant.ofEpochMilli((Long) globalStateMap.get(LAST_SCAN_TIME)).isBefore(Instant.now()), equalTo(true)); + + assertThat(partitionCreationSupplier.apply(globalStateMap), equalTo(Collections.emptyList())); + + verify(listObjectsResponse, times(4)).contents(); + } } From b13a64586838a44840881a50974520873bb8acd2 Mon Sep 17 00:00:00 2001 From: David Venable Date: Wed, 15 Jan 2025 09:27:35 -0800 Subject: [PATCH 29/36] Introduces the experimental plugin feature (#5318) Introduces the experimental plugin feature, allowing plugin developers to mark plugins as experimental. This PR adds a new annotation @Experimental which can be applied to plugin types. When loading plugins, the plugin framework will now check whether the plugin is experimental or not. If the plugin is experimental, Data Prepper will fail if experimental plugins are not enabled. This PR also adds a new configuration in data-prepper-config.yaml to enable all experimental plugins if they are desired. Additionally, I refactored the code that logs deprecated plugin names into a consumer to help the overall code structure and fit the code design I took for checking for experimental plugins. Resolves #2695 Signed-off-by: David Venable --- .../model/annotations/Experimental.java | 33 ++++++ .../plugin/DefaultPluginFactoryIT.java | 22 ++++ .../model/DataPrepperConfiguration.java | 13 ++- .../core/parser/PipelineTransformerTests.java | 1 + .../plugins/TestExperimentalPlugin.java | 19 ++++ data-prepper-plugin-framework/build.gradle | 1 + .../plugin/DefaultPluginFactory.java | 18 ++-- .../dataprepper/plugin/DefinedPlugin.java | 30 ++++++ .../plugin/DeprecatedPluginDetector.java | 35 ++++++ .../plugin/ExperimentalConfiguration.java | 35 ++++++ .../ExperimentalConfigurationContainer.java | 25 +++++ .../plugin/ExperimentalPluginValidator.java | 37 +++++++ .../plugin/DefaultPluginFactoryTest.java | 41 ++++++- .../plugin/DeprecatedPluginDetectorTest.java | 100 ++++++++++++++++++ .../plugin/ExperimentalConfigurationTest.java | 25 +++++ .../ExperimentalPluginValidatorTest.java | 92 ++++++++++++++++ 16 files changed, 517 insertions(+), 10 deletions(-) create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/Experimental.java create mode 100644 data-prepper-core/src/test/java/org/opensearch/dataprepper/plugins/TestExperimentalPlugin.java create mode 100644 data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefinedPlugin.java create mode 100644 data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DeprecatedPluginDetector.java create mode 100644 data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfiguration.java create mode 100644 data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfigurationContainer.java create mode 100644 data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalPluginValidator.java create mode 100644 data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DeprecatedPluginDetectorTest.java create mode 100644 data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ExperimentalConfigurationTest.java create mode 100644 data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ExperimentalPluginValidatorTest.java diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/Experimental.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/Experimental.java new file mode 100644 index 0000000000..8b97475f54 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/Experimental.java @@ -0,0 +1,33 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.model.annotations; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Marks a Data Prepper plugin as experimental. + *

+ * Experimental plugins do not have the same compatibility guarantees as other plugins and may be unstable. + * They may have breaking changes between minor versions and may even be removed. + *

+ * Data Prepper administrators must enable experimental plugins in order to use them. + * Otherwise, they are not available to use with pipelines. + * + * @since 2.11 + */ +@Documented +@Retention(RetentionPolicy.RUNTIME) +@Target({ElementType.TYPE}) +public @interface Experimental { +} diff --git a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java index 27dfa97cef..9230aa7ff8 100644 --- a/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java +++ b/data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java @@ -14,6 +14,7 @@ import org.opensearch.dataprepper.core.event.EventFactoryApplicationContextMarker; import org.opensearch.dataprepper.core.validation.LoggingPluginErrorsHandler; import org.opensearch.dataprepper.core.validation.PluginErrorCollector; +import org.opensearch.dataprepper.model.plugin.NoPluginFoundException; import org.opensearch.dataprepper.plugins.configtest.TestComponentWithConfigInject; import org.opensearch.dataprepper.plugins.configtest.TestDISourceWithConfig; import org.opensearch.dataprepper.validation.PluginErrorsHandler; @@ -27,6 +28,7 @@ import org.opensearch.dataprepper.plugins.test.TestPlugin; import org.springframework.context.annotation.AnnotationConfigApplicationContext; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.UUID; @@ -38,6 +40,7 @@ import static org.junit.jupiter.api.Assertions.assertInstanceOf; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; /** * Integration test of the plugin framework. These tests should not mock any portion @@ -49,6 +52,8 @@ class DefaultPluginFactoryIT { private PipelinesDataFlowModel pipelinesDataFlowModel; @Mock private ExtensionsConfiguration extensionsConfiguration; + @Mock + private ExperimentalConfigurationContainer experimentalConfigurationContainer; private String pluginName; private String objectPluginName; private String pipelineName; @@ -67,6 +72,8 @@ private DefaultPluginFactory createObjectUnderTest() { final AnnotationConfigApplicationContext coreContext = new AnnotationConfigApplicationContext(); coreContext.setParent(publicContext); + when(experimentalConfigurationContainer.getExperimental()).thenReturn(ExperimentalConfiguration.defaultConfiguration()); + coreContext.scan(EventFactoryApplicationContextMarker.class.getPackage().getName()); coreContext.scan(DefaultAcknowledgementSetManager.class.getPackage().getName()); coreContext.scan(DefaultPluginFactory.class.getPackage().getName()); @@ -75,6 +82,7 @@ private DefaultPluginFactory createObjectUnderTest() { coreContext.registerBean(PluginErrorsHandler.class, LoggingPluginErrorsHandler::new); coreContext.registerBean(ExtensionsConfiguration.class, () -> extensionsConfiguration); coreContext.registerBean(PipelinesDataFlowModel.class, () -> pipelinesDataFlowModel); + coreContext.registerBean(ExperimentalConfigurationContainer.class, () -> experimentalConfigurationContainer); coreContext.refresh(); return coreContext.getBean(DefaultPluginFactory.class); @@ -188,6 +196,20 @@ void loadPlugin_should_throw_when_a_plugin_configuration_is_invalid() { assertThat(actualException.getMessage(), equalTo("Plugin test_plugin in pipeline " + pipelineName + " is configured incorrectly: requiredString must not be null")); } + @Test + void loadPlugin_should_throw_when_a_plugin_is_experimental_by_default() { + pluginName = "test_experimental_plugin"; + final PluginSetting pluginSetting = createPluginSettings(Collections.emptyMap()); + + final DefaultPluginFactory objectUnderTest = createObjectUnderTest(); + + final NoPluginFoundException actualException = assertThrows(NoPluginFoundException.class, + () -> objectUnderTest.loadPlugin(TestPluggableInterface.class, pluginSetting)); + + assertThat(actualException.getMessage(), notNullValue()); + assertThat(actualException.getMessage(), equalTo("Unable to create experimental plugin test_experimental_plugin. You must enable experimental plugins in data-prepper-config.yaml in order to use them.")); + } + private PluginSetting createPluginSettings(final Map pluginSettingMap) { final PluginSetting pluginSetting = new PluginSetting(pluginName, pluginSettingMap); pluginSetting.setPipelineName(pipelineName); diff --git a/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/model/DataPrepperConfiguration.java b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/model/DataPrepperConfiguration.java index 9212a9943b..046653dcd6 100644 --- a/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/model/DataPrepperConfiguration.java +++ b/data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/model/DataPrepperConfiguration.java @@ -18,6 +18,8 @@ import org.opensearch.dataprepper.core.pipeline.PipelineShutdownOption; import org.opensearch.dataprepper.model.configuration.PipelineExtensions; import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.plugin.ExperimentalConfiguration; +import org.opensearch.dataprepper.plugin.ExperimentalConfigurationContainer; import org.opensearch.dataprepper.plugin.ExtensionsConfiguration; import java.time.Duration; @@ -31,7 +33,7 @@ /** * Class to hold configuration for DataPrepper, including server port and Log4j settings */ -public class DataPrepperConfiguration implements ExtensionsConfiguration, EventConfigurationContainer { +public class DataPrepperConfiguration implements ExtensionsConfiguration, EventConfigurationContainer, ExperimentalConfigurationContainer { static final Duration DEFAULT_SHUTDOWN_DURATION = Duration.ofSeconds(30L); private static final String DEFAULT_SOURCE_COORDINATION_STORE = "in_memory"; @@ -55,6 +57,7 @@ public class DataPrepperConfiguration implements ExtensionsConfiguration, EventC private PeerForwarderConfiguration peerForwarderConfiguration; private Duration processorShutdownTimeout; private Duration sinkShutdownTimeout; + private ExperimentalConfiguration experimental; private PipelineExtensions pipelineExtensions; public static final DataPrepperConfiguration DEFAULT_CONFIG = new DataPrepperConfiguration(); @@ -96,6 +99,7 @@ public DataPrepperConfiguration( @JsonProperty("source_coordination") final SourceCoordinationConfig sourceCoordinationConfig, @JsonProperty("pipeline_shutdown") final PipelineShutdownOption pipelineShutdown, @JsonProperty("event") final EventConfiguration eventConfiguration, + @JsonProperty("experimental") final ExperimentalConfiguration experimental, @JsonProperty("extensions") @JsonInclude(JsonInclude.Include.NON_NULL) @JsonSetter(nulls = Nulls.SKIP) @@ -126,6 +130,8 @@ public DataPrepperConfiguration( if (this.sinkShutdownTimeout.isNegative()) { throw new IllegalArgumentException("sinkShutdownTimeout must be non-negative."); } + this.experimental = experimental != null ? experimental : ExperimentalConfiguration.defaultConfiguration(); + this.pipelineExtensions = pipelineExtensions; } @@ -239,4 +245,9 @@ public EventConfiguration getEventConfiguration() { public PipelineExtensions getPipelineExtensions() { return pipelineExtensions; } + + @Override + public ExperimentalConfiguration getExperimental() { + return experimental; + } } diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/core/parser/PipelineTransformerTests.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/core/parser/PipelineTransformerTests.java index 300850d9c9..3f2f03a1d9 100644 --- a/data-prepper-core/src/test/java/org/opensearch/dataprepper/core/parser/PipelineTransformerTests.java +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/core/parser/PipelineTransformerTests.java @@ -140,6 +140,7 @@ void setUp() { @AfterEach void tearDown() { verify(dataPrepperConfiguration).getEventConfiguration(); + verify(dataPrepperConfiguration).getExperimental(); verifyNoMoreInteractions(dataPrepperConfiguration); } diff --git a/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugins/TestExperimentalPlugin.java b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugins/TestExperimentalPlugin.java new file mode 100644 index 0000000000..36f334e526 --- /dev/null +++ b/data-prepper-core/src/test/java/org/opensearch/dataprepper/plugins/TestExperimentalPlugin.java @@ -0,0 +1,19 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugins; + +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.annotations.Experimental; +import org.opensearch.dataprepper.plugin.TestPluggableInterface; + +@DataPrepperPlugin(name = "test_experimental_plugin", pluginType = TestPluggableInterface.class) +@Experimental +public class TestExperimentalPlugin { +} diff --git a/data-prepper-plugin-framework/build.gradle b/data-prepper-plugin-framework/build.gradle index 8eff39b1dc..894132b473 100644 --- a/data-prepper-plugin-framework/build.gradle +++ b/data-prepper-plugin-framework/build.gradle @@ -25,4 +25,5 @@ dependencies { implementation libs.reflections.core implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'org.apache.commons:commons-text:1.10.0' + testImplementation 'ch.qos.logback:logback-classic:1.5.16' } \ No newline at end of file diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java index 81fd1c2b5b..0ec3b5a953 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Objects; import java.util.Optional; +import java.util.function.Consumer; import java.util.function.Function; /** @@ -41,6 +42,7 @@ public class DefaultPluginFactory implements PluginFactory { private final PluginBeanFactoryProvider pluginBeanFactoryProvider; private final PluginConfigurationObservableFactory pluginConfigurationObservableFactory; private final ApplicationContextToTypedSuppliers applicationContextToTypedSuppliers; + private final List>> definedPluginConsumers; @Inject DefaultPluginFactory( @@ -49,8 +51,10 @@ public class DefaultPluginFactory implements PluginFactory { final PluginConfigurationConverter pluginConfigurationConverter, final PluginBeanFactoryProvider pluginBeanFactoryProvider, final PluginConfigurationObservableFactory pluginConfigurationObservableFactory, - final ApplicationContextToTypedSuppliers applicationContextToTypedSuppliers) { + final ApplicationContextToTypedSuppliers applicationContextToTypedSuppliers, + final List>> definedPluginConsumers) { this.applicationContextToTypedSuppliers = applicationContextToTypedSuppliers; + this.definedPluginConsumers = definedPluginConsumers; Objects.requireNonNull(pluginProviderLoader); Objects.requireNonNull(pluginConfigurationObservableFactory); this.pluginCreator = Objects.requireNonNull(pluginCreator); @@ -140,15 +144,13 @@ private Class getPluginClass(final Class baseClass, final St .orElseThrow(() -> new NoPluginFoundException( "Unable to find a plugin named '" + pluginName + "'. Please ensure that plugin is annotated with appropriate values.")); - logDeprecatedPluginsNames(pluginClass, pluginName); + handleDefinedPlugins(pluginClass, pluginName); return pluginClass; } - private void logDeprecatedPluginsNames(final Class pluginClass, final String pluginName) { - final String deprecatedName = pluginClass.getAnnotation(DataPrepperPlugin.class).deprecatedName(); - final String name = pluginClass.getAnnotation(DataPrepperPlugin.class).name(); - if (deprecatedName.equals(pluginName)) { - LOG.warn("Plugin name '{}' is deprecated and will be removed in the next major release. Consider using the updated plugin name '{}'.", deprecatedName, name); - } + private void handleDefinedPlugins(final Class pluginClass, final String pluginName) { + final DefinedPlugin definedPlugin = new DefinedPlugin<>(pluginClass, pluginName); + + definedPluginConsumers.forEach(definedPluginConsumer -> definedPluginConsumer.accept(definedPlugin)); } } diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefinedPlugin.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefinedPlugin.java new file mode 100644 index 0000000000..7f0b550a89 --- /dev/null +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefinedPlugin.java @@ -0,0 +1,30 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugin; + +import java.util.Objects; + +class DefinedPlugin { + private final Class pluginClass; + private final String pluginName; + + public DefinedPlugin(final Class pluginClass, final String pluginName) { + this.pluginClass = Objects.requireNonNull(pluginClass); + this.pluginName = Objects.requireNonNull(pluginName); + } + + public Class getPluginClass() { + return pluginClass; + } + + public String getPluginName() { + return pluginName; + } +} diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DeprecatedPluginDetector.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DeprecatedPluginDetector.java new file mode 100644 index 0000000000..aab8a8120f --- /dev/null +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DeprecatedPluginDetector.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugin; + +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.inject.Named; +import java.util.function.Consumer; + +@Named +class DeprecatedPluginDetector implements Consumer> { + private static final Logger LOG = LoggerFactory.getLogger(DeprecatedPluginDetector.class); + + @Override + public void accept(final DefinedPlugin definedPlugin) { + logDeprecatedPluginsNames(definedPlugin.getPluginClass(), definedPlugin.getPluginName()); + } + + private void logDeprecatedPluginsNames(final Class pluginClass, final String pluginName) { + final String deprecatedName = pluginClass.getAnnotation(DataPrepperPlugin.class).deprecatedName(); + final String name = pluginClass.getAnnotation(DataPrepperPlugin.class).name(); + if (deprecatedName.equals(pluginName)) { + LOG.warn("Plugin name '{}' is deprecated and will be removed in the next major release. Consider using the updated plugin name '{}'.", deprecatedName, name); + } + } +} diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfiguration.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfiguration.java new file mode 100644 index 0000000000..5a30e529a4 --- /dev/null +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfiguration.java @@ -0,0 +1,35 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugin; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * Data Prepper configurations for experimental features. + * + * @since 2.11 + */ +public class ExperimentalConfiguration { + @JsonProperty("enable_all") + private boolean enableAll = false; + + public static ExperimentalConfiguration defaultConfiguration() { + return new ExperimentalConfiguration(); + } + + /** + * Gets whether all experimental features are enabled. + * @return true if all experimental features are enabled, false otherwise + * @since 2.11 + */ + public boolean isEnableAll() { + return enableAll; + } +} diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfigurationContainer.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfigurationContainer.java new file mode 100644 index 0000000000..e2ba00657f --- /dev/null +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfigurationContainer.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugin; + +/** + * Interface to decouple how an experimental configuration is defined from + * usage of those configurations. + * + * @since 2.11 + */ +public interface ExperimentalConfigurationContainer { + /** + * Gets the experimental configuration. + * @return the experimental configuration + * @since 2.11 + */ + ExperimentalConfiguration getExperimental(); +} diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalPluginValidator.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalPluginValidator.java new file mode 100644 index 0000000000..ffd835de41 --- /dev/null +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalPluginValidator.java @@ -0,0 +1,37 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugin; + +import org.opensearch.dataprepper.model.annotations.Experimental; +import org.opensearch.dataprepper.model.plugin.NoPluginFoundException; + +import javax.inject.Named; +import java.util.function.Consumer; + +@Named +class ExperimentalPluginValidator implements Consumer> { + private final ExperimentalConfiguration experimentalConfiguration; + + ExperimentalPluginValidator(final ExperimentalConfigurationContainer experimentalConfigurationContainer) { + this.experimentalConfiguration = experimentalConfigurationContainer.getExperimental(); + } + + @Override + public void accept(final DefinedPlugin definedPlugin) { + if(isPluginDisallowedAsExperimental(definedPlugin.getPluginClass())) { + throw new NoPluginFoundException("Unable to create experimental plugin " + definedPlugin.getPluginName() + + ". You must enable experimental plugins in data-prepper-config.yaml in order to use them."); + } + } + + private boolean isPluginDisallowedAsExperimental(final Class pluginClass) { + return pluginClass.isAnnotationPresent(Experimental.class) && !experimentalConfiguration.isEnableAll(); + } +} diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java index 6f54a55c95..2c1bf9e0fa 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryTest.java @@ -30,6 +30,7 @@ import java.util.Map; import java.util.Optional; import java.util.UUID; +import java.util.function.Consumer; import java.util.function.Supplier; import static org.hamcrest.CoreMatchers.equalTo; @@ -38,6 +39,7 @@ import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.sameInstance; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -64,6 +66,7 @@ class DefaultPluginFactoryTest { private PluginConfigurationObservableFactory pluginConfigurationObservableFactory; private PluginConfigObservable pluginConfigObservable; private ApplicationContextToTypedSuppliers applicationContextToTypedSuppliers; + private List>> definedPluginConsumers; @BeforeEach void setUp() { @@ -92,6 +95,8 @@ void setUp() { )).willReturn(pluginConfigObservable); applicationContextToTypedSuppliers = mock(ApplicationContextToTypedSuppliers.class); + + definedPluginConsumers = List.of(mock(Consumer.class), mock(Consumer.class)); } private DefaultPluginFactory createObjectUnderTest() { @@ -99,7 +104,8 @@ private DefaultPluginFactory createObjectUnderTest() { pluginProviderLoader, pluginCreator, pluginConfigurationConverter, beanFactoryProvider, pluginConfigurationObservableFactory, - applicationContextToTypedSuppliers); + applicationContextToTypedSuppliers, + definedPluginConsumers); } @Test @@ -230,6 +236,22 @@ void loadPlugin_should_create_a_new_instance_of_the_first_plugin_found() { verify(beanFactoryProvider).createPluginSpecificContext(new Class[]{}, convertedConfiguration); } + @Test + void loadPlugin_should_call_all_definedPluginConsumers() { + createObjectUnderTest().loadPlugin(baseClass, pluginSetting); + + assertThat("This test is not valid if there are no defined plugin consumers.", + definedPluginConsumers.size(), greaterThanOrEqualTo(2)); + for (final Consumer> definedPluginConsumer : definedPluginConsumers) { + final ArgumentCaptor> definedPluginArgumentCaptor = ArgumentCaptor.forClass(DefinedPlugin.class); + verify(definedPluginConsumer).accept(definedPluginArgumentCaptor.capture()); + + final DefinedPlugin actualDefinedPlugin = definedPluginArgumentCaptor.getValue(); + assertThat(actualDefinedPlugin.getPluginClass(), equalTo(expectedPluginClass)); + assertThat(actualDefinedPlugin.getPluginName(), equalTo(pluginName)); + } + } + @Test void loadPlugins_should_throw_for_null_number_of_instances() { @@ -322,6 +344,23 @@ void loadPlugin_with_varargs_should_return_a_single_instance_when_the_the_number assertThat(plugin, equalTo(expectedInstance)); } + @Test + void loadPlugin_with_varargs_should_call_all_definedPluginConsumers() { + final Object vararg1 = new Object(); + createObjectUnderTest().loadPlugin(baseClass, pluginSetting, vararg1); + + assertThat("This test is not valid if there are no defined plugin consumers.", + definedPluginConsumers.size(), greaterThanOrEqualTo(2)); + for (final Consumer> definedPluginConsumer : definedPluginConsumers) { + final ArgumentCaptor> definedPluginArgumentCaptor = ArgumentCaptor.forClass(DefinedPlugin.class); + verify(definedPluginConsumer).accept(definedPluginArgumentCaptor.capture()); + + final DefinedPlugin actualDefinedPlugin = definedPluginArgumentCaptor.getValue(); + assertThat(actualDefinedPlugin.getPluginClass(), equalTo(expectedPluginClass)); + assertThat(actualDefinedPlugin.getPluginName(), equalTo(pluginName)); + } + } + @Test void loadPlugins_should_return_an_instance_for_the_total_count() { final TestSink expectedInstance1 = mock(TestSink.class); diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DeprecatedPluginDetectorTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DeprecatedPluginDetectorTest.java new file mode 100644 index 0000000000..f5325968a2 --- /dev/null +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/DeprecatedPluginDetectorTest.java @@ -0,0 +1,100 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugin; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.AppenderBase; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin; +import org.opensearch.dataprepper.model.processor.Processor; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.collection.IsEmptyCollection.empty; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class DeprecatedPluginDetectorTest { + @Mock + private DefinedPlugin definedPlugin; + private TestLogAppender testAppender; + + @BeforeEach + void setUp() { + final Logger logger = (Logger) LoggerFactory.getLogger(DeprecatedPluginDetector.class); + + testAppender = new TestLogAppender(); + testAppender.start(); + logger.addAppender(testAppender); + } + + private DeprecatedPluginDetector createObjectUnderTest() { + return new DeprecatedPluginDetector(); + } + + @Test + void accept_on_plugin_without_deprecated_name_does_not_log() { + when(definedPlugin.getPluginClass()).thenReturn(PluginWithoutDeprecatedName.class); + createObjectUnderTest().accept(definedPlugin); + + assertThat(testAppender.getLoggedEvents(), empty()); + } + + @Test + void accept_on_plugin_with_deprecated_name_does_not_log_if_new_name_is_used() { + when(definedPlugin.getPluginClass()).thenReturn(PluginWithDeprecatedName.class); + when(definedPlugin.getPluginName()).thenReturn("test_for_deprecated_detection"); + createObjectUnderTest().accept(definedPlugin); + + assertThat(testAppender.getLoggedEvents(), empty()); + } + + @Test + void accept_on_plugin_with_deprecated_name_logs_if_deprecated_name_is_used() { + when(definedPlugin.getPluginClass()).thenReturn(PluginWithDeprecatedName.class); + when(definedPlugin.getPluginName()).thenReturn("test_for_deprecated_detection_deprecated_name"); + createObjectUnderTest().accept(definedPlugin); + + assertThat(testAppender.getLoggedEvents().stream() + .anyMatch(event -> event.getFormattedMessage().contains("Plugin name 'test_for_deprecated_detection_deprecated_name' is deprecated and will be removed in the next major release. Consider using the updated plugin name 'test_for_deprecated_detection'.")), + equalTo(true)); + } + + @DataPrepperPlugin(name = "test_for_deprecated_detection", pluginType = Processor.class) + private static class PluginWithoutDeprecatedName { + } + + @DataPrepperPlugin(name = "test_for_deprecated_detection", pluginType = Processor.class, deprecatedName = "test_for_deprecated_detection_deprecated_name") + private static class PluginWithDeprecatedName { + } + + public static class TestLogAppender extends AppenderBase { + private final List events = new ArrayList<>(); + + @Override + protected void append(final ILoggingEvent eventObject) { + events.add(eventObject); + } + + public List getLoggedEvents() { + return Collections.unmodifiableList(events); + } + } +} \ No newline at end of file diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ExperimentalConfigurationTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ExperimentalConfigurationTest.java new file mode 100644 index 0000000000..0a3df6a892 --- /dev/null +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ExperimentalConfigurationTest.java @@ -0,0 +1,25 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugin; + +import org.junit.jupiter.api.Test; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; + +class ExperimentalConfigurationTest { + @Test + void defaultConfiguration_should_return_config_with_isEnableAll_false() { + final ExperimentalConfiguration objectUnderTest = ExperimentalConfiguration.defaultConfiguration(); + assertThat(objectUnderTest, notNullValue()); + assertThat(objectUnderTest.isEnableAll(), equalTo(false)); + } +} \ No newline at end of file diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ExperimentalPluginValidatorTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ExperimentalPluginValidatorTest.java new file mode 100644 index 0000000000..78d008b222 --- /dev/null +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/ExperimentalPluginValidatorTest.java @@ -0,0 +1,92 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.dataprepper.plugin; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.annotations.Experimental; +import org.opensearch.dataprepper.model.plugin.NoPluginFoundException; + +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ExperimentalPluginValidatorTest { + + @Mock + private ExperimentalConfigurationContainer experimentalConfigurationContainer; + + @Mock + private ExperimentalConfiguration experimentalConfiguration; + + @Mock + private DefinedPlugin definedPlugin; + + @BeforeEach + void setUp() { + when(experimentalConfigurationContainer.getExperimental()).thenReturn(experimentalConfiguration); + } + + private ExperimentalPluginValidator createObjectUnderTest() { + return new ExperimentalPluginValidator(experimentalConfigurationContainer); + } + + @Test + void accept_with_non_Experimental_plugin_returns() { + when(definedPlugin.getPluginClass()).thenReturn(NonExperimentalPlugin.class); + + createObjectUnderTest().accept(definedPlugin); + } + + @Nested + class WithExperimentalPlugin { + @BeforeEach + void setUp() { + when(definedPlugin.getPluginClass()).thenReturn(ExperimentalPlugin.class); + } + + @Test + void accept_with_Experimental_plugin_throws_if_experimental_is_not_enabled() { + final String pluginName = UUID.randomUUID().toString(); + when(definedPlugin.getPluginName()).thenReturn(pluginName); + + final ExperimentalPluginValidator objectUnderTest = createObjectUnderTest(); + + final NoPluginFoundException actualException = assertThrows(NoPluginFoundException.class, () -> objectUnderTest.accept(definedPlugin)); + + assertThat(actualException.getMessage(), notNullValue()); + assertThat(actualException.getMessage(), containsString(pluginName)); + assertThat(actualException.getMessage(), containsString("experimental plugin")); + } + + @Test + void accept_with_Experimental_plugin_does_not_throw_if_experimental_is_enabled() { + when(experimentalConfiguration.isEnableAll()).thenReturn(true); + + createObjectUnderTest().accept(definedPlugin); + } + } + + private static class NonExperimentalPlugin { + } + + @Experimental + private static class ExperimentalPlugin { + } +} \ No newline at end of file From 7c3681f284070998c1fa5e64c30a9eca5cf4f643 Mon Sep 17 00:00:00 2001 From: Jeremy Michael <60355474+jmsusanto@users.noreply.github.com> Date: Wed, 15 Jan 2025 13:06:21 -0800 Subject: [PATCH 30/36] sqs source: json codec support to split sqs message into multiple events (#5330) * added json codec support and functionality to split message into multiple events Signed-off-by: Jeremy Michael * Added message strategy and improved metadata handler efficiency Signed-off-by: Jeremy Michael * updated license Signed-off-by: Jeremy Michael * minor changes Signed-off-by: Jeremy Michael --------- Signed-off-by: Jeremy Michael Co-authored-by: Jeremy Michael --- .../plugins/source/sqs/AttributeHandler.java | 44 ++++++++++ .../source/sqs/AwsAuthenticationAdapter.java | 5 ++ .../source/sqs/AwsAuthenticationOptions.java | 5 ++ .../sqs/CodecBulkMessageFieldStrategy.java | 41 ++++++++++ .../source/sqs/MessageFieldStrategy.java | 21 +++++ .../plugins/source/sqs/QueueConfig.java | 14 ++++ .../source/sqs/RawSqsMessageHandler.java | 51 +++++------- .../plugins/source/sqs/SqsEventProcessor.java | 5 ++ .../plugins/source/sqs/SqsMessageHandler.java | 6 ++ .../sqs/SqsRetriesExhaustedException.java | 5 ++ .../plugins/source/sqs/SqsService.java | 30 +++++-- .../plugins/source/sqs/SqsSource.java | 13 ++- .../plugins/source/sqs/SqsSourceConfig.java | 5 ++ .../plugins/source/sqs/SqsWorker.java | 10 ++- .../sqs/StandardMessageFieldStrategy.java | 27 +++++++ .../sqs/AwsAuthenticationAdapterTest.java | 5 ++ .../sqs/AwsAuthenticationOptionsTest.java | 5 ++ .../plugins/source/sqs/QueueConfigTest.java | 6 ++ .../source/sqs/RawSqsMessageHandlerTest.java | 81 ++++++++++++++++--- .../source/sqs/SqsEventProcessorTest.java | 5 ++ .../plugins/source/sqs/SqsServiceTest.java | 14 +++- .../source/sqs/SqsSourceConfigTest.java | 5 ++ .../plugins/source/sqs/SqsSourceTest.java | 10 ++- .../plugins/source/sqs/SqsWorkerTest.java | 14 +++- 24 files changed, 367 insertions(+), 60 deletions(-) create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AttributeHandler.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/CodecBulkMessageFieldStrategy.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/MessageFieldStrategy.java create mode 100644 data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/StandardMessageFieldStrategy.java diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AttributeHandler.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AttributeHandler.java new file mode 100644 index 0000000000..338d482f85 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AttributeHandler.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.EventMetadata; +import software.amazon.awssdk.services.sqs.model.Message; +import software.amazon.awssdk.services.sqs.model.MessageAttributeValue; +import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; +import java.util.HashMap; +import java.util.Map; + +public class AttributeHandler { + public static Map collectMetadataAttributes(final Message message, final String queueUrl) { + final Map metadataMap = new HashMap<>(); + metadataMap.put("queueUrl", queueUrl); + + for (Map.Entry entry : message.attributes().entrySet()) { + String originalKey = entry.getKey().toString(); + String key = originalKey.substring(0, 1).toLowerCase() + originalKey.substring(1); + metadataMap.put(key, entry.getValue()); + } + + for (Map.Entry entry : message.messageAttributes().entrySet()) { + String originalKey = entry.getKey().toString(); + String key = originalKey.substring(0, 1).toLowerCase() + originalKey.substring(1); + metadataMap.put(key, entry.getValue().stringValue()); + } + return metadataMap; + } + + public static void applyMetadataAttributes(final Event event, final Map attributes) { + final EventMetadata metadata = event.getMetadata(); + attributes.forEach(metadata::setAttribute); + } +} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapter.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapter.java index 08600cba13..e22fa68652 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapter.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapter.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptions.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptions.java index 99da366de3..3ac9cec011 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptions.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptions.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/CodecBulkMessageFieldStrategy.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/CodecBulkMessageFieldStrategy.java new file mode 100644 index 0000000000..15581169d9 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/CodecBulkMessageFieldStrategy.java @@ -0,0 +1,41 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.opensearch.dataprepper.model.codec.InputCodec; +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.record.Record; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Consumer; + +public class CodecBulkMessageFieldStrategy implements MessageFieldStrategy { + + private final InputCodec codec; + + public CodecBulkMessageFieldStrategy(final InputCodec codec) { + this.codec = codec; + } + + @Override + public List parseEvents(final String messageBody) { + final List events = new ArrayList<>(); + final ByteArrayInputStream inputStream = new ByteArrayInputStream(messageBody.getBytes(StandardCharsets.UTF_8)); + try { + codec.parse(inputStream, (Consumer>) record -> events.add(record.getData())); + } catch (Exception e) { + throw new RuntimeException("Failed to parse events from SQS body.", e); + } + return events; + } +} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/MessageFieldStrategy.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/MessageFieldStrategy.java new file mode 100644 index 0000000000..f7decd3f91 --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/MessageFieldStrategy.java @@ -0,0 +1,21 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.opensearch.dataprepper.model.event.Event; +import java.util.List; + +public interface MessageFieldStrategy { + /** + * Converts the SQS message body into one or more events. + */ + List parseEvents(String messageBody); +} diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfig.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfig.java index ca5566d6cd..47e417bc27 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfig.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfig.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -15,6 +20,7 @@ import org.hibernate.validator.constraints.time.DurationMax; import org.hibernate.validator.constraints.time.DurationMin; +import org.opensearch.dataprepper.model.configuration.PluginModel; public class QueueConfig { @@ -62,6 +68,9 @@ public class QueueConfig { @DurationMax(seconds = 20) private Duration waitTime = DEFAULT_WAIT_TIME_SECONDS; + @JsonProperty("codec") + private PluginModel codec = null; + public String getUrl() { return url; } @@ -93,5 +102,10 @@ public Duration getWaitTime() { public Duration getPollDelay() { return pollDelay; } + + public PluginModel getCodec() { + return codec; + } + } diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandler.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandler.java index 493b7ab7d7..6fc02f238a 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandler.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandler.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -8,21 +13,23 @@ import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; -import org.opensearch.dataprepper.model.event.EventMetadata; -import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.sqs.model.Message; -import software.amazon.awssdk.services.sqs.model.MessageAttributeValue; -import software.amazon.awssdk.services.sqs.model.MessageSystemAttributeName; -import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.ArrayList; public class RawSqsMessageHandler implements SqsMessageHandler { private static final Logger LOG = LoggerFactory.getLogger(RawSqsMessageHandler.class); + private final MessageFieldStrategy messageFieldStrategy; + + public RawSqsMessageHandler(final MessageFieldStrategy messageFieldStrategy) { + this.messageFieldStrategy = messageFieldStrategy; + } @Override public void handleMessage(final Message message, @@ -31,32 +38,18 @@ public void handleMessage(final Message message, final int bufferTimeoutMillis, final AcknowledgementSet acknowledgementSet) { try { - final Map systemAttributes = message.attributes(); - final Map customAttributes = message.messageAttributes(); - final Event event = JacksonEvent.builder() - .withEventType("DOCUMENT") - .withData(Collections.singletonMap("message", message.body())) - .build(); - - final EventMetadata eventMetadata = event.getMetadata(); - eventMetadata.setAttribute("queueUrl", url); - - for (Map.Entry entry : systemAttributes.entrySet()) { - String originalKey = entry.getKey().toString(); - String lowerCamelCaseKey = originalKey.substring(0, 1).toLowerCase() + originalKey.substring(1); - eventMetadata.setAttribute(lowerCamelCaseKey, entry.getValue()); + List events = messageFieldStrategy.parseEvents(message.body()); + Map metadataMap = AttributeHandler.collectMetadataAttributes(message, url); + List> records = new ArrayList<>(); + for (Event event : events) { + AttributeHandler.applyMetadataAttributes(event, metadataMap); + if (acknowledgementSet != null) { + acknowledgementSet.add(event); + } + records.add(new Record<>(event)); } + buffer.writeAll(records, bufferTimeoutMillis); - for (Map.Entry entry : customAttributes.entrySet()) { - String originalKey = entry.getKey(); - String lowerCamelCaseKey = originalKey.substring(0, 1).toLowerCase() + originalKey.substring(1); - eventMetadata.setAttribute(lowerCamelCaseKey, entry.getValue().stringValue()); - } - - if (acknowledgementSet != null) { - acknowledgementSet.add(event); - } - buffer.write(new Record<>(event), bufferTimeoutMillis); } catch (Exception e) { LOG.error("Error processing SQS message: {}", e.getMessage(), e); throw new RuntimeException(e); diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessor.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessor.java index a03c485c37..c3669c92cb 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessor.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessor.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsMessageHandler.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsMessageHandler.java index 79012b5e00..a9585a4a62 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsMessageHandler.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsMessageHandler.java @@ -1,7 +1,13 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ + package org.opensearch.dataprepper.plugins.source.sqs; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSet; diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsRetriesExhaustedException.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsRetriesExhaustedException.java index 4e1f9507e6..e1fd536cb7 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsRetriesExhaustedException.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsRetriesExhaustedException.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsService.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsService.java index d53f269323..672ee9874c 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsService.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsService.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -9,6 +14,10 @@ import org.opensearch.dataprepper.common.concurrent.BackgroundThreadFactory; import org.opensearch.dataprepper.metrics.PluginMetrics; import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; + import org.opensearch.dataprepper.model.codec.InputCodec; + import org.opensearch.dataprepper.model.configuration.PluginModel; + import org.opensearch.dataprepper.model.configuration.PluginSetting; + import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; @@ -35,9 +44,9 @@ public class SqsService { static final double JITTER_RATE = 0.20; private final SqsSourceConfig sqsSourceConfig; - private final SqsEventProcessor sqsEventProcessor; private final SqsClient sqsClient; private final PluginMetrics pluginMetrics; + private final PluginFactory pluginFactory; private final AcknowledgementSetManager acknowledgementSetManager; private final List allSqsUrlExecutorServices; private final List sqsWorkers; @@ -46,13 +55,13 @@ public class SqsService { public SqsService(final Buffer> buffer, final AcknowledgementSetManager acknowledgementSetManager, final SqsSourceConfig sqsSourceConfig, - final SqsEventProcessor sqsEventProcessor, final PluginMetrics pluginMetrics, + final PluginFactory pluginFactory, final AwsCredentialsProvider credentialsProvider) { this.sqsSourceConfig = sqsSourceConfig; - this.sqsEventProcessor = sqsEventProcessor; this.pluginMetrics = pluginMetrics; + this.pluginFactory = pluginFactory; this.acknowledgementSetManager = acknowledgementSetManager; this.allSqsUrlExecutorServices = new ArrayList<>(); this.sqsWorkers = new ArrayList<>(); @@ -70,8 +79,19 @@ public void start() { sqsSourceConfig.getQueues().forEach(queueConfig -> { String queueUrl = queueConfig.getUrl(); String queueName = queueUrl.substring(queueUrl.lastIndexOf('/') + 1); - int numWorkers = queueConfig.getNumWorkers(); + SqsEventProcessor sqsEventProcessor; + MessageFieldStrategy strategy; + if (queueConfig.getCodec() != null) { + final PluginModel codecConfiguration = queueConfig.getCodec(); + final PluginSetting codecPluginSettings = new PluginSetting(codecConfiguration.getPluginName(), codecConfiguration.getPluginSettings()); + final InputCodec codec = pluginFactory.loadPlugin(InputCodec.class, codecPluginSettings); + strategy = new CodecBulkMessageFieldStrategy(codec); + } else { + strategy = new StandardMessageFieldStrategy(); + } + + sqsEventProcessor = new SqsEventProcessor(new RawSqsMessageHandler(strategy)); ExecutorService executorService = Executors.newFixedThreadPool( numWorkers, BackgroundThreadFactory.defaultExecutorThreadFactory("sqs-source" + queueName)); allSqsUrlExecutorServices.add(executorService); @@ -80,10 +100,10 @@ public void start() { buffer, acknowledgementSetManager, sqsClient, - sqsEventProcessor, sqsSourceConfig, queueConfig, pluginMetrics, + sqsEventProcessor, backoff)) .collect(Collectors.toList()); diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSource.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSource.java index 980e59048b..e722b76780 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSource.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSource.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -12,6 +17,7 @@ import org.opensearch.dataprepper.model.annotations.DataPrepperPluginConstructor; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import org.opensearch.dataprepper.model.source.Source; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; @@ -21,6 +27,7 @@ public class SqsSource implements Source> { private final PluginMetrics pluginMetrics; + private final PluginFactory pluginFactory; private final SqsSourceConfig sqsSourceConfig; private SqsService sqsService; private final AcknowledgementSetManager acknowledgementSetManager; @@ -31,10 +38,12 @@ public class SqsSource implements Source> { @DataPrepperPluginConstructor public SqsSource(final PluginMetrics pluginMetrics, final SqsSourceConfig sqsSourceConfig, + final PluginFactory pluginFactory, final AcknowledgementSetManager acknowledgementSetManager, final AwsCredentialsSupplier awsCredentialsSupplier) { this.pluginMetrics = pluginMetrics; + this.pluginFactory = pluginFactory; this.sqsSourceConfig = sqsSourceConfig; this.acknowledgementsEnabled = sqsSourceConfig.getAcknowledgements(); this.acknowledgementSetManager = acknowledgementSetManager; @@ -49,9 +58,7 @@ public void start(Buffer> buffer) { } final AwsAuthenticationAdapter awsAuthenticationAdapter = new AwsAuthenticationAdapter(awsCredentialsSupplier, sqsSourceConfig); final AwsCredentialsProvider credentialsProvider = awsAuthenticationAdapter.getCredentialsProvider(); - final SqsMessageHandler rawSqsMessageHandler = new RawSqsMessageHandler(); - final SqsEventProcessor sqsEventProcessor = new SqsEventProcessor(rawSqsMessageHandler); - sqsService = new SqsService(buffer, acknowledgementSetManager, sqsSourceConfig, sqsEventProcessor, pluginMetrics, credentialsProvider); + sqsService = new SqsService(buffer, acknowledgementSetManager, sqsSourceConfig, pluginMetrics, pluginFactory, credentialsProvider); sqsService.start(); } diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfig.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfig.java index c84a3a3d69..1f90d1a711 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfig.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfig.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorker.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorker.java index 3f58906b33..f6de0b9ee1 100644 --- a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorker.java +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorker.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -68,21 +73,20 @@ public class SqsWorker implements Runnable { public SqsWorker(final Buffer> buffer, final AcknowledgementSetManager acknowledgementSetManager, final SqsClient sqsClient, - final SqsEventProcessor sqsEventProcessor, final SqsSourceConfig sqsSourceConfig, final QueueConfig queueConfig, final PluginMetrics pluginMetrics, + final SqsEventProcessor sqsEventProcessor, final Backoff backoff) { this.sqsClient = sqsClient; - this.sqsEventProcessor = sqsEventProcessor; this.queueConfig = queueConfig; this.acknowledgementSetManager = acknowledgementSetManager; this.standardBackoff = backoff; this.endToEndAcknowledgementsEnabled = sqsSourceConfig.getAcknowledgements(); this.buffer = buffer; this.bufferTimeoutMillis = (int) sqsSourceConfig.getBufferTimeout().toMillis(); - + this.sqsEventProcessor = sqsEventProcessor; messageVisibilityTimesMap = new HashMap<>(); failedAttemptCount = 0; sqsMessagesReceivedCounter = pluginMetrics.counter(SQS_MESSAGES_RECEIVED_METRIC_NAME); diff --git a/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/StandardMessageFieldStrategy.java b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/StandardMessageFieldStrategy.java new file mode 100644 index 0000000000..defbbf7baa --- /dev/null +++ b/data-prepper-plugins/sqs-source/src/main/java/org/opensearch/dataprepper/plugins/source/sqs/StandardMessageFieldStrategy.java @@ -0,0 +1,27 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.sqs; + +import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.JacksonEvent; +import java.util.Collections; +import java.util.List; + +public class StandardMessageFieldStrategy implements MessageFieldStrategy { + @Override + public List parseEvents(final String messageBody) { + final Event event = JacksonEvent.builder() + .withEventType("DOCUMENT") + .withData(Collections.singletonMap("message", messageBody)) + .build(); + return Collections.singletonList(event); + } +} diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapterTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapterTest.java index 04806ff4d3..0fa8cfec6a 100644 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapterTest.java +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationAdapterTest.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptionsTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptionsTest.java index 77eeeb519a..6f485d1a9e 100644 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptionsTest.java +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/AwsAuthenticationOptionsTest.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfigTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfigTest.java index f312d8abc6..c6bdb8b644 100644 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfigTest.java +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/QueueConfigTest.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -22,6 +27,7 @@ void testDefaultValues() { assertEquals(1, queueConfig.getNumWorkers(), "Number of workers should default to 1"); assertNull(queueConfig.getMaximumMessages(), "Maximum messages should be null by default"); assertEquals(Duration.ofSeconds(0), queueConfig.getPollDelay(), "Poll delay should default to 0 seconds"); + assertNull(queueConfig.getCodec(), "Codec should be null by default"); assertNull(queueConfig.getVisibilityTimeout(), "Visibility timeout should be null by default"); assertFalse(queueConfig.getVisibilityDuplicateProtection(), "Visibility duplicate protection should default to false"); assertEquals(Duration.ofHours(2), queueConfig.getVisibilityDuplicateProtectionTimeout(), diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandlerTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandlerTest.java index 4606df45c6..ae6205457b 100644 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandlerTest.java +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/RawSqsMessageHandlerTest.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -8,37 +13,89 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.event.JacksonEvent; import org.opensearch.dataprepper.model.record.Record; import software.amazon.awssdk.services.sqs.model.Message; +import org.opensearch.dataprepper.model.codec.InputCodec; +import java.io.InputStream; +import java.util.List; +import java.util.function.Consumer; + +import static java.util.Collections.singletonMap; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; class RawSqsMessageHandlerTest { - private final RawSqsMessageHandler rawSqsMessageHandler = new RawSqsMessageHandler(); private Buffer> mockBuffer; private int mockBufferTimeoutMillis; @BeforeEach void setUp() { - mockBuffer = mock(Buffer.class); + mockBuffer = Mockito.mock(Buffer.class); mockBufferTimeoutMillis = 10000; } @Test - void handleMessage_callsBufferWriteOnce() throws Exception { - Message message = Message.builder().body("{\"key\":\"value\"}").build(); + void handleMessage_standardStrategy_callsBufferWriteAllOnce() throws Exception { + MessageFieldStrategy standardMessageFieldStrategy = new StandardMessageFieldStrategy(); + RawSqsMessageHandler handler = new RawSqsMessageHandler(standardMessageFieldStrategy); + Message message = Message.builder() + .body("{\"key\":\"value\"}") + .build(); String queueUrl = "https://sqs.us-east-1.amazonaws.com/123456789012/test-queue"; - rawSqsMessageHandler.handleMessage(message, queueUrl, mockBuffer, mockBufferTimeoutMillis, null); - ArgumentCaptor> argumentCaptor = ArgumentCaptor.forClass(Record.class); - verify(mockBuffer, times(1)).write(argumentCaptor.capture(), eq(mockBufferTimeoutMillis)); - Record capturedRecord = argumentCaptor.getValue(); - assertEquals("DOCUMENT", capturedRecord.getData().getMetadata().getEventType(), "Event type should be 'DOCUMENT'"); + handler.handleMessage(message, queueUrl, mockBuffer, mockBufferTimeoutMillis, null); + + @SuppressWarnings("unchecked") + ArgumentCaptor>> argumentCaptor = ArgumentCaptor.forClass(List.class); + Mockito.verify(mockBuffer, Mockito.times(1)).writeAll(argumentCaptor.capture(), eq(mockBufferTimeoutMillis)); + List> capturedRecords = argumentCaptor.getValue(); + assertEquals(1, capturedRecords.size(), "Raw strategy should produce exactly one record"); + assertEquals("DOCUMENT", capturedRecords.get(0).getData().getMetadata().getEventType(), + "Event type should be 'DOCUMENT'"); + } + + @Test + void handleMessage_bulkStrategy_callsBufferWriteAllWithMultipleEvents() throws Exception { + InputCodec mockCodec = Mockito.mock(InputCodec.class); + Mockito.doAnswer(invocation -> { + InputStream inputStream = invocation.getArgument(0); + @SuppressWarnings("unchecked") + Consumer> eventConsumer = invocation.getArgument(1); + Event event1 = JacksonEvent.builder() + .withEventType("DOCUMENT") + .withData(singletonMap("key1", "val1")) + .build(); + Event event2 = JacksonEvent.builder() + .withEventType("DOCUMENT") + .withData(singletonMap("key2", "val2")) + .build(); + eventConsumer.accept(new Record<>(event1)); + eventConsumer.accept(new Record<>(event2)); + return null; + }).when(mockCodec).parse(any(InputStream.class), any()); + MessageFieldStrategy bulkStrategy = new CodecBulkMessageFieldStrategy(mockCodec); + RawSqsMessageHandler handler = new RawSqsMessageHandler(bulkStrategy); + String messageBody = "{\"events\":[{\"foo\":\"bar1\"},{\"foo\":\"bar2\"}]}"; + Message message = Message.builder() + .body(messageBody) + .build(); + String queueUrl = "https://sqs.us-east-1.amazonaws.com/123456789012/test-queue"; + handler.handleMessage(message, queueUrl, mockBuffer, mockBufferTimeoutMillis, null); + @SuppressWarnings("unchecked") + ArgumentCaptor>> argumentCaptor = ArgumentCaptor.forClass(List.class); + Mockito.verify(mockBuffer, Mockito.times(1)).writeAll(argumentCaptor.capture(), eq(mockBufferTimeoutMillis)); + + List> capturedRecords = argumentCaptor.getValue(); + assertEquals(2, capturedRecords.size(), "Bulk strategy should produce two records"); + for (Record record : capturedRecords) { + assertEquals("DOCUMENT", record.getData().getMetadata().getEventType(), + "Event type should be 'DOCUMENT'"); + } } } diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessorTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessorTest.java index e10b8f471f..630e1e8f10 100644 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessorTest.java +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsEventProcessorTest.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsServiceTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsServiceTest.java index 3bbc44bbe6..695164db82 100644 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsServiceTest.java +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsServiceTest.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -11,6 +16,7 @@ import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -27,9 +33,9 @@ class SqsServiceTest { private SqsSourceConfig sqsSourceConfig; - private SqsEventProcessor sqsEventProcessor; private SqsClient sqsClient; private PluginMetrics pluginMetrics; + private PluginFactory pluginFactory; private AcknowledgementSetManager acknowledgementSetManager; private Buffer> buffer; private AwsCredentialsProvider credentialsProvider; @@ -37,9 +43,9 @@ class SqsServiceTest { @BeforeEach void setUp() { sqsSourceConfig = mock(SqsSourceConfig.class); - sqsEventProcessor = mock(SqsEventProcessor.class); sqsClient = mock(SqsClient.class, withSettings()); pluginMetrics = mock(PluginMetrics.class); + pluginFactory = mock(PluginFactory.class); acknowledgementSetManager = mock(AcknowledgementSetManager.class); buffer = mock(Buffer.class); credentialsProvider = mock(AwsCredentialsProvider.class); @@ -55,7 +61,7 @@ void start_with_single_queue_starts_workers() { when(queueConfig.getUrl()).thenReturn("https://sqs.us-east-1.amazonaws.com/123456789012/MyQueue"); when(queueConfig.getNumWorkers()).thenReturn(2); when(sqsSourceConfig.getQueues()).thenReturn(List.of(queueConfig)); - SqsService sqsService = spy(new SqsService(buffer, acknowledgementSetManager, sqsSourceConfig, sqsEventProcessor, pluginMetrics, credentialsProvider)); + SqsService sqsService = spy(new SqsService(buffer, acknowledgementSetManager, sqsSourceConfig, pluginMetrics, pluginFactory, credentialsProvider)); doReturn(sqsClient).when(sqsService).createSqsClient(credentialsProvider); sqsService.start(); // if no exception is thrown here, then workers have been started } @@ -67,7 +73,7 @@ void stop_should_shutdown_executors_and_workers_and_close_client() throws Interr when(queueConfig.getNumWorkers()).thenReturn(1); when(sqsSourceConfig.getQueues()).thenReturn(List.of(queueConfig)); SqsClient sqsClient = mock(SqsClient.class); - SqsService sqsService = new SqsService(buffer, acknowledgementSetManager, sqsSourceConfig, sqsEventProcessor, pluginMetrics, credentialsProvider) { + SqsService sqsService = new SqsService(buffer, acknowledgementSetManager, sqsSourceConfig, pluginMetrics, pluginFactory, credentialsProvider) { @Override SqsClient createSqsClient(final AwsCredentialsProvider credentialsProvider) { return sqsClient; diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfigTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfigTest.java index 29f0443670..6922c9dbd4 100644 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfigTest.java +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceConfigTest.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceTest.java index cf130c102e..b028c67177 100644 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceTest.java +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsSourceTest.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -12,6 +17,7 @@ import org.opensearch.dataprepper.model.acknowledgements.AcknowledgementSetManager; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; import software.amazon.awssdk.regions.Region; @@ -27,6 +33,7 @@ class SqsSourceTest { private final String TEST_PIPELINE_NAME = "test_pipeline"; private SqsSource sqsSource; private PluginMetrics pluginMetrics; + private PluginFactory pluginFactory; private SqsSourceConfig sqsSourceConfig; private AcknowledgementSetManager acknowledgementSetManager; private AwsCredentialsSupplier awsCredentialsSupplier; @@ -36,10 +43,11 @@ class SqsSourceTest { @BeforeEach void setUp() { pluginMetrics = PluginMetrics.fromNames(PLUGIN_NAME, TEST_PIPELINE_NAME); + pluginFactory = mock(PluginFactory.class); sqsSourceConfig = mock(SqsSourceConfig.class); acknowledgementSetManager = mock(AcknowledgementSetManager.class); awsCredentialsSupplier = mock(AwsCredentialsSupplier.class); - sqsSource = new SqsSource(pluginMetrics, sqsSourceConfig, acknowledgementSetManager, awsCredentialsSupplier); + sqsSource = new SqsSource(pluginMetrics, sqsSourceConfig, pluginFactory, acknowledgementSetManager, awsCredentialsSupplier); buffer = mock(Buffer.class); } diff --git a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorkerTest.java b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorkerTest.java index 7bb8e082cc..22bf48596f 100644 --- a/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorkerTest.java +++ b/data-prepper-plugins/sqs-source/src/test/java/org/opensearch/dataprepper/plugins/source/sqs/SqsWorkerTest.java @@ -1,6 +1,11 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.source.sqs; @@ -19,6 +24,7 @@ import org.opensearch.dataprepper.model.acknowledgements.ProgressCheck; import org.opensearch.dataprepper.model.buffer.Buffer; import org.opensearch.dataprepper.model.event.Event; +import org.opensearch.dataprepper.model.plugin.PluginFactory; import org.opensearch.dataprepper.model.record.Record; import software.amazon.awssdk.services.sqs.SqsClient; import software.amazon.awssdk.services.sqs.model.ChangeMessageVisibilityRequest; @@ -72,6 +78,8 @@ class SqsWorkerTest { @Mock private PluginMetrics pluginMetrics; @Mock + private PluginFactory pluginFactory; + @Mock private Backoff backoff; @Mock private Counter sqsMessagesReceivedCounter; @@ -87,17 +95,17 @@ class SqsWorkerTest { private Counter sqsVisibilityTimeoutChangedCount; @Mock private Counter sqsVisibilityTimeoutChangeFailedCount; - private int mockBufferTimeoutMillis = 10000; + private final int mockBufferTimeoutMillis = 10000; private SqsWorker createObjectUnderTest() { return new SqsWorker( buffer, acknowledgementSetManager, sqsClient, - sqsEventProcessor, sqsSourceConfig, queueConfig, pluginMetrics, + sqsEventProcessor, backoff); } @@ -216,7 +224,7 @@ void acknowledgementsEnabled_and_visibilityDuplicateProtectionEnabled_should_cre when(sqsSourceConfig.getAcknowledgements()).thenReturn(true); when(queueConfig.getVisibilityDuplicateProtection()).thenReturn(true); - SqsWorker worker = new SqsWorker(buffer, acknowledgementSetManager, sqsClient, sqsEventProcessor, sqsSourceConfig, queueConfig, pluginMetrics, backoff); + SqsWorker worker = new SqsWorker(buffer, acknowledgementSetManager, sqsClient, sqsSourceConfig, queueConfig, pluginMetrics, sqsEventProcessor, backoff); Message message = Message.builder().messageId("msg-dup").receiptHandle("handle-dup").build(); ReceiveMessageResponse response = ReceiveMessageResponse.builder().messages(message).build(); when(sqsClient.receiveMessage((ReceiveMessageRequest) any())).thenReturn(response); From 09eec9376689b770d93e01052cc3e4171704f981 Mon Sep 17 00:00:00 2001 From: David Venable Date: Wed, 15 Jan 2025 15:14:27 -0800 Subject: [PATCH 31/36] Minor fix to the formatting of the license header. (#5334) Signed-off-by: David Venable --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 38972a8625..8f22e0aec5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,7 @@ If you are modifying existing files with license headers, or including new files * this file be licensed under the Apache-2.0 license or a * compatible open source license. * -*/ + */ ``` ### Shell, Python From 83180414f004e8e9dbece891391dec2a18ec6109 Mon Sep 17 00:00:00 2001 From: Taylor Gray Date: Wed, 15 Jan 2025 17:17:05 -0600 Subject: [PATCH 32/36] Report SQS message delay immediately after message is received, and as 0 when there are no messages found in queue (#5333) Signed-off-by: Taylor Gray --- .../plugins/source/s3/SqsWorker.java | 11 +++++++---- .../plugins/source/s3/SqsWorkerTest.java | 19 +++++++++++++++++-- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java index 2861ffa6d7..08c643b39c 100644 --- a/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java +++ b/data-prepper-plugins/s3-source/src/main/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorker.java @@ -156,6 +156,9 @@ private List getMessagesFromSqs() { final ReceiveMessageRequest receiveMessageRequest = createReceiveMessageRequest(); final List messages = sqsClient.receiveMessage(receiveMessageRequest).messages(); failedAttemptCount = 0; + if (messages.isEmpty()) { + sqsMessageDelayTimer.record(Duration.ZERO); + } return messages; } catch (final SqsException | StsException e) { LOG.error("Error reading from SQS: {}. Retrying with exponential backoff.", e.getMessage()); @@ -228,6 +231,10 @@ && isEventBridgeEventTypeCreated(parsedMessage)) { LOG.info("Received {} messages from SQS. Processing {} messages.", s3EventNotificationRecords.size(), parsedMessagesToRead.size()); for (ParsedMessage parsedMessage : parsedMessagesToRead) { + sqsMessageDelayTimer.record(Duration.between( + Instant.ofEpochMilli(parsedMessage.getEventTime().toInstant().getMillis()), + Instant.now() + )); List waitingForAcknowledgements = new ArrayList<>(); AcknowledgementSet acknowledgementSet = null; final int visibilityTimeout = (int)sqsOptions.getVisibilityTimeout().getSeconds(); @@ -318,10 +325,6 @@ private Optional processS3Object( // SQS messages won't be deleted if we are unable to process S3Objects because of an exception try { s3Service.addS3Object(s3ObjectReference, acknowledgementSet); - sqsMessageDelayTimer.record(Duration.between( - Instant.ofEpochMilli(parsedMessage.getEventTime().toInstant().getMillis()), - Instant.now() - )); return Optional.of(buildDeleteMessageBatchRequestEntry(parsedMessage.getMessage())); } catch (final Exception e) { LOG.error("Error processing from S3: {}. Retrying with exponential backoff.", e.getMessage()); diff --git a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java index ada789cea6..bed9437f85 100644 --- a/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java +++ b/data-prepper-plugins/s3-source/src/test/java/org/opensearch/dataprepper/plugins/source/s3/SqsWorkerTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.params.provider.ArgumentsSource; import org.junit.jupiter.params.provider.ValueSource; import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.metrics.PluginMetrics; @@ -67,6 +68,7 @@ import static org.junit.jupiter.params.provider.Arguments.arguments; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.inOrder; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; import static org.mockito.Mockito.reset; @@ -700,7 +702,10 @@ void processSqsMessages_should_stop_updating_visibility_timeout_after_stop() thr objectUnderTest.stop(); assertThat(messagesProcessed, equalTo(1)); - verify(s3Service).addS3Object(any(S3ObjectReference.class), any()); + + final InOrder inOrder = inOrder(s3Service, sqsMessageDelayTimer); + inOrder.verify(sqsMessageDelayTimer).record(any(Duration.class)); + inOrder.verify(s3Service).addS3Object(any(S3ObjectReference.class), any()); verify(acknowledgementSetManager).create(any(), any(Duration.class)); ArgumentCaptor> progressConsumerArgumentCaptor = ArgumentCaptor.forClass(Consumer.class); @@ -711,7 +716,17 @@ void processSqsMessages_should_stop_updating_visibility_timeout_after_stop() thr verify(sqsClient, never()).changeMessageVisibility(any(ChangeMessageVisibilityRequest.class)); verify(sqsMessagesReceivedCounter).increment(1); - verify(sqsMessageDelayTimer).record(any(Duration.class)); + } + + @Test + void processSqsMessages_should_record_zero_message_delay_when_no_messages_are_found_on_poll() { + final ReceiveMessageResponse receiveMessageResponse = mock(ReceiveMessageResponse.class); + when(receiveMessageResponse.messages()).thenReturn(Collections.emptyList()); + + when(sqsClient.receiveMessage(any(ReceiveMessageRequest.class))).thenReturn(receiveMessageResponse); + final int messagesProcessed = createObjectUnderTest().processSqsMessages(); + assertThat(messagesProcessed, equalTo(0)); + verify(sqsMessageDelayTimer).record(Duration.ZERO); } private static String createPutNotification(final Instant startTime) { From b2cbbae19102ab633e2b621de5ebd6e58f614b85 Mon Sep 17 00:00:00 2001 From: David Venable Date: Wed, 15 Jan 2025 17:01:02 -0800 Subject: [PATCH 33/36] Welcoming Santhosh Gandhe (san81) to the Data Prepper maintainers. (#5329) Signed-off-by: David Venable --- .github/CODEOWNERS | 2 +- MAINTAINERS.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b6d5905f4a..e1e9d8f1eb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams -* @sb2k16 @chenqi0805 @engechas @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh \ No newline at end of file +* @sb2k16 @chenqi0805 @engechas @san81 @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh \ No newline at end of file diff --git a/MAINTAINERS.md b/MAINTAINERS.md index d52c15f225..51a68597ba 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -9,6 +9,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Souvik Bose | [sb2k16](https://github.com/sb2k16) | Amazon | | Qi Chen | [chenqi0805](https://github.com/chenqi0805) | Amazon | | Chase Engelbrecht | [engechas](https://github.com/engechas) | Amazon | +| Santhosh Gandhe | [san81](https://github.com/san81) | Amazon | | Taylor Gray | [graytaylor0](https://github.com/graytaylor0) | Amazon | | Dinu John | [dinujoh](https://github.com/dinujoh) | Amazon | | Krishna Kondaka | [kkondaka](https://github.com/kkondaka) | Amazon | From 41e3227e8d79c2fd271ecf1aa92d48be807bf53f Mon Sep 17 00:00:00 2001 From: Maxwell Brown <55033421+Galactus22625@users.noreply.github.com> Date: Thu, 16 Jan 2025 14:55:32 -0500 Subject: [PATCH 34/36] github actions/ artifacts v3 to v4 (#5339) * github actions/ artifacts v3 to v4 Signed-off-by: Maxwell Brown * update artifacts for opensearch sink tests and gradle.yml Signed-off-by: Maxwell Brown --------- Signed-off-by: Maxwell Brown --- .github/workflows/gradle.yml | 4 ++-- .github/workflows/kafka-plugin-integration-tests.yml | 4 ++-- .github/workflows/kinesis-source-integration-tests.yml | 4 ++-- .../opensearch-sink-opendistro-integration-tests.yml | 4 ++-- .../opensearch-sink-opensearch-integration-tests.yml | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index cb4373ee24..cc495c596d 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -28,7 +28,7 @@ jobs: run: ./gradlew --parallel --max-workers 2 build - name: Upload Unit Test Results if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: data-prepper-test-results-java-${{ matrix.java }} path: '**/test-results/**/*.xml' @@ -45,7 +45,7 @@ jobs: steps: - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: test-results diff --git a/.github/workflows/kafka-plugin-integration-tests.yml b/.github/workflows/kafka-plugin-integration-tests.yml index 72d6645d53..25a1481339 100644 --- a/.github/workflows/kafka-plugin-integration-tests.yml +++ b/.github/workflows/kafka-plugin-integration-tests.yml @@ -70,7 +70,7 @@ jobs: - name: Upload Unit Test Results if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: data-prepper-kafka-integration-tests-kafka-${{ matrix.kafka }}-java-${{ matrix.java }} path: '**/test-results/**/*.xml' @@ -83,7 +83,7 @@ jobs: steps: - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: test-results diff --git a/.github/workflows/kinesis-source-integration-tests.yml b/.github/workflows/kinesis-source-integration-tests.yml index cf28bdedb6..54dd79b902 100644 --- a/.github/workflows/kinesis-source-integration-tests.yml +++ b/.github/workflows/kinesis-source-integration-tests.yml @@ -59,7 +59,7 @@ jobs: -Dtests.kinesis.source.aws.region=us-east-1 --tests KinesisSourceIT - name: Upload Unit Test Results if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: data-prepper-kinesis-source-integration-tests-java-${{ matrix.java }} path: '**/test-results/**/*.xml' @@ -72,7 +72,7 @@ jobs: steps: - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: test-results diff --git a/.github/workflows/opensearch-sink-opendistro-integration-tests.yml b/.github/workflows/opensearch-sink-opendistro-integration-tests.yml index 6f4ac3d457..42d508ce7e 100644 --- a/.github/workflows/opensearch-sink-opendistro-integration-tests.yml +++ b/.github/workflows/opensearch-sink-opendistro-integration-tests.yml @@ -43,7 +43,7 @@ jobs: ./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin -Dtests.opensearch.bundle=true -Dtests.opensearch.version=opendistro:${{ matrix.opendistro }} - name: Upload Unit Test Results if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: data-prepper-opensearch-integration-tests-opendistro-${{ matrix.opendistro }}-java-${{ matrix.java }} path: '**/test-results/**/*.xml' @@ -56,7 +56,7 @@ jobs: steps: - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: test-results diff --git a/.github/workflows/opensearch-sink-opensearch-integration-tests.yml b/.github/workflows/opensearch-sink-opensearch-integration-tests.yml index dc18aec096..be8bb25fde 100644 --- a/.github/workflows/opensearch-sink-opensearch-integration-tests.yml +++ b/.github/workflows/opensearch-sink-opensearch-integration-tests.yml @@ -43,7 +43,7 @@ jobs: ./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin -Dtests.opensearch.bundle=true -Dtests.opensearch.version=opensearch:${{ matrix.opensearch }} - name: Upload Unit Test Results if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: data-prepper-opensearch-integration-tests-opensearch-${{ matrix.opensearch }}-java-${{ matrix.java }} path: '**/test-results/**/*.xml' @@ -56,7 +56,7 @@ jobs: steps: - name: Download Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: path: test-results From 3b12ca1c8a7faf59b8ed30263c67526f1628180b Mon Sep 17 00:00:00 2001 From: Hai Yan <8153134+oeyh@users.noreply.github.com> Date: Thu, 16 Jan 2025 22:07:41 -0600 Subject: [PATCH 35/36] Support null values (#5342) Signed-off-by: Hai Yan --- .../mutateevent/MapToListProcessor.java | 14 ++-- .../mutateevent/MapToListProcessorTest.java | 66 +++++++++++++++++++ 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java index af291ebc05..d5af649c84 100644 --- a/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java +++ b/data-prepper-plugins/mutate-event-processors/src/main/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessor.java @@ -6,7 +6,6 @@ package org.opensearch.dataprepper.plugins.processor.mutateevent; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; import org.opensearch.dataprepper.expression.ExpressionEvaluator; import static org.opensearch.dataprepper.logging.DataPrepperMarkers.EVENT; import static org.opensearch.dataprepper.logging.DataPrepperMarkers.NOISY; @@ -22,6 +21,7 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.HashMap; import java.util.HashSet; @@ -33,7 +33,6 @@ @DataPrepperPlugin(name = "map_to_list", pluginType = Processor.class, pluginConfigurationType = MapToListProcessorConfig.class) public class MapToListProcessor extends AbstractProcessor, Record> { private static final Logger LOG = LoggerFactory.getLogger(MapToListProcessor.class); - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); private final MapToListProcessorConfig config; private final ExpressionEvaluator expressionEvaluator; private final Set excludeKeySet = new HashSet<>(); @@ -72,9 +71,8 @@ public Collection> doExecute(final Collection> recor for (final Map.Entry entry : sourceMap.entrySet()) { if (!excludeKeySet.contains(entry.getKey())) { - targetNestedList.add(List.of(entry.getKey(), entry.getValue())); + targetNestedList.add(Arrays.asList(entry.getKey(), entry.getValue())); } - } removeProcessedFields(sourceMap, recordEvent); recordEvent.put(config.getTarget(), targetNestedList); @@ -82,10 +80,10 @@ public Collection> doExecute(final Collection> recor final List> targetList = new ArrayList<>(); for (final Map.Entry entry : sourceMap.entrySet()) { if (!excludeKeySet.contains(entry.getKey())) { - targetList.add(Map.of( - config.getKeyName(), entry.getKey(), - config.getValueName(), entry.getValue() - )); + final Map listItem = new HashMap<>(); + listItem.put(config.getKeyName(), entry.getKey()); + listItem.put(config.getValueName(), entry.getValue()); + targetList.add(listItem); } } removeProcessedFields(sourceMap, recordEvent); diff --git a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java index 1b2ca68833..6e9717c103 100644 --- a/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java +++ b/data-prepper-plugins/mutate-event-processors/src/test/java/org/opensearch/dataprepper/plugins/processor/mutateevent/MapToListProcessorTest.java @@ -18,7 +18,9 @@ import org.opensearch.dataprepper.model.record.Record; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -27,6 +29,7 @@ import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.nullValue; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; @@ -357,6 +360,51 @@ void testFailureTagsAreAddedWhenException() { assertThat(resultEvent.getMetadata().getTags(), is(new HashSet<>(testTags))); } + @Test + void testMapToListSuccessWithNullValuesInMap() { + + final MapToListProcessor processor = createObjectUnderTest(); + final Record testRecord = createTestRecordWithNullValues(); + final List> resultRecord = (List>) processor.doExecute(Collections.singletonList(testRecord)); + + assertThat(resultRecord.size(), is(1)); + + final Event resultEvent = resultRecord.get(0).getData(); + List> resultList = resultEvent.get("my-list", List.class); + + assertThat(resultList.size(), is(2)); + Map resultMapWithNullValue = new HashMap<>(); + resultMapWithNullValue.put("key", "key2"); + resultMapWithNullValue.put("value", null); + assertThat(resultList, containsInAnyOrder( + Map.of("key", "key1", "value", "value1"), + resultMapWithNullValue + )); + assertThat(resultEvent.containsKey("my-map"), is(true)); + assertSourceMapUnchangedWithNullValues(resultEvent); + } + + @Test + public void testConvertFieldToListSuccessWithNullValuesInMap() { + when(mockConfig.getConvertFieldToList()).thenReturn(true); + + final MapToListProcessor processor = createObjectUnderTest(); + final Record testRecord = createTestRecordWithNullValues(); + final List> resultRecord = (List>) processor.doExecute(Collections.singletonList(testRecord)); + + assertThat(resultRecord.size(), is(1)); + + final Event resultEvent = resultRecord.get(0).getData(); + List> resultList = resultEvent.get("my-list", List.class); + + assertThat(resultList.size(), is(2)); + assertThat(resultList, containsInAnyOrder( + Arrays.asList("key1", "value1"), + Arrays.asList("key2", null) + )); + assertSourceMapUnchangedWithNullValues(resultEvent); + } + private MapToListProcessor createObjectUnderTest() { return new MapToListProcessor(pluginMetrics, mockConfig, expressionEvaluator); } @@ -396,6 +444,18 @@ private Record createTestRecordWithNestedMap() { return new Record<>(event); } + private Record createTestRecordWithNullValues() { + final Map mapData = new HashMap<>(); + mapData.put("key1", "value1"); + mapData.put("key2", null); + final Map> data = Map.of("my-map", mapData); + final Event event = JacksonEvent.builder() + .withData(data) + .withEventType("event") + .build(); + return new Record<>(event); + } + private void assertSourceMapUnchanged(final Event resultEvent) { assertThat(resultEvent.containsKey("my-map"), is(true)); assertThat(resultEvent.get("my-map/key1", String.class), is("value1")); @@ -408,4 +468,10 @@ private void assertSourceMapUnchangedForFlatRecord(final Event resultEvent) { assertThat(resultEvent.get("key2", String.class), is("value2")); assertThat(resultEvent.get("key3", String.class), is("value3")); } + + private void assertSourceMapUnchangedWithNullValues(final Event resultEvent) { + assertThat(resultEvent.containsKey("my-map"), is(true)); + assertThat(resultEvent.get("my-map/key1", String.class), is("value1")); + assertThat(resultEvent.get("my-map/key2", String.class), nullValue()); + } } From 83e949d63447de1ccc0c08689af235b801f7f52b Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Fri, 17 Jan 2025 11:00:54 -0800 Subject: [PATCH 36/36] Update renewed Jira refresh token back to the secrets store (#5324) * Introduced PluginConfigVariable interaface to provide ability for the plugins to get access to their underlying aws secrets store member to be able to update if needed Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * renewed access token and refresh tokens are now updated back in the secrets store Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * better naming Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * fixing the test cases based on the new PluginConfigVariable attribute used for refreshToken Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * improving the coverage Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * Keeping the existing values in the secret. Just updating an existing key Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * Allowing secrets manager update without a key and also some additional test coverage Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * isUpdatable boolean is introduced and its corresponding tests Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * additional coverage Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * implementing newly added method Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * switching PluginConfigVariable from refreshToken to accessToken Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * Only the master node is responsible for Token refresh Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * Added addition parameter in the API to accept the secrets version to set that helps with enforcing idempotency while updating the secret store multiple times Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * better naming Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * removing setting a versionId for idempotency Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> * removed constructor argument to PluginConfigVariable Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --------- Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- ...ledToUpdatePluginConfigValueException.java | 26 +++++ .../plugin/PluginConfigValueTranslator.java | 41 ++++++++ .../model/plugin/PluginConfigVariable.java | 40 ++++++++ ...oUpdatePluginConfigValueExceptionTest.java | 43 ++++++++ .../plugin/ObjectMapperConfiguration.java | 3 +- .../dataprepper/plugin/VariableExpander.java | 13 ++- .../plugin/VariableExpanderTest.java | 97 ++++++++++++++----- data-prepper-plugins/aws-plugin/build.gradle | 2 +- .../plugins/aws/AwsPluginConfigVariable.java | 54 +++++++++++ .../aws/AwsSecretManagerConfiguration.java | 8 ++ ...AwsSecretsPluginConfigValueTranslator.java | 16 +++ .../plugins/aws/AwsSecretsSupplier.java | 52 +++++++++- .../plugins/aws/SecretsSupplier.java | 21 ++++ .../aws/AwsPluginConfigVariableTest.java | 84 ++++++++++++++++ .../AwsSecretManagerConfigurationTest.java | 34 +++++++ ...ecretsPluginConfigValueTranslatorTest.java | 30 ++++++ .../plugins/aws/AwsSecretsSupplierTest.java | 67 +++++++++++++ .../jira-source/build.gradle | 1 + .../jira/configuration/Oauth2Config.java | 5 +- .../source/jira/rest/JiraRestClient.java | 6 +- .../jira/rest/auth/JiraOauthConfig.java | 18 +++- .../source/jira/JiraConfigHelperTest.java | 19 ++-- .../plugins/source/jira/JiraServiceTest.java | 37 ++++++- .../source/jira/JiraSourceConfigTest.java | 39 +++++--- .../rest/CustomRestTemplateConfigTest.java | 14 ++- .../jira/rest/auth/JiraAuthFactoryTest.java | 12 ++- .../utils/MockPluginConfigVariableImpl.java | 40 ++++++++ .../resources/oauth2-auth-jira-pipeline.yaml | 4 +- 28 files changed, 753 insertions(+), 73 deletions(-) create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueException.java create mode 100644 data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigVariable.java create mode 100644 data-prepper-api/src/test/java/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueExceptionTest.java create mode 100644 data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsPluginConfigVariable.java create mode 100644 data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsPluginConfigVariableTest.java create mode 100644 data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/MockPluginConfigVariableImpl.java diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueException.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueException.java new file mode 100644 index 0000000000..41993e6b3b --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueException.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.dataprepper.model.plugin; + +/** + * Exception thrown when a secret could not be updated. + * + * @since 2.11 + */ +public class FailedToUpdatePluginConfigValueException extends RuntimeException { + + public FailedToUpdatePluginConfigValueException(final String message) { + super(message); + } + + public FailedToUpdatePluginConfigValueException(final String message, Throwable e) { + super(message, e); + } +} diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java index 2c3a20fbe6..e6ac2525a0 100644 --- a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java @@ -1,7 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ package org.opensearch.dataprepper.model.plugin; +/** + * Interface for a Plugin configuration value translator. + * It translates a string expression that is describing a secret store Id and secret Key in to a secretValue + * extracted from corresponding secret store. + * + * @since 2.0 + */ + public interface PluginConfigValueTranslator { + /** + * Translates a string expression that is describing a secret store Id and secret Key in to a secretValue + * extracted from corresponding secret store. + * Example expression: ${{aws_secrets:secretId:secretKey}} + * + * @param value the string value to translate + * @return the translated object + */ Object translate(final String value); + /** + * Returns the prefix for this translator. + * + * @return the prefix for this translator + */ String getPrefix(); + + /** + * Translates a string expression that is describing a secret store Id and secret Key in to an instance + * of PluginConfigVariable with secretValue extracted from corresponding secret store. Additionally, + * this PluginConfigVariable helps with updating the secret value in the secret store, if required. + * Example expression: ${{aws_secrets:secretId:secretKey}} + * + * @param value the string value to translate + * @return the translated object + */ + PluginConfigVariable translateToPluginConfigVariable(final String value); } diff --git a/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigVariable.java b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigVariable.java new file mode 100644 index 0000000000..32c5bde678 --- /dev/null +++ b/data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigVariable.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.dataprepper.model.plugin; + +/** + * Interface for a Extension Plugin configuration variable. + * It gives access to the details of a defined extension variable. + * + * @since 2.11 + */ +public interface PluginConfigVariable { + + /** + * Returns the value of this variable. + * + * @return the value of this variable + */ + Object getValue(); + + /** + * If this variable is updatable, this method helps to set a new value for this variable + * + * @param updatedValue the new value to set + */ + void setValue(Object updatedValue); + + /** + * Returns if the variable is updatable. + * + * @return true if this variable is updatable, false otherwise + */ + boolean isUpdatable(); +} diff --git a/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueExceptionTest.java b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueExceptionTest.java new file mode 100644 index 0000000000..8956c9aea9 --- /dev/null +++ b/data-prepper-api/src/test/java/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueExceptionTest.java @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.dataprepper.model.plugin; + + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; + +public class FailedToUpdatePluginConfigValueExceptionTest extends RuntimeException { + private String message; + + @BeforeEach + void setUp() { + message = UUID.randomUUID().toString(); + } + + @Test + void testGetMessage_should_return_correct_message() { + FailedToUpdatePluginConfigValueException failedToUpdateSecretException = new FailedToUpdatePluginConfigValueException(message); + assertThat(failedToUpdateSecretException.getMessage(), equalTo(message)); + } + + @Test + void testGetMessage_should_return_correct_message_with_throwable() { + RuntimeException cause = new RuntimeException("testException"); + FailedToUpdatePluginConfigValueException failedToUpdateSecretException = new FailedToUpdatePluginConfigValueException(message, cause); + assertThat(failedToUpdateSecretException.getMessage(), equalTo(message)); + assertThat(failedToUpdateSecretException.getCause(), equalTo(cause)); + } + +} diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java index 513ff3d2fa..d475392b66 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ObjectMapperConfiguration.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.module.SimpleModule; import org.opensearch.dataprepper.model.event.EventKey; import org.opensearch.dataprepper.model.event.EventKeyFactory; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import org.opensearch.dataprepper.model.types.ByteCount; import org.opensearch.dataprepper.pipeline.parser.ByteCountDeserializer; import org.opensearch.dataprepper.pipeline.parser.DataPrepperDurationDeserializer; @@ -28,7 +29,7 @@ public class ObjectMapperConfiguration { static final Set TRANSLATE_VALUE_SUPPORTED_JAVA_TYPES = Set.of( String.class, Number.class, Long.class, Short.class, Integer.class, Double.class, Float.class, - Boolean.class, Character.class); + Boolean.class, Character.class, PluginConfigVariable.class); @Bean(name = "extensionPluginConfigObjectMapper") ObjectMapper extensionPluginConfigObjectMapper() { diff --git a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java index 5cbb5136e6..7b249f0389 100644 --- a/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java +++ b/data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/VariableExpander.java @@ -9,6 +9,7 @@ import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.databind.ObjectMapper; import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import javax.inject.Inject; import javax.inject.Named; @@ -29,8 +30,7 @@ public class VariableExpander { @Inject public VariableExpander( - @Named("extensionPluginConfigObjectMapper") - final ObjectMapper objectMapper, + @Named("extensionPluginConfigObjectMapper") final ObjectMapper objectMapper, final Set pluginConfigValueTranslators) { this.objectMapper = objectMapper; patternPluginConfigValueTranslatorMap = pluginConfigValueTranslators.stream().collect(Collectors.toMap( @@ -48,8 +48,13 @@ public T translate(final JsonParser jsonParser, final Class destinationTy .filter(entry -> entry.getKey().matches()) .map(entry -> { final String valueReferenceKey = entry.getKey().group(VALUE_REFERENCE_KEY); - return objectMapper.convertValue( - entry.getValue().translate(valueReferenceKey), destinationType); + if (destinationType.equals(PluginConfigVariable.class)) { + return (T) entry.getValue().translateToPluginConfigVariable(valueReferenceKey); + } else { + return objectMapper.convertValue( + entry.getValue().translate(valueReferenceKey), destinationType); + } + }) .findFirst() .orElseGet(() -> objectMapper.convertValue(rawValue, destinationType)); diff --git a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java index 386b7fd826..c8d2559302 100644 --- a/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java +++ b/data-prepper-plugin-framework/src/test/java/org/opensearch/dataprepper/plugin/VariableExpanderTest.java @@ -19,6 +19,7 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import java.io.IOException; import java.math.BigDecimal; @@ -30,6 +31,8 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.when; @@ -43,6 +46,32 @@ class VariableExpanderTest { private VariableExpander objectUnderTest; + private static Stream getNonStringTypeArguments() { + return Stream.of(Arguments.of(Boolean.class, "true", true), + Arguments.of(Short.class, "2", (short) 2), + Arguments.of(Integer.class, "10", 10), + Arguments.of(Long.class, "200", 200L), + Arguments.of(Double.class, "1.23", 1.23d), + Arguments.of(Float.class, "2.15", 2.15f), + Arguments.of(BigDecimal.class, "2.15", BigDecimal.valueOf(2.15)), + Arguments.of(Map.class, "{}", Collections.emptyMap())); + } + + private static Stream getStringTypeArguments() { + final String testRandomValue = "non-secret-prefix-" + RandomStringUtils.randomAlphabetic(5); + return Stream.of(Arguments.of(String.class, String.format("\"%s\"", testRandomValue), + testRandomValue), + Arguments.of(Duration.class, "\"PT15M\"", Duration.parse("PT15M")), + Arguments.of(Boolean.class, "\"true\"", true), + Arguments.of(Short.class, "\"2\"", (short) 2), + Arguments.of(Integer.class, "\"10\"", 10), + Arguments.of(Long.class, "\"200\"", 200L), + Arguments.of(Double.class, "\"1.23\"", 1.23d), + Arguments.of(Float.class, "\"2.15\"", 2.15f), + Arguments.of(BigDecimal.class, "\"2.15\"", BigDecimal.valueOf(2.15)), + Arguments.of(Character.class, "\"c\"", 'c')); + } + @BeforeEach void setUp() { objectUnderTest = new VariableExpander(OBJECT_MAPPER, Set.of(pluginConfigValueTranslator)); @@ -107,29 +136,53 @@ void testTranslateJsonParserWithStringValue_translate_success( assertThat(actualResult, equalTo(expectedResult)); } - private static Stream getNonStringTypeArguments() { - return Stream.of(Arguments.of(Boolean.class, "true", true), - Arguments.of(Short.class, "2", (short) 2), - Arguments.of(Integer.class, "10", 10), - Arguments.of(Long.class, "200", 200L), - Arguments.of(Double.class, "1.23", 1.23d), - Arguments.of(Float.class, "2.15", 2.15f), - Arguments.of(BigDecimal.class, "2.15", BigDecimal.valueOf(2.15)), - Arguments.of(Map.class, "{}", Collections.emptyMap())); + @Test + void testTranslateJsonParserWithSPluginConfigVariableValue_translate_success() throws IOException { + final String testSecretKey = "testSecretKey"; + final String testTranslatorKey = "test_prefix"; + final String testSecretReference = String.format("${{%s:%s}}", testTranslatorKey, testSecretKey); + final JsonParser jsonParser = JSON_FACTORY.createParser(String.format("\"%s\"", testSecretReference)); + jsonParser.nextToken(); + PluginConfigVariable mockPluginConfigVariable = new PluginConfigVariable() { + + String secretValue = "samplePluginConfigValue"; + + @Override + public Object getValue() { + return secretValue; + } + + @Override + public void setValue(Object updatedValue) { + this.secretValue = updatedValue.toString(); + } + + @Override + public boolean isUpdatable() { + return true; + } + }; + when(pluginConfigValueTranslator.getPrefix()).thenReturn(testTranslatorKey); + when(pluginConfigValueTranslator.translateToPluginConfigVariable(eq(testSecretKey))) + .thenReturn(mockPluginConfigVariable); + objectUnderTest = new VariableExpander(OBJECT_MAPPER, Set.of(pluginConfigValueTranslator)); + final Object actualResult = objectUnderTest.translate(jsonParser, PluginConfigVariable.class); + assertNotNull(actualResult); + assertThat(actualResult, equalTo(mockPluginConfigVariable)); } - private static Stream getStringTypeArguments() { - final String testRandomValue = "non-secret-prefix-" + RandomStringUtils.randomAlphabetic(5); - return Stream.of(Arguments.of(String.class, String.format("\"%s\"", testRandomValue), - testRandomValue), - Arguments.of(Duration.class, "\"PT15M\"", Duration.parse("PT15M")), - Arguments.of(Boolean.class, "\"true\"", true), - Arguments.of(Short.class, "\"2\"", (short) 2), - Arguments.of(Integer.class, "\"10\"", 10), - Arguments.of(Long.class, "\"200\"", 200L), - Arguments.of(Double.class, "\"1.23\"", 1.23d), - Arguments.of(Float.class, "\"2.15\"", 2.15f), - Arguments.of(BigDecimal.class, "\"2.15\"", BigDecimal.valueOf(2.15)), - Arguments.of(Character.class, "\"c\"", 'c')); + @Test + void testTranslateJsonParserWithSPluginConfigVariableValue_translate_failure() throws IOException { + final String testSecretKey = "testSecretKey"; + final String testTranslatorKey = "test_prefix"; + final String testSecretReference = String.format("${{%s:%s}}", testTranslatorKey, testSecretKey); + final JsonParser jsonParser = JSON_FACTORY.createParser(String.format("\"%s\"", testSecretReference)); + jsonParser.nextToken(); + when(pluginConfigValueTranslator.getPrefix()).thenReturn(testTranslatorKey); + when(pluginConfigValueTranslator.translateToPluginConfigVariable(eq(testSecretKey))) + .thenThrow(IllegalArgumentException.class); + objectUnderTest = new VariableExpander(OBJECT_MAPPER, Set.of(pluginConfigValueTranslator)); + assertThrows(IllegalArgumentException.class, + () -> objectUnderTest.translate(jsonParser, PluginConfigVariable.class)); } } \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/build.gradle b/data-prepper-plugins/aws-plugin/build.gradle index 6915278023..88015e537b 100644 --- a/data-prepper-plugins/aws-plugin/build.gradle +++ b/data-prepper-plugins/aws-plugin/build.gradle @@ -1,4 +1,3 @@ - dependencies { implementation project(':data-prepper-api') implementation project(':data-prepper-plugins:aws-plugin-api') @@ -11,6 +10,7 @@ dependencies { implementation 'org.hibernate.validator:hibernate-validator:8.0.2.Final' testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml' testImplementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + testImplementation project(':data-prepper-test-common') } test { diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsPluginConfigVariable.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsPluginConfigVariable.java new file mode 100644 index 0000000000..d53528443e --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsPluginConfigVariable.java @@ -0,0 +1,54 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ +package org.opensearch.dataprepper.plugins.aws; + +import org.opensearch.dataprepper.model.plugin.FailedToUpdatePluginConfigValueException; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; + +/** + * AWS Plugin configuration variable implementation. + */ +public class AwsPluginConfigVariable implements PluginConfigVariable { + + private final SecretsSupplier secretsSupplier; + private final String secretId; + private final String secretKey; + private final boolean isUpdatable; + private Object secretValue; + + public AwsPluginConfigVariable(final SecretsSupplier secretsSupplier, + final String secretId, final String secretKey, Object secretValue) { + this.secretsSupplier = secretsSupplier; + this.secretId = secretId; + this.secretKey = secretKey; + this.secretValue = secretValue; + this.isUpdatable = true; + } + + @Override + public Object getValue() { + return secretValue; + } + + @Override + public void setValue(Object newValue) { + if (!isUpdatable()) { + throw new FailedToUpdatePluginConfigValueException( + String.format("Trying to update a secrets that is not updatable. SecretId: %s SecretKey: %s", this.secretId, this.secretKey)); + } + this.secretsSupplier.updateValue(secretId, secretKey, newValue); + this.secretValue = newValue; + } + + @Override + public boolean isUpdatable() { + return isUpdatable; + } +} diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfiguration.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfiguration.java index 1aebb13a51..be95953be6 100644 --- a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfiguration.java +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfiguration.java @@ -15,6 +15,7 @@ import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.PutSecretValueRequest; import software.amazon.awssdk.services.sts.StsClient; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; @@ -78,6 +79,13 @@ public GetSecretValueRequest createGetSecretValueRequest() { .build(); } + public PutSecretValueRequest putSecretValueRequest(String secretKeyValueMapAsString) { + return PutSecretValueRequest.builder() + .secretId(awsSecretId) + .secretString(secretKeyValueMapAsString) + .build(); + } + private AwsCredentialsProvider authenticateAwsConfiguration() { final AwsCredentialsProvider awsCredentialsProvider; diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslator.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslator.java index ba544fdb32..adeb498bfb 100644 --- a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslator.java +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslator.java @@ -6,6 +6,7 @@ package org.opensearch.dataprepper.plugins.aws; import org.opensearch.dataprepper.model.plugin.PluginConfigValueTranslator; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -43,4 +44,19 @@ public Object translate(final String value) { public String getPrefix() { return AWS_SECRETS_PREFIX; } + + @Override + public PluginConfigVariable translateToPluginConfigVariable(String value) { + final Matcher matcher = SECRETS_REF_PATTERN.matcher(value); + if (!matcher.matches()) { + throw new IllegalArgumentException(String.format( + "Unable to parse %s or %s according to pattern %s", + SECRET_CONFIGURATION_ID_GROUP, SECRET_KEY_GROUP, SECRETS_REF_PATTERN.pattern())); + } + final String secretId = matcher.group(SECRET_CONFIGURATION_ID_GROUP); + final String secretKey = matcher.group(SECRET_KEY_GROUP); + final Object secretValue = secretKey != null ? secretsSupplier.retrieveValue(secretId, secretKey) : + secretsSupplier.retrieveValue(secretId); + return new AwsPluginConfigVariable(secretsSupplier, secretId, secretKey, secretValue); + } } diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplier.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplier.java index 84fe975836..8161893d29 100644 --- a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplier.java +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplier.java @@ -1,6 +1,12 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.aws; @@ -8,21 +14,23 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; +import org.opensearch.dataprepper.model.plugin.FailedToUpdatePluginConfigValueException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; +import software.amazon.awssdk.services.secretsmanager.model.PutSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.PutSecretValueResponse; import java.util.Map; import java.util.concurrent.ConcurrentMap; import java.util.stream.Collectors; public class AwsSecretsSupplier implements SecretsSupplier { - private static final Logger LOG = LoggerFactory.getLogger(AwsSecretsSupplier.class); static final TypeReference> MAP_TYPE_REFERENCE = new TypeReference<>() { }; - + private static final Logger LOG = LoggerFactory.getLogger(AwsSecretsSupplier.class); private final SecretValueDecoder secretValueDecoder; private final ObjectMapper objectMapper; private final Map awsSecretManagerConfigurationMap; @@ -94,6 +102,7 @@ public Object retrieveValue(String secretId) { } } + @Override public void refresh(String secretConfigId) { LOG.info("Retrieving latest secrets in aws:secrets:{}.", secretConfigId); @@ -126,4 +135,43 @@ private Object retrieveSecretsFromSecretManager(final AwsSecretManagerConfigurat return secretValueDecoder.decode(getSecretValueResponse); } } + + @Override + public String updateValue(String secretId, Object newValue) { + return updateValue(secretId, null, newValue); + } + + @Override + public String updateValue(String secretId, String keyToUpdate, Object newValue) { + Object currentSecretStore = secretIdToValue.get(secretId); + if (currentSecretStore instanceof Map) { + if (keyToUpdate == null) { + throw new IllegalArgumentException( + String.format("Key to update cannot be null for a key value based secret. secretId: %s", secretId)); + } + final Map keyValuePairs = (Map) currentSecretStore; + keyValuePairs.put(keyToUpdate, newValue); + } else { + //This store is not a key value pair store. It is just a value store. + //If we are here, either KeyToUpdate passed is null or we simply ignore it and just put value in the store + secretIdToValue.put(secretId, newValue); + } + // assuming all the secrets are string based (not binary) + String secretKeyValueMapAsString = (String) retrieveValue(secretId); + AwsSecretManagerConfiguration awsSecretManagerConfiguration = awsSecretManagerConfigurationMap.get(secretId); + PutSecretValueRequest putSecretValueRequest = + awsSecretManagerConfiguration.putSecretValueRequest(secretKeyValueMapAsString); + SecretsManagerClient secretsManagerClient = secretsManagerClientMap.get(secretId); + + try { + final PutSecretValueResponse putSecretValueResponse = secretsManagerClient.putSecretValue(putSecretValueRequest); + LOG.info("Updated key: {} in the secret {}. New version of the store is {}", + keyToUpdate, secretId, putSecretValueResponse.versionId()); + return putSecretValueResponse.versionId(); + } catch (Exception e) { + throw new FailedToUpdatePluginConfigValueException( + String.format("Failed to update the secret: %s to put a new value for the key: %s", + awsSecretManagerConfiguration.getAwsSecretId(), keyToUpdate), e); + } + } } diff --git a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/SecretsSupplier.java b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/SecretsSupplier.java index fad423f7cc..482e9323a0 100644 --- a/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/SecretsSupplier.java +++ b/data-prepper-plugins/aws-plugin/src/main/java/org/opensearch/dataprepper/plugins/aws/SecretsSupplier.java @@ -11,4 +11,25 @@ public interface SecretsSupplier { Object retrieveValue(String secretId); void refresh(String secretId); + + /** + * Update the value of a secret key in the secret store and responds + * with the version id of the secret after the update. + * + * @param secretId The id of the secret to be updated + * @param keyToUpdate The key of the secret to be updated + * @param newValueToSet The value of the secret to be updated + * @return The version id of the secret after the update + */ + String updateValue(String secretId, String keyToUpdate, Object newValueToSet); + + /** + * Update the value of secret store (which is not a key value secret store) and responds + * with the version id of the secret after the update. + * + * @param secretId The id of the secret to be updated + * @param newValueToSet The value of the secret to be updated + * @return The version id of the secret after the update + */ + String updateValue(String secretId, Object newValueToSet); } diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsPluginConfigVariableTest.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsPluginConfigVariableTest.java new file mode 100644 index 0000000000..faad9f72d7 --- /dev/null +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsPluginConfigVariableTest.java @@ -0,0 +1,84 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.aws; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.FailedToUpdatePluginConfigValueException; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; + +import java.util.UUID; + +import static org.hamcrest.CoreMatchers.equalTo; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class AwsPluginConfigVariableTest { + private final String secretId = "valid@secret-manager_name"; + private final String secretKey = UUID.randomUUID().toString(); + private final Object secretValue = UUID.randomUUID().toString(); + @Mock + private SecretsSupplier secretsSupplier; + private AwsPluginConfigVariable objectUnderTest; + + @BeforeEach + void setUp() { + objectUnderTest = new AwsPluginConfigVariable( + secretsSupplier, + secretId, secretKey, + secretValue + ); + } + + @Test + void testGetPrefix() { + assertThat(objectUnderTest.getValue(), equalTo(secretValue)); + } + + @Test + void testSetValueFailure_when_secret_is_not_updatable() throws NoSuchFieldException, IllegalAccessException { + objectUnderTest = new AwsPluginConfigVariable( + secretsSupplier, + secretId, secretKey, + secretValue + ); + ReflectivelySetField.setField(AwsPluginConfigVariable.class, objectUnderTest, "isUpdatable", false); + assertThrows(FailedToUpdatePluginConfigValueException.class, () -> objectUnderTest.setValue("new-secret-to-set")); + } + + @ParameterizedTest + @ValueSource(strings = { + "", + "new-secret-to-set" + }) + void testSetValueFailure(final String input) { + when(secretsSupplier.updateValue(secretId, secretKey, input)).thenThrow(RuntimeException.class); + assertThrows(RuntimeException.class, () -> objectUnderTest.setValue(input)); + } + + @ParameterizedTest + @ValueSource(strings = { + "", + "new-secret-to-set" + }) + void testSetValueSuccess(final String input) { + objectUnderTest.setValue(input); + assertThat(objectUnderTest.getValue(), equalTo(input)); + } + +} \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfigurationTest.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfigurationTest.java index 38c9159fc4..dd6f045a6e 100644 --- a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfigurationTest.java +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretManagerConfigurationTest.java @@ -1,6 +1,12 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.aws; @@ -28,6 +34,7 @@ import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.secretsmanager.SecretsManagerClientBuilder; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.PutSecretValueRequest; import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; import java.io.IOException; @@ -59,9 +66,15 @@ class AwsSecretManagerConfigurationTest { @Mock private GetSecretValueRequest.Builder getSecretValueRequestBuilder; + @Mock + private PutSecretValueRequest.Builder putSecretValueRequestBuilder; + @Mock private GetSecretValueRequest getSecretValueRequest; + @Mock + private PutSecretValueRequest putSecretValueRequest; + @Mock private SecretsManagerClientBuilder secretsManagerClientBuilder; @@ -131,6 +144,27 @@ void testCreateGetSecretValueRequest() throws IOException { verify(getSecretValueRequestBuilder).secretId("test-secret"); } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "secretValue", "{\"keyToUpdate\", \"newValue\"}"}) + void testPutSecretValueRequest_construct_put_request(String secretValueToStore) throws IOException { + when(putSecretValueRequestBuilder.secretId(anyString())).thenReturn(putSecretValueRequestBuilder); + when(putSecretValueRequestBuilder.secretString(anyString())).thenReturn(putSecretValueRequestBuilder); + when(putSecretValueRequestBuilder.build()).thenReturn(putSecretValueRequest); + final InputStream inputStream = AwsSecretPluginConfigTest.class.getResourceAsStream( + "/test-aws-secret-manager-configuration-default.yaml"); + final AwsSecretManagerConfiguration awsSecretManagerConfiguration = objectMapper.readValue( + inputStream, AwsSecretManagerConfiguration.class); + try (final MockedStatic putSecretValueRequestMockedStatic = + mockStatic(PutSecretValueRequest.class)) { + putSecretValueRequestMockedStatic.when(PutSecretValueRequest::builder).thenReturn( + putSecretValueRequestBuilder); + assertThat(awsSecretManagerConfiguration.putSecretValueRequest(secretValueToStore), + is(putSecretValueRequest)); + } + verify(putSecretValueRequestBuilder).secretId("test-secret"); + } + @Test void testCreateSecretManagerClientWithDefaultCredential() throws IOException { final InputStream inputStream = AwsSecretPluginConfigTest.class.getResourceAsStream( diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslatorTest.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslatorTest.java index 8445c34450..e21a47fcdf 100644 --- a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslatorTest.java +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsPluginConfigValueTranslatorTest.java @@ -12,6 +12,7 @@ import org.junit.jupiter.params.provider.ValueSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import java.util.UUID; @@ -65,4 +66,33 @@ void testTranslateSecretIdWithoutKeyMatch() { when(secretsSupplier.retrieveValue(eq(testSecretName))).thenReturn(testSecretValue); assertThat(objectUnderTest.translate(testSecretName), equalTo(testSecretValue)); } + + @Test + void testTranslateToPluginConfigVariableWithoutKeyMatch() { + final String testSecretName = "valid@secret-manager_name"; + final String testSecretValue = UUID.randomUUID().toString(); + when(secretsSupplier.retrieveValue(eq(testSecretName))).thenReturn(testSecretValue); + PluginConfigVariable pluginConfigVariable = objectUnderTest.translateToPluginConfigVariable(testSecretName); + assertThat(pluginConfigVariable.getValue(), equalTo(testSecretValue)); + } + + @Test + void testTranslateToPluginConfigVariableWithKeyMatch() { + final String testSecretName = "valid@secret-manager_name"; + final String testSecretKey = UUID.randomUUID().toString(); + final String testSecretValue = UUID.randomUUID().toString(); + final String input = String.format("%s:%s", testSecretName, testSecretKey); + when(secretsSupplier.retrieveValue(eq(testSecretName), eq(testSecretKey))).thenReturn(testSecretValue); + PluginConfigVariable pluginConfigVariable = objectUnderTest.translateToPluginConfigVariable(input); + assertThat(pluginConfigVariable.getValue(), equalTo(testSecretValue)); + } + + @ParameterizedTest + @ValueSource(strings = { + "", + "invalid secret id with space:secret_key" + }) + void testTranslateToPluginConfigVariableInputNoMatch(final String input) { + assertThrows(IllegalArgumentException.class, () -> objectUnderTest.translateToPluginConfigVariable(input)); + } } \ No newline at end of file diff --git a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplierTest.java b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplierTest.java index 63735d3491..8fc8a07d33 100644 --- a/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplierTest.java +++ b/data-prepper-plugins/aws-plugin/src/test/java/org/opensearch/dataprepper/plugins/aws/AwsSecretsSupplierTest.java @@ -1,6 +1,12 @@ /* * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 + * + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * */ package org.opensearch.dataprepper.plugins.aws; @@ -18,6 +24,8 @@ import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueRequest; import software.amazon.awssdk.services.secretsmanager.model.GetSecretValueResponse; +import software.amazon.awssdk.services.secretsmanager.model.PutSecretValueRequest; +import software.amazon.awssdk.services.secretsmanager.model.PutSecretValueResponse; import software.amazon.awssdk.services.secretsmanager.model.SecretsManagerException; import java.util.Map; @@ -25,7 +33,9 @@ import static org.hamcrest.CoreMatchers.equalTo; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -53,9 +63,15 @@ class AwsSecretsSupplierTest { @Mock private GetSecretValueRequest getSecretValueRequest; + @Mock + private PutSecretValueRequest putSecretValueRequest; + @Mock private GetSecretValueResponse getSecretValueResponse; + @Mock + private PutSecretValueResponse putSecretValueResponse; + @Mock private SecretsManagerException secretsManagerException; @@ -163,4 +179,55 @@ void testRefreshSecretsWithoutKey() { objectUnderTest.refresh(TEST_AWS_SECRET_CONFIGURATION_NAME); assertThat(objectUnderTest.retrieveValue(TEST_AWS_SECRET_CONFIGURATION_NAME), equalTo(newTestValue)); } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "newValue", "{\"key\":\"oldValue\"}", "{\"a\":\"b\"}"}) + void testUpdateValue_successfully_updated(String valueToSet) { + when(awsSecretManagerConfiguration.putSecretValueRequest(any())).thenReturn(putSecretValueRequest); + when(secretsManagerClient.putSecretValue(eq(putSecretValueRequest))).thenReturn(putSecretValueResponse); + String newVersionId = UUID.randomUUID().toString(); + when(putSecretValueResponse.versionId()).thenReturn(newVersionId); + objectUnderTest = new AwsSecretsSupplier(secretValueDecoder, awsSecretPluginConfig, OBJECT_MAPPER); + assertThat(objectUnderTest.updateValue(TEST_AWS_SECRET_CONFIGURATION_NAME, "key", valueToSet), + equalTo(newVersionId)); + } + + @Test + void testUpdateValue_null_key_throws_exception() { + when(secretsManagerClient.getSecretValue(eq(getSecretValueRequest))).thenReturn(getSecretValueResponse); + objectUnderTest = new AwsSecretsSupplier(secretValueDecoder, awsSecretPluginConfig, OBJECT_MAPPER); + assertThrows(IllegalArgumentException.class, + () -> objectUnderTest.updateValue(TEST_AWS_SECRET_CONFIGURATION_NAME, "newValue")); + } + + @ParameterizedTest + @ValueSource(strings = {"", " ", "newValue"}) + void testUpdateValue_null_key_doesnot_throws_exception_when_value_is_not_key_value_pair(String secretValueToSet) { + when(awsSecretManagerConfiguration.createGetSecretValueRequest()).thenReturn(getSecretValueRequest); + when(awsSecretPluginConfig.getAwsSecretManagerConfigurationMap()).thenReturn( + Map.of(TEST_AWS_SECRET_CONFIGURATION_NAME, awsSecretManagerConfiguration) + ); + when(awsSecretManagerConfiguration.createSecretManagerClient()).thenReturn(secretsManagerClient); + when(secretValueDecoder.decode(eq(getSecretValueResponse))).thenReturn(TEST_VALUE); + when(secretsManagerClient.getSecretValue(eq(getSecretValueRequest))).thenReturn(getSecretValueResponse); + when(awsSecretManagerConfiguration.putSecretValueRequest(any())).thenReturn(putSecretValueRequest); + when(secretsManagerClient.putSecretValue(eq(putSecretValueRequest))).thenReturn(putSecretValueResponse); + String versionId = UUID.randomUUID().toString(); + when(putSecretValueResponse.versionId()).thenReturn(versionId); + objectUnderTest = new AwsSecretsSupplier(secretValueDecoder, awsSecretPluginConfig, OBJECT_MAPPER); + String newValue = objectUnderTest.updateValue(TEST_AWS_SECRET_CONFIGURATION_NAME, secretValueToSet); + assertEquals(versionId, newValue); + } + + @Test + void testUpdateValue_failed_to_update() { + when(awsSecretManagerConfiguration.putSecretValueRequest(any())).thenReturn(putSecretValueRequest); + when(secretsManagerClient.putSecretValue(eq(putSecretValueRequest))).thenReturn(putSecretValueResponse); + final String testValue = "{\"key\":\"oldValue\"}"; + when(secretValueDecoder.decode(eq(getSecretValueResponse))).thenReturn(testValue); + when(putSecretValueResponse.versionId()).thenThrow(RuntimeException.class); + objectUnderTest = new AwsSecretsSupplier(secretValueDecoder, awsSecretPluginConfig, OBJECT_MAPPER); + assertThrows(RuntimeException.class, + () -> objectUnderTest.updateValue(TEST_AWS_SECRET_CONFIGURATION_NAME, "key", "newValue")); + } } \ No newline at end of file diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle b/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle index 3edfc6c4f7..1cf3854ab4 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle +++ b/data-prepper-plugins/saas-source-plugins/jira-source/build.gradle @@ -21,6 +21,7 @@ dependencies { annotationProcessor 'org.projectlombok:lombok:1.18.30' testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.13.4' + testImplementation project(path: ':data-prepper-test-common') implementation(libs.spring.context) { exclude group: 'commons-logging', module: 'commons-logging' diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/Oauth2Config.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/Oauth2Config.java index 3282e7b38f..b013bb3527 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/Oauth2Config.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/configuration/Oauth2Config.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import jakarta.validation.constraints.AssertTrue; import lombok.Getter; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; @Getter public class Oauth2Config { @@ -23,10 +24,10 @@ public class Oauth2Config { private String clientSecret; @JsonProperty("access_token") - private String accessToken; + private PluginConfigVariable accessToken; @JsonProperty("refresh_token") - private String refreshToken; + private PluginConfigVariable refreshToken; @AssertTrue(message = "Client ID, Client Secret, Access Token, and Refresh Token are both required for Oauth2") private boolean isOauth2ConfigValid() { diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java index caf5d84ee7..92420ac319 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/JiraRestClient.java @@ -110,7 +110,7 @@ public String getIssue(String issueKey) { return invokeRestApi(uri, String.class).getBody(); } - private ResponseEntity invokeRestApi(URI uri, Class responseType) throws BadRequestException{ + private ResponseEntity invokeRestApi(URI uri, Class responseType) throws BadRequestException { AddressValidation.validateInetAddress(AddressValidation.getInetAddress(uri.toString())); int retryCount = 0; while (retryCount < RETRY_ATTEMPT) { @@ -119,7 +119,7 @@ private ResponseEntity invokeRestApi(URI uri, Class responseType) thro } catch (HttpClientErrorException ex) { HttpStatus statusCode = ex.getStatusCode(); String statusMessage = ex.getMessage(); - log.error("An exception has occurred while getting response from Jira search API {}", ex.getMessage(), ex); + log.error("An exception has occurred while getting response from Jira search API {}", ex.getMessage()); if (statusCode == HttpStatus.FORBIDDEN) { throw new UnAuthorizedException(statusMessage); } else if (statusCode == HttpStatus.UNAUTHORIZED) { @@ -136,7 +136,7 @@ private ResponseEntity invokeRestApi(URI uri, Class responseType) thro } retryCount++; } - String errorMessage = String.format("Exceeded max retry attempts. Failed to execute the Rest API call %s", uri.toString()); + String errorMessage = String.format("Exceeded max retry attempts. Failed to execute the Rest API call %s", uri); log.error(errorMessage); throw new RuntimeException(errorMessage); } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java index 791c9e6ff4..aaa9850a8c 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/main/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraOauthConfig.java @@ -14,6 +14,7 @@ import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; import org.opensearch.dataprepper.plugins.source.jira.exception.UnAuthorizedException; import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; @@ -44,8 +45,7 @@ public class JiraOauthConfig implements JiraAuthConfig { public static final String EXPIRES_IN = "expires_in"; public static final String REFRESH_TOKEN = "refresh_token"; public static final String ACCESS_TOKEN = "access_token"; - private static final Logger log = - org.slf4j.LoggerFactory.getLogger(JiraOauthConfig.class); + private static final Logger log = LoggerFactory.getLogger(JiraOauthConfig.class); private final String clientId; private final String clientSecret; private final JiraSourceConfig jiraSourceConfig; @@ -65,8 +65,10 @@ public class JiraOauthConfig implements JiraAuthConfig { public JiraOauthConfig(JiraSourceConfig jiraSourceConfig) { this.jiraSourceConfig = jiraSourceConfig; - this.accessToken = jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken(); - this.refreshToken = jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getRefreshToken(); + this.accessToken = (String) jiraSourceConfig.getAuthenticationConfig().getOauth2Config() + .getAccessToken().getValue(); + this.refreshToken = (String) jiraSourceConfig.getAuthenticationConfig() + .getOauth2Config().getRefreshToken().getValue(); this.clientId = jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId(); this.clientSecret = jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret(); } @@ -117,7 +119,7 @@ public void renewCredentials() { return; } - log.info("Renewing access-refresh token pair for Jira Connector."); + log.info("Renewing access token and refresh token pair for Jira Connector."); HttpHeaders headers = new HttpHeaders(); headers.setContentType(MediaType.APPLICATION_JSON); String payloadTemplate = "{\"grant_type\": \"%s\", \"client_id\": \"%s\", \"client_secret\": \"%s\", \"refresh_token\": \"%s\"}"; @@ -131,6 +133,12 @@ public void renewCredentials() { this.refreshToken = (String) oauthClientResponse.get(REFRESH_TOKEN); this.expiresInSeconds = (int) oauthClientResponse.get(EXPIRES_IN); this.expireTime = Instant.ofEpochMilli(System.currentTimeMillis() + (expiresInSeconds * 1000L)); + // updating config object's PluginConfigVariable so that it updates the underlying Secret store + jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken() + .setValue(this.accessToken); + jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getRefreshToken() + .setValue(this.refreshToken); + log.info("Access Token and Refresh Token pair is now refreshed. Corresponding Secret store key updated."); } catch (HttpClientErrorException ex) { this.expireTime = Instant.ofEpochMilli(0); this.expiresInSeconds = 0; diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java index eeb9c03941..ceadbaf65b 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraConfigHelperTest.java @@ -14,6 +14,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; import org.opensearch.dataprepper.plugins.source.jira.configuration.BasicConfig; import org.opensearch.dataprepper.plugins.source.jira.configuration.FilterConfig; @@ -51,10 +52,10 @@ public class JiraConfigHelperTest { IssueTypeConfig issueTypeConfig; @Mock - ProjectConfig projectConfig; + ProjectConfig projectConfig; @Mock - NameConfig nameConfig; + NameConfig nameConfig; @Mock AuthenticationConfig authenticationConfig; @@ -63,7 +64,13 @@ public class JiraConfigHelperTest { BasicConfig basicConfig; @Mock - Oauth2Config oauth2Config; + Oauth2Config oauth2Config; + + @Mock + PluginConfigVariable accessTokenPluginConfigVariable; + + @Mock + PluginConfigVariable refreshTokenPluginConfigVariable; @Test void testInitialization() { @@ -153,14 +160,14 @@ void testValidateConfigOauth2() { when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(oauth2Config.getAccessToken()).thenReturn("id"); + when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(oauth2Config.getRefreshToken()).thenReturn("credential"); + when(authenticationConfig.getOauth2Config().getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); when(oauth2Config.getAccessToken()).thenReturn(null); assertThrows(RuntimeException.class, () -> JiraConfigHelper.validateConfig(jiraSourceConfig)); - when(oauth2Config.getAccessToken()).thenReturn("id"); + when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); assertDoesNotThrow(() -> JiraConfigHelper.validateConfig(jiraSourceConfig)); } } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java index 0de0ea47dc..484ca2caac 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraServiceTest.java @@ -18,12 +18,16 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.jira.configuration.Oauth2Config; import org.opensearch.dataprepper.plugins.source.jira.exception.BadRequestException; import org.opensearch.dataprepper.plugins.source.jira.models.IssueBean; import org.opensearch.dataprepper.plugins.source.jira.models.SearchResults; import org.opensearch.dataprepper.plugins.source.jira.rest.JiraRestClient; +import org.opensearch.dataprepper.plugins.source.jira.utils.MockPluginConfigVariableImpl; import org.opensearch.dataprepper.plugins.source.source_crawler.base.PluginExecutorServiceProvider; import org.opensearch.dataprepper.plugins.source.source_crawler.model.ItemInfo; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -84,9 +88,19 @@ private static InputStream getResourceAsStream(String resourceName) { public static JiraSourceConfig createJiraConfigurationFromYaml(String fileName) { ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory()); try (InputStream inputStream = getResourceAsStream(fileName)) { - return objectMapper.readValue(inputStream, JiraSourceConfig.class); + JiraSourceConfig jiraSourceConfig = objectMapper.readValue(inputStream, JiraSourceConfig.class); + Oauth2Config oauth2Config = jiraSourceConfig.getAuthenticationConfig().getOauth2Config(); + if (oauth2Config != null) { + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "accessToken", + new MockPluginConfigVariableImpl("mockAccessToken")); + ReflectivelySetField.setField(Oauth2Config.class, oauth2Config, "refreshToken", + new MockPluginConfigVariableImpl("mockRefreshToken")); + } + return jiraSourceConfig; } catch (IOException ex) { log.error("Failed to parse pipeline Yaml", ex); + } catch (Exception e) { + throw new RuntimeException(e); } return null; } @@ -95,6 +109,8 @@ public static JiraSourceConfig createJiraConfiguration(String auth_type, List issueType, List issueStatus, List projectKey) throws JsonProcessingException { + PluginConfigVariable pcvAccessToken = null; + PluginConfigVariable pcvRefreshToken = null; ObjectMapper objectMapper = new ObjectMapper(); Map authenticationMap = new HashMap<>(); Map basicMap = new HashMap<>(); @@ -103,11 +119,11 @@ public static JiraSourceConfig createJiraConfiguration(String auth_type, basicMap.put("username", "test_username"); basicMap.put("password", "test_password"); authenticationMap.put("basic", basicMap); - } else if (auth_type.equals(OAUTH2)) { + } else if (auth_type.equals(OAUTH2)) { oauth2Map.put("client_id", "test-client-id"); oauth2Map.put("client_secret", "test-client-secret"); - oauth2Map.put("access_token", "test-access-token"); - oauth2Map.put("refresh_token", "test-refresh-token"); + pcvAccessToken = new MockPluginConfigVariableImpl("test-access-token"); + pcvRefreshToken = new MockPluginConfigVariableImpl("test-refresh-token"); authenticationMap.put("oauth2", oauth2Map); } @@ -137,7 +153,18 @@ public static JiraSourceConfig createJiraConfiguration(String auth_type, jiraSourceConfigMap.put("filter", filterMap); String jiraSourceConfigJsonString = objectMapper.writeValueAsString(jiraSourceConfigMap); - return objectMapper.readValue(jiraSourceConfigJsonString, JiraSourceConfig.class); + JiraSourceConfig jiraSourceConfig = objectMapper.readValue(jiraSourceConfigJsonString, JiraSourceConfig.class); + if (jiraSourceConfig.getAuthenticationConfig().getOauth2Config() != null && pcvAccessToken != null) { + try { + ReflectivelySetField.setField(Oauth2Config.class, + jiraSourceConfig.getAuthenticationConfig().getOauth2Config(), "accessToken", pcvAccessToken); + ReflectivelySetField.setField(Oauth2Config.class, + jiraSourceConfig.getAuthenticationConfig().getOauth2Config(), "refreshToken", pcvRefreshToken); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + return jiraSourceConfig; } @AfterEach diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java index 35a2450fdb..b7b30af5f6 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/JiraSourceConfigTest.java @@ -10,9 +10,12 @@ package org.opensearch.dataprepper.plugins.source.jira; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; +import org.opensearch.dataprepper.plugins.source.jira.configuration.Oauth2Config; +import org.opensearch.dataprepper.plugins.source.jira.utils.MockPluginConfigVariableImpl; +import org.opensearch.dataprepper.test.helper.ReflectivelySetField; import java.util.ArrayList; import java.util.HashMap; @@ -27,19 +30,21 @@ import static org.opensearch.dataprepper.plugins.source.source_crawler.base.CrawlerSourceConfig.DEFAULT_NUMBER_OF_WORKERS; public class JiraSourceConfigTest { - private final String accessToken = "access token test"; - private final String refreshToken = "refresh token test"; + private final PluginConfigVariable accessToken = new MockPluginConfigVariableImpl("access token test"); + private final PluginConfigVariable refreshToken = new MockPluginConfigVariableImpl("refresh token test"); private final String clientId = "client id test"; private final String clientSecret = "client secret test"; private final String password = "test Jira Credential"; private final String username = "test Jira Id"; private final String accountUrl = "https://example.atlassian.net"; - private List projectList = new ArrayList<>(); - private List issueTypeList = new ArrayList<>(); - private List statusList = new ArrayList<>(); + private final List projectList = new ArrayList<>(); + private final List issueTypeList = new ArrayList<>(); + private final List statusList = new ArrayList<>(); private JiraSourceConfig jiraSourceConfig; - private JiraSourceConfig createJiraSourceConfig(String authtype, boolean hasToken) throws JsonProcessingException { + private JiraSourceConfig createJiraSourceConfig(String authtype, boolean hasToken) throws Exception { + PluginConfigVariable pcvAccessToken = null; + PluginConfigVariable pcvRefreshToken = null; Map configMap = new HashMap<>(); List hosts = new ArrayList<>(); hosts.add(accountUrl); @@ -48,17 +53,16 @@ private JiraSourceConfig createJiraSourceConfig(String authtype, boolean hasToke Map authenticationMap = new HashMap<>(); Map basicMap = new HashMap<>(); - Map oauth2Map = new HashMap<>(); + Map oauth2Map = new HashMap<>(); if (authtype.equals(BASIC)) { basicMap.put("username", username); basicMap.put("password", password); authenticationMap.put("basic", basicMap); } else if (authtype.equals(OAUTH2)) { if (hasToken) { - oauth2Map.put("access_token", accessToken); - oauth2Map.put("refresh_token", refreshToken); + pcvRefreshToken = refreshToken; + pcvAccessToken = accessToken; } else { - oauth2Map.put("access_token", null); oauth2Map.put("refresh_token", null); } oauth2Map.put("client_id", clientId); @@ -98,11 +102,17 @@ private JiraSourceConfig createJiraSourceConfig(String authtype, boolean hasToke ObjectMapper objectMapper = new ObjectMapper(); String jsonConfig = objectMapper.writeValueAsString(configMap); JiraSourceConfig config = objectMapper.readValue(jsonConfig, JiraSourceConfig.class); + if (config.getAuthenticationConfig().getOauth2Config() != null && pcvAccessToken != null) { + ReflectivelySetField.setField(Oauth2Config.class, + config.getAuthenticationConfig().getOauth2Config(), "accessToken", pcvAccessToken); + ReflectivelySetField.setField(Oauth2Config.class, + config.getAuthenticationConfig().getOauth2Config(), "refreshToken", pcvRefreshToken); + } return config; } @Test - void testGetters() throws JsonProcessingException { + void testGetters() throws Exception { jiraSourceConfig = createJiraSourceConfig(BASIC, false); assertEquals(jiraSourceConfig.getFilterConfig().getIssueTypeConfig().getInclude(), issueTypeList); assertEquals(jiraSourceConfig.getNumWorkers(), DEFAULT_NUMBER_OF_WORKERS); @@ -115,18 +125,17 @@ void testGetters() throws JsonProcessingException { } @Test - void testFetchGivenOauthAttributeWrongAuthType() throws JsonProcessingException { + void testFetchGivenOauthAttributeWrongAuthType() throws Exception { jiraSourceConfig = createJiraSourceConfig(BASIC, true); assertThrows(RuntimeException.class, () -> jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); } @Test - void testFetchGivenOauthAtrribute() throws JsonProcessingException { + void testFetchGivenOauthAtrribute() throws Exception { jiraSourceConfig = createJiraSourceConfig(OAUTH2, true); assertEquals(accessToken, jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getAccessToken()); assertEquals(refreshToken, jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getRefreshToken()); assertEquals(clientId, jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getClientId()); assertEquals(clientSecret, jiraSourceConfig.getAuthenticationConfig().getOauth2Config().getClientSecret()); } - } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java index 11d29c78c0..1f620c822a 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/CustomRestTemplateConfigTest.java @@ -17,6 +17,7 @@ import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; import org.opensearch.dataprepper.plugins.source.jira.configuration.BasicConfig; @@ -49,11 +50,17 @@ class CustomRestTemplateConfigTest { private JiraAuthConfig mockAuthConfig; @Mock - private BasicConfig mockBasicConfig; + private BasicConfig mockBasicConfig; @Mock private Oauth2Config mockOauth2Config; + @Mock + private PluginConfigVariable accessTokenPluginConfigVariable; + + @Mock + private PluginConfigVariable refreshTokenPluginConfigVariable; + @Mock private AuthenticationConfig mockAuthenticationConfig; @@ -77,8 +84,9 @@ void testBasicAuthRestTemplateWithOAuth2(String authType, Class interceptorClass when(mockSourceConfig.getAuthType()).thenReturn(authType); lenient().when(mockSourceConfig.getAuthenticationConfig()).thenReturn(mockAuthenticationConfig); lenient().when(mockAuthenticationConfig.getOauth2Config()).thenReturn(mockOauth2Config); - lenient().when(mockOauth2Config.getAccessToken()).thenReturn("accessToken"); - lenient().when(mockOauth2Config.getRefreshToken()).thenReturn("refreshToken"); + lenient().when(mockOauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + lenient().when(mockOauth2Config.getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); + lenient().when(accessTokenPluginConfigVariable.getValue()).thenReturn("accessToken"); lenient().when(mockOauth2Config.getClientId()).thenReturn("clientId"); lenient().when(mockOauth2Config.getClientSecret()).thenReturn("clientSecret"); lenient().when(mockAuthenticationConfig.getBasicConfig()).thenReturn(mockBasicConfig); diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java index 5106bd8ad1..7ae2fe588d 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/rest/auth/JiraAuthFactoryTest.java @@ -15,6 +15,7 @@ import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; import org.opensearch.dataprepper.plugins.source.jira.JiraSourceConfig; import org.opensearch.dataprepper.plugins.source.jira.configuration.AuthenticationConfig; import org.opensearch.dataprepper.plugins.source.jira.configuration.Oauth2Config; @@ -31,11 +32,17 @@ public class JiraAuthFactoryTest { private JiraSourceConfig sourceConfig; @Mock - private AuthenticationConfig authenticationConfig; + private AuthenticationConfig authenticationConfig; @Mock private Oauth2Config oauth2Config; + @Mock + private PluginConfigVariable accessTokenPluginConfigVariable; + + @Mock + private PluginConfigVariable refreshTokenPluginConfigVariable; + private JiraAuthFactory jiraAuthFactory; @BeforeEach @@ -48,6 +55,9 @@ void testGetObjectOauth2() { when(sourceConfig.getAuthType()).thenReturn(OAUTH2); when(sourceConfig.getAuthenticationConfig()).thenReturn(authenticationConfig); when(authenticationConfig.getOauth2Config()).thenReturn(oauth2Config); + when(oauth2Config.getRefreshToken()).thenReturn(refreshTokenPluginConfigVariable); + when(oauth2Config.getAccessToken()).thenReturn(accessTokenPluginConfigVariable); + when(accessTokenPluginConfigVariable.getValue()).thenReturn("mockRefreshToken"); assertInstanceOf(JiraOauthConfig.class, jiraAuthFactory.getObject()); } diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/MockPluginConfigVariableImpl.java b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/MockPluginConfigVariableImpl.java new file mode 100644 index 0000000000..aa8cfd87ab --- /dev/null +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/java/org/opensearch/dataprepper/plugins/source/jira/utils/MockPluginConfigVariableImpl.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + * + */ + +package org.opensearch.dataprepper.plugins.source.jira.utils; + +import org.opensearch.dataprepper.model.plugin.PluginConfigVariable; + +/** + * Mock implementation of PluginConfigVariable interface used only for Unit Testing. + */ +public class MockPluginConfigVariableImpl implements PluginConfigVariable { + + private Object defaultValue; + + public MockPluginConfigVariableImpl(Object defaultValue) { + this.defaultValue = defaultValue; + } + + @Override + public Object getValue() { + return null; + } + + @Override + public void setValue(Object someValue) { + this.defaultValue = someValue; + } + + @Override + public boolean isUpdatable() { + return true; + } +} diff --git a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml index 09c9e9f2c5..7a4afd3abf 100644 --- a/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml +++ b/data-prepper-plugins/saas-source-plugins/jira-source/src/test/resources/oauth2-auth-jira-pipeline.yaml @@ -1,8 +1,6 @@ -hosts: ["https://jira.com/"] +hosts: [ "https://jira.com/" ] authentication: oauth2: client_id: "client_id" client_secret: "client_secret" - access_token: "access_token" - refresh_token: "refresh_token"