From ff27e16b38ab578c8828c3230e5337a14e558e77 Mon Sep 17 00:00:00 2001 From: Santhosh Gandhe <1909520+san81@users.noreply.github.com> Date: Sun, 17 Nov 2024 12:18:32 -0800 Subject: [PATCH] making the event type default Signed-off-by: Santhosh Gandhe <1909520+san81@users.noreply.github.com> --- .../common/config/LambdaCommonConfig.java | 78 +++++++++---------- .../processor/LambdaProcessorConfig.java | 66 +++++++++------- .../plugins/lambda/sink/LambdaSinkConfig.java | 61 ++++++++------- .../common/config/LambdaCommonConfigTest.java | 67 +++++++++------- 4 files changed, 153 insertions(+), 119 deletions(-) diff --git a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/common/config/LambdaCommonConfig.java b/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/common/config/LambdaCommonConfig.java index c60d050626..8559551ea5 100644 --- a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/common/config/LambdaCommonConfig.java +++ b/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/common/config/LambdaCommonConfig.java @@ -11,47 +11,47 @@ import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; import jakarta.validation.constraints.Size; -import org.opensearch.dataprepper.model.configuration.PluginModel; - -import lombok.Getter; import java.time.Duration; +import lombok.Getter; +import org.opensearch.dataprepper.model.configuration.PluginModel; @Getter -public class LambdaCommonConfig { - public static final int DEFAULT_CONNECTION_RETRIES = 3; - public static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofSeconds(60); - public static final String STS_REGION = "region"; - public static final String STS_ROLE_ARN = "sts_role_arn"; - - @JsonProperty("aws") - @NotNull - @Valid - private AwsAuthenticationOptions awsAuthenticationOptions; - - @JsonPropertyDescription("Lambda Function Name") - @JsonProperty("function_name") - @NotEmpty - @Size(min = 3, max = 500, message = "function name length should be at least 3 characters") - private String functionName; - - @JsonPropertyDescription("Total retries we want before failing") - @JsonProperty("max_retries") - private int maxConnectionRetries = DEFAULT_CONNECTION_RETRIES; - - @JsonPropertyDescription("invocation type defines the way we want to call lambda function") - @JsonProperty("invocation_type") - private InvocationType invocationType = InvocationType.REQUEST_RESPONSE; - - @JsonPropertyDescription("sdk timeout defines the time sdk maintains the connection to the client before timing out") - @JsonProperty("connection_timeout") - private Duration connectionTimeout = DEFAULT_CONNECTION_TIMEOUT; - - @JsonProperty("batch") - private BatchOptions batchOptions; - - @JsonPropertyDescription("Codec configuration for parsing Lambda responses") - @JsonProperty("response_codec") - @Valid - private PluginModel responseCodecConfig; +public abstract class LambdaCommonConfig { + + public static final int DEFAULT_CONNECTION_RETRIES = 3; + public static final Duration DEFAULT_CONNECTION_TIMEOUT = Duration.ofSeconds(60); + public static final String STS_REGION = "region"; + public static final String STS_ROLE_ARN = "sts_role_arn"; + + @JsonProperty("aws") + @NotNull + @Valid + private AwsAuthenticationOptions awsAuthenticationOptions; + + @JsonPropertyDescription("Lambda Function Name") + @JsonProperty("function_name") + @NotEmpty + @Size(min = 3, max = 500, message = "function name length should be at least 3 characters") + private String functionName; + + @JsonPropertyDescription("Total retries we want before failing") + @JsonProperty("max_retries") + private int maxConnectionRetries = DEFAULT_CONNECTION_RETRIES; + + + @JsonPropertyDescription("sdk timeout defines the time sdk maintains the connection to the client before timing out") + @JsonProperty("connection_timeout") + private Duration connectionTimeout = DEFAULT_CONNECTION_TIMEOUT; + + @JsonProperty("batch") + private BatchOptions batchOptions; + + @JsonPropertyDescription("Codec configuration for parsing Lambda responses") + @JsonProperty("response_codec") + @Valid + private PluginModel responseCodecConfig; + + + public abstract InvocationType getInvocationType(); } diff --git a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorConfig.java b/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorConfig.java index 21ae307d17..5d7e05d7c0 100644 --- a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorConfig.java +++ b/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorConfig.java @@ -7,37 +7,47 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonPropertyDescription; -import org.opensearch.dataprepper.plugins.lambda.common.config.LambdaCommonConfig; - import java.util.Collections; import java.util.List; +import org.opensearch.dataprepper.plugins.lambda.common.config.InvocationType; +import org.opensearch.dataprepper.plugins.lambda.common.config.LambdaCommonConfig; public class LambdaProcessorConfig extends LambdaCommonConfig { - @JsonPropertyDescription("Defines the way Data Prepper treats the response from Lambda") - @JsonProperty("response_events_match") - private boolean responseEventsMatch = false; - - @JsonPropertyDescription("defines a condition for event to use this processor") - @JsonProperty("lambda_when") - private String whenCondition; - - @JsonProperty("tags_on_failure") - @JsonPropertyDescription("A List of Strings that specifies the tags to be set in the event when lambda fails to " + - "or exception occurs. This tag may be used in conditional expressions in " + - "other parts of the configuration") - private List tagsOnFailure = Collections.emptyList(); - - public List getTagsOnFailure(){ - return tagsOnFailure; - } - - public String getWhenCondition() { - return whenCondition; - } - - public Boolean getResponseEventsMatch() { - return responseEventsMatch; - } - + @JsonPropertyDescription("invocation type defines the way we want to call lambda function") + @JsonProperty("invocation_type") + private InvocationType invocationType = InvocationType.REQUEST_RESPONSE; + + @JsonPropertyDescription("Defines the way Data Prepper treats the response from Lambda") + @JsonProperty("response_events_match") + private boolean responseEventsMatch = false; + + @JsonPropertyDescription("defines a condition for event to use this processor") + @JsonProperty("lambda_when") + private String whenCondition; + + @JsonProperty("tags_on_failure") + @JsonPropertyDescription( + "A List of Strings that specifies the tags to be set in the event when lambda fails to " + + + "or exception occurs. This tag may be used in conditional expressions in " + + "other parts of the configuration") + private List tagsOnFailure = Collections.emptyList(); + + public List getTagsOnFailure() { + return tagsOnFailure; + } + + public String getWhenCondition() { + return whenCondition; + } + + public Boolean getResponseEventsMatch() { + return responseEventsMatch; + } + + @Override + public InvocationType getInvocationType() { + return invocationType; + } } diff --git a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/sink/LambdaSinkConfig.java b/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/sink/LambdaSinkConfig.java index 2c68d25986..52003f9c1d 100644 --- a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/sink/LambdaSinkConfig.java +++ b/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/sink/LambdaSinkConfig.java @@ -6,35 +6,44 @@ package org.opensearch.dataprepper.plugins.lambda.sink; import com.fasterxml.jackson.annotation.JsonProperty; -import org.opensearch.dataprepper.model.configuration.PluginModel; -import org.opensearch.dataprepper.plugins.lambda.common.config.LambdaCommonConfig; - +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import java.util.Map; import java.util.Objects; +import org.opensearch.dataprepper.model.configuration.PluginModel; +import org.opensearch.dataprepper.plugins.lambda.common.config.InvocationType; +import org.opensearch.dataprepper.plugins.lambda.common.config.LambdaCommonConfig; public class LambdaSinkConfig extends LambdaCommonConfig { - @JsonProperty("dlq") - private PluginModel dlq; - - public PluginModel getDlq() { - return dlq; - } - - public String getDlqStsRoleARN(){ - return Objects.nonNull(getDlqPluginSetting().get(STS_ROLE_ARN)) ? - String.valueOf(getDlqPluginSetting().get(STS_ROLE_ARN)) : - getAwsAuthenticationOptions().getAwsStsRoleArn(); - } - - public String getDlqStsRegion(){ - return Objects.nonNull(getDlqPluginSetting().get(STS_REGION)) ? - String.valueOf(getDlqPluginSetting().get(STS_REGION)) : - getAwsAuthenticationOptions().getAwsRegion().toString(); - } - - public Map getDlqPluginSetting(){ - return dlq != null ? dlq.getPluginSettings() : null; - } - + @JsonPropertyDescription("invocation type defines the way we want to call lambda function") + @JsonProperty("invocation_type") + private InvocationType invocationType = InvocationType.EVENT; + + @JsonProperty("dlq") + private PluginModel dlq; + + public PluginModel getDlq() { + return dlq; + } + + public String getDlqStsRoleARN() { + return Objects.nonNull(getDlqPluginSetting().get(STS_ROLE_ARN)) ? + String.valueOf(getDlqPluginSetting().get(STS_ROLE_ARN)) : + getAwsAuthenticationOptions().getAwsStsRoleArn(); + } + + public String getDlqStsRegion() { + return Objects.nonNull(getDlqPluginSetting().get(STS_REGION)) ? + String.valueOf(getDlqPluginSetting().get(STS_REGION)) : + getAwsAuthenticationOptions().getAwsRegion().toString(); + } + + public Map getDlqPluginSetting() { + return dlq != null ? dlq.getPluginSettings() : null; + } + + @Override + public InvocationType getInvocationType() { + return invocationType; + } } diff --git a/data-prepper-plugins/aws-lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/common/config/LambdaCommonConfigTest.java b/data-prepper-plugins/aws-lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/common/config/LambdaCommonConfigTest.java index 2ddcf08b46..1c440fd7e6 100644 --- a/data-prepper-plugins/aws-lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/common/config/LambdaCommonConfigTest.java +++ b/data-prepper-plugins/aws-lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/common/config/LambdaCommonConfigTest.java @@ -5,37 +5,52 @@ package org.opensearch.dataprepper.plugins.lambda.common.config; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.equalTo; + import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; -import static org.hamcrest.MatcherAssert.assertThat; -import static org.hamcrest.Matchers.equalTo; import org.junit.jupiter.api.Test; +import org.opensearch.dataprepper.plugins.lambda.processor.LambdaProcessorConfig; import software.amazon.awssdk.regions.Region; + public class LambdaCommonConfigTest { - private final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory().enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS)); - - @Test - void test_defaults() { - final LambdaCommonConfig lambdaCommonConfig = new LambdaCommonConfig(); - assertThat(lambdaCommonConfig.getFunctionName(), equalTo(null)); - assertThat(lambdaCommonConfig.getAwsAuthenticationOptions(), equalTo(null)); - assertThat(lambdaCommonConfig.getBatchOptions(), equalTo(null)); - assertThat(lambdaCommonConfig.getResponseCodecConfig(), equalTo(null)); - assertThat(lambdaCommonConfig.getConnectionTimeout(), equalTo(LambdaCommonConfig.DEFAULT_CONNECTION_TIMEOUT)); - assertThat(lambdaCommonConfig.getMaxConnectionRetries(), equalTo(LambdaCommonConfig.DEFAULT_CONNECTION_RETRIES)); - assertThat(lambdaCommonConfig.getInvocationType(), equalTo(InvocationType.REQUEST_RESPONSE)); - } - - @Test - public void testAwsAuthenticationOptionsNotNull() throws JsonProcessingException { - final String config = " function_name: test_function\n" + " aws:\n" + " region: ap-south-1\n" + " sts_role_arn: arn:aws:iam::524239988912:role/app-test\n" + " sts_header_overrides: {\"test\":\"test\"}\n" + " max_retries: 10\n"; - final LambdaCommonConfig lambdaCommonConfig = objectMapper.readValue(config, LambdaCommonConfig.class); - - assertThat(lambdaCommonConfig.getMaxConnectionRetries(), equalTo(10)); - assertThat(lambdaCommonConfig.getAwsAuthenticationOptions().getAwsRegion(), equalTo(Region.AP_SOUTH_1)); - assertThat(lambdaCommonConfig.getAwsAuthenticationOptions().getAwsStsRoleArn(), equalTo("arn:aws:iam::524239988912:role/app-test")); - assertThat(lambdaCommonConfig.getAwsAuthenticationOptions().getAwsStsHeaderOverrides().get("test"), equalTo("test")); - } + + private final ObjectMapper objectMapper = new ObjectMapper( + new YAMLFactory().enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS)); + + @Test + void test_defaults() { + final LambdaCommonConfig lambdaCommonConfig = new LambdaProcessorConfig(); + assertThat(lambdaCommonConfig.getFunctionName(), equalTo(null)); + assertThat(lambdaCommonConfig.getAwsAuthenticationOptions(), equalTo(null)); + assertThat(lambdaCommonConfig.getBatchOptions(), equalTo(null)); + assertThat(lambdaCommonConfig.getResponseCodecConfig(), equalTo(null)); + assertThat(lambdaCommonConfig.getConnectionTimeout(), + equalTo(LambdaCommonConfig.DEFAULT_CONNECTION_TIMEOUT)); + assertThat(lambdaCommonConfig.getMaxConnectionRetries(), + equalTo(LambdaCommonConfig.DEFAULT_CONNECTION_RETRIES)); + assertThat(lambdaCommonConfig.getInvocationType(), equalTo(InvocationType.REQUEST_RESPONSE)); + } + + @Test + public void testAwsAuthenticationOptionsNotNull() throws JsonProcessingException { + final String config = " function_name: test_function\n" + " aws:\n" + + " region: ap-south-1\n" + + " sts_role_arn: arn:aws:iam::524239988912:role/app-test\n" + + " sts_header_overrides: {\"test\":\"test\"}\n" + " max_retries: 10\n"; + final LambdaProcessorConfig lambdaCommonConfig = objectMapper.readValue(config, + LambdaProcessorConfig.class); + + assertThat(lambdaCommonConfig.getMaxConnectionRetries(), equalTo(10)); + assertThat(lambdaCommonConfig.getAwsAuthenticationOptions().getAwsRegion(), + equalTo(Region.AP_SOUTH_1)); + assertThat(lambdaCommonConfig.getAwsAuthenticationOptions().getAwsStsRoleArn(), + equalTo("arn:aws:iam::524239988912:role/app-test")); + assertThat( + lambdaCommonConfig.getAwsAuthenticationOptions().getAwsStsHeaderOverrides().get("test"), + equalTo("test")); + } }