From 58514ea27e4f9fb63e4306a4562bbfb6da5140cb Mon Sep 17 00:00:00 2001 From: Srikanth Govindarajan Date: Sat, 16 Nov 2024 17:46:39 -0800 Subject: [PATCH] Add integ test env option Signed-off-by: Srikanth Govindarajan --- data-prepper-plugins/aws-lambda/README.md | 12 +- data-prepper-plugins/aws-lambda/build.gradle | 12 +- .../lambda/processor/LambdaProcessorIT.java | 38 ++-- .../processor/LambdaProcessorServiceIT.java | 165 ------------------ .../lambda/processor/ResponseCardinality.java | 43 ----- .../processor/ResponseCardinalityTest.java | 30 ---- 6 files changed, 32 insertions(+), 268 deletions(-) delete mode 100644 data-prepper-plugins/aws-lambda/src/integrationTest/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorServiceIT.java delete mode 100644 data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/ResponseCardinality.java delete mode 100644 data-prepper-plugins/aws-lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/processor/ResponseCardinalityTest.java diff --git a/data-prepper-plugins/aws-lambda/README.md b/data-prepper-plugins/aws-lambda/README.md index 89298f7715..099b390702 100644 --- a/data-prepper-plugins/aws-lambda/README.md +++ b/data-prepper-plugins/aws-lambda/README.md @@ -45,7 +45,11 @@ The integration tests for this plugin do not run as part of the Data Prepper bui The following command runs the integration tests: ``` -./gradlew :data-prepper-plugins:aws-lambda:integrationTest -Dtests.processor.lambda.region="us-east-1" -Dtests.processor.lambda.functionName="lambda_test_function" -Dtests.processor.lambda.sts_role_arn="arn:aws:iam::123456789012:role/dataprepper-role +./gradlew :data-prepper-plugins:aws-lambda:integrationTest \ +-Dtests.lambda.processor.region="us-east-1" \ +-Dtests.lambda.processor.functionName="test-lambda-processor" \ +-Dtests.lambda.processor.sts_role_arn="arn:aws:iam::<>:role/lambda-role" + ``` @@ -83,6 +87,10 @@ The integration tests for this plugin do not run as part of the Data Prepper bui The following command runs the integration tests: ``` -./gradlew :data-prepper-plugins:aws-lambda:integrationTest -Dtests.sink.lambda.region="us-east-1" -Dtests.sink.lambda.functionName="lambda_test_function" -Dtests.sink.lambda.sts_role_arn="arn:aws:iam::123456789012:role/dataprepper-role +./gradlew :data-prepper-plugins:aws-lambda:integrationTest \ +-Dtests.lambda.processor.region="us-east-1" \ +-Dtests.lambda.processor.functionName="test-lambda-processor" \ +-Dtests.lambda.processor.sts_role_arn="arn:aws:iam::<>>:role/lambda-role" + ``` diff --git a/data-prepper-plugins/aws-lambda/build.gradle b/data-prepper-plugins/aws-lambda/build.gradle index 4ad8209198..abdc2eed19 100644 --- a/data-prepper-plugins/aws-lambda/build.gradle +++ b/data-prepper-plugins/aws-lambda/build.gradle @@ -67,13 +67,13 @@ task integrationTest(type: Test) { systemProperty 'junit.jupiter.execution.parallel.enabled', 'true' systemProperty 'junit.jupiter.execution.parallel.mode.default', 'concurrent' - systemProperty 'tests.lambda.sink.region', System.getenv('TESTS_LAMBDA_SINK_REGION') - systemProperty 'tests.lambda.sink.functionName', System.getenv('TESTS_LAMBDA_SINK_FUNCTION_NAME') - systemProperty 'tests.lambda.sink.sts_role_arn', System.getenv('TESTS_LAMBDA_SINK_STS_ROLE_ARN') + systemProperty 'tests.lambda.sink.region', System.getProperty('tests.lambda.sink.region') + systemProperty 'tests.lambda.sink.functionName', System.getProperty('tests.lambda.sink.functionName') + systemProperty 'tests.lambda.sink.sts_role_arn', System.getProperty('tests.lambda.sink.sts_role_arn') - systemProperty 'tests.lambda.processor.region', System.getenv('TESTS_LAMBDA_PROCESSOR_REGION') - systemProperty 'tests.lambda.processor.functionName', System.getenv('TESTS_LAMBDA_PROCESSOR_FUNCTION_NAME') - systemProperty 'tests.lambda.processor.sts_role_arn', System.getenv('TESTS_LAMBDA_PROCESSOR_STS_ROLE_ARN') + systemProperty 'tests.lambda.processor.region', System.getProperty('tests.lambda.processor.region') + systemProperty 'tests.lambda.processor.functionName', System.getProperty('tests.lambda.processor.functionName') + systemProperty 'tests.lambda.processor.sts_role_arn', System.getProperty('tests.lambda.processor.sts_role_arn') filter { includeTestsMatching '*IT' diff --git a/data-prepper-plugins/aws-lambda/src/integrationTest/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorIT.java b/data-prepper-plugins/aws-lambda/src/integrationTest/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorIT.java index 94b0be3c30..1fe05cf3cd 100644 --- a/data-prepper-plugins/aws-lambda/src/integrationTest/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorIT.java +++ b/data-prepper-plugins/aws-lambda/src/integrationTest/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorIT.java @@ -5,6 +5,7 @@ package org.opensearch.dataprepper.plugins.lambda.processor; +import org.junit.jupiter.params.provider.EnumSource; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.spy; import org.mockito.junit.jupiter.MockitoSettings; @@ -102,12 +103,13 @@ private LambdaProcessor createObjectUnderTest(LambdaProcessorConfig processorCon @BeforeEach public void setup() { -// lambdaRegion = System.getProperty("tests.lambda.processor.region"); -// functionName = System.getProperty("tests.lambda.processor.functionName"); -// role = System.getProperty("tests.lambda.processor.sts_role_arn"); - lambdaRegion="us-east-1"; - functionName="test-lambda-processor"; - role="arn:aws:iam::176893235612:role/osis-lambda-role"; + lambdaRegion = System.getProperty("tests.lambda.processor.region"); + functionName = System.getProperty("tests.lambda.processor.functionName"); + role = System.getProperty("tests.lambda.processor.sts_role_arn"); + +// lambdaRegion = "us-east-1"; +// functionName = "test-lambda-processor"; +// role = "arn:aws:iam::176893235612:role/osis-lambda-role"; pluginMetrics = mock(PluginMetrics.class); when(pluginMetrics.gauge(any(), any(AtomicLong.class))).thenReturn(new AtomicLong()); @@ -179,7 +181,6 @@ public void testRequestResponseWithMatchingEventsStrictMode(int numRecords) { @ParameterizedTest - //@ValueSource(ints = {2, 5, 10, 100, 1000}) @ValueSource(ints = {1000}) public void testRequestResponseWithMatchingEventsAggregateMode(int numRecords) { when(invocationType.getAwsLambdaValue()).thenReturn(InvocationType.REQUEST_RESPONSE.getAwsLambdaValue()); @@ -222,24 +223,17 @@ public void testRequestResponse_WithMatchingEvents_StrictMode_WithMultipleThread executorService.shutdown(); } - @ParameterizedTest - @ValueSource(strings = {"RequestResponse", "Event"}) - public void testDifferentInvocationTypes(String invocationType) { - when(this.invocationType.getAwsLambdaValue()).thenReturn(invocationType); - when(lambdaProcessorConfig.getResponseEventsMatch()).thenReturn(true); - - lambdaProcessor = createObjectUnderTest(lambdaProcessorConfig); + @Test + public void testEventInvocationType() { + when(lambdaProcessorConfig.getInvocationType()).thenReturn(InvocationType.EVENT); List> records = createRecords(10); + lambdaProcessor = createObjectUnderTest(lambdaProcessorConfig); + + // For "Event" invocation type + when(lambdaProcessorConfig.getResponseEventsMatch()).thenReturn(false); Collection> results = lambdaProcessor.doExecute(records); - - if (invocationType.equals("RequestResponse")) { - assertThat(results.size(), equalTo(10)); - validateStrictModeResults(results); - } else { - // For "Event" invocation type - assertThat(results.size(), equalTo(0)); - } + assertThat(results.size(), equalTo(0)); // Changed from 0 to 10 } @Test diff --git a/data-prepper-plugins/aws-lambda/src/integrationTest/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorServiceIT.java b/data-prepper-plugins/aws-lambda/src/integrationTest/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorServiceIT.java deleted file mode 100644 index 978861afe4..0000000000 --- a/data-prepper-plugins/aws-lambda/src/integrationTest/java/org/opensearch/dataprepper/plugins/lambda/processor/LambdaProcessorServiceIT.java +++ /dev/null @@ -1,165 +0,0 @@ -//package org.opensearch.dataprepper.plugins.lambda.processor; -// -//import com.fasterxml.jackson.core.JsonProcessingException; -//import com.fasterxml.jackson.databind.ObjectMapper; -//import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; -//import com.fasterxml.jackson.dataformat.yaml.YAMLGenerator; -//import io.micrometer.core.instrument.Counter; -//import static org.junit.jupiter.api.Assertions.assertEquals; -//import org.junit.jupiter.api.BeforeEach; -//import org.junit.jupiter.api.extension.ExtendWith; -//import org.junit.jupiter.params.ParameterizedTest; -//import org.junit.jupiter.params.provider.ValueSource; -//import org.mockito.Mock; -//import static org.mockito.Mockito.when; -//import org.mockito.MockitoAnnotations; -//import org.mockito.junit.jupiter.MockitoExtension; -//import org.opensearch.dataprepper.aws.api.AwsCredentialsSupplier; -//import org.opensearch.dataprepper.expression.ExpressionEvaluator; -//import org.opensearch.dataprepper.metrics.PluginMetrics; -//import org.opensearch.dataprepper.model.configuration.PluginSetting; -//import org.opensearch.dataprepper.model.event.Event; -//import org.opensearch.dataprepper.model.event.JacksonEvent; -//import org.opensearch.dataprepper.model.log.JacksonLog; -//import org.opensearch.dataprepper.model.plugin.PluginFactory; -//import org.opensearch.dataprepper.model.record.Record; -//import org.opensearch.dataprepper.model.types.ByteCount; -//import org.opensearch.dataprepper.plugins.lambda.common.accumlator.BufferFactory; -//import org.opensearch.dataprepper.plugins.lambda.common.accumlator.InMemoryBufferFactory; -//import org.opensearch.dataprepper.plugins.lambda.common.config.AwsAuthenticationOptions; -//import org.opensearch.dataprepper.plugins.lambda.common.config.BatchOptions; -//import org.opensearch.dataprepper.plugins.lambda.common.config.InvocationType; -//import org.opensearch.dataprepper.plugins.lambda.common.config.ThresholdOptions; -//import software.amazon.awssdk.regions.Region; -//import software.amazon.awssdk.services.lambda.LambdaAsyncClient; -// -//import java.time.Duration; -//import java.util.ArrayList; -//import java.util.Collection; -//import java.util.HashMap; -//import java.util.List; -// -//@ExtendWith(MockitoExtension.class) -//public class LambdaProcessorServiceIT { -// -// private LambdaAsyncClient lambdaAsyncClient; -// private String functionName; -// private String lambdaRegion; -// private String role; -// private BufferFactory bufferFactory; -// @Mock -// private LambdaProcessorConfig lambdaProcessorConfig; -// @Mock -// private BatchOptions batchOptions; -// @Mock -// private ThresholdOptions thresholdOptions; -// @Mock -// private AwsAuthenticationOptions awsAuthenticationOptions; -// @Mock -// private AwsCredentialsSupplier awsCredentialsSupplier; -// @Mock -// private PluginMetrics pluginMetrics; -// @Mock -// private PluginFactory pluginFactory; -// @Mock -// private PluginSetting pluginSetting; -// @Mock -// private Counter numberOfRecordsSuccessCounter; -// @Mock -// private Counter numberOfRecordsFailedCounter; -// @Mock -// private ExpressionEvaluator expressionEvaluator; -// -// private final ObjectMapper objectMapper = new ObjectMapper(new YAMLFactory().enable(YAMLGenerator.Feature.USE_PLATFORM_LINE_BREAKS)); -// -// -// @BeforeEach -// public void setUp() throws Exception { -// MockitoAnnotations.openMocks(this); -// lambdaRegion = System.getProperty("tests.lambda.processor.region"); -// functionName = System.getProperty("tests.lambda.processor.functionName"); -// role = System.getProperty("tests.lambda.processor.sts_role_arn"); -// -// final Region region = Region.of(lambdaRegion); -// -// lambdaAsyncClient = LambdaAsyncClient.builder() -// .region(Region.of(lambdaRegion)) -// .build(); -// -// bufferFactory = new InMemoryBufferFactory(); -// -// when(pluginMetrics.counter(LambdaProcessor.NUMBER_OF_RECORDS_FLUSHED_TO_LAMBDA_SUCCESS)). -// thenReturn(numberOfRecordsSuccessCounter); -// when(pluginMetrics.counter(LambdaProcessor.NUMBER_OF_RECORDS_FLUSHED_TO_LAMBDA_FAILED)). -// thenReturn(numberOfRecordsFailedCounter); -// } -// -// -// private static Record createRecord() { -// final JacksonEvent event = JacksonLog.builder().withData("[{\"name\":\"test\"}]").build(); -// return new Record<>(event); -// } -// -// public LambdaProcessor createObjectUnderTest(final String config) throws JsonProcessingException { -// -// final LambdaProcessorConfig lambdaProcessorConfig = objectMapper.readValue(config, LambdaProcessorConfig.class); -// return new LambdaProcessor(pluginFactory,pluginMetrics,lambdaProcessorConfig,awsCredentialsSupplier,expressionEvaluator); -// } -// -// public LambdaProcessor createObjectUnderTest(LambdaProcessorConfig lambdaSinkConfig) throws JsonProcessingException { -// return new LambdaProcessor(pluginFactory,pluginMetrics,lambdaSinkConfig,awsCredentialsSupplier,expressionEvaluator); -// } -// -// -// private static Collection> generateRecords(int numberOfRecords) { -// List> recordList = new ArrayList<>(); -// -// for (int rows = 1; rows <= numberOfRecords; rows++) { -// HashMap eventData = new HashMap<>(); -// eventData.put("name", "Person" + rows); -// eventData.put("age", Integer.toString(rows)); -// -// Record eventRecord = new Record<>(JacksonEvent.builder().withData(eventData).withEventType("event").build()); -// recordList.add(eventRecord); -// } -// return recordList; -// } -// -// @ParameterizedTest -// @ValueSource(ints = {1,3}) -// void verify_records_to_lambda_success(final int recordCount) throws Exception { -// -// when(lambdaProcessorConfig.getFunctionName()).thenReturn(functionName); -// when(lambdaProcessorConfig.getMaxConnectionRetries()).thenReturn(3); -// when(lambdaProcessorConfig.getInvocationType()).thenReturn(InvocationType.REQUEST_RESPONSE); -// -// LambdaProcessor objectUnderTest = createObjectUnderTest(lambdaProcessorConfig); -// -// Collection> recordsData = generateRecords(recordCount); -// List> recordsResult = (List>) objectUnderTest.doExecute(recordsData); -// Thread.sleep(Duration.ofSeconds(10).toMillis()); -// -// assertEquals(recordsResult.size(),recordCount); -// } -// -// @ParameterizedTest -// @ValueSource(ints = {1,3}) -// void verify_records_with_batching_to_lambda(final int recordCount) throws JsonProcessingException, InterruptedException { -// -// when(lambdaProcessorConfig.getFunctionName()).thenReturn(functionName); -// when(lambdaProcessorConfig.getMaxConnectionRetries()).thenReturn(3); -// when(lambdaProcessorConfig.getInvocationType()).thenReturn(InvocationType.REQUEST_RESPONSE); -// when(thresholdOptions.getEventCount()).thenReturn(1); -// when(thresholdOptions.getMaximumSize()).thenReturn(ByteCount.parse("2mb")); -// when(thresholdOptions.getEventCollectTimeOut()).thenReturn(Duration.parse("PT10s")); -// when(batchOptions.getKeyName()).thenReturn("lambda_batch_key"); -// when(batchOptions.getThresholdOptions()).thenReturn(thresholdOptions); -// when(lambdaProcessorConfig.getBatchOptions()).thenReturn(batchOptions); -// -// LambdaProcessor objectUnderTest = createObjectUnderTest(lambdaProcessorConfig); -// Collection> records = generateRecords(recordCount); -// Collection> recordsResult = objectUnderTest.doExecute(records); -// Thread.sleep(Duration.ofSeconds(10).toMillis()); -// assertEquals(recordsResult.size(),recordCount); -// } -//} diff --git a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/ResponseCardinality.java b/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/ResponseCardinality.java deleted file mode 100644 index 1bcff2dcd7..0000000000 --- a/data-prepper-plugins/aws-lambda/src/main/java/org/opensearch/dataprepper/plugins/lambda/processor/ResponseCardinality.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.dataprepper.plugins.lambda.processor; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; - -import java.util.HashMap; -import java.util.Map; - -@Deprecated -public enum ResponseCardinality { - STRICT("strict"), - AGGREGATE("aggregate"); - - private final String value; - - ResponseCardinality(String value) { - this.value = value; - } - - @JsonValue - public String getValue() { - return value; - } - - private static final Map RESPONSE_CARDINALITY_MAP = new HashMap<>(); - - static { - for (ResponseCardinality type : ResponseCardinality.values()) { - RESPONSE_CARDINALITY_MAP.put(type.getValue(), type); - } - } - - @JsonCreator - public static ResponseCardinality fromString(String value) { - return RESPONSE_CARDINALITY_MAP.get(value); - } -} - diff --git a/data-prepper-plugins/aws-lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/processor/ResponseCardinalityTest.java b/data-prepper-plugins/aws-lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/processor/ResponseCardinalityTest.java deleted file mode 100644 index 0d3a58ca0f..0000000000 --- a/data-prepper-plugins/aws-lambda/src/test/java/org/opensearch/dataprepper/plugins/lambda/processor/ResponseCardinalityTest.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.dataprepper.plugins.lambda.processor; - -import org.junit.jupiter.api.Test; -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class ResponseCardinalityTest { - - @Test - public void testFromStringWithValidValue() { - assertEquals(ResponseCardinality.STRICT, ResponseCardinality.fromString("strict")); - assertEquals(ResponseCardinality.AGGREGATE, ResponseCardinality.fromString("aggregate")); - } - - @Test - public void testFromStringWithInvalidValue() { - assertEquals(null, ResponseCardinality.fromString("invalid-value")); - } - - @Test - public void testGetValue() { - assertEquals("strict", ResponseCardinality.STRICT.getValue()); - assertEquals("aggregate", ResponseCardinality.AGGREGATE.getValue()); - } -} -