diff --git a/e2e-test/log/README.md b/e2e-test/log/README.md index 55a7c080da..2a5136bf7b 100644 --- a/e2e-test/log/README.md +++ b/e2e-test/log/README.md @@ -13,6 +13,19 @@ or from project root directory ./gradlew :e2e-test:log:basicLogEndToEndTest ``` +## Basic Grok Ingestion Pipeline with AWS secrets End-to-end test + +This test requires AWS secrets `opensearch-sink-basic-credentials` with the basic credentials for opensearch sink to be created in `us-east-1` within the test AWS account. The IAM credentials needs to be passed in through specifying [`AWS_PROFILE` environment variable](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html). + +Run from current directory +``` +AWS_PROFILE= ./gradlew :basicLogWithAwsSecretsEndToEndTest +``` +or from project root directory +``` +AWS_PROFILE= ./gradlew :e2e-test:log:basicLogWithAwsSecretsEndToEndTest +``` + ## Parallel Grok and string substitute End-to-End test Run from current directory diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle index 8198e41bee..60986ef9b3 100644 --- a/e2e-test/log/build.gradle +++ b/e2e-test/log/build.gradle @@ -44,6 +44,14 @@ List logTestConfigurations = [ 'basic-grok-e2e-pipeline.yml', 'data_prepper.yml' ), + new LogTestConfiguration( + 'basicLogWithAwsSecretsEndToEndTest', + 'Runs the basic grok with AWS secrets end-to-end test.', + 'org.opensearch.dataprepper.integration.log.EndToEndBasicLogTest.testPipelineEndToEnd*', + 'data-prepper-basic-log-with-aws-secrets', + 'basic-grok-e2e-pipeline-with-aws-secrets.yml', + 'data_prepper.yml' + ), new LogTestConfiguration( 'parallelGrokStringSubstituteTest', 'Runs the parallel grok and string substitute end-to-end test.', @@ -63,9 +71,13 @@ logTestConfigurations.each { testConfiguration -> exposePorts('tcp', [2021, 4900]) hostConfig.portBindings = ['2021:2021', '4900:4900'] hostConfig.binds = [ + "${System.getProperty('user.home')}/.aws" : '/root/.aws', (project.file("src/integrationTest/resources/${testConfiguration.pipelineConfiguration}").toString()) : '/usr/share/data-prepper/pipelines/log-pipeline.yaml', (project.file("src/integrationTest/resources/${testConfiguration.dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml' ] + envVars = [ + 'AWS_PROFILE' : System.getenv('AWS_PROFILE') ?: '' + ] hostConfig.network = createDataPrepperNetwork.getNetworkName() targetImageId dataPrepperDockerImage.imageId } diff --git a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml new file mode 100644 index 0000000000..34be38f12b --- /dev/null +++ b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml @@ -0,0 +1,20 @@ +pipeline_configurations: + aws: + secrets: + opensearch-sink: + secret_id: "opensearch-sink-basic-credentials" +grok-pipeline: + source: + http: + path: "/${pipelineName}/logs" + processor: + - grok: + match: + log: [ "%{COMMONAPACHELOG}" ] + sink: + - opensearch: + hosts: [ "https://node-0.example.com:9200" ] + username: "${{aws_secrets:opensearch-sink:username}}" + password: "${{aws_secrets:opensearch-sink:password}}" + index: "test-grok-index" + flush_timeout: 5000 \ No newline at end of file