E2E: aws secrets tests (opensearch-project#3654)

* E2E: basicLogWithAwsSecretsEndToEndTest Signed-off-by: George Chen <[email protected]>
kkondaka · Nov 17, 2023 · c453dd0 · c453dd0
1 parent 1c28285
commit c453dd0
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 0 deletions.
diff --git a/e2e-test/log/README.md b/e2e-test/log/README.md
@@ -13,6 +13,19 @@ or from project root directory
 ./gradlew :e2e-test:log:basicLogEndToEndTest
 ```
 
+## Basic Grok Ingestion Pipeline with AWS secrets End-to-end test
+
+This test requires AWS secrets `opensearch-sink-basic-credentials` with the basic credentials for opensearch sink to be created in `us-east-1` within the test AWS account. The IAM credentials needs to be passed in through specifying [`AWS_PROFILE` environment variable](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html).
+
+Run from current directory
+```
+AWS_PROFILE=<your-aws-profile-name> ./gradlew :basicLogWithAwsSecretsEndToEndTest
+```
+or from project root directory
+```
+AWS_PROFILE=<your-aws-profile-name> ./gradlew :e2e-test:log:basicLogWithAwsSecretsEndToEndTest
+```
+
 ## Parallel Grok and string substitute End-to-End test
 
 Run from current directory

diff --git a/e2e-test/log/build.gradle b/e2e-test/log/build.gradle
@@ -44,6 +44,14 @@ List<LogTestConfiguration> logTestConfigurations = [
                 'basic-grok-e2e-pipeline.yml',
                 'data_prepper.yml'
         ),
+        new LogTestConfiguration(
+                'basicLogWithAwsSecretsEndToEndTest',
+                'Runs the basic grok with AWS secrets end-to-end test.',
+                'org.opensearch.dataprepper.integration.log.EndToEndBasicLogTest.testPipelineEndToEnd*',
+                'data-prepper-basic-log-with-aws-secrets',
+                'basic-grok-e2e-pipeline-with-aws-secrets.yml',
+                'data_prepper.yml'
+        ),
         new LogTestConfiguration(
                 'parallelGrokStringSubstituteTest',
                 'Runs the parallel grok and string substitute end-to-end test.',
@@ -63,9 +71,13 @@ logTestConfigurations.each { testConfiguration ->
         exposePorts('tcp', [2021, 4900])
         hostConfig.portBindings = ['2021:2021', '4900:4900']
         hostConfig.binds = [
+                "${System.getProperty('user.home')}/.aws" : '/root/.aws',
                 (project.file("src/integrationTest/resources/${testConfiguration.pipelineConfiguration}").toString())   : '/usr/share/data-prepper/pipelines/log-pipeline.yaml',
                 (project.file("src/integrationTest/resources/${testConfiguration.dataPrepperConfiguration}").toString()): '/usr/share/data-prepper/config/data-prepper-config.yaml'
         ]
+        envVars = [
+                'AWS_PROFILE' : System.getenv('AWS_PROFILE') ?: ''
+        ]
         hostConfig.network = createDataPrepperNetwork.getNetworkName()
         targetImageId dataPrepperDockerImage.imageId
     }

diff --git a/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml b/e2e-test/log/src/integrationTest/resources/basic-grok-e2e-pipeline-with-aws-secrets.yml
@@ -0,0 +1,20 @@
+pipeline_configurations:
+  aws:
+    secrets:
+      opensearch-sink:
+        secret_id: "opensearch-sink-basic-credentials"
+grok-pipeline:
+  source:
+    http:
+      path: "/${pipelineName}/logs"
+  processor:
+    - grok:
+        match:
+          log: [ "%{COMMONAPACHELOG}" ]
+  sink:
+    - opensearch:
+        hosts: [ "https://node-0.example.com:9200" ]
+        username: "${{aws_secrets:opensearch-sink:username}}"
+        password: "${{aws_secrets:opensearch-sink:password}}"
+        index: "test-grok-index"
+        flush_timeout: 5000