Merge branch 'opensearch-project:main' into migrate-plugins-off-of-pl…

…ugin-setting-for-configuration-classes

.github/CODEOWNERS

@@ -1,2 +1,2 @@
 # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams
-*   @sb2k16 @chenqi0805 @engechas @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh
+*   @sb2k16 @chenqi0805 @engechas @san81 @graytaylor0 @dinujoh @kkondaka @KarstenSchnitter @dlvenable @oeyh

.github/workflows/gradle.yml

@@ -28,7 +28,7 @@ jobs:
       run: ./gradlew --parallel --max-workers 2 build
     - name: Upload Unit Test Results
       if: always()
-      uses: actions/upload-artifact@v3
+      uses: actions/upload-artifact@v4
       with:
         name: data-prepper-test-results-java-${{ matrix.java }}
         path: '**/test-results/**/*.xml'
@@ -45,7 +45,7 @@ jobs:
 
     steps:
       - name: Download Artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: test-results
 

.github/workflows/kafka-plugin-integration-tests.yml

@@ -70,7 +70,7 @@ jobs:
 
       - name: Upload Unit Test Results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: data-prepper-kafka-integration-tests-kafka-${{ matrix.kafka }}-java-${{ matrix.java }}
           path: '**/test-results/**/*.xml'
@@ -83,7 +83,7 @@ jobs:
 
     steps:
       - name: Download Artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: test-results
 

.github/workflows/kinesis-source-integration-tests.yml

@@ -59,7 +59,7 @@ jobs:
             -Dtests.kinesis.source.aws.region=us-east-1 --tests KinesisSourceIT
       - name: Upload Unit Test Results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: data-prepper-kinesis-source-integration-tests-java-${{ matrix.java }}
           path: '**/test-results/**/*.xml'
@@ -72,7 +72,7 @@ jobs:
 
     steps:
       - name: Download Artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: test-results
 

.github/workflows/opensearch-sink-opendistro-integration-tests.yml

@@ -43,7 +43,7 @@ jobs:
           ./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin -Dtests.opensearch.bundle=true -Dtests.opensearch.version=opendistro:${{ matrix.opendistro }}
       - name: Upload Unit Test Results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: data-prepper-opensearch-integration-tests-opendistro-${{ matrix.opendistro }}-java-${{ matrix.java }}
           path: '**/test-results/**/*.xml'
@@ -56,7 +56,7 @@ jobs:
 
     steps:
       - name: Download Artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: test-results
 

.github/workflows/opensearch-sink-opensearch-integration-tests.yml

@@ -43,7 +43,7 @@ jobs:
           ./gradlew :data-prepper-plugins:opensearch:integrationTest -Dtests.opensearch.host=localhost:9200 -Dtests.opensearch.user=admin -Dtests.opensearch.password=admin -Dtests.opensearch.bundle=true -Dtests.opensearch.version=opensearch:${{ matrix.opensearch }}
       - name: Upload Unit Test Results
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: data-prepper-opensearch-integration-tests-opensearch-${{ matrix.opensearch }}-java-${{ matrix.java }}
           path: '**/test-results/**/*.xml'
@@ -56,7 +56,7 @@ jobs:
 
     steps:
       - name: Download Artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: test-results
 

CONTRIBUTING.md

@@ -106,7 +106,7 @@ If you are modifying existing files with license headers, or including new files
  * this file be licensed under the Apache-2.0 license or a
  * compatible open source license.
  *
-*/
+ */
 ```
 
 ### Shell, Python

MAINTAINERS.md

@@ -9,6 +9,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Souvik Bose          | [sb2k16](https://github.com/sb2k16)               | Amazon      |
 | Qi Chen              | [chenqi0805](https://github.com/chenqi0805)               | Amazon      |
 | Chase Engelbrecht    | [engechas](https://github.com/engechas)                   | Amazon      |
+| Santhosh Gandhe      | [san81](https://github.com/san81)             | Amazon      |
 | Taylor Gray          | [graytaylor0](https://github.com/graytaylor0)             | Amazon      |
 | Dinu John            | [dinujoh](https://github.com/dinujoh)                     | Amazon      |
 | Krishna Kondaka      | [kkondaka](https://github.com/kkondaka)                   | Amazon      |

build.gradle

@@ -350,7 +350,8 @@ coreProjects.each { coreProject ->
     def assembleTasks = collectTasksRecursively(coreProject, 'assemble')
     def publishTasks = collectTasksRecursively(coreProject, 'publish')
 
-    // Add these tasks as dependencies of the release task
-    release.dependsOn assembleTasks
-    release.dependsOn publishTasks
+    // Explicitly declare release task for better gradle compatibility
+    def releaseTask = tasks.named('release').get()
+    releaseTask.dependsOn assembleTasks
+    releaseTask.dependsOn publishTasks
 }

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/Experimental.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.dataprepper.model.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marks a Data Prepper plugin as experimental.
+ * <p>
+ * Experimental plugins do not have the same compatibility guarantees as other plugins and may be unstable.
+ * They may have breaking changes between minor versions and may even be removed.
+ * <p>
+ * Data Prepper administrators must enable experimental plugins in order to use them.
+ * Otherwise, they are not available to use with pipelines.
+ *
+ * @since 2.11
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE})
+public @interface Experimental {
+}

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/Event.java

@@ -110,6 +110,18 @@ public interface Event extends Serializable {
      */
     void clear();
 
+    /**
+     * Merges another Event into the current Event.
+     * The values from the other Event will overwrite the values in the current Event for all keys in the current Event.
+     * If the other Event has keys that are not in the current Event, they will be unmodified.
+     *
+     * @param other the other Event to merge into this Event
+     * @throws IllegalArgumentException if the input event is not compatible to merge.
+     * @throws UnsupportedOperationException if the current Event does not support merging.
+     * @since 2.11
+     */
+    void merge(Event other);
+
     /**
      * Generates a serialized Json string of the entire Event
      *

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/event/JacksonEvent.java

@@ -123,7 +123,6 @@ public static Event fromMessage(String message) {
     }
 
     private JsonNode getInitialJsonNode(final Object data) {
-
         if (data == null) {
             return mapper.valueToTree(new HashMap<>());
         } else if (data instanceof String) {
@@ -348,14 +347,30 @@ public void clear() {
         }
     }
 
+    @Override
+    public void merge(final Event other) {
+        if(!(other instanceof JacksonEvent))
+            throw new IllegalArgumentException("Unable to merge the Event. The input Event must be a JacksonEvent.");
+        final JacksonEvent otherJacksonEvent = (JacksonEvent) other;
+        if(!(otherJacksonEvent.jsonNode instanceof ObjectNode)) {
+            throw new IllegalArgumentException("Unable to merge the Event. The input Event must be a JacksonEvent with object data.");
+        }
+        final ObjectNode otherObjectNode = (ObjectNode) otherJacksonEvent.jsonNode;
+
+        if(!(jsonNode instanceof ObjectNode)) {
+            throw new UnsupportedOperationException("Unable to merge the Event. The current Event must have object data.");
+        }
+
+        ((ObjectNode) jsonNode).setAll(otherObjectNode);
+    }
+
     @Override
     public String toJsonString() {
         return jsonNode.toString();
     }
 
     @Override
     public String getAsJsonString(EventKey key) {
-
         JacksonEventKey jacksonEventKey = asJacksonEventKey(key);
         final JsonNode node = getNode(jacksonEventKey);
         if (node.isMissingNode()) {

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/FailedToUpdatePluginConfigValueException.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.dataprepper.model.plugin;
+
+/**
+ * Exception thrown when a secret could not be updated.
+ *
+ * @since 2.11
+ */
+public class FailedToUpdatePluginConfigValueException extends RuntimeException {
+
+    public FailedToUpdatePluginConfigValueException(final String message) {
+        super(message);
+    }
+
+    public FailedToUpdatePluginConfigValueException(final String message, Throwable e) {
+        super(message, e);
+    }
+}

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigValueTranslator.java

@@ -1,7 +1,48 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
 package org.opensearch.dataprepper.model.plugin;
 
+/**
+ * Interface for a Plugin configuration value translator.
+ * It translates a string expression that is describing a secret store Id and secret Key in to a secretValue
+ * extracted from corresponding secret store.
+ *
+ * @since 2.0
+ */
+
 public interface PluginConfigValueTranslator {
+    /**
+     * Translates a string expression that is describing a secret store Id and secret Key in to a secretValue
+     * extracted from corresponding secret store.
+     * Example expression:  ${{aws_secrets:secretId:secretKey}}
+     *
+     * @param value the string value to translate
+     * @return the translated object
+     */
     Object translate(final String value);
 
+    /**
+     * Returns the prefix for this translator.
+     *
+     * @return the prefix for this translator
+     */
     String getPrefix();
+
+    /**
+     * Translates a string expression that is describing a secret store Id and secret Key in to an instance
+     * of PluginConfigVariable with secretValue extracted from corresponding secret store. Additionally,
+     * this PluginConfigVariable helps with updating the secret value in the secret store, if required.
+     * Example expression:  ${{aws_secrets:secretId:secretKey}}
+     *
+     * @param value the string value to translate
+     * @return the translated object
+     */
+    PluginConfigVariable translateToPluginConfigVariable(final String value);
 }

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/plugin/PluginConfigVariable.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ */
+package org.opensearch.dataprepper.model.plugin;
+
+/**
+ * Interface for a Extension Plugin configuration variable.
+ * It gives access to the details of a defined extension variable.
+ *
+ * @since 2.11
+ */
+public interface PluginConfigVariable {
+
+    /**
+     * Returns the value of this variable.
+     *
+     * @return the value of this variable
+     */
+    Object getValue();
+
+    /**
+     * If this variable is updatable, this method helps to set a new value for this variable
+     *
+     * @param updatedValue the new value to set
+     */
+    void setValue(Object updatedValue);
+
+    /**
+     * Returns if the variable is updatable.
+     *
+     * @return true if this variable is updatable, false otherwise
+     */
+    boolean isUpdatable();
+}

data-prepper-api/src/test/java/org/opensearch/dataprepper/model/event/JacksonEventTest.java

@@ -446,6 +446,52 @@ public void testClear() {
         assertThat(event.toMap().size(), equalTo(0));
     }
 
+    @Test
+    void merge_with_non_JacksonEvent_throws() {
+        final Event otherEvent = mock(Event.class);
+        assertThrows(IllegalArgumentException.class, () -> event.merge(otherEvent));
+    }
+
+    @Test
+    void merge_with_array_JsonNode_throws() {
+        final JacksonEvent otherEvent = (JacksonEvent) event;
+        event = JacksonEvent.builder().withEventType(EventType.DOCUMENT.toString()).withData(List.of(UUID.randomUUID().toString())).build();
+        assertThrows(UnsupportedOperationException.class, () -> event.merge(otherEvent));
+    }
+
+    @Test
+    void merge_with_array_JsonNode_in_other_throws() {
+        Event otherEvent = JacksonEvent.builder().withEventType(EventType.DOCUMENT.toString()).withData(List.of(UUID.randomUUID().toString())).build();
+        assertThrows(IllegalArgumentException.class, () -> event.merge(otherEvent));
+    }
+
+    @Test
+    void merge_sets_all_values() {
+        final String jsonString = "{\"a\": \"alpha\", \"info\": {\"ids\": {\"id\":\"idx\"}}}";
+        event.put("b", "original");
+        Event otherEvent = JacksonEvent.builder().withEventType(EventType.DOCUMENT.toString()).withData(jsonString).build();
+        event.merge(otherEvent);
+
+        assertThat(event.get("b", Object.class), equalTo("original"));
+        assertThat(event.get("a", Object.class), equalTo("alpha"));
+        assertThat(event.containsKey("info"), equalTo(true));
+        assertThat(event.get("info/ids/id", String.class), equalTo("idx"));
+    }
+
+    @Test
+    void merge_overrides_existing_values() {
+        final String jsonString = "{\"a\": \"alpha\", \"info\": {\"ids\": {\"id\":\"idx\"}}}";
+        event.put("a", "original");
+        event.put("b", "original");
+        Event otherEvent = JacksonEvent.builder().withEventType(EventType.DOCUMENT.toString()).withData(jsonString).build();
+        event.merge(otherEvent);
+
+        assertThat(event.get("b", Object.class), equalTo("original"));
+        assertThat(event.get("a", Object.class), equalTo("alpha"));
+        assertThat(event.containsKey("info"), equalTo(true));
+        assertThat(event.get("info/ids/id", String.class), equalTo("idx"));
+    }
+
     @ParameterizedTest
     @ValueSource(strings = {"/", "foo", "/foo", "/foo/bar", "foo/bar", "foo/bar/", "/foo/bar/leaf/key"})
     public void testDelete_withNonexistentKey(final String key) {