Introduces the experimental plugin feature

data-prepper-api/src/main/java/org/opensearch/dataprepper/model/annotations/Experimental.java

@@ -0,0 +1,29 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.model.annotations;
+
+import java.lang.annotation.Documented;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Marks a Data Prepper plugin as experimental.
+ * <p>
+ * Experimental plugins do not have the same compatibility guarantees as other plugins and may be unstable.
+ * They may have breaking changes between minor versions and may even be removed.
+ * <p>
+ * Data Prepper administrators must enable experimental plugins in order to use them.
+ * Otherwise, they are not available to use with pipelines.
+ *
+ * @since 2.11
+ */
+@Documented
+@Retention(RetentionPolicy.RUNTIME)
+@Target({ElementType.TYPE})
+public @interface Experimental {
+}

data-prepper-core/src/integrationTest/java/org/opensearch/dataprepper/plugin/DefaultPluginFactoryIT.java

@@ -14,6 +14,7 @@
 import org.opensearch.dataprepper.core.event.EventFactoryApplicationContextMarker;
 import org.opensearch.dataprepper.core.validation.LoggingPluginErrorsHandler;
 import org.opensearch.dataprepper.core.validation.PluginErrorCollector;
+import org.opensearch.dataprepper.model.plugin.NoPluginFoundException;
 import org.opensearch.dataprepper.plugins.configtest.TestComponentWithConfigInject;
 import org.opensearch.dataprepper.plugins.configtest.TestDISourceWithConfig;
 import org.opensearch.dataprepper.validation.PluginErrorsHandler;
@@ -27,6 +28,7 @@
 import org.opensearch.dataprepper.plugins.test.TestPlugin;
 import org.springframework.context.annotation.AnnotationConfigApplicationContext;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.UUID;
@@ -38,6 +40,7 @@
 import static org.junit.jupiter.api.Assertions.assertInstanceOf;
 import static org.junit.jupiter.api.Assertions.assertNotNull;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.when;
 
 /**
  * Integration test of the plugin framework. These tests should not mock any portion
@@ -49,6 +52,8 @@ class DefaultPluginFactoryIT {
     private PipelinesDataFlowModel pipelinesDataFlowModel;
     @Mock
     private ExtensionsConfiguration extensionsConfiguration;
+    @Mock
+    private ExperimentalConfigurationContainer experimentalConfigurationContainer;
     private String pluginName;
     private String objectPluginName;
     private String pipelineName;
@@ -67,6 +72,8 @@ private DefaultPluginFactory createObjectUnderTest() {
         final AnnotationConfigApplicationContext coreContext = new AnnotationConfigApplicationContext();
         coreContext.setParent(publicContext);
 
+        when(experimentalConfigurationContainer.getExperimental()).thenReturn(ExperimentalConfiguration.defaultConfiguration());
+
         coreContext.scan(EventFactoryApplicationContextMarker.class.getPackage().getName());
         coreContext.scan(DefaultAcknowledgementSetManager.class.getPackage().getName());
         coreContext.scan(DefaultPluginFactory.class.getPackage().getName());
@@ -75,6 +82,7 @@ private DefaultPluginFactory createObjectUnderTest() {
         coreContext.registerBean(PluginErrorsHandler.class, LoggingPluginErrorsHandler::new);
         coreContext.registerBean(ExtensionsConfiguration.class, () -> extensionsConfiguration);
         coreContext.registerBean(PipelinesDataFlowModel.class, () -> pipelinesDataFlowModel);
+        coreContext.registerBean(ExperimentalConfigurationContainer.class, () -> experimentalConfigurationContainer);
         coreContext.refresh();
 
         return coreContext.getBean(DefaultPluginFactory.class);
@@ -188,6 +196,20 @@ void loadPlugin_should_throw_when_a_plugin_configuration_is_invalid() {
         assertThat(actualException.getMessage(), equalTo("Plugin test_plugin in pipeline " + pipelineName + " is configured incorrectly: requiredString must not be null"));
     }
 
+    @Test
+    void loadPlugin_should_throw_when_a_plugin_is_experimental_by_default() {
+        pluginName = "test_experimental_plugin";
+        final PluginSetting pluginSetting = createPluginSettings(Collections.emptyMap());
+
+        final DefaultPluginFactory objectUnderTest = createObjectUnderTest();
+
+        final NoPluginFoundException actualException = assertThrows(NoPluginFoundException.class,
+                () -> objectUnderTest.loadPlugin(TestPluggableInterface.class, pluginSetting));
+
+        assertThat(actualException.getMessage(), notNullValue());
+        assertThat(actualException.getMessage(), equalTo("Unable to create experimental plugin test_experimental_plugin. You must enable experimental plugins in data-prepper-config.yaml in order to use them."));
+    }
+
     private PluginSetting createPluginSettings(final Map<String, Object> pluginSettingMap) {
         final PluginSetting pluginSetting = new PluginSetting(pluginName, pluginSettingMap);
         pluginSetting.setPipelineName(pipelineName);

data-prepper-core/src/main/java/org/opensearch/dataprepper/core/parser/model/DataPrepperConfiguration.java

@@ -18,6 +18,8 @@
 import org.opensearch.dataprepper.core.pipeline.PipelineShutdownOption;
 import org.opensearch.dataprepper.model.configuration.PipelineExtensions;
 import org.opensearch.dataprepper.model.configuration.PluginModel;
+import org.opensearch.dataprepper.plugin.ExperimentalConfiguration;
+import org.opensearch.dataprepper.plugin.ExperimentalConfigurationContainer;
 import org.opensearch.dataprepper.plugin.ExtensionsConfiguration;
 
 import java.time.Duration;
@@ -31,7 +33,7 @@
 /**
  * Class to hold configuration for DataPrepper, including server port and Log4j settings
  */
-public class DataPrepperConfiguration implements ExtensionsConfiguration, EventConfigurationContainer {
+public class DataPrepperConfiguration implements ExtensionsConfiguration, EventConfigurationContainer, ExperimentalConfigurationContainer {
     static final Duration DEFAULT_SHUTDOWN_DURATION = Duration.ofSeconds(30L);
 
     private static final String DEFAULT_SOURCE_COORDINATION_STORE = "in_memory";
@@ -55,6 +57,7 @@ public class DataPrepperConfiguration implements ExtensionsConfiguration, EventC
     private PeerForwarderConfiguration peerForwarderConfiguration;
     private Duration processorShutdownTimeout;
     private Duration sinkShutdownTimeout;
+    private ExperimentalConfiguration experimental;
     private PipelineExtensions pipelineExtensions;
 
     public static final DataPrepperConfiguration DEFAULT_CONFIG = new DataPrepperConfiguration();
@@ -96,6 +99,7 @@ public DataPrepperConfiguration(
             @JsonProperty("source_coordination") final SourceCoordinationConfig sourceCoordinationConfig,
             @JsonProperty("pipeline_shutdown") final PipelineShutdownOption pipelineShutdown,
             @JsonProperty("event") final EventConfiguration eventConfiguration,
+            @JsonProperty("experimental") final ExperimentalConfiguration experimental,
             @JsonProperty("extensions")
             @JsonInclude(JsonInclude.Include.NON_NULL)
             @JsonSetter(nulls = Nulls.SKIP)
@@ -126,6 +130,8 @@ public DataPrepperConfiguration(
         if (this.sinkShutdownTimeout.isNegative()) {
             throw new IllegalArgumentException("sinkShutdownTimeout must be non-negative.");
         }
+        this.experimental = experimental != null ? experimental : ExperimentalConfiguration.defaultConfiguration();
+
         this.pipelineExtensions = pipelineExtensions;
     }
 
@@ -239,4 +245,9 @@ public EventConfiguration getEventConfiguration() {
     public PipelineExtensions getPipelineExtensions() {
         return pipelineExtensions;
     }
+
+    @Override
+    public ExperimentalConfiguration getExperimental() {
+        return experimental;
+    }
 }

data-prepper-core/src/test/java/org/opensearch/dataprepper/core/parser/PipelineTransformerTests.java

@@ -140,6 +140,7 @@ void setUp() {
     @AfterEach
     void tearDown() {
         verify(dataPrepperConfiguration).getEventConfiguration();
+        verify(dataPrepperConfiguration).getExperimental();
         verifyNoMoreInteractions(dataPrepperConfiguration);
     }
 

data-prepper-core/src/test/java/org/opensearch/dataprepper/plugins/TestExperimentalPlugin.java

@@ -0,0 +1,15 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.plugins;
+
+import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
+import org.opensearch.dataprepper.model.annotations.Experimental;
+import org.opensearch.dataprepper.plugin.TestPluggableInterface;
+
+@DataPrepperPlugin(name = "test_experimental_plugin", pluginType = TestPluggableInterface.class)
+@Experimental
+public class TestExperimentalPlugin {
+}

data-prepper-plugin-framework/build.gradle

@@ -25,4 +25,5 @@ dependencies {
     implementation libs.reflections.core
     implementation 'com.fasterxml.jackson.core:jackson-databind'
     implementation 'org.apache.commons:commons-text:1.10.0'
+    testImplementation 'ch.qos.logback:logback-classic:1.5.16'
 }

data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefaultPluginFactory.java

@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.function.Consumer;
 import java.util.function.Function;
 
 /**
@@ -41,6 +42,7 @@ public class DefaultPluginFactory implements PluginFactory {
     private final PluginBeanFactoryProvider pluginBeanFactoryProvider;
     private final PluginConfigurationObservableFactory pluginConfigurationObservableFactory;
     private final ApplicationContextToTypedSuppliers applicationContextToTypedSuppliers;
+    private final List<Consumer<DefinedPlugin<?>>> definedPluginConsumers;
 
     @Inject
     DefaultPluginFactory(
@@ -49,8 +51,10 @@ public class DefaultPluginFactory implements PluginFactory {
             final PluginConfigurationConverter pluginConfigurationConverter,
             final PluginBeanFactoryProvider pluginBeanFactoryProvider,
             final PluginConfigurationObservableFactory pluginConfigurationObservableFactory,
-            final ApplicationContextToTypedSuppliers applicationContextToTypedSuppliers) {
+            final ApplicationContextToTypedSuppliers applicationContextToTypedSuppliers,
+            final List<Consumer<DefinedPlugin<?>>> definedPluginConsumers) {
         this.applicationContextToTypedSuppliers = applicationContextToTypedSuppliers;
+        this.definedPluginConsumers = definedPluginConsumers;
         Objects.requireNonNull(pluginProviderLoader);
         Objects.requireNonNull(pluginConfigurationObservableFactory);
         this.pluginCreator = Objects.requireNonNull(pluginCreator);
@@ -140,15 +144,13 @@ private <T> Class<? extends T> getPluginClass(final Class<T> baseClass, final St
                 .orElseThrow(() -> new NoPluginFoundException(
                         "Unable to find a plugin named '" + pluginName + "'. Please ensure that plugin is annotated with appropriate values."));
 
-        logDeprecatedPluginsNames(pluginClass, pluginName);
+        handleDefinedPlugins(pluginClass, pluginName);
         return pluginClass;
     }
 
-    private <T> void logDeprecatedPluginsNames(final Class<? extends T> pluginClass, final String pluginName) {
-        final String deprecatedName = pluginClass.getAnnotation(DataPrepperPlugin.class).deprecatedName();
-        final String name = pluginClass.getAnnotation(DataPrepperPlugin.class).name();
-        if (deprecatedName.equals(pluginName)) {
-            LOG.warn("Plugin name '{}' is deprecated and will be removed in the next major release. Consider using the updated plugin name '{}'.", deprecatedName, name);
-        }
+    private <T> void handleDefinedPlugins(final Class<? extends T> pluginClass, final String pluginName) {
+        final DefinedPlugin<? extends T> definedPlugin = new DefinedPlugin<>(pluginClass, pluginName);
+
+        definedPluginConsumers.forEach(definedPluginConsumer -> definedPluginConsumer.accept(definedPlugin));
     }
 }

data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DefinedPlugin.java

@@ -0,0 +1,26 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.plugin;
+
+import java.util.Objects;
+
+class DefinedPlugin<T> {
+    private final Class<? extends T> pluginClass;
+    private final String pluginName;
+
+    public DefinedPlugin(final Class<? extends T> pluginClass, final String pluginName) {
+        this.pluginClass = Objects.requireNonNull(pluginClass);
+        this.pluginName = Objects.requireNonNull(pluginName);
+    }
+
+    public Class<? extends T> getPluginClass() {
+        return pluginClass;
+    }
+
+    public String getPluginName() {
+        return pluginName;
+    }
+}

data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/DeprecatedPluginDetector.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.plugin;
+
+import org.opensearch.dataprepper.model.annotations.DataPrepperPlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Named;
+import java.util.function.Consumer;
+
+@Named
+class DeprecatedPluginDetector implements Consumer<DefinedPlugin<?>> {
+    private static final Logger LOG = LoggerFactory.getLogger(DeprecatedPluginDetector.class);
+
+    @Override
+    public void accept(final DefinedPlugin<?> definedPlugin) {
+        logDeprecatedPluginsNames(definedPlugin.getPluginClass(), definedPlugin.getPluginName());
+    }
+
+    private void logDeprecatedPluginsNames(final Class<?> pluginClass, final String pluginName) {
+        final String deprecatedName = pluginClass.getAnnotation(DataPrepperPlugin.class).deprecatedName();
+        final String name = pluginClass.getAnnotation(DataPrepperPlugin.class).name();
+        if (deprecatedName.equals(pluginName)) {
+            LOG.warn("Plugin name '{}' is deprecated and will be removed in the next major release. Consider using the updated plugin name '{}'.", deprecatedName, name);
+        }
+    }
+}

data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfiguration.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.plugin;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+
+/**
+ * Data Prepper configurations for experimental features.
+ *
+ * @since 2.11
+ */
+public class ExperimentalConfiguration {
+    @JsonProperty("enable_all")
+    private boolean enableAll = false;
+
+    public static ExperimentalConfiguration defaultConfiguration() {
+        return new ExperimentalConfiguration();
+    }
+
+    /**
+     * Gets whether all experimental features are enabled.
+     * @return true if all experimental features are enabled, false otherwise
+     * @since 2.11
+     */
+    public boolean isEnableAll() {
+        return enableAll;
+    }
+}

data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalConfigurationContainer.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.plugin;
+
+/**
+ * Interface to decouple how an experimental configuration is defined from
+ * usage of those configurations.
+ *
+ * @since 2.11
+ */
+public interface ExperimentalConfigurationContainer {
+    /**
+     * Gets the experimental configuration.
+     * @return the experimental configuration
+     * @since 2.11
+     */
+    ExperimentalConfiguration getExperimental();
+}

data-prepper-plugin-framework/src/main/java/org/opensearch/dataprepper/plugin/ExperimentalPluginValidator.java

@@ -0,0 +1,33 @@
+/*
+ * Copyright OpenSearch Contributors
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package org.opensearch.dataprepper.plugin;
+
+import org.opensearch.dataprepper.model.annotations.Experimental;
+import org.opensearch.dataprepper.model.plugin.NoPluginFoundException;
+
+import javax.inject.Named;
+import java.util.function.Consumer;
+
+@Named
+class ExperimentalPluginValidator implements Consumer<DefinedPlugin<?>> {
+    private final ExperimentalConfiguration experimentalConfiguration;
+
+    ExperimentalPluginValidator(final ExperimentalConfigurationContainer experimentalConfigurationContainer) {
+        this.experimentalConfiguration = experimentalConfigurationContainer.getExperimental();
+    }
+
+    @Override
+    public void accept(final DefinedPlugin<?> definedPlugin) {
+        if(isPluginDisallowedAsExperimental(definedPlugin.getPluginClass())) {
+            throw new NoPluginFoundException("Unable to create experimental plugin " + definedPlugin.getPluginName() +
+                    ". You must enable experimental plugins in data-prepper-config.yaml in order to use them.");
+        }
+    }
+
+    private boolean isPluginDisallowedAsExperimental(final Class<?> pluginClass) {
+        return pluginClass.isAnnotationPresent(Experimental.class) && !experimentalConfiguration.isEnableAll();
+    }
+}