[Deprecation API] Refactor resource deprecation checkers and add new …

…resources. (elastic#120505)

docs/changelog/120505.yaml

@@ -0,0 +1,5 @@
+pr: 120505
+summary: "introduce new categories for deprecated resources in deprecation API"
+area: Indices APIs
+type: enhancement
+issues: []

qa/rolling-upgrade/src/javaRestTest/java/org/elasticsearch/upgrades/SourceModeRollingUpgradeIT.java

@@ -50,22 +50,9 @@ public void testConfigureStoredSourceBeforeIndexCreationLegacy() throws IOExcept
             putComponentTemplateRequest.setOptions(expectWarnings(SourceFieldMapper.DEPRECATION_WARNING));
             putComponentTemplateRequest.setJsonEntity(storedSourceMapping);
             assertOK(client().performRequest(putComponentTemplateRequest));
-
-            var request = new Request("GET", "/_migration/deprecations");
-            var nodeSettings = (Map<?, ?>) ((List<?>) entityAsMap(client().performRequest(request)).get("node_settings")).getFirst();
-            assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
-            assertThat(
-                (String) nodeSettings.get("details"),
-                containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]")
-            );
+            assertDeprecationWarningForTemplate(templateName);
         } else if (isUpgradedCluster()) {
-            var request = new Request("GET", "/_migration/deprecations");
-            var nodeSettings = (Map<?, ?>) ((List<?>) entityAsMap(client().performRequest(request)).get("node_settings")).getFirst();
-            assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
-            assertThat(
-                (String) nodeSettings.get("details"),
-                containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]")
-            );
+            assertDeprecationWarningForTemplate(templateName);
         }
     }
 
@@ -87,17 +74,24 @@ public void testConfigureStoredSourceWhenIndexIsCreatedLegacy() throws IOExcepti
             putComponentTemplateRequest.setOptions(expectWarnings(SourceFieldMapper.DEPRECATION_WARNING));
             putComponentTemplateRequest.setJsonEntity(storedSourceMapping);
             assertOK(client().performRequest(putComponentTemplateRequest));
-
-            var request = new Request("GET", "/_migration/deprecations");
-            var nodeSettings = (Map<?, ?>) ((List<?>) entityAsMap(client().performRequest(request)).get("node_settings")).getFirst();
-            assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
-            assertThat(
-                (String) nodeSettings.get("details"),
-                containsString(SourceFieldMapper.DEPRECATION_WARNING + " Affected component templates: [" + templateName + "]")
-            );
+            assertDeprecationWarningForTemplate(templateName);
         } else if (isUpgradedCluster()) {
-            var request = new Request("GET", "/_migration/deprecations");
-            var nodeSettings = (Map<?, ?>) ((List<?>) entityAsMap(client().performRequest(request)).get("node_settings")).getFirst();
+            assertDeprecationWarningForTemplate(templateName);
+        }
+    }
+
+    private void assertDeprecationWarningForTemplate(String templateName) throws IOException {
+        var request = new Request("GET", "/_migration/deprecations");
+        var response = entityAsMap(client().performRequest(request));
+        if (response.containsKey("templates")) {
+            // Check the newer version of the deprecation API that contains the templates section
+            Map<?, ?> issuesByTemplate = (Map<?, ?>) response.get("templates");
+            assertThat(issuesByTemplate.containsKey(templateName), equalTo(true));
+            var templateIssues = (List<?>) issuesByTemplate.get(templateName);
+            assertThat(((Map<?, ?>) templateIssues.getFirst()).get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
+        } else {
+            // Bwc version with 8.18 until https://github.com/elastic/elasticsearch/pull/120505/ gets backported, clean up after backport
+            var nodeSettings = (Map<?, ?>) ((List<?>) response.get("node_settings")).getFirst();
             assertThat(nodeSettings.get("message"), equalTo(SourceFieldMapper.DEPRECATION_WARNING));
             assertThat(
                 (String) nodeSettings.get("details"),

server/src/main/java/org/elasticsearch/TransportVersions.java

@@ -167,6 +167,7 @@ static TransportVersion def(int id) {
     public static final TransportVersion RANK_DOC_OPTIONAL_METADATA_FOR_EXPLAIN = def(8_833_00_0);
     public static final TransportVersion ILM_ADD_SEARCHABLE_SNAPSHOT_ADD_REPLICATE_FOR = def(8_834_00_0);
     public static final TransportVersion INGEST_REQUEST_INCLUDE_SOURCE_ON_ERROR = def(8_835_00_0);
+    public static final TransportVersion RESOURCE_DEPRECATION_CHECKS = def(8_836_00_0);
 
     /*
      * STOP! READ THIS FIRST! No, really,

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/deprecation/DeprecationIssue.java

@@ -240,14 +240,7 @@ public static DeprecationIssue getIntersectionOfRemovableSettings(List<Deprecati
      *    }
      * }
      */
-    private static final class Meta {
-        private final List<Action> actions;
-        private final Map<String, Object> nonActionMetadata;
-
-        Meta(List<Action> actions, Map<String, Object> nonActionMetadata) {
-            this.actions = actions;
-            this.nonActionMetadata = nonActionMetadata;
-        }
+    private record Meta(List<Action> actions, Map<String, Object> nonActionMetadata) {
 
         private static Meta fromRemovableSettings(List<String> removableSettings) {
             List<Action> actions;
@@ -358,12 +351,7 @@ private interface Action {
     /*
      * This class a represents remove_settings action within the actions list in a meta Map.
      */
-    private static final class RemovalAction implements Action {
-        private final List<String> removableSettings;
-
-        RemovalAction(List<String> removableSettings) {
-            this.removableSettings = removableSettings;
-        }
+    private record RemovalAction(List<String> removableSettings) implements Action {
 
         @SuppressWarnings("unchecked")
         private static RemovalAction fromActionMap(Map<String, Object> actionMap) {
@@ -398,12 +386,7 @@ public Map<String, Object> toActionMap() {
     /*
      * This represents an action within the actions list in a meta Map that is *not* a removal_action.
      */
-    private static class UnknownAction implements Action {
-        private final Map<String, Object> actionMap;
-
-        private UnknownAction(Map<String, Object> actionMap) {
-            this.actionMap = actionMap;
-        }
+    private record UnknownAction(Map<String, Object> actionMap) implements Action {
 
         private static Action fromActionMap(Map<String, Object> actionMap) {
             return new UnknownAction(actionMap);

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecks.java → x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DataStreamDeprecationChecker.java

@@ -7,20 +7,65 @@
 
 package org.elasticsearch.xpack.deprecation;
 
+import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.metadata.DataStream;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.index.Index;
 import org.elasticsearch.xpack.core.deprecation.DeprecatedIndexPredicate;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Set;
+import java.util.function.BiFunction;
 import java.util.stream.Collectors;
 
 import static java.util.Map.entry;
 import static java.util.Map.ofEntries;
+import static org.elasticsearch.xpack.deprecation.DeprecationInfoAction.filterChecks;
+
+/**
+ * Checks the data streams for deprecation warnings.
+ */
+public class DataStreamDeprecationChecker implements ResourceDeprecationChecker {
+
+    public static final String NAME = "data_streams";
+    private static final List<BiFunction<DataStream, ClusterState, DeprecationIssue>> DATA_STREAM_CHECKS = List.of(
+        DataStreamDeprecationChecker::oldIndicesCheck,
+        DataStreamDeprecationChecker::ignoredOldIndicesCheck
+    );
+    private final IndexNameExpressionResolver indexNameExpressionResolver;
+
+    public DataStreamDeprecationChecker(IndexNameExpressionResolver indexNameExpressionResolver) {
+        this.indexNameExpressionResolver = indexNameExpressionResolver;
+    }
+
+    /**
+     * @param clusterState The cluster state provided for the checker
+     * @return the name of the data streams that have violated the checks with their respective warnings.
+     */
+    @Override
+    public Map<String, List<DeprecationIssue>> check(ClusterState clusterState, DeprecationInfoAction.Request request) {
+        List<String> dataStreamNames = indexNameExpressionResolver.dataStreamNames(
+            clusterState,
+            IndicesOptions.LENIENT_EXPAND_OPEN_CLOSED_HIDDEN
+        );
+        if (dataStreamNames.isEmpty()) {
+            return Map.of();
+        }
+        Map<String, List<DeprecationIssue>> dataStreamIssues = new HashMap<>();
+        for (String dataStreamName : dataStreamNames) {
+            DataStream dataStream = clusterState.metadata().dataStreams().get(dataStreamName);
+            List<DeprecationIssue> issuesForSingleDataStream = filterChecks(DATA_STREAM_CHECKS, c -> c.apply(dataStream, clusterState));
+            if (issuesForSingleDataStream.isEmpty() == false) {
+                dataStreamIssues.put(dataStreamName, issuesForSingleDataStream);
+            }
+        }
+        return dataStreamIssues.isEmpty() ? Map.of() : dataStreamIssues;
+    }
 
-public class DataStreamDeprecationChecks {
     static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clusterState) {
         List<Index> backingIndices = dataStream.getIndices();
 
@@ -47,9 +92,7 @@ static DeprecationIssue oldIndicesCheck(DataStream dataStream, ClusterState clus
 
     static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ClusterState clusterState) {
         List<Index> backingIndices = dataStream.getIndices();
-
         Set<String> ignoredIndices = getReindexRequiredIndices(backingIndices, clusterState, true);
-
         if (ignoredIndices.isEmpty() == false) {
             return new DeprecationIssue(
                 DeprecationIssue.Level.WARNING,
@@ -66,7 +109,6 @@ static DeprecationIssue ignoredOldIndicesCheck(DataStream dataStream, ClusterSta
                 )
             );
         }
-
         return null;
     }
 
@@ -80,4 +122,9 @@ private static Set<String> getReindexRequiredIndices(
             .map(Index::getName)
             .collect(Collectors.toUnmodifiableSet());
     }
+
+    @Override
+    public String getName() {
+        return NAME;
+    }
 }

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/DeprecationChecks.java

@@ -8,16 +8,13 @@
 
 import org.elasticsearch.action.admin.cluster.node.info.PluginsAndModules;
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.cluster.metadata.DataStream;
-import org.elasticsearch.cluster.metadata.IndexMetadata;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
 
 import java.util.List;
 import java.util.Objects;
-import java.util.function.BiFunction;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 
@@ -88,25 +85,9 @@ private DeprecationChecks() {}
                 NodeDeprecationChecks::checkEqlEnabledSetting,
                 NodeDeprecationChecks::checkNodeAttrData,
                 NodeDeprecationChecks::checkWatcherBulkConcurrentRequestsSetting,
-                NodeDeprecationChecks::checkTracingApmSettings,
-                NodeDeprecationChecks::checkSourceModeInComponentTemplates
+                NodeDeprecationChecks::checkTracingApmSettings
             );
 
-    static List<BiFunction<IndexMetadata, ClusterState, DeprecationIssue>> INDEX_SETTINGS_CHECKS = List.of(
-        IndexDeprecationChecks::oldIndicesCheck,
-        IndexDeprecationChecks::ignoredOldIndicesCheck,
-        IndexDeprecationChecks::translogRetentionSettingCheck,
-        IndexDeprecationChecks::checkIndexDataPath,
-        IndexDeprecationChecks::storeTypeSettingCheck,
-        IndexDeprecationChecks::frozenIndexSettingCheck,
-        IndexDeprecationChecks::deprecatedCamelCasePattern
-    );
-
-    static List<BiFunction<DataStream, ClusterState, DeprecationIssue>> DATA_STREAM_CHECKS = List.of(
-        DataStreamDeprecationChecks::oldIndicesCheck,
-        DataStreamDeprecationChecks::ignoredOldIndicesCheck
-    );
-
     /**
      * helper utility function to reduce repeat of running a specific {@link List} of checks.
      *