Add remove_index_block arg to _create_from api (elastic#120548) (elas…

…tic#120561)

Add a new boolean parameter remove_index_blocks from the _create_from API.
If this parameter is set to true, all index block will be filtered out when creating the destination index.

docs/changelog/120548.yaml

@@ -0,0 +1,5 @@
+pr: 120548
+summary: Add `remove_index_block` arg to `_create_from` api
+area: Indices APIs
+type: enhancement
+issues: []

rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json

@@ -29,7 +29,7 @@
       ]
     },
     "body":{
-      "description":"The body contains the fields `mappings_override` and `settings_override`.",
+      "description":"The body contains the fields `mappings_override`, `settings_override`, and `remove_index_blocks`.",
       "required":false
     }
   }

x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java

@@ -177,7 +177,7 @@ public void testSettingsOverridden() throws Exception {
         assertAcked(
             client().execute(
                 CreateIndexFromSourceAction.INSTANCE,
-                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of())
+                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of(), randomBoolean())
             )
         );
 
@@ -193,7 +193,10 @@ public void testSettingsNullOverride() throws Exception {
         assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
 
         var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
-        var sourceSettings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build();
+        var sourceSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_BLOCKS_WRITE, true)
+            .put(IndexMetadata.SETTING_BLOCKS_READ, true)
+            .build();
         indicesAdmin().create(new CreateIndexRequest(sourceIndex, sourceSettings)).get();
 
         Settings settingsOverride = Settings.builder().putNull(IndexMetadata.SETTING_BLOCKS_WRITE).build();
@@ -203,13 +206,55 @@ public void testSettingsNullOverride() throws Exception {
         assertAcked(
             client().execute(
                 CreateIndexFromSourceAction.INSTANCE,
-                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of())
+                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of(), false)
             )
         );
 
         // assert settings overridden
         var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
-        assertNull(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE));
+        var destSettings = settingsResponse.getIndexToSettings().get(destIndex);
+
+        // sanity check
+        assertTrue(destSettings.getAsBoolean(IndexMetadata.SETTING_BLOCKS_READ, false));
+
+        // override null removed
+        assertNull(destSettings.get(IndexMetadata.SETTING_BLOCKS_WRITE));
+    }
+
+    public void testRemoveIndexBlocks() throws Exception {
+        assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled());
+
+        var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+
+        var sourceSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean())
+            .put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, randomBoolean())
+            .build();
+        indicesAdmin().create(new CreateIndexRequest(sourceIndex, sourceSettings)).get();
+
+        var settingsOverride = Settings.builder()
+            .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean())
+            .put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, randomBoolean())
+            .put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean())
+            .build();
+
+        // create from source
+        var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT);
+        assertAcked(
+            client().execute(
+                CreateIndexFromSourceAction.INSTANCE,
+                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of(), true)
+            )
+        );
+
+        // assert settings overridden
+        var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet();
+        var destSettings = settingsResponse.getIndexToSettings().get(destIndex);
+
+        // remove block settings override both source settings and override settings
+        assertNull(destSettings.get(IndexMetadata.SETTING_BLOCKS_WRITE));
+        assertNull(destSettings.get(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE));
+        assertNull(destSettings.get(IndexMetadata.SETTING_BLOCKS_READ));
     }
 
     public void testMappingsOverridden() {
@@ -255,7 +300,7 @@ public void testMappingsOverridden() {
         assertAcked(
             client().execute(
                 CreateIndexFromSourceAction.INSTANCE,
-                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, Settings.EMPTY, mappingOverride)
+                new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, Settings.EMPTY, mappingOverride, randomBoolean())
             )
         );
 

x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java

@@ -38,10 +38,12 @@ private CreateIndexFromSourceAction() {
     public static class Request extends ActionRequest implements IndicesRequest, ToXContent {
         private final String sourceIndex;
         private final String destIndex;
-        private Settings settingsOverride = Settings.EMPTY;
-        private Map<String, Object> mappingsOverride = Map.of();
+        private Settings settingsOverride;
+        private Map<String, Object> mappingsOverride;
+        private boolean removeIndexBlocks;
         private static final ParseField SETTINGS_OVERRIDE_FIELD = new ParseField("settings_override");
         private static final ParseField MAPPINGS_OVERRIDE_FIELD = new ParseField("mappings_override");
+        private static final ParseField REMOVE_INDEX_BLOCKS_FIELD = new ParseField("remove_index_blocks");
         private static final ObjectParser<Request, Void> PARSER = new ObjectParser<>("create_index_from_source_request");
 
         static {
@@ -50,25 +52,36 @@ public static class Request extends ActionRequest implements IndicesRequest, ToX
                 SETTINGS_OVERRIDE_FIELD,
                 ObjectParser.ValueType.OBJECT
             );
-
             PARSER.declareField(
                 (parser, request, context) -> request.mappingsOverride(Map.of("_doc", parser.map())),
                 MAPPINGS_OVERRIDE_FIELD,
                 ObjectParser.ValueType.OBJECT
             );
+            PARSER.declareField(
+                (parser, request, context) -> request.removeIndexBlocks(parser.booleanValue()),
+                REMOVE_INDEX_BLOCKS_FIELD,
+                ObjectParser.ValueType.BOOLEAN
+            );
         }
 
         public Request(String sourceIndex, String destIndex) {
-            this(sourceIndex, destIndex, Settings.EMPTY, Map.of());
+            this(sourceIndex, destIndex, Settings.EMPTY, Map.of(), false);
         }
 
-        public Request(String sourceIndex, String destIndex, Settings settingsOverride, Map<String, Object> mappingsOverride) {
+        public Request(
+            String sourceIndex,
+            String destIndex,
+            Settings settingsOverride,
+            Map<String, Object> mappingsOverride,
+            boolean removeIndexBlocks
+        ) {
             Objects.requireNonNull(settingsOverride);
             Objects.requireNonNull(mappingsOverride);
             this.sourceIndex = sourceIndex;
             this.destIndex = destIndex;
             this.settingsOverride = settingsOverride;
             this.mappingsOverride = mappingsOverride;
+            this.removeIndexBlocks = removeIndexBlocks;
         }
 
         @SuppressWarnings("unchecked")
@@ -78,6 +91,7 @@ public Request(StreamInput in) throws IOException {
             this.destIndex = in.readString();
             this.settingsOverride = Settings.readSettingsFromStream(in);
             this.mappingsOverride = (Map<String, Object>) in.readGenericValue();
+            this.removeIndexBlocks = in.readBoolean();
         }
 
         @Override
@@ -87,6 +101,7 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeString(destIndex);
             settingsOverride.writeTo(out);
             out.writeGenericValue(mappingsOverride);
+            out.writeBoolean(removeIndexBlocks);
         }
 
         @Override
@@ -110,6 +125,10 @@ public Map<String, Object> mappingsOverride() {
             return mappingsOverride;
         }
 
+        public boolean removeIndexBlocks() {
+            return removeIndexBlocks;
+        }
+
         public void settingsOverride(Settings settingsOverride) {
             this.settingsOverride = settingsOverride;
         }
@@ -118,6 +137,10 @@ public void mappingsOverride(Map<String, Object> mappingsOverride) {
             this.mappingsOverride = mappingsOverride;
         }
 
+        public void removeIndexBlocks(boolean removeIndexBlocks) {
+            this.removeIndexBlocks = removeIndexBlocks;
+        }
+
         public void fromXContent(XContentParser parser) throws IOException {
             PARSER.parse(parser, this, null);
         }
@@ -136,6 +159,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
                 settingsOverride.toXContent(builder, params);
                 builder.endObject();
             }
+            builder.field(REMOVE_INDEX_BLOCKS_FIELD.getPreferredName(), removeIndexBlocks);
 
             return builder;
         }
@@ -148,12 +172,14 @@ public boolean equals(Object o) {
             return Objects.equals(sourceIndex, request.sourceIndex)
                 && Objects.equals(destIndex, request.destIndex)
                 && Objects.equals(settingsOverride, request.settingsOverride)
-                && Objects.equals(mappingsOverride, request.mappingsOverride);
+                && Objects.equals(mappingsOverride, request.mappingsOverride)
+                && removeIndexBlocks == request.removeIndexBlocks;
+
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(sourceIndex, destIndex, settingsOverride, mappingsOverride);
+            return Objects.hash(sourceIndex, destIndex, settingsOverride, mappingsOverride, removeIndexBlocks);
         }
 
         @Override

x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java

@@ -46,6 +46,13 @@ public class CreateIndexFromSourceTransportAction extends HandledTransportAction
     private final ClusterService clusterService;
     private final Client client;
     private final IndexScopedSettings indexScopedSettings;
+    private static final Settings REMOVE_INDEX_BLOCKS_SETTING_OVERRIDE = Settings.builder()
+        .putNull(IndexMetadata.SETTING_READ_ONLY)
+        .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE)
+        .putNull(IndexMetadata.SETTING_BLOCKS_WRITE)
+        .putNull(IndexMetadata.SETTING_BLOCKS_METADATA)
+        .putNull(IndexMetadata.SETTING_BLOCKS_READ)
+        .build();
 
     @Inject
     public CreateIndexFromSourceTransportAction(
@@ -80,12 +87,15 @@ protected void doExecute(Task task, CreateIndexFromSourceAction.Request request,
 
         logger.debug("Creating destination index [{}] for source index [{}]", request.destIndex(), request.sourceIndex());
 
-        Settings settings = Settings.builder()
-            // add source settings
+        Settings.Builder settings = Settings.builder()
+            // first settings from source index
             .put(filterSettings(sourceIndex))
-            // add override settings from request
-            .put(request.settingsOverride())
-            .build();
+            // then override with request settings
+            .put(request.settingsOverride());
+        if (request.removeIndexBlocks()) {
+            // lastly, override with settings to remove index blocks if requested
+            settings.put(REMOVE_INDEX_BLOCKS_SETTING_OVERRIDE);
+        }
 
         Map<String, Object> mergeMappings;
         try {

x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java

@@ -192,21 +192,17 @@ private void createIndex(
     ) {
         logger.debug("Creating destination index [{}] for source index [{}]", destIndexName, sourceIndex.getIndex().getName());
 
-        var removeReadOnlyOverride = Settings.builder()
-            // remove read-only settings if they exist
-            .putNull(IndexMetadata.SETTING_READ_ONLY)
-            .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE)
-            .putNull(IndexMetadata.SETTING_BLOCKS_WRITE)
-            // settings to optimize reindex
+        var settingsOverride = Settings.builder()
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
             .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1)
             .build();
 
         var request = new CreateIndexFromSourceAction.Request(
             sourceIndex.getIndex().getName(),
             destIndexName,
-            removeReadOnlyOverride,
-            Map.of()
+            settingsOverride,
+            Map.of(),
+            true
         );
         request.setParentTask(parentTaskId);
         var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", request.destIndex());

x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java

@@ -63,7 +63,7 @@ protected Request createTestInstance() {
         if (randomBoolean()) {
             return new Request(source, dest);
         } else {
-            return new Request(source, dest, randomSettings(), randomMappings());
+            return new Request(source, dest, randomSettings(), randomMappings(), randomBoolean());
         }
     }
 
@@ -74,14 +74,16 @@ protected Request mutateInstance(Request instance) throws IOException {
         String destIndex = instance.destIndex();
         Settings settingsOverride = instance.settingsOverride();
         Map<String, Object> mappingsOverride = instance.mappingsOverride();
+        boolean removeIndexBlocks = instance.removeIndexBlocks();
 
-        switch (between(0, 3)) {
+        switch (between(0, 4)) {
             case 0 -> sourceIndex = randomValueOtherThan(sourceIndex, () -> randomAlphaOfLength(30));
             case 1 -> destIndex = randomValueOtherThan(destIndex, () -> randomAlphaOfLength(30));
             case 2 -> settingsOverride = randomValueOtherThan(settingsOverride, CreateFromSourceIndexRequestTests::randomSettings);
             case 3 -> mappingsOverride = randomValueOtherThan(mappingsOverride, CreateFromSourceIndexRequestTests::randomMappings);
+            case 4 -> removeIndexBlocks = removeIndexBlocks == false;
         }
-        return new Request(sourceIndex, destIndex, settingsOverride, mappingsOverride);
+        return new Request(sourceIndex, destIndex, settingsOverride, mappingsOverride, removeIndexBlocks);
     }
 
     public static Map<String, Object> randomMappings() {

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml

@@ -119,3 +119,39 @@ teardown:
   - match: {dest-index-1.mappings.properties.foo.type: boolean}
   - match: {dest-index-1.mappings.properties.baz.type: integer}
 
+---
+"Test create_from with remove_index_blocks":
+  - requires:
+      reason: "migration reindex is behind a feature flag"
+      test_runner_features: [capabilities]
+      capabilities:
+        - method: POST
+          path: /_migration/reindex
+          capabilities: [migration_reindex]
+  - do:
+      indices.create:
+        index: source-index-1
+        body:
+          settings:
+            index:
+              blocks.write: true
+              blocks.read: true
+              number_of_shards: 1
+              number_of_replicas: 0
+  - do:
+      migrate.create_from:
+        source: "source-index-1"
+        dest: "dest-index-1"
+        body:
+          settings_override:
+            index:
+              blocks.write: false
+              blocks.read: true
+          remove_index_blocks: true
+  - do:
+      indices.get_settings:
+        index: dest-index-1
+  - match: { dest-index-1.settings.index.number_of_shards: "1" }
+  - match: { dest-index-1.settings.index.blocks.write: null }
+  - match: { dest-index-1.settings.index.blocks.read: null }
+