diff --git a/docs/changelog/120548.yaml b/docs/changelog/120548.yaml new file mode 100644 index 0000000000000..a6a63d0c8d782 --- /dev/null +++ b/docs/changelog/120548.yaml @@ -0,0 +1,5 @@ +pr: 120548 +summary: Add `remove_index_block` arg to `_create_from` api +area: Indices APIs +type: enhancement +issues: [] diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json index e17a69a77b252..c6b5aae6789d6 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/migrate.create_from.json @@ -29,7 +29,7 @@ ] }, "body":{ - "description":"The body contains the fields `mappings_override` and `settings_override`.", + "description":"The body contains the fields `mappings_override`, `settings_override`, and `remove_index_blocks`.", "required":false } } diff --git a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java index 3c097f26eb353..20ac25b91783a 100644 --- a/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java +++ b/x-pack/plugin/migrate/src/internalClusterTest/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceActionIT.java @@ -177,7 +177,7 @@ public void testSettingsOverridden() throws Exception { assertAcked( client().execute( CreateIndexFromSourceAction.INSTANCE, - new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of()) + new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of(), randomBoolean()) ) ); @@ -193,7 +193,10 @@ public void testSettingsNullOverride() throws Exception { assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); - var sourceSettings = Settings.builder().put(IndexMetadata.SETTING_BLOCKS_WRITE, true).build(); + var sourceSettings = Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_WRITE, true) + .put(IndexMetadata.SETTING_BLOCKS_READ, true) + .build(); indicesAdmin().create(new CreateIndexRequest(sourceIndex, sourceSettings)).get(); Settings settingsOverride = Settings.builder().putNull(IndexMetadata.SETTING_BLOCKS_WRITE).build(); @@ -203,13 +206,55 @@ public void testSettingsNullOverride() throws Exception { assertAcked( client().execute( CreateIndexFromSourceAction.INSTANCE, - new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of()) + new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of(), false) ) ); // assert settings overridden var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); - assertNull(settingsResponse.getSetting(destIndex, IndexMetadata.SETTING_BLOCKS_WRITE)); + var destSettings = settingsResponse.getIndexToSettings().get(destIndex); + + // sanity check + assertTrue(destSettings.getAsBoolean(IndexMetadata.SETTING_BLOCKS_READ, false)); + + // override null removed + assertNull(destSettings.get(IndexMetadata.SETTING_BLOCKS_WRITE)); + } + + public void testRemoveIndexBlocks() throws Exception { + assumeTrue("requires the migration reindex feature flag", REINDEX_DATA_STREAM_FEATURE_FLAG.isEnabled()); + + var sourceIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + + var sourceSettings = Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, randomBoolean()) + .build(); + indicesAdmin().create(new CreateIndexRequest(sourceIndex, sourceSettings)).get(); + + var settingsOverride = Settings.builder() + .put(IndexMetadata.SETTING_BLOCKS_WRITE, randomBoolean()) + .put(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE, randomBoolean()) + .put(IndexMetadata.SETTING_BLOCKS_READ, randomBoolean()) + .build(); + + // create from source + var destIndex = randomAlphaOfLength(20).toLowerCase(Locale.ROOT); + assertAcked( + client().execute( + CreateIndexFromSourceAction.INSTANCE, + new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, settingsOverride, Map.of(), true) + ) + ); + + // assert settings overridden + var settingsResponse = indicesAdmin().getSettings(new GetSettingsRequest().indices(destIndex)).actionGet(); + var destSettings = settingsResponse.getIndexToSettings().get(destIndex); + + // remove block settings override both source settings and override settings + assertNull(destSettings.get(IndexMetadata.SETTING_BLOCKS_WRITE)); + assertNull(destSettings.get(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE)); + assertNull(destSettings.get(IndexMetadata.SETTING_BLOCKS_READ)); } public void testMappingsOverridden() { @@ -255,7 +300,7 @@ public void testMappingsOverridden() { assertAcked( client().execute( CreateIndexFromSourceAction.INSTANCE, - new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, Settings.EMPTY, mappingOverride) + new CreateIndexFromSourceAction.Request(sourceIndex, destIndex, Settings.EMPTY, mappingOverride, randomBoolean()) ) ); diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java index b2c6a3ea4b779..3b1841472209d 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceAction.java @@ -38,10 +38,12 @@ private CreateIndexFromSourceAction() { public static class Request extends ActionRequest implements IndicesRequest, ToXContent { private final String sourceIndex; private final String destIndex; - private Settings settingsOverride = Settings.EMPTY; - private Map mappingsOverride = Map.of(); + private Settings settingsOverride; + private Map mappingsOverride; + private boolean removeIndexBlocks; private static final ParseField SETTINGS_OVERRIDE_FIELD = new ParseField("settings_override"); private static final ParseField MAPPINGS_OVERRIDE_FIELD = new ParseField("mappings_override"); + private static final ParseField REMOVE_INDEX_BLOCKS_FIELD = new ParseField("remove_index_blocks"); private static final ObjectParser PARSER = new ObjectParser<>("create_index_from_source_request"); static { @@ -50,25 +52,36 @@ public static class Request extends ActionRequest implements IndicesRequest, ToX SETTINGS_OVERRIDE_FIELD, ObjectParser.ValueType.OBJECT ); - PARSER.declareField( (parser, request, context) -> request.mappingsOverride(Map.of("_doc", parser.map())), MAPPINGS_OVERRIDE_FIELD, ObjectParser.ValueType.OBJECT ); + PARSER.declareField( + (parser, request, context) -> request.removeIndexBlocks(parser.booleanValue()), + REMOVE_INDEX_BLOCKS_FIELD, + ObjectParser.ValueType.BOOLEAN + ); } public Request(String sourceIndex, String destIndex) { - this(sourceIndex, destIndex, Settings.EMPTY, Map.of()); + this(sourceIndex, destIndex, Settings.EMPTY, Map.of(), false); } - public Request(String sourceIndex, String destIndex, Settings settingsOverride, Map mappingsOverride) { + public Request( + String sourceIndex, + String destIndex, + Settings settingsOverride, + Map mappingsOverride, + boolean removeIndexBlocks + ) { Objects.requireNonNull(settingsOverride); Objects.requireNonNull(mappingsOverride); this.sourceIndex = sourceIndex; this.destIndex = destIndex; this.settingsOverride = settingsOverride; this.mappingsOverride = mappingsOverride; + this.removeIndexBlocks = removeIndexBlocks; } @SuppressWarnings("unchecked") @@ -78,6 +91,7 @@ public Request(StreamInput in) throws IOException { this.destIndex = in.readString(); this.settingsOverride = Settings.readSettingsFromStream(in); this.mappingsOverride = (Map) in.readGenericValue(); + this.removeIndexBlocks = in.readBoolean(); } @Override @@ -87,6 +101,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(destIndex); settingsOverride.writeTo(out); out.writeGenericValue(mappingsOverride); + out.writeBoolean(removeIndexBlocks); } @Override @@ -110,6 +125,10 @@ public Map mappingsOverride() { return mappingsOverride; } + public boolean removeIndexBlocks() { + return removeIndexBlocks; + } + public void settingsOverride(Settings settingsOverride) { this.settingsOverride = settingsOverride; } @@ -118,6 +137,10 @@ public void mappingsOverride(Map mappingsOverride) { this.mappingsOverride = mappingsOverride; } + public void removeIndexBlocks(boolean removeIndexBlocks) { + this.removeIndexBlocks = removeIndexBlocks; + } + public void fromXContent(XContentParser parser) throws IOException { PARSER.parse(parser, this, null); } @@ -136,6 +159,7 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par settingsOverride.toXContent(builder, params); builder.endObject(); } + builder.field(REMOVE_INDEX_BLOCKS_FIELD.getPreferredName(), removeIndexBlocks); return builder; } @@ -148,12 +172,14 @@ public boolean equals(Object o) { return Objects.equals(sourceIndex, request.sourceIndex) && Objects.equals(destIndex, request.destIndex) && Objects.equals(settingsOverride, request.settingsOverride) - && Objects.equals(mappingsOverride, request.mappingsOverride); + && Objects.equals(mappingsOverride, request.mappingsOverride) + && removeIndexBlocks == request.removeIndexBlocks; + } @Override public int hashCode() { - return Objects.hash(sourceIndex, destIndex, settingsOverride, mappingsOverride); + return Objects.hash(sourceIndex, destIndex, settingsOverride, mappingsOverride, removeIndexBlocks); } @Override diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java index dcd8dc0ee3fce..81edb0e716f51 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/CreateIndexFromSourceTransportAction.java @@ -46,6 +46,13 @@ public class CreateIndexFromSourceTransportAction extends HandledTransportAction private final ClusterService clusterService; private final Client client; private final IndexScopedSettings indexScopedSettings; + private static final Settings REMOVE_INDEX_BLOCKS_SETTING_OVERRIDE = Settings.builder() + .putNull(IndexMetadata.SETTING_READ_ONLY) + .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE) + .putNull(IndexMetadata.SETTING_BLOCKS_WRITE) + .putNull(IndexMetadata.SETTING_BLOCKS_METADATA) + .putNull(IndexMetadata.SETTING_BLOCKS_READ) + .build(); @Inject public CreateIndexFromSourceTransportAction( @@ -80,12 +87,15 @@ protected void doExecute(Task task, CreateIndexFromSourceAction.Request request, logger.debug("Creating destination index [{}] for source index [{}]", request.destIndex(), request.sourceIndex()); - Settings settings = Settings.builder() - // add source settings + Settings.Builder settings = Settings.builder() + // first settings from source index .put(filterSettings(sourceIndex)) - // add override settings from request - .put(request.settingsOverride()) - .build(); + // then override with request settings + .put(request.settingsOverride()); + if (request.removeIndexBlocks()) { + // lastly, override with settings to remove index blocks if requested + settings.put(REMOVE_INDEX_BLOCKS_SETTING_OVERRIDE); + } Map mergeMappings; try { diff --git a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java index e020e59c8a9e1..58a90ce15e98b 100644 --- a/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java +++ b/x-pack/plugin/migrate/src/main/java/org/elasticsearch/xpack/migrate/action/ReindexDataStreamIndexTransportAction.java @@ -192,12 +192,7 @@ private void createIndex( ) { logger.debug("Creating destination index [{}] for source index [{}]", destIndexName, sourceIndex.getIndex().getName()); - var removeReadOnlyOverride = Settings.builder() - // remove read-only settings if they exist - .putNull(IndexMetadata.SETTING_READ_ONLY) - .putNull(IndexMetadata.SETTING_READ_ONLY_ALLOW_DELETE) - .putNull(IndexMetadata.SETTING_BLOCKS_WRITE) - // settings to optimize reindex + var settingsOverride = Settings.builder() .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), -1) .build(); @@ -205,8 +200,9 @@ private void createIndex( var request = new CreateIndexFromSourceAction.Request( sourceIndex.getIndex().getName(), destIndexName, - removeReadOnlyOverride, - Map.of() + settingsOverride, + Map.of(), + true ); request.setParentTask(parentTaskId); var errorMessage = String.format(Locale.ROOT, "Could not create index [%s]", request.destIndex()); diff --git a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java index 21dab66fccd5e..426a244267dcd 100644 --- a/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java +++ b/x-pack/plugin/migrate/src/test/java/org/elasticsearch/xpack/migrate/action/CreateFromSourceIndexRequestTests.java @@ -63,7 +63,7 @@ protected Request createTestInstance() { if (randomBoolean()) { return new Request(source, dest); } else { - return new Request(source, dest, randomSettings(), randomMappings()); + return new Request(source, dest, randomSettings(), randomMappings(), randomBoolean()); } } @@ -74,14 +74,16 @@ protected Request mutateInstance(Request instance) throws IOException { String destIndex = instance.destIndex(); Settings settingsOverride = instance.settingsOverride(); Map mappingsOverride = instance.mappingsOverride(); + boolean removeIndexBlocks = instance.removeIndexBlocks(); - switch (between(0, 3)) { + switch (between(0, 4)) { case 0 -> sourceIndex = randomValueOtherThan(sourceIndex, () -> randomAlphaOfLength(30)); case 1 -> destIndex = randomValueOtherThan(destIndex, () -> randomAlphaOfLength(30)); case 2 -> settingsOverride = randomValueOtherThan(settingsOverride, CreateFromSourceIndexRequestTests::randomSettings); case 3 -> mappingsOverride = randomValueOtherThan(mappingsOverride, CreateFromSourceIndexRequestTests::randomMappings); + case 4 -> removeIndexBlocks = removeIndexBlocks == false; } - return new Request(sourceIndex, destIndex, settingsOverride, mappingsOverride); + return new Request(sourceIndex, destIndex, settingsOverride, mappingsOverride, removeIndexBlocks); } public static Map randomMappings() { diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml index dfffea9a049a1..3569276c17f77 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/migrate/30_create_from.yml @@ -119,3 +119,39 @@ teardown: - match: {dest-index-1.mappings.properties.foo.type: boolean} - match: {dest-index-1.mappings.properties.baz.type: integer} +--- +"Test create_from with remove_index_blocks": + - requires: + reason: "migration reindex is behind a feature flag" + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_migration/reindex + capabilities: [migration_reindex] + - do: + indices.create: + index: source-index-1 + body: + settings: + index: + blocks.write: true + blocks.read: true + number_of_shards: 1 + number_of_replicas: 0 + - do: + migrate.create_from: + source: "source-index-1" + dest: "dest-index-1" + body: + settings_override: + index: + blocks.write: false + blocks.read: true + remove_index_blocks: true + - do: + indices.get_settings: + index: dest-index-1 + - match: { dest-index-1.settings.index.number_of_shards: "1" } + - match: { dest-index-1.settings.index.blocks.write: null } + - match: { dest-index-1.settings.index.blocks.read: null } +