From f4db40081aaf998151fe54b00087ca7c3655c401 Mon Sep 17 00:00:00 2001 From: vikasvb90 Date: Thu, 15 Feb 2024 20:25:11 +0530 Subject: [PATCH] Splitting a shard in-place --- .idea/inspectionProfiles/Project_Default.xml | 5 +- .idea/runConfigurations/Debug_OpenSearch.xml | 6 +- .../allocation/AllocationBenchmark.java | 1 + .../processor/ApiAnnotationProcessor.java | 10 + .../core/common/io/stream/StreamInput.java | 12 + .../core/common/io/stream/StreamOutput.java | 7 + .../repositories/azure/AzureBlobStore.java | 2 +- .../gcs/GoogleCloudStorageBlobStore.java | 2 +- .../repositories/hdfs/HdfsBlobContainer.java | 2 +- .../repositories/s3/S3BlobContainer.java | 115 +++- .../s3/async/CopyObjectHelper.java | 327 +++++++++++ .../s3/async/CopyObjectMetadata.java | 44 ++ .../gateway/RecoveryFromGatewayIT.java | 3 +- .../opensearch/index/shard/IndexShardIT.java | 3 +- .../mocks/MockFsAsyncBlobContainer.java | 7 + .../shardsplit/InPlaceShardSplitIT.java | 243 ++++++++ .../InPlaceShardSplitRemoteStoreIT.java | 52 ++ .../opensearch/OpenSearchServerException.java | 9 + .../org/opensearch/action/ActionModule.java | 5 + .../action/PrimaryShardSplitException.java | 29 + .../ClusterAllocationExplanation.java | 4 + .../split/InPlaceShardSplitAction.java | 21 + ...ceShardSplitClusterStateUpdateRequest.java | 42 ++ .../split/InPlaceShardSplitRequest.java | 133 +++++ .../split/InPlaceShardSplitResponse.java | 88 +++ .../TransportInPlaceShardSplitAction.java | 162 ++++++ .../action/bulk/TransportBulkAction.java | 76 ++- .../action/bulk/TransportShardBulkAction.java | 71 ++- .../action/search/TransportSearchAction.java | 40 +- .../ReplicationModeAwareProxy.java | 3 +- .../replication/ReplicationOperation.java | 55 +- .../support/replication/ReplicationProxy.java | 3 +- .../replication/ReplicationProxyRequest.java | 7 + .../replication/ReplicationRequest.java | 2 +- .../TransportReplicationAction.java | 179 +++++- .../replication/TransportWriteAction.java | 1 + .../opensearch/client/IndicesAdminClient.java | 14 + .../client/support/AbstractClient.java | 13 + .../org/opensearch/cluster/ClusterModule.java | 2 + .../org/opensearch/cluster/ClusterState.java | 8 +- .../action/shard/ShardStateAction.java | 317 ++++++++--- .../ClusterAwarenessAttributeValueHealth.java | 45 +- .../ClusterAwarenessAttributesHealth.java | 12 +- .../cluster/health/ClusterShardHealth.java | 1 + .../cluster/metadata/IndexMetadata.java | 165 +++++- .../MetadataInPlaceShardSplitService.java | 166 ++++++ .../cluster/metadata/ShardRange.java | 134 +++++ .../cluster/metadata/SplitShardsMetadata.java | 441 +++++++++++++++ .../cluster/routing/AllocationId.java | 144 ++++- .../cluster/routing/IndexRoutingTable.java | 57 +- .../routing/IndexShardRoutingTable.java | 10 + .../cluster/routing/OperationRouting.java | 24 +- .../cluster/routing/RecoverySource.java | 33 +- .../routing/RoutingChangesObserver.java | 109 ++++ .../cluster/routing/RoutingNode.java | 33 ++ .../cluster/routing/RoutingNodes.java | 326 ++++++++++- .../cluster/routing/RoutingTable.java | 29 +- .../routing/RoutingTableIncrementalDiff.java | 18 +- .../cluster/routing/ShardRouting.java | 445 ++++++++++++++- .../cluster/routing/ShardRoutingState.java | 8 +- .../cluster/routing/UnassignedInfo.java | 6 +- .../allocation/AllocationDecision.java | 8 +- .../routing/allocation/AllocationService.java | 18 +- .../allocation/IndexMetadataUpdater.java | 123 +++- .../routing/allocation/MoveDecision.java | 9 + .../RoutingNodesChangedObserver.java | 42 ++ .../allocator/BalancedShardsAllocator.java | 1 + .../allocator/LocalShardsBalancer.java | 218 +++++-- .../allocator/RemoteShardsBalancer.java | 4 + .../allocation/allocator/ShardsBalancer.java | 5 + .../command/CancelAllocationCommand.java | 4 +- .../routing/allocation/decider/Decision.java | 14 +- .../InPlaceShardSplitAllocationDecider.java | 33 ++ ...caAfterPrimaryActiveAllocationDecider.java | 4 +- .../decider/ThrottlingAllocationDecider.java | 26 +- .../service/ClusterManagerTaskKeys.java | 1 + .../AsyncMultiStreamBlobContainer.java | 14 + ...syncMultiStreamEncryptedBlobContainer.java | 6 + .../common/blobstore/BlobContainer.java | 3 +- .../common/blobstore/BlobMetadata.java | 5 + .../blobstore/EncryptedBlobMetadata.java | 5 + .../common/blobstore/fs/FsBlobContainer.java | 11 +- .../common/blobstore/fs/FsBlobStore.java | 6 + .../blobstore/support/PlainBlobMetadata.java | 10 +- .../org/opensearch/index/IndexService.java | 10 +- .../index/engine/InternalEngine.java | 17 +- .../index/engine/NRTReplicationEngine.java | 1 + .../seqno/GlobalCheckpointSyncAction.java | 4 +- .../index/seqno/ReplicationTracker.java | 134 ++++- .../index/seqno/RetentionLeaseActions.java | 6 + .../RetentionLeaseBackgroundSyncAction.java | 4 +- .../index/seqno/RetentionLeaseSyncAction.java | 4 +- .../shard/AbstractIndexShardComponent.java | 7 + .../opensearch/index/shard/IndexShard.java | 288 ++++++++-- .../shard/IndexShardOperationPermits.java | 13 + .../shard/RemoteStoreRefreshListener.java | 50 +- .../index/shard/ReplicationGroup.java | 11 + .../index/shard/ShardSplittingQuery.java | 12 +- .../opensearch/index/shard/StoreRecovery.java | 49 +- .../index/store/RemoteDirectory.java | 22 +- .../store/RemoteSegmentStoreDirectory.java | 62 +- .../translog/InternalTranslogManager.java | 6 + .../index/translog/NoOpTranslogManager.java | 8 + ...emoteBlobStoreInternalTranslogFactory.java | 2 + .../RemoteFsTimestampAwareTranslog.java | 6 +- .../opensearch/index/translog/Translog.java | 18 +- .../index/translog/TranslogManager.java | 9 + .../opensearch/indices/IndicesService.java | 64 ++- .../cluster/IndicesClusterStateService.java | 243 +++++++- .../LocalStorePeerRecoverySourceHandler.java | 126 +++-- .../recovery/PeerRecoverySourceService.java | 32 +- .../recovery/PeerRecoveryTargetService.java | 5 +- .../indices/recovery/RecoveryResponse.java | 16 + .../recovery/RecoverySourceHandler.java | 352 +++++++++--- .../RecoverySourceHandlerFactory.java | 16 +- .../indices/recovery/RecoveryState.java | 5 +- .../indices/recovery/RecoveryTarget.java | 48 +- .../RemoteStorePeerRecoverySourceHandler.java | 49 +- .../recovery/StartRecoveryRequest.java | 28 +- .../InPlaceShardRecoveryContext.java | 47 ++ .../InPlaceShardSplitRecoveryListener.java | 62 ++ .../InPlaceShardSplitRecoveryService.java | 352 ++++++++++++ ...nPlaceShardSplitRecoverySourceHandler.java | 534 ++++++++++++++++++ ...nPlaceShardSplitRecoveryTargetHandler.java | 439 ++++++++++++++ .../InPlaceShardSplitRemoteSourceHandler.java | 332 +++++++++++ .../InPlaceShardSplitResponseHandler.java | 96 ++++ .../InPlaceShardsRecoveryFailedException.java | 30 + .../inplacesplit/SplitCommitMetadata.java | 33 ++ .../RemoteStoreReplicationSource.java | 33 +- .../SegmentReplicationSourceService.java | 2 +- .../replication/SegmentReplicationTarget.java | 12 +- .../SegmentReplicationTargetService.java | 34 +- .../replication/SegmentReplicator.java | 2 +- .../checkpoint/PublishCheckpointAction.java | 7 +- .../checkpoint/ReplicationCheckpoint.java | 12 + .../common/ReplicationLuceneIndex.java | 3 + .../replication/common/ReplicationTarget.java | 5 +- .../replication/common/ReplicationTimer.java | 6 +- .../main/java/org/opensearch/node/Node.java | 26 +- .../java/org/opensearch/plugins/Plugin.java | 6 + .../opensearch/plugins/PluginsService.java | 23 + .../indices/RestInPlaceShardSplitHandler.java | 83 +++ .../rest/action/cat/RestShardsAction.java | 13 +- .../opensearch/snapshots/RestoreService.java | 3 +- .../org/opensearch/bootstrap/security.policy | 7 + .../ReplicationOperationTests.java | 3 +- .../TransportReplicationActionTests.java | 5 +- .../TransportWriteActionTests.java | 5 +- ...rdFailedClusterStateTaskExecutorTests.java | 30 +- ...dStartedClusterStateTaskExecutorTests.java | 28 +- .../action/shard/ShardStateActionTests.java | 5 +- ...terAwarenessAttributeValueHealthTests.java | 3 +- ...essAttributesHealthSerializationTests.java | 9 +- .../allocation/InSyncAllocationIdTests.java | 8 +- .../common/io/stream/BytesStreamsTests.java | 4 +- ...RemoteClusterStateCleanupManagerTests.java | 34 +- .../RemoteClusterStateServiceTests.java | 9 +- .../index/remote/RemoteStoreUtilsTests.java | 10 +- ...PeerRecoveryRetentionLeaseExpiryTests.java | 2 + .../index/seqno/ReplicationTrackerTests.java | 12 +- .../index/shard/IndexShardTests.java | 46 +- .../SegmentReplicationIndexShardTests.java | 11 +- .../index/shard/ShardSplittingQueryTests.java | 2 +- .../index/shard/StoreRecoveryTests.java | 4 +- .../index/store/RemoteDirectoryTests.java | 12 +- ...toreDirectoryWithPinnedTimestampTests.java | 2 +- .../RemoteFsTimestampAwareTranslogTests.java | 6 +- .../BlobStoreTransferServiceTests.java | 6 + .../TranslogTransferManagerTests.java | 34 +- ...dicesLifecycleListenerSingleNodeTests.java | 4 +- .../indices/IndicesRequestCacheTests.java | 4 +- ...actIndicesClusterStateServiceTestCase.java | 30 + .../indices/cluster/ClusterStateChanges.java | 9 +- ...ClusterStateServiceRandomUpdatesTests.java | 1 + ...alStorePeerRecoverySourceHandlerTests.java | 113 ++-- .../PeerRecoverySourceServiceTests.java | 12 +- ...teStorePeerRecoverySourceHandlerTests.java | 18 +- .../recovery/StartRecoveryRequestTests.java | 3 +- .../SegmentReplicationTargetServiceTests.java | 5 + .../SegmentReplicationTargetTests.java | 14 +- .../recovery/ReplicationCollectionTests.java | 2 + .../snapshots/BlobStoreFormatTests.java | 7 + .../InternalSnapshotsInfoServiceTests.java | 2 +- .../snapshots/SnapshotResiliencyTests.java | 7 +- .../MockEventuallyConsistentRepository.java | 2 +- .../cluster/routing/TestShardRouting.java | 8 +- ...enSearchIndexLevelReplicationTestCase.java | 3 +- .../index/shard/IndexShardTestCase.java | 9 +- .../java/org/opensearch/node/MockNode.java | 27 +- .../AbstractThirdPartyRepositoryTestCase.java | 2 +- .../opensearch/test/BackgroundIndexer.java | 11 +- .../opensearch/test/InternalTestCluster.java | 14 +- 192 files changed, 8619 insertions(+), 808 deletions(-) create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/CopyObjectHelper.java create mode 100644 plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/CopyObjectMetadata.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/shardsplit/InPlaceShardSplitIT.java create mode 100644 server/src/internalClusterTest/java/org/opensearch/shardsplit/InPlaceShardSplitRemoteStoreIT.java create mode 100644 server/src/main/java/org/opensearch/action/PrimaryShardSplitException.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/split/InPlaceShardSplitAction.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/split/InPlaceShardSplitClusterStateUpdateRequest.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/split/InPlaceShardSplitRequest.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/split/InPlaceShardSplitResponse.java create mode 100644 server/src/main/java/org/opensearch/action/admin/indices/split/TransportInPlaceShardSplitAction.java create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/MetadataInPlaceShardSplitService.java create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/ShardRange.java create mode 100644 server/src/main/java/org/opensearch/cluster/metadata/SplitShardsMetadata.java create mode 100644 server/src/main/java/org/opensearch/cluster/routing/allocation/decider/InPlaceShardSplitAllocationDecider.java create mode 100644 server/src/main/java/org/opensearch/indices/recovery/inplacesplit/InPlaceShardRecoveryContext.java create mode 100644 server/src/main/java/org/opensearch/indices/recovery/inplacesplit/InPlaceShardSplitRecoveryListener.java create mode 100644 server/src/main/java/org/opensearch/indices/recovery/inplacesplit/InPlaceShardSplitRecoveryService.java create mode 100644 server/src/main/java/org/opensearch/indices/recovery/inplacesplit/InPlaceShardSplitRecoverySourceHandler.java create mode 100644 server/src/main/java/org/opensearch/indices/recovery/inplacesplit/InPlaceShardSplitRecoveryTargetHandler.java create mode 100644 server/src/main/java/org/opensearch/indices/recovery/inplacesplit/InPlaceShardSplitRemoteSourceHandler.java create mode 100644 server/src/main/java/org/opensearch/indices/recovery/inplacesplit/InPlaceShardSplitResponseHandler.java create mode 100644 server/src/main/java/org/opensearch/indices/recovery/inplacesplit/InPlaceShardsRecoveryFailedException.java create mode 100644 server/src/main/java/org/opensearch/indices/recovery/inplacesplit/SplitCommitMetadata.java create mode 100644 server/src/main/java/org/opensearch/rest/action/admin/indices/RestInPlaceShardSplitHandler.java diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index f090532aecfc2..f06f9e44615a1 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -1,6 +1,9 @@ - + \ No newline at end of file diff --git a/.idea/runConfigurations/Debug_OpenSearch.xml b/.idea/runConfigurations/Debug_OpenSearch.xml index 0d8bf59823acf..c18046f873477 100644 --- a/.idea/runConfigurations/Debug_OpenSearch.xml +++ b/.idea/runConfigurations/Debug_OpenSearch.xml @@ -6,6 +6,10 @@