Merge branch 'main' into dependabot/gradle/libs/grok/org.jruby.jcodin…

…gs-jcodings-1.0.61 Signed-off-by: Craig Perkins <[email protected]>
opensearch-project · Jan 21, 2025 · f6490a7 · f6490a7
2 parents 845fb3f + 827aa63
commit f6490a7
Show file tree

Hide file tree

Showing 11 changed files with 281 additions and 112 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -66,6 +66,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `opentelemetry-semconv` from 1.27.0-alpha to 1.29.0-alpha ([#16700](https://github.com/opensearch-project/OpenSearch/pull/16700))
 - Bump `com.google.re2j:re2j` from 1.7 to 1.8 ([#17012](https://github.com/opensearch-project/OpenSearch/pull/17012))
 - Bump `com.squareup.okio:okio` from 3.9.1 to 3.10.2 ([#17060](https://github.com/opensearch-project/OpenSearch/pull/17060))
+- Bump `com.diffplug.spotless` from 6.25.0 to 7.0.2 ([#17058](https://github.com/opensearch-project/OpenSearch/pull/17058))
 - Bump `org.jruby.jcodings:jcodings` from 1.0.58 to 1.0.61 ([#17061](https://github.com/opensearch-project/OpenSearch/pull/17061))
 
 ### Changed

diff --git a/build.gradle b/build.gradle
@@ -54,7 +54,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.25.0" apply false
+  id "com.diffplug.spotless" version "7.0.2" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }

diff --git a/...src/test/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessorTests.java b/...src/test/java/org/opensearch/common/annotation/processor/ApiAnnotationProcessorTests.java
diff --git a/libs/common/src/test/java/org/opensearch/common/annotation/processor/CompilerSupport.java b/libs/common/src/test/java/org/opensearch/common/annotation/processor/CompilerSupport.java
@@ -29,6 +29,7 @@
 import java.net.URI;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
 import java.security.AccessController;
 import java.security.PrivilegedAction;
 import java.util.Arrays;
@@ -39,12 +40,12 @@
 import java.util.stream.Stream;
 
 interface CompilerSupport {
-    default CompilerResult compile(String name, String... names) {
-        return compileWithPackage(ApiAnnotationProcessorTests.class.getPackageName(), name, names);
+    default CompilerResult compile(Path outputDirectory, String name, String... names) {
+        return compileWithPackage(outputDirectory, ApiAnnotationProcessorTests.class.getPackageName(), name, names);
     }
 
     @SuppressWarnings("removal")
-    default CompilerResult compileWithPackage(String pck, String name, String... names) {
+    default CompilerResult compileWithPackage(Path outputDirectory, String pck, String name, String... names) {
         final JavaCompiler compiler = ToolProvider.getSystemJavaCompiler();
         final DiagnosticCollector<JavaFileObject> collector = new DiagnosticCollector<>();
 
@@ -54,11 +55,18 @@ default CompilerResult compileWithPackage(String pck, String name, String... nam
                 .map(f -> asSource(pck, f))
                 .collect(Collectors.toList());
 
-            final CompilationTask task = compiler.getTask(out, fileManager, collector, null, null, files);
+            final CompilationTask task = compiler.getTask(
+                out,
+                fileManager,
+                collector,
+                List.of("-d", outputDirectory.toString()),
+                null,
+                files
+            );
             task.setProcessors(Collections.singleton(new ApiAnnotationProcessor()));
 
             if (AccessController.doPrivileged((PrivilegedAction<Boolean>) () -> task.call())) {
-                return new Success();
+                return new Success(collector.getDiagnostics());
             } else {
                 return new Failure(collector.getDiagnostics());
             }
@@ -81,16 +89,10 @@ public CharSequence getCharContent(boolean ignoreEncodingErrors) throws IOExcept
         };
     }
 
-    class CompilerResult {}
-
-    class Success extends CompilerResult {
-
-    }
-
-    class Failure extends CompilerResult {
+    class CompilerResult {
         private final List<Diagnostic<? extends JavaFileObject>> diagnotics;
 
-        Failure(List<Diagnostic<? extends JavaFileObject>> diagnotics) {
+        CompilerResult(List<Diagnostic<? extends JavaFileObject>> diagnotics) {
             this.diagnotics = diagnotics;
         }
 
@@ -99,6 +101,18 @@ List<Diagnostic<? extends JavaFileObject>> diagnotics() {
         }
     }
 
+    class Success extends CompilerResult {
+        Success(List<Diagnostic<? extends JavaFileObject>> diagnotics) {
+            super(diagnotics);
+        }
+    }
+
+    class Failure extends CompilerResult {
+        Failure(List<Diagnostic<? extends JavaFileObject>> diagnotics) {
+            super(diagnotics);
+        }
+    }
+
     class HasDiagnostic extends TypeSafeMatcher<Diagnostic<? extends JavaFileObject>> {
         private final Diagnostic.Kind kind;
         private final Matcher<String> matcher;

diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
@@ -206,7 +206,8 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans
         } catch (Exception ex) {
             logger.error(() -> new ParameterizedMessage("Transfer failed for snapshot {}", transferSnapshot), ex);
             captureStatsOnUploadFailure();
-            translogTransferListener.onUploadFailed(transferSnapshot, ex);
+            Exception exWithoutSuppressed = new TranslogUploadFailedException(ex.getMessage());
+            translogTransferListener.onUploadFailed(transferSnapshot, exWithoutSuppressed);
             return false;
         }
     }

diff --git a/...r/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java b/...r/src/test/java/org/opensearch/cluster/routing/remote/RemoteRoutingTableServiceTests.java
@@ -800,9 +800,7 @@ public void testDeleteStaleIndexRoutingPathsThrowsIOException() throws IOExcepti
         doThrow(new IOException("test exception")).when(blobContainer).deleteBlobsIgnoringIfNotExists(Mockito.anyList());
 
         remoteRoutingTableService.doStart();
-        IOException thrown = assertThrows(IOException.class, () -> {
-            remoteRoutingTableService.deleteStaleIndexRoutingPaths(stalePaths);
-        });
+        IOException thrown = assertThrows(IOException.class, () -> { remoteRoutingTableService.deleteStaleIndexRoutingPaths(stalePaths); });
         assertEquals("test exception", thrown.getMessage());
         verify(blobContainer).deleteBlobsIgnoringIfNotExists(stalePaths);
     }
@@ -823,9 +821,10 @@ public void testDeleteStaleIndexRoutingDiffPathsThrowsIOException() throws IOExc
         doThrow(new IOException("test exception")).when(blobContainer).deleteBlobsIgnoringIfNotExists(Mockito.anyList());
 
         remoteRoutingTableService.doStart();
-        IOException thrown = assertThrows(IOException.class, () -> {
-            remoteRoutingTableService.deleteStaleIndexRoutingDiffPaths(stalePaths);
-        });
+        IOException thrown = assertThrows(
+            IOException.class,
+            () -> { remoteRoutingTableService.deleteStaleIndexRoutingDiffPaths(stalePaths); }
+        );
         assertEquals("test exception", thrown.getMessage());
         verify(blobContainer).deleteBlobsIgnoringIfNotExists(stalePaths);
     }

diff --git a/.../org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java b/.../org/opensearch/index/compositeindex/datacube/startree/builder/StarTreesBuilderTests.java
@@ -104,16 +104,32 @@ public void test_buildWithNoStarTreeFields() throws IOException {
     public void test_getStarTreeBuilder() throws IOException {
         when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType));
         StarTreesBuilder starTreesBuilder = new StarTreesBuilder(segmentWriteState, mapperService, new AtomicInteger());
-        StarTreeBuilder starTreeBuilder = starTreesBuilder.getStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService);
+        StarTreeBuilder starTreeBuilder = starTreesBuilder.getStarTreeBuilder(
+            metaOut,
+            dataOut,
+            starTreeField,
+            segmentWriteState,
+            mapperService
+        );
         assertTrue(starTreeBuilder instanceof OnHeapStarTreeBuilder);
     }
 
     public void test_getStarTreeBuilder_illegalArgument() throws IOException {
         when(mapperService.getCompositeFieldTypes()).thenReturn(Set.of(starTreeFieldType));
-        StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration(1, new HashSet<>(), StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP);
+        StarTreeFieldConfiguration starTreeFieldConfiguration = new StarTreeFieldConfiguration(
+            1,
+            new HashSet<>(),
+            StarTreeFieldConfiguration.StarTreeBuildMode.OFF_HEAP
+        );
         StarTreeField starTreeField = new StarTreeField("star_tree", new ArrayList<>(), new ArrayList<>(), starTreeFieldConfiguration);
         StarTreesBuilder starTreesBuilder = new StarTreesBuilder(segmentWriteState, mapperService, new AtomicInteger());
-        StarTreeBuilder starTreeBuilder = starTreesBuilder.getStarTreeBuilder(metaOut, dataOut, starTreeField, segmentWriteState, mapperService);
+        StarTreeBuilder starTreeBuilder = starTreesBuilder.getStarTreeBuilder(
+            metaOut,
+            dataOut,
+            starTreeField,
+            segmentWriteState,
+            mapperService
+        );
         assertTrue(starTreeBuilder instanceof OffHeapStarTreeBuilder);
         starTreeBuilder.close();
     }

diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
@@ -135,9 +135,12 @@ public void testGetPrimaryTermGenerationUuid() {
     }
 
     public void testInitException() throws IOException {
-        when(remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, METADATA_FILES_TO_FETCH)).thenThrow(
-            new IOException("Error")
-        );
+        when(
+            remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
+                RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
+                METADATA_FILES_TO_FETCH
+            )
+        ).thenThrow(new IOException("Error"));
 
         assertThrows(IOException.class, () -> remoteSegmentStoreDirectory.init());
     }
@@ -155,9 +158,12 @@ public void testInitNoMetadataFile() throws IOException {
     }
 
     public void testInitMultipleMetadataFile() throws IOException {
-        when(remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX, METADATA_FILES_TO_FETCH)).thenReturn(
-            List.of(metadataFilename, metadataFilenameDup)
-        );
+        when(
+            remoteMetadataDirectory.listFilesByPrefixInLexicographicOrder(
+                RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX,
+                METADATA_FILES_TO_FETCH
+            )
+        ).thenReturn(List.of(metadataFilename, metadataFilenameDup));
         assertThrows(IllegalStateException.class, () -> remoteSegmentStoreDirectory.init());
     }
 

diff --git a/.../java/org/opensearch/index/store/RemoteSegmentStoreDirectoryWithPinnedTimestampTests.java b/.../java/org/opensearch/index/store/RemoteSegmentStoreDirectoryWithPinnedTimestampTests.java
@@ -187,7 +187,8 @@ public void testDeleteStaleCommitsPinnedTimestampMdFile() throws Exception {
             )
         ).thenReturn(List.of(metadataFilename, metadataFilename2, metadataFilename3));
 
-        long pinnedTimestampMatchingMetadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getTimestamp(metadataFilename2) + 10;
+        long pinnedTimestampMatchingMetadataFilename2 = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getTimestamp(metadataFilename2)
+            + 10;
         String blobName = "snapshot1__" + pinnedTimestampMatchingMetadataFilename2;
         when(blobContainer.listBlobs()).thenReturn(Map.of(blobName, new PlainBlobMetadata(blobName, 100)));
 

diff --git a/...er/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/...er/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
@@ -206,6 +206,80 @@ public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) {
         assertEquals(4, fileTransferTracker.allUploaded().size());
     }
 
+    public void testTransferSnapshotOnFileTransferUploadFail() throws Exception {
+        AtomicInteger fileTransferSucceeded = new AtomicInteger();
+        AtomicInteger fileTransferFailed = new AtomicInteger();
+        AtomicInteger translogTransferSucceeded = new AtomicInteger();
+        AtomicInteger translogTransferFailed = new AtomicInteger();
+
+        doAnswer(invocationOnMock -> {
+            ActionListener<TransferFileSnapshot> listener = (ActionListener<TransferFileSnapshot>) invocationOnMock.getArguments()[2];
+            Set<TransferFileSnapshot> transferFileSnapshots = (Set<TransferFileSnapshot>) invocationOnMock.getArguments()[0];
+
+            TransferFileSnapshot actualFileSnapshot = transferFileSnapshots.iterator().next();
+            FileTransferException testException = new FileTransferException(
+                actualFileSnapshot,
+                new RuntimeException("FileTransferUploadNeedsToFail-Exception")
+            );
+
+            listener.onFailure(testException);
+            transferFileSnapshots.stream().skip(1).forEach(listener::onResponse);
+            return null;
+        }).when(transferService).uploadBlobs(anySet(), anyMap(), any(ActionListener.class), any(WritePriority.class));
+
+        FileTransferTracker fileTransferTracker = new FileTransferTracker(
+            new ShardId("index", "indexUUid", 0),
+            remoteTranslogTransferTracker
+        ) {
+            @Override
+            public void onSuccess(TransferFileSnapshot fileSnapshot) {
+                fileTransferSucceeded.incrementAndGet();
+                super.onSuccess(fileSnapshot);
+            }
+
+            @Override
+            public void onFailure(TransferFileSnapshot fileSnapshot, Exception e) {
+                fileTransferFailed.incrementAndGet();
+                super.onFailure(fileSnapshot, e);
+            }
+        };
+
+        TranslogTransferManager translogTransferManager = new TranslogTransferManager(
+            shardId,
+            transferService,
+            remoteBaseTransferPath.add(TRANSLOG.getName()),
+            remoteBaseTransferPath.add(METADATA.getName()),
+            fileTransferTracker,
+            remoteTranslogTransferTracker,
+            DefaultRemoteStoreSettings.INSTANCE,
+            isTranslogMetadataEnabled
+        );
+
+        SetOnce<Exception> exception = new SetOnce<>();
+        assertFalse(translogTransferManager.transferSnapshot(createTransferSnapshot(), new TranslogTransferListener() {
+            @Override
+            public void onUploadComplete(TransferSnapshot transferSnapshot) {
+                translogTransferSucceeded.incrementAndGet();
+            }
+
+            @Override
+            public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) {
+                translogTransferFailed.incrementAndGet();
+                exception.set(ex);
+            }
+        }));
+
+        assertNotNull(exception.get());
+        assertTrue(exception.get() instanceof TranslogUploadFailedException);
+        assertEquals("Failed to upload 1 files during transfer", exception.get().getMessage());
+        assertEquals(0, exception.get().getSuppressed().length);
+        assertEquals(3, fileTransferSucceeded.get());
+        assertEquals(1, fileTransferFailed.get());
+        assertEquals(0, translogTransferSucceeded.get());
+        assertEquals(1, translogTransferFailed.get());
+        assertEquals(3, fileTransferTracker.allUploaded().size());
+    }
+
     public void testTransferSnapshotOnUploadTimeout() throws Exception {
         doAnswer(invocationOnMock -> {
             Set<TransferFileSnapshot> transferFileSnapshots = invocationOnMock.getArgument(0);

diff --git a/...r/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java b/...r/src/test/java/org/opensearch/indices/replication/RemoteStoreReplicationSourceTests.java
@@ -166,8 +166,12 @@ private void buildIndexShardBehavior(IndexShard mockShard, IndexShard indexShard
         when(mockShard.getSegmentInfosSnapshot()).thenReturn(indexShard.getSegmentInfosSnapshot());
         Store remoteStore = mock(Store.class);
         when(mockShard.remoteStore()).thenReturn(remoteStore);
-        RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) indexShard.remoteStore().directory()).getDelegate()).getDelegate();
-        FilterDirectory remoteStoreFilterDirectory = new RemoteStoreRefreshListenerTests.TestFilterDirectory(new RemoteStoreRefreshListenerTests.TestFilterDirectory(remoteSegmentStoreDirectory));
+        RemoteSegmentStoreDirectory remoteSegmentStoreDirectory =
+            (RemoteSegmentStoreDirectory) ((FilterDirectory) ((FilterDirectory) indexShard.remoteStore().directory()).getDelegate())
+                .getDelegate();
+        FilterDirectory remoteStoreFilterDirectory = new RemoteStoreRefreshListenerTests.TestFilterDirectory(
+            new RemoteStoreRefreshListenerTests.TestFilterDirectory(remoteSegmentStoreDirectory)
+        );
         when(remoteStore.directory()).thenReturn(remoteStoreFilterDirectory);
     }
 }