From d785d2a09b33294056e36c7c68dc84dad6dedd1d Mon Sep 17 00:00:00 2001 From: Kirill Vasilenko Date: Tue, 6 Feb 2024 10:46:26 +0100 Subject: [PATCH 1/3] remove the index folder in the SearchMode.dropIndex(...) lack of this caused impossibility to create an index with the same name afterward --- .../server/IndexManagerServiceImpl.java | 8 ++-- .../vectoriadb/server/IndexManagerTest.java | 41 +++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/vectoriadb-server/src/main/java/jetbrains/vectoriadb/server/IndexManagerServiceImpl.java b/vectoriadb-server/src/main/java/jetbrains/vectoriadb/server/IndexManagerServiceImpl.java index 0e48528fe..204738a01 100644 --- a/vectoriadb-server/src/main/java/jetbrains/vectoriadb/server/IndexManagerServiceImpl.java +++ b/vectoriadb-server/src/main/java/jetbrains/vectoriadb/server/IndexManagerServiceImpl.java @@ -902,6 +902,8 @@ public void dropIndex(IndexManagerOuterClass.IndexNameRequest request, StreamObs @SuppressWarnings("resource") var indexReader = fetchIndexReader(indexName); indexReader.deleteIndex(); + var indexDir = indexMetadatas.get(indexName).dir; + FileUtils.deleteDirectory(indexDir.toFile()); //noinspection resource indexReaders.remove(indexName); indexStates.remove(indexName); @@ -917,7 +919,7 @@ public void dropIndex(IndexManagerOuterClass.IndexNameRequest request, StreamObs @NotNull private IndexReader fetchIndexReader(final String indexName) { - return indexReaders.computeIfAbsent(indexName, r -> { + return indexReaders.computeIfAbsent(indexName, _ -> { var metadata = indexMetadatas.get(indexName); return new IndexReader(indexName, dimensions, maxConnectionsPerVertex, maxCandidatesReturned, compressionRatio, metadata.dir, metadata.distance, diskCache); @@ -1029,7 +1031,7 @@ public void createIndex(IndexManagerOuterClass.CreateIndexRequest request, public void buildIndex(IndexManagerOuterClass.IndexNameRequest request, StreamObserver responseObserver) { try { var indexName = request.getIndexName(); - var indexState = indexStates.compute(indexName, (k, state) -> { + var indexState = indexStates.compute(indexName, (_, state) -> { if (state == IndexState.UPLOADED || state == IndexState.CREATED) { return IndexState.IN_BUILD_QUEUE; } else { @@ -1295,7 +1297,7 @@ public void dropIndex(IndexManagerOuterClass.IndexNameRequest request, StreamObs try { var indexName = request.getIndexName(); - indexStates.compute(indexName, (k, state) -> { + indexStates.compute(indexName, (_, state) -> { if (state == IndexState.CREATED || state == IndexState.BUILT || state == IndexState.UPLOADED) { return IndexState.BROKEN; } else { diff --git a/vectoriadb-server/src/test/java/jetbrains/vectoriadb/server/IndexManagerTest.java b/vectoriadb-server/src/test/java/jetbrains/vectoriadb/server/IndexManagerTest.java index ffb2ad6fb..81dd97c1f 100644 --- a/vectoriadb-server/src/test/java/jetbrains/vectoriadb/server/IndexManagerTest.java +++ b/vectoriadb-server/src/test/java/jetbrains/vectoriadb/server/IndexManagerTest.java @@ -156,7 +156,7 @@ public void testListIndexes() throws Exception { indexes = listIndexes(indexManagerService); Assert.assertEquals(1, indexes.size()); - Assert.assertEquals(indexName + 1, indexes.get(0)); + Assert.assertEquals(indexName + 1, indexes.getFirst()); createIndex(indexName + 2, indexManagerService, IndexManagerOuterClass.Distance.L2); indexes = listIndexes(indexManagerService); @@ -247,6 +247,26 @@ public void testDropSearchIndex() throws Exception { }); } + @Test + public void testDropIndexInSearchModeAndCreateItAgain() throws Exception { + var indexOne = "testDropSearchIndexOne"; + + executeInServiceContext(indexManagerService -> { + generateIndex(indexOne, L2DistanceFunction.INSTANCE, 64, 1, indexManagerService); + + switchToSearchMode(indexManagerService); + + dropIndex(indexOne, indexManagerService); + + var indexes = listIndexes(indexManagerService); + Assert.assertEquals(0, indexes.size()); + + switchToBuildMode(indexManagerService); + + generateIndex(indexOne, L2DistanceFunction.INSTANCE, 64, 1, indexManagerService); + }); + } + @Test public void testTwoIndexesSimultaniously() throws Exception { var buildDir = System.getProperty("exodus.tests.buildDirectory"); @@ -471,7 +491,7 @@ private static List listIndexes(IndexManagerServiceImpl indexManagerServ indexManagerService.listIndexes(Empty.newBuilder().build(), listIndexesRecorder); checkCompleteness(listIndexesRecorder); - return listIndexesRecorder.getValues().get(0).getIndexNamesList(); + return listIndexesRecorder.getValues().getFirst().getIndexNamesList(); } @@ -492,7 +512,7 @@ private static void buildIndex(String indexName, IndexManagerServiceImpl indexMa checkCompleteness(indexStateRecorder); - var response = indexStateRecorder.getValues().get(0); + var response = indexStateRecorder.getValues().getFirst(); var indexState = response.getState(); if (indexState == IndexManagerOuterClass.IndexState.BUILDING || indexState == IndexManagerOuterClass.IndexState.IN_BUILD_QUEUE) { @@ -531,6 +551,13 @@ private static void switchToSearchMode(IndexManagerServiceImpl indexManagerServi checkCompleteness(switchToSearchModeRecorder); } + private static void switchToBuildMode(IndexManagerServiceImpl indexManagerService) throws Exception { + var streamRecorder = StreamRecorder.create(); + indexManagerService.switchToBuildMode(Empty.newBuilder().build(), streamRecorder); + + checkCompleteness(streamRecorder); + } + private static byte[][] findNearestNeighbours(IndexManagerServiceImpl indexManagerService, float[] queryVector, String indexName, int k) throws Exception { var findNearestVectorsRecorder = StreamRecorder.create(); @@ -546,7 +573,7 @@ private static byte[][] findNearestNeighbours(IndexManagerServiceImpl indexManag checkCompleteness(findNearestVectorsRecorder); - var response = findNearestVectorsRecorder.getValues().get(0); + var response = findNearestVectorsRecorder.getValues().getFirst(); var nearestVectors = response.getIdsList(); var result = new byte[nearestVectors.size()][]; @@ -674,9 +701,9 @@ private static void searchNeighbours(String indexName, int vectorsCount, int vec @NotNull private static IndexManagerOuterClass.Distance convertDistanceFunction(DistanceFunction distanceFunction) { return switch (distanceFunction) { - case L2DistanceFunction l2DistanceFunction -> IndexManagerOuterClass.Distance.L2; - case DotDistanceFunction dotDistanceFunction -> IndexManagerOuterClass.Distance.DOT; - case CosineDistanceFunction cosineDistanceFunction -> IndexManagerOuterClass.Distance.COSINE; + case L2DistanceFunction _ -> IndexManagerOuterClass.Distance.L2; + case DotDistanceFunction _ -> IndexManagerOuterClass.Distance.DOT; + case CosineDistanceFunction _ -> IndexManagerOuterClass.Distance.COSINE; case null, default -> throw new IllegalArgumentException("Unknown distance function " + distanceFunction); }; } From 00e439c71bb76f2b018c4d43e7d7839aa98b7bd5 Mon Sep 17 00:00:00 2001 From: Kirill Vasilenko Date: Tue, 6 Feb 2024 10:53:29 +0100 Subject: [PATCH 2/3] fix handling the case when the context is cancelled there is no sense to do anything further, so add the return operator at the end of the if listener should be removed before we close responseObserver, otherwise on any progress notification we will have an exception that makes the whole server not operable --- .../jetbrains/vectoriadb/server/IndexManagerServiceImpl.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vectoriadb-server/src/main/java/jetbrains/vectoriadb/server/IndexManagerServiceImpl.java b/vectoriadb-server/src/main/java/jetbrains/vectoriadb/server/IndexManagerServiceImpl.java index 204738a01..849192304 100644 --- a/vectoriadb-server/src/main/java/jetbrains/vectoriadb/server/IndexManagerServiceImpl.java +++ b/vectoriadb-server/src/main/java/jetbrains/vectoriadb/server/IndexManagerServiceImpl.java @@ -1367,11 +1367,13 @@ public ServiceIndexBuildProgressListener(StreamObserver Date: Tue, 6 Feb 2024 16:25:50 +0100 Subject: [PATCH 3/3] complete requestObserver even if there is no progressIndicator --- .../java/jetbrains/vectoriadb/client/VectoriaDBClient.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/vectoriadb-java-client/src/main/java/jetbrains/vectoriadb/client/VectoriaDBClient.java b/vectoriadb-java-client/src/main/java/jetbrains/vectoriadb/client/VectoriaDBClient.java index 6a9c3bd2d..8c8c860b6 100644 --- a/vectoriadb-java-client/src/main/java/jetbrains/vectoriadb/client/VectoriaDBClient.java +++ b/vectoriadb-java-client/src/main/java/jetbrains/vectoriadb/client/VectoriaDBClient.java @@ -202,7 +202,10 @@ public void onCompleted() { progressIndicator.accept(-1, -1); requestObserver.onCompleted(); progressIndicator.accept(Integer.MAX_VALUE, Integer.MAX_VALUE); + } else { + requestObserver.onCompleted(); } + try { finishedLatch.await(); } catch (InterruptedException e) {