updatedPages, final boolean async
protected void flushPage(final MutablePage page) throws IOException {
if (fileManager.existsFile(page.pageId.getFileId())) {
- final PaginatedFile file = fileManager.getFile(page.pageId.getFileId());
+ final PaginatedComponentFile file = (PaginatedComponentFile) fileManager.getFile(page.pageId.getFileId());
if (!file.isOpen())
throw new DatabaseMetadataException("Cannot flush pages on disk because file '" + file.getFileName() + "' is closed");
@@ -322,7 +322,7 @@ protected void flushPage(final MutablePage page) throws IOException {
}
private CachedPage loadPage(final PageId pageId, final int size, final boolean createIfNotExists, final boolean cache) throws IOException {
- final PaginatedFile file = fileManager.getFile(pageId.getFileId());
+ final PaginatedComponentFile file = (PaginatedComponentFile) fileManager.getFile(pageId.getFileId());
final boolean isNewPage = pageId.getPageNumber() >= file.getTotalPages();
if (!createIfNotExists && isNewPage)
diff --git a/engine/src/main/java/com/arcadedb/engine/PaginatedComponent.java b/engine/src/main/java/com/arcadedb/engine/PaginatedComponent.java
index 16d27e671..3db1053f6 100644
--- a/engine/src/main/java/com/arcadedb/engine/PaginatedComponent.java
+++ b/engine/src/main/java/com/arcadedb/engine/PaginatedComponent.java
@@ -26,43 +26,35 @@
import java.util.concurrent.atomic.*;
/**
+ * Extends a FileComponent by supporting pages.
+ *
* HEADER = [recordCount(int:4)] CONTENT-PAGES = [version(long:8),recordCountInPage(short:2),recordOffsetsInPage(512*ushort=2048)]
+ *
+ * @author Luca Garulli (l.garulli@arcadedata.com)
*/
-public abstract class PaginatedComponent {
- protected final DatabaseInternal database;
- protected final String name;
- protected final PaginatedFile file;
- protected final int id;
- protected final int pageSize;
- protected final int version;
- protected final AtomicInteger pageCount = new AtomicInteger();
+public abstract class PaginatedComponent extends Component {
+ protected final PaginatedComponentFile file;
+ protected final int pageSize;
+ protected final AtomicInteger pageCount = new AtomicInteger();
- protected PaginatedComponent(final DatabaseInternal database, final String name, final String filePath, final String ext, final PaginatedFile.MODE mode,
+ protected PaginatedComponent(final DatabaseInternal database, final String name, final String filePath, final String ext, final ComponentFile.MODE mode,
final int pageSize, final int version) throws IOException {
this(database, name, filePath, ext, database.getFileManager().newFileId(), mode, pageSize, version);
}
private PaginatedComponent(final DatabaseInternal database, final String name, final String filePath, final String ext, final int id,
- final PaginatedFile.MODE mode, final int pageSize, final int version) throws IOException {
+ final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException {
this(database, name, filePath + "." + id + "." + pageSize + ".v" + version + "." + ext, id, mode, pageSize, version);
}
- protected PaginatedComponent(final DatabaseInternal database, final String name, final String filePath, final int id, final PaginatedFile.MODE mode,
+ protected PaginatedComponent(final DatabaseInternal database, final String name, final String filePath, final int id, final ComponentFile.MODE mode,
final int pageSize, final int version) throws IOException {
+ super(database, name, id, version, filePath);
if (pageSize <= 0)
throw new IllegalArgumentException("Invalid page size " + pageSize);
- if (id < 0)
- throw new IllegalArgumentException("Invalid file id " + id);
- if (name == null || name.isEmpty())
- throw new IllegalArgumentException("Invalid file name " + name);
- this.database = database;
- this.name = name;
- this.id = id;
this.pageSize = pageSize;
- this.version = version;
-
- this.file = database.getFileManager().getOrCreateFile(name, filePath, mode);
+ this.file = (PaginatedComponentFile) database.getFileManager().getOrCreateFile(name, filePath, mode);
if (file.getSize() == 0)
// NEW FILE, CREATE HEADER PAGE
@@ -71,16 +63,12 @@ protected PaginatedComponent(final DatabaseInternal database, final String name,
pageCount.set((int) (file.getSize() / getPageSize()));
}
- public File getOSFile() {
- return file.getOSFile();
+ public PaginatedComponentFile getComponentFile() {
+ return file;
}
- public void onAfterLoad() {
- // NO ACTIONS
- }
-
- public void onAfterCommit() {
- // NO ACTIONS
+ public File getOSFile() {
+ return file.getOSFile();
}
public int getPageSize() {
@@ -89,26 +77,11 @@ public int getPageSize() {
public void setPageCount(final int value) {
if (value <= pageCount.get())
- throw new ConcurrentModificationException("Unable to update page count for component '" + name + "' (" + value + "<=" + pageCount.get() + ")");
+ throw new ConcurrentModificationException("Unable to update page count for component '" + componentName + "' (" + value + "<=" + pageCount.get() + ")");
pageCount.set(value);
}
- public String getName() {
- return name;
- }
-
- public int getId() {
- return id;
- }
-
- public int getVersion() {
- return version;
- }
-
- public DatabaseInternal getDatabase() {
- return database;
- }
-
+ @Override
public void close() {
if (file != null)
file.close();
@@ -117,14 +90,10 @@ public void close() {
public int getTotalPages() {
final TransactionContext tx = database.getTransaction();
if (tx != null) {
- final Integer txPageCounter = tx.getPageCounter(id);
+ final Integer txPageCounter = tx.getPageCounter(fileId);
if (txPageCounter != null)
return txPageCounter;
}
return pageCount.get();
}
-
- public Object getMainComponent() {
- return this;
- }
}
diff --git a/engine/src/main/java/com/arcadedb/engine/PaginatedFile.java b/engine/src/main/java/com/arcadedb/engine/PaginatedComponentFile.java
similarity index 73%
rename from engine/src/main/java/com/arcadedb/engine/PaginatedFile.java
rename to engine/src/main/java/com/arcadedb/engine/PaginatedComponentFile.java
index 5ca726399..7c2efa544 100644
--- a/engine/src/main/java/com/arcadedb/engine/PaginatedFile.java
+++ b/engine/src/main/java/com/arcadedb/engine/PaginatedComponentFile.java
@@ -23,37 +23,23 @@
import java.io.*;
import java.nio.*;
import java.nio.channels.*;
-import java.nio.file.*;
import java.util.logging.*;
import java.util.zip.*;
-public class PaginatedFile {
- public enum MODE {
- READ_ONLY, READ_WRITE
- }
+public class PaginatedComponentFile extends ComponentFile {
+
+ private RandomAccessFile file;
+ private FileChannel channel;
+ private int pageSize;
- private RandomAccessFile file;
- private final MODE mode;
- private String filePath;
- private String fileName;
- private File osFile;
- private FileChannel channel;
- private int fileId;
- private int pageSize;
- private int version = 0; // STARTING FROM 21.10.2 COMPONENTS HAVE VERSION IN THE FILE NAME
- private String componentName;
- private String fileExtension;
- private boolean open;
-
- public PaginatedFile() {
- this.mode = MODE.READ_ONLY;
+ public PaginatedComponentFile() {
}
- protected PaginatedFile(final String filePath, final MODE mode) throws FileNotFoundException {
- this.mode = mode;
- open(filePath, mode);
+ protected PaginatedComponentFile(final String filePath, final MODE mode) throws FileNotFoundException {
+ super(filePath, mode);
}
+ @Override
public void close() {
try {
LogManager.instance().log(this, Level.FINE, "Closing file %s (id=%d)...", null, filePath, fileId);
@@ -74,34 +60,18 @@ public void close() {
this.open = false;
}
- public void rename(final String newFileName) throws IOException {
- close();
- LogManager.instance().log(this, Level.FINE, "Renaming file %s (id=%d) to %s...", null, filePath, fileId, newFileName);
- final File newFile = new File(newFileName);
- new File(filePath).renameTo(newFile);
- open(newFile.getAbsolutePath(), mode);
- }
-
- public void drop() throws IOException {
- close();
- LogManager.instance().log(this, Level.FINE, "Deleting file %s (id=%d) to %s...", null, filePath, fileId);
- Files.delete(Paths.get(getFilePath()));
- }
-
+ @Override
public long getSize() throws IOException {
return channel.size();
}
- public long getTotalPages() throws IOException {
- return channel.size() / pageSize;
- }
-
+ @Override
public void flush() throws IOException {
channel.force(true);
}
- public String getFileName() {
- return fileName;
+ public long getTotalPages() throws IOException {
+ return channel.size() / pageSize;
}
public long calculateChecksum() throws IOException {
@@ -187,52 +157,17 @@ public void read(final CachedPage page) throws IOException {
}
}
- public boolean isOpen() {
- return open;
- }
-
- public String getFilePath() {
- return filePath;
- }
-
- public String getComponentName() {
- return componentName;
- }
-
- public String getFileExtension() {
- return fileExtension;
- }
-
- public int getFileId() {
- return fileId;
- }
-
- public File getOSFile() {
- return osFile;
- }
-
- public void setFileId(final int fileId) {
- this.fileId = fileId;
- }
-
public int getPageSize() {
return pageSize;
}
- public int getVersion() {
- return version;
- }
-
@Override
- public String toString() {
- return filePath;
- }
-
- private void open(final String filePath, final MODE mode) throws FileNotFoundException {
+ protected void open(final String filePath, final MODE mode) throws FileNotFoundException {
this.filePath = filePath;
- String filePrefix = filePath.substring(0, filePath.lastIndexOf("."));
- this.fileExtension = filePath.substring(filePath.lastIndexOf(".") + 1);
+ final int lastDotPos = filePath.lastIndexOf(".");
+ String filePrefix = filePath.substring(0, lastDotPos);
+ this.fileExtension = filePath.substring(lastDotPos + 1);
final int versionPos = filePrefix.lastIndexOf(".");
if (filePrefix.charAt(versionPos + 1) == 'v') {
diff --git a/engine/src/main/java/com/arcadedb/engine/TransactionManager.java b/engine/src/main/java/com/arcadedb/engine/TransactionManager.java
index f26282e86..803ac4c96 100644
--- a/engine/src/main/java/com/arcadedb/engine/TransactionManager.java
+++ b/engine/src/main/java/com/arcadedb/engine/TransactionManager.java
@@ -54,7 +54,7 @@ public TransactionManager(final DatabaseInternal database) {
this.logContext = LogManager.instance().getContext();
- if (database.getMode() == PaginatedFile.MODE.READ_WRITE) {
+ if (database.getMode() == ComponentFile.MODE.READ_WRITE) {
createWALFilePool();
task = new Timer("ArcadeDB TransactionManager " + database.getName());
@@ -276,7 +276,7 @@ public boolean applyChanges(final WALFile.WALTransaction tx, final boolean ignor
LogManager.instance().log(this, Level.FINE, "- applying changes from txId=%d", null, tx.txId);
for (final WALFile.WALPage txPage : tx.pages) {
- final PaginatedFile file;
+ final PaginatedComponentFile file;
final PageId pageId = new PageId(txPage.fileId, txPage.pageNumber);
@@ -289,7 +289,7 @@ public boolean applyChanges(final WALFile.WALTransaction tx, final boolean ignor
}
try {
- file = database.getFileManager().getFile(txPage.fileId);
+ file = (PaginatedComponentFile) database.getFileManager().getFile(txPage.fileId);
} catch (final Exception e) {
LogManager.instance().log(this, Level.SEVERE, "Error on applying tx changes for page %s", e, txPage);
throw e;
@@ -339,7 +339,7 @@ public boolean applyChanges(final WALFile.WALTransaction tx, final boolean ignor
database.getPageManager().removePageFromCache(modifiedPage.pageId);
- final PaginatedComponent component = database.getSchema().getFileById(txPage.fileId);
+ final PaginatedComponent component = (PaginatedComponent) database.getSchema().getFileById(txPage.fileId);
if (component != null) {
final int newPageCount = (int) (file.getSize() / file.getPageSize());
if (newPageCount > component.pageCount.get())
diff --git a/engine/src/main/java/com/arcadedb/graph/EdgeIteratorFilter.java b/engine/src/main/java/com/arcadedb/graph/EdgeIteratorFilter.java
index 8ab50f7b5..b3bc67eb5 100644
--- a/engine/src/main/java/com/arcadedb/graph/EdgeIteratorFilter.java
+++ b/engine/src/main/java/com/arcadedb/graph/EdgeIteratorFilter.java
@@ -20,7 +20,7 @@
import com.arcadedb.database.DatabaseInternal;
import com.arcadedb.database.RID;
-import com.arcadedb.engine.PaginatedFile;
+import com.arcadedb.engine.ComponentFile;
import com.arcadedb.exception.RecordNotFoundException;
import com.arcadedb.exception.SchemaException;
import com.arcadedb.log.LogManager;
@@ -87,7 +87,7 @@ public Edge next() {
@Override
protected void handleCorruption(final Exception e, final RID edge, final RID vertex) {
if ((e instanceof RecordNotFoundException || e instanceof SchemaException) &&//
- database.getMode() == PaginatedFile.MODE.READ_WRITE) {
+ database.getMode() == ComponentFile.MODE.READ_WRITE) {
LogManager.instance().log(this, Level.WARNING, "Error on loading edge %s %s. Fixing it...", e, edge, vertex != null ? "vertex " + vertex : "");
diff --git a/engine/src/main/java/com/arcadedb/graph/GraphEngine.java b/engine/src/main/java/com/arcadedb/graph/GraphEngine.java
index 0a5104715..a69771a85 100644
--- a/engine/src/main/java/com/arcadedb/graph/GraphEngine.java
+++ b/engine/src/main/java/com/arcadedb/graph/GraphEngine.java
@@ -558,7 +558,7 @@ public boolean isVertexConnectedTo(final VertexInternal vertex, final Identifiab
if (edgeType == null)
throw new IllegalArgumentException("Edge type is null");
- final int[] bucketFilter = vertex.getDatabase().getSchema().getType(edgeType).getBuckets(true).stream().mapToInt(x -> x.getId()).toArray();
+ final int[] bucketFilter = vertex.getDatabase().getSchema().getType(edgeType).getBuckets(true).stream().mapToInt(x -> x.getFileId()).toArray();
if (direction == Vertex.DIRECTION.OUT || direction == Vertex.DIRECTION.BOTH) {
final EdgeLinkedList outEdges = getEdgeHeadChunk(vertex, Vertex.DIRECTION.OUT);
diff --git a/engine/src/main/java/com/arcadedb/index/IndexFactory.java b/engine/src/main/java/com/arcadedb/index/IndexFactory.java
index 1d19ca461..b39b3d40f 100644
--- a/engine/src/main/java/com/arcadedb/index/IndexFactory.java
+++ b/engine/src/main/java/com/arcadedb/index/IndexFactory.java
@@ -18,10 +18,7 @@
*/
package com.arcadedb.index;
-import com.arcadedb.database.DatabaseInternal;
-import com.arcadedb.engine.PaginatedFile;
-import com.arcadedb.index.lsm.LSMTreeIndexAbstract;
-import com.arcadedb.schema.Type;
+import com.arcadedb.schema.IndexBuilder;
import java.util.*;
@@ -32,14 +29,13 @@ public void register(final String type, final IndexFactoryHandler handler) {
map.put(type, handler);
}
- public IndexInternal createIndex(final String indexType, final DatabaseInternal database, final String indexName, final boolean unique, final String filePath,
- final PaginatedFile.MODE mode, final Type[] keyTypes, final int pageSize, final LSMTreeIndexAbstract.NULL_STRATEGY nullStrategy,
- final Index.BuildIndexCallback callback) {
+ public IndexInternal createIndex(final IndexBuilder builder) {
+ final String indexType = builder.getIndexType().name();
final IndexFactoryHandler handler = map.get(indexType);
if (handler == null)
throw new IllegalArgumentException("Cannot create index of type '" + indexType + "'");
- return handler.create(database, indexName, unique, filePath, mode, keyTypes, pageSize, nullStrategy, callback);
+ return handler.create(builder);
}
}
diff --git a/engine/src/main/java/com/arcadedb/index/IndexFactoryHandler.java b/engine/src/main/java/com/arcadedb/index/IndexFactoryHandler.java
index 31311f240..ce568f141 100644
--- a/engine/src/main/java/com/arcadedb/index/IndexFactoryHandler.java
+++ b/engine/src/main/java/com/arcadedb/index/IndexFactoryHandler.java
@@ -18,12 +18,8 @@
*/
package com.arcadedb.index;
-import com.arcadedb.database.DatabaseInternal;
-import com.arcadedb.engine.PaginatedFile;
-import com.arcadedb.index.lsm.LSMTreeIndexAbstract;
-import com.arcadedb.schema.Type;
+import com.arcadedb.schema.IndexBuilder;
public interface IndexFactoryHandler {
- IndexInternal create(DatabaseInternal database, String name, boolean unique, String filePath, PaginatedFile.MODE mode, Type[] keyTypes, int pageSize,
- LSMTreeIndexAbstract.NULL_STRATEGY nullStrategy, Index.BuildIndexCallback callback);
+ IndexInternal create(IndexBuilder builder);
}
diff --git a/engine/src/main/java/com/arcadedb/index/IndexInternal.java b/engine/src/main/java/com/arcadedb/index/IndexInternal.java
index b204ba563..a1ddb4601 100644
--- a/engine/src/main/java/com/arcadedb/index/IndexInternal.java
+++ b/engine/src/main/java/com/arcadedb/index/IndexInternal.java
@@ -18,7 +18,7 @@
*/
package com.arcadedb.index;
-import com.arcadedb.engine.PaginatedComponent;
+import com.arcadedb.engine.Component;
import com.arcadedb.schema.Type;
import com.arcadedb.serializer.json.JSONObject;
@@ -29,7 +29,7 @@
* Internal Index interface.
*/
public interface IndexInternal extends Index {
- long build(int batchSize, BuildIndexCallback callback);
+ long build(int buildIndexBatchSize, BuildIndexCallback callback);
boolean compact() throws IOException, InterruptedException;
@@ -43,7 +43,7 @@ public interface IndexInternal extends Index {
int getFileId();
- PaginatedComponent getPaginatedComponent();
+ Component getComponent();
Type[] getKeyTypes();
diff --git a/engine/src/main/java/com/arcadedb/index/TypeIndex.java b/engine/src/main/java/com/arcadedb/index/TypeIndex.java
index 13a52f08d..13b712e58 100644
--- a/engine/src/main/java/com/arcadedb/index/TypeIndex.java
+++ b/engine/src/main/java/com/arcadedb/index/TypeIndex.java
@@ -295,11 +295,11 @@ public int getPageSize() {
}
@Override
- public long build(final int batchSize, final BuildIndexCallback callback) {
+ public long build(final int buildIndexBatchSize, final BuildIndexCallback callback) {
checkIsValid();
long total = 0;
for (final IndexInternal index : indexesOnBuckets)
- total += index.build(batchSize, callback);
+ total += index.build(buildIndexBatchSize, callback);
return total;
}
@@ -366,7 +366,7 @@ public int getFileId() {
}
@Override
- public PaginatedComponent getPaginatedComponent() {
+ public PaginatedComponent getComponent() {
throw new UnsupportedOperationException("getPaginatedComponent");
}
@@ -436,7 +436,7 @@ public List getIndexesByKeys(final Object[] keys) {
// USE THE SHARDED INDEX
final List propNames = getPropertyNames();
- List polymorphicIndexesOnKeys = type.getPolymorphicBucketIndexByBucketId(type.getBuckets(false).get(bucketIndex).getId(), propNames);
+ List polymorphicIndexesOnKeys = type.getPolymorphicBucketIndexByBucketId(type.getBuckets(false).get(bucketIndex).getFileId(), propNames);
final List subTypes = type.getSubTypes();
if (!subTypes.isEmpty()) {
@@ -444,7 +444,7 @@ public List getIndexesByKeys(final Object[] keys) {
polymorphicIndexesOnKeys = new ArrayList<>(polymorphicIndexesOnKeys);
for (DocumentType s : subTypes) {
- final List subIndexes = s.getPolymorphicBucketIndexByBucketId(s.getBuckets(false).get(bucketIndex).getId(), propNames);
+ final List subIndexes = s.getPolymorphicBucketIndexByBucketId(s.getBuckets(false).get(bucketIndex).getFileId(), propNames);
polymorphicIndexesOnKeys.addAll(subIndexes);
}
diff --git a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java
index 722f40667..4fbf5318f 100644
--- a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java
+++ b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeFullTextIndex.java
@@ -21,9 +21,9 @@
import com.arcadedb.database.DatabaseInternal;
import com.arcadedb.database.Identifiable;
import com.arcadedb.database.RID;
+import com.arcadedb.engine.ComponentFile;
import com.arcadedb.engine.PaginatedComponent;
-import com.arcadedb.engine.PaginatedComponentFactory;
-import com.arcadedb.engine.PaginatedFile;
+import com.arcadedb.engine.ComponentFactory;
import com.arcadedb.index.Index;
import com.arcadedb.index.IndexCursor;
import com.arcadedb.index.IndexCursorEntry;
@@ -32,13 +32,14 @@
import com.arcadedb.index.TempIndexCursor;
import com.arcadedb.index.TypeIndex;
import com.arcadedb.schema.EmbeddedSchema;
+import com.arcadedb.schema.IndexBuilder;
import com.arcadedb.schema.Schema;
import com.arcadedb.schema.Type;
+import com.arcadedb.serializer.json.JSONObject;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-import com.arcadedb.serializer.json.JSONObject;
import java.io.*;
import java.util.*;
@@ -70,18 +71,19 @@ public class LSMTreeFullTextIndex implements Index, IndexInternal {
public static class IndexFactoryHandler implements com.arcadedb.index.IndexFactoryHandler {
@Override
- public IndexInternal create(final DatabaseInternal database, final String name, final boolean unique, final String filePath, final PaginatedFile.MODE mode,
- final Type[] keyTypes, final int pageSize, final LSMTreeIndexAbstract.NULL_STRATEGY nullStrategy, final BuildIndexCallback callback) {
- if (unique)
+ public IndexInternal create(final IndexBuilder builder) {
+ if (builder.isUnique())
throw new IllegalArgumentException("Full text index cannot be unique");
- return new LSMTreeFullTextIndex(database, name, filePath, mode, pageSize, nullStrategy);
+
+ return new LSMTreeFullTextIndex(builder.getDatabase(), builder.getIndexName(), builder.getFilePath(), ComponentFile.MODE.READ_WRITE, builder.getPageSize(),
+ builder.getNullStrategy());
}
}
- public static class PaginatedComponentFactoryHandlerNotUnique implements PaginatedComponentFactory.PaginatedComponentFactoryHandler {
+ public static class PaginatedComponentFactoryHandlerNotUnique implements ComponentFactory.PaginatedComponentFactoryHandler {
@Override
public PaginatedComponent createOnLoad(final DatabaseInternal database, final String name, final String filePath, final int id,
- final PaginatedFile.MODE mode, final int pageSize, final int version) {
+ final ComponentFile.MODE mode, final int pageSize, final int version) {
final LSMTreeFullTextIndex mainIndex = new LSMTreeFullTextIndex(database, name, filePath, id, mode, pageSize, version);
return mainIndex.underlyingIndex.mutable;
}
@@ -90,7 +92,7 @@ public PaginatedComponent createOnLoad(final DatabaseInternal database, final St
/**
* Creation time.
*/
- public LSMTreeFullTextIndex(final DatabaseInternal database, final String name, final String filePath, final PaginatedFile.MODE mode, final int pageSize,
+ public LSMTreeFullTextIndex(final DatabaseInternal database, final String name, final String filePath, final ComponentFile.MODE mode, final int pageSize,
final LSMTreeIndexAbstract.NULL_STRATEGY nullStrategy) {
analyzer = new StandardAnalyzer();
underlyingIndex = new LSMTreeIndex(database, name, false, filePath, mode, new Type[] { Type.STRING }, pageSize, nullStrategy);
@@ -99,7 +101,7 @@ public LSMTreeFullTextIndex(final DatabaseInternal database, final String name,
/**
* Loading time.
*/
- public LSMTreeFullTextIndex(final DatabaseInternal database, final String name, final String filePath, final int fileId, final PaginatedFile.MODE mode,
+ public LSMTreeFullTextIndex(final DatabaseInternal database, final String name, final String filePath, final int fileId, final ComponentFile.MODE mode,
final int pageSize, final int version) {
try {
underlyingIndex = new LSMTreeIndex(database, name, false, filePath, fileId, mode, pageSize, version);
@@ -262,8 +264,8 @@ public boolean isUnique() {
}
@Override
- public PaginatedComponent getPaginatedComponent() {
- return underlyingIndex.getPaginatedComponent();
+ public PaginatedComponent getComponent() {
+ return underlyingIndex.getComponent();
}
@Override
@@ -312,8 +314,8 @@ public TypeIndex getTypeIndex() {
}
@Override
- public long build(final int batchSize, final BuildIndexCallback callback) {
- return underlyingIndex.build(batchSize, callback);
+ public long build(final int buildIndexBatchSize, final BuildIndexCallback callback) {
+ return underlyingIndex.build(buildIndexBatchSize, callback);
}
@Override
diff --git a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndex.java b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndex.java
index 65ff7254e..754d9e1df 100644
--- a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndex.java
+++ b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndex.java
@@ -25,11 +25,11 @@
import com.arcadedb.database.TransactionContext;
import com.arcadedb.database.TransactionIndexContext;
import com.arcadedb.engine.BasePage;
+import com.arcadedb.engine.ComponentFile;
import com.arcadedb.engine.MutablePage;
import com.arcadedb.engine.PageId;
import com.arcadedb.engine.PaginatedComponent;
-import com.arcadedb.engine.PaginatedComponentFactory;
-import com.arcadedb.engine.PaginatedFile;
+import com.arcadedb.engine.ComponentFactory;
import com.arcadedb.exception.DatabaseIsReadOnlyException;
import com.arcadedb.exception.NeedRetryException;
import com.arcadedb.exception.TimeoutException;
@@ -43,11 +43,13 @@
import com.arcadedb.index.TypeIndex;
import com.arcadedb.log.LogManager;
import com.arcadedb.schema.EmbeddedSchema;
+import com.arcadedb.schema.IndexBuilder;
import com.arcadedb.schema.Schema;
import com.arcadedb.schema.Type;
import com.arcadedb.serializer.BinaryComparator;
import com.arcadedb.serializer.BinaryTypes;
import com.arcadedb.serializer.json.JSONObject;
+import com.arcadedb.utility.FileUtils;
import com.arcadedb.utility.LockManager;
import com.arcadedb.utility.RWLockContext;
@@ -76,16 +78,16 @@ public enum INDEX_STATUS {UNAVAILABLE, AVAILABLE, COMPACTION_SCHEDULED, COMPACTI
public static class IndexFactoryHandler implements com.arcadedb.index.IndexFactoryHandler {
@Override
- public IndexInternal create(final DatabaseInternal database, final String name, final boolean unique, final String filePath, final PaginatedFile.MODE mode,
- final Type[] keyTypes, final int pageSize, final LSMTreeIndexAbstract.NULL_STRATEGY nullStrategy, final BuildIndexCallback callback) {
- return new LSMTreeIndex(database, name, unique, filePath, mode, keyTypes, pageSize, nullStrategy);
+ public IndexInternal create(final IndexBuilder builder) {
+ return new LSMTreeIndex(builder.getDatabase(), builder.getIndexName(), builder.isUnique(), builder.getFilePath(), ComponentFile.MODE.READ_WRITE,
+ builder.getKeyTypes(), builder.getPageSize(), builder.getNullStrategy());
}
}
- public static class PaginatedComponentFactoryHandlerUnique implements PaginatedComponentFactory.PaginatedComponentFactoryHandler {
+ public static class PaginatedComponentFactoryHandlerUnique implements ComponentFactory.PaginatedComponentFactoryHandler {
@Override
public PaginatedComponent createOnLoad(final DatabaseInternal database, final String name, final String filePath, final int id,
- final PaginatedFile.MODE mode, final int pageSize, final int version) throws IOException {
+ final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException {
if (filePath.endsWith(LSMTreeIndexCompacted.UNIQUE_INDEX_EXT))
return new LSMTreeIndexCompacted(null, database, name, true, filePath, id, mode, pageSize, version);
@@ -93,10 +95,10 @@ public PaginatedComponent createOnLoad(final DatabaseInternal database, final St
}
}
- public static class PaginatedComponentFactoryHandlerNotUnique implements PaginatedComponentFactory.PaginatedComponentFactoryHandler {
+ public static class PaginatedComponentFactoryHandlerNotUnique implements ComponentFactory.PaginatedComponentFactoryHandler {
@Override
public PaginatedComponent createOnLoad(final DatabaseInternal database, final String name, final String filePath, final int id,
- final PaginatedFile.MODE mode, final int pageSize, final int version) throws IOException {
+ final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException {
if (filePath.endsWith(LSMTreeIndexCompacted.UNIQUE_INDEX_EXT))
return new LSMTreeIndexCompacted(null, database, name, false, filePath, id, mode, pageSize, version);
@@ -107,10 +109,10 @@ public PaginatedComponent createOnLoad(final DatabaseInternal database, final St
/**
* Called at creation time.
*/
- public LSMTreeIndex(final DatabaseInternal database, final String name, final boolean unique, final String filePath, final PaginatedFile.MODE mode,
+ public LSMTreeIndex(final DatabaseInternal database, final String name, final boolean unique, final String filePath, final ComponentFile.MODE mode,
final Type[] keyTypes, final int pageSize, final LSMTreeIndexAbstract.NULL_STRATEGY nullStrategy) {
try {
- this.name = name;
+ this.name = FileUtils.encode(name, database.getSchema().getEncoding());
this.mutable = new LSMTreeIndexMutable(this, database, name, unique, filePath, mode, keyTypes, pageSize, nullStrategy);
} catch (final IOException e) {
throw new IndexException("Error on creating index '" + name + "'", e);
@@ -121,8 +123,8 @@ public LSMTreeIndex(final DatabaseInternal database, final String name, final bo
* Called at load time (1st page only).
*/
public LSMTreeIndex(final DatabaseInternal database, final String name, final boolean unique, final String filePath, final int id,
- final PaginatedFile.MODE mode, final int pageSize, final int version) throws IOException {
- this.name = name;
+ final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException {
+ this.name = FileUtils.encode(name, database.getSchema().getEncoding());
this.mutable = new LSMTreeIndexMutable(this, database, name, unique, filePath, id, mode, pageSize, version);
}
@@ -216,7 +218,7 @@ public List getPropertyNames() {
@Override
public boolean compact() throws IOException, InterruptedException {
checkIsValid();
- if (getDatabase().getMode() == PaginatedFile.MODE.READ_ONLY)
+ if (getDatabase().getMode() == ComponentFile.MODE.READ_ONLY)
throw new DatabaseIsReadOnlyException("Cannot update the index '" + getName() + "'");
if (getDatabase().getPageManager().isPageFlushingSuspended())
@@ -475,7 +477,7 @@ public LSMTreeIndexMutable getMutableIndex() {
}
@Override
- public PaginatedComponent getPaginatedComponent() {
+ public PaginatedComponent getComponent() {
return mutable;
}
@@ -573,7 +575,7 @@ protected LSMTreeIndexMutable splitIndex(final int startingFromPage, final LSMTr
}
}
- public long build(final int batchSize, final BuildIndexCallback callback) {
+ public long build(final int buildIndexBatchSize, final BuildIndexCallback callback) {
checkIsValid();
final AtomicLong total = new AtomicLong();
@@ -588,15 +590,15 @@ public long build(final int batchSize, final BuildIndexCallback callback) {
db.getIndexer().addToIndex(LSMTreeIndex.this, record.getIdentity(), (Document) record);
total.incrementAndGet();
- if (callback != null)
- callback.onDocumentIndexed((Document) record, total.get());
-
- if (total.get() % batchSize == 0) {
+ if (total.get() % buildIndexBatchSize == 0) {
// CHUNK OF 100K
db.getWrappedDatabaseInstance().commit();
db.getWrappedDatabaseInstance().begin();
}
+ if (callback != null)
+ callback.onDocumentIndexed((Document) record, total.get());
+
return true;
});
diff --git a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexAbstract.java b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexAbstract.java
index cd368f8f6..80c579be2 100644
--- a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexAbstract.java
+++ b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexAbstract.java
@@ -23,9 +23,9 @@
import com.arcadedb.database.DatabaseInternal;
import com.arcadedb.database.RID;
import com.arcadedb.engine.BasePage;
+import com.arcadedb.engine.ComponentFile;
import com.arcadedb.engine.MutablePage;
import com.arcadedb.engine.PaginatedComponent;
-import com.arcadedb.engine.PaginatedFile;
import com.arcadedb.index.IndexCursorEntry;
import com.arcadedb.index.IndexException;
import com.arcadedb.log.LogManager;
@@ -92,7 +92,7 @@ public LookupResult(final boolean found, final boolean outside, final int keyInd
* Called at creation time.
*/
protected LSMTreeIndexAbstract(final LSMTreeIndex mainIndex, final DatabaseInternal database, final String name, final boolean unique, final String filePath,
- final String ext, final PaginatedFile.MODE mode, final Type[] keyTypes, final int pageSize, final int version, final NULL_STRATEGY nullStrategy)
+ final String ext, final ComponentFile.MODE mode, final Type[] keyTypes, final int pageSize, final int version, final NULL_STRATEGY nullStrategy)
throws IOException {
super(database, name, filePath, ext, mode, pageSize, version);
@@ -118,7 +118,7 @@ protected LSMTreeIndexAbstract(final LSMTreeIndex mainIndex, final DatabaseInter
*/
protected LSMTreeIndexAbstract(final LSMTreeIndex mainIndex, final DatabaseInternal database, final String name, final boolean unique, final String filePath,
final String ext, final Type[] keyTypes, final byte[] binaryKeyTypes, final int pageSize, final int version) throws IOException {
- super(database, name, filePath, TEMP_EXT + ext, PaginatedFile.MODE.READ_WRITE, pageSize, version);
+ super(database, name, filePath, TEMP_EXT + ext, ComponentFile.MODE.READ_WRITE, pageSize, version);
this.mainIndex = mainIndex;
this.serializer = database.getSerializer();
this.comparator = serializer.getComparator();
@@ -132,7 +132,7 @@ protected LSMTreeIndexAbstract(final LSMTreeIndex mainIndex, final DatabaseInter
* Called at load time (1st page only).
*/
protected LSMTreeIndexAbstract(final LSMTreeIndex mainIndex, final DatabaseInternal database, final String name, final boolean unique, final String filePath,
- final int id, final PaginatedFile.MODE mode, final int pageSize, final int version) throws IOException {
+ final int id, final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException {
super(database, name, filePath, id, mode, pageSize, version);
this.mainIndex = mainIndex;
this.serializer = database.getSerializer();
@@ -165,7 +165,7 @@ public NULL_STRATEGY getNullStrategy() {
@Override
public String toString() {
- return name + "(" + getFileId() + ")";
+ return componentName + "(" + getFileId() + ")";
}
public Type[] getKeyTypes() {
diff --git a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCompacted.java b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCompacted.java
index f3d8eb88f..dc1dab981 100644
--- a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCompacted.java
+++ b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexCompacted.java
@@ -23,9 +23,9 @@
import com.arcadedb.database.RID;
import com.arcadedb.database.TrackableBinary;
import com.arcadedb.engine.BasePage;
+import com.arcadedb.engine.ComponentFile;
import com.arcadedb.engine.MutablePage;
import com.arcadedb.engine.PageId;
-import com.arcadedb.engine.PaginatedFile;
import com.arcadedb.exception.DatabaseOperationException;
import com.arcadedb.index.IndexCursorEntry;
import com.arcadedb.log.LogManager;
@@ -58,7 +58,7 @@ public LSMTreeIndexCompacted(final LSMTreeIndex mainIndex, final DatabaseInterna
* Called at load time (1st page only).
*/
protected LSMTreeIndexCompacted(final LSMTreeIndex mainIndex, final DatabaseInternal database, final String name, final boolean unique, final String filePath,
- final int id, final PaginatedFile.MODE mode, final int pageSize, final int version) throws IOException {
+ final int id, final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException {
super(mainIndex, database, name, unique, filePath, id, mode, pageSize, version);
}
@@ -80,7 +80,7 @@ public Set get(final Object[] keys, final int limit) {
return set;
} catch (final IOException e) {
- throw new DatabaseOperationException("Cannot lookup key '" + Arrays.toString(keys) + "' in index '" + name + "'", e);
+ throw new DatabaseOperationException("Cannot lookup key '" + Arrays.toString(keys) + "' in index '" + componentName + "'", e);
}
}
diff --git a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexMutable.java b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexMutable.java
index 03afa619d..fdc7813ef 100644
--- a/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexMutable.java
+++ b/engine/src/main/java/com/arcadedb/index/lsm/LSMTreeIndexMutable.java
@@ -26,9 +26,9 @@
import com.arcadedb.database.TrackableBinary;
import com.arcadedb.database.async.DatabaseAsyncExecutorImpl;
import com.arcadedb.engine.BasePage;
+import com.arcadedb.engine.ComponentFile;
import com.arcadedb.engine.MutablePage;
import com.arcadedb.engine.PageId;
-import com.arcadedb.engine.PaginatedFile;
import com.arcadedb.exception.DatabaseIsReadOnlyException;
import com.arcadedb.exception.DatabaseOperationException;
import com.arcadedb.index.IndexCursor;
@@ -59,7 +59,7 @@ public class LSMTreeIndexMutable extends LSMTreeIndexAbstract {
* Called at creation time.
*/
protected LSMTreeIndexMutable(final LSMTreeIndex mainIndex, final DatabaseInternal database, final String name, final boolean unique, final String filePath,
- final PaginatedFile.MODE mode, final Type[] keyTypes, final int pageSize, final NULL_STRATEGY nullStrategy) throws IOException {
+ final ComponentFile.MODE mode, final Type[] keyTypes, final int pageSize, final NULL_STRATEGY nullStrategy) throws IOException {
super(mainIndex, database, name, unique, filePath, unique ? UNIQUE_INDEX_EXT : NOTUNIQUE_INDEX_EXT, mode, keyTypes, pageSize, CURRENT_VERSION,
nullStrategy);
database.checkTransactionIsActive(database.isAutoTransaction());
@@ -81,7 +81,7 @@ protected LSMTreeIndexMutable(final LSMTreeIndex mainIndex, final DatabaseIntern
* Called at load time (1st page only).
*/
protected LSMTreeIndexMutable(final LSMTreeIndex mainIndex, final DatabaseInternal database, final String name, final boolean unique, final String filePath,
- final int id, final PaginatedFile.MODE mode, final int pageSize, final int version) throws IOException {
+ final int id, final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException {
super(mainIndex, database, name, unique, filePath, id, mode, pageSize, version);
onAfterLoad();
}
@@ -127,9 +127,9 @@ public void onAfterLoad() {
} catch (final Exception e) {
LogManager.instance().log(this, Level.SEVERE,
"Invalid sub-index for index '%s', ignoring it. WARNING: This could lead on using partial indexes. Please recreate the index from scratch (error=%s)",
- null, name, e.getMessage());
+ null, componentName, e.getMessage());
- database.getSchema().dropIndex(name);
+ database.getSchema().dropIndex(componentName);
}
}
@@ -137,7 +137,7 @@ public void onAfterLoad() {
public void onAfterCommit() {
if (minPagesToScheduleACompaction > 0 && currentMutablePages >= minPagesToScheduleACompaction) {
LogManager.instance()
- .log(this, Level.FINE, "Scheduled compaction of index '%s' (currentMutablePages=%d totalPages=%d)", null, name, currentMutablePages, getTotalPages());
+ .log(this, Level.FINE, "Scheduled compaction of index '%s' (currentMutablePages=%d totalPages=%d)", null, componentName, currentMutablePages, getTotalPages());
((DatabaseAsyncExecutorImpl) database.async()).compact(mainIndex);
}
}
@@ -158,8 +158,8 @@ public void remove(final Object[] keys, final Identifiable rid) {
}
public LSMTreeIndexCompacted createNewForCompaction() throws IOException {
- final int last_ = name.lastIndexOf('_');
- final String newName = name.substring(0, last_) + "_" + System.nanoTime();
+ final int last_ = componentName.lastIndexOf('_');
+ final String newName = componentName.substring(0, last_) + "_" + System.nanoTime();
return new LSMTreeIndexCompacted(mainIndex, database, newName, unique, database.getDatabasePath() + File.separator + newName, keyTypes, binaryKeyTypes,
pageSize);
@@ -371,7 +371,7 @@ protected MutablePage createNewPage() throws IOException {
pos += INT_SERIALIZED_SIZE;
if (txPageCounter == 0) {
- currentPage.writeInt(pos, subIndex != null ? subIndex.getId() : -1); // SUB-INDEX FILE ID
+ currentPage.writeInt(pos, subIndex != null ? subIndex.getFileId() : -1); // SUB-INDEX FILE ID
pos += INT_SERIALIZED_SIZE;
currentPage.writeByte(pos++, (byte) binaryKeyTypes.length);
@@ -410,8 +410,8 @@ protected void internalPut(final Object[] keys, final RID[] rids) {
if (keys == null)
throw new IllegalArgumentException("Keys parameter is null");
- if (database.getMode() == PaginatedFile.MODE.READ_ONLY)
- throw new DatabaseIsReadOnlyException("Cannot update the index '" + name + "'");
+ if (database.getMode() == ComponentFile.MODE.READ_ONLY)
+ throw new DatabaseIsReadOnlyException("Cannot update the index '" + componentName + "'");
if (keys.length != binaryKeyTypes.length)
throw new IllegalArgumentException("Cannot put an entry in the index with a partial key");
@@ -427,7 +427,7 @@ protected void internalPut(final Object[] keys, final RID[] rids) {
final int txPageCounter = getTotalPages();
if (txPageCounter < 1)
- throw new IllegalArgumentException("Cannot update the index '" + name + "' because the file is invalid");
+ throw new IllegalArgumentException("Cannot update the index '" + componentName + "' because the file is invalid");
int pageNum = txPageCounter - 1;
@@ -489,11 +489,11 @@ protected void internalPut(final Object[] keys, final RID[] rids) {
if (LogManager.instance().isDebugEnabled())
LogManager.instance()
.log(this, Level.FINE, "Put entry %s=%s in index '%s' (page=%s countInPage=%d newPage=%s thread=%d)", Arrays.toString(keys), Arrays.toString(rids),
- name, currentPage.getPageId(), count + 1, newPage, Thread.currentThread().getId());
+ componentName, currentPage.getPageId(), count + 1, newPage, Thread.currentThread().getId());
} catch (final IOException e) {
throw new DatabaseOperationException(
- "Cannot index key '" + Arrays.toString(keys) + "' with value '" + Arrays.toString(rids) + "' in index '" + name + "'", e);
+ "Cannot index key '" + Arrays.toString(keys) + "' with value '" + Arrays.toString(rids) + "' in index '" + componentName + "'", e);
}
}
@@ -501,8 +501,8 @@ protected void internalRemove(final Object[] keys, final Identifiable rid) {
if (keys == null)
throw new IllegalArgumentException("Keys parameter is null");
- if (database.getMode() == PaginatedFile.MODE.READ_ONLY)
- throw new DatabaseIsReadOnlyException("Cannot update the index '" + name + "'");
+ if (database.getMode() == ComponentFile.MODE.READ_ONLY)
+ throw new DatabaseIsReadOnlyException("Cannot update the index '" + componentName + "'");
if (keys.length != binaryKeyTypes.length)
throw new IllegalArgumentException("Cannot remove an entry in the index with a partial key");
@@ -518,7 +518,7 @@ protected void internalRemove(final Object[] keys, final Identifiable rid) {
final int txPageCounter = getTotalPages();
if (txPageCounter < 1)
- throw new IllegalArgumentException("Cannot update the index '" + name + "' because the file is invalid");
+ throw new IllegalArgumentException("Cannot update the index '" + componentName + "' because the file is invalid");
int pageNum = txPageCounter - 1;
@@ -610,10 +610,10 @@ protected void internalRemove(final Object[] keys, final Identifiable rid) {
if (LogManager.instance().isDebugEnabled())
LogManager.instance()
.log(this, Level.FINE, "Put removed entry %s=%s (original=%s) in index '%s' (page=%s countInPage=%d newPage=%s)", null, Arrays.toString(keys),
- removedRID, rid, name, currentPage.getPageId(), count + 1, newPage);
+ removedRID, rid, componentName, currentPage.getPageId(), count + 1, newPage);
} catch (final IOException e) {
- throw new DatabaseOperationException("Cannot index key '" + Arrays.toString(keys) + "' with value '" + rid + "' in index '" + name + "'", e);
+ throw new DatabaseOperationException("Cannot index key '" + Arrays.toString(keys) + "' with value '" + rid + "' in index '" + componentName + "'", e);
}
}
diff --git a/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndex.java b/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndex.java
new file mode 100644
index 000000000..0349de8a2
--- /dev/null
+++ b/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndex.java
@@ -0,0 +1,1054 @@
+/*
+ * Copyright 2023 Arcade Data Ltd
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package com.arcadedb.index.vector;
+
+import com.arcadedb.database.DatabaseInternal;
+import com.arcadedb.database.Identifiable;
+import com.arcadedb.database.RID;
+import com.arcadedb.engine.Component;
+import com.arcadedb.engine.ComponentFactory;
+import com.arcadedb.engine.ComponentFile;
+import com.arcadedb.graph.MutableVertex;
+import com.arcadedb.graph.Vertex;
+import com.arcadedb.index.IndexCursor;
+import com.arcadedb.index.IndexException;
+import com.arcadedb.index.IndexInternal;
+import com.arcadedb.index.TypeIndex;
+import com.arcadedb.index.lsm.LSMTreeIndexAbstract;
+import com.arcadedb.index.vector.distance.DistanceFunctionFactory;
+import com.arcadedb.schema.EmbeddedSchema;
+import com.arcadedb.schema.IndexBuilder;
+import com.arcadedb.schema.Schema;
+import com.arcadedb.schema.Type;
+import com.arcadedb.schema.VectorIndexBuilder;
+import com.arcadedb.serializer.json.JSONObject;
+import com.arcadedb.utility.FileUtils;
+import com.arcadedb.utility.Pair;
+import com.github.jelmerk.knn.DistanceFunction;
+import com.github.jelmerk.knn.Index;
+import com.github.jelmerk.knn.SearchResult;
+import com.github.jelmerk.knn.util.Murmur3;
+import org.eclipse.collections.api.list.primitive.MutableIntList;
+
+import java.io.*;
+import java.lang.reflect.*;
+import java.util.*;
+import java.util.concurrent.locks.*;
+import java.util.stream.*;
+
+/**
+ * This work is derived from the excellent work made by Jelmer Kuperus on https://github.com/jelmerk/hnswlib.
+ *
+ * Implementation of {@link Index} that implements the hnsw algorithm.
+ *
+ * @author Luca Garulli (l.garulli@arcadedata.com)
+ * @see
+ * Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs
+ */
+public class HnswVectorIndex extends Component implements com.arcadedb.index.Index, IndexInternal {
+ public static final String FILE_EXT = "hnswidx";
+ public static final int CURRENT_VERSION = 0;
+
+ private final DistanceFunction distanceFunction;
+ private final Comparator distanceComparator;
+ private final MaxValueComparator maxValueDistanceComparator;
+ private final int dimensions;
+ private final int maxItemCount;
+ private final int m;
+ private final int maxM;
+ private final int maxM0;
+ private final double levelLambda;
+ private int ef;
+ private final int efConstruction;
+ public volatile Vertex entryPoint;
+
+ private TypeIndex underlyingIndex;
+ private final ReentrantLock globalLock;
+ private final Set excludedCandidates = new HashSet<>();
+ private final String vertexType;
+ private final String edgeType;
+ private final String vectorPropertyName;
+ private final String idPropertyName;
+ private final Map cache;
+ private String indexName;
+
+ public static class IndexFactoryHandler implements com.arcadedb.index.IndexFactoryHandler {
+ @Override
+ public IndexInternal create(final IndexBuilder builder) {
+ if (!(builder instanceof VectorIndexBuilder))
+ throw new IndexException("Expected VectorIndexBuilder but received " + builder);
+
+ return new HnswVectorIndex<>((VectorIndexBuilder) builder);
+ }
+ }
+
+ public static class PaginatedComponentFactoryHandlerUnique implements ComponentFactory.PaginatedComponentFactoryHandler {
+ @Override
+ public Component createOnLoad(final DatabaseInternal database, final String name, final String filePath, final int id, final ComponentFile.MODE mode,
+ final int pageSize, final int version) throws IOException {
+ return new HnswVectorIndex(database, name, filePath, id, version);
+ }
+ }
+
+ protected HnswVectorIndex(final VectorIndexBuilder builder) {
+ super(builder.getDatabase(), builder.getFilePath(), builder.getDatabase().getFileManager().newFileId(), CURRENT_VERSION, builder.getFilePath());
+
+ this.dimensions = builder.getDimensions();
+ this.maxItemCount = builder.getMaxItemCount();
+ this.distanceFunction = builder.getDistanceFunction();
+ this.distanceComparator = builder.getDistanceComparator();
+ this.maxValueDistanceComparator = new MaxValueComparator<>(this.distanceComparator);
+
+ this.m = builder.getM();
+ this.maxM = m;
+ this.maxM0 = m * 2;
+ this.levelLambda = 1 / Math.log(this.m);
+ this.efConstruction = Math.max(builder.getEfConstruction(), m);
+ this.ef = builder.getEf();
+
+ this.vertexType = builder.getVertexType();
+ this.edgeType = builder.getEdgeType();
+ this.vectorPropertyName = builder.getVectorPropertyName();
+ this.idPropertyName = builder.getIdPropertyName();
+
+ this.cache = builder.getCache();
+
+ this.underlyingIndex = builder.getDatabase().getSchema().buildTypeIndex(builder.getVertexType(), new String[] { idPropertyName }).withUnique(true)
+ .withIgnoreIfExists(true).withType(Schema.INDEX_TYPE.LSM_TREE).create();
+
+ this.globalLock = new ReentrantLock();
+ this.indexName = vertexType + "[" + idPropertyName + "," + vectorPropertyName + "]";
+ }
+
+ /**
+ * Load time.
+ */
+ protected HnswVectorIndex(final DatabaseInternal database, final String indexName, final String filePath, final int id, final int version)
+ throws IOException {
+ super(database, indexName, id, version, filePath);
+
+ final String fileContent = FileUtils.readFileAsString(new File(filePath));
+
+ final JSONObject json = new JSONObject(fileContent);
+
+ this.distanceFunction = DistanceFunctionFactory.getImplementationByClassName(json.getString("distanceFunction"));
+ if (distanceFunction == null)
+ throw new IllegalArgumentException("distance function '" + json.getString("distanceFunction") + "' not supported");
+
+ this.dimensions = json.getInt("dimensions");
+ this.distanceComparator = (Comparator) Comparator.naturalOrder();
+ this.maxValueDistanceComparator = new MaxValueComparator<>(this.distanceComparator);
+ this.maxItemCount = json.getInt("maxItemCount");
+ this.m = json.getInt("m");
+ this.maxM = json.getInt("maxM");
+ this.maxM0 = json.getInt("maxM0");
+ this.levelLambda = json.getDouble("levelLambda");
+ this.ef = json.getInt("ef");
+ this.efConstruction = json.getInt("efConstruction");
+
+ if (json.getString("entryPoint").length() > 0) {
+ this.entryPoint = new RID(database, json.getString("entryPoint")).asVertex();
+ } else
+ this.entryPoint = null;
+
+ this.vertexType = json.getString("vertexType");
+ this.edgeType = json.getString("edgeType");
+ this.idPropertyName = json.getString("idPropertyName");
+ this.vectorPropertyName = json.getString("vectorPropertyName");
+
+ this.globalLock = new ReentrantLock();
+ this.cache = null;
+ this.indexName = vertexType + "[" + idPropertyName + "," + vectorPropertyName + "]";
+ }
+
+ @Override
+ public void onAfterSchemaLoad() {
+ this.underlyingIndex = database.getSchema().buildTypeIndex(vertexType, new String[] { idPropertyName }).withIgnoreIfExists(true).withUnique(true)
+ .withType(Schema.INDEX_TYPE.LSM_TREE).create();
+ }
+
+ @Override
+ public String getName() {
+ return indexName;
+ }
+
+ public List> findNeighbors(final TId id, final int k) {
+ final Vertex start = get(id);
+ if (start == null)
+ return Collections.emptyList();
+
+ final List> neighbors = findNearest(getVectorFromVertex(start), k + 1).stream()
+ .filter(result -> !getIdFromVertex(result.item()).equals(id)).limit(k).collect(Collectors.toList());
+
+ final List> result = new ArrayList<>(neighbors.size());
+ for (SearchResult neighbor : neighbors)
+ result.add(new Pair(neighbor.item(), neighbor.distance()));
+ return result;
+
+ }
+
+ public boolean add(Vertex vertex) {
+ final TVector vertexVector = getVectorFromVertex(vertex);
+ if (Array.getLength(vertexVector) != dimensions)
+ throw new IllegalArgumentException("Item does not have dimensionality of " + dimensions);
+
+ final TId vertexId = getIdFromVertex(vertex);
+ final int vertexMaxLevel = getMaxLevelFromVertex(vertex);
+
+ final int randomLevel = assignLevel(vertexId, this.levelLambda);
+
+ final ArrayList[] connections = new ArrayList[randomLevel + 1];
+
+ for (int level = 0; level <= randomLevel; level++) {
+ final int levelM = level == 0 ? maxM0 : maxM;
+ connections[level] = new ArrayList<>(levelM);
+ }
+
+ globalLock.lock();
+ try {
+
+ final long totalEdges = vertex.countEdges(Vertex.DIRECTION.OUT, getEdgeType(0));
+ if (totalEdges > 0)
+ // ALREADY INSERTED
+ return true;
+
+ vertex = vertex.modify().set("vectorMaxLevel", randomLevel).save();
+ if (cache != null)
+ cache.put(vertex.getIdentity(), vertex);
+
+ final RID vertexRID = vertex.getIdentity();
+ synchronized (excludedCandidates) {
+ excludedCandidates.add(vertexRID);
+ }
+
+ final Vertex entryPointCopy = entryPoint;
+ try {
+ if (entryPoint != null && randomLevel <= getMaxLevelFromVertex(entryPoint)) {
+ globalLock.unlock();
+ }
+
+ Vertex currObj = entryPointCopy;
+ final int entryPointCopyMaxLevel = getMaxLevelFromVertex(entryPointCopy);
+
+ if (currObj != null) {
+ if (vertexMaxLevel < entryPointCopyMaxLevel) {
+ TDistance curDist = distanceFunction.distance(vertexVector, getVectorFromVertex(currObj));
+ for (int activeLevel = entryPointCopyMaxLevel; activeLevel > vertexMaxLevel; activeLevel--) {
+ boolean changed = true;
+
+ while (changed) {
+ changed = false;
+
+ synchronized (currObj) {
+ final Iterator candidateConnections = getConnectionsFromVertex(currObj, activeLevel);
+ while (candidateConnections.hasNext()) {
+ final Vertex candidateNode = candidateConnections.next();
+ final TDistance candidateDistance = distanceFunction.distance(vertexVector, getVectorFromVertex(candidateNode));
+
+ if (lt(candidateDistance, curDist)) {
+ curDist = candidateDistance;
+ currObj = candidateNode;
+ changed = true;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ for (int level = Math.min(randomLevel, entryPointCopyMaxLevel); level >= 0; level--) {
+ final PriorityQueue> topCandidates = searchBaseLayer(currObj, vertexVector, efConstruction, level);
+
+ // TODO: MANAGE DELETE OF ENTRYPOINT
+// if (entryPointCopy.deleted) {
+// TDistance distance = distanceFunction.distance(vertex.vector(), entryPointCopy.item.vector());
+// topCandidates.add(new NodeIdAndDistance<>(entryPointCopy.id, distance, maxValueDistanceComparator));
+//
+// if (topCandidates.size() > efConstruction) {
+// topCandidates.poll();
+// }
+// }
+
+ mutuallyConnectNewElement(vertex, topCandidates, level);
+
+ }
+ }
+
+ // zoom out to the highest level
+ if (entryPoint == null || vertexMaxLevel > entryPointCopyMaxLevel)
+ // this is thread safe because we get the global lock when we add a level
+ this.entryPoint = vertex;
+
+ return true;
+
+ } finally {
+ synchronized (excludedCandidates) {
+ excludedCandidates.remove(vertexRID);
+ }
+ }
+ } finally {
+ if (globalLock.isHeldByCurrentThread()) {
+ globalLock.unlock();
+ }
+ }
+ }
+
+ private Iterator getConnectionsFromVertex(final Vertex vertex, final int level) {
+ return vertex.getVertices(Vertex.DIRECTION.OUT, edgeType + level).iterator();
+ }
+
+ private int countConnectionsFromVertex(final Vertex vertex, final int level) {
+ return (int) vertex.countEdges(Vertex.DIRECTION.OUT, edgeType + level);
+ }
+
+ private int getMaxLevelFromVertex(final Vertex vertex) {
+ if (vertex == null)
+ return 0;
+ final Integer vectorMaxLevel = vertex.getInteger("vectorMaxLevel");
+ return vectorMaxLevel != null ? vectorMaxLevel : 0;
+ }
+
+ private Vertex loadVertexFromRID(final Identifiable rid) {
+ if (rid instanceof Vertex)
+ return (Vertex) rid;
+
+ Vertex vertex = null;
+ if (cache != null)
+ vertex = cache.get(rid);
+ if (vertex == null)
+ vertex = rid.asVertex();
+ return vertex;
+ }
+
+ private void mutuallyConnectNewElement(final Vertex newNode, final PriorityQueue> topCandidates, final int level) {
+ final int bestN = level == 0 ? this.maxM0 : this.maxM;
+ final RID newNodeId = newNode.getIdentity();
+ final TVector newItemVector = getVectorFromVertex(newNode);
+
+ getNeighborsByHeuristic2(topCandidates, m);
+
+ while (!topCandidates.isEmpty()) {
+ final RID selectedNeighbourId = topCandidates.poll().nodeId;
+ synchronized (excludedCandidates) {
+ if (excludedCandidates.contains(selectedNeighbourId)) {
+ continue;
+ }
+ }
+
+ // CREATE THE EDGE TYPE IF NOT PRESENT
+ final String edgeTypeName = getEdgeType(level);
+ database.getSchema().getOrCreateEdgeType(edgeTypeName);
+
+ newNode.newEdge(edgeTypeName, selectedNeighbourId, false);
+
+ final Vertex neighbourNode = loadVertexFromRID(selectedNeighbourId);
+ final TVector neighbourVector = getVectorFromVertex(neighbourNode);
+ final int neighbourConnectionsAtLevelTotal = countConnectionsFromVertex(neighbourNode, level);
+ final Iterator neighbourConnectionsAtLevel = getConnectionsFromVertex(neighbourNode, level);
+
+ if (neighbourConnectionsAtLevelTotal < bestN) {
+ neighbourNode.newEdge(edgeTypeName, newNode, false);
+ } else {
+ // finding the "weakest" element to replace it with the new one
+ final TDistance dMax = distanceFunction.distance(newItemVector, neighbourVector);
+ final Comparator> comparator = Comparator.>naturalOrder().reversed();
+ final PriorityQueue> candidates = new PriorityQueue<>(comparator);
+ candidates.add(new NodeIdAndDistance<>(newNodeId, dMax, maxValueDistanceComparator));
+
+ neighbourConnectionsAtLevel.forEachRemaining(neighbourConnection -> {
+ final TDistance dist = distanceFunction.distance(neighbourVector, getVectorFromVertex(neighbourConnection));
+ candidates.add(new NodeIdAndDistance<>(neighbourConnection.getIdentity(), dist, maxValueDistanceComparator));
+ });
+
+ getNeighborsByHeuristic2(candidates, bestN);
+
+ while (!candidates.isEmpty()) {
+ neighbourNode.newEdge(edgeTypeName, candidates.poll().nodeId, false);
+ }
+ }
+ }
+ }
+
+ private void getNeighborsByHeuristic2(final PriorityQueue> topCandidates, final int m) {
+ if (topCandidates.size() < m)
+ return;
+
+ final PriorityQueue> queueClosest = new PriorityQueue<>();
+ final List> returnList = new ArrayList<>();
+
+ while (!topCandidates.isEmpty()) {
+ queueClosest.add(topCandidates.poll());
+ }
+
+ while (!queueClosest.isEmpty()) {
+ if (returnList.size() >= m)
+ break;
+
+ final NodeIdAndDistance currentPair = queueClosest.poll();
+ final TDistance distToQuery = currentPair.distance;
+
+ boolean good = true;
+ for (NodeIdAndDistance secondPair : returnList) {
+
+ final TDistance curdist = distanceFunction.distance(//
+ getVectorFromVertex(loadVertexFromRID(secondPair.nodeId)),//
+ getVectorFromVertex(loadVertexFromRID(currentPair.nodeId)));
+
+ if (lt(curdist, distToQuery)) {
+ good = false;
+ break;
+ }
+
+ }
+ if (good) {
+ returnList.add(currentPair);
+ }
+ }
+
+ topCandidates.addAll(returnList);
+ }
+
+ public List> findNearest(final TVector destination, final int k) {
+ if (entryPoint == null)
+ return Collections.emptyList();
+
+ final Vertex entryPointCopy = entryPoint;
+ Vertex currObj = entryPointCopy;
+
+ TDistance curDist = distanceFunction.distance(destination, getVectorFromVertex(currObj));
+
+ for (int activeLevel = getMaxLevelFromVertex(entryPointCopy); activeLevel > 0; activeLevel--) {
+ boolean changed = true;
+ while (changed) {
+ changed = false;
+
+ final Iterator candidateConnections = getConnectionsFromVertex(currObj, activeLevel);
+
+ while (candidateConnections.hasNext()) {
+ final Vertex candidateNode = candidateConnections.next();
+
+ TDistance candidateDistance = distanceFunction.distance(destination, getVectorFromVertex(candidateNode));
+ if (lt(candidateDistance, curDist)) {
+ curDist = candidateDistance;
+ currObj = candidateNode;
+ changed = true;
+ }
+ }
+
+ }
+ }
+
+ final PriorityQueue> topCandidates = searchBaseLayer(currObj, destination, Math.max(ef, k), 0);
+
+ while (topCandidates.size() > k) {
+ topCandidates.poll();
+ }
+
+ List> results = new ArrayList<>(topCandidates.size());
+ while (!topCandidates.isEmpty()) {
+ NodeIdAndDistance pair = topCandidates.poll();
+ results.add(0, new SearchResult<>(loadVertexFromRID(pair.nodeId), pair.distance, maxValueDistanceComparator));
+ }
+
+ return results;
+ }
+
+ private PriorityQueue> searchBaseLayer(final Vertex entryPointNode, final TVector destination, final int k, final int layer) {
+ final Set visitedNodes = new HashSet<>();
+
+ final PriorityQueue> topCandidates = new PriorityQueue<>(Comparator.>naturalOrder().reversed());
+ final PriorityQueue> candidateSet = new PriorityQueue<>();
+
+ TDistance lowerBound;
+
+ final TVector entryPointVector = getVectorFromVertex(entryPointNode);
+
+ final TDistance distance = distanceFunction.distance(destination, entryPointVector);
+ final NodeIdAndDistance pair = new NodeIdAndDistance<>(entryPointNode.getIdentity(), distance, maxValueDistanceComparator);
+
+ topCandidates.add(pair);
+ lowerBound = distance;
+ candidateSet.add(pair);
+
+ // TODO: MANAGE WHEN ENTRY POINT WAS DELETED
+// lowerBound = MaxValueComparator.maxValue();
+// NodeIdAndDistance pair = new NodeIdAndDistance<>(entryPointNode.id, lowerBound, maxValueDistanceComparator);
+// candidateSet.add(pair);
+
+ visitedNodes.add(entryPointNode.getIdentity());
+
+ while (!candidateSet.isEmpty()) {
+ final NodeIdAndDistance currentPair = candidateSet.poll();
+
+ if (gt(currentPair.distance, lowerBound))
+ break;
+
+ final Vertex node = loadVertexFromRID(currentPair.nodeId);
+
+ final Iterator candidates = getConnectionsFromVertex(node, layer);
+ while (candidates.hasNext()) {
+ final Vertex candidateNode = candidates.next();
+
+ if (!visitedNodes.contains(candidateNode.getIdentity())) {
+ visitedNodes.add(candidateNode.getIdentity());
+
+ final TDistance candidateDistance = distanceFunction.distance(destination, getVectorFromVertex(candidateNode));
+ if (topCandidates.size() < k || gt(lowerBound, candidateDistance)) {
+ final NodeIdAndDistance candidatePair = new NodeIdAndDistance<>(candidateNode.getIdentity(), candidateDistance,
+ maxValueDistanceComparator);
+
+ candidateSet.add(candidatePair);
+ topCandidates.add(candidatePair);
+
+ if (topCandidates.size() > k)
+ topCandidates.poll();
+
+ if (!topCandidates.isEmpty())
+ lowerBound = topCandidates.peek().distance;
+ }
+ }
+ }
+ }
+
+ return topCandidates;
+ }
+
+ /**
+ * Returns the dimensionality of the items stored in this index.
+ *
+ * @return the dimensionality of the items stored in this index
+ */
+ public int getDimensions() {
+ return dimensions;
+ }
+
+ /**
+ * Returns the number of bi-directional links created for every new element during construction.
+ *
+ * @return the number of bi-directional links created for every new element during construction
+ */
+ public int getM() {
+ return m;
+ }
+
+ /**
+ * The size of the dynamic list for the nearest neighbors (used during the search)
+ *
+ * @return The size of the dynamic list for the nearest neighbors
+ */
+ public int getEf() {
+ return ef;
+ }
+
+ /**
+ * Set the size of the dynamic list for the nearest neighbors (used during the search)
+ *
+ * @param ef The size of the dynamic list for the nearest neighbors
+ */
+ public void setEf(int ef) {
+ this.ef = ef;
+ }
+
+ /**
+ * Returns the parameter has the same meaning as ef, but controls the index time / index precision.
+ *
+ * @return the parameter has the same meaning as ef, but controls the index time / index precision
+ */
+ public int getEfConstruction() {
+ return efConstruction;
+ }
+
+ /**
+ * Returns the distance function.
+ *
+ * @return the distance function
+ */
+ public DistanceFunction getDistanceFunction() {
+ return distanceFunction;
+ }
+
+ /**
+ * Returns the comparator used to compare distances.
+ *
+ * @return the comparator used to compare distance
+ */
+ public Comparator getDistanceComparator() {
+ return distanceComparator;
+ }
+
+ /**
+ * Returns the maximum number of items the index can hold.
+ *
+ * @return the maximum number of items the index can hold
+ */
+ public int getMaxItemCount() {
+ return maxItemCount;
+ }
+
+ public void save(OutputStream out) throws IOException {
+ try (ObjectOutputStream oos = new ObjectOutputStream(out)) {
+ oos.writeObject(this);
+ }
+ }
+
+ private int assignLevel(final TId value, final double lambda) {
+ // by relying on the external id to come up with the level, the graph construction should be a lot more stable
+ // see : https://github.com/nmslib/hnswlib/issues/28
+ final int hashCode = value.hashCode();
+ final byte[] bytes = new byte[] { (byte) (hashCode >> 24), (byte) (hashCode >> 16), (byte) (hashCode >> 8), (byte) hashCode };
+ final double random = Math.abs((double) Murmur3.hash32(bytes) / (double) Integer.MAX_VALUE);
+ final double r = -Math.log(random) * lambda;
+ return (int) r;
+ }
+
+ private boolean lt(final TDistance x, final TDistance y) {
+ return maxValueDistanceComparator.compare(x, y) < 0;
+ }
+
+ private boolean gt(final TDistance x, final TDistance y) {
+ return maxValueDistanceComparator.compare(x, y) > 0;
+ }
+
+ public TId getIdFromVertex(final Vertex vertex) {
+ return (TId) vertex.get(idPropertyName);
+ }
+
+ public TVector getVectorFromVertex(final Vertex vertex) {
+ return (TVector) vertex.get(vectorPropertyName);
+ }
+
+ public int getDimensionFromVertex(final Vertex vertex) {
+ return Array.getLength(getVectorFromVertex(vertex));
+ }
+
+ public String getEdgeType(final int level) {
+ return edgeType + level;
+ }
+
+ static class NodeIdAndDistance implements Comparable> {
+ final RID nodeId;
+ final TDistance distance;
+ final Comparator distanceComparator;
+
+ NodeIdAndDistance(final RID nodeId, final TDistance distance, final Comparator distanceComparator) {
+ this.nodeId = nodeId;
+ this.distance = distance;
+ this.distanceComparator = distanceComparator;
+ }
+
+ @Override
+ public int compareTo(NodeIdAndDistance o) {
+ return distanceComparator.compare(distance, o.distance);
+ }
+ }
+
+ static class MaxValueComparator implements Comparator, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ private final Comparator delegate;
+
+ MaxValueComparator(Comparator delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public int compare(final TDistance o1, final TDistance o2) {
+ return o1 == null ? o2 == null ? 0 : 1 : o2 == null ? -1 : delegate.compare(o1, o2);
+ }
+
+ static TDistance maxValue() {
+ return null;
+ }
+ }
+
+ public void save() throws IOException {
+ FileUtils.writeFile(new File(filePath), toJSON().toString());
+ }
+
+ @Override
+ public JSONObject toJSON() {
+ final JSONObject json = new JSONObject();
+ json.put("indexName", getName());
+ json.put("version", CURRENT_VERSION);
+ json.put("dimensions", dimensions);
+ json.put("distanceFunction", distanceFunction.getClass().getSimpleName());
+ json.put("distanceComparator", distanceComparator.getClass().getSimpleName());
+ json.put("maxItemCount", maxItemCount);
+ json.put("m", m);
+ json.put("maxM", maxM);
+ json.put("maxM0", maxM0);
+ json.put("levelLambda", levelLambda);
+ json.put("ef", ef);
+ json.put("efConstruction", efConstruction);
+ json.put("levelLambda", levelLambda);
+ json.put("entryPoint", entryPoint == null ? "" : entryPoint.getIdentity().toString());
+
+ json.put("vertexType", vertexType);
+ json.put("edgeType", edgeType);
+ json.put("idPropertyName", idPropertyName);
+ json.put("vectorPropertyName", vectorPropertyName);
+ return json;
+ }
+
+ @Override
+ public void drop() {
+ underlyingIndex.drop();
+ }
+
+ @Override
+ public Map getStats() {
+ return underlyingIndex.getStats();
+ }
+
+ @Override
+ public LSMTreeIndexAbstract.NULL_STRATEGY getNullStrategy() {
+ return underlyingIndex.getNullStrategy();
+ }
+
+ @Override
+ public void setNullStrategy(final LSMTreeIndexAbstract.NULL_STRATEGY nullStrategy) {
+ underlyingIndex.setNullStrategy(nullStrategy);
+ }
+
+ @Override
+ public boolean isUnique() {
+ return underlyingIndex.isUnique();
+ }
+
+ @Override
+ public boolean supportsOrderedIterations() {
+ return underlyingIndex.supportsOrderedIterations();
+ }
+
+ @Override
+ public boolean isAutomatic() {
+ return underlyingIndex.isAutomatic();
+ }
+
+ @Override
+ public int getPageSize() {
+ return underlyingIndex.getPageSize();
+ }
+
+ @Override
+ public long build(final int buildIndexBatchSize, final BuildIndexCallback callback) {
+ return underlyingIndex.build(buildIndexBatchSize, callback);
+ }
+
+ public long build(final HnswVectorIndexRAM origin, final int buildIndexBatchSize, final BuildIndexCallback vertexCreationCallback,
+ final BuildIndexCallback edgeCallback) {
+ if (origin != null) {
+ // IMPORT FROM RAM Index
+ final RID[] pointersToRIDMapping = new RID[origin.size()];
+
+ database.begin();
+
+ // SAVE ALL THE NODES AS VERTICES AND KEEP AN ARRAY OF RIDS TO BUILD EDGES LATER
+ int maxLevel = 0;
+ HnswVectorIndexRAM.ItemIterator iter = origin.iterateNodes();
+ for (int totalVertices = 0; iter.hasNext(); ++totalVertices) {
+ final HnswVectorIndexRAM.Node node = iter.next();
+
+ final int nodeMaxLevel = node.maxLevel();
+ if (nodeMaxLevel > maxLevel)
+ maxLevel = nodeMaxLevel;
+
+ final MutableVertex vertex = database.newVertex(vertexType).set(idPropertyName, node.item.id()).set(vectorPropertyName, node.item.vector());
+ if (nodeMaxLevel > 0)
+ // SAVE MAX LEVEL INTO THE VERTEX. IF NOT PRESENT, MEANS 0
+ vertex.set("vectorMaxLevel", nodeMaxLevel);
+
+ vertex.save();
+
+ if (vertexCreationCallback != null)
+ vertexCreationCallback.onDocumentIndexed(vertex, totalVertices);
+
+ pointersToRIDMapping[node.id] = vertex.getIdentity();
+
+ if (totalVertices % buildIndexBatchSize == 0) {
+ database.commit();
+ database.begin();
+ }
+ }
+
+ database.commit();
+
+ final Integer entryPoint = origin.getEntryPoint();
+ if (entryPoint != null)
+ this.entryPoint = pointersToRIDMapping[entryPoint].asVertex();
+
+ // BUILD ALL EDGE TYPES (ONE PER LEVEL)
+ for (int level = 0; level <= maxLevel; level++) {
+ // ASSURE THE EDGE TYPE IS CREATED IN THE DATABASE
+ database.getSchema().getOrCreateEdgeType(getEdgeType(level));
+ }
+
+ database.begin();
+
+ // BUILD THE EDGES
+ long totalVertices = 0L;
+ long totalEdges = 0L;
+ iter = origin.iterateNodes();
+ for (int txCounter = 0; iter.hasNext(); ++txCounter) {
+ final HnswVectorIndexRAM.Node node = iter.next();
+
+ final Vertex source = pointersToRIDMapping[node.id].asVertex();
+ ++totalVertices;
+
+ final MutableIntList[] connections = node.connections();
+ for (int level = 0; level < connections.length; level++) {
+ final String edgeTypeLevel = getEdgeType(level);
+
+ final MutableIntList pointers = connections[level];
+ for (int i = 0; i < pointers.size(); i++) {
+ final int pointer = pointers.get(i);
+
+ final RID destination = pointersToRIDMapping[pointer];
+ source.newEdge(edgeTypeLevel, destination, false);
+ ++totalEdges;
+ }
+ }
+
+ if (txCounter % buildIndexBatchSize == 0) {
+ database.commit();
+ database.begin();
+ }
+
+ if (edgeCallback != null)
+ edgeCallback.onDocumentIndexed(source, totalEdges);
+ }
+
+ database.commit();
+ return totalVertices;
+ }
+
+ // TODO: NOT SUPPORTED WITHOUT RAM INDEX
+ return 0L;
+ }
+
+ @Override
+ public boolean equals(final Object obj) {
+ if (!(obj instanceof HnswVectorIndex))
+ return false;
+ return componentName.equals(((HnswVectorIndex) obj).componentName) && underlyingIndex.equals(obj);
+ }
+
+ public List getSubIndexes() {
+ return underlyingIndex.getSubIndexes();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(componentName, underlyingIndex.hashCode());
+ }
+
+ @Override
+ public String toString() {
+ final String baseIndex = underlyingIndex.toString();
+ return baseIndex.substring(0, baseIndex.length() - 1) + "," + vectorPropertyName + "]";
+ }
+
+ @Override
+ public void setMetadata(final String name, final String[] propertyNames, final int associatedBucketId) {
+ underlyingIndex.setMetadata(name, propertyNames, associatedBucketId);
+ }
+
+ @Override
+ public Component getComponent() {
+ return this;
+ }
+
+ @Override
+ public Type[] getKeyTypes() {
+ return underlyingIndex.getKeyTypes();
+ }
+
+ @Override
+ public byte[] getBinaryKeyTypes() {
+ return underlyingIndex.getBinaryKeyTypes();
+ }
+
+ @Override
+ public List getFileIds() {
+ return underlyingIndex.getFileIds();
+ }
+
+ @Override
+ public void setTypeIndex(final TypeIndex typeIndex) {
+ underlyingIndex.setTypeIndex(typeIndex);
+ }
+
+ @Override
+ public TypeIndex getTypeIndex() {
+ return underlyingIndex.getTypeIndex();
+ }
+
+ @Override
+ public int getAssociatedBucketId() {
+ return underlyingIndex.getAssociatedBucketId();
+ }
+
+ public void addIndexOnBucket(final IndexInternal index) {
+ underlyingIndex.addIndexOnBucket(index);
+ }
+
+ public void removeIndexOnBucket(final IndexInternal index) {
+ underlyingIndex.removeIndexOnBucket(index);
+ }
+
+ public IndexInternal[] getIndexesOnBuckets() {
+ return underlyingIndex.getIndexesOnBuckets();
+ }
+
+ public List getIndexesByKeys(final Object[] keys) {
+ return underlyingIndex.getIndexesByKeys(keys);
+ }
+
+ public IndexCursor iterator(final boolean ascendingOrder) {
+ return underlyingIndex.iterator(ascendingOrder);
+ }
+
+ public IndexCursor iterator(final boolean ascendingOrder, final Object[] fromKeys, final boolean inclusive) {
+ return underlyingIndex.iterator(ascendingOrder, fromKeys, inclusive);
+ }
+
+ public IndexCursor range(final boolean ascending, final Object[] beginKeys, final boolean beginKeysInclusive, final Object[] endKeys,
+ boolean endKeysInclusive) {
+ return underlyingIndex.range(ascending, beginKeys, beginKeysInclusive, endKeys, endKeysInclusive);
+ }
+
+ @Override
+ public IndexCursor get(final Object[] keys) {
+ return underlyingIndex.get(keys);
+ }
+
+ @Override
+ public IndexCursor get(final Object[] keys, final int limit) {
+ return underlyingIndex.get(keys, limit);
+ }
+
+ @Override
+ public void put(final Object[] keys, RID[] rid) {
+ underlyingIndex.put(keys, rid);
+ }
+
+ @Override
+ public void remove(final Object[] keys) {
+ globalLock.lock();
+ try {
+ final IndexCursor cursor = underlyingIndex.get(keys);
+ if (!cursor.hasNext())
+ return;
+
+ final Vertex vertex = loadVertexFromRID(cursor.next());
+ if (vertex.equals(entryPoint)) {
+ // TODO: CHANGE THE ENTRYPOINT
+ }
+
+ vertex.delete();
+ } finally {
+ globalLock.unlock();
+ }
+ }
+
+ @Override
+ public void remove(final Object[] keys, final Identifiable rid) {
+ globalLock.lock();
+ try {
+ final IndexCursor cursor = underlyingIndex.get(keys);
+ if (!cursor.hasNext())
+ return;
+
+ final Identifiable itemRID = cursor.next();
+ if (!itemRID.equals(rid))
+ return;
+
+ final Vertex vertex = loadVertexFromRID(itemRID);
+ if (vertex.equals(entryPoint)) {
+ // TODO: CHANGE THE ENTRYPOINT
+ }
+
+ vertex.delete();
+ } finally {
+ globalLock.unlock();
+ }
+ }
+
+ @Override
+ public long countEntries() {
+ return underlyingIndex.countEntries();
+ }
+
+ @Override
+ public boolean compact() throws IOException, InterruptedException {
+ return underlyingIndex.compact();
+ }
+
+ @Override
+ public boolean isCompacting() {
+ return underlyingIndex.isCompacting();
+ }
+
+ @Override
+ public boolean scheduleCompaction() {
+ return underlyingIndex.scheduleCompaction();
+ }
+
+ @Override
+ public String getMostRecentFileName() {
+ return underlyingIndex.getMostRecentFileName();
+ }
+
+ @Override
+ public EmbeddedSchema.INDEX_TYPE getType() {
+ return EmbeddedSchema.INDEX_TYPE.HSNW;
+ }
+
+ @Override
+ public String getTypeName() {
+ return vertexType;
+ }
+
+ @Override
+ public List getPropertyNames() {
+ return List.of(idPropertyName, vectorPropertyName);
+ }
+
+ @Override
+ public void close() {
+ underlyingIndex.close();
+ }
+
+ private Vertex get(final Object id) {
+ globalLock.lock();
+ try {
+ final IndexCursor cursor = underlyingIndex.get(new Object[] { id });
+ if (!cursor.hasNext())
+ return null;
+
+ return loadVertexFromRID(cursor.next());
+ } finally {
+ globalLock.unlock();
+ }
+ }
+}
diff --git a/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndexRAM.java b/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndexRAM.java
new file mode 100644
index 000000000..f4e32a464
--- /dev/null
+++ b/engine/src/main/java/com/arcadedb/index/vector/HnswVectorIndexRAM.java
@@ -0,0 +1,1047 @@
+package com.arcadedb.index.vector;
+
+import com.arcadedb.database.Database;
+import com.arcadedb.schema.VectorIndexBuilder;
+import com.github.jelmerk.knn.DistanceFunction;
+import com.github.jelmerk.knn.Index;
+import com.github.jelmerk.knn.Item;
+import com.github.jelmerk.knn.SearchResult;
+import com.github.jelmerk.knn.hnsw.SizeLimitExceededException;
+import com.github.jelmerk.knn.util.ArrayBitSet;
+import com.github.jelmerk.knn.util.ClassLoaderObjectInputStream;
+import com.github.jelmerk.knn.util.GenericObjectPool;
+import com.github.jelmerk.knn.util.Murmur3;
+import org.eclipse.collections.api.list.primitive.MutableIntList;
+import org.eclipse.collections.api.map.primitive.MutableObjectIntMap;
+import org.eclipse.collections.api.map.primitive.MutableObjectLongMap;
+import org.eclipse.collections.impl.list.mutable.primitive.IntArrayList;
+import org.eclipse.collections.impl.map.mutable.primitive.ObjectIntHashMap;
+import org.eclipse.collections.impl.map.mutable.primitive.ObjectLongHashMap;
+
+import java.io.*;
+import java.nio.file.*;
+import java.util.*;
+import java.util.concurrent.atomic.*;
+import java.util.concurrent.locks.*;
+
+/**
+ * This work is derived from the excellent work made by Jelmer Kuperus on https://github.com/jelmerk/hnswlib. We forked the entire class only because it was
+ * not extensible (private members).
+ *
+ * Implementation of {@link Index} that implements the hnsw algorithm.
+ *
+ * @param Type of the external identifier of an item
+ * @param Type of the vector to perform distance calculation on
+ * @param Type of items stored in the index
+ * @param Type of distance between items (expect any numeric type: float, double, int, ..)
+ *
+ * @see
+ * Efficient and robust approximate nearest neighbor search using Hierarchical Navigable Small World graphs
+ */
+public class HnswVectorIndexRAM, TDistance> implements Index {
+
+ private static final byte VERSION_1 = 0x01;
+
+ private static final long serialVersionUID = 1L;
+
+ private static final int NO_NODE_ID = -1;
+
+ private DistanceFunction distanceFunction;
+ private Comparator