diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 00000000..6971fb87 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,8 @@ +# Description & motivation +Please provide some context for your reviewers +- *What does this PR do?* +- *Why does this PR do it?* + +# Verification +Please describe how you verified that the change works as intended +- *Did you add a unit test?* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..e975fee3 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +name: CI pipeline + +on: + pull_request: + paths: + - '**' + push: + branches: + - '*' + tags-ignore: + - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + ci-pipeline: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + spark: +# - 2.4.8 +# - 3.0.2 +# - 3.1.3 + - 3.2.4 +# - 3.3.2 +# - 3.4.1 +# - 3.5.0 + env: + ENV: 'ci' + SPARK_VERSION: ${{ matrix.spark }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Build and test + run: | + sbt -java-home "$JAVA_HOME_17_X64" clean +test +publishM2 pyTest -DsparkVersion="$SPARK_VERSION" + - name: Publish Unit test results + uses: mikepenz/action-junit-report@v4 + with: + report_paths: '**/target/test-reports/TEST-*.xml' + check_name: Unit test results + if: always() diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..4646f7d4 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,43 @@ +name: Publish pipeline + +on: + workflow_dispatch: + + push: + tags: + - 'v[0-9]+.[0-9]+.[0-9]+' + +jobs: + publish-artifacts: + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + spark: +# - 2.4.8 +# - 3.0.2 +# - 3.1.3 + - 3.2.4 +# - 3.3.2 +# - 3.4.1 +# - 3.5.0 + + env: + ENV: 'ci' + SPARK_VERSION: ${{ matrix.spark }} + NEXUS_USER: ${{ secrets.NEXUS_USER }} + NEXUS_PASSWORD: ${{ secrets.NEXUS_PASSWORD }} + + steps: + - name: Checkout + uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Import GPG Key + uses: crazy-max/ghaction-import-gpg@v1 + env: + GPG_PRIVATE_KEY: ${{ secrets.GPG_PRIVATE_KEY }} + PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }} + - name: Publish artifacts + run: | + sbt -java-home "$JAVA_HOME_17_X64" clean +publishSigned -DsparkVersion="$SPARK_VERSION" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..201abf5b --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,21 @@ +name: Release pipeline + +on: + workflow_dispatch: + inputs: + version: + description: Semantic version. For example 1.0.0 + required: true + +jobs: + ci-pipeline: + runs-on: ubuntu-22.04 + steps: + - name: Checkout main branch + uses: actions/checkout@v3 + - name: Release + run: | + git config --global user.email "action@github.com" + git config --global user.name "GitHub Action" + git tag -a v${{ github.event.inputs.version }} -m "next release" + git push --tags diff --git a/.gitignore b/.gitignore index 4bfa5506..a68f2928 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,26 @@ +# Intellij idea .idea/ +.bsp/ *.iml + +# vscode with metals +.bloop/ +.vscode/ +.metals/ +metals.sbt + +# output folders target/ +dist/ + +# MacOS +.DS_Store + +# Virtual env +.venv + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +.pytest_cache/ \ No newline at end of file diff --git a/.mvn/wrapper/MavenWrapperDownloader.java b/.mvn/wrapper/MavenWrapperDownloader.java deleted file mode 100755 index fa4f7b49..00000000 --- a/.mvn/wrapper/MavenWrapperDownloader.java +++ /dev/null @@ -1,110 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one -or more contributor license agreements. See the NOTICE file -distributed with this work for additional information -regarding copyright ownership. The ASF licenses this file -to you under the Apache License, Version 2.0 (the -"License"); you may not use this file except in compliance -with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, -software distributed under the License is distributed on an -"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -KIND, either express or implied. See the License for the -specific language governing permissions and limitations -under the License. -*/ - -import java.net.*; -import java.io.*; -import java.nio.channels.*; -import java.util.Properties; - -public class MavenWrapperDownloader { - - /** - * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. - */ - private static final String DEFAULT_DOWNLOAD_URL = - "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar"; - - /** - * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to - * use instead of the default one. - */ - private static final String MAVEN_WRAPPER_PROPERTIES_PATH = - ".mvn/wrapper/maven-wrapper.properties"; - - /** - * Path where the maven-wrapper.jar will be saved to. - */ - private static final String MAVEN_WRAPPER_JAR_PATH = - ".mvn/wrapper/maven-wrapper.jar"; - - /** - * Name of the property which should be used to override the default download url for the wrapper. - */ - private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; - - public static void main(String args[]) { - System.out.println("- Downloader started"); - File baseDirectory = new File(args[0]); - System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); - - // If the maven-wrapper.properties exists, read it and check if it contains a custom - // wrapperUrl parameter. - File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); - String url = DEFAULT_DOWNLOAD_URL; - if(mavenWrapperPropertyFile.exists()) { - FileInputStream mavenWrapperPropertyFileInputStream = null; - try { - mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); - Properties mavenWrapperProperties = new Properties(); - mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); - url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); - } catch (IOException e) { - System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); - } finally { - try { - if(mavenWrapperPropertyFileInputStream != null) { - mavenWrapperPropertyFileInputStream.close(); - } - } catch (IOException e) { - // Ignore ... - } - } - } - System.out.println("- Downloading from: : " + url); - - File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); - if(!outputFile.getParentFile().exists()) { - if(!outputFile.getParentFile().mkdirs()) { - System.out.println( - "- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'"); - } - } - System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); - try { - downloadFileFromURL(url, outputFile); - System.out.println("Done"); - System.exit(0); - } catch (Throwable e) { - System.out.println("- Error downloading"); - e.printStackTrace(); - System.exit(1); - } - } - - private static void downloadFileFromURL(String urlString, File destination) throws Exception { - URL website = new URL(urlString); - ReadableByteChannel rbc; - rbc = Channels.newChannel(website.openStream()); - FileOutputStream fos = new FileOutputStream(destination); - fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); - fos.close(); - rbc.close(); - } - -} diff --git a/.mvn/wrapper/maven-wrapper.jar b/.mvn/wrapper/maven-wrapper.jar deleted file mode 100755 index 01e67997..00000000 Binary files a/.mvn/wrapper/maven-wrapper.jar and /dev/null differ diff --git a/.mvn/wrapper/maven-wrapper.properties b/.mvn/wrapper/maven-wrapper.properties deleted file mode 100755 index 00d32aab..00000000 --- a/.mvn/wrapper/maven-wrapper.properties +++ /dev/null @@ -1 +0,0 @@ -distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.5.4/apache-maven-3.5.4-bin.zip \ No newline at end of file diff --git a/.sbtopts b/.sbtopts new file mode 100644 index 00000000..e2220ae7 --- /dev/null +++ b/.sbtopts @@ -0,0 +1 @@ +-J--enable-preview -J--add-modules=jdk.incubator.vector -Dsbt.argsfile=false \ No newline at end of file diff --git a/.sdkmanrc b/.sdkmanrc new file mode 100644 index 00000000..31a23dca --- /dev/null +++ b/.sdkmanrc @@ -0,0 +1,5 @@ +# Enable auto-env through the sdkman_auto_env config +# Add key=value pairs of SDKs to use below +java=17.0.9-amzn +scala=2.12.18 +sbt=1.9.8 \ No newline at end of file diff --git a/.travis-toolchains.xml b/.travis-toolchains.xml deleted file mode 100644 index e615b756..00000000 --- a/.travis-toolchains.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - jdk - - 8 - sun - - - /usr/lib/jvm/java-8-openjdk-amd64 - - - - jdk - - 17 - oracle - - - /home/travis/openjdk17 - - - \ No newline at end of file diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 75c2ff11..00000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -sudo: required - -dist: xenial - -language: java - -script: - - ./crossbuild.sh clean install - -jdk: - - openjdk8 - -cache: - directories: - - $HOME/.m2 - -before_install: - - curl -s -o ~/bin/install-jdk.sh "https://raw.githubusercontent.com/sormuras/bach/master/install-jdk.sh" - - chmod +x ~/bin/install-jdk.sh - - ~/bin/install-jdk.sh --target "/home/travis/openjdk17" --workspace "/home/travis/.cache/install-jdk" --feature "17" --license "GPL" --cacerts - - cp .travis-toolchains.xml ~/.m2/toolchains.xml \ No newline at end of file diff --git a/README.md b/README.md index 48a0381e..1bc9bc96 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://app.travis-ci.com/jelmerk/hnswlib.svg?branch=master)](https://app.travis-ci.com/jelmerk/hnswlib) +[![Build Status](https://github.com/jelmerk/hnswlib/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/jelmerk/hnswlib/actions/workflows/ci.yml) Hnswlib ======= diff --git a/RELEASING.md b/RELEASING.md deleted file mode 100644 index 5a9c1c4a..00000000 --- a/RELEASING.md +++ /dev/null @@ -1,37 +0,0 @@ -Releasing hnswlib -================= - -Releasing a new version to maven central is a bit cumbersome because maven does not play nice with cross releasing scala libraries - -1. run export GPG_TTY=$(tty) - -2. update the version number to the release version in every pom - -3. commit - - git commit -am "Prepare release" - -4. tag the release - - git tag v0.x.x HEAD - -5. ./crossbuild.sh clean deploy -DperformRelease=true - -6. release the pyspark module with - - rm -rf build ; rm -rf dist ; rm -rf pyspark_hnsw.egg-info ; python2.7 setup.py clean --all bdist_wheel && python2.7 -m twine upload dist/* - rm -rf build ; rm -rf dist ; rm -rf pyspark_hnsw.egg-info ; python3.7 setup.py clean --all bdist_wheel && python3.7 -m twine upload dist/* - -7. update the version number to the development version version in every pom to the new development version - -8. update the version in hnswlib-pyspark/setup.py - -9. commit - - git commit -am "Next development version" - -10. push - - git push - git push --tags - diff --git a/build.sbt b/build.sbt new file mode 100644 index 00000000..3096fb5d --- /dev/null +++ b/build.sbt @@ -0,0 +1,209 @@ +import Path.relativeTo +import sys.process.* + +ThisBuild / organization := "com.github.jelmerk" +ThisBuild / scalaVersion := "2.12.18" + +ThisBuild / fork := true + +ThisBuild / dynverSonatypeSnapshots := true +ThisBuild / publishMavenStyle := true + +val java8Home = sys.env.getOrElse("JAVA_HOME_8_X64", s"${sys.props("user.home")}/.sdkman/candidates/java/8.0.382-amzn") + +lazy val publishSettings = Seq( + pomIncludeRepository := { _ => false }, + publishTo := { + val nexus = "https://oss.sonatype.org/" + if (isSnapshot.value) Some("snapshots" at nexus + "content/repositories/snapshots") + else Some("releases" at nexus + "service/local/staging/deploy/maven2") + }, + + licenses := Seq("Apache License 2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.html")), + + homepage := Some(url("https://github.com/jelmerk/hnswlib")), + + scmInfo := Some(ScmInfo( + url("https://github.com/jelmerk/hnswlib.git"), + "scm:git@github.com:jelmerk/hnswlib.git" + )), + + developers := List( + Developer("jelmerk", "Jelmer Kuperus", "jkuperus@gmail.com", url("https://github.com/jelmerk")) + ), + + ThisBuild / credentials += Credentials( + "Sonatype Nexus Repository Manager", + "oss.sonatype.org", + sys.env.getOrElse("NEXUS_USER", ""), + sys.env.getOrElse("NEXUS_PASSWORD", "") + ) +) + +lazy val noPublishSettings = + publish / skip := true + +val scalaTestVersion = "3.2.17" +val junitVersion = "5.5.2" +val hamcrestVersion = "2.1" +val mockitoVersion = "3.0.0" + +val sparkVersion = settingKey[String]("Spark version") + +lazy val pyTest = taskKey[Unit]("Run the python tests") +lazy val pyPublish = taskKey[Unit]("Publish the python sources to a pypi repo") + +lazy val root = (project in file(".")) + .aggregate(hnswlibUtils, hnswlibCore, hnswlibCoreJdk17, hnswlibMetricsDropwizard, hnswlibScala, hnswlibSpark) + .settings(noPublishSettings) + +lazy val hnswlibUtils = (project in file("hnswlib-utils")) + .settings( + name := "hnswlib-utils", + autoScalaLibrary := false, + crossPaths := false, + publishSettings, + Compile / compile / javacOptions ++= Seq( + "-source", "8", + "-target", "8" + ), + libraryDependencies ++= Seq( + "org.hamcrest" % "hamcrest-library" % hamcrestVersion % Test, + "org.junit.jupiter" % "junit-jupiter-engine" % junitVersion % Test, + "org.junit.jupiter" % "junit-jupiter-api" % junitVersion % Test, + "net.aichler" % "jupiter-interface" % JupiterKeys.jupiterVersion.value % Test + ) + ) + +lazy val hnswlibCore = (project in file("hnswlib-core")) + .dependsOn(hnswlibUtils % "test->compile") + .settings( + name := "hnswlib-core", + autoScalaLibrary := false, + crossPaths := false, + publishSettings, + Compile / compile / javacOptions ++= Seq( + "-source", "8", + "-target", "8" + ), + libraryDependencies ++= Seq( + "org.eclipse.collections" % "eclipse-collections" % "9.2.0", + "org.hamcrest" % "hamcrest-library" % hamcrestVersion % Test, + "org.junit.jupiter" % "junit-jupiter-engine" % junitVersion % Test, + "org.junit.jupiter" % "junit-jupiter-api" % junitVersion % Test, + "net.aichler" % "jupiter-interface" % JupiterKeys.jupiterVersion.value % Test + ) + ) + +lazy val hnswlibCoreJdk17 = (project in file("hnswlib-core-jdk17")) + .dependsOn(hnswlibCore) + .settings( + name := "hnswlib-core-jdk17", + autoScalaLibrary := false, + crossPaths := false, + publishSettings, + Compile / compile / javacOptions ++= Seq( + "-source", "17", + "-target", "17", + "--enable-preview", + "--add-modules", "jdk.incubator.vector" + ), + Compile / doc / javacOptions ++= Seq( + "-source", "17", + "--enable-preview", + "--add-modules", "jdk.incubator.vector" + ), + Test / javaOptions ++= Seq( + "--enable-preview", + "--add-modules", "jdk.incubator.vector" + ), + libraryDependencies ++= Seq( + "org.hamcrest" % "hamcrest-library" % hamcrestVersion % Test, + "org.junit.jupiter" % "junit-jupiter-engine" % junitVersion % Test, + "org.junit.jupiter" % "junit-jupiter-api" % junitVersion % Test, + "net.aichler" % "jupiter-interface" % JupiterKeys.jupiterVersion.value % Test + ) + ) + +lazy val hnswlibMetricsDropwizard = (project in file("hnswlib-metrics-dropwizard")) + .dependsOn(hnswlibCore) + .settings( + name := "hnswlib-metrics-dropwizard", + autoScalaLibrary := false, + crossPaths := false, + publishSettings, + Compile / compile / javacOptions ++= Seq( + "-source", "8", + "-target", "8" + ), + libraryDependencies ++= Seq( + "io.dropwizard.metrics" % "metrics-core" % "4.1.0", + "org.awaitility" % "awaitility" % "4.0.1" % Test, + "org.mockito" % "mockito-junit-jupiter" % mockitoVersion % Test, + "org.mockito" % "mockito-core" % mockitoVersion % Test, + "org.hamcrest" % "hamcrest-library" % hamcrestVersion % Test, + "org.junit.jupiter" % "junit-jupiter-engine" % junitVersion % Test, + "org.junit.jupiter" % "junit-jupiter-api" % junitVersion % Test, + "net.aichler" % "jupiter-interface" % JupiterKeys.jupiterVersion.value % Test + ) + ) + +lazy val hnswlibScala = (project in file("hnswlib-scala")) + .dependsOn(hnswlibCore) + .dependsOn(hnswlibMetricsDropwizard % Optional) + .settings( + name := "hnswlib-scala", + crossScalaVersions := List("2.11.12", "2.12.18", "2.13.10"), + publishSettings, + scalacOptions := Seq( + "-target:jvm-1.8", + "-encoding", "UTF-8" + ), + libraryDependencies ++= Seq( + "org.scalatest" %% "scalatest" % scalaTestVersion % Test + ) + ) + +lazy val hnswlibSpark = (project in file("hnswlib-spark")) + .dependsOn(hnswlibUtils) + .dependsOn(hnswlibScala) + .settings( + name := s"hnswlib-spark_${sparkVersion.value.split('.').take(2).mkString("-")}", + publishSettings, + crossScalaVersions := { + if (sparkVersion.value >= "3.2.0") { +// Seq("2.12.18", "2.13.10") + // TODO fix this + Seq("2.12.18") + } else if (sparkVersion.value >= "3.0.0") { + Seq("2.12.18") + } else { + Seq("2.12.18", "2.11.12") + } + }, + javaHome := Some(file(java8Home)), + Compile / unmanagedSourceDirectories += baseDirectory.value / "src" / "main" / "python", + Test / unmanagedSourceDirectories += baseDirectory.value / "src" / "test" / "python", + Compile / packageBin / mappings ++= { + val base = baseDirectory.value / "src" / "main" / "python" + val srcs = base ** "*.py" + srcs pair relativeTo(base) + }, + sparkVersion := sys.props.getOrElse("sparkVersion", "3.3.2"), + pyTest := { + val artifactName = s"${organization.value}:${name.value}_${scalaBinaryVersion.value}:${version.value}" + val ret = Process( + Seq("./run-pyspark-tests.sh", sparkVersion.value), + cwd = baseDirectory.value, + extraEnv = "JAVA_HOME" -> java8Home, "ARTIFACT" -> artifactName + ).! + require(ret == 0, "Python tests failed") + }, + pyTest := pyTest.dependsOn(publishM2).value, + libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-hive" % sparkVersion.value % Provided, + "org.apache.spark" %% "spark-mllib" % sparkVersion.value % Provided, + "com.holdenkarau" %% "spark-testing-base" % s"${sparkVersion.value}_1.4.7" % Test, + "org.scalatest" %% "scalatest" % scalaTestVersion % Test + ) + ) \ No newline at end of file diff --git a/crossbuild.sh b/crossbuild.sh deleted file mode 100755 index 2d1d04bf..00000000 --- a/crossbuild.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash - -set -e - -function cleanup() -{ - git checkout ./hnswlib-scala/pom.xml - git checkout ./hnswlib-spark/pom.xml -} - -trap cleanup EXIT - -ARGS=$* - -mvn $ARGS -N - -mvn $ARGS -pl hnswlib-utils - -mvn $ARGS -pl hnswlib-core - -mvn $ARGS -pl hnswlib-metrics-dropwizard - -mvn $ARGS -pl hnswlib-core-jdk17 - -cp ./hnswlib-scala/pom-scala-2_11.xml ./hnswlib-scala/pom.xml -mvn $ARGS -pl hnswlib-scala - -cp ./hnswlib-scala/pom-scala-2_12.xml ./hnswlib-scala/pom.xml -mvn $ARGS -pl hnswlib-scala - -cp ./hnswlib-scala/pom-scala-2_13.xml ./hnswlib-scala/pom.xml -mvn $ARGS -pl hnswlib-scala - -cp ./hnswlib-spark/pom-spark-2.3-scala-2_11.xml ./hnswlib-spark/pom.xml -mvn $ARGS -pl hnswlib-spark - -cp ./hnswlib-spark/pom-spark-2.4-scala-2_11.xml ./hnswlib-spark/pom.xml -mvn $ARGS -pl hnswlib-spark - -cp ./hnswlib-spark/pom-spark-2.4-scala-2_12.xml ./hnswlib-spark/pom.xml -mvn $ARGS -pl hnswlib-spark - -cp ./hnswlib-spark/pom-spark-3.0-scala-2_12.xml ./hnswlib-spark/pom.xml -mvn $ARGS -pl hnswlib-spark - -cp ./hnswlib-spark/pom-spark-3.1-scala-2_12.xml ./hnswlib-spark/pom.xml -mvn $ARGS -pl hnswlib-spark - -cp ./hnswlib-spark/pom-spark-3.2-scala-2_12.xml ./hnswlib-spark/pom.xml -mvn $ARGS -pl hnswlib-spark - -cp ./hnswlib-spark/pom-spark-3.3-scala-2_12.xml ./hnswlib-spark/pom.xml -mvn $ARGS -pl hnswlib-spark - - -mvn $ARGS -pl hnswlib-examples \ No newline at end of file diff --git a/hnswlib-core-jdk17/pom.xml b/hnswlib-core-jdk17/pom.xml deleted file mode 100644 index 92cf42b1..00000000 --- a/hnswlib-core-jdk17/pom.xml +++ /dev/null @@ -1,91 +0,0 @@ - - - 4.0.0 - - hnswlib-core-jdk17 - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - Jdk 17 vector intrinsics integration - - - - - com.github.jelmerk - hnswlib-core - ${project.version} - - - - org.hamcrest - hamcrest-library - - - - org.junit.jupiter - junit-jupiter-api - - - - org.junit.jupiter - junit-jupiter-engine - - - - - - - - - maven-surefire-plugin - - --enable-preview --add-modules jdk.incubator.vector - - - - - org.apache.maven.plugins - maven-compiler-plugin - - 17 - 17 - - --enable-preview - --add-modules - jdk.incubator.vector - - - 17 - - - - - - maven-toolchains-plugin - - - - toolchain - - - - - - - 17 - oracle - - - - - - - - diff --git a/hnswlib-core/pom.xml b/hnswlib-core/pom.xml deleted file mode 100644 index ab196ed2..00000000 --- a/hnswlib-core/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - 4.0.0 - - hnswlib-core - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - Core java library - - - - - org.eclipse.collections - eclipse-collections - - - - com.github.jelmerk - hnswlib-utils - ${project.version} - test - - - - org.hamcrest - hamcrest-library - - - - org.junit.jupiter - junit-jupiter-api - - - - org.junit.jupiter - junit-jupiter-engine - - - - - diff --git a/hnswlib-examples/hnswlib-examples-java/pom.xml b/hnswlib-examples/hnswlib-examples-java/pom.xml index 6167ee02..77b26a8e 100644 --- a/hnswlib-examples/hnswlib-examples-java/pom.xml +++ b/hnswlib-examples/hnswlib-examples-java/pom.xml @@ -4,17 +4,22 @@ 4.0.0 + com.github.jelmerk hnswlib-examples-java - - - com.github.jelmerk - hnswlib-examples-parent-pom - 1.1.0 - .. - + 0.0.1-SNAPSHOT Examples for the core java library + + UTF-8 + UTF-8 + + 1.8 + 1.8 + + 1.1.0 + + @@ -39,13 +44,13 @@ com.github.jelmerk hnswlib-core - ${project.version} + ${hnswlib.version} com.github.jelmerk hnswlib-utils - ${project.version} + ${hnswlib.version} diff --git a/hnswlib-examples/hnswlib-examples-scala/README.md b/hnswlib-examples/hnswlib-examples-scala/README.md index d4465001..9fc45052 100644 --- a/hnswlib-examples/hnswlib-examples-scala/README.md +++ b/hnswlib-examples/hnswlib-examples-scala/README.md @@ -5,4 +5,4 @@ Example scala application. To run execute - mvn exec:scala \ No newline at end of file + sbt run \ No newline at end of file diff --git a/hnswlib-examples/hnswlib-examples-scala/build.sbt b/hnswlib-examples/hnswlib-examples-scala/build.sbt new file mode 100644 index 00000000..c67f74f3 --- /dev/null +++ b/hnswlib-examples/hnswlib-examples-scala/build.sbt @@ -0,0 +1,9 @@ +organization := "com.github.jelmerk" +name := "hnswlib-examples-scala" +version := "0.1" + +scalaVersion := "2.12.18" + +Compile / mainClass := Some("com.github.jelmerk.knn.examples.FastText") + +libraryDependencies += "com.github.jelmerk" %% "hnswlib-scala" % "1.1.0" diff --git a/hnswlib-examples/hnswlib-examples-scala/pom.xml b/hnswlib-examples/hnswlib-examples-scala/pom.xml deleted file mode 100644 index 1b64ff01..00000000 --- a/hnswlib-examples/hnswlib-examples-scala/pom.xml +++ /dev/null @@ -1,123 +0,0 @@ - - - 4.0.0 - - hnswlib-examples-scala - - - com.github.jelmerk - hnswlib-examples-parent-pom - 1.1.0 - .. - - - - 2.11.12 - 2.11 - - - Examples for the scala integration - - - src/main/scala - src/test/scala - - - - org.codehaus.mojo - exec-maven-plugin - 1.2.1 - - - - java - - - - - com.github.jelmerk.knn.examples.FastText - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - ${scala.version} - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -optimise - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - - - - com.github.jelmerk - hnswlib-scala_${scala.binary.version} - ${project.version} - - - - com.github.jelmerk - hnswlib-metrics-dropwizard - ${project.version} - - - - org.scala-lang - scala-library - ${scala.version} - - - - - diff --git a/hnswlib-examples/hnswlib-examples-scala/project/build.properties b/hnswlib-examples/hnswlib-examples-scala/project/build.properties new file mode 100644 index 00000000..abbbce5d --- /dev/null +++ b/hnswlib-examples/hnswlib-examples-scala/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.9.8 diff --git a/hnswlib-examples/pom.xml b/hnswlib-examples/pom.xml deleted file mode 100644 index c7a5003c..00000000 --- a/hnswlib-examples/pom.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - 4.0.0 - - hnswlib-examples-parent-pom - pom - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - Example projects for hnswlib - - - hnswlib-examples-java - hnswlib-examples-scala - - - diff --git a/hnswlib-pyspark/DEVELOPMENT.md b/hnswlib-pyspark/DEVELOPMENT.md deleted file mode 100644 index a3f9348e..00000000 --- a/hnswlib-pyspark/DEVELOPMENT.md +++ /dev/null @@ -1,22 +0,0 @@ -Development ------------ - -You can run the integration tests by first running the following command in the root of the project - - mvn clean install - -Followed by running the following commands in the hnswlib-pyspark module - - export SPARK_HOME=/path/to/spark-2.3.0-bin-hadoop2.6 - rm -rf ~/.ivy2/cache/com.github.jelmerk - rm -rf ~/.ivy2/jars/com.github.jelmerk_hnswlib-* - pip install -e .[test] - py.test - -The easiest way to test changes on a real cluster is to produce an egg file with - - python setup.py bdist_egg - -And then reference it from spark with - - spark-submit --py-files hnswlib-pyspark/dist/pyspark_hnsw-*.egg YOUR_SCRIPT diff --git a/hnswlib-pyspark/README.md b/hnswlib-pyspark/README.md deleted file mode 100644 index 754293f1..00000000 --- a/hnswlib-pyspark/README.md +++ /dev/null @@ -1,108 +0,0 @@ -[![PyPI](https://img.shields.io/pypi/v/pyspark-hnsw)](https://pypi.org/project/pyspark-hnsw/) - -hnswlib-pyspark -=============== - -[PySpark](https://spark.apache.org/) integration for hnswlib. - -Setup ------ - -Find the package appropriate for your spark setup - -| | Scala 2.11 | Scala 2.12 | -|-------------|-------------------------------------------------|-------------------------------------------------| -| Spark 2.3.x | com.github.jelmerk:hnswlib-spark_2.3_2.11:1.1.0 | | -| Spark 2.4.x | com.github.jelmerk:hnswlib-spark_2.4_2.11:1.1.0 | com.github.jelmerk:hnswlib-spark_2.4_2.12:1.1.0 | -| Spark 3.0.x | | com.github.jelmerk:hnswlib-spark_3.0_2.12:1.1.0 | -| Spark 3.1.x | | com.github.jelmerk:hnswlib-spark_3.1_2.12:1.1.0 | -| Spark 3.2.x | | com.github.jelmerk:hnswlib-spark_3.2_2.12:1.1.0 | -| Spark 3.3.x | | com.github.jelmerk:hnswlib-spark_3.3_2.12:1.1.0 | - - -Pass this as an argument to spark - - --packages 'com.github.jelmerk:hnswlib-spark_2.3_2.11:1.1.0' - -Then install the python module with - - pip install pyspark-hnsw --upgrade - - -Example usage -------------- - -Basic: - -```python -from pyspark_hnsw.knn import HnswSimilarity - -hnsw = HnswSimilarity(identifierCol='id', featuresCol='features', distanceFunction='cosine', m=48, ef=5, k=200, - efConstruction=200, numPartitions=2, excludeSelf=True) - -model = hnsw.fit(index_items) - -model.transform(index_items).write.parquet('/path/to/output', mode='overwrite') -``` - -Advanced: - -```python -from pyspark.ml import Pipeline -from pyspark_hnsw.evaluation import KnnSimilarityEvaluator -from pyspark_hnsw.knn import * -from pyspark_hnsw.linalg import Normalizer -from pyspark_hnsw.conversion import VectorConverter - -# often it is acceptable to use float instead of double precision. -# this uses less memory and will be faster -converter = VectorConverter(inputCol='features_as_ml_lib_vector', outputCol='features') - -# The cosine distance is obtained with the inner product after normalizing all vectors to unit norm -# this is much faster than calculating the cosine distance directly - -normalizer = Normalizer(inputCol='features', outputCol='normalized_features') - -hnsw = HnswSimilarity(identifierCol='id', queryIdentifierCol='id', featuresCol='normalized_features', distanceFunction='inner-product', m=48, ef=5, k=200, - efConstruction=200, numPartitions=2, excludeSelf=True, similarityThreshold=0.4, predictionCol='approximate') - -brute_force = BruteForceSimilarity(identifierCol='id', queryIdentifierCol='id', featuresCol='normalized_features', distanceFunction='inner-product', - k=200, numPartitions=2, excludeSelf=True, similarityThreshold=0.4, predictionCol='exact') - -pipeline = Pipeline(stages=[converter, normalizer, hnsw, brute_force]) - -model = pipeline.fit(index_items) - -# computing the exact similarity is expensive so only take a small sample -query_items = index_items.sample(0.01) - -output = model.transform(query_items) - -evaluator = KnnSimilarityEvaluator(approximateNeighborsCol='approximate', exactNeighborsCol='exact') - -accuracy = evaluator.evaluate(output) - -print(accuracy) - -# save the model -model.write().overwrite().save('/path/to/model') -``` - -Suggested configuration ------------------------ - -- set `executor.instances` to the same value as the numPartitions property of your Hnsw instance -- set `spark.executor.cores` to as high a value as feasible on your executors while not making your jobs impossible to schedule -- set `spark.task.cpus` to the same value as `spark.executor.cores` -- set `spark.scheduler.minRegisteredResourcesRatio` to `1.0` -- set `spark.scheduler.maxRegisteredResourcesWaitingTime` to `3600` -- set `spark.speculation` to `false` -- set `spark.task.maxFailures` to `1` -- set `spark.dynamicAllocation.enabled` to `false` -- set `spark.driver.memory`: to some arbitrary low value for instance `2g` will do because the model does not run on the driver -- set `spark.executor.memory`: to a value appropriate to the size of your data, typically this will be a large value -- set `spark.yarn.executor.memoryOverhead` to a value higher than `executorMemory * 0.10` if you get the "Container killed by YARN for exceeding memory limits" error -- set `spark.hnswlib.settings.index.cache_folder` to a folder with plenty of space that you can write to. Defaults to /tmp - -Note that as it stands increasing the number of partitions will speed up fitting the model but not querying the model. The only way to speed up querying is by increasing the number of replicas - diff --git a/hnswlib-pyspark/setup.py b/hnswlib-pyspark/setup.py deleted file mode 100644 index 861b11e0..00000000 --- a/hnswlib-pyspark/setup.py +++ /dev/null @@ -1,29 +0,0 @@ -from setuptools import setup, find_packages -try: - from setupext_janitor import janitor - CleanCommand = janitor.CleanCommand -except ImportError: - CleanCommand = None - -cmd_classes = {} -if CleanCommand is not None: - cmd_classes['clean'] = CleanCommand - -setup( - name="pyspark_hnsw", - url="https://github.com/jelmerk/hnswlib/tree/master/hnswlib-pyspark", - version="1.1.0", - zip_safe=True, - packages=find_packages(exclude=['tests']), - extras_require={ - 'dev': ['findspark', 'pytest'], - 'test': ['findspark', 'pytest'], - }, - setup_requires=['setupext_janitor'], - cmdclass=cmd_classes, - entry_points={ - # normal parameters, ie. console_scripts[] - 'distutils.commands': [ - ' clean = setupext_janitor.janitor:CleanCommand'] - } -) \ No newline at end of file diff --git a/hnswlib-pyspark/tests/conftest.py b/hnswlib-pyspark/tests/conftest.py deleted file mode 100644 index f1c16c16..00000000 --- a/hnswlib-pyspark/tests/conftest.py +++ /dev/null @@ -1,23 +0,0 @@ -# coding=utf-8 - -import pytest -import findspark -findspark.init() - -from pyspark import SparkContext, SparkConf - - -APP_NAME = 'hnswlib-pyspark-tests' - - -@pytest.fixture(scope="session", autouse=True) -def spark_context(request): - """ fixture for creating a spark context - Args: - request: pytest.FixtureRequest object - """ - conf = (SparkConf().set('spark.jars.packages', 'com.github.jelmerk:hnswlib-spark_2.3_2.11:1.1.0').setMaster("local[2]").setAppName(APP_NAME)) - sc = SparkContext(conf=conf) - request.addfinalizer(lambda: sc.stop()) - - return sc diff --git a/hnswlib-scala/pom-scala-2_11.xml b/hnswlib-scala/pom-scala-2_11.xml deleted file mode 100644 index 6a4ee56f..00000000 --- a/hnswlib-scala/pom-scala-2_11.xml +++ /dev/null @@ -1,138 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-scala_2.11 - - Scala integration - - - - - org.scala-lang - scala-library - 2.11.12 - - - - com.github.jelmerk - hnswlib-core - ${project.version} - - - - com.github.jelmerk - hnswlib-metrics-dropwizard - ${project.version} - true - - - - org.scalatest - scalatest_2.11 - 3.1.0 - test - - - - - src/main/scala - src/test/scala - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.11.12 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -optimise - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - diff --git a/hnswlib-scala/pom-scala-2_12.xml b/hnswlib-scala/pom-scala-2_12.xml deleted file mode 100644 index 88e1efa3..00000000 --- a/hnswlib-scala/pom-scala-2_12.xml +++ /dev/null @@ -1,139 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-scala_2.12 - - Scala integration - - - - - org.scala-lang - scala-library - 2.12.7 - - - - com.github.jelmerk - hnswlib-core - ${project.version} - - - - com.github.jelmerk - hnswlib-metrics-dropwizard - ${project.version} - true - - - - org.scalatest - scalatest_2.12 - 3.1.0 - test - - - - - src/main/scala - src/test/scala - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.12.7 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -optimise - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - - diff --git a/hnswlib-scala/pom-scala-2_13.xml b/hnswlib-scala/pom-scala-2_13.xml deleted file mode 100644 index a0049741..00000000 --- a/hnswlib-scala/pom-scala-2_13.xml +++ /dev/null @@ -1,139 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-scala_2.13 - - Scala integration - - - - - org.scala-lang - scala-library - 2.13.1 - - - - com.github.jelmerk - hnswlib-core - ${project.version} - - - - com.github.jelmerk - hnswlib-metrics-dropwizard - ${project.version} - true - - - - org.scalatest - scalatest_2.13 - 3.1.0 - test - - - - - - src/main/scala - src/test/scala - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.13.1 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -optimise - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - diff --git a/hnswlib-scala/pom.xml b/hnswlib-scala/pom.xml deleted file mode 100644 index 6a4ee56f..00000000 --- a/hnswlib-scala/pom.xml +++ /dev/null @@ -1,138 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-scala_2.11 - - Scala integration - - - - - org.scala-lang - scala-library - 2.11.12 - - - - com.github.jelmerk - hnswlib-core - ${project.version} - - - - com.github.jelmerk - hnswlib-metrics-dropwizard - ${project.version} - true - - - - org.scalatest - scalatest_2.11 - 3.1.0 - test - - - - - src/main/scala - src/test/scala - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.11.12 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -optimise - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - diff --git a/hnswlib-spark/DEVELOPMENT.md b/hnswlib-spark/DEVELOPMENT.md deleted file mode 100644 index 1524746b..00000000 --- a/hnswlib-spark/DEVELOPMENT.md +++ /dev/null @@ -1,11 +0,0 @@ -Development ------------ - -The easiest way to test changes to the hnswlib codebase is to produce an assembly file with - - mvn clean assembly:assembly - -And then reference it from spark with - - spark-submit --jars hnswlib-spark/target/hnswlib-spark-*-jar-with-dependencies.jar your.jar - diff --git a/hnswlib-spark/pom-spark-2.3-scala-2_11.xml b/hnswlib-spark/pom-spark-2.3-scala-2_11.xml deleted file mode 100644 index fc3c4c3f..00000000 --- a/hnswlib-spark/pom-spark-2.3-scala-2_11.xml +++ /dev/null @@ -1,188 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-spark_2.3_2.11 - - Spark integration - - - - - org.scala-lang - scala-library - 2.11.12 - - - - com.github.jelmerk - hnswlib-utils - 1.1.0 - - - - com.github.jelmerk - hnswlib-scala_2.11 - 1.1.0 - - - - - org.apache.spark - spark-hive_2.11 - 2.3.0 - provided - - - org.scala-lang - scalap - - - - - - org.apache.spark - spark-mllib_2.11 - 2.3.0 - provided - - - - - org.scalatest - scalatest_2.11 - 3.0.8 - test - - - - com.holdenkarau - spark-testing-base_2.11 - 2.3.0_0.12.0 - test - - - - - - src/main/scala - src/test/scala - - - - ${project.basedir}/../hnswlib-pyspark/pyspark_hnsw - pyspark_hnsw - - __pycache__ - **/*.pyc - - - - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.11.12 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - diff --git a/hnswlib-spark/pom-spark-2.4-scala-2_11.xml b/hnswlib-spark/pom-spark-2.4-scala-2_11.xml deleted file mode 100644 index 87f6e16e..00000000 --- a/hnswlib-spark/pom-spark-2.4-scala-2_11.xml +++ /dev/null @@ -1,174 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-spark_2.4_2.11 - - Spark integration - - - - - org.scala-lang - scala-library - 2.11.12 - - - - com.github.jelmerk - hnswlib-utils - 1.1.0 - - - - com.github.jelmerk - hnswlib-scala_2.11 - 1.1.0 - - - - org.apache.spark - spark-mllib_2.11 - 2.4.5 - provided - - - - - org.scalatest - scalatest_2.11 - 3.0.8 - test - - - - com.holdenkarau - spark-testing-base_2.11 - 2.4.5_0.14.0 - test - - - - - - src/main/scala - src/test/scala - - - - ${project.basedir}/../hnswlib-pyspark/pyspark_hnsw - pyspark_hnsw - - __pycache__ - **/*.pyc - - - - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.11.12 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - diff --git a/hnswlib-spark/pom-spark-2.4-scala-2_12.xml b/hnswlib-spark/pom-spark-2.4-scala-2_12.xml deleted file mode 100644 index b1b3906a..00000000 --- a/hnswlib-spark/pom-spark-2.4-scala-2_12.xml +++ /dev/null @@ -1,174 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-spark_2.4_2.12 - - Spark integration - - - - - org.scala-lang - scala-library - 2.12.7 - - - - com.github.jelmerk - hnswlib-utils - 1.1.0 - - - - com.github.jelmerk - hnswlib-scala_2.12 - 1.1.0 - - - - org.apache.spark - spark-mllib_2.12 - 2.4.5 - provided - - - - - org.scalatest - scalatest_2.12 - 3.0.8 - test - - - - com.holdenkarau - spark-testing-base_2.12 - 2.4.5_0.14.0 - test - - - - - - src/main/scala - src/test/scala - - - - ${project.basedir}/../hnswlib-pyspark/pyspark_hnsw - pyspark_hnsw - - __pycache__ - **/*.pyc - - - - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.12.7 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - diff --git a/hnswlib-spark/pom-spark-3.0-scala-2_12.xml b/hnswlib-spark/pom-spark-3.0-scala-2_12.xml deleted file mode 100644 index b4107dcb..00000000 --- a/hnswlib-spark/pom-spark-3.0-scala-2_12.xml +++ /dev/null @@ -1,174 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-spark_3.0_2.12 - - Spark integration - - - - - org.scala-lang - scala-library - 2.12.7 - - - - com.github.jelmerk - hnswlib-utils - 1.1.0 - - - - com.github.jelmerk - hnswlib-scala_2.12 - 1.1.0 - - - - org.apache.spark - spark-mllib_2.12 - 3.0.2 - provided - - - - - org.scalatest - scalatest_2.12 - 3.0.8 - test - - - - com.holdenkarau - spark-testing-base_2.12 - 3.0.2_1.1.0 - test - - - - - - src/main/scala - src/test/scala - - - - ${project.basedir}/../hnswlib-pyspark/pyspark_hnsw - pyspark_hnsw - - __pycache__ - **/*.pyc - - - - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.12.7 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - diff --git a/hnswlib-spark/pom-spark-3.1-scala-2_12.xml b/hnswlib-spark/pom-spark-3.1-scala-2_12.xml deleted file mode 100644 index 8923d2b6..00000000 --- a/hnswlib-spark/pom-spark-3.1-scala-2_12.xml +++ /dev/null @@ -1,174 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-spark_3.1_2.12 - - Spark integration - - - - - org.scala-lang - scala-library - 2.12.7 - - - - com.github.jelmerk - hnswlib-utils - 1.1.0 - - - - com.github.jelmerk - hnswlib-scala_2.12 - 1.1.0 - - - - org.apache.spark - spark-mllib_2.12 - 3.1.2 - provided - - - - - org.scalatest - scalatest_2.12 - 3.0.8 - test - - - - com.holdenkarau - spark-testing-base_2.12 - 3.1.2_1.1.0 - test - - - - - - src/main/scala - src/test/scala - - - - ${project.basedir}/../hnswlib-pyspark/pyspark_hnsw - pyspark_hnsw - - __pycache__ - **/*.pyc - - - - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.12.7 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - diff --git a/hnswlib-spark/pom-spark-3.2-scala-2_12.xml b/hnswlib-spark/pom-spark-3.2-scala-2_12.xml deleted file mode 100644 index 9f04b1df..00000000 --- a/hnswlib-spark/pom-spark-3.2-scala-2_12.xml +++ /dev/null @@ -1,174 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-spark_3.2_2.12 - - Spark integration - - - - - org.scala-lang - scala-library - 2.12.7 - - - - com.github.jelmerk - hnswlib-utils - 1.1.0 - - - - com.github.jelmerk - hnswlib-scala_2.12 - 1.1.0 - - - - org.apache.spark - spark-mllib_2.12 - 3.2.1 - provided - - - - - org.scalatest - scalatest_2.12 - 3.0.8 - test - - - - com.holdenkarau - spark-testing-base_2.12 - 3.2.1_1.2.0 - test - - - - - - src/main/scala - src/test/scala - - - - ${project.basedir}/../hnswlib-pyspark/pyspark_hnsw - pyspark_hnsw - - __pycache__ - **/*.pyc - - - - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.12.7 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - diff --git a/hnswlib-spark/pom-spark-3.3-scala-2_12.xml b/hnswlib-spark/pom-spark-3.3-scala-2_12.xml deleted file mode 100644 index 37329a3f..00000000 --- a/hnswlib-spark/pom-spark-3.3-scala-2_12.xml +++ /dev/null @@ -1,182 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-spark_3.3_2.12 - - Spark integration - - - - - org.scala-lang - scala-library - 2.12.7 - - - - com.github.jelmerk - hnswlib-utils - 1.1.0 - - - - com.github.jelmerk - hnswlib-scala_2.12 - 1.1.0 - - - - org.apache.spark - spark-mllib_2.12 - 3.3.0 - provided - - - - - com.fasterxml.jackson.core - jackson-databind - 2.13.3 - provided - - - - - org.scalatest - scalatest_2.12 - 3.0.8 - test - - - - com.holdenkarau - spark-testing-base_2.12 - 3.3.0_1.2.0 - test - - - - - - src/main/scala - src/test/scala - - - - ${project.basedir}/../hnswlib-pyspark/pyspark_hnsw - pyspark_hnsw - - __pycache__ - **/*.pyc - - - - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.12.7 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - diff --git a/hnswlib-spark/pom.xml b/hnswlib-spark/pom.xml deleted file mode 100644 index fc3c4c3f..00000000 --- a/hnswlib-spark/pom.xml +++ /dev/null @@ -1,188 +0,0 @@ - - - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - .. - - - 4.0.0 - - hnswlib-spark_2.3_2.11 - - Spark integration - - - - - org.scala-lang - scala-library - 2.11.12 - - - - com.github.jelmerk - hnswlib-utils - 1.1.0 - - - - com.github.jelmerk - hnswlib-scala_2.11 - 1.1.0 - - - - - org.apache.spark - spark-hive_2.11 - 2.3.0 - provided - - - org.scala-lang - scalap - - - - - - org.apache.spark - spark-mllib_2.11 - 2.3.0 - provided - - - - - org.scalatest - scalatest_2.11 - 3.0.8 - test - - - - com.holdenkarau - spark-testing-base_2.11 - 2.3.0_0.12.0 - test - - - - - - src/main/scala - src/test/scala - - - - ${project.basedir}/../hnswlib-pyspark/pyspark_hnsw - pyspark_hnsw - - __pycache__ - **/*.pyc - - - - - - - net.alchim31.maven - scala-maven-plugin - - - scala-compile-first - process-resources - - add-source - compile - - - - scala-test-compile - process-test-resources - - testCompile - - - - attach-javadocs - - doc-jar - - - - - 2.11.12 - false - - -Xmx512m - - - -nobootcp - -target:jvm-1.8 - -deprecation - -unchecked - -encoding - UTF-8 - -feature - -Xlog-reflective-calls - - - - - - maven-compiler-plugin - - - compile - - compile - - - - - - - org.apache.maven.plugins - maven-surefire-plugin - - true - - - - - org.scalatest - scalatest-maven-plugin - 1.0 - - ${project.build.directory}/surefire-reports - . - WDF TestSuite.txt - - - - test - - test - - - - - - - org.apache.maven.plugins - maven-assembly-plugin - - - jar-with-dependencies - - - - - - - diff --git a/hnswlib-spark/run-pyspark-tests.sh b/hnswlib-spark/run-pyspark-tests.sh new file mode 100755 index 00000000..f7099d91 --- /dev/null +++ b/hnswlib-spark/run-pyspark-tests.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +SPARK_VERSION=$1 + +# add python sources on the path +export PYTHONPATH=src/main/python + +# unset SPARK_HOME or it will use whatever is configured on the host system instead of the pip packages +unset SPARK_HOME + +# create a virtual environment +python3.9 -m venv "target/spark-$SPARK_VERSION-venv" +source "target/spark-$SPARK_VERSION-venv/bin/activate" + +# install packages +pip install pytest==7.4.3 +pip install 'pyspark[ml]'=="$SPARK_VERSION" + +# run unit tests +pytest --junitxml=target/test-reports/TEST-python.xml \ No newline at end of file diff --git a/hnswlib-pyspark/com/__init__.py b/hnswlib-spark/src/main/python/__init__.py similarity index 100% rename from hnswlib-pyspark/com/__init__.py rename to hnswlib-spark/src/main/python/__init__.py diff --git a/hnswlib-pyspark/com/github/__init__.py b/hnswlib-spark/src/main/python/com/__init__.py similarity index 100% rename from hnswlib-pyspark/com/github/__init__.py rename to hnswlib-spark/src/main/python/com/__init__.py diff --git a/hnswlib-pyspark/com/github/jelmerk/__init__.py b/hnswlib-spark/src/main/python/com/github/__init__.py similarity index 100% rename from hnswlib-pyspark/com/github/jelmerk/__init__.py rename to hnswlib-spark/src/main/python/com/github/__init__.py diff --git a/hnswlib-pyspark/com/github/jelmerk/spark/__init__.py b/hnswlib-spark/src/main/python/com/github/jelmerk/__init__.py similarity index 100% rename from hnswlib-pyspark/com/github/jelmerk/spark/__init__.py rename to hnswlib-spark/src/main/python/com/github/jelmerk/__init__.py diff --git a/hnswlib-pyspark/com/github/jelmerk/spark/knn/__init__.py b/hnswlib-spark/src/main/python/com/github/jelmerk/spark/__init__.py similarity index 100% rename from hnswlib-pyspark/com/github/jelmerk/spark/knn/__init__.py rename to hnswlib-spark/src/main/python/com/github/jelmerk/spark/__init__.py diff --git a/hnswlib-pyspark/com/github/jelmerk/spark/conversion/__init__.py b/hnswlib-spark/src/main/python/com/github/jelmerk/spark/conversion/__init__.py similarity index 100% rename from hnswlib-pyspark/com/github/jelmerk/spark/conversion/__init__.py rename to hnswlib-spark/src/main/python/com/github/jelmerk/spark/conversion/__init__.py diff --git a/hnswlib-spark/src/main/python/com/github/jelmerk/spark/knn/__init__.py b/hnswlib-spark/src/main/python/com/github/jelmerk/spark/knn/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/hnswlib-pyspark/com/github/jelmerk/spark/knn/bruteforce/__init__.py b/hnswlib-spark/src/main/python/com/github/jelmerk/spark/knn/bruteforce/__init__.py similarity index 100% rename from hnswlib-pyspark/com/github/jelmerk/spark/knn/bruteforce/__init__.py rename to hnswlib-spark/src/main/python/com/github/jelmerk/spark/knn/bruteforce/__init__.py diff --git a/hnswlib-pyspark/com/github/jelmerk/spark/knn/evaluation/__init__.py b/hnswlib-spark/src/main/python/com/github/jelmerk/spark/knn/evaluation/__init__.py similarity index 100% rename from hnswlib-pyspark/com/github/jelmerk/spark/knn/evaluation/__init__.py rename to hnswlib-spark/src/main/python/com/github/jelmerk/spark/knn/evaluation/__init__.py diff --git a/hnswlib-pyspark/com/github/jelmerk/spark/knn/hnsw/__init__.py b/hnswlib-spark/src/main/python/com/github/jelmerk/spark/knn/hnsw/__init__.py similarity index 100% rename from hnswlib-pyspark/com/github/jelmerk/spark/knn/hnsw/__init__.py rename to hnswlib-spark/src/main/python/com/github/jelmerk/spark/knn/hnsw/__init__.py diff --git a/hnswlib-pyspark/com/github/jelmerk/spark/linalg/__init__.py b/hnswlib-spark/src/main/python/com/github/jelmerk/spark/linalg/__init__.py similarity index 100% rename from hnswlib-pyspark/com/github/jelmerk/spark/linalg/__init__.py rename to hnswlib-spark/src/main/python/com/github/jelmerk/spark/linalg/__init__.py diff --git a/hnswlib-pyspark/pyspark_hnsw/__init__.py b/hnswlib-spark/src/main/python/pyspark_hnsw/__init__.py similarity index 100% rename from hnswlib-pyspark/pyspark_hnsw/__init__.py rename to hnswlib-spark/src/main/python/pyspark_hnsw/__init__.py diff --git a/hnswlib-pyspark/pyspark_hnsw/conversion.py b/hnswlib-spark/src/main/python/pyspark_hnsw/conversion.py similarity index 100% rename from hnswlib-pyspark/pyspark_hnsw/conversion.py rename to hnswlib-spark/src/main/python/pyspark_hnsw/conversion.py diff --git a/hnswlib-pyspark/pyspark_hnsw/evaluation.py b/hnswlib-spark/src/main/python/pyspark_hnsw/evaluation.py similarity index 100% rename from hnswlib-pyspark/pyspark_hnsw/evaluation.py rename to hnswlib-spark/src/main/python/pyspark_hnsw/evaluation.py diff --git a/hnswlib-pyspark/pyspark_hnsw/knn.py b/hnswlib-spark/src/main/python/pyspark_hnsw/knn.py similarity index 100% rename from hnswlib-pyspark/pyspark_hnsw/knn.py rename to hnswlib-spark/src/main/python/pyspark_hnsw/knn.py diff --git a/hnswlib-pyspark/pyspark_hnsw/linalg.py b/hnswlib-spark/src/main/python/pyspark_hnsw/linalg.py similarity index 100% rename from hnswlib-pyspark/pyspark_hnsw/linalg.py rename to hnswlib-spark/src/main/python/pyspark_hnsw/linalg.py diff --git a/hnswlib-spark/src/main/scala/com/github/jelmerk/spark/knn/KnnAlgorithm.scala b/hnswlib-spark/src/main/scala/com/github/jelmerk/spark/knn/KnnAlgorithm.scala index 7d600131..6096d900 100644 --- a/hnswlib-spark/src/main/scala/com/github/jelmerk/spark/knn/KnnAlgorithm.scala +++ b/hnswlib-spark/src/main/scala/com/github/jelmerk/spark/knn/KnnAlgorithm.scala @@ -23,7 +23,6 @@ import org.apache.spark.sql.functions._ import org.apache.spark.sql.types._ import org.json4s.jackson.JsonMethods._ import org.json4s._ -import org.json4s.JsonDSL._ import com.github.jelmerk.knn.scalalike._ import com.github.jelmerk.knn.util.NamedThreadFactory import com.github.jelmerk.spark.linalg.functions.VectorDistanceFunctions @@ -289,21 +288,26 @@ private[knn] class KnnModelWriter[ extends MLWriter { override protected def saveImpl(path: String): Unit = { - val params = + val params = JObject( instance.extractParamMap().toSeq.toList // cannot use parse because of incompatibilities between json4s 3.2.11 used by spark 2.3 and 3.6.6 used by spark 2.4 - .map { case ParamPair(param, value) => param.name -> mapper.readValue(param.jsonEncode(value), classOf[JValue]) } - .toMap - - val metaData: JObject = - ("class" -> instance.getClass.getName) ~ - ("timestamp" -> System.currentTimeMillis()) ~ - ("sparkVersion", sc.version) ~ - ("uid", instance.uid) ~ - ("identifierType", typeDescription[TId]) ~ - ("vectorType", typeDescription[TVector]) ~ - ("partitions", instance.getNumPartitions) ~ - ("paramMap", params) + .map { case ParamPair(param, value) => + val fieldName = param.name + val fieldValue = mapper.readValue(param.jsonEncode(value), classOf[JValue]) + JField(fieldName, fieldValue) + } + ) + + val metaData = JObject(List( + JField("class", JString(instance.getClass.getName)), + JField("timestamp", JLong(System.currentTimeMillis())), + JField("sparkVersion", JString(sc.version)), + JField("uid", JString(instance.uid)), + JField("identifierType", JString(typeDescription[TId])), + JField("vectorType", JString(typeDescription[TVector])), + JField("partitions", JInt(instance.getNumPartitions)), + JField("paramMap", params) + )) val metadataPath = new Path(path, "metadata").toString sc.parallelize(Seq(compact(metaData)), numSlices = 1).saveAsTextFile(metadataPath) diff --git a/hnswlib-spark/src/test/python/conftest.py b/hnswlib-spark/src/test/python/conftest.py new file mode 100644 index 00000000..3743af3a --- /dev/null +++ b/hnswlib-spark/src/test/python/conftest.py @@ -0,0 +1,18 @@ +# coding=utf-8 + +import os + +import pytest + +from pyspark.sql import SparkSession + +@pytest.fixture(scope="session", autouse=True) +def spark(request): + sc = SparkSession.builder \ + .config("spark.jars.packages", os.environ["ARTIFACT"]) \ + .master("local[*]") \ + .getOrCreate() + + request.addfinalizer(lambda: sc.stop()) + + return sc diff --git a/hnswlib-pyspark/tests/test_bruteforce.py b/hnswlib-spark/src/test/python/test_bruteforce.py similarity index 80% rename from hnswlib-pyspark/tests/test_bruteforce.py rename to hnswlib-spark/src/test/python/test_bruteforce.py index eebb1433..e405713d 100644 --- a/hnswlib-pyspark/tests/test_bruteforce.py +++ b/hnswlib-spark/src/test/python/test_bruteforce.py @@ -2,14 +2,11 @@ from pyspark_hnsw.knn import BruteForceSimilarity from pyspark.ml.linalg import Vectors -from pyspark.sql import SQLContext -def test_bruteforce(spark_context): +def test_bruteforce(spark): - sql_context = SQLContext(spark_context) - - df = sql_context.createDataFrame([ + df = spark.createDataFrame([ [1, Vectors.dense([0.2, 0.9])], [2, Vectors.dense([0.2, 1.0])], [3, Vectors.dense([0.2, 0.1])], diff --git a/hnswlib-pyspark/tests/test_hnsw.py b/hnswlib-spark/src/test/python/test_hnsw.py similarity index 79% rename from hnswlib-pyspark/tests/test_hnsw.py rename to hnswlib-spark/src/test/python/test_hnsw.py index 0a90583f..cb960588 100644 --- a/hnswlib-pyspark/tests/test_hnsw.py +++ b/hnswlib-spark/src/test/python/test_hnsw.py @@ -2,14 +2,11 @@ from pyspark_hnsw.knn import HnswSimilarity from pyspark.ml.linalg import Vectors -from pyspark.sql import SQLContext -def test_hnsw(spark_context): +def test_hnsw(spark): - sql_context = SQLContext(spark_context) - - df = sql_context.createDataFrame([ + df = spark.createDataFrame([ [1, Vectors.dense([0.2, 0.9])], [2, Vectors.dense([0.2, 1.0])], [3, Vectors.dense([0.2, 0.1])], diff --git a/hnswlib-pyspark/tests/test_integration.py b/hnswlib-spark/src/test/python/test_integration.py similarity index 72% rename from hnswlib-pyspark/tests/test_integration.py rename to hnswlib-spark/src/test/python/test_integration.py index d0cd7e1b..3e2b9eea 100644 --- a/hnswlib-pyspark/tests/test_integration.py +++ b/hnswlib-spark/src/test/python/test_integration.py @@ -2,15 +2,12 @@ from pyspark_hnsw.knn import HnswSimilarity from pyspark.ml.linalg import Vectors -from pyspark.sql import SQLContext from pyspark.sql import functions as F -def test_incremental_models(spark_context, tmp_path): +def test_incremental_models(spark, tmp_path): - sql_context = SQLContext(spark_context) - - df1 = sql_context.createDataFrame([ + df1 = spark.createDataFrame([ [1, Vectors.dense([0.1, 0.2, 0.3])] ], ['id', 'features']) @@ -20,7 +17,7 @@ def test_incremental_models(spark_context, tmp_path): model1.write().overwrite().save(tmp_path.as_posix()) - df2 = sql_context.createDataFrame([ + df2 = spark.createDataFrame([ [2, Vectors.dense([0.9, 0.1, 0.2])] ], ['id', 'features']) diff --git a/hnswlib-pyspark/tests/test_knn_evaluator.py b/hnswlib-spark/src/test/python/test_knn_evaluator.py similarity index 79% rename from hnswlib-pyspark/tests/test_knn_evaluator.py rename to hnswlib-spark/src/test/python/test_knn_evaluator.py index aec6b64f..6735eaa2 100644 --- a/hnswlib-pyspark/tests/test_knn_evaluator.py +++ b/hnswlib-spark/src/test/python/test_knn_evaluator.py @@ -1,20 +1,16 @@ # coding=utf-8 from pyspark_hnsw.evaluation import KnnSimilarityEvaluator -from pyspark.ml.linalg import Vectors -from pyspark.sql import SQLContext from pyspark.sql.types import * -def test_evaluator(spark_context): - - sql_context = SQLContext(spark_context) +def test_evaluator(spark): neighbors_list_schema = ArrayType(StructType([StructField("neighbor", IntegerType()), StructField("distance", FloatType())])) schema = StructType([StructField("approximate", neighbors_list_schema), StructField("exact", neighbors_list_schema)]) - df = sql_context.createDataFrame([ + df = spark.createDataFrame([ [[{'neighbor': 1, 'distance': 0.1}], [{'neighbor': 1, 'distance': 0.1}]], [[{'neighbor': 2, 'distance': 0.1}], [{'neighbor': 2, 'distance': 0.1}, {'neighbor': 3, 'distance': 0.9}]] ], schema=schema) diff --git a/hnswlib-pyspark/tests/test_normalizer.py b/hnswlib-spark/src/test/python/test_normalizer.py similarity index 54% rename from hnswlib-pyspark/tests/test_normalizer.py rename to hnswlib-spark/src/test/python/test_normalizer.py index 8b2a4374..61b911e6 100644 --- a/hnswlib-pyspark/tests/test_normalizer.py +++ b/hnswlib-spark/src/test/python/test_normalizer.py @@ -2,13 +2,10 @@ from pyspark_hnsw.linalg import Normalizer from pyspark.ml.linalg import Vectors -from pyspark.sql import SQLContext -def test_normalizer(spark_context): +def test_normalizer(spark): - sql_context = SQLContext(spark_context) - - df = sql_context.createDataFrame([[Vectors.dense([0.01, 0.02, 0.03])]], ['vector']) + df = spark.createDataFrame([[Vectors.dense([0.01, 0.02, 0.03])]], ['vector']) normalizer = Normalizer(inputCol="vector", outputCol="normalized_vector") diff --git a/hnswlib-pyspark/tests/test_vector_converter.py b/hnswlib-spark/src/test/python/test_vector_converter.py similarity index 56% rename from hnswlib-pyspark/tests/test_vector_converter.py rename to hnswlib-spark/src/test/python/test_vector_converter.py index 44912934..1f296a39 100644 --- a/hnswlib-pyspark/tests/test_vector_converter.py +++ b/hnswlib-spark/src/test/python/test_vector_converter.py @@ -2,13 +2,10 @@ from pyspark_hnsw.conversion import VectorConverter from pyspark.ml.linalg import Vectors -from pyspark.sql import SQLContext -def test_vector_converter(spark_context): +def test_vector_converter(spark): - sql_context = SQLContext(spark_context) - - df = sql_context.createDataFrame([[Vectors.dense([0.01, 0.02, 0.03])]], ['vector']) + df = spark.createDataFrame([[Vectors.dense([0.01, 0.02, 0.03])]], ['vector']) converter = VectorConverter(inputCol="vector", outputCol="array", outputType="array") diff --git a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/SharedSparkContext.scala b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/SharedSparkContext.scala new file mode 100644 index 00000000..7aeebdeb --- /dev/null +++ b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/SharedSparkContext.scala @@ -0,0 +1,43 @@ +package com.github.jelmerk.spark + +import org.apache.spark.SparkConf +import org.apache.spark.sql.{SQLContext, SparkSession} +import org.scalatest.{BeforeAndAfterAll, Suite} + +/** Shares a local `SparkContext` between all tests in a suite and closes it at the end */ +trait SharedSparkContext extends BeforeAndAfterAll { + self: Suite => + + @transient private var sparkSession: SparkSession = _ + + def appID: String = this.getClass.getName + math.floor(math.random * 10E4).toLong.toString + + def conf: SparkConf = { + new SparkConf(). + setMaster("local[*]"). + setAppName("test"). + set("spark.ui.enabled", "false"). + set("spark.app.id", appID). + set("spark.driver.host", "localhost") + } + + def spark: SQLContext = sparkSession.sqlContext + + override def beforeAll(): Unit = { + sparkSession = SparkSession.builder().config(conf).getOrCreate() + super.beforeAll() + } + + override def afterAll(): Unit = { + try { + Option(sparkSession).foreach { _.stop() } + // To avoid Akka rebinding to the same port, since it doesn't + // unbind immediately on shutdown. + System.clearProperty("spark.driver.port") + sparkSession = null + } finally { + super.afterAll() + } + } + +} \ No newline at end of file diff --git a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/SparkSessionProvider.scala b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/SparkSessionProvider.scala new file mode 100644 index 00000000..89975594 --- /dev/null +++ b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/SparkSessionProvider.scala @@ -0,0 +1,9 @@ +package com.github.jelmerk.spark + +import org.apache.spark.sql.{EvilSessionTools, SQLContext, SparkSession} + +object SparkSessionProvider { + @transient var _sparkSession: SparkSession = _ + def sqlContext: SQLContext = EvilSessionTools.extractSQLContext(_sparkSession) + def sparkSession = _sparkSession +} \ No newline at end of file diff --git a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/conversion/VectorConverterSpec.scala b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/conversion/VectorConverterSpec.scala index d41f22af..dd89e482 100644 --- a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/conversion/VectorConverterSpec.scala +++ b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/conversion/VectorConverterSpec.scala @@ -3,14 +3,14 @@ package com.github.jelmerk.spark.conversion import com.holdenkarau.spark.testing.DataFrameSuiteBase import org.apache.spark.ml.linalg.Vectors import org.apache.spark.sql.DataFrame -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import org.scalatest.prop.TableDrivenPropertyChecks._ case class InputRow[TVector](vector: TVector) case class OutputRow[TVectorIn, TVectorOut](vector: TVectorIn, array: TVectorOut) -class VectorConverterSpec extends FunSuite with DataFrameSuiteBase { +class VectorConverterSpec extends AnyFunSuite with DataFrameSuiteBase { test("convert vectors") { @@ -45,15 +45,20 @@ class VectorConverterSpec extends FunSuite with DataFrameSuiteBase { ) ) - forAll (scenarios) { case (input, expectedOutput, outputType) => + val input = Seq(InputRow(Array(1d, 2d, 3d))).toDF() + val expectedOutput = Seq(OutputRow(Array(1d, 2d, 3d), Vectors.dense(Array(1d, 2d, 3d)))).toDF() + val outputType = "vector" + +// forAll (scenarios) { case (input, expectedOutput, outputType) => val converter = new VectorConverter() .setInputCol("vector") .setOutputCol("array") .setOutputType(outputType) +// converter.transform(input).show() assertDataFrameEquals(converter.transform(input), expectedOutput) - } +// } } } diff --git a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/SerializerSpec.scala b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/SerializerSpec.scala index d8e5e6a6..22a1d009 100644 --- a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/SerializerSpec.scala +++ b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/SerializerSpec.scala @@ -4,10 +4,10 @@ import java.io.{ByteArrayInputStream, ByteArrayOutputStream, ObjectInputStream, import com.github.jelmerk.knn.scalalike.{Item, ObjectSerializer} import org.apache.spark.ml.linalg.{Vectors, Vector} -import org.scalatest.FunSuite -import org.scalatest.Matchers._ +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers._ -class SerializerSpec extends FunSuite { +class SerializerSpec extends AnyFunSuite { test("serialize objects") { validateSerializability(IntSerializer, 1) diff --git a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/evaluation/KnnSimilarityEvaluatorSpec.scala b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/evaluation/KnnSimilarityEvaluatorSpec.scala index ba259e37..30a3e736 100644 --- a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/evaluation/KnnSimilarityEvaluatorSpec.scala +++ b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/evaluation/KnnSimilarityEvaluatorSpec.scala @@ -1,15 +1,15 @@ package com.github.jelmerk.spark.knn.evaluation -import com.holdenkarau.spark.testing.DataFrameSuiteBase -import org.scalatest.FunSuite -import org.scalatest.Matchers._ +import com.github.jelmerk.spark.SharedSparkContext +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers._ case class Neighbor[TId, TDistance](neighbor: TId, distance: TDistance) -class KnnSimilarityEvaluatorSpec extends FunSuite with DataFrameSuiteBase { +class KnnSimilarityEvaluatorSpec extends AnyFunSuite with SharedSparkContext { test("evaluate performance") { - val sqlCtx = sqlContext + val sqlCtx = spark import sqlCtx.implicits._ val evaluator = new KnnSimilarityEvaluator() @@ -25,7 +25,7 @@ class KnnSimilarityEvaluatorSpec extends FunSuite with DataFrameSuiteBase { } test("evaluate performance empty lists") { - val sqlCtx = sqlContext + val sqlCtx = spark import sqlCtx.implicits._ val evaluator = new KnnSimilarityEvaluator() diff --git a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/hnsw/HnswSimilaritySpec.scala b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/hnsw/HnswSimilaritySpec.scala index f3de0d78..f2ffa2bf 100644 --- a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/hnsw/HnswSimilaritySpec.scala +++ b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/knn/hnsw/HnswSimilaritySpec.scala @@ -10,8 +10,8 @@ import org.apache.commons.lang.builder.{EqualsBuilder, HashCodeBuilder} import org.apache.spark.SparkConf import org.apache.spark.ml.linalg.{DenseVector, SparseVector, Vectors} import org.apache.spark.sql.DataFrame -import org.scalatest.FunSuite -import org.scalatest.Matchers._ +import org.scalatest.funsuite.AnyFunSuite +import org.scalatest.matchers.should.Matchers._ import org.scalatest.prop.TableDrivenPropertyChecks._ case class PrePartitionedInputRow[TId, TVector](partition: Int, id: TId, vector: TVector) @@ -36,7 +36,7 @@ case class MinimalOutputRow[TId, TDistance](id: TId, neighbors: Seq[Neighbor[TId override def hashCode(): Int = HashCodeBuilder.reflectionHashCode(this) } -class HnswSimilaritySpec extends FunSuite with DataFrameSuiteBase { +class HnswSimilaritySpec extends AnyFunSuite with DataFrameSuiteBase { // for some reason kryo cannot serialize the hnswindex so configure it to make sure it never gets serialized override def conf: SparkConf = super.conf diff --git a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/linalg/NormalizerSpec.scala b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/linalg/NormalizerSpec.scala index fd363853..e9c40698 100644 --- a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/linalg/NormalizerSpec.scala +++ b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/linalg/NormalizerSpec.scala @@ -3,14 +3,14 @@ package com.github.jelmerk.spark.linalg import com.holdenkarau.spark.testing.DataFrameSuiteBase import org.apache.spark.ml.linalg.{DenseVector, SparseVector} import org.apache.spark.sql.DataFrame -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import org.scalatest.prop.TableDrivenPropertyChecks._ case class InputRow[TVector](vector: TVector) case class OutputRow[TVector](vector: TVector, normalized: TVector) -class NormalizerSpec extends FunSuite with DataFrameSuiteBase { +class NormalizerSpec extends AnyFunSuite with DataFrameSuiteBase { test("normalize vector") { diff --git a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/linalg/functions/SparseVectorDistanceFunctionsSpec.scala b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/linalg/functions/SparseVectorDistanceFunctionsSpec.scala index 6d763cd6..0e7fffdb 100644 --- a/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/linalg/functions/SparseVectorDistanceFunctionsSpec.scala +++ b/hnswlib-spark/src/test/scala/com/github/jelmerk/spark/linalg/functions/SparseVectorDistanceFunctionsSpec.scala @@ -3,11 +3,11 @@ package com.github.jelmerk.spark.linalg.functions import org.apache.spark.ml.linalg.Vector import org.apache.spark.ml.linalg.Vectors import org.scalactic.{Equality, TolerantNumerics} -import org.scalatest.FunSuite +import org.scalatest.funsuite.AnyFunSuite import scala.util.Random -class SparseVectorDistanceFunctionsSpec extends FunSuite { +class SparseVectorDistanceFunctionsSpec extends AnyFunSuite { private implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.001) private val random = new Random(1000L) diff --git a/mvnw b/mvnw deleted file mode 100755 index 5551fde8..00000000 --- a/mvnw +++ /dev/null @@ -1,286 +0,0 @@ -#!/bin/sh -# ---------------------------------------------------------------------------- -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# ---------------------------------------------------------------------------- - -# ---------------------------------------------------------------------------- -# Maven2 Start Up Batch script -# -# Required ENV vars: -# ------------------ -# JAVA_HOME - location of a JDK home dir -# -# Optional ENV vars -# ----------------- -# M2_HOME - location of maven2's installed home dir -# MAVEN_OPTS - parameters passed to the Java VM when running Maven -# e.g. to debug Maven itself, use -# set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -# MAVEN_SKIP_RC - flag to disable loading of mavenrc files -# ---------------------------------------------------------------------------- - -if [ -z "$MAVEN_SKIP_RC" ] ; then - - if [ -f /etc/mavenrc ] ; then - . /etc/mavenrc - fi - - if [ -f "$HOME/.mavenrc" ] ; then - . "$HOME/.mavenrc" - fi - -fi - -# OS specific support. $var _must_ be set to either true or false. -cygwin=false; -darwin=false; -mingw=false -case "`uname`" in - CYGWIN*) cygwin=true ;; - MINGW*) mingw=true;; - Darwin*) darwin=true - # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home - # See https://developer.apple.com/library/mac/qa/qa1170/_index.html - if [ -z "$JAVA_HOME" ]; then - if [ -x "/usr/libexec/java_home" ]; then - export JAVA_HOME="`/usr/libexec/java_home`" - else - export JAVA_HOME="/Library/Java/Home" - fi - fi - ;; -esac - -if [ -z "$JAVA_HOME" ] ; then - if [ -r /etc/gentoo-release ] ; then - JAVA_HOME=`java-config --jre-home` - fi -fi - -if [ -z "$M2_HOME" ] ; then - ## resolve links - $0 may be a link to maven's home - PRG="$0" - - # need this for relative symlinks - while [ -h "$PRG" ] ; do - ls=`ls -ld "$PRG"` - link=`expr "$ls" : '.*-> \(.*\)$'` - if expr "$link" : '/.*' > /dev/null; then - PRG="$link" - else - PRG="`dirname "$PRG"`/$link" - fi - done - - saveddir=`pwd` - - M2_HOME=`dirname "$PRG"`/.. - - # make it fully qualified - M2_HOME=`cd "$M2_HOME" && pwd` - - cd "$saveddir" - # echo Using m2 at $M2_HOME -fi - -# For Cygwin, ensure paths are in UNIX format before anything is touched -if $cygwin ; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --unix "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --unix "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --unix "$CLASSPATH"` -fi - -# For Mingw, ensure paths are in UNIX format before anything is touched -if $mingw ; then - [ -n "$M2_HOME" ] && - M2_HOME="`(cd "$M2_HOME"; pwd)`" - [ -n "$JAVA_HOME" ] && - JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" - # TODO classpath? -fi - -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - if $darwin ; then - javaHome="`dirname \"$javaExecutable\"`" - javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" - else - javaExecutable="`readlink -f \"$javaExecutable\"`" - fi - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "$JAVACMD" ] ; then - if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." >&2 - echo " We cannot execute $JAVACMD" >&2 - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi - -CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher - -# traverses directory structure from process work directory to filesystem root -# first directory with .mvn subdirectory is considered project base directory -find_maven_basedir() { - - if [ -z "$1" ] - then - echo "Path not specified to find_maven_basedir" - return 1 - fi - - basedir="$1" - wdir="$1" - while [ "$wdir" != '/' ] ; do - if [ -d "$wdir"/.mvn ] ; then - basedir=$wdir - break - fi - # workaround for JBEAP-8937 (on Solaris 10/Sparc) - if [ -d "${wdir}" ]; then - wdir=`cd "$wdir/.."; pwd` - fi - # end of workaround - done - echo "${basedir}" -} - -# concatenates all lines of a file -concat_lines() { - if [ -f "$1" ]; then - echo "$(tr -s '\n' ' ' < "$1")" - fi -} - -BASE_DIR=`find_maven_basedir "$(pwd)"` -if [ -z "$BASE_DIR" ]; then - exit 1; -fi - -########################################################################################## -# Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -# This allows using the maven wrapper in projects that prohibit checking in binary data. -########################################################################################## -if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found .mvn/wrapper/maven-wrapper.jar" - fi -else - if [ "$MVNW_VERBOSE" = true ]; then - echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." - fi - jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" - while IFS="=" read key value; do - case "$key" in (wrapperUrl) jarUrl="$value"; break ;; - esac - done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" - if [ "$MVNW_VERBOSE" = true ]; then - echo "Downloading from: $jarUrl" - fi - wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" - - if command -v wget > /dev/null; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found wget ... using wget" - fi - wget "$jarUrl" -O "$wrapperJarPath" - elif command -v curl > /dev/null; then - if [ "$MVNW_VERBOSE" = true ]; then - echo "Found curl ... using curl" - fi - curl -o "$wrapperJarPath" "$jarUrl" - else - if [ "$MVNW_VERBOSE" = true ]; then - echo "Falling back to using Java to download" - fi - javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" - if [ -e "$javaClass" ]; then - if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then - if [ "$MVNW_VERBOSE" = true ]; then - echo " - Compiling MavenWrapperDownloader.java ..." - fi - # Compiling the Java class - ("$JAVA_HOME/bin/javac" "$javaClass") - fi - if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then - # Running the downloader - if [ "$MVNW_VERBOSE" = true ]; then - echo " - Running MavenWrapperDownloader.java ..." - fi - ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") - fi - fi - fi -fi -########################################################################################## -# End of extension -########################################################################################## - -export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} -if [ "$MVNW_VERBOSE" = true ]; then - echo $MAVEN_PROJECTBASEDIR -fi -MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" - -# For Cygwin, switch paths to Windows format before running java -if $cygwin; then - [ -n "$M2_HOME" ] && - M2_HOME=`cygpath --path --windows "$M2_HOME"` - [ -n "$JAVA_HOME" ] && - JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` - [ -n "$CLASSPATH" ] && - CLASSPATH=`cygpath --path --windows "$CLASSPATH"` - [ -n "$MAVEN_PROJECTBASEDIR" ] && - MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` -fi - -WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -exec "$JAVACMD" \ - $MAVEN_OPTS \ - -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ - "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ - ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" diff --git a/mvnw.cmd b/mvnw.cmd deleted file mode 100755 index e5cfb0ae..00000000 --- a/mvnw.cmd +++ /dev/null @@ -1,161 +0,0 @@ -@REM ---------------------------------------------------------------------------- -@REM Licensed to the Apache Software Foundation (ASF) under one -@REM or more contributor license agreements. See the NOTICE file -@REM distributed with this work for additional information -@REM regarding copyright ownership. The ASF licenses this file -@REM to you under the Apache License, Version 2.0 (the -@REM "License"); you may not use this file except in compliance -@REM with the License. You may obtain a copy of the License at -@REM -@REM http://www.apache.org/licenses/LICENSE-2.0 -@REM -@REM Unless required by applicable law or agreed to in writing, -@REM software distributed under the License is distributed on an -@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -@REM KIND, either express or implied. See the License for the -@REM specific language governing permissions and limitations -@REM under the License. -@REM ---------------------------------------------------------------------------- - -@REM ---------------------------------------------------------------------------- -@REM Maven2 Start Up Batch script -@REM -@REM Required ENV vars: -@REM JAVA_HOME - location of a JDK home dir -@REM -@REM Optional ENV vars -@REM M2_HOME - location of maven2's installed home dir -@REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands -@REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending -@REM MAVEN_OPTS - parameters passed to the Java VM when running Maven -@REM e.g. to debug Maven itself, use -@REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 -@REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files -@REM ---------------------------------------------------------------------------- - -@REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' -@echo off -@REM set title of command window -title %0 -@REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' -@if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% - -@REM set %HOME% to equivalent of $HOME -if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") - -@REM Execute a user defined script before this one -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre -@REM check for pre script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" -if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" -:skipRcPre - -@setlocal - -set ERROR_CODE=0 - -@REM To isolate internal variables from possible post scripts, we use another setlocal -@setlocal - -@REM ==== START VALIDATION ==== -if not "%JAVA_HOME%" == "" goto OkJHome - -echo. -echo Error: JAVA_HOME not found in your environment. >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -:OkJHome -if exist "%JAVA_HOME%\bin\java.exe" goto init - -echo. -echo Error: JAVA_HOME is set to an invalid directory. >&2 -echo JAVA_HOME = "%JAVA_HOME%" >&2 -echo Please set the JAVA_HOME variable in your environment to match the >&2 -echo location of your Java installation. >&2 -echo. -goto error - -@REM ==== END VALIDATION ==== - -:init - -@REM Find the project base dir, i.e. the directory that contains the folder ".mvn". -@REM Fallback to current working directory if not found. - -set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% -IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir - -set EXEC_DIR=%CD% -set WDIR=%EXEC_DIR% -:findBaseDir -IF EXIST "%WDIR%"\.mvn goto baseDirFound -cd .. -IF "%WDIR%"=="%CD%" goto baseDirNotFound -set WDIR=%CD% -goto findBaseDir - -:baseDirFound -set MAVEN_PROJECTBASEDIR=%WDIR% -cd "%EXEC_DIR%" -goto endDetectBaseDir - -:baseDirNotFound -set MAVEN_PROJECTBASEDIR=%EXEC_DIR% -cd "%EXEC_DIR%" - -:endDetectBaseDir - -IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig - -@setlocal EnableExtensions EnableDelayedExpansion -for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a -@endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% - -:endReadAdditionalConfig - -SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" -set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" -set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain - -set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" -FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO ( - IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B -) - -@REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central -@REM This allows using the maven wrapper in projects that prohibit checking in binary data. -if exist %WRAPPER_JAR% ( - echo Found %WRAPPER_JAR% -) else ( - echo Couldn't find %WRAPPER_JAR%, downloading it ... - echo Downloading from: %DOWNLOAD_URL% - powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')" - echo Finished downloading %WRAPPER_JAR% -) -@REM End of extension - -%MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* -if ERRORLEVEL 1 goto error -goto end - -:error -set ERROR_CODE=1 - -:end -@endlocal & set ERROR_CODE=%ERROR_CODE% - -if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost -@REM check for post script, once with legacy .bat ending and once with .cmd ending -if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" -if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" -:skipRcPost - -@REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' -if "%MAVEN_BATCH_PAUSE%" == "on" pause - -if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% - -exit /B %ERROR_CODE% diff --git a/pom.xml b/pom.xml deleted file mode 100644 index 9085800a..00000000 --- a/pom.xml +++ /dev/null @@ -1,306 +0,0 @@ - - 4.0.0 - - com.github.jelmerk - hnswlib-parent-pom - 1.1.0 - hnswlib - pom - https://github.com/jelmerk/hnswlib - Fast approximative nearest neighbour search using the HNSW algorithm - - - - The Apache Software License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - - - Jelmer Kuperus - - Developer - - - - - - UTF-8 - UTF-8 - - 1.8 - 1.8 - - 3.0.0 - 5.5.2 - - - - scm:git:git://github.com/jelmerk/hnswlib.git - scm:git:git@github.com:jelmerk/hnswlib.git - https://github.com/jelmerk/hnswlib/ - HEAD - - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - - hnswlib-utils - hnswlib-core - hnswlib-core-jdk17 - hnswlib-metrics-dropwizard - hnswlib-scala - hnswlib-spark - hnswlib-examples - - - - - - - net.alchim31.maven - scala-maven-plugin - 4.3.1 - - - - maven-compiler-plugin - 3.8.1 - - - - maven-surefire-plugin - 3.0.0-M5 - - - - maven-assembly-plugin - 2.4 - - - - maven-jar-plugin - 3.1.0 - - - - maven-deploy-plugin - 2.8.2 - - - - maven-release-plugin - 2.5.3 - - - - maven-source-plugin - 3.2.0 - - - - maven-javadoc-plugin - 3.3.1 - - - - org.sonatype.plugins - nexus-staging-maven-plugin - 1.6.7 - - - - maven-gpg-plugin - 1.6 - - - - maven-toolchains-plugin - 1.1 - - - - - - - - maven-surefire-plugin - - - - maven-deploy-plugin - - - default-deploy - deploy - - deploy - - - - - - - maven-release-plugin - - true - false - forked-path - -Dgpg.passphrase=${gpg.passphrase} - - - - org.apache.maven.scm - maven-scm-provider-gitexe - 1.9.5 - - - - - - org.sonatype.plugins - nexus-staging-maven-plugin - true - - ossrh - https://oss.sonatype.org/ - true - - - - - org.apache.maven.plugins - maven-source-plugin - - - attach-sources - - jar - - - - - - - maven-javadoc-plugin - - UTF-8 - true - 8 - - - - attach-javadocs - - jar - - - - - - - - - - - - org.eclipse.collections - eclipse-collections - 9.2.0 - - - - io.dropwizard.metrics - metrics-core - 4.1.0 - - - - org.awaitility - awaitility - 4.0.1 - test - - - - org.hamcrest - hamcrest-library - 2.1 - test - - - - org.mockito - mockito-core - ${mockito.version} - test - - - - org.mockito - mockito-junit-jupiter - ${mockito.version} - test - - - - - org.junit.jupiter - junit-jupiter-api - ${junit.version} - test - - - - org.junit.jupiter - junit-jupiter-engine - ${junit.version} - test - - - - - - - - release-sign-artifacts - - - performRelease - true - - - - - - maven-gpg-plugin - - - sign-artifacts - verify - - sign - - - - - - - - - - diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 00000000..abbbce5d --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version=1.9.8 diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 00000000..9fa87dce --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,3 @@ +addSbtPlugin("com.github.sbt" % "sbt-dynver" % "5.0.1") +addSbtPlugin("net.aichler" % "sbt-jupiter-interface" % "0.11.1") +addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1") \ No newline at end of file