Refactor third party (#94)

.github/workflows/build.yml

@@ -88,6 +88,7 @@ jobs:
             TARGET_ARCH=${{ matrix.arch }}
             LIBOMP_USE_HIDDEN_HELPER_TASK=0
             LIBOMP_NUM_HIDDEN_HELPER_THREADS=0
+            KMP_DUPLICATE_LIB_OK=TRUE
             MACOSX_DEPLOYMENT_TARGET=10.14
           CIBW_ENVIRONMENT_WINDOWS: >
             FAISS_OPT_LEVEL=${{ matrix.opt_level }}
@@ -96,7 +97,10 @@ jobs:
             LIB="${LIB};${CMAKE_PREFIX_PATH}\\lib;${CONDA}\\Library\\lib"
             CPATH="${CPATH};${CMAKE_PREFIX_PATH}\\include;${CONDA}\\Library\\include"
           CIBW_BEFORE_ALL: bash scripts/build_${{ runner.os }}.sh
-          CIBW_TEST_COMMAND: env -u FAISS_OPT_LEVEL pytest {project}/faiss/tests
+          CIBW_BEFORE_TEST_LINUX: pip install torch --index-url https://download.pytorch.org/whl/cpu
+          CIBW_TEST_COMMAND: >
+            env -u FAISS_OPT_LEVEL pytest {project}/faiss/tests &&
+            env -u FAISS_OPT_LEVEL pytest -s {project}/faiss/tests/torch_test_contrib.py
 
       - uses: actions/upload-artifact@v4
         with:

pyproject.toml

@@ -32,27 +32,24 @@ classifiers = [
     "Programming Language :: Python :: 3.12",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
 ]
-dependencies = [
-    "numpy",
-]
+dependencies = ["numpy"]
 
 [project.urls]
 Repository = "https://github.com/kyamagu/faiss-wheels"
 
 [tool.pytest.ini_options]
 minversion = "6.0"
 addopts = [
-    "--capture=fd",
     # Failing tests for numerical issues.
     "--deselect=faiss/tests/test_local_search_quantizer.py::TestProductLocalSearchQuantizer::test_lut",
 ]
 testpaths = ["faiss/tests"]
 
 [tool.cibuildwheel]
 skip = "pp* *-musllinux*"
-test-skip = "*-macosx_arm64 *-manylinux_aarch64 *-win_amd64"
+test-skip = "*-macosx_arm64 *-manylinux_aarch64 *-win_amd64 cp312-*"
 
-test-requires = ["pytest", "scipy"]
+test-requires = ["pytest", "scipy", "torch"]
 test-command = "pytest {project}/faiss/tests"
 
 [tool.cibuildwheel.windows]

scripts/build_Linux.sh

@@ -1,14 +1,14 @@
 #!/usr/bin/env bash
 
+set -eux
+
 export CXXFLAGS="-fvisibility=hidden -fdata-sections -ffunction-sections"
 
 FAISS_ENABLE_GPU=${FAISS_ENABLE_GPU:-"OFF"}
-FAISS_OPT_LEVEL=${FAISS_OPT_LEVEL:-"generic"}
 
-CUDA_VERSION="11.0"
-CUDA_PKG_VERSION="11-0"
+CUDA_VERSION=${CUDA_VERSION:-"12.3"}
+CUDA_PKG_VERSION=${CUDA_PKG_VERSION:-${CUDA_VERSION//./-}}
 NVIDIA_REPO_URL="http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo"
-CMAKE_CUDA_ARCHITECTURES="60-real;70-real;75-real;80"
 
 # Fix manylinux2014 aarch64
 if [[ $(uname -m) == "aarch64" ]]; then
@@ -37,14 +37,11 @@ if [[ ${FAISS_ENABLE_GPU} == "ON" ]]; then
         ldconfig
 
     export PATH="/opt/rh/devtoolset-7/root/usr/bin:/usr/local/cuda/bin:${PATH}"
-    export CUDAFLAGS="--compiler-options=-fvisibility=hidden,-fdata-sections,-ffunction-sections"
+    export CUDAFLAGS="--compiler-options=${CXXFLAGS// /,}"
 fi
 
 # Install system dependencies
-yum install -y \
-    openblas-devel \
-    openblas-static \
-    swig3
+yum install -y openblas-devel openblas-static swig3
 
 # Build and patch faiss
 cd faiss && \
@@ -53,8 +50,7 @@ cd faiss && \
         -DFAISS_ENABLE_GPU=${FAISS_ENABLE_GPU} \
         -DFAISS_ENABLE_PYTHON=OFF \
         -DBUILD_TESTING=OFF \
-        -DCMAKE_CUDA_ARCHITECTURES=${CMAKE_CUDA_ARCHITECTURES} \
-        -DFAISS_OPT_LEVEL=${FAISS_OPT_LEVEL} \
+        -DFAISS_OPT_LEVEL=${FAISS_OPT_LEVEL:-"generic"} \
         -DCMAKE_BUILD_TYPE=Release && \
     cmake --build build --config Release -j3 && \
     cmake --install build && \

scripts/build_Windows.sh

@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
 
+set -eux
+
 CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-"c:\\opt"}
-FAISS_OPT_LEVEL=${FAISS_OPT_LEVEL:-"generic"}
 
 # Install system dependencies
 conda.bat install -c conda-forge openblas
@@ -14,12 +15,12 @@ cd faiss && \
         -A x64 \
         -DFAISS_ENABLE_GPU=OFF \
         -DFAISS_ENABLE_PYTHON=OFF \
-        -DFAISS_OPT_LEVEL=${FAISS_OPT_LEVEL} \
+        -DFAISS_OPT_LEVEL=${FAISS_OPT_LEVEL:-"generic"} \
         -DBUILD_TESTING=OFF \
         -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}" \
         -DCMAKE_BUILD_TYPE=Release \
         -DBLA_STATIC=ON && \
     cmake --build build --config Release -j && \
-    cmake --install build --prefix ${CMAKE_PREFIX_PATH} && \
+    cmake --install build --prefix "${CMAKE_PREFIX_PATH}" && \
     git apply ../patch/faiss-rename-swigfaiss.patch && \
     cd ..

scripts/build_macOS.sh

@@ -1,7 +1,6 @@
 #!/usr/bin/env bash
 
-FAISS_OPT_LEVEL=${FAISS_OPT_LEVEL:-"generic"}
-LLVM_VERSION="llvmorg-16.0.2"
+set -eux
 
 HOST_ARCH=${HOST_ARCH:-$(uname -m)}
 TARGET_ARCH=${TARGET_ARCH:-$HOST_ARCH}
@@ -13,25 +12,26 @@ echo "TARGET_ARCH=${TARGET_ARCH}"
 # Install system dependencies
 brew install swig
 
-# Build libomp
+# Build libomp: needed for cross compilation
 echo "Building libomp"
 git clone \
         --depth 1 \
         --filter=blob:none \
         --sparse \
-        --branch ${LLVM_VERSION} \
-        https://github.com/llvm/llvm-project.git && \
-    cd llvm-project && \
+        --branch ${LLVM_VERSION:-"llvmorg-17.0.6"} \
+        https://github.com/llvm/llvm-project.git \
+        third-party/llvm-project && \
+    cd third-party/llvm-project && \
     git sparse-checkout set openmp cmake && \
     cd openmp && \
     cmake . \
         -B build \
         -DCMAKE_BUILD_TYPE=Release \
         -DCMAKE_OSX_ARCHITECTURES=${TARGET_ARCH} \
-        -DLIBOMP_ENABLE_SHARED=OFF && \
+        -DLIBOMP_ENABLE_SHARED=ON && \
     cmake --build build -j && \
     cmake --install build && \
-    cd ../..
+    cd ../../..
 
 # Build and patch faiss
 echo "Building faiss"
@@ -41,7 +41,7 @@ cd faiss && \
         -DFAISS_ENABLE_GPU=OFF \
         -DFAISS_ENABLE_PYTHON=OFF \
         -DBUILD_TESTING=OFF \
-        -DFAISS_OPT_LEVEL=${FAISS_OPT_LEVEL} \
+        -DFAISS_OPT_LEVEL=${FAISS_OPT_LEVEL:-"generic"} \
         -DCMAKE_BUILD_TYPE=Release \
         -DCMAKE_OSX_ARCHITECTURES=${TARGET_ARCH} && \
     cmake --build build --config Release -j && \