Added Faiss, and Faiss_GPU

* Added mingw32 patch: To make posix_memalign etc. available on mingw32. * Dropped CUDA archs 60, 61 for CUDA 11.8+. * Update CMake wrt. CUDA arch detection: https://github.com/Kitware/CMake/blob/v3.23.3/Modules/CMakeDetermineCUDACompiler.cmake#L654 * Fixed CUDA cmake configure: Added symlink from $prefix/cuda/lib64 to $prefix/cuda/lib * Updated to v1.9.0 * Added header file products * Added additional patches for mingw32
JuliaPackaging · Dec 4, 2024 · 70c112e · 70c112e
1 parent f04be74
commit 70c112e
Show file tree

Hide file tree

Showing 6 changed files with 245 additions and 0 deletions.
diff --git a/F/Faiss/Faiss/build_tarballs.jl b/F/Faiss/Faiss/build_tarballs.jl
@@ -0,0 +1,11 @@
+# Note that this script can accept some limited command-line arguments, run
+# `julia build_tarballs.jl --help` to see a usage message.
+using BinaryBuilder, Pkg
+
+name = "Faiss"
+version = v"1.9.0"
+
+include(joinpath(@__DIR__, "..", "common.jl"))
+
+# Build the tarballs, and possibly a `build.jl` as well.
+build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version=v"7")
diff --git a/F/Faiss/Faiss_GPU/build_tarballs.jl b/F/Faiss/Faiss_GPU/build_tarballs.jl
@@ -0,0 +1,48 @@
+# Note that this script can accept some limited command-line arguments, run
+# `julia build_tarballs.jl --help` to see a usage message.
+using BinaryBuilder, Pkg
+
+const YGGDRASIL_DIR = "../../.."
+include(joinpath(YGGDRASIL_DIR, "fancy_toys.jl"))
+include(joinpath(YGGDRASIL_DIR, "platforms", "cuda.jl"))
+
+name = "Faiss_GPU"
+version = v"1.9.0"
+
+# Conda GPU packages build on 11.4 and 12.1: https://github.com/facebookresearch/faiss/blob/v1.9.0/.github/workflows/build.yml#L182-L260
+cuda_versions = [
+    # 11.4 does not provide cuda_profiler_api (used by faiss)
+    "11.8", # 11.8 to have cuda_profiler_api, and SM 87;89;90 support.
+    "12.1",
+] 
+
+include(joinpath(@__DIR__, "..", "common.jl"))
+
+# Override the default platforms
+platforms = CUDA.supported_platforms()
+filter!(p -> arch(p) == "x86_64", platforms)
+
+filter!(p -> p["cuda"] in cuda_versions, platforms)
+
+# Override the default products
+products = [
+    products...,
+    FileProduct("include/faiss/gpu/GpuIndex.h", :faiss_gpu_gpuindex_h),
+    FileProduct("include/faiss/c_api/gpu/GpuIndex_c.h", :faiss_c_api_gpu_gpuindex_c_h),
+]
+
+# Build for all supported CUDA toolkits
+for platform in platforms
+    should_build_platform(triplet(platform)) || continue
+
+    cuda_deps = CUDA.required_dependencies(platform; static_sdk=true)
+
+    build_tarballs(ARGS, name, version, sources, script, [platform], products, [dependencies; cuda_deps];
+                   lazy_artifacts=true,
+                   julia_compat="1.6",
+                   preferred_gcc_version=v"7",
+                   augment_platform_block=CUDA.augment,
+                   skip_audit=true,
+                   dont_dlopen=true)
+end
+# trigger yggy
diff --git a/F/Faiss/bundled/patches/faiss-mingw32-InvertedListsIOHook.patch b/F/Faiss/bundled/patches/faiss-mingw32-InvertedListsIOHook.patch
@@ -0,0 +1,25 @@
+diff --git a/faiss/invlists/InvertedListsIOHook.cpp b/faiss/invlists/InvertedListsIOHook.cpp
+index 0081c4f9..2c3a6006 100644
+--- a/faiss/invlists/InvertedListsIOHook.cpp
++++ b/faiss/invlists/InvertedListsIOHook.cpp
+@@ -13,9 +13,9 @@
+
+ #include <faiss/invlists/BlockInvertedLists.h>
+
+-#ifndef _MSC_VER
++#ifndef _WIN32
+ #include <faiss/invlists/OnDiskInvertedLists.h>
+-#endif // !_MSC_VER
++#endif // !_WIN32
+
+ namespace faiss {
+
+@@ -33,7 +33,7 @@ namespace {
+ /// std::vector that deletes its contents
+ struct IOHookTable : std::vector<InvertedListsIOHook*> {
+     IOHookTable() {
+-#ifndef _MSC_VER
++#ifndef _WIN32
+         push_back(new OnDiskInvertedListsIOHook());
+ #endif
+         push_back(new BlockInvertedListsIOHook());
diff --git a/F/Faiss/bundled/patches/faiss-mingw32-cmake.patch b/F/Faiss/bundled/patches/faiss-mingw32-cmake.patch
@@ -0,0 +1,15 @@
+diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt
+index 2871d974..bc017d10 100644
+--- a/faiss/CMakeLists.txt
++++ b/faiss/CMakeLists.txt
+@@ -326,6 +326,10 @@ if(WIN32)
+   target_compile_definitions(faiss_sve PRIVATE FAISS_MAIN_LIB)
+ endif()
+
++if(WIN32)
++  set_target_properties(faiss PROPERTIES LINK_FLAGS "-Wl,--export-all-symbols")
++endif()
++
+ string(FIND "${CMAKE_CXX_FLAGS}" "FINTEGER" finteger_idx)
+ if (${finteger_idx} EQUAL -1)
+   target_compile_definitions(faiss PRIVATE FINTEGER=int)
diff --git a/F/Faiss/bundled/patches/faiss-mingw32.patch b/F/Faiss/bundled/patches/faiss-mingw32.patch
@@ -0,0 +1,44 @@
+diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h
+index 2aecc512..0c792190 100644
+--- a/faiss/impl/platform_macros.h
++++ b/faiss/impl/platform_macros.h
+@@ -11,7 +11,7 @@
+ #include <cstdint>
+ #include <cstdio>
+
+-#ifdef _MSC_VER
++#ifdef _WIN32
+
+ /*******************************************************
+  * Windows specific macros
+@@ -23,11 +23,11 @@
+ #define FAISS_API __declspec(dllimport)
+ #endif // FAISS_MAIN_LIB
+
+-#ifdef _MSC_VER
+ #define strtok_r strtok_s
+-#endif // _MSC_VER
+
++#ifdef _MSC_VER
+ #define __PRETTY_FUNCTION__ __FUNCSIG__
++#endif // _MSC_VER
+
+ #define posix_memalign(p, a, s) \
+     (((*(p)) = _aligned_malloc((s), (a))), *(p) ? 0 : errno)
+@@ -37,6 +37,7 @@
+ #define ALIGNED(x) __declspec(align(x))
+
+ // redefine the GCC intrinsics with Windows equivalents
++#ifdef _MSC_VER
+
+ #include <intrin.h>
+
+@@ -91,6 +92,8 @@ inline int __builtin_clzll(uint64_t x) {
+ #define __F16C__ 1
+ #endif
+
++#endif // _MSC_VER
++
+ #define FAISS_ALWAYS_INLINE __forceinline
+
+ #else
diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl
@@ -0,0 +1,102 @@
+# Collection of sources required to complete build
+sources = [
+    GitSource("https://github.com/facebookresearch/faiss.git", "d243e628880676332263347817b3fe7f474b8b5b"),
+    DirectorySource(joinpath(@__DIR__, "bundled")),
+]
+
+# Bash recipe for building across all platforms
+script = raw"""
+# Needs CMake >= 3.23.1 provided via HostBuildDependency
+apk del cmake
+
+cd faiss
+
+atomic_patch -p1 ../patches/faiss-mingw32-cmake.patch
+atomic_patch -p1 ../patches/faiss-mingw32-InvertedListsIOHook.patch
+atomic_patch -p1 ../patches/faiss-mingw32.patch
+
+cmake_extra_args=()
+
+if [[ $bb_full_target == *cuda* ]]; then
+    cuda_version=${bb_full_target##*-cuda+}
+    if [[ $cuda_version == "11.8" ]]; then
+        cuda_archs="60-real;61-real;62-real;70-real;72-real;75-real;80;86-real;87;89;90"
+    elif [[ $cuda_version == "12.1" ]]; then
+        cuda_archs="70-real;72-real;75-real;80;86-real;87;89;90"
+    else
+        false # Fail for unexpected CUDA version
+    fi
+
+    # CUDA compilation can run out of storage
+    mkdir $WORKSPACE/tmpdir
+    export TMPDIR=$WORKSPACE/tmpdir
+
+    export CUDA_PATH=$prefix/cuda
+    ln -s $prefix/cuda/lib $prefix/cuda/lib64
+    cmake_extra_args+=(
+        -DFAISS_ENABLE_GPU=ON
+        -DCUDAToolkit_ROOT=$CUDA_PATH
+        -DCMAKE_CUDA_ARCHITECTURES=$cuda_archs
+    )
+fi
+
+configure() {
+cmake -B build \
+    -DCMAKE_INSTALL_PREFIX=$prefix \
+    -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DFAISS_ENABLE_GPU=OFF \
+    -DFAISS_ENABLE_PYTHON=OFF \
+    -DBUILD_TESTING=OFF \
+    -DBUILD_SHARED_LIBS=ON \
+    -DFAISS_ENABLE_C_API=ON \
+    ${cmake_extra_args[@]}
+}
+configure || configure || configure
+cmake --build build --parallel ${nproc}
+cmake --install build
+
+install -Dvm 755 build/c_api/libfaiss_c.$dlext $libdir/libfaiss_c.$dlext
+
+if [[ $bb_full_target == *cuda* ]]; then
+    unlink $prefix/cuda/lib64
+fi
+"""
+
+# These are the platforms we will build for by default, unless further
+# platforms are passed in on the command line
+platforms = supported_platforms()
+
+mkl_platforms = Platform[
+    Platform("x86_64", "Linux"),
+    Platform("i686", "Linux"),
+    Platform("x86_64", "MacOS"),
+    Platform("x86_64", "Windows"),
+]
+
+openblas_platforms = filter(p -> p ∉ mkl_platforms, platforms)
+
+platforms = expand_cxxstring_abis(platforms)
+mkl_platforms = expand_cxxstring_abis(mkl_platforms)
+openblas_platforms = expand_cxxstring_abis(openblas_platforms)
+
+# The products that we will ensure are always built
+products = Product[
+    FileProduct("include/faiss/Index.h", :faiss_index_h),
+    FileProduct("include/faiss/c_api/faiss_c.h", :faiss_c_api_faiss_c_h),
+    LibraryProduct(["libfaiss", "faiss"], :libfaiss),
+    LibraryProduct(["libfaiss_c", "faiss_c"], :libfaiss_c),
+]
+
+# Dependencies that must be installed before this package can be built
+dependencies = [
+    # For OpenMP we use libomp from `LLVMOpenMP_jll` where we use LLVM as compiler (BSD
+     # systems), and libgomp from `CompilerSupportLibraries_jll` everywhere else.
+    Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae"); platforms=filter(!Sys.isbsd, platforms)),
+    Dependency(PackageSpec(name="LLVMOpenMP_jll", uuid="1d63c593-3942-5779-bab2-d838dc0a180e"); platforms=filter(Sys.isbsd, platforms)),
+    Dependency("LAPACK_jll"; platforms = openblas_platforms),
+    Dependency("MKL_jll"; platforms = mkl_platforms),
+    BuildDependency("MKL_Headers_jll"; platforms = mkl_platforms),
+    Dependency("OpenBLAS32_jll"; platforms = openblas_platforms),
+    HostBuildDependency(PackageSpec("CMake_jll", v"3.28.1")),
+]