From 70c112e4c31d624550a44b5995d41ba5c08da927 Mon Sep 17 00:00:00 2001 From: Jesper Stemann Andersen Date: Wed, 4 Dec 2024 09:18:06 +0100 Subject: [PATCH] Added Faiss, and Faiss_GPU * Added mingw32 patch: To make posix_memalign etc. available on mingw32. * Dropped CUDA archs 60, 61 for CUDA 11.8+. * Update CMake wrt. CUDA arch detection: https://github.com/Kitware/CMake/blob/v3.23.3/Modules/CMakeDetermineCUDACompiler.cmake#L654 * Fixed CUDA cmake configure: Added symlink from $prefix/cuda/lib64 to $prefix/cuda/lib * Updated to v1.9.0 * Added header file products * Added additional patches for mingw32 --- F/Faiss/Faiss/build_tarballs.jl | 11 ++ F/Faiss/Faiss_GPU/build_tarballs.jl | 48 +++++++++ .../faiss-mingw32-InvertedListsIOHook.patch | 25 +++++ .../bundled/patches/faiss-mingw32-cmake.patch | 15 +++ F/Faiss/bundled/patches/faiss-mingw32.patch | 44 ++++++++ F/Faiss/common.jl | 102 ++++++++++++++++++ 6 files changed, 245 insertions(+) create mode 100644 F/Faiss/Faiss/build_tarballs.jl create mode 100644 F/Faiss/Faiss_GPU/build_tarballs.jl create mode 100644 F/Faiss/bundled/patches/faiss-mingw32-InvertedListsIOHook.patch create mode 100644 F/Faiss/bundled/patches/faiss-mingw32-cmake.patch create mode 100644 F/Faiss/bundled/patches/faiss-mingw32.patch create mode 100644 F/Faiss/common.jl diff --git a/F/Faiss/Faiss/build_tarballs.jl b/F/Faiss/Faiss/build_tarballs.jl new file mode 100644 index 00000000000..19277ef34ec --- /dev/null +++ b/F/Faiss/Faiss/build_tarballs.jl @@ -0,0 +1,11 @@ +# Note that this script can accept some limited command-line arguments, run +# `julia build_tarballs.jl --help` to see a usage message. +using BinaryBuilder, Pkg + +name = "Faiss" +version = v"1.9.0" + +include(joinpath(@__DIR__, "..", "common.jl")) + +# Build the tarballs, and possibly a `build.jl` as well. +build_tarballs(ARGS, name, version, sources, script, platforms, products, dependencies; julia_compat="1.6", preferred_gcc_version=v"7") diff --git a/F/Faiss/Faiss_GPU/build_tarballs.jl b/F/Faiss/Faiss_GPU/build_tarballs.jl new file mode 100644 index 00000000000..ca6adfbdd95 --- /dev/null +++ b/F/Faiss/Faiss_GPU/build_tarballs.jl @@ -0,0 +1,48 @@ +# Note that this script can accept some limited command-line arguments, run +# `julia build_tarballs.jl --help` to see a usage message. +using BinaryBuilder, Pkg + +const YGGDRASIL_DIR = "../../.." +include(joinpath(YGGDRASIL_DIR, "fancy_toys.jl")) +include(joinpath(YGGDRASIL_DIR, "platforms", "cuda.jl")) + +name = "Faiss_GPU" +version = v"1.9.0" + +# Conda GPU packages build on 11.4 and 12.1: https://github.com/facebookresearch/faiss/blob/v1.9.0/.github/workflows/build.yml#L182-L260 +cuda_versions = [ + # 11.4 does not provide cuda_profiler_api (used by faiss) + "11.8", # 11.8 to have cuda_profiler_api, and SM 87;89;90 support. + "12.1", +] + +include(joinpath(@__DIR__, "..", "common.jl")) + +# Override the default platforms +platforms = CUDA.supported_platforms() +filter!(p -> arch(p) == "x86_64", platforms) + +filter!(p -> p["cuda"] in cuda_versions, platforms) + +# Override the default products +products = [ + products..., + FileProduct("include/faiss/gpu/GpuIndex.h", :faiss_gpu_gpuindex_h), + FileProduct("include/faiss/c_api/gpu/GpuIndex_c.h", :faiss_c_api_gpu_gpuindex_c_h), +] + +# Build for all supported CUDA toolkits +for platform in platforms + should_build_platform(triplet(platform)) || continue + + cuda_deps = CUDA.required_dependencies(platform; static_sdk=true) + + build_tarballs(ARGS, name, version, sources, script, [platform], products, [dependencies; cuda_deps]; + lazy_artifacts=true, + julia_compat="1.6", + preferred_gcc_version=v"7", + augment_platform_block=CUDA.augment, + skip_audit=true, + dont_dlopen=true) +end +# trigger yggy diff --git a/F/Faiss/bundled/patches/faiss-mingw32-InvertedListsIOHook.patch b/F/Faiss/bundled/patches/faiss-mingw32-InvertedListsIOHook.patch new file mode 100644 index 00000000000..a29dc7b4000 --- /dev/null +++ b/F/Faiss/bundled/patches/faiss-mingw32-InvertedListsIOHook.patch @@ -0,0 +1,25 @@ +diff --git a/faiss/invlists/InvertedListsIOHook.cpp b/faiss/invlists/InvertedListsIOHook.cpp +index 0081c4f9..2c3a6006 100644 +--- a/faiss/invlists/InvertedListsIOHook.cpp ++++ b/faiss/invlists/InvertedListsIOHook.cpp +@@ -13,9 +13,9 @@ + + #include + +-#ifndef _MSC_VER ++#ifndef _WIN32 + #include +-#endif // !_MSC_VER ++#endif // !_WIN32 + + namespace faiss { + +@@ -33,7 +33,7 @@ namespace { + /// std::vector that deletes its contents + struct IOHookTable : std::vector { + IOHookTable() { +-#ifndef _MSC_VER ++#ifndef _WIN32 + push_back(new OnDiskInvertedListsIOHook()); + #endif + push_back(new BlockInvertedListsIOHook()); diff --git a/F/Faiss/bundled/patches/faiss-mingw32-cmake.patch b/F/Faiss/bundled/patches/faiss-mingw32-cmake.patch new file mode 100644 index 00000000000..ef0455a1658 --- /dev/null +++ b/F/Faiss/bundled/patches/faiss-mingw32-cmake.patch @@ -0,0 +1,15 @@ +diff --git a/faiss/CMakeLists.txt b/faiss/CMakeLists.txt +index 2871d974..bc017d10 100644 +--- a/faiss/CMakeLists.txt ++++ b/faiss/CMakeLists.txt +@@ -326,6 +326,10 @@ if(WIN32) + target_compile_definitions(faiss_sve PRIVATE FAISS_MAIN_LIB) + endif() + ++if(WIN32) ++ set_target_properties(faiss PROPERTIES LINK_FLAGS "-Wl,--export-all-symbols") ++endif() ++ + string(FIND "${CMAKE_CXX_FLAGS}" "FINTEGER" finteger_idx) + if (${finteger_idx} EQUAL -1) + target_compile_definitions(faiss PRIVATE FINTEGER=int) diff --git a/F/Faiss/bundled/patches/faiss-mingw32.patch b/F/Faiss/bundled/patches/faiss-mingw32.patch new file mode 100644 index 00000000000..1d825e0d7ce --- /dev/null +++ b/F/Faiss/bundled/patches/faiss-mingw32.patch @@ -0,0 +1,44 @@ +diff --git a/faiss/impl/platform_macros.h b/faiss/impl/platform_macros.h +index 2aecc512..0c792190 100644 +--- a/faiss/impl/platform_macros.h ++++ b/faiss/impl/platform_macros.h +@@ -11,7 +11,7 @@ + #include + #include + +-#ifdef _MSC_VER ++#ifdef _WIN32 + + /******************************************************* + * Windows specific macros +@@ -23,11 +23,11 @@ + #define FAISS_API __declspec(dllimport) + #endif // FAISS_MAIN_LIB + +-#ifdef _MSC_VER + #define strtok_r strtok_s +-#endif // _MSC_VER + ++#ifdef _MSC_VER + #define __PRETTY_FUNCTION__ __FUNCSIG__ ++#endif // _MSC_VER + + #define posix_memalign(p, a, s) \ + (((*(p)) = _aligned_malloc((s), (a))), *(p) ? 0 : errno) +@@ -37,6 +37,7 @@ + #define ALIGNED(x) __declspec(align(x)) + + // redefine the GCC intrinsics with Windows equivalents ++#ifdef _MSC_VER + + #include + +@@ -91,6 +92,8 @@ inline int __builtin_clzll(uint64_t x) { + #define __F16C__ 1 + #endif + ++#endif // _MSC_VER ++ + #define FAISS_ALWAYS_INLINE __forceinline + + #else diff --git a/F/Faiss/common.jl b/F/Faiss/common.jl new file mode 100644 index 00000000000..11027e69fbd --- /dev/null +++ b/F/Faiss/common.jl @@ -0,0 +1,102 @@ +# Collection of sources required to complete build +sources = [ + GitSource("https://github.com/facebookresearch/faiss.git", "d243e628880676332263347817b3fe7f474b8b5b"), + DirectorySource(joinpath(@__DIR__, "bundled")), +] + +# Bash recipe for building across all platforms +script = raw""" +# Needs CMake >= 3.23.1 provided via HostBuildDependency +apk del cmake + +cd faiss + +atomic_patch -p1 ../patches/faiss-mingw32-cmake.patch +atomic_patch -p1 ../patches/faiss-mingw32-InvertedListsIOHook.patch +atomic_patch -p1 ../patches/faiss-mingw32.patch + +cmake_extra_args=() + +if [[ $bb_full_target == *cuda* ]]; then + cuda_version=${bb_full_target##*-cuda+} + if [[ $cuda_version == "11.8" ]]; then + cuda_archs="60-real;61-real;62-real;70-real;72-real;75-real;80;86-real;87;89;90" + elif [[ $cuda_version == "12.1" ]]; then + cuda_archs="70-real;72-real;75-real;80;86-real;87;89;90" + else + false # Fail for unexpected CUDA version + fi + + # CUDA compilation can run out of storage + mkdir $WORKSPACE/tmpdir + export TMPDIR=$WORKSPACE/tmpdir + + export CUDA_PATH=$prefix/cuda + ln -s $prefix/cuda/lib $prefix/cuda/lib64 + cmake_extra_args+=( + -DFAISS_ENABLE_GPU=ON + -DCUDAToolkit_ROOT=$CUDA_PATH + -DCMAKE_CUDA_ARCHITECTURES=$cuda_archs + ) +fi + +configure() { +cmake -B build \ + -DCMAKE_INSTALL_PREFIX=$prefix \ + -DCMAKE_TOOLCHAIN_FILE=${CMAKE_TARGET_TOOLCHAIN} \ + -DCMAKE_BUILD_TYPE=Release \ + -DFAISS_ENABLE_GPU=OFF \ + -DFAISS_ENABLE_PYTHON=OFF \ + -DBUILD_TESTING=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DFAISS_ENABLE_C_API=ON \ + ${cmake_extra_args[@]} +} +configure || configure || configure +cmake --build build --parallel ${nproc} +cmake --install build + +install -Dvm 755 build/c_api/libfaiss_c.$dlext $libdir/libfaiss_c.$dlext + +if [[ $bb_full_target == *cuda* ]]; then + unlink $prefix/cuda/lib64 +fi +""" + +# These are the platforms we will build for by default, unless further +# platforms are passed in on the command line +platforms = supported_platforms() + +mkl_platforms = Platform[ + Platform("x86_64", "Linux"), + Platform("i686", "Linux"), + Platform("x86_64", "MacOS"), + Platform("x86_64", "Windows"), +] + +openblas_platforms = filter(p -> p ∉ mkl_platforms, platforms) + +platforms = expand_cxxstring_abis(platforms) +mkl_platforms = expand_cxxstring_abis(mkl_platforms) +openblas_platforms = expand_cxxstring_abis(openblas_platforms) + +# The products that we will ensure are always built +products = Product[ + FileProduct("include/faiss/Index.h", :faiss_index_h), + FileProduct("include/faiss/c_api/faiss_c.h", :faiss_c_api_faiss_c_h), + LibraryProduct(["libfaiss", "faiss"], :libfaiss), + LibraryProduct(["libfaiss_c", "faiss_c"], :libfaiss_c), +] + +# Dependencies that must be installed before this package can be built +dependencies = [ + # For OpenMP we use libomp from `LLVMOpenMP_jll` where we use LLVM as compiler (BSD + # systems), and libgomp from `CompilerSupportLibraries_jll` everywhere else. + Dependency(PackageSpec(name="CompilerSupportLibraries_jll", uuid="e66e0078-7015-5450-92f7-15fbd957f2ae"); platforms=filter(!Sys.isbsd, platforms)), + Dependency(PackageSpec(name="LLVMOpenMP_jll", uuid="1d63c593-3942-5779-bab2-d838dc0a180e"); platforms=filter(Sys.isbsd, platforms)), + Dependency("LAPACK_jll"; platforms = openblas_platforms), + Dependency("MKL_jll"; platforms = mkl_platforms), + BuildDependency("MKL_Headers_jll"; platforms = mkl_platforms), + Dependency("OpenBLAS32_jll"; platforms = openblas_platforms), + HostBuildDependency(PackageSpec("CMake_jll", v"3.28.1")), +]