conda

conda/Dockerfile.cpu

@@ -0,0 +1,19 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+FROM nvidia/cuda:10.2-devel-ubuntu18.04
+
+RUN apt-get update && apt-get install -y wget git
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+        bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda3
+ENV PATH="/root/miniconda3/condabin:${PATH}"
+
+RUN conda install conda-build
+
+COPY ./ faiss
+WORKDIR /faiss/conda
+
+RUN conda build faiss --no-anaconda-upload -c pytorch

conda/Dockerfile.cuda10.2

@@ -0,0 +1,24 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+FROM nvidia/cuda:10.2-devel-centos8
+
+RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+
+RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+
+RUN yum update -y --nogpgcheck
+
+RUN yum install -y wget git libcublas-devel-10-2
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+        bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda3
+ENV PATH="/root/miniconda3/condabin:${PATH}"
+
+RUN conda install -y -q conda-build anaconda-client
+RUN conda config --set anaconda_upload yes
+
+COPY ./ faiss
+WORKDIR /faiss/conda

conda/Dockerfile.cuda11.3

@@ -0,0 +1,24 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+FROM nvidia/cuda:11.3.1-devel-centos8
+
+RUN sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-*
+
+RUN sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-*
+
+RUN yum update -y --nogpgcheck
+
+RUN yum install -y --nogpgcheck wget git libcublas-devel-11-3
+
+RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
+        bash Miniconda3-latest-Linux-x86_64.sh -b -p ~/miniconda3
+ENV PATH="/root/miniconda3/condabin:${PATH}"
+
+RUN conda install -y -q conda-build anaconda-client
+RUN conda config --set anaconda_upload yes
+
+COPY ./ faiss
+WORKDIR /faiss/conda

conda/conda_build_config.yaml

@@ -0,0 +1,6 @@
+CONDA_BUILD_SYSROOT:
+  - /opt/MacOSX10.9.sdk        # [osx]
+python:
+  - 3.7
+  - 3.8
+  - 3.9

conda/faiss-gpu/build-lib.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+
+# Build libfaiss.so/libfaiss_avx2.so.
+cmake -B _build \
+      -DBUILD_SHARED_LIBS=ON \
+      -DBUILD_TESTING=OFF \
+      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_ENABLE_GPU=ON \
+      -DCMAKE_CUDA_ARCHITECTURES="${CUDA_ARCHS}" \
+      -DFAISS_ENABLE_PYTHON=OFF \
+      -DBLA_VENDOR=Intel10_64lp \
+      -DCMAKE_INSTALL_LIBDIR=lib \
+      -DCMAKE_BUILD_TYPE=Release .
+
+make -C _build -j $CPU_COUNT faiss faiss_avx2
+
+cmake --install _build --prefix $PREFIX
+cmake --install _build --prefix _libfaiss_stage/

conda/faiss-gpu/build-pkg.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+
+# Build swigfaiss.so/swigfaiss_avx2.so.
+cmake -B _build_python_${PY_VER} \
+      -Dfaiss_ROOT=_libfaiss_stage/ \
+      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_ENABLE_GPU=ON \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DPython_EXECUTABLE=$PYTHON \
+      faiss/python
+
+make -C _build_python_${PY_VER} -j $CPU_COUNT swigfaiss swigfaiss_avx2
+
+# Build actual python module.
+cd _build_python_${PY_VER}/
+$PYTHON setup.py install --single-version-externally-managed --record=record.txt --prefix=$PREFIX

conda/faiss-gpu/install-cmake.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+wget -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | tar xzf -
+cp -R cmake-3.17.1-Linux-x86_64/* $PREFIX

conda/faiss-gpu/meta.yaml

@@ -0,0 +1,90 @@
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+{% set version = environ.get('GIT_DESCRIBE_TAG').lstrip('v') %}
+{% set suffix = "_nightly" if environ.get('PACKAGE_TYPE') == 'nightly' else "" %}
+{% set number = GIT_DESCRIBE_NUMBER %}
+
+package:
+  name: faiss-pkg
+  version: {{ version }}
+
+build:
+  number: {{ number }}
+
+about:
+  home: https://github.com/facebookresearch/faiss
+  license: MIT
+  license_family: MIT
+  license_file: LICENSE
+  summary: A library for efficient similarity search and clustering of dense vectors.
+
+source:
+  git_url: ../../
+
+outputs:
+  - name: libfaiss
+    script: build-lib.sh
+    build:
+      string: "h{{ PKG_HASH }}_{{ number }}_cuda{{ cudatoolkit }}{{ suffix }}"
+      run_exports:
+        - {{ pin_compatible('libfaiss', exact=True) }}
+      script_env:
+        - CUDA_ARCHS
+    requirements:
+      build:
+        - {{ compiler('cxx') }}
+        - llvm-openmp  # [osx]
+        - cmake >=3.18
+        - make  # [not win]
+        - mkl-devel =2021
+      host:
+        - mkl =2021
+        - cudatoolkit {{ cudatoolkit }}
+      run:
+        - mkl >=2021
+        - {{ pin_compatible('cudatoolkit', max_pin='x.x') }}
+    test:
+      requires:
+        - conda-build
+      commands:
+        - test -f $PREFIX/lib/libfaiss.so              # [linux]
+        - test -f $PREFIX/lib/libfaiss.dylib           # [osx]
+        - conda inspect linkages -p $PREFIX $PKG_NAME  # [not win]
+        - conda inspect objects -p $PREFIX $PKG_NAME   # [osx]
+
+  - name: faiss-gpu
+    script: build-pkg.sh
+    build:
+      string: "py{{ PY_VER }}_h{{ PKG_HASH }}_{{ number }}_cuda{{ cudatoolkit }}{{ suffix }}"
+    requirements:
+      build:
+        - {{ compiler('cxx') }}
+        - swig
+        - cmake >=3.17
+        - make  # [not win]
+      host:
+        - python {{ python }}
+        - numpy =1.16
+        - {{ pin_subpackage('libfaiss', exact=True) }}
+      run:
+        - python {{ python }}
+        - numpy >=1.11,<2
+        - {{ pin_subpackage('libfaiss', exact=True) }}
+    test:
+      requires:
+        - numpy
+        - scipy
+        - pytorch
+      commands:
+        - python -m unittest discover tests/
+        - cp tests/common_faiss_tests.py faiss/gpu/test
+        - python -m unittest discover faiss/gpu/test/
+        - sh test_cpu_dispatch.sh  # [linux]
+      files:
+        - test_cpu_dispatch.sh  # [linux]
+      source_files:
+        - tests/
+        - faiss/gpu/test/

conda/faiss-gpu/test_cpu_dispatch.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+FAISS_DISABLE_CPU_FEATURES=AVX2 LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss.so
+LD_DEBUG=libs python -c "import faiss" 2>&1 | grep libfaiss_avx2.so

conda/faiss/build-lib.bat

@@ -0,0 +1,23 @@
+:: Copyright (c) Facebook, Inc. and its affiliates.
+::
+:: This source code is licensed under the MIT license found in the
+:: LICENSE file in the root directory of this source tree.
+
+:: Build libfaiss.so.
+cmake -B _build ^
+      -T v141 ^
+      -A x64 ^
+      -G "Visual Studio 16 2019" ^
+      -DBUILD_SHARED_LIBS=ON ^
+      -DBUILD_TESTING=OFF ^
+      -DFAISS_ENABLE_GPU=OFF ^
+      -DFAISS_ENABLE_PYTHON=OFF ^
+      -DBLA_VENDOR=Intel10_64_dyn ^
+      .
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+cmake --build _build --config Release -j %CPU_COUNT%
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+cmake --install _build --config Release --prefix %PREFIX%
+if %errorlevel% neq 0 exit /b %errorlevel%

conda/faiss/build-lib.sh

@@ -0,0 +1,24 @@
+#!/bin/sh
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+
+# Build libfaiss.so/libfaiss_avx2.so.
+cmake -B _build \
+      -DBUILD_SHARED_LIBS=ON \
+      -DBUILD_TESTING=OFF \
+      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_ENABLE_GPU=OFF \
+      -DFAISS_ENABLE_PYTHON=OFF \
+      -DBLA_VENDOR=Intel10_64lp \
+      -DCMAKE_INSTALL_LIBDIR=lib \
+      -DCMAKE_BUILD_TYPE=Release .
+
+make -C _build -j $CPU_COUNT faiss faiss_avx2
+
+cmake --install _build --prefix $PREFIX
+cmake --install _build --prefix _libfaiss_stage/

conda/faiss/build-pkg.bat

@@ -0,0 +1,23 @@
+:: Copyright (c) Facebook, Inc. and its affiliates.
+::
+:: This source code is licensed under the MIT license found in the
+:: LICENSE file in the root directory of this source tree.
+
+:: Build vanilla version (no avx).
+cmake -B _build_python_%PY_VER% ^
+      -T v141 ^
+      -A x64 ^
+      -G "Visual Studio 16 2019" ^
+      -DFAISS_ENABLE_GPU=OFF ^
+      -DPython_EXECUTABLE=%PYTHON% ^
+      faiss/python
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+cmake --build _build_python_%PY_VER% --config Release -j %CPU_COUNT%
+if %errorlevel% neq 0 exit /b %errorlevel%
+
+
+:: Build actual python module.
+cd _build_python_%PY_VER%/
+%PYTHON% setup.py install --single-version-externally-managed --record=record.txt --prefix=%PREFIX%
+if %errorlevel% neq 0 exit /b %errorlevel%

conda/faiss/build-pkg.sh

@@ -0,0 +1,23 @@
+#!/bin/sh
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+
+# Build swigfaiss.so/swigfaiss_avx2.so.
+cmake -B _build_python_${PY_VER} \
+      -Dfaiss_ROOT=_libfaiss_stage/ \
+      -DFAISS_OPT_LEVEL=avx2 \
+      -DFAISS_ENABLE_GPU=OFF \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DPython_EXECUTABLE=$PYTHON \
+      faiss/python
+
+make -C _build_python_${PY_VER} -j $CPU_COUNT swigfaiss swigfaiss_avx2
+
+# Build actual python module.
+cd _build_python_${PY_VER}/
+$PYTHON setup.py install --single-version-externally-managed --record=record.txt --prefix=$PREFIX

conda/faiss/install-cmake.sh

@@ -0,0 +1,10 @@
+#!/bin/sh#
+# Copyright (c) Facebook, Inc. and its affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+
+wget -O - https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz | tar xzf -
+cp -R cmake-3.17.1-Linux-x86_64/* $PREFIX