Fixes to allow docker build

environment.yml

@@ -6,7 +6,7 @@ channels:
 dependencies:
   - pip:
       - biopython==1.79
-      - deepspeed==0.5.3
+      - deepspeed==0.5.10
       - dm-tree==0.1.6
       - ml-collections==0.1.0
       - numpy==1.21.2
@@ -16,15 +16,18 @@ dependencies:
       - tqdm==4.62.2
       - typing-extensions==3.10.0.2
       - pytorch_lightning==1.5.0
-      - nvidia-pyindex
-      - nvidia-dllogger
-  - pytorch::pytorch=1.10.*
+      - wandb==0.12.21
+      - git+https://github.com/NVIDIA/dllogger.git
   - conda-forge::python=3.7
   - conda-forge::setuptools=59.5.0
   - conda-forge::pip
   - conda-forge::openmm=7.5.1
   - conda-forge::pdbfixer
+  - conda-forge::cudatoolkit==11.3.*
   - bioconda::aria2
   - bioconda::hmmer==3.3.2
   - bioconda::hhsuite==3.3.0
   - bioconda::kalign2==2.04
+  - pytorch::pytorch=1.12.*
+
+

scripts/utils.py

@@ -1,22 +1,21 @@
 import argparse
+import ctypes
 from datetime import date
+import sys
 
 
 def add_data_args(parser: argparse.ArgumentParser):
     parser.add_argument(
-        'uniref90_database_path', type=str, 
+        '--uniref90_database_path', type=str, default=None,
     )
     parser.add_argument(
-        'mgnify_database_path', type=str, 
+        '--mgnify_database_path', type=str, default=None,
     )
     parser.add_argument(
-        'pdb70_database_path', type=str,
+        '--pdb70_database_path', type=str, default=None,
     )
     parser.add_argument(
-        'template_mmcif_dir', type=str,
-    )
-    parser.add_argument(
-        'uniclust30_database_path', type=str,
+        '--uniclust30_database_path', type=str, default=None,
     )
     parser.add_argument(
         '--bfd_database_path', type=str, default=None,
@@ -34,7 +33,7 @@ def add_data_args(parser: argparse.ArgumentParser):
         '--kalign_binary_path', type=str, default='/usr/bin/kalign'
     )
     parser.add_argument(
-        '--max_template_date', type=str, 
+        '--max_template_date', type=str,
         default=date.today().strftime("%Y-%m-%d"),
     )
     parser.add_argument(
@@ -43,3 +42,67 @@ def add_data_args(parser: argparse.ArgumentParser):
     parser.add_argument(
         '--release_dates_path', type=str, default=None
     )
+
+
+def get_nvidia_cc():
+    """
+    Returns a tuple containing the Compute Capability of the first GPU
+    installed in the system (formatted as a tuple of strings) and an error
+    message. When the former is provided, the latter is None, and vice versa.
+
+    Adapted from script by Jan Schlüte t
+    https://gist.github.com/f0k/63a664160d016a491b2cbea15913d549
+    """
+    CUDA_SUCCESS = 0
+
+    libnames = [
+        'libcuda.so', 
+        'libcuda.dylib', 
+        'cuda.dll',
+        '/usr/local/cuda/compat/libcuda.so', # For Docker
+    ]
+    for libname in libnames:
+        try:
+            cuda = ctypes.CDLL(libname)
+        except OSError:
+            continue
+        else:
+            break
+    else:
+        return None, "Could not load any of: " + ' '.join(libnames)
+
+    nGpus = ctypes.c_int()
+    cc_major = ctypes.c_int()
+    cc_minor = ctypes.c_int()
+
+    result = ctypes.c_int()
+    device = ctypes.c_int()
+    error_str = ctypes.c_char_p()
+
+    result = cuda.cuInit(0)
+    if result != CUDA_SUCCESS:
+        cuda.cuGetErrorString(result, ctypes.byref(error_str))
+        if error_str.value:
+            return None, error_str.value.decode()
+        else:
+            return None, "Unknown error: cuInit returned %d" % result
+    result = cuda.cuDeviceGetCount(ctypes.byref(nGpus))
+    if result != CUDA_SUCCESS:
+        cuda.cuGetErrorString(result, ctypes.byref(error_str))
+        return None, error_str.value.decode()
+
+    if nGpus.value < 1:
+        return None, "No GPUs detected"
+
+    result = cuda.cuDeviceGet(ctypes.byref(device), 0)
+    if result != CUDA_SUCCESS:
+        cuda.cuGetErrorString(result, ctypes.byref(error_str))
+        return None, error_str.value.decode()
+
+    if cuda.cuDeviceComputeCapability(ctypes.byref(cc_major), ctypes.byref(cc_minor), device) != CUDA_SUCCESS:
+        return None, "Compute Capability not found"
+
+    major = cc_major.value
+    minor = cc_minor.value
+
+    return (major, minor), None

setup.py

@@ -0,0 +1,133 @@
+# Copyright 2021 AlQuraishi Laboratory
+# Copyright 2021 DeepMind Technologies Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import os
+from setuptools import setup, Extension, find_packages
+import subprocess
+
+from torch.utils.cpp_extension import BuildExtension, CppExtension, CUDAExtension, CUDA_HOME
+
+from scripts.utils import get_nvidia_cc
+
+
+version_dependent_macros = [
+    '-DVERSION_GE_1_1',
+    '-DVERSION_GE_1_3',
+    '-DVERSION_GE_1_5',
+]
+
+extra_cuda_flags = [
+    '-std=c++14',
+    '-maxrregcount=50',
+    '-U__CUDA_NO_HALF_OPERATORS__',
+    '-U__CUDA_NO_HALF_CONVERSIONS__',
+    '--expt-relaxed-constexpr',
+    '--expt-extended-lambda'
+]
+
+def get_cuda_bare_metal_version(cuda_dir):
+    if cuda_dir==None:
+        print("CUDA is not found, cpu version is installed")
+        return None, -1, 0
+    else:
+        raw_output = subprocess.check_output([cuda_dir + "/bin/nvcc", "-V"], universal_newlines=True)
+        output = raw_output.split()
+        release_idx = output.index("release") + 1
+        release = output[release_idx].split(".")
+        bare_metal_major = release[0]
+        bare_metal_minor = release[1][0]
+
+        return raw_output, bare_metal_major, bare_metal_minor
+
+compute_capabilities = set([
+    (3, 7), # K80, e.g.
+    (5, 2), # Titan X
+    (6, 1), # GeForce 1000-series
+])
+
+compute_capabilities.add((7, 0))
+_, bare_metal_major, _ = get_cuda_bare_metal_version(CUDA_HOME)
+if int(bare_metal_major) >= 11:
+    compute_capabilities.add((8, 0))
+
+compute_capability, _ = get_nvidia_cc()
+if compute_capability is not None:
+    compute_capabilities = set([compute_capability])
+
+cc_flag = []
+for major, minor in list(compute_capabilities):
+    cc_flag.extend([
+        '-gencode',
+        f'arch=compute_{major}{minor},code=sm_{major}{minor}',
+    ])
+
+extra_cuda_flags += cc_flag
+
+if bare_metal_major != -1:
+    modules = [CUDAExtension(
+        name="attn_core_inplace_cuda",
+        sources=[
+            "openfold/utils/kernel/csrc/softmax_cuda.cpp",
+            "openfold/utils/kernel/csrc/softmax_cuda_kernel.cu",
+        ],
+        include_dirs=[
+            os.path.join(
+                os.path.dirname(os.path.abspath(__file__)),
+                'openfold/utils/kernel/csrc/'
+            )
+        ],
+        extra_compile_args={
+            'cxx': ['-O3'] + version_dependent_macros,
+            'nvcc': (
+                ['-O3', '--use_fast_math'] +
+                version_dependent_macros +
+                extra_cuda_flags
+            ),
+        }
+    )]
+else:
+    modules = [CppExtension(
+        name="attn_core_inplace_cuda",
+        sources=[
+            "openfold/utils/kernel/csrc/softmax_cuda.cpp",
+            "openfold/utils/kernel/csrc/softmax_cuda_stub.cpp",
+        ],
+        extra_compile_args={
+            'cxx': ['-O3'],
+        }
+    )]
+
+setup(
+    name='openfold',
+    version='1.0.1',
+    description='A PyTorch reimplementation of DeepMind\'s AlphaFold 2',
+    author='Gustaf Ahdritz & DeepMind',
+    author_email='[email protected]',
+    license='Apache License, Version 2.0',
+    url='https://github.com/aqlaboratory/openfold',
+    packages=find_packages(exclude=["tests", "scripts"]),
+    include_package_data=True,
+    package_data={
+        "openfold": ['utils/kernel/csrc/*'],
+        "": ["resources/stereo_chemical_props.txt"]
+    },
+    ext_modules=modules,
+    cmdclass={'build_ext': BuildExtension},
+    classifiers=[
+        'License :: OSI Approved :: Apache Software License',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python :: 3.7,'
+        'Topic :: Scientific/Engineering :: Artificial Intelligence',
+    ],
+)