New release 3.4.0

.gitignore

@@ -7,3 +7,4 @@ prody2/protocols/__pycache__/
 
 # IDE files
 .idea
+.vscode

CHANGES.txt

@@ -1,3 +1,20 @@
+12 April 2024 - major release v3.4.0
+    1. Added logistic regression analysis (LRA) instead of LDA as a much better way for comparing ensemble subsets
+    2. Added protocols for fetching data from the BioExcel-CV19 simulation database
+    3. Many fixes, especially to ensembles, PCA and viewers
+
+8 February 2024
+    1. Added linear discriminant analysis (LDA) for comparing ensemble subsets
+    2. Many fixes, especially to ensemble building
+
+20 October 2023
+    1. Overhauled installation to install ProDy itself in scipion3 env 
+    and also ProDy, OpenMM, PDBFixer and TEMPy in its own env with activation for apps
+    2. Added new ClustENM fitting based on MDeNMD-EMFit
+
+16 October
+    General improvements to ensemble analysis
+
 13 August 2023 - major release v3.3.0
     Substantial integration and improvements of everything:
     1. Improved ensembles and alignments including custom chain matching

README.rst

@@ -36,6 +36,12 @@ python with conda ones. One example of this could can be seen below but
 depending on your conda version and shell you will need something different:
 CONDA_ACTIVATION_CMD = eval "$(/extra/miniconda3/bin/conda shell.bash hook)"
 
+*PRODY_ENV_ACT*: If undefined, it will point to the prody-github as the default:
+PRODY_ENV_ACT = conda activate prody-github
+
+It could be changed as follows:
+PRODY_ENV_ACT = conda activate prody-2.4.1
+
 
 Protocols
 ----------

prody2/__init__.py

@@ -23,7 +23,7 @@
 # *  e-mail address '[email protected]'
 # *
 # **************************************************************************
-
+from collections import OrderedDict
 import os
 import pwem
 import pyworkflow.utils as pwutils
@@ -32,15 +32,19 @@
 from .constants import *
 
 
-__version__ = "3.3.0"
+__version__ = "3.4.0"
 _logo = "icon.png"
-_references = ['Zhang2021']
+_references = ['ProDy2']
 
 
 class Plugin(pwem.Plugin):
     _supportedVersions = VERSIONS
     _url = "https://github.com/scipion-em/scipion-em-prody"
 
+    @classmethod
+    def _defineVariables(cls):
+        cls._defineVar(PRODY_ENV_ACT, "conda activate prody-{0}".format(PRODY_DEFAULT_VER_NUM))
+
     @classmethod
     def getEnviron(cls):
         """ Setup the environment variables needed to launch ProDy. """
@@ -65,65 +69,98 @@ def getDependencies(cls):
     def defineBinaries(cls, env):
         for ver in VERSIONS:
             cls.addProDyPackage(env, ver,
-                                default=ver == PRODY_DEFAULT_VER_NUM)
+                                default=(ver==PRODY_DEFAULT_VER_NUM))
 
     @classmethod
     def addProDyPackage(cls, env, version, default=False):
-        PRODY_INSTALLED = 'prody_%s_installed' % version
-        installCmd = []
-
-        if version == DEVEL:
-            # Use latest scipion branch of prody on my github
-            installCmd.append('cd .. &&')
-            clonePath = os.path.join(pwem.Config.EM_ROOT, "ProDy")
-            if not os.path.exists(clonePath):
-                installCmd.append('git clone -b master https://github.com/prody/ProDy.git ProDy &&')
-            installCmd.append('cd ProDy &&')
 
-        # Install downloaded code
-        installCmd.append('pip install -U -e . && python setup.py build_ext --inplace --force &&')
+        ENV_NAME = getProDyEnvName(version)
 
-        if version == DEVEL:
-            installCmd.append('cd .. && cd prody-github &&')
-
-        installCmd.append('python -c "import os; os.environ.setdefault(\'HOME\', \'{0}\')" &&'.format(Config.SCIPION_HOME + os.path.sep))
+        installCmd = [
+            cls.getCondaActivationCmd(),
+            f'conda create -y -n {ENV_NAME} python=3.9 &&',
+            f'conda activate {ENV_NAME} &&']
 
-        # Flag installation finished
-        installCmd.append('touch %s' % PRODY_INSTALLED)
+        # Install TEMPy for ClustENM fitting, scikit-learn-extra for Kmedoids
+        # and threadpoolctl for control of thread pools for apps generally
+        TEMPY_INSTALLED = 'tempy_installed'
+        installTEMPy = installCmd.copy()
+        installTEMPy.append('pip install biotempy==2.0.0 scikit-learn-extra '
+                            'threadpoolctl requests mdtraj pyparsing==3.1.1 && touch %s' % TEMPY_INSTALLED)
+        installCmd.pop(1) # remove conda create to only do it the first time
 
         # Install PDBFixer and OpenMM for ClustENM
-        OPEN_MM_INSTALLED = 'openmm_installed'
-        installOpenMM = 'conda install -c conda-forge pdbfixer -y && touch %s' % OPEN_MM_INSTALLED
-
-        prody_commands = [(" ".join(installCmd), PRODY_INSTALLED),
-                          (installOpenMM, OPEN_MM_INSTALLED)]
+        OPENMM_INSTALLED = 'openmm_installed'
+        installOpenMM = installCmd.copy()
+        installOpenMM.append('conda install -c conda-forge openmm==7.6 pdbfixer -y && touch %s' % OPENMM_INSTALLED)
+
+        prodyCommands = [(" ".join(installTEMPy), TEMPY_INSTALLED),
+                         (" ".join(installOpenMM), OPENMM_INSTALLED)]
+
+        PRODY_INSTALLED_OWN = 'prody_%s_installed_own_env' % version
+        PRODY_INSTALLED_SCIPION = 'prody_%s_installed_scipion_env' % version
+        for i, PRODY_INSTALLED in enumerate([PRODY_INSTALLED_OWN, PRODY_INSTALLED_SCIPION]):
+            if i == 0:
+
+                installCmd.append('git clone https://github.com/jamesmkrieger/ProDy.git ProDy &&')
+                installCmd.append('cd ProDy &&')
+                installCmd.append('git fetch &&')
+
+                installCmd.append('git remote add upstream https://github.com/prody/ProDy.git &&')
+                installCmd.append('git fetch upstream &&')
+                installCmd.append('git checkout -t upstream/main &&')
+
+                installCmd.append('git checkout scipion &&')
+                installCmd.append('git pull &&')
+
+                installCmd.append('pip install -Ue . && python setup.py build_ext --inplace --force &&')
+            else:
+                installCmd = []
+                installCmd.append('cd ProDy &&')
+                installCmd.append('pip install -Ue . &&')
+
+            installCmd.append('cd .. && touch %s' % PRODY_INSTALLED)
+
+            prodyCommands.append((" ".join(installCmd.copy()), PRODY_INSTALLED))
 
         envHome = os.environ.get('HOME', "")
         envPath = os.environ.get('PATH', "")
         # keep path since conda likely in there, and home since prody needs it to configure
         installEnvVars = {'PATH': envPath, 'HOME': envHome} if envPath else {'HOME': envHome}
 
-        if version == DEVEL:
-            env.addPackage('prody', version=version,
-                            tar='void.tgz',
-                            commands=prody_commands,
-                            neededProgs=cls.getDependencies(),
-                            default=default,
-                            vars=installEnvVars)
-        else:
-            env.addPackage('prody', version=version,
-                           url='https://github.com/prody/ProDy/archive/refs/tags/v{0}.tar.gz'.format(version),
-                           buildDir='ProDy-{0}'.format(version),
-                           commands=prody_commands,
-                           neededProgs=cls.getDependencies(),
-                           default=default,
-                           vars=installEnvVars)            
+        env.addPackage('prody', version=version,
+                        tar='void.tgz',
+                        buildDir='ProDy',
+                        commands=prodyCommands,
+                        neededProgs=cls.getDependencies(),
+                        default=default,
+                        vars=installEnvVars)
 
     @classmethod
-    def getProgram(cls, program):
+    def getProgram(cls, program, script=False):
         """ Create ProDy command line. """
-        fullProgram = '%s && prody %s' % (
-            cls.getCondaActivationCmd(),
-            program)
+        if script:
+            fullProgram = '%s %s && python %s' % (
+                cls.getCondaActivationCmd(), cls.getEnvActivation(),
+                PRODY_SCRIPTS+'/'+program)
+        else:
+            fullProgram = '%s %s && prody %s' % (
+                cls.getCondaActivationCmd(), cls.getEnvActivation(),
+                program)
 
         return fullProgram
+
+    @classmethod
+    def getEnvActivation(cls):
+        return cls.getVar(PRODY_ENV_ACT)
+
+def parseMatchDict(cls):
+    if cls.chainOrders.get() != "":
+        cls.matchDic = eval(cls.chainOrders.get())
+    else:
+        cls.matchDic = OrderedDict()
+
+    if not isinstance(cls.matchDic, OrderedDict):
+        cls.matchDic = OrderedDict()
+
+    cls.labels = list(cls.matchDic.keys())

prody2/constants.py

@@ -24,14 +24,38 @@
 # *
 # **************************************************************************
 
+import os
+import prody2
 
 def getProDyEnvName(version):
     return "prody-%s" % version
 
 DEVEL = 'github'
-LATEST = '2.4.1'
-VERSIONS = [DEVEL, LATEST]
-PRODY_DEFAULT_VER_NUM = LATEST
+LATEST = 'master'
+RELEASE = '2.4.1'
+VERSIONS = [DEVEL]
+PRODY_DEFAULT_VER_NUM = DEVEL
+
+PRODY_ENV_ACT = "PRODY_ENV_ACT"
 
 PROJ_COEFFS = "_prodyProjCoefficients"
-CLUSTENM_WEIGHTS = "_prodyWeight"
+ENSEMBLE_WEIGHTS = "_prodyWeights"
+MEASURES = "_prodyMeasures"
+
+PRODY_FRACT_VARS = "_prodyFractVars"
+
+PRODY_SCRIPTS = os.path.join(os.path.dirname(prody2.__file__),
+                             "protocols", "scripts")
+
+
+# chain matching methods
+BEST_MATCH = 0
+SAME_CHID = 1
+SAME_POS = 2
+CUSTOM = 3
+
+# residue mapping methods
+NOTHING = 0 # stop trivial mapping if trivial mapping fails
+PWALIGN = 1 # biopython pwalign local pairwise sequence alignment after trivial mapping
+CEALIGN = 2 # combinatorial extension (CE) as in PyMOL
+DEFAULT = 3 # try pwalign then CE