Use python3 and rucio client with cmssw 10 (#5237)

* use Python3 for CMSSW_10, add our LumiList, fix LumiList import, remove
pycurl from crab.py

* fix PSetCores check in CMSSWConfig.py

* remove unused, obsolete code from report.py

bin/crab

@@ -10,7 +10,7 @@ CMSSW_Serie=$(echo $CMSSW_VERSION | cut -d_ -f3)  # e.g. from CMSSW_10_6_x this
 # would be good to do also for CMSSW_10_6 + (for UL) but we use
 # FWCore.PythonUtilities.LumiList which imports urllib2 which
 # is not available in python3
-[ $CMSSW_Major -ge 11 ] && python_cmd="python3"
+[ $CMSSW_Major -ge 10 ] && python_cmd="python3"
 
 if [ $python_cmd = "python3" ] ; then
   # can also include Rucio client (we do not want to deal with old python2 client)

bin/crab.py

@@ -29,12 +29,6 @@
     print('\nError: using a version of python < 2.6. Exiting...\n')
     sys.exit()
 
-import pycurl
-if not 'OpenSSL' in pycurl.version:
-    print('\nError: missing SSL support in pycurl. Make sure you do cmsenv first. Exiting...')
-    print('pycurl version is: %s\n' % pycurl.version)
-    sys.exit()
-
 if 'crab-dev' in __file__:
     print('BEWARE: this is the development version of CRAB Client.\nBe sure to have a good reason for using it\n')
 
@@ -178,13 +172,6 @@ def log_exception(exc_type, exc_value, tback):
             errorId = re.search(r'(?<=X-Error-Id:\s)[^\n]*', err).group(0)
             client.logger.info('Error Id: %s', errorId)
         logging.getLogger('CRAB3').exception('Caught RESTInterfaceException exception')
-    except pycurl.error as pe:
-        client.logger.error(pe)
-        logging.getLogger('CRAB3').exception('Caught pycurl.error exception')
-        exitcode = pe.args[0]
-        # SB following line is wrong, the exception (pe) is not a list. Anyhow we want to remove pycurl
-        if pe[1].find('(DNS server returned answer with no data)'):
-            client.logger.info(schedInterv)
     except ClientException as ce:
         client.logger.error(ce)
         logging.getLogger('CRAB3').exception('Caught ClientException exception')

src/python/CRABClient/Commands/report.py

@@ -5,7 +5,10 @@
 import tarfile
 from ast import literal_eval
 
-from FWCore.PythonUtilities.LumiList import LumiList
+try:
+    from FWCore.PythonUtilities.LumiList import LumiList
+except Exception:
+    from CRABClient.LumiList import LumiList
 
 from CRABClient.ClientUtilities import colors, execute_command
 from CRABClient.Commands.SubCommand import SubCommand
@@ -486,35 +489,3 @@ def validateOptions(self):
             msg = "%sError%s:" % (colors.RED, colors.NORMAL)
             msg += " The --recovery option only accepts the following values: %s" % (recoveryMethods)
             raise ConfigurationException(msg)
-
-################# Unused functions moved here just in case we find out why this code is here ###############
-
-    def compactLumis(self, datasetInfo):
-        """ Help function that allow to convert from runLumis divided per file (result of listDatasetFileDetails)
-            to an aggregated result.
-        """
-        lumilist = {}
-        for _, info in datasetInfo.items():
-            for run, lumis in info['Lumis'].items():
-                lumilist.setdefault(str(run), []).extend(lumis)
-        return lumilist
-
-    def prepareCurl(self):
-        import pycurl
-        curl = pycurl.Curl()
-        curl.setopt(pycurl.NOSIGNAL, 0)
-        curl.setopt(pycurl.TIMEOUT, 30)
-        curl.setopt(pycurl.CONNECTTIMEOUT, 30)
-        curl.setopt(pycurl.FOLLOWLOCATION, 0)
-        curl.setopt(pycurl.MAXREDIRS, 0)
-        return curl
-
-    def myPerform(self, curl, url):
-        import pycurl
-        try:
-            curl.perform()
-        except pycurl.error as e:
-            raise ClientException(("Failed to contact Grid scheduler when getting URL %s. "
-                                   "This might be a temporary error, please retry later and "
-                                   "contact %s if the error persist. Error from curl: %s" % \
-                                   (url, FEEDBACKMAIL, str(e))))

src/python/CRABClient/JobType/Analysis.py

@@ -15,7 +15,10 @@
 except:  # pylint: disable=bare-except
     from httplib import HTTPException  # old Python 2 version in CMSSW_7
 
-from FWCore.PythonUtilities.LumiList import LumiList
+try:
+    from FWCore.PythonUtilities.LumiList import LumiList
+except Exception:
+    from CRABClient.LumiList import LumiList
 
 from ServerUtilities import BOOTSTRAP_CFGFILE_DUMP
 

src/python/CRABClient/JobType/BasicJobType.py

@@ -6,7 +6,10 @@
 """
 from ast import literal_eval
 
-from FWCore.PythonUtilities.LumiList import LumiList
+try:
+    from FWCore.PythonUtilities.LumiList import LumiList
+except Exception:
+    from CRABClient.LumiList import LumiList
 
 from CRABClient.ClientExceptions import ConfigurationException
 

src/python/CRABClient/JobType/CMSSWConfig.py

@@ -271,8 +271,8 @@ def validateConfig(self):
             return False, msg
 
         #Assumes a default of 1 if the parameter is not specified
-        cfgNumCores = getattr(self.config.JobType, 'numCores', None)
-        numPSetCores = getattr(getattr(self.fullConfig.process, 'options', object), 'numberOfThreads', None)
+        cfgNumCores = getattr(self.config.JobType, 'numCores', 1)
+        numPSetCores = getattr(getattr(self.fullConfig.process, 'options', object), 'numberOfThreads', uint32(1))
         if numPSetCores and not isinstance(numPSetCores, uint32):
             msg = "The only accepted type for process.options.numberOfThreads is uint32. Please, change its type in the PSet."
             msg += "See https://github.com/cms-sw/cmssw/issues/32070 for details"

src/python/CRABClient/JobType/CopyCat.py

@@ -14,7 +14,10 @@
 from functools import reduce
 from ast import literal_eval
 
-from FWCore.PythonUtilities.LumiList import LumiList
+try:
+    from FWCore.PythonUtilities.LumiList import LumiList
+except Exception:
+    from CRABClient.LumiList import LumiList
 
 from ServerUtilities import BOOTSTRAP_CFGFILE_DUMP, getProxiedWebDir, NEW_USER_SANDBOX_EXCLUSIONS
 from ServerUtilities import SERVICE_INSTANCES

src/python/CRABClient/JobType/LumiMask.py

@@ -7,7 +7,10 @@
 if sys.version_info < (3, 0):
     from urlparse import urlparse
 
-from FWCore.PythonUtilities.LumiList import LumiList
+try:
+    from FWCore.PythonUtilities.LumiList import LumiList
+except Exception:
+    from CRABClient.LumiList import LumiList
 
 from CRABClient.ClientExceptions import ConfigurationException