Merge pull request #47 in WSCLIENT/nda-tools-python from bugfix/NDA-5…

…080-python-nda-tools-package-tries-to-load-configuration-file-before-it-has-been-copied to master

* commit '6473cf8382d7a12f8edd42b263d7444255919385':
  NDA-5080 - create .NDATools/settings.cfg on initialization of Configuration class * Updated release version
  NDA-5080 - create .NDATools/settings.cfg on initialization of Configuration class * Updated Utils.py to initialize Configuration class with template settings.cfg file from python package installation directory * Updated changelog
  NDA-5076 - Python2 compatibility fixes * Updated changelog
  NDA-5076 - Python2 compatibility fixes * Removed use of star operator to unpack objects in print statements * Removed Python3 specific exceptions in favor of python2/3 compatible OSError and IOError exceptions * Small PEP-8 fixes (spacing) - more to do.

CHANGELOG.md

@@ -1,8 +1,12 @@
 
 # CHANGELOG
 
+## 0.1.20 - 2019-07-15
+* [`nda-tools`] Python2 compatibility fixes and some PEP-8 style changes; fixes [GitHub Issue 9](https://github.com/NDAR/nda-tools/issues/9)
+* [`nda-tools`] Addressed issue where configuration file cannot be located on fresh installations.
+
 ## 0.1.19 - 2019-07-02
-* [`vtcmd`] Fixed issue with writing debug log files to directory before it is created
+* [`vtcmd`] Fixed issue with writing debug log files to directory before it is created; fixes [GitHub Issue 12](https://github.com/NDAR/nda-tools/issues/12)
 * [`vtcmd`] Removed write mode when opening associated files for upload to S3
 * [`vtcmd`] Update to only fetch multi-part credentials once during resume option in check_submitted_files()
 * [`downloadcmd`] Add message regarding transfer limit to download(5TB)

NDATools/BuildPackage.py

@@ -11,6 +11,7 @@
 from NDATools.Configuration import *
 from NDATools.Utils import *
 
+
 class SubmissionPackage:
     def __init__(self, uuid, associated_files, config, username=None, password=None, collection=None, title=None,
                  description=None, alternate_location=None, allow_exit=False):
@@ -159,9 +160,9 @@ def check_read_permissions(self, file):
         try:
             open(file)
             return True
-        except IOError:
-            return False
-        except PermissionError:
+        except (OSError, IOError) as err:
+            if err.errno == 13:
+                print('Permission Denied: {}'.format(file))
             return False
 
     def file_search(self, directories=None, source_bucket=None, source_prefix=None, access_key=None, secret_key=None, retry_allowed=False):

NDATools/Configuration.py

@@ -12,6 +12,7 @@
 import os
 from pkg_resources import resource_filename
 
+
 class ClientConfiguration:
     def __init__(self, settings_file, username=None, password=None, access_key=None, secret_key=None):
         self.config = configparser.ConfigParser()

NDATools/Download.py

@@ -90,9 +90,9 @@ def __init__(self, directory, config=None, verbose=False):
     def get_protocol(cls):
         return cls.XML
 
-    def verbose_print(self, *args, **kwargs):
+    def verbose_print(self, *args):
         if self.verbose:
-            print(*args, **kwargs)
+            print(' '.join(list(args)))
 
     def useDataManager(self):
         """ Download package files (not associated files) """
@@ -175,10 +175,6 @@ def useDataStructure(self, data_structure):
             self.verbose_print(
                 '{} not found. Please enter the correct path to your file and try again.'.format(self.dataStructure))
             raise e
-        except FileNotFoundError:
-            self.verbose_print(
-                '{} not found. Please enter the correct path to your file and try again.'.format(self.dataStructure))
-            raise FileNotFoundError
 
 
     def get_links(self, links, files, filters=None):
@@ -253,7 +249,7 @@ def download_path(self, path, resume, prev_directory):
 
             try:
                 s3transfer.download_file(bucket, key, local_filename)
-                self.verbose_print('downloaded: ', path)
+                self.verbose_print('downloaded: {}'.format(path))
 
             except botocore.exceptions.ClientError as e:
                 # If a client error is thrown, then check that it was a 404 error.

NDATools/Submission.py

@@ -223,9 +223,9 @@ def check_read_permissions(self, file):
         try:
             open(file)
             return True
-        except IOError:
-            return False
-        except PermissionError:
+        except (OSError, IOError) as err:
+            if err.errno == 13:
+                print('Permission Denied: {}'.format(file))
             return False
 
     def found_all_files(self, directories=None, source_bucket=None, source_prefix=None, access_key=None, secret_key=None, retry_allowed=False):

NDATools/Utils.py

@@ -15,11 +15,11 @@
 
 from NDATools.Configuration import ClientConfiguration
 
-config = ClientConfiguration(os.path.join(os.path.expanduser('~'), '.NDATools/settings.cfg'))
-if config.validation_results:
-    validation_results_dir = os.path.join(os.path.expanduser('~'), config.validation_results)
+if os.path.isfile(os.path.join(os.path.expanduser('~'), '.NDATools/settings.cfg')):
+    config = ClientConfiguration(os.path.join(os.path.expanduser('~'), '.NDATools/settings.cfg'))
 else:
-    validation_results_dir = os.path.join(os.path.expanduser('~'), "NDAValidationResults")
+    config = ClientConfiguration('clientscripts/config/settings.cfg')
+validation_results_dir = os.path.join(os.path.expanduser('~'), config.validation_results)
 if not os.path.exists(validation_results_dir):
     os.mkdir(validation_results_dir)
 log_file = os.path.join(validation_results_dir, "debug_log_{}.txt").format(time.strftime("%Y%m%dT%H%M%S"))

NDATools/Validation.py

@@ -300,7 +300,6 @@ def __init__(self, _validation_result, hide):
                 manifests.append(Validation.ValidationManifest(_manifest, hide))
             self.manifests = manifests
 
-
     class ValidationTask(threading.Thread, Protocol):
         def __init__(self, file_queue, result_queue, api, scope, responses, validation_progress, exit):
             threading.Thread.__init__(self)
@@ -337,15 +336,6 @@ def run(self):
                         error = " ".join(['FileNotFound:', message])
                         raise Exception(error)
 
-                except FileNotFoundError:
-                    message = 'This file does not exist in current directory: {}'.format(file_name)
-                    if self.progress_bar:
-                        self.progress_bar.close()
-                    if self.exit:
-                        exit_client(signal=signal.SIGTERM, message=message)
-                    else:
-                        error = " ".join(['FileNotFound:', message])
-                        raise Exception(error)
                 data = file.read()
 
                 response, session = api_request(self, "POST", self.api_scope, data)

NDATools/__init__.py

@@ -3,7 +3,7 @@
 import json
 import sys
 
-__version__ = '0.1.19'
+__version__ = '0.1.20'
 pypi_version = None
 version_checked = False
 
@@ -12,7 +12,7 @@ def check_version():
         from packaging.version import parse
     except ImportError:
         from pip._vendor.packaging.version import parse
-
+    # use https://test.pypi.org/pypi/{package}/json on test/release branches, use https://pypi.org on master
     url_pattern = 'https://pypi.org/pypi/{package}/json'
     package = 'nda-tools'
     """Return version of package on pypi.python.org using json."""

NDATools/clientscripts/downloadcmd.py

@@ -53,7 +53,6 @@ def parse_args():
     return args
 
 
-
 def configure(username, password):
 
     NDATools.Utils.logging.getLogger().setLevel(logging.INFO)
@@ -109,12 +108,12 @@ def main():
     s3Download.start_workers(resume, prev_directory, args.workerThreads)
 
     # download associated files from package
-    #if args.package:
+    # if args.package:
     #    s3Download.searchForDataStructure(resume, prev_directory)
 
-
     if verbose:
         print('Finished downloading all files.')
 
+
 if __name__ == "__main__":
     main()

NDATools/clientscripts/vtcmd.py

@@ -9,6 +9,7 @@
 import os
 import pkg_resources
 
+
 def parse_args():
     parser = argparse.ArgumentParser(
         description='This application allows you to validate files and submit data into NDA. '