diff --git a/CHANGELOG.md b/CHANGELOG.md index 3100e3c..3ad7ab0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,12 @@ # CHANGELOG +## 0.1.20 - 2019-07-15 +* [`nda-tools`] Python2 compatibility fixes and some PEP-8 style changes; fixes [GitHub Issue 9](https://github.com/NDAR/nda-tools/issues/9) +* [`nda-tools`] Addressed issue where configuration file cannot be located on fresh installations. + ## 0.1.19 - 2019-07-02 -* [`vtcmd`] Fixed issue with writing debug log files to directory before it is created +* [`vtcmd`] Fixed issue with writing debug log files to directory before it is created; fixes [GitHub Issue 12](https://github.com/NDAR/nda-tools/issues/12) * [`vtcmd`] Removed write mode when opening associated files for upload to S3 * [`vtcmd`] Update to only fetch multi-part credentials once during resume option in check_submitted_files() * [`downloadcmd`] Add message regarding transfer limit to download(5TB) diff --git a/NDATools/BuildPackage.py b/NDATools/BuildPackage.py index a85b806..3a2720b 100644 --- a/NDATools/BuildPackage.py +++ b/NDATools/BuildPackage.py @@ -11,6 +11,7 @@ from NDATools.Configuration import * from NDATools.Utils import * + class SubmissionPackage: def __init__(self, uuid, associated_files, config, username=None, password=None, collection=None, title=None, description=None, alternate_location=None, allow_exit=False): @@ -159,9 +160,9 @@ def check_read_permissions(self, file): try: open(file) return True - except IOError: - return False - except PermissionError: + except (OSError, IOError) as err: + if err.errno == 13: + print('Permission Denied: {}'.format(file)) return False def file_search(self, directories=None, source_bucket=None, source_prefix=None, access_key=None, secret_key=None, retry_allowed=False): diff --git a/NDATools/Configuration.py b/NDATools/Configuration.py index 9992f38..5a510f8 100644 --- a/NDATools/Configuration.py +++ b/NDATools/Configuration.py @@ -12,6 +12,7 @@ import os from pkg_resources import resource_filename + class ClientConfiguration: def __init__(self, settings_file, username=None, password=None, access_key=None, secret_key=None): self.config = configparser.ConfigParser() diff --git a/NDATools/Download.py b/NDATools/Download.py index 8639654..6bfcf8a 100644 --- a/NDATools/Download.py +++ b/NDATools/Download.py @@ -90,9 +90,9 @@ def __init__(self, directory, config=None, verbose=False): def get_protocol(cls): return cls.XML - def verbose_print(self, *args, **kwargs): + def verbose_print(self, *args): if self.verbose: - print(*args, **kwargs) + print(' '.join(list(args))) def useDataManager(self): """ Download package files (not associated files) """ @@ -175,10 +175,6 @@ def useDataStructure(self, data_structure): self.verbose_print( '{} not found. Please enter the correct path to your file and try again.'.format(self.dataStructure)) raise e - except FileNotFoundError: - self.verbose_print( - '{} not found. Please enter the correct path to your file and try again.'.format(self.dataStructure)) - raise FileNotFoundError def get_links(self, links, files, filters=None): @@ -253,7 +249,7 @@ def download_path(self, path, resume, prev_directory): try: s3transfer.download_file(bucket, key, local_filename) - self.verbose_print('downloaded: ', path) + self.verbose_print('downloaded: {}'.format(path)) except botocore.exceptions.ClientError as e: # If a client error is thrown, then check that it was a 404 error. diff --git a/NDATools/Submission.py b/NDATools/Submission.py index 4769af7..8a9d4e8 100644 --- a/NDATools/Submission.py +++ b/NDATools/Submission.py @@ -223,9 +223,9 @@ def check_read_permissions(self, file): try: open(file) return True - except IOError: - return False - except PermissionError: + except (OSError, IOError) as err: + if err.errno == 13: + print('Permission Denied: {}'.format(file)) return False def found_all_files(self, directories=None, source_bucket=None, source_prefix=None, access_key=None, secret_key=None, retry_allowed=False): diff --git a/NDATools/Utils.py b/NDATools/Utils.py index e29c584..48aaafa 100644 --- a/NDATools/Utils.py +++ b/NDATools/Utils.py @@ -15,11 +15,11 @@ from NDATools.Configuration import ClientConfiguration -config = ClientConfiguration(os.path.join(os.path.expanduser('~'), '.NDATools/settings.cfg')) -if config.validation_results: - validation_results_dir = os.path.join(os.path.expanduser('~'), config.validation_results) +if os.path.isfile(os.path.join(os.path.expanduser('~'), '.NDATools/settings.cfg')): + config = ClientConfiguration(os.path.join(os.path.expanduser('~'), '.NDATools/settings.cfg')) else: - validation_results_dir = os.path.join(os.path.expanduser('~'), "NDAValidationResults") + config = ClientConfiguration('clientscripts/config/settings.cfg') +validation_results_dir = os.path.join(os.path.expanduser('~'), config.validation_results) if not os.path.exists(validation_results_dir): os.mkdir(validation_results_dir) log_file = os.path.join(validation_results_dir, "debug_log_{}.txt").format(time.strftime("%Y%m%dT%H%M%S")) diff --git a/NDATools/Validation.py b/NDATools/Validation.py index 65fcccc..0132cc4 100644 --- a/NDATools/Validation.py +++ b/NDATools/Validation.py @@ -300,7 +300,6 @@ def __init__(self, _validation_result, hide): manifests.append(Validation.ValidationManifest(_manifest, hide)) self.manifests = manifests - class ValidationTask(threading.Thread, Protocol): def __init__(self, file_queue, result_queue, api, scope, responses, validation_progress, exit): threading.Thread.__init__(self) @@ -337,15 +336,6 @@ def run(self): error = " ".join(['FileNotFound:', message]) raise Exception(error) - except FileNotFoundError: - message = 'This file does not exist in current directory: {}'.format(file_name) - if self.progress_bar: - self.progress_bar.close() - if self.exit: - exit_client(signal=signal.SIGTERM, message=message) - else: - error = " ".join(['FileNotFound:', message]) - raise Exception(error) data = file.read() response, session = api_request(self, "POST", self.api_scope, data) diff --git a/NDATools/__init__.py b/NDATools/__init__.py index b9b0f32..1ecb905 100644 --- a/NDATools/__init__.py +++ b/NDATools/__init__.py @@ -3,7 +3,7 @@ import json import sys -__version__ = '0.1.19' +__version__ = '0.1.20' pypi_version = None version_checked = False @@ -12,7 +12,7 @@ def check_version(): from packaging.version import parse except ImportError: from pip._vendor.packaging.version import parse - + # use https://test.pypi.org/pypi/{package}/json on test/release branches, use https://pypi.org on master url_pattern = 'https://pypi.org/pypi/{package}/json' package = 'nda-tools' """Return version of package on pypi.python.org using json.""" diff --git a/NDATools/clientscripts/downloadcmd.py b/NDATools/clientscripts/downloadcmd.py index b1b83cb..d6d1af4 100644 --- a/NDATools/clientscripts/downloadcmd.py +++ b/NDATools/clientscripts/downloadcmd.py @@ -53,7 +53,6 @@ def parse_args(): return args - def configure(username, password): NDATools.Utils.logging.getLogger().setLevel(logging.INFO) @@ -109,12 +108,12 @@ def main(): s3Download.start_workers(resume, prev_directory, args.workerThreads) # download associated files from package - #if args.package: + # if args.package: # s3Download.searchForDataStructure(resume, prev_directory) - if verbose: print('Finished downloading all files.') + if __name__ == "__main__": main() \ No newline at end of file diff --git a/NDATools/clientscripts/vtcmd.py b/NDATools/clientscripts/vtcmd.py index 9133dd5..3232683 100644 --- a/NDATools/clientscripts/vtcmd.py +++ b/NDATools/clientscripts/vtcmd.py @@ -9,6 +9,7 @@ import os import pkg_resources + def parse_args(): parser = argparse.ArgumentParser( description='This application allows you to validate files and submit data into NDA. '