Squashed commit of the following

commit 9418e0e5 Author: alkohlanyrf <[email protected]> Date: Mon Dec 04 2023 16:00:31 GMT-0500 (Eastern Standard Time) Set Release version to 0.2.26 commit 1a285aa2 Author: alkohlanyrf <[email protected]> Date: Mon Dec 04 2023 15:31:02 GMT-0500 (Eastern Standard Time) Updated swagger links commit 47b15da5 Author: alkohlanyrf <[email protected]> Date: Mon Dec 04 2023 14:19:14 GMT-0500 (Eastern Standard Time) version bump commit 083d5402 Author: alkohlanyrf <[email protected]> Date: Fri Dec 01 2023 10:44:12 GMT-0500 (Eastern Standard Time) Added buildspec.yml commit 51a6b91 Author: gregmagdits <[email protected]> Date: Wed Nov 29 2023 13:50:09 GMT-0500 (Eastern Standard Time) fixed issues found during testing commit acecabb Author: gregmagdits <[email protected]> Date: Tue Nov 28 2023 16:57:04 GMT-0500 (Eastern Standard Time) version bump commit cce30c6 Author: gregmagdits <[email protected]> Date: Tue Nov 28 2023 16:53:55 GMT-0500 (Eastern Standard Time) updated code to initalize files in .NDATools dir the same way as the files in downloadcmd are initialized commit 4efce99 Author: gregmagdits <[email protected]> Date: Tue Nov 28 2023 15:08:13 GMT-0500 (Eastern Standard Time) changed option name commit 0855f35 Author: Reem Al-kohlany <[email protected]> Date: Mon Nov 27 2023 12:04:27 GMT-0500 (Eastern Standard Time) Squashed commit of the following commit c8850129 Author: alkohlanyrf <[email protected]> Date: Mon Nov 27 2023 11:30:39 GMT-0500 (Eastern Standard Time) Removed --password as parser option for vtcmd and downloadcmd commit 6fd6fe4 Author: gregmagdits <[email protected]> Date: Wed Nov 22 2023 18:07:06 GMT-0500 (Eastern Standard Time) add step to download metadata file if its not already present commit e9b2e54 Author: gregmagdits <[email protected]> Date: Wed Nov 22 2023 10:07:27 GMT-0500 (Eastern Standard Time) updated this to not require requests library when installing commit 49e10a1 Author: gregmagdits <[email protected]> Date: Wed Nov 22 2023 07:40:59 GMT-0500 (Eastern Standard Time) removed botocore as direct dependency commit ae371d6 Author: gregmagdits <[email protected]> Date: Wed Nov 22 2023 06:28:48 GMT-0500 (Eastern Standard Time) moved mock to extra-require commit 03a8c8d Author: gregmagdits <[email protected]> Date: Tue Nov 21 2023 14:30:49 GMT-0500 (Eastern Standard Time) change error logic to help with tests commit 01a7b08 Author: gregmagdits <[email protected]> Date: Thu Nov 16 2023 13:04:49 GMT-0500 (Eastern Standard Time) fixed regression affecting -d flag of downloadcmd. moved package-metadata to metadata directory to avoid potentially downloading it multiple times commit 2f583ed Author: gregmagdits <[email protected]> Date: Thu Nov 16 2023 11:55:35 GMT-0500 (Eastern Standard Time) removed sleeps and bumped version commit bc4fbd7 Author: alkohlanyrf <[email protected]> Date: Tue Nov 14 2023 14:10:13 GMT-0500 (Eastern Standard Time) Fixed validation warning incorrect path output commit 621cb2b Author: alkohlanyrf <[email protected]> Date: Wed Nov 01 2023 17:53:18 GMT-0400 (Eastern Daylight Time) Bugfix commit 96a5db3 Author: Reem Al-kohlany <[email protected]> Date: Tue Oct 31 2023 11:13:51 GMT-0400 (Eastern Daylight Time) REV-549 Bugfix the package-metadata file unique name commit fbf0075 Author: greg <[email protected]> Date: Fri Oct 27 2023 09:22:15 GMT-0400 (Eastern Daylight Time) Squashed commit of the following commit 74023065 Author: gregmagdits <[email protected]> Date: Fri Oct 20 2023 13:12:44 GMT-0400 (Eastern Daylight Time) udpated name of metadata-file commit dfc81b3 Author: Reem Al-kohlany <[email protected]> Date: Thu Oct 26 2023 12:23:28 GMT-0400 (Eastern Daylight Time) Bugfix settings.cfg file location output Updated -t flag info commit a94abf2 Author: Reem Al-kohlany <[email protected]> Date: Wed Oct 25 2023 10:57:36 GMT-0400 (Eastern Daylight Time) REV-550 Bugfix DownloadRequest() NDA Tools Python - add command line arg to enable debug logging commit 6b1893f Author: Reem Al-kohlany <[email protected]> Date: Fri Oct 20 2023 14:51:37 GMT-0400 (Eastern Daylight Time) updated logic to handle expired credentials when downloading locally 52_expired_credentials_lead_to_'This_is_a_private_bucket'_errors Retrying cred requests a limited number of times when encountering a 403 error commit db0f6cd Author: gregmagdits <[email protected]> Date: Fri Oct 20 2023 12:18:53 GMT-0400 (Eastern Daylight Time) removed documentation around -qa command line arg commit eb4b87b Author: Reem Al-kohlany <[email protected]> Date: Fri Oct 20 2023 11:49:28 GMT-0400 (Eastern Daylight Time) REV-277 fix regresssion_in_s3_transfer_functionality
NDAR · Dec 4, 2023 · e4069d4 · e4069d4
1 parent 9246af7
commit e4069d4
Show file tree

Hide file tree

Showing 15 changed files with 449 additions and 509 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,12 @@
 
 # CHANGELOG
+## 0.2.26 - 2023-12-04
+* [`nda-tools`] Added --verbose parser option. Enables debug logging.
+* [`nda-tools`] Removed obsolete --password and -qa as parser option.
+* [`downloadcmd`] Updated the package-metadata file naming conventions to be unique per package to avoid name collisions.
+* [`downloadcmd`] Updated the logic to regenerate credentials if they expire.
+* [`downloadcmd`] Fixed regression in s3 bucket download functionality.
+
 ## 0.2.23 - 2023-1-23
 * [`vtcmd`] updated logic to allow alt-endpoints to contain sub-folders
 * [`downloadcmd`] Added validation for -dp arg

diff --git a/NDATools/BuildPackage.py b/NDATools/BuildPackage.py
@@ -5,12 +5,9 @@
 import botocore
 import requests.packages.urllib3.util
 
-import NDATools
-
 if sys.version_info[0] < 3:
     input = raw_input
 import requests.packages.urllib3.util
-from tqdm import tqdm
 from NDATools.Configuration import *
 from NDATools.Utils import *
 
@@ -112,8 +109,7 @@ def set_upload_destination(self, hide_input=True):
                         else:
                             message = 'Incorrect/Missing collection ID or alternate endpoint.'
                             if self.exit:
-                                exit_client(signal=signal.SIGTERM,
-                                            message=message)
+                                exit_error(message=message)
                             else:
                                 raise Exception(message)
                 except (AttributeError, ValueError, TypeError) as e:
@@ -124,13 +120,13 @@ def set_upload_destination(self, hide_input=True):
                     else:
                         message = 'Incorrect/Missing collection ID or alternate endpoint.'
                         if self.exit:
-                            exit_client(signal=signal.SIGTERM, message=message)
+                            exit_error(message=message)
                         else:
                             raise Exception(message)
         else:
             message = 'The user {} does not have permission to submit to any collections or alternate upload locations.'.format(self.config.username)
             if self.exit:
-                exit_client(signal=signal.SIGTERM, message=message)
+                exit_error(message=message)
             else:
                 raise Exception(message)
 
@@ -204,7 +200,7 @@ def raise_error(error, l = []):
                     logger.info('\n%s', message)
                     for b in set(no_access_buckets):
                         logger.info(b)
-                    exit_client()
+                    exit_error()
                 else:
                     error = "".join(['Bucket Access:', message])
                     raise_error(error, no_access_buckets)
@@ -271,7 +267,7 @@ def raise_error(value):
             if not self.config.force:
                 user_input = evaluate_yes_no_input("Are you sure you want to continue?", 'n')
                 if user_input.lower() == 'n':
-                    exit_client(signal.SIGTERM, message='Exiting...')
+                    exit_error(message='Exiting...')
 
 
         json_data = json.dumps(self.package_info)
@@ -289,16 +285,13 @@ def raise_error(value):
                 if response['status'] == Status.ERROR:
                     message = response['errors'][0]['message']
                 if self.exit:
-                    exit_client(signal.SIGTERM, message=message)
+                    exit_error(message=message)
                 else:
                     raise Exception(message)
 
-            polling = 0
             while response['package_info']['status'] == Status.PROCESSING:
                 time.sleep(1.1)
                 response = get_request("/".join([self.api, self.package_id]), auth=self.auth)
-                polling += 1
-                self.package_id = response['submission_package_uuid']
             if response['package_info']['status'] == Status.COMPLETE:
                 for f in [f for f in response['files']
                           if f['type'] in ('Submission Memento', 'Submission Data Package')]:
@@ -312,54 +305,16 @@ def raise_error(value):
                 elif 'has changed since validation' in response['errors']:
                     message = response['errors']
                 if self.exit:
-                    exit_client(signal.SIGTERM, message=message)
+                    exit_error(message=message)
                 else:
                     raise Exception(message)
         else:
             message='There was an error with your package request.'
             if self.exit:
-                exit_client(signal.SIGTERM, message=message)
+                exit_error(message=message)
             else:
                 raise Exception(message)
 
-    def download_package(self, hide_progress):
-        folder = self.download_links[0][1]
-        folder = folder.split('/')
-        self.package_folder = folder[0]
-        session = requests.session()
-        total_package_size = 0
-        for i, (url, file_name) in enumerate(self.download_links):
-            r = session.get(url, auth=(self.username, self.password), stream=True)
-            size = r.headers['content-length']
-            total_package_size += int(size)
-            self.download_links[i] = (url, file_name, int(size))
-            r.close()
-        if not hide_progress:
-            package_download = tqdm(total=total_package_size,
-                                    unit="bytes",
-                                    unit_scale=True,
-                                    desc="Submission Package Download",
-                                    ascii=os.name == 'nt')
-        # logger.info('dl_links: {}'.format(self.download_links))
-        for url, file_name, size in self.download_links:
-            path = os.path.join(NDATools.NDA_TOOLS_SUB_PACKAGE_FOLDER, self.package_folder)
-            if not os.path.exists(path):
-                os.mkdir(path)
-            file_name = file_name.split('/')
-            file_name = file_name[1]
-            file = os.path.join(path, file_name)
-            r = session.get(url, auth=(self.username, self.password), stream=True)
-            with open(file, 'wb') as out_file:
-                for chunk in r.iter_content(chunk_size=1024 * 1024):
-                    if chunk:
-                        out_file.write(chunk)
-                        if not hide_progress:
-                            package_download.update(sys.getsizeof(chunk))
-            session.close()
-        if not hide_progress:
-            if package_download.total > package_download.n:
-                package_download.update(package_download.total - package_download.n)
-            package_download.close()
 
     def print_replacement_summary(self):
         logger.info('Below is a summary of what your submission will look like with the validation files provided:')

diff --git a/NDATools/Configuration.py b/NDATools/Configuration.py
@@ -5,7 +5,6 @@
 import logging
 import logging.config
 import platform
-import shutil
 import sys
 import time
 
@@ -14,8 +13,9 @@
 import yaml
 from requests import HTTPError
 
+import NDATools
 from NDATools import NDA_TOOLS_LOGGING_YML_FILE, Utils
-from NDATools.Utils import exit_client, HttpErrorHandlingStrategy
+from NDATools.Utils import exit_error, HttpErrorHandlingStrategy
 
 if sys.version_info[0] < 3:
     import ConfigParser as configparser
@@ -34,39 +34,26 @@ def __init__(self):
         pass
 
     @staticmethod
-    def load_config(logs_directory):
-        def make_config():
-            file_path = os.path.join(os.path.expanduser('~'), '.NDATools')
-            if not os.path.exists(file_path):
-                os.makedirs(file_path)
-
-            if not os.path.exists(NDA_TOOLS_LOGGING_YML_FILE):
-                shutil.copyfile(resource_filename(__name__, 'clientscripts/config/logging.yml'),
-                                NDA_TOOLS_LOGGING_YML_FILE)
-
-        if not os.path.exists(NDA_TOOLS_LOGGING_YML_FILE):
-            make_config()
+    def load_config(logs_directory, verbose=False):
 
         with open(NDA_TOOLS_LOGGING_YML_FILE, 'r') as stream:
             config = yaml.load(stream, Loader=yaml.FullLoader)
         log_file = os.path.join(logs_directory, "debug_log_{}.txt").format(time.strftime("%Y%m%dT%H%M%S"))
         config['handlers']['file']['filename']=log_file
+        if verbose:
+            config['loggers']['NDATools']['level']='DEBUG'
+            config['handlers']['console']['formatter']='detailed'
         logging.config.dictConfig(config)
 
 class ClientConfiguration:
 
     SERVICE_NAME = 'nda-tools'
 
-    def __init__(self, settings_file, username=None, access_key=None, secret_key=None):
+    def __init__(self, username=None, access_key=None, secret_key=None):
         self.config = configparser.ConfigParser()
-        if settings_file == os.path.join(os.path.expanduser('~'), '.NDATools/settings.cfg'):
-            self.config_location = settings_file
-            self._check_and_fix_missing_options(self.config_location)
-        else:
-            self.config_location = resource_filename(__name__, settings_file)
-        logger.info('Using configuration file from {}'.format(self.config_location))
-
-        self.config.read(self.config_location)
+        logger.info('Using configuration file from {}'.format(NDATools.NDA_TOOLS_SETTINGS_CFG_FILE))
+        user_settings = self.config.read(NDATools.NDA_TOOLS_SETTINGS_CFG_FILE)
+        self._check_and_fix_missing_options()
         self.validation_api = self.config.get("Endpoints", "validation")
         self.submission_package_api = self.config.get("Endpoints", "submission_package")
         self.submission_api = self.config.get("Endpoints", "submission")
@@ -80,13 +67,11 @@ def __init__(self, settings_file, username=None, access_key=None, secret_key=Non
         self.aws_session_token = self.config.get('User', 'session_token')
         self.username = self.config.get("User", "username")
 
-        self.check_deprecated_settings()
-
         if username:
             self.username = username
         elif self.username:
             logger.warning("-u/--username argument not provided. Using default value of '%s' which was saved in %s",
-                           self.username, os.path.join(os.path.expanduser('~'), '.NDATools' + os.sep + 'settings.cfg'))
+                           self.username, NDATools.NDA_TOOLS_SETTINGS_CFG_FILE)
 
         if access_key:
             self.aws_access_key = access_key
@@ -106,48 +91,27 @@ def __init__(self, settings_file, username=None, access_key=None, secret_key=Non
         self.skip_local_file_check = False
         logger.info('proceeding as nda user: {}'.format(self.username))
 
-    def _check_and_fix_missing_options(self, config_location):
+    def _check_and_fix_missing_options(self):
         default_config = configparser.ConfigParser()
         default_file_path = resource_filename(__name__, 'clientscripts/config/settings.cfg')
         default_config.read(default_file_path)
-        default_settings = dict(default_config._sections)
-        self.config.read(config_location)
-        user_settings = dict(self.config._sections)
-
-        for section in default_settings:
-            for option in default_settings[section]:
-                if option not in user_settings[section]:
+        change_detected = False
+        for section in default_config.sections():
+            if not section in self.config.sections():
+                logger.debug(f'adding {section} to settings.cfg')
+                self.config.add_section(section)
+                change_detected = True
+            for option in default_config[section]:
+                if option not in self.config[section]:
                     logger.debug('[{}][{}] is missing'.format(section, option))
-                    with open(config_location, 'w') as configfile:
-                        self.config.set(section, option, default_settings[section][option])
-                        self.config.write(configfile)
-
-    def make_config(self):
-        file_path = os.path.join(os.path.expanduser('~'), '.NDATools')
-        if not os.path.exists(file_path):
-            os.makedirs(file_path)
-        config_path = os.path.join(os.path.expanduser('~'), '.NDATools/settings.cfg')
-
-        copy_config = configparser.ConfigParser()
-
-        copy_config.add_section("Endpoints")
-        copy_config.set("Endpoints", "user", self.user_api)
-        copy_config.set("Endpoints", "package", self.package_api)
-        copy_config.set("Endpoints", "validation", self.validation_api)
-        copy_config.set("Endpoints", "submission_package", self.submission_package_api)
-        copy_config.set("Endpoints", "submission", self.submission_api)
-        copy_config.set("Endpoints", "validationtool", self.validationtool_api)
-        copy_config.set("Endpoints", "datadictionary", self.datadictionary_api)
-        copy_config.set("Endpoints", "package_creation", self.package_creation_api)
-
-        copy_config.add_section("User")
-        copy_config.set("User", "username", self.username)
-        copy_config.set("User", "access_key", self.aws_access_key)
-        copy_config.set("User", "secret_key", self.aws_secret_key)
-        copy_config.set("User", "session_token", self.aws_session_token)
-
-        with open(config_path, 'w') as configfile:
-            copy_config.write(configfile)
+                    self.config.set(section, option, default_config[section][option])
+                    change_detected = True
+        if change_detected:
+            logger.debug(f'updating settings.cfg')
+            with open(NDATools.NDA_TOOLS_SETTINGS_CFG_FILE, 'w') as configfile:
+                self.config.write(configfile)
+        else:
+            logger.debug(f'settings.cfg is up to date')
 
     def read_user_credentials(self, auth_req=True):
 
@@ -157,7 +121,7 @@ def read_user_credentials(self, auth_req=True):
                 try:
                     while not self.username:
                         self.username = input('Enter your NIMH Data Archives username:')
-                        with open(self.config_location, 'w') as configfile:
+                        with open(NDATools.NDA_TOOLS_SETTINGS_CFG_FILE, 'w') as configfile:
                             self.config.set('User', 'username', self.username)
                             self.config.write(configfile)
                     self.password = keyring.get_password(self.SERVICE_NAME, self.username)
@@ -166,7 +130,7 @@ def read_user_credentials(self, auth_req=True):
                     while not self.is_valid_nda_credentials():
                         logger.error("The password that was entered for user '%s' is invalid ...", self.username)
                         logger.error('If your username was previously entered incorrectly, you may update it in your '
-                                     'settings.cfg located at \n{}'.format(self.config_location))
+                                     'settings.cfg located at \n{}'.format(NDATools.NDA_TOOLS_SETTINGS_CFG_FILE))
                         self.password = getpass.getpass('Enter your NIMH Data Archives password:')
                         while self.password == '':
                             self.password = getpass.getpass('Enter your NIMH Data Archives password:')
@@ -191,11 +155,6 @@ def read_aws_credentials(self):
         if not self.aws_secret_key:
             self.aws_secret_key = getpass.getpass('Enter your aws_secret_key:')
 
-    def check_deprecated_settings(self):
-        if self.config.has_option('User', 'password') and self.config.get("User", "password"):
-            print('Warning: Detected non-empty value for "password" in settings.cfg. Support for this setting has been '
-                  'deprecated and will no longer be used by this tool. Password storage is not recommended for security'
-                  ' considerations')
 
     def is_valid_nda_credentials(self):
         try:
@@ -211,10 +170,10 @@ def is_valid_nda_credentials(self):
                     tmp = json.loads(e.response.text)
                     logger.error('\nError: %s', tmp['message'])
                     logger.error('\nPlease contact [email protected] for help in resolving this error')
-                    exit_client()
+                    exit_error()
                 else:
                     return False
             else:
                 logger.error('\nSystemError while checking credentials for user %s', self.username)
                 logger.error('\nPlease contact [email protected] for help in resolving this error')
-                exit_client()
+                exit_error()