From f84afeeefb5ac27ce4b76c0f749be485c8f49dec Mon Sep 17 00:00:00 2001
From: Michael Hall <mlh0079@uah.edu>
Date: Wed, 31 Jul 2024 10:12:52 -0500
Subject: [PATCH]  - Verifying the output is now configurable.

---
 README.md                  | 12 ++++++++++--
 dmrpp_generator/main.py    | 35 +++++++++++++----------------------
 dmrpp_generator/version.py |  2 +-
 3 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/README.md b/README.md
index fa97beb..7a0c0ac 100644
--- a/README.md
+++ b/README.md
@@ -213,10 +213,18 @@ variable.
 When making the subprocess call, stdout and stderr will default to `None` to prevent an issue from occurring where the 
 timeout is not respected. This can be configured in two ways.
 1. Setting the `ENABLE_SUBPROCESS_LOGGING` environment variable in terraform
-2. Adding `enable_subprocess_logging` to the collection definition: `collection.meta.dmrpp`. Can be `true` or `false`.
+2. Adding `enable_subprocess_logging` to the collection definition: `collection.meta.dmrpp`.  
 
 If the value is provided in the collection definition this will take precedence over the environment
-variable.
+variable. Can be `true` or `false`.
+
+## Verify Output Configuration
+The processing code will verify that dmrpp outputs are produced and if not an exception will be raised. This behavior can be disabled if needed. This can be configured in two ways.
+1. Setting the `VERIFY_OUTPUT` environment variable in terraform
+2. Adding `verify_output` to the collection definition: `collection.meta.dmrpp`.  
+
+If the value is provided in the collection definition this will take precedence over the environment
+variable. Can be `true` or `false`.
 
 # DMR++ Python CLI
 # How to install
diff --git a/dmrpp_generator/main.py b/dmrpp_generator/main.py
index 91fd4fa..bcbcb1b 100644
--- a/dmrpp_generator/main.py
+++ b/dmrpp_generator/main.py
@@ -40,7 +40,7 @@ def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.path = self.path.rstrip('/') + "/"
         # Enable logging the default is True
-        enable_logging = os.getenv('ENABLE_CW_LOGGING', 'True') in [True, "true", "t", 1]
+        enable_logging = (os.getenv('ENABLE_CW_LOGGING', 'true').lower() == 'true')
         self.dmrpp_version = f"DMRPP {__version__}"
         self.logger_to_cw = LOGGER_TO_CW if enable_logging else logging
         self.logger_to_cw.info(f'config: {self.config}')
@@ -48,12 +48,15 @@ def __init__(self, **kwargs):
             'get_dmrpp_timeout', os.getenv('GET_DMRPP_TIMEOUT', '600'))
         )
         self.enable_subprocess_logging = self.dmrpp_meta.get(
-            'enable_subprocess_logging', os.getenv('ENABLE_SUBPROCESS_LOGGING', False)
+            'enable_subprocess_logging', os.getenv('ENABLE_SUBPROCESS_LOGGING', 'false').lower() == 'true'
+        )
+        self.verify_output = self.dmrpp_meta.get(
+            'verify_output', os.getenv('VERIFY_OUTPUT', 'true').lower() == 'true'
         )
-
         self.logger_to_cw.info(f'get_dmrpp_timeout: {self.timeout}')
         self.logger_to_cw.info(f'enagled_cw_logging: {enable_logging}')
         self.logger_to_cw.info(f'enable_subprocess_logging: {self.enable_subprocess_logging}')
+        self.logger_to_cw.info(f'verify_output: {self.verify_output}')
 
     @property
     def input_keys(self):
@@ -129,14 +132,16 @@ def process(self):
                         dmrpp_files.append(dmrpp_file)
                         self.upload_file_to_s3(output_file_path, f's3://{dmrpp_file["bucket"]}/{dmrpp_file["key"]}')
 
-            if dmrpp_files == 0:
-                raise Exception(f'No dmrpp files were produced for {granule}')
-
+            if len(dmrpp_files) == 0:
+                message = f'No dmrpp files were produced for {granule}'
+                if self.verify_output:
+                    raise Exception(message)
+                else:
+                    self.logger_to_cw.warning(message)
+                
             self.strip_old_dmrpp_files(granule)
             granule['files'] += dmrpp_files
 
-        self.verify_outputs_produced(granules)
-
         return self.input
 
     @staticmethod
@@ -150,20 +155,6 @@ def strip_old_dmrpp_files(granule):
             else:
                 i += 1
 
-    @staticmethod
-    def verify_outputs_produced(granules):
-        has_output = False
-        for granule in granules:
-            for file in granule['files']:
-                if str(file.get('fileName')).endswith('dmrpp'):
-                    has_output = True
-                    break
-            if has_output:
-                break
-
-        if not has_output:
-            raise Exception('No dmrpp outputs produced.')
-
     def get_dmrpp_command(self, dmrpp_meta, input_path, output_filename, local=False):
         """
         Getting the command line to create DMRPP files
diff --git a/dmrpp_generator/version.py b/dmrpp_generator/version.py
index e311457..fe781cc 100644
--- a/dmrpp_generator/version.py
+++ b/dmrpp_generator/version.py
@@ -1 +1 @@
-__version__ = "v5.1.0"
+__version__ = "v5.1.1"