Merge pull request #1274 from GSA/the_great_debugging

fix job retrieval

app/aws/s3.py

@@ -173,7 +173,7 @@ def get_job_and_metadata_from_s3(service_id, job_id):
 
 def get_job_from_s3(service_id, job_id):
     retries = 0
-    max_retries = 5
+    max_retries = 3
     backoff_factor = 1
     while retries < max_retries:
 
@@ -190,11 +190,20 @@ def get_job_from_s3(service_id, job_id):
                 sleep_time = backoff_factor * (2**retries)  # Exponential backoff
                 time.sleep(sleep_time)
                 continue
-        except Exception as e:
-            current_app.logger.error(f"Failed to get object from bucket {e}")
-            raise
+            else:
+                current_app.logger.error(
+                    f"Failed to get job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)} from bucket",
+                    exc_info=True,
+                )
+                return None
+        except Exception:
+            current_app.logger.error(
+                f"Failed to get job {FILE_LOCATION_STRUCTURE.format(service_id, job_id)} from bucket",
+                exc_info=True,
+            )
+            return None
 
-    raise Exception("Failed to get object after 5 attempts")
+    raise Exception("Failed to get object after 3 attempts")
 
 
 def incr_jobs_cache_misses():

tests/app/aws/test_s3.py

@@ -102,7 +102,7 @@ def test_get_job_from_s3_exponential_backoff(mocker):
     mocker.patch("app.aws.s3.get_s3_object", side_effect=mock_s3_get_object_slowdown)
     with pytest.raises(Exception) as exc_info:
         get_job_from_s3("service_id", "job_id")
-    assert "Failed to get object after 5 attempts" in str(exc_info)
+    assert "Failed to get object after 3 attempts" in str(exc_info)
 
 
 @pytest.mark.parametrize(