retry on gcloud crashed (OSError) (#100)

Capture and retry on `unexpected end of data` error

experiment/builder_runner.py

@@ -324,6 +324,12 @@ def get_coverage_local(
 
 class CloudBuilderRunner(BuilderRunner):
   """Cloud BuilderRunner."""
+  _RETRYABLE_ERRORS = [
+      # Temp workaround for issue #12.
+      'You do not currently have an active account selected',
+      # Workaround for issue #85.
+      'gcloud crashed (OSError): unexpected end of data'
+  ]
 
   def __init__(self, *args, experiment_name: str, experiment_bucket: str,
                **kwargs):
@@ -370,11 +376,11 @@ def build_and_run(self, generated_project: str, target_path: str,
                cwd=oss_fuzz_checkout.OSS_FUZZ_DIR)
         break
       except sp.CalledProcessError as e:
-        stdout = e.stdout.decode("utf-8")
-        stderr = e.stderr.decode("utf-8")
-        # Temp workaround for issue #12.
-        if ('You do not currently have an active account selected'
-            in stdout + stderr and attempt_id < CLOUD_EXP_MAX_ATTEMPT):
+        # Remove \n for single log entry on cloud.
+        stdout = e.stdout.decode('utf-8').replace('\n', '\t')
+        stderr = e.stderr.decode('utf-8').replace('\n', '\t')
+        if (any(error in stdout + stderr for error in self._RETRYABLE_ERRORS)
+            and attempt_id < CLOUD_EXP_MAX_ATTEMPT):
           delay = 5 * 2**attempt_id
           logging.warning(f'Failed to evaluate {os.path.realpath(target_path)} '
                           f'on cloud, attempt {attempt_id}:\n'