Retry streaming storage reads with callback method.

app/lib/package/backend.dart

@@ -902,8 +902,8 @@ class PackageBackend {
         throw PackageRejectedException.archiveTooLarge(
             UploadSignerService.maxUploadSize);
       }
-      await _saveTarballToFS(
-          _incomingBucket.read(tmpObjectName(guid)), filename);
+      await _incomingBucket.readWithRetry(
+          tmpObjectName(guid), (input) => _saveTarballToFS(input, filename));
       _logger.info('Examining tarball content ($guid).');
       final sw = Stopwatch()..start();
       final file = File(filename);
@@ -1756,6 +1756,11 @@ class InviteStatus {
 Future _saveTarballToFS(Stream<List<int>> data, String filename) async {
   final sw = Stopwatch()..start();
   final targetFile = File(filename);
+
+  // cleanup the leftover if previous attempt failed
+  if (await targetFile.exists()) {
+    await targetFile.delete();
+  }
   try {
     int receivedBytes = 0;
     final stream = data.transform<List<int>>(

app/lib/package/tarball_storage.dart

@@ -120,18 +120,20 @@ class TarballStorage {
     final raf = await file.open();
     var remainingLength = info.length;
     try {
-      await for (final chunk in _canonicalBucket.read(objectName)) {
-        if (chunk.isEmpty) continue;
-        remainingLength -= chunk.length;
-        if (remainingLength < 0) {
-          return ContentMatchStatus.different;
+      await _canonicalBucket.readWithRetry(objectName, (input) async {
+        await for (final chunk in input) {
+          if (chunk.isEmpty) continue;
+          remainingLength -= chunk.length;
+          if (remainingLength < 0) {
+            return ContentMatchStatus.different;
+          }
+          // TODO: consider rewriting to fixed-length chunk comparison
+          final fileChunk = await raf.read(chunk.length);
+          if (!fileChunk.byteToByteEquals(chunk)) {
+            return ContentMatchStatus.different;
+          }
         }
-        // TODO: consider rewriting to fixed-length chunk comparison
-        final fileChunk = await raf.read(chunk.length);
-        if (!fileChunk.byteToByteEquals(chunk)) {
-          return ContentMatchStatus.different;
-        }
-      }
+      });
     } finally {
       await raf.close();
     }

app/lib/service/download_counts/backend.dart

@@ -50,10 +50,13 @@ class DownloadCountsBackend {
       }
       final data = (await storageService
               .bucket(activeConfiguration.reportsBucketName!)
-              .read(downloadCounts30DaysTotalsFileName)
-              .transform(utf8.decoder)
-              .transform(json.decoder)
-              .single as Map<String, dynamic>)
+              .readWithRetry(
+                downloadCounts30DaysTotalsFileName,
+                (input) async => await input
+                    .transform(utf8.decoder)
+                    .transform(json.decoder)
+                    .single as Map<String, dynamic>,
+              ))
           .cast<String, int>();
       _lastData = (data: data, etag: info.etag);
       return data;

app/lib/shared/storage.dart

@@ -180,6 +180,14 @@ extension BucketExt on Bucket {
     );
   }
 
+  /// Read object content as byte stream using the callback function to receive data chunks.
+  ///
+  /// When network error occurs, the entire stream is restarted and [fn] is called again.
+  Future<T> readWithRetry<T>(
+      String objectName, Future<T> Function(Stream<List<int>> input) fn) async {
+    return await _retry(() async => fn(read(objectName)));
+  }
+
   /// The HTTP URL of a publicly accessable GCS object.
   String objectUrl(String objectName) {
     return '${activeConfiguration.storageBaseUrl}/$bucketName/$objectName';
@@ -399,12 +407,14 @@ class VersionedJsonStorage {
     }
     final objectName = _objectName(version);
     _logger.info('Loading snapshot: $objectName');
-    final map = await _bucket
-        .read(objectName)
-        .transform(_gzip.decoder)
-        .transform(utf8.decoder)
-        .transform(json.decoder)
-        .single;
+    final map = await _bucket.readWithRetry(
+      objectName,
+      (input) => input
+          .transform(_gzip.decoder)
+          .transform(utf8.decoder)
+          .transform(json.decoder)
+          .single,
+    );
     return map as Map<String, dynamic>;
   }
 

pkg/fake_gcloud/lib/retry_enforcer_storage.dart

@@ -164,7 +164,7 @@ class _RetryEnforcerBucket implements Bucket {
 
   @override
   Stream<List<int>> read(String objectName, {int? offset, int? length}) {
-    // TODO: verify retry wrapper here
+    _verifyRetryOnStack();
     return _bucket.read(objectName, offset: offset, length: length);
   }