Skip to content

Commit

Permalink
sdks/python: update delete test case for GCS
Browse files Browse the repository at this point in the history
In this commit, we update the delete test to verify
recursive deletion of directories (prefixes) in GCS.

Changes include:
- Added test for deleting a GCS directory (prefix) with multiple files.
- Verified files under a directory are deleted recursively when using the delete method.
  • Loading branch information
mohamedawnallah committed Feb 2, 2025
1 parent 97192c4 commit 661f822
Showing 1 changed file with 28 additions and 1 deletion.
29 changes: 28 additions & 1 deletion sdks/python/apache_beam/io/gcp/gcsio_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,10 @@ def delete_blob(self, name, **kwargs):
if name in bucket.blobs:
del bucket.blobs[name]

def list_blobs(self, prefix=None, **kwargs):
bucket = self._get_canonical_bucket()
return self.client.list_blobs(bucket, prefix, **kwargs)


class FakeBlob(object):
def __init__(
Expand Down Expand Up @@ -445,20 +449,43 @@ def test_bad_file_modes(self):
self.gcs.open(file_name, 'r+b')

def test_delete(self):
# File path.
file_name = 'gs://gcsio-test/delete_me'
file_size = 1024
bucket_name, blob_name = gcsio.parse_gcs_path(file_name)

# Test deletion of non-existent file.
bucket = self.client.get_bucket(bucket_name)
self.gcs.delete(file_name)

# Insert a random file for testing.
self._insert_random_file(self.client, file_name, file_size)
self.assertTrue(blob_name in bucket.blobs)

# Deleting the file.
self.gcs.delete(file_name)

self.assertFalse(blob_name in bucket.blobs)

# Now test deleting a directory (prefix) with multiple files.
prefix = 'gs://gcsio-test/directory_to_delete/'
file_names = [f"{prefix}file1", f"{prefix}file2", f"{prefix}file3"]
blobs = [gcsio.parse_gcs_path(file_name) for file_name in file_names]

# Insert random files under the prefix.
for file_name in file_names:
self._insert_random_file(self.client, file_name, file_size)

# Verify the files exist before deletion
for blob in blobs:
self.assertTrue(blob[1] in bucket.blobs)

# Deleting the directory (all files under the prefix).
self.gcs.delete(prefix, recursive=True)

# Verify that the files are deleted.
for blob in blobs:
self.assertFalse(blob[1] in bucket.blobs)

def test_copy(self):
src_file_name = 'gs://gcsio-test/source'
dest_file_name = 'gs://gcsio-test/dest'
Expand Down

0 comments on commit 661f822

Please sign in to comment.