Merge pull request #31 from toppk/common_prefix

support common_prefixes response object parsing.

README.md

@@ -61,10 +61,10 @@ async with ClientSession(raise_for_status=True) as session:
         assert resp == HTTPStatus.NO_CONTENT
 
     # List objects by prefix
-    async for result in client.list_objects_v2("bucket/", prefix="prefix"):
+    async for result, prefixes in client.list_objects_v2("bucket/", prefix="prefix"):
         # Each result is a list of metadata objects representing an object
-        # stored in the bucket.
-        do_work(result)
+        # stored in the bucket.  Each prefixes is a list of common prefixes
+        do_work(result, prefixes)
 ```
 
 Bucket may be specified as subdomain or in object name:

aiohttp_s3_client/client.py

@@ -739,7 +739,7 @@ async def list_objects_v2(
         delimiter: t.Optional[str] = None,
         max_keys: t.Optional[int] = None,
         start_after: t.Optional[str] = None,
-    ) -> t.AsyncIterator[t.List[AwsObjectMeta]]:
+    ) -> t.AsyncIterator[t.Tuple[t.List[AwsObjectMeta], t.List[str]]]:
         """
         List objects in bucket.
 
@@ -787,10 +787,10 @@ async def list_objects_v2(
                         ),
                     )
                 payload = await resp.read()
-                metadata, continuation_token = parse_list_objects(payload)
-                if not metadata:
+                metadata, prefixes, cont_token = parse_list_objects(payload)
+                if not metadata and not prefixes:
                     break
-                yield metadata
-                if not continuation_token:
+                yield metadata, prefixes
+                if not cont_token:
                     break
-                params["continuation-token"] = continuation_token
+                params["continuation-token"] = cont_token

aiohttp_s3_client/xml.py

@@ -43,10 +43,15 @@ def create_complete_upload_request(parts: List[Tuple[int, str]]) -> bytes:
 
 
 def parse_list_objects(payload: bytes) -> Tuple[
-    List[AwsObjectMeta], Optional[str],
+    List[AwsObjectMeta], List[str], Optional[str],
 ]:
     root = ET.fromstring(payload)
     result = []
+    prefixes = [
+        el.text
+        for el in root.findall(f"{{{NS}}}CommonPrefixes/{{{NS}}}Prefix")
+        if el.text
+    ]
     for el in root.findall(f"{{{NS}}}Contents"):
         etag = key = last_modified = size = storage_class = None
         for child in el:
@@ -78,4 +83,4 @@ def parse_list_objects(payload: bytes) -> Tuple[
             result.append(meta)
     nct_el = root.find(f"{{{NS}}}NextContinuationToken")
     continuation_token = nct_el.text if nct_el is not None else None
-    return result, continuation_token
+    return result, prefixes, continuation_token

tests/test_simple.py

@@ -76,7 +76,7 @@ async def test_list_objects_v2(s3_client: S3Client, s3_read, tmp_path):
 
         # Test list file
         batch = 0
-        async for result in s3_client.list_objects_v2(
+        async for result, prefixes in s3_client.list_objects_v2(
             prefix="test/list/",
             delimiter="/",
             max_keys=1,
@@ -86,6 +86,32 @@ async def test_list_objects_v2(s3_client: S3Client, s3_read, tmp_path):
             assert result[0].size == len(data)
 
 
+async def test_list_objects_v2_prefix(s3_client: S3Client, s3_read, tmp_path):
+    data = b"hello, world"
+
+    with (tmp_path / "hello.txt").open("wb") as f:
+        f.write(data)
+        f.flush()
+
+        resp = await s3_client.put_file("/test2/list1/test1", f.name)
+        assert resp.status == HTTPStatus.OK
+
+        resp = await s3_client.put_file("/test2/list2/test2", f.name)
+        assert resp.status == HTTPStatus.OK
+
+        # Test list file
+        batch = 0
+
+        async for result, prefixes in s3_client.list_objects_v2(
+            prefix="test2/",
+            delimiter="/",
+        ):
+            batch += 1
+            assert len(result) == 0
+            assert prefixes[0] == "test2/list1/"
+            assert prefixes[1] == "test2/list2/"
+
+
 async def test_url_path_with_colon(s3_client: S3Client, s3_read):
     data = b"hello, world"
     key = "/some-path:with-colon.txt"