diff --git a/README.md b/README.md index 92a537e..85ef3ec 100644 --- a/README.md +++ b/README.md @@ -61,10 +61,10 @@ async with ClientSession(raise_for_status=True) as session: assert resp == HTTPStatus.NO_CONTENT # List objects by prefix - async for result in client.list_objects_v2("bucket/", prefix="prefix"): + async for result, prefixes in client.list_objects_v2("bucket/", prefix="prefix"): # Each result is a list of metadata objects representing an object - # stored in the bucket. - do_work(result) + # stored in the bucket. Each prefixes is a list of common prefixes + do_work(result, prefixes) ``` Bucket may be specified as subdomain or in object name: diff --git a/aiohttp_s3_client/client.py b/aiohttp_s3_client/client.py index 6580991..d0b3fe0 100644 --- a/aiohttp_s3_client/client.py +++ b/aiohttp_s3_client/client.py @@ -739,7 +739,7 @@ async def list_objects_v2( delimiter: t.Optional[str] = None, max_keys: t.Optional[int] = None, start_after: t.Optional[str] = None, - ) -> t.AsyncIterator[t.List[AwsObjectMeta]]: + ) -> t.AsyncIterator[t.Tuple[t.List[AwsObjectMeta], t.List[str]]]: """ List objects in bucket. @@ -787,10 +787,10 @@ async def list_objects_v2( ), ) payload = await resp.read() - metadata, continuation_token = parse_list_objects(payload) - if not metadata: + metadata, prefixes, cont_token = parse_list_objects(payload) + if not metadata and not prefixes: break - yield metadata - if not continuation_token: + yield metadata, prefixes + if not cont_token: break - params["continuation-token"] = continuation_token + params["continuation-token"] = cont_token diff --git a/aiohttp_s3_client/xml.py b/aiohttp_s3_client/xml.py index 7a485fa..5fe73b5 100644 --- a/aiohttp_s3_client/xml.py +++ b/aiohttp_s3_client/xml.py @@ -43,10 +43,15 @@ def create_complete_upload_request(parts: List[Tuple[int, str]]) -> bytes: def parse_list_objects(payload: bytes) -> Tuple[ - List[AwsObjectMeta], Optional[str], + List[AwsObjectMeta], List[str], Optional[str], ]: root = ET.fromstring(payload) result = [] + prefixes = [ + el.text + for el in root.findall(f"{{{NS}}}CommonPrefixes/{{{NS}}}Prefix") + if el.text + ] for el in root.findall(f"{{{NS}}}Contents"): etag = key = last_modified = size = storage_class = None for child in el: @@ -78,4 +83,4 @@ def parse_list_objects(payload: bytes) -> Tuple[ result.append(meta) nct_el = root.find(f"{{{NS}}}NextContinuationToken") continuation_token = nct_el.text if nct_el is not None else None - return result, continuation_token + return result, prefixes, continuation_token diff --git a/tests/test_simple.py b/tests/test_simple.py index cfffd48..dbc6d6c 100644 --- a/tests/test_simple.py +++ b/tests/test_simple.py @@ -76,7 +76,7 @@ async def test_list_objects_v2(s3_client: S3Client, s3_read, tmp_path): # Test list file batch = 0 - async for result in s3_client.list_objects_v2( + async for result, prefixes in s3_client.list_objects_v2( prefix="test/list/", delimiter="/", max_keys=1, @@ -86,6 +86,32 @@ async def test_list_objects_v2(s3_client: S3Client, s3_read, tmp_path): assert result[0].size == len(data) +async def test_list_objects_v2_prefix(s3_client: S3Client, s3_read, tmp_path): + data = b"hello, world" + + with (tmp_path / "hello.txt").open("wb") as f: + f.write(data) + f.flush() + + resp = await s3_client.put_file("/test2/list1/test1", f.name) + assert resp.status == HTTPStatus.OK + + resp = await s3_client.put_file("/test2/list2/test2", f.name) + assert resp.status == HTTPStatus.OK + + # Test list file + batch = 0 + + async for result, prefixes in s3_client.list_objects_v2( + prefix="test2/", + delimiter="/", + ): + batch += 1 + assert len(result) == 0 + assert prefixes[0] == "test2/list1/" + assert prefixes[1] == "test2/list2/" + + async def test_url_path_with_colon(s3_client: S3Client, s3_read): data = b"hello, world" key = "/some-path:with-colon.txt"