From d2a4f4134e15668be96e07fb6a91af3c063a10fb Mon Sep 17 00:00:00 2001 From: Josh Moore Date: Thu, 1 Aug 2024 09:14:49 +0200 Subject: [PATCH] Support bioformats2raw.layout --- src/ome2024_ngff_challenge/resave.py | 72 ++++++++++++++++++-- tests/data/bf2raw.fake | 0 tests/data/bf2raw.fake.ini | 3 + tests/data/bf2raw.zarr/.zattrs | 3 + tests/data/bf2raw.zarr/.zgroup | 3 + tests/data/bf2raw.zarr/0/.zattrs | 54 +++++++++++++++ tests/data/bf2raw.zarr/0/.zgroup | 3 + tests/data/bf2raw.zarr/0/0/.zarray | 17 +++++ tests/data/bf2raw.zarr/0/0/0/0/0/0/0 | Bin 0 -> 80 bytes tests/data/bf2raw.zarr/1/.zattrs | 54 +++++++++++++++ tests/data/bf2raw.zarr/1/.zgroup | 3 + tests/data/bf2raw.zarr/1/0/.zarray | 17 +++++ tests/data/bf2raw.zarr/1/0/0/0/0/0/0 | Bin 0 -> 80 bytes tests/data/bf2raw.zarr/OME/.zattrs | 3 + tests/data/bf2raw.zarr/OME/.zgroup | 3 + tests/data/bf2raw.zarr/OME/METADATA.ome.xml | 1 + tests/test_resave.py | 38 ++++++++--- 17 files changed, 260 insertions(+), 14 deletions(-) create mode 100644 tests/data/bf2raw.fake create mode 100644 tests/data/bf2raw.fake.ini create mode 100644 tests/data/bf2raw.zarr/.zattrs create mode 100644 tests/data/bf2raw.zarr/.zgroup create mode 100644 tests/data/bf2raw.zarr/0/.zattrs create mode 100644 tests/data/bf2raw.zarr/0/.zgroup create mode 100644 tests/data/bf2raw.zarr/0/0/.zarray create mode 100644 tests/data/bf2raw.zarr/0/0/0/0/0/0/0 create mode 100644 tests/data/bf2raw.zarr/1/.zattrs create mode 100644 tests/data/bf2raw.zarr/1/.zgroup create mode 100644 tests/data/bf2raw.zarr/1/0/.zarray create mode 100644 tests/data/bf2raw.zarr/1/0/0/0/0/0/0 create mode 100644 tests/data/bf2raw.zarr/OME/.zattrs create mode 100644 tests/data/bf2raw.zarr/OME/.zgroup create mode 100644 tests/data/bf2raw.zarr/OME/METADATA.ome.xml diff --git a/src/ome2024_ngff_challenge/resave.py b/src/ome2024_ngff_challenge/resave.py index aecfc44..a1746f1 100755 --- a/src/ome2024_ngff_challenge/resave.py +++ b/src/ome2024_ngff_challenge/resave.py @@ -380,7 +380,11 @@ def write_rocrate(write_store): sync(write_store.set(filename, text)) -def main(ns: argparse.Namespace): +def main(ns: argparse.Namespace) -> int: + """ + Returns the number of images converted + """ + converted = 0 configs = create_configs(ns) stores = [] @@ -444,9 +448,10 @@ def main(ns: argparse.Namespace): ns.output_write_details, ns.output_script, ) + converted += 1 # plate... - elif read_root.attrs.get("plate"): + elif plate_attrs := read_root.attrs.get("plate"): ome_attrs = {"version": NGFF_VERSION} for key, value in read_root.attrs.items(): # ...replaces all other versions - remove @@ -457,7 +462,6 @@ def main(ns: argparse.Namespace): # dev2: everything is under 'ome' key write_root.attrs["ome"] = ome_attrs - plate_attrs = read_root.attrs.get("plate") wells = plate_attrs.get("wells") for well in tqdm.tqdm( @@ -506,9 +510,65 @@ def main(ns: argparse.Namespace): ns.output_script, img_path, # TODO: review ) + converted += 1 + # Note: plates can *also* contain this metadata + elif layout := read_root.attrs.get("bioformats2raw.layout"): + assert layout == 3 + ome_attrs = {"version": NGFF_VERSION} + for key, value in read_root.attrs.items(): + # ...replaces all other versions - remove + strip_version(value) + ome_attrs[key] = value + + if write_root is not None: # otherwise dry run + # dev2: everything is under 'ome' key + write_root.attrs["ome"] = ome_attrs + + ome_group = zarr.open_group(store=stores[0], path="OME", zarr_format=2) + series = ome_group.attrs.get("series", []) + assert series # TODO: support implicit case where OME-XML must be read + + for img_path in tqdm.tqdm( + series, position=0, desc="i", leave=False, colour="green", ncols=80 + ): + # TODO: duplicated with plate section (move to class) + out_path = ns.output_path / img_path + input_path = ns.input_path / img_path + img_v2 = zarr.open_group(store=stores[0], path=str(img_path), zarr_format=2) + + if write_root is not None: # otherwise dry-run + image_group = write_root.create_group(str(img_path)) + else: + image_group = None + + convert_image( + configs, + img_v2, + input_path, + write_store, # TODO: review + image_group, + out_path, + ns.output_overwrite, + ns.output_chunks, + ns.output_shards, + ns.output_read_details, + ns.output_write_details, + ns.output_script, + img_path, # TODO: review + ) + converted += 1 + else: + LOGGER.warning(f"no convertible metadata: {read_root.attrs.keys()}") + + return converted def cli(args=sys.argv): + """ + Parses the arguments contained in `args` and passes + them to `main`. If no images are converted, raises + SystemExit. Otherwise, return the number of images. + """ parser = argparse.ArgumentParser() parser.add_argument("--input-bucket") parser.add_argument("--input-endpoint") @@ -544,7 +604,11 @@ def cli(args=sys.argv): ns = parser.parse_args(args) logging.basicConfig() - main(ns) + + converted = main(ns) + if converted == 0: + raise SystemExit(1) + return converted if __name__ == "__main__": diff --git a/tests/data/bf2raw.fake b/tests/data/bf2raw.fake new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/bf2raw.fake.ini b/tests/data/bf2raw.fake.ini new file mode 100644 index 0000000..282039a --- /dev/null +++ b/tests/data/bf2raw.fake.ini @@ -0,0 +1,3 @@ +sizeX=8 +sizeY=8 +series=2 diff --git a/tests/data/bf2raw.zarr/.zattrs b/tests/data/bf2raw.zarr/.zattrs new file mode 100644 index 0000000..80da91b --- /dev/null +++ b/tests/data/bf2raw.zarr/.zattrs @@ -0,0 +1,3 @@ +{ + "bioformats2raw.layout" : 3 +} diff --git a/tests/data/bf2raw.zarr/.zgroup b/tests/data/bf2raw.zarr/.zgroup new file mode 100644 index 0000000..0308776 --- /dev/null +++ b/tests/data/bf2raw.zarr/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} diff --git a/tests/data/bf2raw.zarr/0/.zattrs b/tests/data/bf2raw.zarr/0/.zattrs new file mode 100644 index 0000000..c5cac4c --- /dev/null +++ b/tests/data/bf2raw.zarr/0/.zattrs @@ -0,0 +1,54 @@ +{ + "multiscales" : [ { + "metadata" : { + "method" : "loci.common.image.SimpleImageScaler", + "version" : "Bio-Formats 7.1.0" + }, + "axes" : [ { + "name" : "t", + "type" : "time" + }, { + "name" : "c", + "type" : "channel" + }, { + "name" : "z", + "type" : "space" + }, { + "name" : "y", + "type" : "space" + }, { + "name" : "x", + "type" : "space" + } ], + "name" : "bf2raw", + "datasets" : [ { + "path" : "0", + "coordinateTransformations" : [ { + "scale" : [ 1.0, 1.0, 1.0, 1.0, 1.0 ], + "type" : "scale" + } ] + } ], + "version" : "0.4" + } ], + "omero" : { + "channels" : [ { + "color" : "808080", + "coefficient" : 1, + "active" : true, + "label" : "Channel 0", + "window" : { + "min" : 0.0, + "max" : 0.0, + "start" : 0.0, + "end" : 0.0 + }, + "family" : "linear", + "inverted" : false + } ], + "rdefs" : { + "defaultT" : 0, + "model" : "greyscale", + "defaultZ" : 0 + } + } +} diff --git a/tests/data/bf2raw.zarr/0/.zgroup b/tests/data/bf2raw.zarr/0/.zgroup new file mode 100644 index 0000000..0308776 --- /dev/null +++ b/tests/data/bf2raw.zarr/0/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} diff --git a/tests/data/bf2raw.zarr/0/0/.zarray b/tests/data/bf2raw.zarr/0/0/.zarray new file mode 100644 index 0000000..755a000 --- /dev/null +++ b/tests/data/bf2raw.zarr/0/0/.zarray @@ -0,0 +1,17 @@ +{ + "chunks" : [ 1, 1, 1, 8, 8 ], + "compressor" : { + "clevel" : 5, + "blocksize" : 0, + "shuffle" : 1, + "cname" : "lz4", + "id" : "blosc" + }, + "dtype" : "|u1", + "fill_value" : 0, + "filters" : null, + "order" : "C", + "shape" : [ 1, 1, 1, 8, 8 ], + "dimension_separator" : "/", + "zarr_format" : 2 +} diff --git a/tests/data/bf2raw.zarr/0/0/0/0/0/0/0 b/tests/data/bf2raw.zarr/0/0/0/0/0/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..33567ca165190ee00842be656b34dc3f5dfb568b GIT binary patch literal 80 VcmZQ#G-h;QU|;~@03fCm002u?0SEv9 literal 0 HcmV?d00001 diff --git a/tests/data/bf2raw.zarr/1/.zattrs b/tests/data/bf2raw.zarr/1/.zattrs new file mode 100644 index 0000000..d34f6ef --- /dev/null +++ b/tests/data/bf2raw.zarr/1/.zattrs @@ -0,0 +1,54 @@ +{ + "multiscales" : [ { + "metadata" : { + "method" : "loci.common.image.SimpleImageScaler", + "version" : "Bio-Formats 7.1.0" + }, + "axes" : [ { + "name" : "t", + "type" : "time" + }, { + "name" : "c", + "type" : "channel" + }, { + "name" : "z", + "type" : "space" + }, { + "name" : "y", + "type" : "space" + }, { + "name" : "x", + "type" : "space" + } ], + "name" : "bf2raw 2", + "datasets" : [ { + "path" : "0", + "coordinateTransformations" : [ { + "scale" : [ 1.0, 1.0, 1.0, 1.0, 1.0 ], + "type" : "scale" + } ] + } ], + "version" : "0.4" + } ], + "omero" : { + "channels" : [ { + "color" : "808080", + "coefficient" : 1, + "active" : true, + "label" : "Channel 0", + "window" : { + "min" : 1.0, + "max" : 1.0, + "start" : 1.0, + "end" : 1.0 + }, + "family" : "linear", + "inverted" : false + } ], + "rdefs" : { + "defaultT" : 0, + "model" : "greyscale", + "defaultZ" : 0 + } + } +} diff --git a/tests/data/bf2raw.zarr/1/.zgroup b/tests/data/bf2raw.zarr/1/.zgroup new file mode 100644 index 0000000..0308776 --- /dev/null +++ b/tests/data/bf2raw.zarr/1/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} diff --git a/tests/data/bf2raw.zarr/1/0/.zarray b/tests/data/bf2raw.zarr/1/0/.zarray new file mode 100644 index 0000000..755a000 --- /dev/null +++ b/tests/data/bf2raw.zarr/1/0/.zarray @@ -0,0 +1,17 @@ +{ + "chunks" : [ 1, 1, 1, 8, 8 ], + "compressor" : { + "clevel" : 5, + "blocksize" : 0, + "shuffle" : 1, + "cname" : "lz4", + "id" : "blosc" + }, + "dtype" : "|u1", + "fill_value" : 0, + "filters" : null, + "order" : "C", + "shape" : [ 1, 1, 1, 8, 8 ], + "dimension_separator" : "/", + "zarr_format" : 2 +} diff --git a/tests/data/bf2raw.zarr/1/0/0/0/0/0/0 b/tests/data/bf2raw.zarr/1/0/0/0/0/0/0 new file mode 100644 index 0000000000000000000000000000000000000000..d2e8e244c3c270ad2440ac8276db7fa63291a36e GIT binary patch literal 80 WcmZQ#G-h;QU|;~@03c?h5C8yEb^%BL literal 0 HcmV?d00001 diff --git a/tests/data/bf2raw.zarr/OME/.zattrs b/tests/data/bf2raw.zarr/OME/.zattrs new file mode 100644 index 0000000..be7e9ed --- /dev/null +++ b/tests/data/bf2raw.zarr/OME/.zattrs @@ -0,0 +1,3 @@ +{ + "series" : [ "0", "1" ] +} diff --git a/tests/data/bf2raw.zarr/OME/.zgroup b/tests/data/bf2raw.zarr/OME/.zgroup new file mode 100644 index 0000000..0308776 --- /dev/null +++ b/tests/data/bf2raw.zarr/OME/.zgroup @@ -0,0 +1,3 @@ +{ + "zarr_format" : 2 +} diff --git a/tests/data/bf2raw.zarr/OME/METADATA.ome.xml b/tests/data/bf2raw.zarr/OME/METADATA.ome.xml new file mode 100644 index 0000000..4d77df2 --- /dev/null +++ b/tests/data/bf2raw.zarr/OME/METADATA.ome.xml @@ -0,0 +1 @@ + diff --git a/tests/test_resave.py b/tests/test_resave.py index 7667c2c..b7c71dd 100644 --- a/tests/test_resave.py +++ b/tests/test_resave.py @@ -90,18 +90,36 @@ def test_remote_simple_with_download(tmp_path): def test_local_2d(tmp_path): - resave.cli( - [ - "data/2d.zarr", - str(tmp_path / "out.zarr"), - ] + assert ( + resave.cli( + [ + "data/2d.zarr", + str(tmp_path / "out.zarr"), + ] + ) + == 1 + ) + + +def test_local_bf2raw(tmp_path): + assert ( + resave.cli( + [ + "data/bf2raw.zarr", + str(tmp_path / "out.zarr"), + ] + ) + == 2 ) def test_local_hcs(tmp_path): - resave.cli( - [ - "data/hcs.zarr", - str(tmp_path / "out.zarr"), - ] + assert ( + resave.cli( + [ + "data/hcs.zarr", + str(tmp_path / "out.zarr"), + ] + ) + == 8 )