Merge pull request #631 from fractal-analytics-platform/600-refactor-…

…core-library-modules

Refactor core-library modules

.gitignore

@@ -1,4 +1,5 @@
 .pytest_cache/
+tree.sh
 .coverage*
 __pycache__
 *.cpython-*.pyc

examples/01_cardio_tiny_dataset/run_workflow.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.yokogawa_to_ome_zarr import yokogawa_to_ome_zarr
 

examples/01_cardio_tiny_dataset/run_workflow_partial_channel_inputs.py

@@ -16,7 +16,7 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
+from fractal_tasks_core.channels import OmeroChannel
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.yokogawa_to_ome_zarr import yokogawa_to_ome_zarr
 

examples/02_cardio_tiny_dataset_sparse/run_workflow.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.yokogawa_to_ome_zarr import yokogawa_to_ome_zarr
 

examples/03_cardio_tiny_dataset_with_illumination_correction/run_workflow.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.illumination_correction import (
     illumination_correction,

examples/04_cardio_tiny_dataset_napari_worfklows/run_example_workflow_1.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.napari_workflows_wrapper import (
     napari_workflows_wrapper,

examples/04_cardio_tiny_dataset_napari_worfklows/run_example_workflow_2.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.napari_workflows_wrapper import (
     napari_workflows_wrapper,

examples/04_cardio_tiny_dataset_napari_worfklows/run_example_workflow_3.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.napari_workflows_wrapper import (
     napari_workflows_wrapper,

examples/04_cardio_tiny_dataset_napari_worfklows/run_example_workflow_4.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.cellpose_segmentation import (
     cellpose_segmentation,
 )

examples/04_cardio_tiny_dataset_napari_worfklows/run_example_workflow_5.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.napari_workflows_wrapper import (
     napari_workflows_wrapper,

examples/04_cardio_tiny_dataset_napari_worfklows/run_example_workflow_6.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.napari_workflows_wrapper import (
     napari_workflows_wrapper,

examples/04_cardio_tiny_dataset_napari_worfklows/run_example_workflow_7.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.cellpose_segmentation import (
     cellpose_segmentation,
 )

examples/05_cardio_tiny_dataset_cellpose/run_example_cellpose.py

@@ -16,8 +16,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.cellpose_segmentation import (
     cellpose_segmentation,
 )

examples/06_cellpose_dev_example/run_example_cellpose_dual_channel.py

@@ -15,8 +15,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.cellpose_segmentation import (
     cellpose_segmentation,
 )

examples/08_subset_of_data/run_example_single_channel.py

@@ -2,8 +2,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.copy_ome_zarr import copy_ome_zarr
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.maximum_intensity_projection import (

examples/08_subset_of_data/run_example_single_channel_few_Z_planes.py

@@ -2,8 +2,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.copy_ome_zarr import copy_ome_zarr
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.maximum_intensity_projection import (

examples/08_subset_of_data/run_example_single_channel_single_Z_plane.py

@@ -2,8 +2,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.copy_ome_zarr import copy_ome_zarr
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.maximum_intensity_projection import (

examples/08_subset_of_data/run_example_two_channel_two_Z_planes.py

@@ -2,8 +2,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.copy_ome_zarr import copy_ome_zarr
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.maximum_intensity_projection import (

examples/08_subset_of_data/run_example_two_channel_two_Z_planes_single_FOV.py

@@ -2,8 +2,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.copy_ome_zarr import copy_ome_zarr
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.maximum_intensity_projection import (

examples/09_secondary_labeling/01_images_to_mip.py

@@ -3,8 +3,8 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_channels import OmeroChannel
-from fractal_tasks_core.lib_channels import Window
+from fractal_tasks_core.channels import OmeroChannel
+from fractal_tasks_core.channels import Window
 from fractal_tasks_core.tasks.copy_ome_zarr import copy_ome_zarr
 from fractal_tasks_core.tasks.create_ome_zarr import create_ome_zarr
 from fractal_tasks_core.tasks.maximum_intensity_projection import (

examples/09_secondary_labeling/02_cellpose_two_steps_segmentation.py

@@ -4,7 +4,7 @@
 
 from devtools import debug
 
-from fractal_tasks_core.lib_input_models import Channel
+from fractal_tasks_core.channels import ChannelInputModel
 from fractal_tasks_core.tasks.cellpose_segmentation import (
     cellpose_segmentation,
 )
@@ -31,7 +31,7 @@
         output_path=zarr_path_mip,
         metadata=metadata,
         component=component,
-        channel=Channel(wavelength_id="A01_C01"),
+        channel=ChannelInputModel(wavelength_id="A01_C01"),
         level=2,
         relabeling=True,
         diameter_level0=400.0,
@@ -53,7 +53,7 @@
         output_path=zarr_path_mip,
         metadata=metadata,
         component=component,
-        channel=Channel(wavelength_id="A01_C01"),
+        channel=ChannelInputModel(wavelength_id="A01_C01"),
         level=2,
         relabeling=True,
         diameter_level0=20.0,

examples/09_secondary_labeling/check_ROI_table.py

@@ -1,6 +1,6 @@
 import sys
 
-from fractal_tasks_core.lib_regions_of_interest import _inspect_ROI_table
+from fractal_tasks_core.roi import _inspect_ROI_table
 
 
 table = sys.argv[1]

examples/tools/lib_inspect_ROI_table.py

@@ -2,7 +2,7 @@
 
 import anndata as ad
 
-from fractal_tasks_core.lib_regions_of_interest import (
+from fractal_tasks_core.roi import (
     convert_ROI_table_to_indices,
 )
 

examples/tools/show_FOV_ROIs/show_FOV_ROIs.py

@@ -3,7 +3,9 @@
 """
 import matplotlib.pyplot as plt
 
-from fractal_tasks_core.lib_metadata_parsing import parse_yokogawa_metadata
+from fractal_tasks_core.cellvoyager.metadata import (
+    parse_yokogawa_metadata,
+)
 from fractal_tasks_core.lib_ROI_overlaps import run_overlap_check
 
 

fractal_tasks_core/__FRACTAL_MANIFEST__.json

@@ -393,12 +393,12 @@
             "description": "Pyramid level of the image to be segmented. Choose `0` to process at full resolution."
           },
           "channel": {
-            "$ref": "#/definitions/Channel",
+            "$ref": "#/definitions/ChannelInputModel",
             "title": "Channel",
             "description": "Primary channel for segmentation; requires either `wavelength_id` (e.g. `A01_C01`) or `label` (e.g. `DAPI`)."
           },
           "channel2": {
-            "$ref": "#/definitions/Channel",
+            "$ref": "#/definitions/ChannelInputModel",
             "title": "Channel2",
             "description": "Second channel for segmentation (in the same format as `channel`). If specified, cellpose runs in dual channel mode. For dual channel segmentation of cells, the first channel should contain the membrane marker, the second channel should contain the nuclear marker."
           },
@@ -505,8 +505,8 @@
         ],
         "additionalProperties": false,
         "definitions": {
-          "Channel": {
-            "title": "Channel",
+          "ChannelInputModel": {
+            "title": "ChannelInputModel",
             "description": "A channel which is specified by either `wavelength_id` or `label`.",
             "type": "object",
             "properties": {
@@ -709,8 +709,8 @@
         ],
         "additionalProperties": false,
         "definitions": {
-          "Channel": {
-            "title": "Channel",
+          "ChannelInputModel": {
+            "title": "ChannelInputModel",
             "description": "A channel which is specified by either `wavelength_id` or `label`.",
             "type": "object",
             "properties": {
@@ -746,9 +746,9 @@
                 "description": "Label name (for label inputs only)."
               },
               "channel": {
-                "$ref": "#/definitions/Channel",
+                "$ref": "#/definitions/ChannelInputModel",
                 "title": "Channel",
-                "description": "Channel object (for image inputs only)."
+                "description": "`ChannelInputModel` object (for image inputs only)."
               }
             },
             "required": [

fractal_tasks_core/__init__.py

@@ -1,5 +1,6 @@
 import logging
 
+
 logging.basicConfig(
     level=logging.INFO, format="%(asctime)s; %(levelname)s; %(message)s"
 )

fractal_tasks_core/cellvoyager/__init__.py

@@ -0,0 +1,3 @@
+"""
+Subpackage with utilities for tasks converting CellVoyager images to OME-Zarr.
+"""

fractal_tasks_core/lib_parse_filename_metadata.py → fractal_tasks_core/cellvoyager/filenames.py

@@ -10,10 +10,54 @@
 # Institute for Biomedical Research and Pelkmans Lab from the University of
 # Zurich.
 """
-Extract metadata from image filename.
+Auxiliary functions related to filenames of Yokogawa-microscope images.
 """
+import logging
 import re
+from glob import glob
 from pathlib import Path
+from typing import Sequence
+
+
+def glob_with_multiple_patterns(
+    *,
+    folder: str,
+    patterns: Sequence[str] = None,
+) -> set[str]:
+    """
+    List all the items (files and folders) in a given folder that
+    simultaneously match a series of glob patterns.
+
+    Args:
+        folder: Base folder where items will be searched.
+        patterns: If specified, the list of patterns (defined as in
+            https://docs.python.org/3/library/fnmatch.html) that item
+            names will match with.
+    """
+
+    # Sanitize base-folder path
+    if folder.endswith("/"):
+        actual_folder = folder[:-1]
+    else:
+        actual_folder = folder[:]
+
+    # If not pattern is specified, look for *all* items in the base folder
+    if not patterns:
+        patterns = ["*"]
+
+    # Combine multiple glob searches (via set intersection)
+    logging.info(f"[glob_with_multiple_patterns] {patterns=}")
+    items = None
+    for pattern in patterns:
+        new_matches = glob(f"{actual_folder}/{pattern}")
+        if items is None:
+            items = set(new_matches)
+        else:
+            items = items.intersection(new_matches)
+    items = items or set()
+    logging.info(f"[glob_with_multiple_patterns] Found {len(items)} items")
+
+    return items
 
 
 def _get_plate_name(plate_prefix: str) -> str: