Integrating labeling tasks into fractal (ref #64 #104) and clean up p…

…art of the examples folder
fractal-analytics-platform · Jul 18, 2022 · 3c1c431 · 3c1c431
1 parent 12deb67
commit 3c1c431
Show file tree

Hide file tree

Showing 15 changed files with 189 additions and 79 deletions.
diff --git a/.gitignore b/.gitignore
@@ -17,3 +17,7 @@ LOG_*
 slurm-*.out
 Local_backup
 mprofile_*.dat
+
+.nfs*
+nohup
+nohup.out
diff --git a/examples/example_uzh_10_well_5x5_sites.sh → ...x5_sites/example_uzh_10_well_5x5_sites.sh b/examples/example_uzh_10_well_5x5_sites.sh → ...x5_sites/example_uzh_10_well_5x5_sites.sh
@@ -7,11 +7,14 @@ WFPARAMS=wf_params_uzh_10_well_5x5_sites.json
 MWE_DIR=/data/active/fractal/tests
 PATH_OUTPUT=${MWE_DIR}/Temporary_data_UZH_10_well_5x5_sites
 
-CMD='poetry run python ../fractal/fractal_cmd.py'
+CMD='poetry run python ../../fractal/fractal_cmd.py'
 
 #echo 'Re-install poetry'
 #poetry install
 
+date
+echo
+
 echo 'Clean up'
 rm -rf $PATH_OUTPUT
 mkdir -p $PATH_OUTPUT
@@ -37,61 +40,73 @@ echo
 
 echo 'Add yokogawa_to_zarr task'
 $CMD task add yokogawa_to_zarr zarr zarr well
-$CMD task list
 echo
 
-echo 'Add replicate_zarr_structure'
-$CMD task add replicate_zarr_structure zarr zarr plate
-$CMD task list
-echo
+#echo 'Add replicate_zarr_structure'
+#$CMD task add replicate_zarr_structure zarr zarr plate
+#echo
 
 echo 'Add illumination_correction'
 $CMD task add illumination_correction zarr zarr well
-$CMD task list
 echo
 
 echo 'Add replicate_zarr_structure_mip'
 $CMD task add replicate_zarr_structure_mip zarr zarr plate
-$CMD task list
+echo
+
+echo 'Add image_labeling'
+$CMD task add image_labeling zarr zarr well
+echo
+
+echo 'Add image_labeling_whole_well'
+$CMD task add image_labeling_whole_well zarr zarr well
 echo
 
 echo 'Add maximum_intensity_projection'
 $CMD task add maximum_intensity_projection zarr zarr well
+echo
+
+echo
 $CMD task list
 echo
 
 #############################################
 
 echo 'Create workflow'
 $CMD workflow new mwe-test wftest create_zarr_structure
-$CMD workflow list mwe-test
 echo
 
 echo 'Add yokogawa_to_zarr task'
 $CMD workflow add-task mwe-test wftest yokogawa_to_zarr
-$CMD workflow list mwe-test
 echo
 
-#echo 'Add replicate_zarr_structure'
-#$CMD workflow add-task mwe-test wftest replicate_zarr_structure
-#$CMD workflow list mwe-test
-#echo
-
 echo 'Add illumination_correction'
 $CMD workflow add-task mwe-test wftest illumination_correction
-$CMD workflow list mwe-test
+echo
+
+echo 'Add image_labeling'
+$CMD workflow add-task mwe-test wftest image_labeling
 echo
 
 echo 'Add replicate_zarr_structure_mip'
 $CMD workflow add-task mwe-test wftest replicate_zarr_structure_mip
-$CMD workflow list mwe-test
 echo
 
 echo 'Add maximum_intensity_projection'
 $CMD workflow add-task mwe-test wftest maximum_intensity_projection
+echo
+
+echo 'Add image_labeling_whole_well'
+$CMD workflow add-task mwe-test wftest image_labeling_whole_well
+echo
+
+echo 'Final list:'
 $CMD workflow list mwe-test
 echo
 
 echo 'Execute workflow'
 $CMD workflow apply mwe-test wftest dstest dstest $PATH_INPUT $PATH_OUTPUT $WFPARAMS
 echo
+
+echo
+date
diff --git a/examples/UZH_10_well_5x5_sites/wf_params_uzh_10_well_5x5_sites.json b/examples/UZH_10_well_5x5_sites/wf_params_uzh_10_well_5x5_sites.json
@@ -0,0 +1,11 @@
+{
+"workflow_name": "uzh_10_well_5x5_sites",
+"dims": [5, 5],
+"coarsening_xy": 2,
+"coarsening_z": 1,
+"num_levels": 5,
+"channel_file": "../wf_params_uzh_cardiac_channels.json",
+"path_dict_corr": "../wf_params_uzh_cardiac_illumination.json",
+"image_labeling": {"coarsening_xy": 2, "labeling_level": 0, "labeling_channel": "A01_C01", "num_threads": 2, "relabeling": 1, "anisotropy": 6.1538, "diameter": 35.0, "cellprob_threshold": 0.0},
+"image_labeling_whole_well": {"coarsening_xy": 2, "labeling_level": 2, "labeling_channel": "A01_C01", "diameter_level0": 35.0, "cellprob_threshold": 0.0}
+}
diff --git a/examples/example_uzh_1_well_2x2_sites.sh → ...2x2_sites/example_uzh_1_well_2x2_sites.sh b/examples/example_uzh_1_well_2x2_sites.sh → ...2x2_sites/example_uzh_1_well_2x2_sites.sh
@@ -5,13 +5,16 @@ PATH_INPUT=/data/active/fractal/3D/PelkmansLab/CardiacMultiplexing/Cycle1_testSu
 WFPARAMS=wf_params_uzh_1_well_2x2_sites.json
 
 MWE_DIR=/data/active/fractal/tests
-PATH_OUTPUT=${MWE_DIR}/Temporary_data_UZH_1_well_2x2_sites_singlefov
+PATH_OUTPUT=${MWE_DIR}/Temporary_data_UZH_1_well_2x2_sites
 
-CMD='poetry run python ../fractal/fractal_cmd.py'
+CMD='poetry run python ../../fractal/fractal_cmd.py'
 
 #echo 'Re-install poetry'
 #poetry install
 
+date
+echo
+
 echo 'Clean up'
 rm -rf $PATH_OUTPUT
 mkdir -p $PATH_OUTPUT
@@ -55,8 +58,15 @@ echo 'Add image_labeling'
 $CMD task add image_labeling zarr zarr well
 echo
 
+echo 'Add image_labeling_whole_well'
+$CMD task add image_labeling_whole_well zarr zarr well
+echo
+
 echo 'Add maximum_intensity_projection'
 $CMD task add maximum_intensity_projection zarr zarr well
+echo
+
+echo
 $CMD task list
 echo
 
@@ -86,10 +96,17 @@ echo 'Add maximum_intensity_projection'
 $CMD workflow add-task mwe-test wftest maximum_intensity_projection
 echo
 
+echo 'Add image_labeling_whole_well'
+$CMD workflow add-task mwe-test wftest image_labeling_whole_well
+echo
+
 echo 'Final list:'
 $CMD workflow list mwe-test
 echo
 
 echo 'Execute workflow'
 $CMD workflow apply mwe-test wftest dstest dstest $PATH_INPUT $PATH_OUTPUT $WFPARAMS
 echo
+
+echo
+date
diff --git a/examples/UZH_1_well_2x2_sites/wf_params_uzh_1_well_2x2_sites.json b/examples/UZH_1_well_2x2_sites/wf_params_uzh_1_well_2x2_sites.json
@@ -0,0 +1,11 @@
+{
+"workflow_name": "uzh_1_well_2x2_sites",
+"dims": [2, 2],
+"coarsening_xy": 2,
+"coarsening_z": 1,
+"num_levels": 5,
+"channel_file": "../wf_params_uzh_cardiac_channels.json",
+"path_dict_corr": "../wf_params_uzh_cardiac_illumination.json",
+"image_labeling": {"coarsening_xy": 2, "labeling_level": 0, "labeling_channel": "A01_C01", "num_threads": 2, "relabeling": 1, "anisotropy": 6.1538, "diameter": 35.0, "cellprob_threshold": 0.0},
+"image_labeling_whole_well": {"coarsening_xy": 2, "labeling_level": 2, "labeling_channel": "A01_C01", "diameter_level0": 35.0, "cellprob_threshold": 0.0}
+}
diff --git a/examples/example_uzh_1_well_9x8_sites.sh → ...9x8_sites/example_uzh_1_well_9x8_sites.sh b/examples/example_uzh_1_well_9x8_sites.sh → ...9x8_sites/example_uzh_1_well_9x8_sites.sh
@@ -7,11 +7,14 @@ WFPARAMS=wf_params_uzh_1_well_9x8_sites.json
 MWE_DIR=/data/active/fractal/tests
 PATH_OUTPUT=${MWE_DIR}/Temporary_data_UZH_1_well_9x8_sites_singlefov
 
-CMD='poetry run python ../fractal/fractal_cmd.py'
+CMD='poetry run python ../../fractal/fractal_cmd.py'
 
 #echo 'Re-install poetry'
 #poetry install
 
+date
+echo
+
 echo 'Clean up'
 rm -rf $PATH_OUTPUT
 mkdir -p $PATH_OUTPUT
@@ -55,8 +58,15 @@ echo 'Add image_labeling'
 $CMD task add image_labeling zarr zarr well
 echo
 
+echo 'Add image_labeling_whole_well'
+$CMD task add image_labeling_whole_well zarr zarr well
+echo
+
 echo 'Add maximum_intensity_projection'
 $CMD task add maximum_intensity_projection zarr zarr well
+echo
+
+echo
 $CMD task list
 echo
 
@@ -86,10 +96,17 @@ echo 'Add maximum_intensity_projection'
 $CMD workflow add-task mwe-test wftest maximum_intensity_projection
 echo
 
+echo 'Add image_labeling_whole_well'
+$CMD workflow add-task mwe-test wftest image_labeling_whole_well
+echo
+
 echo 'Final list:'
 $CMD workflow list mwe-test
 echo
 
 echo 'Execute workflow'
 $CMD workflow apply mwe-test wftest dstest dstest $PATH_INPUT $PATH_OUTPUT $WFPARAMS
 echo
+
+echo
+date
diff --git a/examples/UZH_1_well_9x8_sites/wf_params_uzh_1_well_9x8_sites.json b/examples/UZH_1_well_9x8_sites/wf_params_uzh_1_well_9x8_sites.json
@@ -0,0 +1,11 @@
+{
+"workflow_name": "uzh_1_well_9x8_sites",
+"dims": [9, 8],
+"coarsening_xy": 2,
+"coarsening_z": 1,
+"num_levels": 5,
+"channel_file": "../wf_params_uzh_cardiac_channels.json",
+"path_dict_corr": "../wf_params_uzh_cardiac_illumination.json",
+"image_labeling": {"coarsening_xy": 2, "labeling_level": 0, "labeling_channel": "A01_C01", "num_threads": 2, "relabeling": 1, "anisotropy": 6.1538, "diameter": 35.0, "cellprob_threshold": 0.0},
+"image_labeling_whole_well": {"coarsening_xy": 2, "labeling_level": 2, "labeling_channel": "A01_C01", "diameter_level0": 35.0, "cellprob_threshold": 0.0}
+}
diff --git a/examples/example_uzh_4_well_2x2_sites.sh → ...2x2_sites/example_uzh_4_well_2x2_sites.sh b/examples/example_uzh_4_well_2x2_sites.sh → ...2x2_sites/example_uzh_4_well_2x2_sites.sh
@@ -7,11 +7,14 @@ WFPARAMS=wf_params_uzh_4_well_2x2_sites.json
 MWE_DIR=/data/active/fractal/tests
 PATH_OUTPUT=${MWE_DIR}/Temporary_data_UZH_4_well_2x2_sites
 
-CMD='poetry run python ../fractal/fractal_cmd.py'
+CMD='poetry run python ../../fractal/fractal_cmd.py'
 
 #echo 'Re-install poetry'
 #poetry install
 
+date
+echo
+
 echo 'Clean up'
 rm -rf $PATH_OUTPUT
 mkdir -p $PATH_OUTPUT
@@ -37,61 +40,73 @@ echo
 
 echo 'Add yokogawa_to_zarr task'
 $CMD task add yokogawa_to_zarr zarr zarr well
-$CMD task list
 echo
 
-echo 'Add replicate_zarr_structure'
-$CMD task add replicate_zarr_structure zarr zarr plate
-$CMD task list
-echo
+#echo 'Add replicate_zarr_structure'
+#$CMD task add replicate_zarr_structure zarr zarr plate
+#echo
 
 echo 'Add illumination_correction'
 $CMD task add illumination_correction zarr zarr well
-$CMD task list
 echo
 
 echo 'Add replicate_zarr_structure_mip'
 $CMD task add replicate_zarr_structure_mip zarr zarr plate
-$CMD task list
+echo
+
+echo 'Add image_labeling'
+$CMD task add image_labeling zarr zarr well
+echo
+
+echo 'Add image_labeling_whole_well'
+$CMD task add image_labeling_whole_well zarr zarr well
 echo
 
 echo 'Add maximum_intensity_projection'
 $CMD task add maximum_intensity_projection zarr zarr well
+echo
+
+echo
 $CMD task list
 echo
 
 #############################################
 
 echo 'Create workflow'
 $CMD workflow new mwe-test wftest create_zarr_structure
-$CMD workflow list mwe-test
 echo
 
 echo 'Add yokogawa_to_zarr task'
 $CMD workflow add-task mwe-test wftest yokogawa_to_zarr
-$CMD workflow list mwe-test
 echo
 
-#echo 'Add replicate_zarr_structure'
-#$CMD workflow add-task mwe-test wftest replicate_zarr_structure
-#$CMD workflow list mwe-test
-#echo
-
 echo 'Add illumination_correction'
 $CMD workflow add-task mwe-test wftest illumination_correction
-$CMD workflow list mwe-test
+echo
+
+echo 'Add image_labeling'
+$CMD workflow add-task mwe-test wftest image_labeling
 echo
 
 echo 'Add replicate_zarr_structure_mip'
 $CMD workflow add-task mwe-test wftest replicate_zarr_structure_mip
-$CMD workflow list mwe-test
 echo
 
 echo 'Add maximum_intensity_projection'
 $CMD workflow add-task mwe-test wftest maximum_intensity_projection
+echo
+
+echo 'Add image_labeling_whole_well'
+$CMD workflow add-task mwe-test wftest image_labeling_whole_well
+echo
+
+echo 'Final list:'
 $CMD workflow list mwe-test
 echo
 
 echo 'Execute workflow'
 $CMD workflow apply mwe-test wftest dstest dstest $PATH_INPUT $PATH_OUTPUT $WFPARAMS
 echo
+
+echo
+date
diff --git a/examples/UZH_4_well_2x2_sites/wf_params_uzh_4_well_2x2_sites.json b/examples/UZH_4_well_2x2_sites/wf_params_uzh_4_well_2x2_sites.json
@@ -0,0 +1,11 @@
+{
+"workflow_name": "uzh_4_well_2x2_sites",
+"dims": [2, 2],
+"coarsening_xy": 2,
+"coarsening_z": 1,
+"num_levels": 5,
+"channel_file": "../wf_params_uzh_cardiac_channels.json",
+"path_dict_corr": "../wf_params_uzh_cardiac_illumination.json",
+"image_labeling": {"coarsening_xy": 2, "labeling_level": 0, "labeling_channel": "A01_C01", "num_threads": 1, "relabeling": 1, "anisotropy": 6.1538, "diameter": 35.0, "cellprob_threshold": 0.0},
+"image_labeling_whole_well": {"coarsening_xy": 2, "labeling_level": 2, "labeling_channel": "A01_C01", "diameter_level0": 35.0, "cellprob_threshold": 0.0}
+}
diff --git a/examples/wf_params_uzh_1_well_2x2_sites.json b/examples/wf_params_uzh_1_well_2x2_sites.json
diff --git a/examples/wf_params_uzh_1_well_9x8_sites.json b/examples/wf_params_uzh_1_well_9x8_sites.json