From 204370a2e4769482feec2de28f8be7eb67554c08 Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:17:58 +0200 Subject: [PATCH 1/5] added notebook for batch analysis --- .../demo_analyze_everything_batch.ipynb | 1312 +++++++++++++++++ 1 file changed, 1312 insertions(+) create mode 100644 docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb diff --git a/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb new file mode 100644 index 00000000..acac2c4f --- /dev/null +++ b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb @@ -0,0 +1,1312 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "id": "75bf2599-5a29-4dfc-b0b0-7cecc3c7ab33", + "metadata": {}, + "source": [ + "(toolboxes:analyze_everything)=\n", + "# Analyze everything\n", + "\n", + "This notebook demonstrates how to run a complete STRESS analysis in a batched way. This requires you to assemble a list of complete filenames and then submit all of them for processing. It is strongly required to do this on a sufficiently powerful processing PC.\n", + "\n", + "## Additional dependencies\n", + "\n", + "In case you want to work with proprietory file formats (e.g., czi, nd2, etc), you'll need to install some packages that can handle them. As a one-fits-all solution, install the bioio package. To find out exactly what extension of bioio you need, [check the documentation](https://pypi.org/project/bioio/). For instance, for working with czi images you'll need these dependencies:\n", + "\n", + "```bash\n", + "pip install bioio bioio-czi\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": 71, + "id": "92cdbcda-ad6d-4048-a1eb-f2a452969326", + "metadata": {}, + "outputs": [], + "source": [ + "import napari\n", + "import numpy as np\n", + "from napari_stress import reconstruction, measurements, TimelapseConverter, utils, stress_backend\n", + "import os\n", + "from pathlib import Path\n", + "import datetime\n", + "from bioio import BioImage\n", + "\n", + "from dask.distributed import Client, get_client\n", + "\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib as mpl\n", + "from skimage import io\n", + "import napari_vedo_bridge as nvb" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "e440ed15-c9bd-454f-8583-2aaa071142bb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Assistant skips harvesting pyclesperanto as it's not installed.\n" + ] + } + ], + "source": [ + "viewer = napari.Viewer(ndisplay=3)" + ] + }, + { + "cell_type": "markdown", + "id": "c5fa4385", + "metadata": {}, + "source": [ + "## Set up parallelization\n", + "\n", + "IT's important to set this up beforehand. The process may require a lot of memory - and if you don't set it here, napari-stress will set it for you, which may not be suitable for the load we will be working with in this usecase." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "11e4c0cb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Link to progress dashboard: http://127.0.0.1:8787/status\n" + ] + } + ], + "source": [ + "try:\n", + " client = get_client()\n", + "except Exception:\n", + " client = Client(n_workers=8, memory_limit='32GB')\n", + "print('Link to progress dashboard: ', client.dashboard_link)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "87a6fa3c-9eae-48cd-8eca-52dd0cbaed49", + "metadata": {}, + "source": [ + "## Collect data to be analyzed\n", + "\n", + "The easiest way to find samples in a given root folder is to use `os.walk`. In essence, it recursively walks all subdirectories of a given folder, which allows you to look for files that are matching a specific pattern:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "09557687", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pathpixel_size_zpixel_size_ypixel_size_x
0D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.370.0828640.082864
1D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.380.1035800.103580
2D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.380.1035800.103580
3D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.380.0941640.094164
4D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.380.0828640.082864
5D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.380.0828640.082864
6D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.380.1035800.103580
7D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.380.0828640.082864
8D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.380.0828640.082864
9D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.370.0900700.090070
10D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.370.1035800.103580
11D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.370.1150890.115089
12D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8....0.370.1035800.103580
13D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1150890.115089
14D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1150890.115089
15D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1150890.115089
16D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1035800.103580
17D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1150890.115089
18D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1150890.115089
19D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1035800.103580
20D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1294750.129475
21D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1150890.115089
22D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1150890.115089
23D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.380.1150890.115089
24D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.370.1479720.147972
25D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.370.1150890.115089
26D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.370.1381070.138107
27D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.370.1726340.172634
28D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.370.1381070.138107
29D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.370.1150890.115089
30D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9....0.370.1035800.103580
31D:\\Marc\\Marc_genistein_droplets_Aug2024\\F1G4 d...0.370.1035800.103580
32D:\\Marc\\Marc_genistein_droplets_Aug2024\\F1G4 d...0.370.1035800.103580
33D:\\Marc\\Marc_genistein_droplets_Aug2024\\F1G4 d...0.370.0941640.094164
34D:\\Marc\\Marc_genistein_droplets_Aug2024\\GFP-NL...0.400.0828640.082864
35D:\\Marc\\Marc_genistein_droplets_Aug2024\\GFP-NL...0.400.0690540.069054
36D:\\Marc\\Marc_genistein_droplets_Aug2024\\GFP-NL...0.400.0828640.082864
37D:\\Marc\\Marc_genistein_droplets_Aug2024\\GFP-NL...0.400.0828640.082864
38D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.480.0796770.079677
39D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.1035800.103580
40D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.0828640.082864
41D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.0941640.094164
42D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.1035800.103580
43D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.1090320.109032
44D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.1150890.115089
45D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.1035800.103580
46D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.0828640.082864
47D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.1035800.103580
48D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4...0.370.1035800.103580
49D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.1035800.103580
50D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0828640.082864
51D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0690540.069054
52D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0690540.069054
53D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0690540.069054
54D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.1035800.103580
55D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0690540.069054
56D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0690540.069054
57D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0690540.069054
58D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0690540.069054
59D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0739860.073986
60D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0941640.094164
61D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.370.0739860.073986
62D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.1035800.103580
63D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.1035800.103580
64D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.0739860.073986
65D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.0690540.069054
66D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.0690540.069054
67D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.0690540.069054
68D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.0828640.082864
69D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.0767260.076726
70D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.0690540.069054
71D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.0690540.069054
72D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-...0.400.0690540.069054
73D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0739860.073986
74D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0690540.069054
75D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0739860.073986
76D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0690540.069054
77D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0690540.069054
78D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0690540.069054
79D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0767260.076726
80D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0690540.069054
81D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0690540.069054
82D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ...0.370.0690540.069054
83D:\\Marc\\Marc_human_timepoint_data_Aug2024\\60h ...0.660.0690540.069054
84D:\\Marc\\Marc_human_timepoint_data_Aug2024\\60h ...0.660.0690540.069054
85D:\\Marc\\Marc_human_timepoint_data_Aug2024\\60h ...0.660.0690540.069054
\n", + "
" + ], + "text/plain": [ + " path pixel_size_z \\\n", + "0 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.37 \n", + "1 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.38 \n", + "2 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.38 \n", + "3 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.38 \n", + "4 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.38 \n", + "5 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.38 \n", + "6 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.38 \n", + "7 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.38 \n", + "8 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.38 \n", + "9 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.37 \n", + "10 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.37 \n", + "11 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.37 \n", + "12 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E8.... 0.37 \n", + "13 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "14 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "15 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "16 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "17 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "18 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "19 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "20 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "21 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "22 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "23 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.38 \n", + "24 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.37 \n", + "25 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.37 \n", + "26 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.37 \n", + "27 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.37 \n", + "28 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.37 \n", + "29 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.37 \n", + "30 D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\E9.... 0.37 \n", + "31 D:\\Marc\\Marc_genistein_droplets_Aug2024\\F1G4 d... 0.37 \n", + "32 D:\\Marc\\Marc_genistein_droplets_Aug2024\\F1G4 d... 0.37 \n", + "33 D:\\Marc\\Marc_genistein_droplets_Aug2024\\F1G4 d... 0.37 \n", + "34 D:\\Marc\\Marc_genistein_droplets_Aug2024\\GFP-NL... 0.40 \n", + "35 D:\\Marc\\Marc_genistein_droplets_Aug2024\\GFP-NL... 0.40 \n", + "36 D:\\Marc\\Marc_genistein_droplets_Aug2024\\GFP-NL... 0.40 \n", + "37 D:\\Marc\\Marc_genistein_droplets_Aug2024\\GFP-NL... 0.40 \n", + "38 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.48 \n", + "39 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "40 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "41 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "42 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "43 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "44 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "45 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "46 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "47 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "48 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\F1G4... 0.37 \n", + "49 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "50 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "51 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "52 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "53 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "54 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "55 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "56 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "57 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "58 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "59 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "60 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "61 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.37 \n", + "62 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "63 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "64 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "65 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "66 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "67 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "68 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "69 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "70 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "71 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "72 D:\\Marc\\Marc_mouse_timepoint_data_Aug2024\\GFP-... 0.40 \n", + "73 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "74 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "75 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "76 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "77 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "78 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "79 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "80 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "81 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "82 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\48h ... 0.37 \n", + "83 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\60h ... 0.66 \n", + "84 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\60h ... 0.66 \n", + "85 D:\\Marc\\Marc_human_timepoint_data_Aug2024\\60h ... 0.66 \n", + "\n", + " pixel_size_y pixel_size_x \n", + "0 0.082864 0.082864 \n", + "1 0.103580 0.103580 \n", + "2 0.103580 0.103580 \n", + "3 0.094164 0.094164 \n", + "4 0.082864 0.082864 \n", + "5 0.082864 0.082864 \n", + "6 0.103580 0.103580 \n", + "7 0.082864 0.082864 \n", + "8 0.082864 0.082864 \n", + "9 0.090070 0.090070 \n", + "10 0.103580 0.103580 \n", + "11 0.115089 0.115089 \n", + "12 0.103580 0.103580 \n", + "13 0.115089 0.115089 \n", + "14 0.115089 0.115089 \n", + "15 0.115089 0.115089 \n", + "16 0.103580 0.103580 \n", + "17 0.115089 0.115089 \n", + "18 0.115089 0.115089 \n", + "19 0.103580 0.103580 \n", + "20 0.129475 0.129475 \n", + "21 0.115089 0.115089 \n", + "22 0.115089 0.115089 \n", + "23 0.115089 0.115089 \n", + "24 0.147972 0.147972 \n", + "25 0.115089 0.115089 \n", + "26 0.138107 0.138107 \n", + "27 0.172634 0.172634 \n", + "28 0.138107 0.138107 \n", + "29 0.115089 0.115089 \n", + "30 0.103580 0.103580 \n", + "31 0.103580 0.103580 \n", + "32 0.103580 0.103580 \n", + "33 0.094164 0.094164 \n", + "34 0.082864 0.082864 \n", + "35 0.069054 0.069054 \n", + "36 0.082864 0.082864 \n", + "37 0.082864 0.082864 \n", + "38 0.079677 0.079677 \n", + "39 0.103580 0.103580 \n", + "40 0.082864 0.082864 \n", + "41 0.094164 0.094164 \n", + "42 0.103580 0.103580 \n", + "43 0.109032 0.109032 \n", + "44 0.115089 0.115089 \n", + "45 0.103580 0.103580 \n", + "46 0.082864 0.082864 \n", + "47 0.103580 0.103580 \n", + "48 0.103580 0.103580 \n", + "49 0.103580 0.103580 \n", + "50 0.082864 0.082864 \n", + "51 0.069054 0.069054 \n", + "52 0.069054 0.069054 \n", + "53 0.069054 0.069054 \n", + "54 0.103580 0.103580 \n", + "55 0.069054 0.069054 \n", + "56 0.069054 0.069054 \n", + "57 0.069054 0.069054 \n", + "58 0.069054 0.069054 \n", + "59 0.073986 0.073986 \n", + "60 0.094164 0.094164 \n", + "61 0.073986 0.073986 \n", + "62 0.103580 0.103580 \n", + "63 0.103580 0.103580 \n", + "64 0.073986 0.073986 \n", + "65 0.069054 0.069054 \n", + "66 0.069054 0.069054 \n", + "67 0.069054 0.069054 \n", + "68 0.082864 0.082864 \n", + "69 0.076726 0.076726 \n", + "70 0.069054 0.069054 \n", + "71 0.069054 0.069054 \n", + "72 0.069054 0.069054 \n", + "73 0.073986 0.073986 \n", + "74 0.069054 0.069054 \n", + "75 0.073986 0.073986 \n", + "76 0.069054 0.069054 \n", + "77 0.069054 0.069054 \n", + "78 0.069054 0.069054 \n", + "79 0.076726 0.076726 \n", + "80 0.069054 0.069054 \n", + "81 0.069054 0.069054 \n", + "82 0.069054 0.069054 \n", + "83 0.069054 0.069054 \n", + "84 0.069054 0.069054 \n", + "85 0.069054 0.069054 " + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "paths = [\n", + " os.path.join(r'D:\\Marc', 'Marc_mouse_embryo_droplets_Aug2024'),\n", + " os.path.join(r'D:\\Marc', 'Marc_genistein_droplets_Aug2024'),\n", + " os.path.join(r'D:\\Marc', 'Marc_mouse_timepoint_data_Aug2024'),\n", + " os.path.join(r'D:\\Marc', 'Marc_human_timepoint_data_Aug2024')\n", + "]\n", + "\n", + "files_to_analyze = []\n", + "\n", + "for path in paths:\n", + " for root, subdirs, filenames in os.walk(path):\n", + " for filename in filenames:\n", + " if filename.endswith('.czi'):\n", + " files_to_analyze.append(os.path.join(root, filename))\n", + "\n", + "data_records = pd.DataFrame()\n", + "data_records['path'] = files_to_analyze\n", + "data_records['pixel_size_z'] = data_records['path'].apply(lambda x: BioImage(x).physical_pixel_sizes[0])\n", + "data_records['pixel_size_y'] = data_records['path'].apply(lambda x: BioImage(x).physical_pixel_sizes[1])\n", + "data_records['pixel_size_x'] = data_records['path'].apply(lambda x: BioImage(x).physical_pixel_sizes[2])\n", + "\n", + "# show all rows\n", + "pd.set_option('display.max_rows', None)\n", + "data_records" + ] + }, + { + "cell_type": "markdown", + "id": "11f155c6", + "metadata": {}, + "source": [ + "## Set up output folder\n", + "\n", + "Select a folder where you would like to store the results:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "ad934956", + "metadata": {}, + "outputs": [], + "source": [ + "output_folder = r'D:\\Marc\\Marc_mouse_embryo_droplets_Aug2024\\stress_analysis_2024-08-21'" + ] + }, + { + "cell_type": "markdown", + "id": "b57e9a6f", + "metadata": {}, + "source": [ + "\n", + "Let's pick a sample and put it to napari to determine feasible parameters for the analysis." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9cb1afe7", + "metadata": {}, + "outputs": [], + "source": [ + "image = BioImage(r'D:\\\\Marc\\\\Marc_mouse_embryo_droplets_Aug2024\\\\E8.5 embryos\\\\Mouse embryo E8.5_6_tailbud_droplets_40X_droplet1.czi')\n", + "image_data = image.get_image_data()\n", + "\n", + "for idx, ch in enumerate(image.channel_names):\n", + " viewer.add_image(image_data[:, idx], name=ch, scale=[1] + list(image.physical_pixel_sizes), blending='additive')" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "dab632d6-e1fe-4265-b6dd-c0739906ca17", + "metadata": {}, + "source": [ + "### Analysis parameters\n", + "\n", + "In case you ran the reconstruction previously interactively from the napari viewer (as explained [here](toolboxes:droplet_reconstruction:interactive)) and exported the settings, you can import the settings here, too. To do so, simply uncomment the line below (remove the `#`) and provide the path to the saved settings file:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "bc558d67-0cb5-40c9-aa41-8a90548a43ea", + "metadata": {}, + "outputs": [], + "source": [ + "reconstruction_parameters = utils.import_settings(file_name=os.path.join(output_folder, 'reconstruction_settings.yaml'))\n", + "reconstruction_parameters['return_intermediate_results'] = False\n", + "\n", + "measurement_parameters = {\n", + " 'max_degree': 20, # spherical harmonics degree\n", + " 'n_quadrature_points': 590, # number of quadrature points to measure on (maximum is 5810)\n", + " 'gamma': 1.1} # interfacial tension of droplet\n", + "alpha = 0.05 # lower and upper boundary in cumulative distribution function which should be used to calculate the stress anisotropy" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "aa170d40-5d31-4298-9637-27162d9478db", + "metadata": {}, + "source": [ + "## Analysis\n", + "\n", + "In this step, we put the entire analysis as demonstrated in [this notebook](toolboxes:analyze_everything) into a single function. This function will then be called in a parallel fashion on all the datasets to make analysis faster.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "567a32fe", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Processing D:\\Marc\\Marc_human_timepoint_data_Aug2024\\60h samples\\H2B-GFP p33 800 cells Gastruloid 60h_1_droplet3.czi: 100%|██████████| 86/86 [18:18<00:00, 12.77s/it] \n" + ] + } + ], + "source": [ + "import tqdm\n", + "\n", + "jobs = []\n", + "iterator = tqdm.tqdm(data_records.iterrows(), total=len(data_records))\n", + "for idx, row in iterator:\n", + " iterator.set_description(f'Processing {row[\"path\"]}')\n", + "\n", + " # there's something weird with this sample\n", + " if idx== 4:\n", + " continue\n", + " # load data\n", + " image = BioImage(row['path'])\n", + " reconstruction_parameters['voxelsize'] = np.asarray(image.physical_pixel_sizes)\n", + " reconstruction_parameters['use_dask'] = False\n", + " raw_image = image.get_image_data().squeeze()[:, 1]\n", + "\n", + " jobs.append(client.submit(reconstruction.reconstruct_droplet,\n", + " raw_image,\n", + " **reconstruction_parameters))" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "ee6e6adc", + "metadata": {}, + "outputs": [], + "source": [ + "results_reconstruction = []\n", + "for job in jobs:\n", + " try:\n", + " result = client.gather(job)\n", + " except Exception as e:\n", + " result = None\n", + " results_reconstruction.append(result)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "26777755", + "metadata": {}, + "outputs": [], + "source": [ + "_ = stress_backend.lbdv_info(Max_SPH_Deg=measurement_parameters['max_degree'],\n", + " Num_Quad_Pts=measurement_parameters['n_quadrature_points'])\n", + "\n", + "jobs_measurement = []\n", + "for result_reconstruction_single in results_reconstruction:\n", + " if result_reconstruction_single is None:\n", + " jobs_measurement.append(None)\n", + " continue\n", + " jobs_measurement.append(client.submit(measurements.comprehensive_analysis, result_reconstruction_single[1][0], **measurement_parameters))" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "id": "2e160c4b", + "metadata": {}, + "outputs": [], + "source": [ + "finished_jobs = []\n", + "for job in jobs_measurement:\n", + " if job is None:\n", + " finished_jobs.append(None)\n", + " continue\n", + " if job.status == 'finished':\n", + " finished_jobs.append(job)\n", + " else:\n", + " finished_jobs.append(None)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "id": "c90a54ca", + "metadata": {}, + "outputs": [], + "source": [ + "results_measurement = client.gather(finished_jobs)" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "id": "e5e62d48", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reconstructed droplets: 85, measured droplets: 85\n" + ] + } + ], + "source": [ + "print(f'Reconstructed droplets: {len(results_reconstruction)}, measured droplets: {len(results_measurement)}')" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "567245aa", + "metadata": {}, + "outputs": [], + "source": [ + "viewer.layers.clear()\n", + "\n", + "index_visualize = 0\n", + "image_raw = io.imread(data_records.iloc[index_visualize]['path']).squeeze()\n", + "scale = np.asarray(AICSImage(data_records.iloc[index_visualize]['path'], reader=BioformatsReader).physical_pixel_sizes)\n", + "viewer.add_image(image_raw, scale=scale)\n", + "\n", + "for res in results_reconstruction[index_visualize] + results_measurement[index_visualize]:\n", + " layer = napari.layers.Layer.create(res[0], res[1], res[2])\n", + " viewer.add_layer(layer)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "id": "adf3125e", + "metadata": {}, + "source": [ + "To make handling further down easier, we store all data and metadata in a few simple dataframes" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "id": "e23bbc53", + "metadata": {}, + "outputs": [], + "source": [ + "root_export = r'D:\\Marc\\stress_analysis_2024-08-21'" + ] + }, + { + "cell_type": "code", + "execution_count": 65, + "id": "c94e39d2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Error processing D:\\Marc\\Marc_genistein_droplets_Aug2024\\GFP-NLS data\\20240728_Genistein_Gastruloid_102h_1_droplet.czi: object of type 'numpy.float64' has no len()\n" + ] + } + ], + "source": [ + "df_to_export = pd.DataFrame()\n", + "\n", + "for result, result_reconstruction, (idx, row) in zip(results_measurement, results_reconstruction, data_records.iterrows()):\n", + " if result is None:\n", + " continue\n", + "\n", + " try:\n", + " # Compile data\n", + " n_frames = len(np.unique(result[0][0][:, 0]))\n", + " df_over_time, df_nearest_pairs, df_all_pairs, df_autocorrelations = utils.compile_data_from_layers(\n", + " result, n_frames=n_frames, time_step=0)\n", + " \n", + " # create folder for results in separate directory with same name as the original data\n", + " folder_name = os.path.join(root_export, str(Path(row['path']).stem))\n", + " os.makedirs(folder_name, exist_ok=True)\n", + "\n", + " # remove non-scalar properties and save separately\n", + " ellipsoid_contribution_matrix = np.stack(df_over_time['Elipsoid_deviation_contribution_matrix'].values)\n", + " io.imsave(os.path.join(folder_name, 'ellipsoid_contribution_matrix.tif'), ellipsoid_contribution_matrix)\n", + " df_over_time.drop(columns='Elipsoid_deviation_contribution_matrix', inplace=True)\n", + "\n", + " Tissue_stress_tensor_cartesian = np.stack(df_over_time['Tissue_stress_tensor_cartesian'].values)\n", + " np.save(os.path.join(folder_name, 'Tissue_stress_tensor_cartesian.npy'), Tissue_stress_tensor_cartesian)\n", + " df_over_time.drop(columns='Tissue_stress_tensor_cartesian', inplace=True)\n", + "\n", + " Tissue_stress_tensor_elliptical = np.stack(df_over_time['Tissue_stress_tensor_elliptical'].values)\n", + " np.save(os.path.join(folder_name, 'Tissue_stress_tensor_elliptical.npy'), Tissue_stress_tensor_elliptical)\n", + " df_over_time.drop(columns='Tissue_stress_tensor_elliptical', inplace=True)\n", + "\n", + " df_over_time.drop(columns=['stress_cell_all_pair_distance',\n", + " 'autocorrelations_spatial_total',\n", + " 'autocorrelations_spatial_cell',\n", + " 'autocorrelations_spatial_tissue',\n", + " 'stress_cell_nearest_pair_anisotropy',\n", + " 'stress_cell_nearest_pair_distance',\n", + " 'stress_cell_all_pair_anisotropy'], inplace=True)\n", + " df_over_time['sample'] = Path(row['path']).stem\n", + " df_nearest_pairs['sample'] = Path(row['path']).stem\n", + " df_all_pairs['sample'] = Path(row['path']).stem\n", + " df_autocorrelations['sample'] = Path(row['path']).stem\n", + "\n", + " df_over_time.to_csv(os.path.join(folder_name, 'results_over_time.csv'), index=False)\n", + " df_nearest_pairs.to_csv(os.path.join(folder_name, 'results_nearest_pairs.csv'), index=False)\n", + " df_all_pairs.to_csv(os.path.join(folder_name, 'results_all_pairs.csv'), index=False)\n", + " df_autocorrelations.to_csv(os.path.join(folder_name, 'results_autocorrelations.csv'), index=False)\n", + "\n", + " df_over_time['sample'] = Path(row['path']).stem\n", + " df_to_export = pd.concat([df_to_export, df_over_time], axis=0)\n", + "\n", + " # export pointclouds\n", + " except Exception as e:\n", + " print(f'Error processing {row[\"path\"]}: {e}')" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "id": "e8932a65", + "metadata": {}, + "outputs": [], + "source": [ + "df_to_export.to_csv(os.path.join(root_export, 'results_over_time_all_samples.csv'), index=False) " + ] + }, + { + "cell_type": "code", + "execution_count": 77, + "id": "85e541f1", + "metadata": {}, + "outputs": [], + "source": [ + "utils.export_settings(reconstruction_parameters, file_name=os.path.join(root_export, 'reconstruction_settings.yaml'))\n", + "utils.export_settings(measurement_parameters, file_name=os.path.join(root_export, 'measurement_settings.yaml'))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "stress", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.19" + }, + "vscode": { + "interpreter": { + "hash": "e94545d40d04f852d50c4a3e54178cb617d42c4dde55aeec8654f84a500b04b2" + } + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 52538dece440359fc5dbbb2adc9cd87199b1aada Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:20:53 +0200 Subject: [PATCH 2/5] added contents of batch processing to toc --- .../demo_analyze_everything_batch.ipynb | 14 +++++++------- .../01_toolboxes/reconstruction_settings.yaml | 17 +++++++++++++++++ docs/_toc.yml | 1 + 3 files changed, 25 insertions(+), 7 deletions(-) create mode 100644 docs/01_code_usage/01_toolboxes/reconstruction_settings.yaml diff --git a/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb index acac2c4f..12602c73 100644 --- a/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb +++ b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb @@ -6,11 +6,15 @@ "id": "75bf2599-5a29-4dfc-b0b0-7cecc3c7ab33", "metadata": {}, "source": [ - "(toolboxes:analyze_everything)=\n", + "(toolboxes:analyze_everything_batch)=\n", "# Analyze everything\n", "\n", "This notebook demonstrates how to run a complete STRESS analysis in a batched way. This requires you to assemble a list of complete filenames and then submit all of them for processing. It is strongly required to do this on a sufficiently powerful processing PC.\n", "\n", + "```{note}\n", + "Visualizing the results is not part of this tutorial. To find out more, check the [analyze-everything](toolboxes:analyze_everything) tutorial.\n", + "```\n", + "\n", "## Additional dependencies\n", "\n", "In case you want to work with proprietory file formats (e.g., czi, nd2, etc), you'll need to install some packages that can handle them. As a one-fits-all solution, install the bioio package. To find out exactly what extension of bioio you need, [check the documentation](https://pypi.org/project/bioio/). For instance, for working with czi images you'll need these dependencies:\n", @@ -29,7 +33,7 @@ "source": [ "import napari\n", "import numpy as np\n", - "from napari_stress import reconstruction, measurements, TimelapseConverter, utils, stress_backend\n", + "from napari_stress import reconstruction, measurements, utils, stress_backend\n", "import os\n", "from pathlib import Path\n", "import datetime\n", @@ -38,11 +42,7 @@ "from dask.distributed import Client, get_client\n", "\n", "import pandas as pd\n", - "import seaborn as sns\n", - "import matplotlib.pyplot as plt\n", - "import matplotlib as mpl\n", - "from skimage import io\n", - "import napari_vedo_bridge as nvb" + "from skimage import io" ] }, { diff --git a/docs/01_code_usage/01_toolboxes/reconstruction_settings.yaml b/docs/01_code_usage/01_toolboxes/reconstruction_settings.yaml new file mode 100644 index 00000000..1e2e15ab --- /dev/null +++ b/docs/01_code_usage/01_toolboxes/reconstruction_settings.yaml @@ -0,0 +1,17 @@ +edge_type: interior +fit_type: fancy +interpolation_method: linear +n_points: 256 +n_smoothing_iterations: 15 +n_tracing_iterations: 2 +outlier_tolerance: 1.5 +remove_outliers: true +resampling_length: 10.0 +sampling_distance: 2.0 +smoothing_sigma: 2.0 +target_voxelsize: 0.23 +trace_length: 80.0 +voxelsize: !!ndarray +- 0.37 +- 0.0901 +- 0.0901 diff --git a/docs/_toc.yml b/docs/_toc.yml index 78cfc297..b03f2647 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -18,6 +18,7 @@ parts: - file: 01_code_usage/01_toolboxes/demo_droplet_reconstruction_toolbox_code - file: 01_code_usage/01_toolboxes/demo_anisotropic_stresses - file: 01_code_usage/01_toolboxes/demo_analyze_everything + - file: 01_code_usage/01_toolboxes/demo_analyze_everything_batch - file: 01_code_usage/02_example_workflows/Readme sections: From 83fe963610cfabd2a5cfdd68a91e50871408a37a Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:23:25 +0200 Subject: [PATCH 3/5] removed unnused dependency --- .../01_toolboxes/demo_analyze_everything_batch.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb index 12602c73..07b97ce0 100644 --- a/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb +++ b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb @@ -36,7 +36,6 @@ "from napari_stress import reconstruction, measurements, utils, stress_backend\n", "import os\n", "from pathlib import Path\n", - "import datetime\n", "from bioio import BioImage\n", "\n", "from dask.distributed import Client, get_client\n", From 63e99bd1caafb494f31a2d04d7d9ab2c253d41f5 Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:45:32 +0200 Subject: [PATCH 4/5] added export of pointclouds and features to nnotebook --- .../demo_analyze_everything_batch.ipynb | 70 +++++++++++++++++-- 1 file changed, 66 insertions(+), 4 deletions(-) diff --git a/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb index 07b97ce0..f98611ab 100644 --- a/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb +++ b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb @@ -26,14 +26,14 @@ }, { "cell_type": "code", - "execution_count": 71, + "execution_count": 82, "id": "92cdbcda-ad6d-4048-a1eb-f2a452969326", "metadata": {}, "outputs": [], "source": [ "import napari\n", "import numpy as np\n", - "from napari_stress import reconstruction, measurements, utils, stress_backend\n", + "from napari_stress import reconstruction, measurements, utils, stress_backend, TimelapseConverter\n", "import os\n", "from pathlib import Path\n", "from bioio import BioImage\n", @@ -41,7 +41,18 @@ "from dask.distributed import Client, get_client\n", "\n", "import pandas as pd\n", - "from skimage import io" + "from skimage import io\n", + "import vedo" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "id": "ac43a0f8", + "metadata": {}, + "outputs": [], + "source": [ + "Converter = TimelapseConverter()" ] }, { @@ -1193,7 +1204,7 @@ }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 119, "id": "c94e39d2", "metadata": {}, "outputs": [ @@ -1255,11 +1266,62 @@ " df_over_time['sample'] = Path(row['path']).stem\n", " df_to_export = pd.concat([df_to_export, df_over_time], axis=0)\n", "\n", + " pointclouds_reconstruction = Converter._ldtuple_to_list_of_ldtuple(results_reconstruction[0][1])\n", + " for t_idx, pointcloud in enumerate(pointclouds_reconstruction):\n", + " pointcloud_vedo = vedo.Points(pointcloud[0])\n", + " vedo.save(pointcloud_vedo, os.path.join(folder_name, f'reconstructed_points_t{t_idx}.vtk'))\n", + "\n", + " # same for result[0]\n", + " pointclouds = Converter._ldtuple_to_list_of_ldtuple(result[0])\n", + " for t_idx, pointcloud in enumerate(pointclouds):\n", + " pointcloud_vedo = vedo.Points(pointcloud[0])\n", + " vedo.save(pointcloud_vedo, os.path.join(folder_name, f'spherical_harmonics_expansion_t{t_idx}.vtk'))\n", + "\n", + " # same for result[3]\n", + " pointclouds = Converter._ldtuple_to_list_of_ldtuple(result[3])\n", + " for t_idx, pointcloud in enumerate(pointclouds):\n", + " pointcloud_vedo = vedo.Points(pointcloud[0])\n", + " for col in pointcloud[1]['features'].columns:\n", + " pointcloud_vedo.pointdata[col] = pointcloud[1]['features'][col]\n", + " vedo.save(pointcloud_vedo, os.path.join(folder_name, f'lebedev_points_ellipsoid_t{t_idx}.vtk'))\n", + "\n", + " # same for result[4]\n", + " pointclouds = Converter._ldtuple_to_list_of_ldtuple(result[4])\n", + " for t_idx, pointcloud in enumerate(pointclouds):\n", + " pointcloud_vedo = vedo.Points(pointcloud[0])\n", + " for col in pointcloud[1]['features'].columns:\n", + " pointcloud_vedo.pointdata[col] = pointcloud[1]['features'][col]\n", + " vedo.save(pointcloud_vedo, os.path.join(folder_name, f'lebedev_points_t{t_idx}.vtk'))\n", + "\n", " # export pointclouds\n", " except Exception as e:\n", " print(f'Error processing {row[\"path\"]}: {e}')" ] }, + { + "cell_type": "code", + "execution_count": 122, + "id": "671298b8", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0 0.000004\n", + "1 0.000003\n", + "2 0.000008\n", + "Name: Gauss_Bonnet_relative_error, dtype: float64" + ] + }, + "execution_count": 122, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_over_time['Gauss_Bonnet_relative_error']" + ] + }, { "cell_type": "code", "execution_count": 76, From 97751334d9047f71896d3220a273e0f669c4ee47 Mon Sep 17 00:00:00 2001 From: Johannes Soltwedel <38459088+jo-mueller@users.noreply.github.com> Date: Mon, 26 Aug 2024 13:45:53 +0200 Subject: [PATCH 5/5] removed unneccessary cell --- .../demo_analyze_everything_batch.ipynb | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb index f98611ab..4e366b8d 100644 --- a/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb +++ b/docs/01_code_usage/01_toolboxes/demo_analyze_everything_batch.ipynb @@ -1298,30 +1298,6 @@ " print(f'Error processing {row[\"path\"]}: {e}')" ] }, - { - "cell_type": "code", - "execution_count": 122, - "id": "671298b8", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "0 0.000004\n", - "1 0.000003\n", - "2 0.000008\n", - "Name: Gauss_Bonnet_relative_error, dtype: float64" - ] - }, - "execution_count": 122, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df_over_time['Gauss_Bonnet_relative_error']" - ] - }, { "cell_type": "code", "execution_count": 76,