From 4dab4eae366776487b8d55ca5c61fb4241b9c6b8 Mon Sep 17 00:00:00 2001 From: C-Achard Date: Tue, 17 Dec 2024 08:27:04 +0100 Subject: [PATCH] Fix use of deprecated arg in colab training --- .../dev_scripts/colab_training.py | 2 +- notebooks/Colab_WNet3D_training.ipynb | 332 +++++++++++------- 2 files changed, 198 insertions(+), 136 deletions(-) diff --git a/napari_cellseg3d/dev_scripts/colab_training.py b/napari_cellseg3d/dev_scripts/colab_training.py index a5020fec..ce30d013 100644 --- a/napari_cellseg3d/dev_scripts/colab_training.py +++ b/napari_cellseg3d/dev_scripts/colab_training.py @@ -330,7 +330,7 @@ def train( wandb.init( config=config_dict, project="CellSeg3D (Colab)", - name=f"{self.config.model_info.name} training - {utils.get_date_time()}", + name=f"WNet3D training - {utils.get_date_time()}", mode=self.wandb_config.mode, tags=["WNet3D", "Colab"], ) diff --git a/notebooks/Colab_WNet3D_training.ipynb b/notebooks/Colab_WNet3D_training.ipynb index 0c5ed172..3aa52f46 100644 --- a/notebooks/Colab_WNet3D_training.ipynb +++ b/notebooks/Colab_WNet3D_training.ipynb @@ -1,27 +1,10 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": { - "colab": { - "provenance": [], - "gpuType": "T4", - "include_colab_link": true - }, - "kernelspec": { - "name": "python3", - "display_name": "Python 3" - }, - "language_info": { - "name": "python" - }, - "accelerator": "GPU" - }, "cells": [ { "cell_type": "markdown", "metadata": { - "id": "view-in-github", - "colab_type": "text" + "colab_type": "text", + "id": "view-in-github" }, "source": [ "\"Open" @@ -29,6 +12,9 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "BTUVNXX7R3Go" + }, "source": [ "# **WNet3D: self-supervised 3D cell segmentation**\n", "\n", @@ -37,20 +23,17 @@ "This notebook is part of the [CellSeg3D project](https://github.com/AdaptiveMotorControlLab/CellSeg3d) in the [Mathis Lab of Adaptive Intelligence](https://www.mackenziemathislab.org/).\n", "\n", "- 💜 The foundation of this notebook owes much to the **[ZeroCostDL4Mic](https://github.com/HenriquesLab/ZeroCostDL4Mic)** project and to the **[DeepLabCut](https://github.com/DeepLabCut/DeepLabCut)** team for bringing Colab into scientific open software." - ], - "metadata": { - "id": "BTUVNXX7R3Go" - } + ] }, { "cell_type": "markdown", + "metadata": { + "id": "zmVCksV0EfVT" + }, "source": [ "#**1. Installing dependencies**\n", "---" - ], - "metadata": { - "id": "zmVCksV0EfVT" - } + ] }, { "cell_type": "code", @@ -66,22 +49,17 @@ }, { "cell_type": "markdown", + "metadata": { + "id": "nqctRognFGDT" + }, "source": [ "##**1.2 Load key dependencies**\n", "---" - ], - "metadata": { - "id": "nqctRognFGDT" - } + ] }, { "cell_type": "code", - "source": [ - "# @title\n", - "from pathlib import Path\n", - "from napari_cellseg3d.dev_scripts import colab_training as c\n", - "from napari_cellseg3d.config import WNetTrainingWorkerConfig, WandBConfig, WeightsInfo, PRETRAINED_WEIGHTS_DIR" - ], + "execution_count": 2, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -89,55 +67,63 @@ "id": "wOOhJjkxjXz-", "outputId": "8f94416d-a482-4ec6-f980-a728e908d90d" }, - "execution_count": 2, "outputs": [ { - "output_type": "stream", "name": "stderr", + "output_type": "stream", "text": [ "INFO:napari_cellseg3d.utils:wandb not installed, wandb config will not be taken into account\n", "WARNING:napari_cellseg3d.utils:wandb not installed, wandb config will not be taken into account\n" ] } + ], + "source": [ + "# @title\n", + "from pathlib import Path\n", + "from napari_cellseg3d.dev_scripts import colab_training as c\n", + "from napari_cellseg3d.config import WNetTrainingWorkerConfig, WandBConfig, WeightsInfo, PRETRAINED_WEIGHTS_DIR" ] }, { "cell_type": "markdown", + "metadata": { + "id": "Ax-vJAWRwIKi" + }, "source": [ "## (optional) **1.3 Initialize Weights & Biases integration **\n", "---\n", "If you wish to utilize Weights & Biases (WandB) for monitoring and logging your training session, execute the cell below.\n", "To enable it, just input your API key in the space provided." - ], - "metadata": { - "id": "Ax-vJAWRwIKi" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "QNgC3awjwb7G" + }, + "outputs": [], "source": [ "!pip install -q wandb\n", "import wandb\n", "wandb.login()" - ], - "metadata": { - "id": "QNgC3awjwb7G" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "Zi9gRBHFFyX-" + }, "source": [ "# **2. Complete the Colab session**\n", "---\n" - ], - "metadata": { - "id": "Zi9gRBHFFyX-" - } + ] }, { "cell_type": "markdown", + "metadata": { + "id": "zSU-LYTfFnvF" + }, "source": [ "\n", "## **2.1. Check for GPU access**\n", @@ -150,27 +136,11 @@ "For Runtime type, ensure it's set to Python 3 (the programming language this program is written in).\n", "\n", "Under Accelerator, choose GPU (Graphics Processing Unit).\n" - ], - "metadata": { - "id": "zSU-LYTfFnvF" - } + ] }, { "cell_type": "code", - "source": [ - "#@markdown ##Execute the cell below to verify if GPU access is available.\n", - "\n", - "import torch\n", - "if not torch.cuda.is_available():\n", - " print('You do not have GPU access.')\n", - " print('Did you change your runtime?')\n", - " print('If the runtime setting is correct then Google did not allocate a GPU for your session')\n", - " print('Expect slow performance. To access GPU try reconnecting later')\n", - "\n", - "else:\n", - " print('You have GPU access')\n", - " !nvidia-smi\n" - ], + "execution_count": 3, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -178,11 +148,10 @@ "id": "Ie7bXiMgFtPH", "outputId": "3276444c-5109-47b4-f507-ea9acaab15ad" }, - "execution_count": 3, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "You have GPU access\n", "Fri May 3 17:19:13 2024 \n", @@ -207,10 +176,27 @@ "+---------------------------------------------------------------------------------------+\n" ] } + ], + "source": [ + "#@markdown ##Execute the cell below to verify if GPU access is available.\n", + "\n", + "import torch\n", + "if not torch.cuda.is_available():\n", + " print('You do not have GPU access.')\n", + " print('Did you change your runtime?')\n", + " print('If the runtime setting is correct then Google did not allocate a GPU for your session')\n", + " print('Expect slow performance. To access GPU try reconnecting later')\n", + "\n", + "else:\n", + " print('You have GPU access')\n", + " !nvidia-smi\n" ] }, { "cell_type": "markdown", + "metadata": { + "id": "X_bbk7RAF2yw" + }, "source": [ "## **2.2. Mount Google Drive**\n", "---\n", @@ -223,18 +209,11 @@ "3. Copy the generated authorization code and paste it into the cell, then press 'Enter'. This grants Colab access to read and write data to your Google Drive.\n", "\n", "4. After completion, you can view your data in the notebook. Simply click the Files tab on the top left and select 'Refresh'." - ], - "metadata": { - "id": "X_bbk7RAF2yw" - } + ] }, { "cell_type": "code", - "source": [ - "# mount user's Google Drive to Google Colab.\n", - "from google.colab import drive\n", - "drive.mount('/content/gdrive')" - ], + "execution_count": 4, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -242,54 +221,61 @@ "id": "AsIARCablq1V", "outputId": "77ffdbd1-4c89-4a56-e3da-7777a607a328" }, - "execution_count": 4, "outputs": [ { - "output_type": "stream", "name": "stdout", + "output_type": "stream", "text": [ "Mounted at /content/gdrive\n" ] } + ], + "source": [ + "# mount user's Google Drive to Google Colab.\n", + "from google.colab import drive\n", + "drive.mount('/content/gdrive')" ] }, { "cell_type": "markdown", + "metadata": { + "id": "r6FI22lkQLTv" + }, "source": [ "** If you cannot see your files, reactivate your session by connecting to your hosted runtime.**\n", "\n", "\n", "\"Example
Connect to a hosted runtime.
" - ], - "metadata": { - "id": "r6FI22lkQLTv" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "EtsK08ECwlnJ" + }, + "outputs": [], "source": [ "# @title\n", "# import wandb\n", "# wandb.login()" - ], - "metadata": { - "id": "EtsK08ECwlnJ" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "IkOpxYjaGM0m" + }, "source": [ "# **3. Select your parameters and paths**\n", "---" - ], - "metadata": { - "id": "IkOpxYjaGM0m" - } + ] }, { "cell_type": "markdown", + "metadata": { + "id": "65FhTkYlGKRt" + }, "source": [ "## **3.1. Choosing parameters**\n", "\n", @@ -327,13 +313,16 @@ "\n", "* **`n_cuts_weight`** is the weight of the NCuts loss in the weighted sum for the backward pass. Default: 0.5\n", "* **`rec_loss_weight`** is the weight of the reconstruction loss. Default: 0.005\n" - ], - "metadata": { - "id": "65FhTkYlGKRt" - } + ] }, { "cell_type": "code", + "execution_count": 7, + "metadata": { + "cellView": "form", + "id": "tTSCC6ChGuuA" + }, + "outputs": [], "source": [ "#@markdown ###Path to the training data:\n", "training_source = \"./gdrive/MyDrive/CELLSEG_BENCHMARK/DATA/WNET/VIP_full\" #@param {type:\"string\"}\n", @@ -368,45 +357,44 @@ "#@markdown Weighted sum of losses:\n", "n_cuts_weight = 0.5 #@param {type:\"number\"}\n", "rec_loss_weight = 0.005 #@param {type:\"number\"}" - ], - "metadata": { - "cellView": "form", - "id": "tTSCC6ChGuuA" - }, - "execution_count": 7, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [], "metadata": { "id": "HtoIo5GcKIXX" - } + }, + "source": [] }, { "cell_type": "markdown", + "metadata": { + "id": "arWhMU6aKsri" + }, "source": [ "# **4. Train the network**\n", "---\n", "\n", "Important Reminder: Google Colab imposes a maximum session time to prevent extended GPU usage, such as for data mining. Ensure your training duration stays under 12 hours. If your training is projected to exceed this limit, consider reducing the `number_of_epochs`." - ], - "metadata": { - "id": "arWhMU6aKsri" - } + ] }, { "cell_type": "markdown", + "metadata": { + "id": "L59J90S_Kva3" + }, "source": [ "## **4.1. Initialize the config**\n", "---" - ], - "metadata": { - "id": "L59J90S_Kva3" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YOgLyUwPjvUX" + }, + "outputs": [], "source": [ "# @title\n", "train_data_folder = Path(training_source)\n", @@ -463,36 +451,110 @@ " mode=\"disabled\" if not WANDB_INSTALLED else \"online\",\n", " save_model_artifact=False,\n", ")" - ], - "metadata": { - "id": "YOgLyUwPjvUX" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", + "metadata": { + "id": "idowGpeQPIm2" + }, "source": [ "## **4.2. Start training**\n", "---" - ], - "metadata": { - "id": "idowGpeQPIm2" - } + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "OXxKZhGMqguz" + }, + "outputs": [], "source": [ "# @title\n", "worker = c.get_colab_worker(worker_config=train_config, wandb_config=wandb_config)\n", "for epoch_loss in worker.train():\n", " continue" - ], - "metadata": { - "id": "OXxKZhGMqguz" - }, + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Once you are done training, you will get a .pth file in the model folder you specified.\n", + "from tiffffile import imread\n", + "from napari_cellseg3d.dev_scripts import remote_inference as cs3d\n", + "from napari_cellseg3d.config import InferenceWorkerConfig, ModelInfo, WeightsInfo, PostProcessConfig, InstanceSegConfig\n", + "from napari_cellseg3d.code_models.instance_segmentation import VoronoiOtsu\n", + "# Add image path below\n", + "demo_image_path = \"/content/CellSeg3D/examples/c5image.tif\"\n", + "demo_image = imread(demo_image_path)\n", + "inference_config = InferenceWorkerConfig(\n", + " device=\"cuda:0\",\n", + " model_info=ModelInfo(\n", + " name=\"WNet3D\",\n", + " num_classes=2,\n", + " ),\n", + " weights_config=WeightsInfo(\n", + " path=\"./path/to/your/model.pth\",\n", + " use_custom=True,\n", + " ),\n", + " results_path=\"./results\",\n", + ")\n", + "\n", + "# select cle device for colab\n", + "import pyclesperanto_prototype as cle\n", + "cle.select_device(\"cupy\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = cs3d.inference_on_images(\n", + " demo_image,\n", + " config=inference_config,\n", + ")" + ] + }, + { + "cell_type": "code", "execution_count": null, - "outputs": [] + "metadata": {}, + "outputs": [], + "source": [ + "semantic = result.semantic_segmentation\n", + "\n", + "# Save the prediction\n", + "from tifffile import imwrite\n", + "dataset = \"Current dataset here...\"\n", + "imwrite(f\"{dataset}_raw_pred.tif\", semantic)\n", + "\n", + "# This should net you the raw prediction to use in the notebooks for plots\n", + "# To make the plots and post-processing, see https://github.com/C-Achard/cellseg3d-figures/blob/main/figures/FIgure3/self-supervised-extra.ipynb\n", + "# To find threshold value, I recommend the scripts in https://github.com/C-Achard/cellseg3d-figures/blob/main/thresholds_opti/wnet_find_thresholds.ipynb\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "include_colab_link": true, + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" } - ] + }, + "nbformat": 4, + "nbformat_minor": 0 }