From 7bf29b3c4c59c6f5f637ef9fd20712d75102a4c2 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Fri, 22 Apr 2022 14:44:34 +0200
Subject: [PATCH 01/19] Add strawberry segmentation

---
 samples/strawberry/README.md     |   1 +
 samples/strawberry/strawberry.py | 314 +++++++++++++++++++++++++++++++
 2 files changed, 315 insertions(+)
 create mode 100644 samples/strawberry/README.md
 create mode 100644 samples/strawberry/strawberry.py

diff --git a/samples/strawberry/README.md b/samples/strawberry/README.md
new file mode 100644
index 0000000000..7421b57de7
--- /dev/null
+++ b/samples/strawberry/README.md
@@ -0,0 +1 @@
+# Strawberry
diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
new file mode 100644
index 0000000000..9ddd664634
--- /dev/null
+++ b/samples/strawberry/strawberry.py
@@ -0,0 +1,314 @@
+import os
+import sys
+import json
+import datetime
+import numpy as np
+import skimage.draw
+
+# Root directory of the project
+ROOT_DIR = os.path.abspath("../../")
+
+# Import Mask RCNN
+sys.path.append(ROOT_DIR)  # To find local version of the library
+from mrcnn.config import Config
+from mrcnn import model as modellib, utils
+
+# Path to trained weights file
+COCO_WEIGHTS_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")
+
+# Directory to save logs and model checkpoints, if not provided
+# through the command line argument --logs
+DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs")
+
+############################################################
+#  Configurations
+############################################################
+
+
+class StrawberryConfig(Config):
+    """Configuration for training on the strawberry dataset.
+    Derives from the base Config class and overrides some values.
+    """
+    # Give the configuration a recognizable name
+    NAME = "strawberry"
+
+    # We use a GPU with 12GB memory, which can fit two images.
+    # Adjust down if you use a smaller GPU.
+    IMAGES_PER_GPU = 1
+
+    # Number of classes (including background)
+    NUM_CLASSES = 1 + 1  # Background + strawberry
+
+    # Number of training steps per epoch
+    STEPS_PER_EPOCH = 100
+
+    # Skip detections with < 90% confidence
+    DETECTION_MIN_CONFIDENCE = 0.9
+
+
+############################################################
+#  Dataset
+############################################################
+
+class StrawberryDataset(utils.Dataset):
+
+    def load_strawberry(self, dataset_dir, subset):
+        """Load a subset of the Strawberry dataset.
+        dataset_dir: Root directory of the dataset.
+        subset: Subset to load: train or val
+        """
+        # Add classes. We have only one class to add.
+        self.add_class("strawberry", 1, "strawberry")
+
+        # Train or validation dataset?
+        assert subset in ["train", "val"]
+        dataset_dir = os.path.join(dataset_dir, subset)
+
+        annotations = json.load(open(os.path.join(dataset_dir, "labels.json")))
+        annotations = list(annotations.values())  # don't need the dict keys
+
+        # Add images
+        for a in annotations[:-1]:
+            annotation = a[0]
+            print('hi')
+            polygons = annotation['polygon']
+            width, height = 4000, 3000
+            image_name = annotation['file'].split("/")[-1]
+            image_path = os.path.join(dataset_dir, image_name)
+
+            self.add_image(
+                "strawberry",
+                image_id=image_name,  # use file name as a unique image id
+                path=image_path,
+                width=width, height=height,
+                polygons=polygons)
+
+    def load_mask(self, image_id):
+        """Generate instance masks for an image.
+       Returns:
+        masks: A bool array of shape [height, width, instance count] with
+            one mask per instance.
+        class_ids: a 1D array of class IDs of the instance masks.
+        """
+        # If not a strawberry dataset image, delegate to parent class.
+        image_info = self.image_info[image_id]
+        if image_info["source"] != "strawberry":
+            return super(self.__class__, self).load_mask(image_id)
+
+        # Convert polygons to a bitmap mask of shape
+        # [height, width, instance_count]
+        info = self.image_info[image_id]
+        mask = np.zeros([info["height"], info["width"], len(info["polygons"])],
+                        dtype=np.uint8)
+        for i, polygon in enumerate(info["polygons"]):
+            # Get indexes of pixels inside the polygon and set them to 1
+            x = [coord[0] for coord in polygon]
+            y = [coord[1] for coord in polygon]
+            rr, cc = skimage.draw.polygon(y, x)
+            mask[rr, cc, i] = 1
+
+        # Return mask, and array of class IDs of each instance. Since we have
+        # one class ID only, we return an array of 1s
+        return mask.astype(np.bool), np.ones([mask.shape[-1]], dtype=np.int32)
+
+    def image_reference(self, image_id):
+        """Return the path of the image."""
+        info = self.image_info[image_id]
+        if info["source"] == "strawberry":
+            return info["path"]
+        else:
+            super(self.__class__, self).image_reference(image_id)
+
+
+def train(model):
+    """Train the model."""
+    # Training dataset.
+    dataset_train = StrawberryDataset()
+    dataset_train.load_strawberry(args.dataset, "train")
+    dataset_train.prepare()
+
+    # Validation dataset
+    dataset_val = StrawberryDataset()
+    dataset_val.load_strawberry(args.dataset, "val")
+    dataset_val.prepare()
+
+    # *** This training schedule is an example. Update to your needs ***
+    # Since we're using a very small dataset, and starting from
+    # COCO trained weights, we don't need to train too long. Also,
+    # no need to train all layers, just the heads should do it.
+    print("Training network heads")
+    model.train(dataset_train, dataset_val,
+                learning_rate=config.LEARNING_RATE,
+                epochs=1,
+                layers='heads')
+
+
+def color_splash(image, mask):
+    """Apply color splash effect.
+    image: RGB image [height, width, 3]
+    mask: instance segmentation mask [height, width, instance count]
+
+    Returns result image.
+    """
+    # Make a grayscale copy of the image. The grayscale copy still
+    # has 3 RGB channels, though.
+    gray = skimage.color.gray2rgb(skimage.color.rgb2gray(image)) * 255
+    # Copy color pixels from the original color image where mask is set
+    if mask.shape[-1] > 0:
+        # We're treating all instances as one, so collapse the mask into one layer
+        mask = (np.sum(mask, -1, keepdims=True) >= 1)
+        splash = np.where(mask, image, gray).astype(np.uint8)
+    else:
+        splash = gray.astype(np.uint8)
+    return splash
+
+
+def detect_and_color_splash(model, image_path=None, video_path=None):
+    assert image_path or video_path
+
+    # Image or video?
+    if image_path:
+        # Run model detection and generate the color splash effect
+        print("Running on {}".format(args.image))
+        # Read image
+        image = skimage.io.imread(args.image)
+        # Detect objects
+        r = model.detect([image], verbose=1)[0]
+        # Color splash
+        splash = color_splash(image, r['masks'])
+        # Save output
+        file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
+        skimage.io.imsave(file_name, splash)
+    elif video_path:
+        import cv2
+        # Video capture
+        vcapture = cv2.VideoCapture(video_path)
+        width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
+        height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
+        fps = vcapture.get(cv2.CAP_PROP_FPS)
+
+        # Define codec and create video writer
+        file_name = "splash_{:%Y%m%dT%H%M%S}.avi".format(datetime.datetime.now())
+        vwriter = cv2.VideoWriter(file_name,
+                                  cv2.VideoWriter_fourcc(*'MJPG'),
+                                  fps, (width, height))
+
+        count = 0
+        success = True
+        while success:
+            print("frame: ", count)
+            # Read next image
+            success, image = vcapture.read()
+            if success:
+                # OpenCV returns images as BGR, convert to RGB
+                image = image[..., ::-1]
+                # Detect objects
+                r = model.detect([image], verbose=0)[0]
+                # Color splash
+                splash = color_splash(image, r['masks'])
+                # RGB -> BGR to save image to video
+                splash = splash[..., ::-1]
+                # Add image to video writer
+                vwriter.write(splash)
+                count += 1
+        vwriter.release()
+    print("Saved to ", file_name)
+
+
+############################################################
+#  Training
+############################################################
+
+if __name__ == '__main__':
+    import argparse
+
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(
+        description='Train Mask R-CNN to detect strawberries.')
+    parser.add_argument("command",
+                        metavar="<command>",
+                        help="'train' or 'splash'")
+    parser.add_argument('--dataset', required=False,
+                        metavar="/path/to/strawberry/dataset/",
+                        help='Directory of the Strawberry dataset')
+    parser.add_argument('--weights', required=True,
+                        metavar="/path/to/weights.h5",
+                        help="Path to weights .h5 file or 'coco'")
+    parser.add_argument('--logs', required=False,
+                        default=DEFAULT_LOGS_DIR,
+                        metavar="/path/to/logs/",
+                        help='Logs and checkpoints directory (default=logs/)')
+    parser.add_argument('--image', required=False,
+                        metavar="path or URL to image",
+                        help='Image to apply the color splash effect on')
+    parser.add_argument('--video', required=False,
+                        metavar="path or URL to video",
+                        help='Video to apply the color splash effect on')
+    args = parser.parse_args()
+
+    # Validate arguments
+    if args.command == "train":
+        assert args.dataset, "Argument --dataset is required for training"
+    elif args.command == "splash":
+        assert args.image or args.video,\
+               "Provide --image or --video to apply color splash"
+
+    print("Weights: ", args.weights)
+    print("Dataset: ", args.dataset)
+    print("Logs: ", args.logs)
+
+    # Configurations
+    if args.command == "train":
+        config = StrawberryConfig()
+    else:
+        class InferenceConfig(StrawberryConfig):
+            # Set batch size to 1 since we'll be running inference on
+            # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
+            GPU_COUNT = 1
+            IMAGES_PER_GPU = 1
+        config = InferenceConfig()
+    config.display()
+
+    # Create model
+    if args.command == "train":
+        model = modellib.MaskRCNN(mode="training", config=config,
+                                  model_dir=args.logs)
+    else:
+        model = modellib.MaskRCNN(mode="inference", config=config,
+                                  model_dir=args.logs)
+
+    # Select weights file to load
+    if args.weights.lower() == "coco":
+        weights_path = COCO_WEIGHTS_PATH
+        # Download weights file
+        if not os.path.exists(weights_path):
+            utils.download_trained_weights(weights_path)
+    elif args.weights.lower() == "last":
+        # Find last trained weights
+        weights_path = model.find_last()
+    elif args.weights.lower() == "imagenet":
+        # Start from ImageNet trained weights
+        weights_path = model.get_imagenet_weights()
+    else:
+        weights_path = args.weights
+
+    # Load weights
+    print("Loading weights ", weights_path)
+    if args.weights.lower() == "coco":
+        # Exclude the last layers because they require a matching
+        # number of classes
+        model.load_weights(weights_path, by_name=True, exclude=[
+            "mrcnn_class_logits", "mrcnn_bbox_fc",
+            "mrcnn_bbox", "mrcnn_mask"])
+    else:
+        model.load_weights(weights_path, by_name=True)
+
+    # Train or evaluate
+    if args.command == "train":
+        train(model)
+    elif args.command == "splash":
+        detect_and_color_splash(model, image_path=args.image,
+                                video_path=args.video)
+    else:
+        print("'{}' is not recognized. "
+              "Use 'train' or 'splash'".format(args.command))

From b4dd3ff27b7e6f6fba05e8308b895e1992e89302 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sat, 23 Apr 2022 11:02:33 +0200
Subject: [PATCH 02/19] Various fixes

---
 .gitignore                       |  3 +++
 samples/strawberry/strawberry.py | 17 +++++++++++------
 2 files changed, 14 insertions(+), 6 deletions(-)

diff --git a/.gitignore b/.gitignore
index 12da7dd781..c45302d513 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,3 +56,6 @@ pip-delete-this-directory.txt
 .venv
 venv/
 ENV/
+
+# Cees
+samples/strawberry/output/
\ No newline at end of file
diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 9ddd664634..807b660eb9 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -10,6 +10,7 @@
 
 # Import Mask RCNN
 sys.path.append(ROOT_DIR)  # To find local version of the library
+sys.path.append('../../mrcnn')  # ...Actually use local version
 from mrcnn.config import Config
 from mrcnn import model as modellib, utils
 
@@ -40,7 +41,7 @@ class StrawberryConfig(Config):
     NUM_CLASSES = 1 + 1  # Background + strawberry
 
     # Number of training steps per epoch
-    STEPS_PER_EPOCH = 100
+    STEPS_PER_EPOCH = 10
 
     # Skip detections with < 90% confidence
     DETECTION_MIN_CONFIDENCE = 0.9
@@ -64,13 +65,11 @@ def load_strawberry(self, dataset_dir, subset):
         assert subset in ["train", "val"]
         dataset_dir = os.path.join(dataset_dir, subset)
 
-        annotations = json.load(open(os.path.join(dataset_dir, "labels.json")))
+        annotations = json.load(open(os.path.join(dataset_dir, "my_labels.json")))
         annotations = list(annotations.values())  # don't need the dict keys
 
         # Add images
-        for a in annotations[:-1]:
-            annotation = a[0]
-            print('hi')
+        for annotation in annotations[0]:
             polygons = annotation['polygon']
             width, height = 4000, 3000
             image_name = annotation['file'].split("/")[-1]
@@ -101,6 +100,12 @@ def load_mask(self, image_id):
         mask = np.zeros([info["height"], info["width"], len(info["polygons"])],
                         dtype=np.uint8)
         for i, polygon in enumerate(info["polygons"]):
+            # Avoid single-dimension polygons 
+            # (i.e. [x, y] instead of [[x, y], [x, y], ...])
+            try:
+                len(polygon[0])
+            except:
+                continue
             # Get indexes of pixels inside the polygon and set them to 1
             x = [coord[0] for coord in polygon]
             y = [coord[1] for coord in polygon]
@@ -178,7 +183,7 @@ def detect_and_color_splash(model, image_path=None, video_path=None):
         splash = color_splash(image, r['masks'])
         # Save output
         file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
-        skimage.io.imsave(file_name, splash)
+        skimage.io.imsave(os.path.join('output', file_name), splash)
     elif video_path:
         import cv2
         # Video capture

From f9cc7d7fe079c750f83666d24379ba20e84fc810 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sat, 23 Apr 2022 11:39:30 +0200
Subject: [PATCH 03/19] Remove video option

---
 samples/strawberry/strawberry.py | 76 ++++++++------------------------
 1 file changed, 19 insertions(+), 57 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 807b660eb9..b40ee6f69c 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -168,56 +168,22 @@ def color_splash(image, mask):
     return splash
 
 
-def detect_and_color_splash(model, image_path=None, video_path=None):
-    assert image_path or video_path
-
-    # Image or video?
-    if image_path:
-        # Run model detection and generate the color splash effect
-        print("Running on {}".format(args.image))
-        # Read image
-        image = skimage.io.imread(args.image)
-        # Detect objects
-        r = model.detect([image], verbose=1)[0]
-        # Color splash
-        splash = color_splash(image, r['masks'])
-        # Save output
-        file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
-        skimage.io.imsave(os.path.join('output', file_name), splash)
-    elif video_path:
-        import cv2
-        # Video capture
-        vcapture = cv2.VideoCapture(video_path)
-        width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
-        height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
-        fps = vcapture.get(cv2.CAP_PROP_FPS)
-
-        # Define codec and create video writer
-        file_name = "splash_{:%Y%m%dT%H%M%S}.avi".format(datetime.datetime.now())
-        vwriter = cv2.VideoWriter(file_name,
-                                  cv2.VideoWriter_fourcc(*'MJPG'),
-                                  fps, (width, height))
-
-        count = 0
-        success = True
-        while success:
-            print("frame: ", count)
-            # Read next image
-            success, image = vcapture.read()
-            if success:
-                # OpenCV returns images as BGR, convert to RGB
-                image = image[..., ::-1]
-                # Detect objects
-                r = model.detect([image], verbose=0)[0]
-                # Color splash
-                splash = color_splash(image, r['masks'])
-                # RGB -> BGR to save image to video
-                splash = splash[..., ::-1]
-                # Add image to video writer
-                vwriter.write(splash)
-                count += 1
-        vwriter.release()
-    print("Saved to ", file_name)
+def detect_and_color_splash(model, image_path=None):
+    assert image_path
+
+    # Run model detection and generate the color splash effect
+    print("Running on {}".format(args.image))
+    # Read image
+    image = skimage.io.imread(args.image)
+    # Detect objects
+    r = model.detect([image], verbose=1)[0]
+    # Color splash
+    splash = color_splash(image, r['masks'])
+    # Save output
+    file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
+    skimage.io.imsave(os.path.join('output', file_name), splash)
+
+    print("Saved to", file_name)
 
 
 ############################################################
@@ -246,17 +212,14 @@ def detect_and_color_splash(model, image_path=None, video_path=None):
     parser.add_argument('--image', required=False,
                         metavar="path or URL to image",
                         help='Image to apply the color splash effect on')
-    parser.add_argument('--video', required=False,
-                        metavar="path or URL to video",
-                        help='Video to apply the color splash effect on')
     args = parser.parse_args()
 
     # Validate arguments
     if args.command == "train":
         assert args.dataset, "Argument --dataset is required for training"
     elif args.command == "splash":
-        assert args.image or args.video,\
-               "Provide --image or --video to apply color splash"
+        assert args.image,\
+               "Provide --image to apply color splash"
 
     print("Weights: ", args.weights)
     print("Dataset: ", args.dataset)
@@ -312,8 +275,7 @@ class InferenceConfig(StrawberryConfig):
     if args.command == "train":
         train(model)
     elif args.command == "splash":
-        detect_and_color_splash(model, image_path=args.image,
-                                video_path=args.video)
+        detect_and_color_splash(model, image_path=args.image)
     else:
         print("'{}' is not recognized. "
               "Use 'train' or 'splash'".format(args.command))

From 856de1344ce8cd0e99af571b6dea06cc3f94d841 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sat, 23 Apr 2022 13:42:34 +0200
Subject: [PATCH 04/19] Return cropped segmentated images

---
 samples/strawberry/strawberry.py | 68 +++++++++++++++++++-------------
 1 file changed, 40 insertions(+), 28 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index b40ee6f69c..3b67eade68 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -148,8 +148,8 @@ def train(model):
                 layers='heads')
 
 
-def color_splash(image, mask):
-    """Apply color splash effect.
+def segment(image, mask):
+    """Segment the image.
     image: RGB image [height, width, 3]
     mask: instance segmentation mask [height, width, instance count]
 
@@ -157,33 +157,45 @@ def color_splash(image, mask):
     """
     # Make a grayscale copy of the image. The grayscale copy still
     # has 3 RGB channels, though.
-    gray = skimage.color.gray2rgb(skimage.color.rgb2gray(image)) * 255
-    # Copy color pixels from the original color image where mask is set
-    if mask.shape[-1] > 0:
-        # We're treating all instances as one, so collapse the mask into one layer
-        mask = (np.sum(mask, -1, keepdims=True) >= 1)
-        splash = np.where(mask, image, gray).astype(np.uint8)
-    else:
-        splash = gray.astype(np.uint8)
-    return splash
-
-
-def detect_and_color_splash(model, image_path=None):
+    black = skimage.color.gray2rgb(skimage.color.rgb2gray(image)) * 255
+    # Make image completely black
+    # TODO: make them transparent instead
+    black[:, :] = 0
+    # Transpose to (nSegments, height, width)
+    mask_transpose = np.transpose(mask, [2, 0, 1])
+    segments = []
+    for mask_ in mask_transpose[0:1]:
+      # Convert (3000, 4000) to (3000, 4000, 3)
+      # TODO: find more efficient way
+      # TODO: when more efficient, remove the [0:1] above
+      mask_ = mask_ >= 1
+      arr_new = np.zeros((3000,4000,3))
+      for i, x in enumerate(mask_):
+          for j, y in enumerate(x):
+              arr_new[i][j] = [y, y, y]
+      mask_ = arr_new
+      # Use image values on mask, black otherwise
+      res = np.where(mask_, image, black).astype(np.uint8)
+      segments.append(res)
+    return segments
+
+
+def detect_and_segment(model, image_path=None):
     assert image_path
 
-    # Run model detection and generate the color splash effect
+    # Run model detection and generate segment
     print("Running on {}".format(args.image))
     # Read image
     image = skimage.io.imread(args.image)
     # Detect objects
     r = model.detect([image], verbose=1)[0]
-    # Color splash
-    splash = color_splash(image, r['masks'])
+    # Segmentation
+    segments = segment(image, r['masks'])
     # Save output
-    file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
-    skimage.io.imsave(os.path.join('output', file_name), splash)
-
-    print("Saved to", file_name)
+    for seg, roi in zip(segments, r['rois']):
+      file_name = "segment_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
+      skimage.io.imsave(os.path.join('output/aapje', file_name), seg[roi[0]:roi[2], roi[1]:roi[3]])
+      print("Saved to", file_name)
 
 
 ############################################################
@@ -198,7 +210,7 @@ def detect_and_color_splash(model, image_path=None):
         description='Train Mask R-CNN to detect strawberries.')
     parser.add_argument("command",
                         metavar="<command>",
-                        help="'train' or 'splash'")
+                        help="'train' or 'segment'")
     parser.add_argument('--dataset', required=False,
                         metavar="/path/to/strawberry/dataset/",
                         help='Directory of the Strawberry dataset')
@@ -211,15 +223,15 @@ def detect_and_color_splash(model, image_path=None):
                         help='Logs and checkpoints directory (default=logs/)')
     parser.add_argument('--image', required=False,
                         metavar="path or URL to image",
-                        help='Image to apply the color splash effect on')
+                        help='Image to apply the segmentation on')
     args = parser.parse_args()
 
     # Validate arguments
     if args.command == "train":
         assert args.dataset, "Argument --dataset is required for training"
-    elif args.command == "splash":
+    elif args.command == "segment":
         assert args.image,\
-               "Provide --image to apply color splash"
+               "Provide --image to apply segmentation"
 
     print("Weights: ", args.weights)
     print("Dataset: ", args.dataset)
@@ -274,8 +286,8 @@ class InferenceConfig(StrawberryConfig):
     # Train or evaluate
     if args.command == "train":
         train(model)
-    elif args.command == "splash":
-        detect_and_color_splash(model, image_path=args.image)
+    elif args.command == "segment":
+        detect_and_segment(model, image_path=args.image)
     else:
         print("'{}' is not recognized. "
-              "Use 'train' or 'splash'".format(args.command))
+              "Use 'train' or 'segment'".format(args.command))

From 85557c0ab9c41859ddb6b012945587f82fff36f9 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sun, 24 Apr 2022 09:13:30 +0200
Subject: [PATCH 05/19] Move cropping earlier to improve efficiency

---
 samples/strawberry/strawberry.py | 44 ++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 16 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 3b67eade68..0af5e18950 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -147,35 +147,47 @@ def train(model):
                 epochs=1,
                 layers='heads')
 
+def crop(image, roi):
+    """Crop an image to a region of interest."""
+    x1, y1, x2, y2 = roi
+    return image[x1:x2, y1:y2]
 
-def segment(image, mask):
+
+def segment(image, mask, roi):
     """Segment the image.
     image: RGB image [height, width, 3]
     mask: instance segmentation mask [height, width, instance count]
 
     Returns result image.
     """
-    # Make a grayscale copy of the image. The grayscale copy still
-    # has 3 RGB channels, though.
-    black = skimage.color.gray2rgb(skimage.color.rgb2gray(image)) * 255
-    # Make image completely black
-    # TODO: make them transparent instead
-    black[:, :] = 0
     # Transpose to (nSegments, height, width)
     mask_transpose = np.transpose(mask, [2, 0, 1])
+
     segments = []
-    for mask_ in mask_transpose[0:1]:
-      # Convert (3000, 4000) to (3000, 4000, 3)
+
+    for mask_, roi_ in zip(mask_transpose[0:2], roi[0:2]):
+      # Crop image and mask
+      image_ = crop(image, roi_)
+      mask_ = crop(mask_, roi_)
+
+      # Make a grayscale copy of the image. The grayscale copy still
+      # has 3 RGB channels, though.
+      black = skimage.color.gray2rgb(skimage.color.rgb2gray(image_)) * 255
+
+      # Make image completely black
+      # TODO: make them transparent instead
+      black[:, :] = 0
+
+      # Convert (width, height) to (width, height, 3)
       # TODO: find more efficient way
-      # TODO: when more efficient, remove the [0:1] above
-      mask_ = mask_ >= 1
-      arr_new = np.zeros((3000,4000,3))
+      arr_new = np.zeros((*mask_.shape, 3))
       for i, x in enumerate(mask_):
           for j, y in enumerate(x):
               arr_new[i][j] = [y, y, y]
       mask_ = arr_new
+
       # Use image values on mask, black otherwise
-      res = np.where(mask_, image, black).astype(np.uint8)
+      res = np.where(mask_, image_, black).astype(np.uint8)
       segments.append(res)
     return segments
 
@@ -190,11 +202,11 @@ def detect_and_segment(model, image_path=None):
     # Detect objects
     r = model.detect([image], verbose=1)[0]
     # Segmentation
-    segments = segment(image, r['masks'])
+    segments = segment(image, r['masks'], r['rois'])
     # Save output
-    for seg, roi in zip(segments, r['rois']):
+    for seg in segments:
       file_name = "segment_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
-      skimage.io.imsave(os.path.join('output/aapje', file_name), seg[roi[0]:roi[2], roi[1]:roi[3]])
+      skimage.io.imsave(os.path.join('output/aapje', file_name), seg)
       print("Saved to", file_name)
 
 

From 79370404e2f9dfe1ee1a5d9fc3e5ea5b5d96151e Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sun, 24 Apr 2022 09:29:02 +0200
Subject: [PATCH 06/19] Store segmented images in folders

---
 samples/strawberry/strawberry.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 0af5e18950..dfa748d15d 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -195,7 +195,7 @@ def segment(image, mask, roi):
 def detect_and_segment(model, image_path=None):
     assert image_path
 
-    # Run model detection and generate segment
+    # 1. Run model detection and generate segment
     print("Running on {}".format(args.image))
     # Read image
     image = skimage.io.imread(args.image)
@@ -203,10 +203,16 @@ def detect_and_segment(model, image_path=None):
     r = model.detect([image], verbose=1)[0]
     # Segmentation
     segments = segment(image, r['masks'], r['rois'])
-    # Save output
-    for seg in segments:
-      file_name = "segment_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
-      skimage.io.imsave(os.path.join('output/aapje', file_name), seg)
+
+    # 2. Save output
+    # Create output directory
+    date = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    path = os.path.join('output', date)
+    os.mkdir(path)
+    # Output each segment
+    for idx, seg in enumerate(segments):
+      file_name = args.image.split('/')[-1].split('.')[0] + '_seg' + str(idx) + '.png'
+      skimage.io.imsave(os.path.join('output', date, file_name), seg)
       print("Saved to", file_name)
 
 

From e3974cd3ef6cbc4bb78da48d028d0af1ab6a42b4 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sun, 24 Apr 2022 18:37:32 +0200
Subject: [PATCH 07/19] Small improvements

---
 samples/strawberry/strawberry.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index dfa748d15d..6f71c8d8f4 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -41,7 +41,10 @@ class StrawberryConfig(Config):
     NUM_CLASSES = 1 + 1  # Background + strawberry
 
     # Number of training steps per epoch
-    STEPS_PER_EPOCH = 10
+    STEPS_PER_EPOCH = 100
+
+    # Validation steps per epoch
+    # VALIDATION_STEPS = 50
 
     # Skip detections with < 90% confidence
     DETECTION_MIN_CONFIDENCE = 0.9
@@ -165,7 +168,7 @@ def segment(image, mask, roi):
 
     segments = []
 
-    for mask_, roi_ in zip(mask_transpose[0:2], roi[0:2]):
+    for mask_, roi_ in zip(mask_transpose, roi):
       # Crop image and mask
       image_ = crop(image, roi_)
       mask_ = crop(mask_, roi_)

From cdf614859d6fbb8f3e93905ed737170557cf6fb8 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Mon, 25 Apr 2022 15:45:00 +0200
Subject: [PATCH 08/19] Use transparent pixels in bg of seg results

---
 requirements.txt                 |  2 +-
 samples/strawberry/strawberry.py | 22 ++++++++--------------
 2 files changed, 9 insertions(+), 15 deletions(-)

diff --git a/requirements.txt b/requirements.txt
index bd64785cab..429408ac3c 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,7 +3,7 @@ scipy
 Pillow
 cython
 matplotlib
-scikit-image
+scikit-image==0.19
 tensorflow>=1.3.0
 keras>=2.0.8
 opencv-python
diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 6f71c8d8f4..6638c1282a 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -174,20 +174,14 @@ def segment(image, mask, roi):
       mask_ = crop(mask_, roi_)
 
       # Make a grayscale copy of the image. The grayscale copy still
-      # has 3 RGB channels, though.
-      black = skimage.color.gray2rgb(skimage.color.rgb2gray(image_)) * 255
-
-      # Make image completely black
-      # TODO: make them transparent instead
-      black[:, :] = 0
-
-      # Convert (width, height) to (width, height, 3)
-      # TODO: find more efficient way
-      arr_new = np.zeros((*mask_.shape, 3))
-      for i, x in enumerate(mask_):
-          for j, y in enumerate(x):
-              arr_new[i][j] = [y, y, y]
-      mask_ = arr_new
+      # has 4 RGBA channels, though.
+      black = skimage.color.gray2rgba(skimage.color.rgb2gray(image_), alpha=0) * 255
+
+      # Add alpha dim to image
+      image_ = np.dstack((image_, np.full((image_.shape[0], image_.shape[1]), 255)))
+
+      # Convert (width, height) to (width, height, 4)
+      mask_ = np.dstack((mask_, np.full((image_.shape[0], image_.shape[1], 3), 255)))
 
       # Use image values on mask, black otherwise
       res = np.where(mask_, image_, black).astype(np.uint8)

From f6b767335bd2a334d639cbcfefb4bf329a78bf43 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Mon, 25 Apr 2022 16:56:58 +0200
Subject: [PATCH 09/19] Small fix

---
 samples/strawberry/strawberry.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 6638c1282a..970b652e9c 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -181,7 +181,11 @@ def segment(image, mask, roi):
       image_ = np.dstack((image_, np.full((image_.shape[0], image_.shape[1]), 255)))
 
       # Convert (width, height) to (width, height, 4)
-      mask_ = np.dstack((mask_, np.full((image_.shape[0], image_.shape[1], 3), 255)))
+      arr_new = np.ones((*mask_.shape, 4))
+      for i, x in enumerate(mask_):
+          for j, y in enumerate(x):
+              arr_new[i][j] = [y for _ in range(4)]
+      mask_ = arr_new
 
       # Use image values on mask, black otherwise
       res = np.where(mask_, image_, black).astype(np.uint8)

From 89444808fe1c71874c5e1d708a1ef14d6063a700 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sat, 30 Apr 2022 14:30:13 +0200
Subject: [PATCH 10/19] Implement OCN segmentation

---
 samples/strawberry/strawberry.py | 51 ++++++++++++++++++++++++--------
 1 file changed, 38 insertions(+), 13 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 970b652e9c..1968b679ca 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -25,6 +25,8 @@
 #  Configurations
 ############################################################
 
+# Specify whether images are RGB or OCN.
+IMAGE_TYPE = "OCN"
 
 class StrawberryConfig(Config):
     """Configuration for training on the strawberry dataset.
@@ -38,7 +40,10 @@ class StrawberryConfig(Config):
     IMAGES_PER_GPU = 1
 
     # Number of classes (including background)
-    NUM_CLASSES = 1 + 1  # Background + strawberry
+    if IMAGE_TYPE == "RGB":
+      NUM_CLASSES = 1 + 1  # Background + strawberry
+    else:
+      NUM_CLASSES = 1 + 3 # Background + strawberry + flower + note
 
     # Number of training steps per epoch
     STEPS_PER_EPOCH = 100
@@ -46,8 +51,8 @@ class StrawberryConfig(Config):
     # Validation steps per epoch
     # VALIDATION_STEPS = 50
 
-    # Skip detections with < 90% confidence
-    DETECTION_MIN_CONFIDENCE = 0.9
+    # Skip detections with < 80% confidence
+    DETECTION_MIN_CONFIDENCE = 0.8
 
 
 ############################################################
@@ -61,8 +66,12 @@ def load_strawberry(self, dataset_dir, subset):
         dataset_dir: Root directory of the dataset.
         subset: Subset to load: train or val
         """
-        # Add classes. We have only one class to add.
+        # Add classes. For RGB, we have only one class to add.
         self.add_class("strawberry", 1, "strawberry")
+        if IMAGE_TYPE == "OCN":
+          # For OCN, also add the flower and note classes.
+          self.add_class("flower", 2, "flower")
+          self.add_class("note", 3, "note")
 
         # Train or validation dataset?
         assert subset in ["train", "val"]
@@ -78,12 +87,22 @@ def load_strawberry(self, dataset_dir, subset):
             image_name = annotation['file'].split("/")[-1]
             image_path = os.path.join(dataset_dir, image_name)
 
-            self.add_image(
-                "strawberry",
-                image_id=image_name,  # use file name as a unique image id
-                path=image_path,
-                width=width, height=height,
-                polygons=polygons)
+            if IMAGE_TYPE == "RGB":
+              self.add_image(
+                  "strawberry",
+                  image_id=image_name,  # use file name as a unique image id
+                  path=image_path,
+                  width=width, height=height,
+                  polygons=polygons)
+            else:
+              labels = [label.lower() for label in annotation['label']]
+              self.add_image(
+                  "strawberry",
+                  image_id=image_name,  # use file name as a unique image id
+                  path=image_path,
+                  width=width, height=height,
+                  polygons=polygons,
+                  annotations=labels)
 
     def load_mask(self, image_id):
         """Generate instance masks for an image.
@@ -115,9 +134,14 @@ def load_mask(self, image_id):
             rr, cc = skimage.draw.polygon(y, x)
             mask[rr, cc, i] = 1
 
-        # Return mask, and array of class IDs of each instance. Since we have
-        # one class ID only, we return an array of 1s
-        return mask.astype(np.bool), np.ones([mask.shape[-1]], dtype=np.int32)
+        if IMAGE_TYPE == "RGB":
+          # Return mask, and array of class IDs of each instance. Since we have
+          # one class ID only, we return an array of 1s
+          return mask.astype(np.bool), np.ones([mask.shape[-1]], dtype=np.int32)
+        else:
+          # Return mask, and array of class IDs of each instance.
+          class_ids = np.array([1 if a == 'strawberry' else (2 if a == 'flower' else 3) for a in info['annotations']]).astype(np.int32)
+          return mask.astype(np.bool), class_ids
 
     def image_reference(self, image_id):
         """Return the path of the image."""
@@ -202,6 +226,7 @@ def detect_and_segment(model, image_path=None):
     image = skimage.io.imread(args.image)
     # Detect objects
     r = model.detect([image], verbose=1)[0]
+    print('r class ids', r['class_ids'])
     # Segmentation
     segments = segment(image, r['masks'], r['rois'])
 

From 3e2e4be84827c0b98ce4bef8c2912f5dfd514579 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sun, 1 May 2022 16:18:45 +0200
Subject: [PATCH 11/19] Calculate IoU (not perfect yet)

---
 samples/strawberry/strawberry.py | 124 +++++++++++++++++++++++++++++--
 1 file changed, 119 insertions(+), 5 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 1968b679ca..50c857c1cc 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -26,7 +26,7 @@
 ############################################################
 
 # Specify whether images are RGB or OCN.
-IMAGE_TYPE = "OCN"
+IMAGE_TYPE = "RGB"
 
 class StrawberryConfig(Config):
     """Configuration for training on the strawberry dataset.
@@ -217,20 +217,134 @@ def segment(image, mask, roi):
     return segments
 
 
+def calculate_iou(seg, seg_bbox, mask, mask_bbox_array):
+  highest_iou = -99999999
+  for mask_bbox in mask_bbox_array:
+      # If bboxes dont overlap, return 0
+      if seg_bbox[2] < mask_bbox[0] or seg_bbox[0] > mask_bbox[2] or seg_bbox[3] < mask_bbox[1] or seg_bbox[1] > mask_bbox[3]:
+        continue
+
+      print('seg_bbox', seg_bbox)
+      print('mask_bbox', mask_bbox)
+
+      # Calculate intersection
+      x_range = [max(seg_bbox[0], mask_bbox[0]), min(seg_bbox[2], mask_bbox[2])]
+      y_range = [max(seg_bbox[1], mask_bbox[1]), min(seg_bbox[3], mask_bbox[3])]
+      intersect_size = (x_range[1] - x_range[0]) * (y_range[1] - y_range[0])
+      print('intersect_size', intersect_size)
+      intersection = 0
+      for x in range(x_range[0], x_range[1]):
+        for y in range(y_range[0], y_range[1]):
+          if mask[y, x] == seg[y, x]:
+            intersection += 1
+      # intersection /= intersect_size
+      print('intersection', intersection)
+
+      if intersection == 0:
+        print('no intersection')
+        continue
+
+      # Calculate union
+      seg_bbox_size = abs(seg_bbox[2] - seg_bbox[0]) * abs(seg_bbox[3] - seg_bbox[1])
+      print('seg_bbox_size', seg_bbox_size)
+      mask_bbox_size = abs(mask_bbox[2] - mask_bbox[0]) * abs(mask_bbox[3] - mask_bbox[1])
+      print('mask_bbox_size', mask_bbox_size)
+      seg_union = 0
+      for x in range(seg_bbox[0], seg_bbox[2]):
+        for y in range(seg_bbox[1], seg_bbox[3]):
+          if mask[y, x] == seg[y, x]:
+            seg_union += 1
+      # seg_union /= seg_bbox_size
+      print('seg_union', seg_union)
+      mask_union = 0
+      for x in range(mask_bbox[0], mask_bbox[2]):
+        for y in range(mask_bbox[1], mask_bbox[3]):
+          if mask[y, x] == seg[y, x]:
+            mask_union += 1
+      # mask_union /= mask_bbox_size
+      print('mask_union', mask_union)
+      union = seg_union + mask_union - intersection     
+      print('union', union)
+
+      if union == 0:
+        print('no union')
+        continue
+
+      cur = intersection / union
+      print('cur', cur)
+      if cur > highest_iou:
+        highest_iou = cur
+    
+  return highest_iou
+
+
 def detect_and_segment(model, image_path=None):
     assert image_path
 
-    # 1. Run model detection and generate segment
+    #
+    # 1. Run model detection
+    #
     print("Running on {}".format(args.image))
     # Read image
     image = skimage.io.imread(args.image)
     # Detect objects
     r = model.detect([image], verbose=1)[0]
     print('r class ids', r['class_ids'])
-    # Segmentation
-    segments = segment(image, r['masks'], r['rois'])
 
-    # 2. Save output
+    #
+    # 2. Calculate accuracy
+    #
+    if not len(r['masks']):
+      print('no masks')
+      return
+    transposed_masks = np.transpose(r['masks'], [2, 0, 1])
+    for seg, seg_bbox in zip(transposed_masks, r['rois']):
+    # seg = transposed_masks[-2]
+    # seg_bbox = r['rois'][-2]
+        # flip x and y
+        seg_bbox = [seg_bbox[1], seg_bbox[0], seg_bbox[3], seg_bbox[2]]
+
+        # Get annotation
+        # test_annotations = json.load(open(os.path.join(ROOT_DIR, "datasets/RGBCAM1_copy_manual_split/test/my_labels.json")))
+        test_annotations = json.load(open(os.path.join(ROOT_DIR, "datasets/RGBCAM1_copy_splitted/test/my_labels.json")))
+        test_annotations = list(test_annotations.values())  # don't need the dict keys
+        image_name = image_path.split("/")[-1]
+        ann = [x for x in test_annotations[0] if image_name in x['file']]
+        if (len(ann) == 0):
+          print('No annotation found for image', image_name)
+          return
+        ann = ann[0]
+        # Convert polygons to a bitmap mask of shape
+        # [height, width, instance_count]
+        polygons = ann['polygon']
+        mask = np.zeros([3000, 4000], dtype=np.int32)
+        for polygon in polygons:
+            # Avoid single-dimension polygons 
+            # (i.e. [x, y] instead of [[x, y], [x, y], ...])
+            try:
+                len(polygon[0])
+            except:
+                continue
+            # Get indexes of pixels inside the polygon and set them to 1
+            x = [coord[0] for coord in polygon]
+            y = [coord[1] for coord in polygon]
+            rr, cc = skimage.draw.polygon(y, x)
+            mask[rr, cc] = 1
+
+        # Calculate IoU
+        mask_bbox = ann['bbox']
+        iou = calculate_iou(seg, seg_bbox, mask, mask_bbox)
+        print('iou', iou)
+        print('---------------------------')
+
+    #
+    # 3. Generate segmentation images.
+    #
+    segments = segment(image, r['masks'], r['rois'])
+    
+    #
+    # 4. Save output.
+    #
     # Create output directory
     date = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
     path = os.path.join('output', date)

From 47e6103cd209487d85313ad533da4379109ef23a Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sun, 1 May 2022 18:20:14 +0200
Subject: [PATCH 12/19] Cleanup and crop output images

---
 samples/strawberry/strawberry.py | 57 ++++++++++++++++----------------
 1 file changed, 28 insertions(+), 29 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 50c857c1cc..ebdb0f777c 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -218,60 +218,44 @@ def segment(image, mask, roi):
 
 
 def calculate_iou(seg, seg_bbox, mask, mask_bbox_array):
-  highest_iou = -99999999
+  highest_iou = 0
   for mask_bbox in mask_bbox_array:
-      # If bboxes dont overlap, return 0
+      # If bboxes dont overlap, continue
       if seg_bbox[2] < mask_bbox[0] or seg_bbox[0] > mask_bbox[2] or seg_bbox[3] < mask_bbox[1] or seg_bbox[1] > mask_bbox[3]:
         continue
 
-      print('seg_bbox', seg_bbox)
-      print('mask_bbox', mask_bbox)
-
       # Calculate intersection
       x_range = [max(seg_bbox[0], mask_bbox[0]), min(seg_bbox[2], mask_bbox[2])]
       y_range = [max(seg_bbox[1], mask_bbox[1]), min(seg_bbox[3], mask_bbox[3])]
-      intersect_size = (x_range[1] - x_range[0]) * (y_range[1] - y_range[0])
-      print('intersect_size', intersect_size)
       intersection = 0
       for x in range(x_range[0], x_range[1]):
         for y in range(y_range[0], y_range[1]):
           if mask[y, x] == seg[y, x]:
             intersection += 1
-      # intersection /= intersect_size
-      print('intersection', intersection)
 
       if intersection == 0:
         print('no intersection')
         continue
 
       # Calculate union
-      seg_bbox_size = abs(seg_bbox[2] - seg_bbox[0]) * abs(seg_bbox[3] - seg_bbox[1])
-      print('seg_bbox_size', seg_bbox_size)
-      mask_bbox_size = abs(mask_bbox[2] - mask_bbox[0]) * abs(mask_bbox[3] - mask_bbox[1])
-      print('mask_bbox_size', mask_bbox_size)
+      # Union = area of seg + area of mask - intersection
       seg_union = 0
       for x in range(seg_bbox[0], seg_bbox[2]):
         for y in range(seg_bbox[1], seg_bbox[3]):
           if mask[y, x] == seg[y, x]:
             seg_union += 1
-      # seg_union /= seg_bbox_size
-      print('seg_union', seg_union)
       mask_union = 0
       for x in range(mask_bbox[0], mask_bbox[2]):
         for y in range(mask_bbox[1], mask_bbox[3]):
           if mask[y, x] == seg[y, x]:
             mask_union += 1
-      # mask_union /= mask_bbox_size
-      print('mask_union', mask_union)
       union = seg_union + mask_union - intersection     
-      print('union', union)
 
       if union == 0:
         print('no union')
         continue
 
       cur = intersection / union
-      print('cur', cur)
       if cur > highest_iou:
         highest_iou = cur
     
@@ -284,6 +268,7 @@ def detect_and_segment(model, image_path=None):
     #
     # 1. Run model detection
     #
+    print('Running model on image...')
     print("Running on {}".format(args.image))
     # Read image
     image = skimage.io.imread(args.image)
@@ -294,25 +279,32 @@ def detect_and_segment(model, image_path=None):
     #
     # 2. Calculate accuracy
     #
+    print('Calculating accuracy...')
     if not len(r['masks']):
-      print('no masks')
+      print('No strawberries were detected, exiting')
       return
     transposed_masks = np.transpose(r['masks'], [2, 0, 1])
+    iou_sum = 0
     for seg, seg_bbox in zip(transposed_masks, r['rois']):
-    # seg = transposed_masks[-2]
-    # seg_bbox = r['rois'][-2]
         # flip x and y
         seg_bbox = [seg_bbox[1], seg_bbox[0], seg_bbox[3], seg_bbox[2]]
 
+        # Crop predicted bbox
+        # Commented out since it doesn't seem to improve IoU
+        if False:
+          x, y = np.nonzero(seg)
+          x_min, x_max = x.min(), x.max()
+          y_min, y_max = y.min(), y.max()
+          seg_bbox = [y_min, x_min, y_max, x_max]
+
         # Get annotation
         # test_annotations = json.load(open(os.path.join(ROOT_DIR, "datasets/RGBCAM1_copy_manual_split/test/my_labels.json")))
         test_annotations = json.load(open(os.path.join(ROOT_DIR, "datasets/RGBCAM1_copy_splitted/test/my_labels.json")))
         test_annotations = list(test_annotations.values())  # don't need the dict keys
         image_name = image_path.split("/")[-1]
         ann = [x for x in test_annotations[0] if image_name in x['file']]
-        if (len(ann) == 0):
-          print('No annotation found for image', image_name)
-          return
+        if len(ann) == 0:
+          raise Exception('No annotation found for image', image_name)
         ann = ann[0]
         # Convert polygons to a bitmap mask of shape
         # [height, width, instance_count]
@@ -334,17 +326,24 @@ def detect_and_segment(model, image_path=None):
         # Calculate IoU
         mask_bbox = ann['bbox']
         iou = calculate_iou(seg, seg_bbox, mask, mask_bbox)
-        print('iou', iou)
-        print('---------------------------')
+        iou_sum += iou
 
+    print('average IoU', iou_sum / len(transposed_masks))
     #
-    # 3. Generate segmentation images.
+    # 3. Generate segmentation images and crop them.
     #
+    print('Creating segmented images...')
     segments = segment(image, r['masks'], r['rois'])
+    for i, s in enumerate(segments):
+      x, y = np.nonzero(s[:,:,-1])
+      x_min, x_max = x.min(), x.max()
+      y_min, y_max = y.min(), y.max()
+      segments[i] = s[x_min:x_max + 1, y_min:y_max + 1, 0:4]
     
     #
     # 4. Save output.
     #
+    print('Saving images...')
     # Create output directory
     date = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
     path = os.path.join('output', date)
@@ -353,7 +352,7 @@ def detect_and_segment(model, image_path=None):
     for idx, seg in enumerate(segments):
       file_name = args.image.split('/')[-1].split('.')[0] + '_seg' + str(idx) + '.png'
       skimage.io.imsave(os.path.join('output', date, file_name), seg)
-      print("Saved to", file_name)
+    print('Done')
 
 
 ############################################################

From 296bf056607ae1018fd6fd2f626c73969357b1f5 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sun, 8 May 2022 13:36:14 +0200
Subject: [PATCH 13/19] Small fixes, improve efficiency of accuracy

---
 samples/strawberry/strawberry.py | 67 +++++++++++++++++++-------------
 1 file changed, 39 insertions(+), 28 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index ebdb0f777c..9654134f7d 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -4,10 +4,14 @@
 import datetime
 import numpy as np
 import skimage.draw
+from os.path import exists
 
 # Root directory of the project
 ROOT_DIR = os.path.abspath("../../")
 
+# Main directory that contains project and data
+MAIN_DIR = os.path.abspath("../../../")
+
 # Import Mask RCNN
 sys.path.append(ROOT_DIR)  # To find local version of the library
 sys.path.append('../../mrcnn')  # ...Actually use local version
@@ -83,9 +87,16 @@ def load_strawberry(self, dataset_dir, subset):
         # Add images
         for annotation in annotations[0]:
             polygons = annotation['polygon']
+            # Skip empty labels or annotations with very little data
+            if len(polygons) < 20:
+                continue
             width, height = 4000, 3000
             image_name = annotation['file'].split("/")[-1]
             image_path = os.path.join(dataset_dir, image_name)
+            # Skip non-existing images
+            file_exists = exists(image_path)
+            if not file_exists:
+                continue
 
             if IMAGE_TYPE == "RGB":
               self.add_image(
@@ -285,6 +296,34 @@ def detect_and_segment(model, image_path=None):
       return
     transposed_masks = np.transpose(r['masks'], [2, 0, 1])
     iou_sum = 0
+
+    # Get image annotation
+    test_annotations = json.load(open(os.path.join(MAIN_DIR, "data/Images/RGBCAM1_split/test/my_labels.json")))
+    test_annotations = list(test_annotations.values())  # don't need the dict keys
+    image_name = image_path.split("/")[-1]
+    ann = [x for x in test_annotations[0] if image_name in x['file']]
+    if len(ann) == 0:
+      raise Exception('No annotation found for image', image_name)
+    ann = ann[0]
+    
+    # Convert polygons to a bitmap mask of shape
+    # [height, width, instance_count]
+    polygons = ann['polygon']
+    mask = np.zeros([3000, 4000], dtype=np.int32)
+    for polygon in polygons:
+        # Avoid single-dimension polygons 
+        # (i.e. [x, y] instead of [[x, y], [x, y], ...])
+        try:
+            len(polygon[0])
+        except:
+            continue
+        # Get indexes of pixels inside the polygon and set them to 1
+        x = [coord[0] for coord in polygon]
+        y = [coord[1] for coord in polygon]
+        rr, cc = skimage.draw.polygon(y, x)
+        mask[rr, cc] = 1
+    mask_bbox = ann['bbox']
+
     for seg, seg_bbox in zip(transposed_masks, r['rois']):
         # flip x and y
         seg_bbox = [seg_bbox[1], seg_bbox[0], seg_bbox[3], seg_bbox[2]]
@@ -297,34 +336,6 @@ def detect_and_segment(model, image_path=None):
           y_min, y_max = y.min(), y.max()
           seg_bbox = [y_min, x_min, y_max, x_max]
 
-        # Get annotation
-        # test_annotations = json.load(open(os.path.join(ROOT_DIR, "datasets/RGBCAM1_copy_manual_split/test/my_labels.json")))
-        test_annotations = json.load(open(os.path.join(ROOT_DIR, "datasets/RGBCAM1_copy_splitted/test/my_labels.json")))
-        test_annotations = list(test_annotations.values())  # don't need the dict keys
-        image_name = image_path.split("/")[-1]
-        ann = [x for x in test_annotations[0] if image_name in x['file']]
-        if len(ann) == 0:
-          raise Exception('No annotation found for image', image_name)
-        ann = ann[0]
-        # Convert polygons to a bitmap mask of shape
-        # [height, width, instance_count]
-        polygons = ann['polygon']
-        mask = np.zeros([3000, 4000], dtype=np.int32)
-        for polygon in polygons:
-            # Avoid single-dimension polygons 
-            # (i.e. [x, y] instead of [[x, y], [x, y], ...])
-            try:
-                len(polygon[0])
-            except:
-                continue
-            # Get indexes of pixels inside the polygon and set them to 1
-            x = [coord[0] for coord in polygon]
-            y = [coord[1] for coord in polygon]
-            rr, cc = skimage.draw.polygon(y, x)
-            mask[rr, cc] = 1
-
-        # Calculate IoU
-        mask_bbox = ann['bbox']
         iou = calculate_iou(seg, seg_bbox, mask, mask_bbox)
         iou_sum += iou
 

From 4f834279508af403b03e5af07c48a53778200bf9 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sat, 21 May 2022 12:37:14 +0200
Subject: [PATCH 14/19] Output track ids during segmentation

---
 mrcnn/model.py                   |  4 ++--
 requirements.txt                 |  9 ++++++--
 samples/strawberry/strawberry.py | 39 +++++++++++++++++++++++---------
 3 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/mrcnn/model.py b/mrcnn/model.py
index 62cb2b0951..c223996cfe 100644
--- a/mrcnn/model.py
+++ b/mrcnn/model.py
@@ -2370,8 +2370,8 @@ def train(self, train_dataset, val_dataset, learning_rate, epochs, layers,
             validation_data=val_generator,
             validation_steps=self.config.VALIDATION_STEPS,
             max_queue_size=100,
-            workers=workers,
-            use_multiprocessing=True,
+            workers=1,
+            use_multiprocessing=False,
         )
         self.epoch = max(self.epoch, epochs)
 
diff --git a/requirements.txt b/requirements.txt
index 429408ac3c..82de52d15a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -3,9 +3,14 @@ scipy
 Pillow
 cython
 matplotlib
-scikit-image==0.19
+# scikit-image
+# 3.6: use ==0.16.2
+# 3.7: use ==0.19
+scikit-image==0.16.2
 tensorflow>=1.3.0
-keras>=2.0.8
+# cluster: use ==2.1.0
+# computer: use >=2.0.8
+keras==2.1.0
 opencv-python
 h5py
 imgaug
diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 9654134f7d..e971c2477b 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -30,7 +30,7 @@
 ############################################################
 
 # Specify whether images are RGB or OCN.
-IMAGE_TYPE = "RGB"
+IMAGE_TYPE = "OCN"
 
 class StrawberryConfig(Config):
     """Configuration for training on the strawberry dataset.
@@ -228,9 +228,12 @@ def segment(image, mask, roi):
     return segments
 
 
-def calculate_iou(seg, seg_bbox, mask, mask_bbox_array):
+def calculate_iou(seg, seg_bbox, mask, mask_bbox_array, track_id_array):
+  # Keep track of IoU and track id of best overlapping bbox
   highest_iou = 0
-  for mask_bbox in mask_bbox_array:
+  best_track_id = -1
+  
+  for mask_bbox, track_id in zip(mask_bbox_array, track_id_array):
       # If bboxes dont overlap, continue
       if seg_bbox[2] < mask_bbox[0] or seg_bbox[0] > mask_bbox[2] or seg_bbox[3] < mask_bbox[1] or seg_bbox[1] > mask_bbox[3]:
         continue
@@ -269,8 +272,9 @@ def calculate_iou(seg, seg_bbox, mask, mask_bbox_array):
       cur = intersection / union
       if cur > highest_iou:
         highest_iou = cur
+        best_track_id = track_id
     
-  return highest_iou
+  return highest_iou, best_track_id
 
 
 def detect_and_segment(model, image_path=None):
@@ -295,17 +299,19 @@ def detect_and_segment(model, image_path=None):
       print('No strawberries were detected, exiting')
       return
     transposed_masks = np.transpose(r['masks'], [2, 0, 1])
-    iou_sum = 0
 
     # Get image annotation
-    test_annotations = json.load(open(os.path.join(MAIN_DIR, "data/Images/RGBCAM1_split/test/my_labels.json")))
+    if args.labels:
+      test_annotations = json.load(open(args.labels))
+    else:
+      test_annotations = json.load(open(os.path.join(MAIN_DIR, "data/Images/RGBCAM1_split/test/my_labels.json")))
     test_annotations = list(test_annotations.values())  # don't need the dict keys
     image_name = image_path.split("/")[-1]
     ann = [x for x in test_annotations[0] if image_name in x['file']]
     if len(ann) == 0:
       raise Exception('No annotation found for image', image_name)
     ann = ann[0]
-    
+
     # Convert polygons to a bitmap mask of shape
     # [height, width, instance_count]
     polygons = ann['polygon']
@@ -322,8 +328,11 @@ def detect_and_segment(model, image_path=None):
         y = [coord[1] for coord in polygon]
         rr, cc = skimage.draw.polygon(y, x)
         mask[rr, cc] = 1
-    mask_bbox = ann['bbox']
+    mask_bbox_array = ann['bbox']
+    track_id_array = ann['track_id']
 
+    iou_sum = 0
+    track_id_estimations = []
     for seg, seg_bbox in zip(transposed_masks, r['rois']):
         # flip x and y
         seg_bbox = [seg_bbox[1], seg_bbox[0], seg_bbox[3], seg_bbox[2]]
@@ -336,8 +345,9 @@ def detect_and_segment(model, image_path=None):
           y_min, y_max = y.min(), y.max()
           seg_bbox = [y_min, x_min, y_max, x_max]
 
-        iou = calculate_iou(seg, seg_bbox, mask, mask_bbox)
+        iou, track_id = calculate_iou(seg, seg_bbox, mask, mask_bbox_array, track_id_array)
         iou_sum += iou
+        track_id_estimations.append(track_id)
 
     print('average IoU', iou_sum / len(transposed_masks))
     #
@@ -350,6 +360,9 @@ def detect_and_segment(model, image_path=None):
       x_min, x_max = x.min(), x.max()
       y_min, y_max = y.min(), y.max()
       segments[i] = s[x_min:x_max + 1, y_min:y_max + 1, 0:4]
+
+    print(track_id_array)
+    print(track_id_estimations)
     
     #
     # 4. Save output.
@@ -360,8 +373,9 @@ def detect_and_segment(model, image_path=None):
     path = os.path.join('output', date)
     os.mkdir(path)
     # Output each segment
-    for idx, seg in enumerate(segments):
-      file_name = args.image.split('/')[-1].split('.')[0] + '_seg' + str(idx) + '.png'
+    for idx, (seg, track_id) in enumerate(zip(segments, track_id_estimations)):
+      file_name = args.image.split('/')[-1].split('.')[0] + \
+                  '_seg' + str(idx) + '_' + str(track_id) + '.png'
       skimage.io.imsave(os.path.join('output', date, file_name), seg)
     print('Done')
 
@@ -392,6 +406,9 @@ def detect_and_segment(model, image_path=None):
     parser.add_argument('--image', required=False,
                         metavar="path or URL to image",
                         help='Image to apply the segmentation on')
+    parser.add_argument('--labels', required=False,
+                        metavar="path or URL to labels JSON file",
+                        help='Labels')
     args = parser.parse_args()
 
     # Validate arguments

From e498173cd8560cf5f9f78301eb90495e0f1a9757 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sat, 21 May 2022 20:48:48 +0200
Subject: [PATCH 15/19] Segment multiple images at once

---
 mrcnn/model.py                   |   5 +-
 samples/strawberry/strawberry.py | 229 +++++++++++++++++--------------
 2 files changed, 132 insertions(+), 102 deletions(-)

diff --git a/mrcnn/model.py b/mrcnn/model.py
index c223996cfe..1eb997586a 100644
--- a/mrcnn/model.py
+++ b/mrcnn/model.py
@@ -2268,8 +2268,9 @@ def set_log_dir(self, model_path=None):
             self.config.NAME.lower(), now))
 
         # Path to save after each epoch. Include placeholders that get filled by Keras.
-        self.checkpoint_path = os.path.join(self.log_dir, "mask_rcnn_{}_*epoch*.h5".format(
-            self.config.NAME.lower()))
+        path_name_model = "mask_rcnn_{}_*epoch*.h5".format(
+            self.config.NAME.lower())
+        self.checkpoint_path = os.path.join(self.log_dir, path_name_model)
         self.checkpoint_path = self.checkpoint_path.replace(
             "*epoch*", "{epoch:04d}")
 
diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index e971c2477b..e22100471f 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -5,6 +5,7 @@
 import numpy as np
 import skimage.draw
 from os.path import exists
+import csv
 
 # Root directory of the project
 ROOT_DIR = os.path.abspath("../../")
@@ -277,106 +278,122 @@ def calculate_iou(seg, seg_bbox, mask, mask_bbox_array, track_id_array):
   return highest_iou, best_track_id
 
 
-def detect_and_segment(model, image_path=None):
-    assert image_path
-
-    #
-    # 1. Run model detection
-    #
-    print('Running model on image...')
-    print("Running on {}".format(args.image))
-    # Read image
-    image = skimage.io.imread(args.image)
-    # Detect objects
-    r = model.detect([image], verbose=1)[0]
-    print('r class ids', r['class_ids'])
-
-    #
-    # 2. Calculate accuracy
-    #
-    print('Calculating accuracy...')
-    if not len(r['masks']):
-      print('No strawberries were detected, exiting')
-      return
-    transposed_masks = np.transpose(r['masks'], [2, 0, 1])
-
-    # Get image annotation
-    if args.labels:
-      test_annotations = json.load(open(args.labels))
-    else:
-      test_annotations = json.load(open(os.path.join(MAIN_DIR, "data/Images/RGBCAM1_split/test/my_labels.json")))
-    test_annotations = list(test_annotations.values())  # don't need the dict keys
-    image_name = image_path.split("/")[-1]
-    ann = [x for x in test_annotations[0] if image_name in x['file']]
-    if len(ann) == 0:
-      raise Exception('No annotation found for image', image_name)
-    ann = ann[0]
-
-    # Convert polygons to a bitmap mask of shape
-    # [height, width, instance_count]
-    polygons = ann['polygon']
-    mask = np.zeros([3000, 4000], dtype=np.int32)
-    for polygon in polygons:
-        # Avoid single-dimension polygons 
-        # (i.e. [x, y] instead of [[x, y], [x, y], ...])
-        try:
-            len(polygon[0])
-        except:
-            continue
-        # Get indexes of pixels inside the polygon and set them to 1
-        x = [coord[0] for coord in polygon]
-        y = [coord[1] for coord in polygon]
-        rr, cc = skimage.draw.polygon(y, x)
-        mask[rr, cc] = 1
-    mask_bbox_array = ann['bbox']
-    track_id_array = ann['track_id']
-
-    iou_sum = 0
-    track_id_estimations = []
-    for seg, seg_bbox in zip(transposed_masks, r['rois']):
-        # flip x and y
-        seg_bbox = [seg_bbox[1], seg_bbox[0], seg_bbox[3], seg_bbox[2]]
-
-        # Crop predicted bbox
-        # Commented out since it doesn't seem to improve IoU
-        if False:
-          x, y = np.nonzero(seg)
-          x_min, x_max = x.min(), x.max()
-          y_min, y_max = y.min(), y.max()
-          seg_bbox = [y_min, x_min, y_max, x_max]
-
-        iou, track_id = calculate_iou(seg, seg_bbox, mask, mask_bbox_array, track_id_array)
-        iou_sum += iou
-        track_id_estimations.append(track_id)
-
-    print('average IoU', iou_sum / len(transposed_masks))
-    #
-    # 3. Generate segmentation images and crop them.
-    #
-    print('Creating segmented images...')
-    segments = segment(image, r['masks'], r['rois'])
-    for i, s in enumerate(segments):
-      x, y = np.nonzero(s[:,:,-1])
-      x_min, x_max = x.min(), x.max()
-      y_min, y_max = y.min(), y.max()
-      segments[i] = s[x_min:x_max + 1, y_min:y_max + 1, 0:4]
-
-    print(track_id_array)
-    print(track_id_estimations)
-    
-    #
-    # 4. Save output.
-    #
-    print('Saving images...')
+def detect_and_segment(model, image_paths, output_dir='output'):
     # Create output directory
     date = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-    path = os.path.join('output', date)
+    path = os.path.join(output_dir, date)
     os.mkdir(path)
-    # Output each segment
-    for idx, (seg, track_id) in enumerate(zip(segments, track_id_estimations)):
-      file_name = args.image.split('/')[-1].split('.')[0] + \
-                  '_seg' + str(idx) + '_' + str(track_id) + '.png'
-      skimage.io.imsave(os.path.join('output', date, file_name), seg)
+
+    # Create output csv
+    # f = open('/Users/ceesjol/Documents/csv_output/' + date + '.csv', 'w')
+    # f = open(os.path.join(output_dir, date, 'output.csv'), 'w')
+    # writer = csv.writer(f)
+    # writer.writerow(["Name", "Track ID", "Pixel width"])
+
+    for image_path in image_paths:
+      # Skip JSON, DS_Store, etc.
+      if not (image_path.lower().endswith("jpg") or \
+        image_path.lower().endswith("png")):
+        continue
+
+      #
+      # 1. Run model detection
+      #
+      print('1. Running model on image...')
+      print("Running on {}".format(image_path))
+      # Read image
+      image = skimage.io.imread(image_path)
+      # Detect objects
+      r = model.detect([image], verbose=1)[0]
+
+      #
+      # 2. Calculate accuracy
+      #
+      print('2. Calculating accuracy...')
+      if not len(r['masks']):
+        print('No strawberries were detected, exiting')
+        return
+      transposed_masks = np.transpose(r['masks'], [2, 0, 1])
+
+      # Get image annotation
+      if args.labels:
+        test_annotations = json.load(open(args.labels))
+      else:
+        test_annotations = json.load(open(os.path.join(MAIN_DIR, "data/Images/RGBCAM1_split/test/my_labels.json")))
+      test_annotations = list(test_annotations.values())  # don't need the dict keys
+      image_name = image_path.split("/")[-1]
+      ann = [x for x in test_annotations[0] if image_name in x['file']]
+      if len(ann) == 0:
+        raise Exception('No annotation found for image', image_name)
+      ann = ann[0]
+
+      # Convert polygons to a bitmap mask of shape
+      # [height, width, instance_count]
+      polygons = ann['polygon']
+      mask = np.zeros([3000, 4000], dtype=np.int32)
+      for polygon in polygons:
+          # Avoid single-dimension polygons 
+          # (i.e. [x, y] instead of [[x, y], [x, y], ...])
+          try:
+              len(polygon[0])
+          except:
+              continue
+          # Get indexes of pixels inside the polygon and set them to 1
+          x = [coord[0] for coord in polygon]
+          y = [coord[1] for coord in polygon]
+          rr, cc = skimage.draw.polygon(y, x)
+          mask[rr, cc] = 1
+      mask_bbox_array = ann['bbox']
+      track_id_array = ann['track_id']
+
+      iou_sum = 0
+      track_id_estimations = []
+      for seg, seg_bbox in zip(transposed_masks, r['rois']):
+          # flip x and y
+          seg_bbox = [seg_bbox[1], seg_bbox[0], seg_bbox[3], seg_bbox[2]]
+
+          # Crop predicted bbox
+          # Commented out since it doesn't seem to improve IoU
+          if False:
+            x, y = np.nonzero(seg)
+            x_min, x_max = x.min(), x.max()
+            y_min, y_max = y.min(), y.max()
+            seg_bbox = [y_min, x_min, y_max, x_max]
+
+          iou, track_id = calculate_iou(seg, seg_bbox, mask, mask_bbox_array, track_id_array)
+          iou_sum += iou
+          track_id_estimations.append(track_id)
+
+      print('average IoU', iou_sum / len(transposed_masks))
+
+      #
+      # 3. Generate segmentation images and crop them.
+      #
+      print('3. Creating segmented images...')
+      segments = segment(image, r['masks'], r['rois'])
+      for i, s in enumerate(segments):
+        x, y = np.nonzero(s[:,:,-1])
+        x_min, x_max = x.min(), x.max()
+        y_min, y_max = y.min(), y.max()
+        segments[i] = s[x_min:x_max + 1, y_min:y_max + 1, 0:4]
+
+      #
+      # 4. Save output.
+      #
+      print('4. Outputting results...')
+
+      # Output each segment
+      for idx, (seg, track_id) in enumerate(zip(segments, track_id_estimations)):
+        # Image
+        image_name = image_path.split('/')[-1].split('.')[0]
+        seg_name = '_seg' + str(idx) + '_' + str(track_id)
+        file_name = image_name + seg_name + '.png'
+        skimage.io.imsave(os.path.join(output_dir, date, file_name), seg)
+
+        # csv
+        # writer.writerow([image_name, track_id, seg.shape[1]])
+
+      # f.close()
     print('Done')
 
 
@@ -406,17 +423,23 @@ def detect_and_segment(model, image_path=None):
     parser.add_argument('--image', required=False,
                         metavar="path or URL to image",
                         help='Image to apply the segmentation on')
+    parser.add_argument('--folder', required=False,
+                        metavar="path or URL to folder",
+                        help='Folder to apply the segmentation on')                    
     parser.add_argument('--labels', required=False,
                         metavar="path or URL to labels JSON file",
                         help='Labels')
+    parser.add_argument('--output', required=False,
+                        metavar="Segmentation images output folder",
+                        help='Output')
     args = parser.parse_args()
 
     # Validate arguments
     if args.command == "train":
         assert args.dataset, "Argument --dataset is required for training"
     elif args.command == "segment":
-        assert args.image,\
-               "Provide --image to apply segmentation"
+        assert args.image or args.folder,\
+               "Provide --image or --folder to apply segmentation"
 
     print("Weights: ", args.weights)
     print("Dataset: ", args.dataset)
@@ -472,7 +495,13 @@ class InferenceConfig(StrawberryConfig):
     if args.command == "train":
         train(model)
     elif args.command == "segment":
-        detect_and_segment(model, image_path=args.image)
+        if args.image:
+          image_paths = [args.image]
+        else:
+          image_paths = os.listdir(args.folder)
+          image_paths = [os.path.join(args.folder, image_path) for image_path in image_paths]
+        print('image_paths: ', image_paths)
+        detect_and_segment(model, image_paths, args.output)
     else:
         print("'{}' is not recognized. "
               "Use 'train' or 'segment'".format(args.command))

From 79cfaf85a189397e20280838e011cc20c5c92e8c Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sat, 21 May 2022 21:17:39 +0200
Subject: [PATCH 16/19] Output track id csv

---
 samples/strawberry/strawberry.py | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index e22100471f..7f7dca4269 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -285,10 +285,11 @@ def detect_and_segment(model, image_paths, output_dir='output'):
     os.mkdir(path)
 
     # Create output csv
-    # f = open('/Users/ceesjol/Documents/csv_output/' + date + '.csv', 'w')
-    # f = open(os.path.join(output_dir, date, 'output.csv'), 'w')
-    # writer = csv.writer(f)
-    # writer.writerow(["Name", "Track ID", "Pixel width"])
+    f = open(os.path.join(output_dir, date, 'output.csv'), 'w')
+    writer = csv.writer(f)
+    writer.writerow(["Track ID", "Name", "Pixel width"])
+
+    track_ids = {}
 
     for image_path in image_paths:
       # Skip JSON, DS_Store, etc.
@@ -384,16 +385,26 @@ def detect_and_segment(model, image_paths, output_dir='output'):
 
       # Output each segment
       for idx, (seg, track_id) in enumerate(zip(segments, track_id_estimations)):
-        # Image
         image_name = image_path.split('/')[-1].split('.')[0]
         seg_name = '_seg' + str(idx) + '_' + str(track_id)
         file_name = image_name + seg_name + '.png'
         skimage.io.imsave(os.path.join(output_dir, date, file_name), seg)
 
-        # csv
-        # writer.writerow([image_name, track_id, seg.shape[1]])
+        # Store track id if it's not in track_ids, or if it's a newer image
+        if not track_id in track_ids or track_ids[track_id]['name'] < image_name:
+          track_ids[track_id] = {
+            'name': image_name,
+            'width': seg.shape[1],
+          }    
+
+    for key in track_ids.keys():
+      arr  = [key]
+      for val in track_ids[key]:
+          arr.append(track_ids[key][val])
+      writer.writerow(arr)
+
+    f.close()
 
-      # f.close()
     print('Done')
 
 

From acbc818aa9a8541118a96c022c1d1f9cdd8fb616 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Fri, 26 Aug 2022 14:45:44 +0200
Subject: [PATCH 17/19] Many improvements

---
 samples/strawberry/strawberry.py | 130 ++++++++++++++++---------------
 1 file changed, 69 insertions(+), 61 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 7f7dca4269..219215979c 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -6,6 +6,9 @@
 import skimage.draw
 from os.path import exists
 import csv
+import glob
+
+DETECTIONS_PATH = '/Users/ceesjol/Documents/Thesis/data_original/Detections/'
 
 # Root directory of the project
 ROOT_DIR = os.path.abspath("../../")
@@ -30,9 +33,6 @@
 #  Configurations
 ############################################################
 
-# Specify whether images are RGB or OCN.
-IMAGE_TYPE = "OCN"
-
 class StrawberryConfig(Config):
     """Configuration for training on the strawberry dataset.
     Derives from the base Config class and overrides some values.
@@ -45,21 +45,50 @@ class StrawberryConfig(Config):
     IMAGES_PER_GPU = 1
 
     # Number of classes (including background)
-    if IMAGE_TYPE == "RGB":
-      NUM_CLASSES = 1 + 1  # Background + strawberry
-    else:
-      NUM_CLASSES = 1 + 3 # Background + strawberry + flower + note
+    NUM_CLASSES = 1 + 3 # Background + strawberry + flower + note
 
     # Number of training steps per epoch
-    STEPS_PER_EPOCH = 100
+    STEPS_PER_EPOCH = 1
 
     # Validation steps per epoch
-    # VALIDATION_STEPS = 50
+    VALIDATION_STEPS = 1
 
     # Skip detections with < 80% confidence
     DETECTION_MIN_CONFIDENCE = 0.8
 
 
+############################################################
+#  Util
+############################################################
+
+labels = []
+def get_labels():
+  global labels
+  if len(labels) == 0:
+    labels = []
+
+    for filepath in glob.iglob(DETECTIONS_PATH + 'v1/*.json', recursive=True):
+        print(filepath)
+        my_json = json.load(open(str(filepath)))
+        my_json = my_json["images"]
+        labels += my_json
+
+    len(labels)
+
+  return labels
+
+def get_polygon(label):
+  try:
+    polygon_array = None
+    if 'predicted_polygon' in label:
+        polygon_array = label['predicted_polygon']
+    if not polygon_array or not len(polygon_array) or not polygon_array[0]:
+        polygon_array = label['polygon']
+    return polygon_array
+  except:
+    return []
+
+
 ############################################################
 #  Dataset
 ############################################################
@@ -71,25 +100,22 @@ def load_strawberry(self, dataset_dir, subset):
         dataset_dir: Root directory of the dataset.
         subset: Subset to load: train or val
         """
-        # Add classes. For RGB, we have only one class to add.
+        # Add classes. 
         self.add_class("strawberry", 1, "strawberry")
-        if IMAGE_TYPE == "OCN":
-          # For OCN, also add the flower and note classes.
-          self.add_class("flower", 2, "flower")
-          self.add_class("note", 3, "note")
+        self.add_class("flower", 2, "flower")
+        self.add_class("note", 3, "note")
 
         # Train or validation dataset?
         assert subset in ["train", "val"]
         dataset_dir = os.path.join(dataset_dir, subset)
 
-        annotations = json.load(open(os.path.join(dataset_dir, "my_labels.json")))
-        annotations = list(annotations.values())  # don't need the dict keys
+        annotations = get_labels()
 
         # Add images
-        for annotation in annotations[0]:
-            polygons = annotation['polygon']
+        for annotation in annotations:
+            polygons = get_polygon(annotation)
             # Skip empty labels or annotations with very little data
-            if len(polygons) < 20:
+            if len(polygons) <= 2:
                 continue
             width, height = 4000, 3000
             image_name = annotation['file'].split("/")[-1]
@@ -99,22 +125,14 @@ def load_strawberry(self, dataset_dir, subset):
             if not file_exists:
                 continue
 
-            if IMAGE_TYPE == "RGB":
-              self.add_image(
-                  "strawberry",
-                  image_id=image_name,  # use file name as a unique image id
-                  path=image_path,
-                  width=width, height=height,
-                  polygons=polygons)
-            else:
-              labels = [label.lower() for label in annotation['label']]
-              self.add_image(
-                  "strawberry",
-                  image_id=image_name,  # use file name as a unique image id
-                  path=image_path,
-                  width=width, height=height,
-                  polygons=polygons,
-                  annotations=labels)
+            labels = [label.lower() for label in annotation['label']]
+            self.add_image(
+                "strawberry",
+                image_id=image_name,  # use file name as a unique image id
+                path=image_path,
+                width=width, height=height,
+                polygons=polygons,
+                annotations=labels)
 
     def load_mask(self, image_id):
         """Generate instance masks for an image.
@@ -146,14 +164,9 @@ def load_mask(self, image_id):
             rr, cc = skimage.draw.polygon(y, x)
             mask[rr, cc, i] = 1
 
-        if IMAGE_TYPE == "RGB":
-          # Return mask, and array of class IDs of each instance. Since we have
-          # one class ID only, we return an array of 1s
-          return mask.astype(np.bool), np.ones([mask.shape[-1]], dtype=np.int32)
-        else:
-          # Return mask, and array of class IDs of each instance.
-          class_ids = np.array([1 if a == 'strawberry' else (2 if a == 'flower' else 3) for a in info['annotations']]).astype(np.int32)
-          return mask.astype(np.bool), class_ids
+        # Return mask, and array of class IDs of each instance.
+        class_ids = np.array([1 if a == 'strawberry' else (2 if a == 'flower' else 3) for a in info['annotations']]).astype(np.int32)
+        return mask.astype(np.bool), class_ids
 
     def image_reference(self, image_id):
         """Return the path of the image."""
@@ -180,6 +193,8 @@ def train(model):
     # Since we're using a very small dataset, and starting from
     # COCO trained weights, we don't need to train too long. Also,
     # no need to train all layers, just the heads should do it.
+    # TODO: why train only the heads?
+    # TODO: why use pretraining (COCO)?
     print("Training network heads")
     model.train(dataset_train, dataset_val,
                 learning_rate=config.LEARNING_RATE,
@@ -217,18 +232,14 @@ def segment(image, mask, roi):
       image_ = np.dstack((image_, np.full((image_.shape[0], image_.shape[1]), 255)))
 
       # Convert (width, height) to (width, height, 4)
-      arr_new = np.ones((*mask_.shape, 4))
-      for i, x in enumerate(mask_):
-          for j, y in enumerate(x):
-              arr_new[i][j] = [y for _ in range(4)]
-      mask_ = arr_new
+      mask_ = np.dstack([mask_ for _ in range(4)])
 
       # Use image values on mask, black otherwise
       res = np.where(mask_, image_, black).astype(np.uint8)
       segments.append(res)
     return segments
 
-
+# TODO: write simplified iou function
 def calculate_iou(seg, seg_bbox, mask, mask_bbox_array, track_id_array):
   # Keep track of IoU and track id of best overlapping bbox
   highest_iou = 0
@@ -312,18 +323,17 @@ def detect_and_segment(model, image_paths, output_dir='output'):
       #
       print('2. Calculating accuracy...')
       if not len(r['masks']):
-        print('No strawberries were detected, exiting')
-        return
+        print('No strawberries were detected, continuing')
+        continue
       transposed_masks = np.transpose(r['masks'], [2, 0, 1])
 
       # Get image annotation
       if args.labels:
         test_annotations = json.load(open(args.labels))
       else:
-        test_annotations = json.load(open(os.path.join(MAIN_DIR, "data/Images/RGBCAM1_split/test/my_labels.json")))
-      test_annotations = list(test_annotations.values())  # don't need the dict keys
+        test_annotations = get_labels()
       image_name = image_path.split("/")[-1]
-      ann = [x for x in test_annotations[0] if image_name in x['file']]
+      ann = [x for x in test_annotations if image_name in x['file']]
       if len(ann) == 0:
         raise Exception('No annotation found for image', image_name)
       ann = ann[0]
@@ -354,12 +364,10 @@ def detect_and_segment(model, image_paths, output_dir='output'):
           seg_bbox = [seg_bbox[1], seg_bbox[0], seg_bbox[3], seg_bbox[2]]
 
           # Crop predicted bbox
-          # Commented out since it doesn't seem to improve IoU
-          if False:
-            x, y = np.nonzero(seg)
-            x_min, x_max = x.min(), x.max()
-            y_min, y_max = y.min(), y.max()
-            seg_bbox = [y_min, x_min, y_max, x_max]
+          x, y = np.nonzero(seg)
+          x_min, x_max = x.min(), x.max()
+          y_min, y_max = y.min(), y.max()
+          seg_bbox = [y_min, x_min, y_max, x_max]
 
           iou, track_id = calculate_iou(seg, seg_bbox, mask, mask_bbox_array, track_id_array)
           iou_sum += iou
@@ -393,7 +401,7 @@ def detect_and_segment(model, image_paths, output_dir='output'):
         # Store track id if it's not in track_ids, or if it's a newer image
         if not track_id in track_ids or track_ids[track_id]['name'] < image_name:
           track_ids[track_id] = {
-            'name': image_name,
+            'name': file_name,
             'width': seg.shape[1],
           }    
 

From dc35726589b4518a6926c2640baccae9d02f8331 Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Mon, 5 Sep 2022 22:30:42 +0200
Subject: [PATCH 18/19] Cluster fixes

---
 mrcnn/model.py                               |   4 +-
 samples/strawberry/strawberry.py             |   3 +-
 samples/strawberry/strawberry_cluster_old.py | 495 +++++++++++++++++++
 3 files changed, 499 insertions(+), 3 deletions(-)
 create mode 100644 samples/strawberry/strawberry_cluster_old.py

diff --git a/mrcnn/model.py b/mrcnn/model.py
index 1eb997586a..8911a797d3 100644
--- a/mrcnn/model.py
+++ b/mrcnn/model.py
@@ -2170,7 +2170,7 @@ def compile(self, learning_rate, momentum):
             if layer.output in self.keras_model.losses:
                 continue
             loss = (
-                tf.reduce_mean(layer.output, keepdims=True)
+                tf.reduce_mean(layer.output, keep_dims=True)
                 * self.config.LOSS_WEIGHTS.get(name, 1.))
             self.keras_model.add_loss(loss)
 
@@ -2194,7 +2194,7 @@ def compile(self, learning_rate, momentum):
             layer = self.keras_model.get_layer(name)
             self.keras_model.metrics_names.append(name)
             loss = (
-                tf.reduce_mean(layer.output, keepdims=True)
+                tf.reduce_mean(layer.output, keep_dims=True)
                 * self.config.LOSS_WEIGHTS.get(name, 1.))
             self.keras_model.metrics_tensors.append(loss)
 
diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 219215979c..6bc7e049cb 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -8,7 +8,8 @@
 import csv
 import glob
 
-DETECTIONS_PATH = '/Users/ceesjol/Documents/Thesis/data_original/Detections/'
+# DETECTIONS_PATH = '/Users/ceesjol/Documents/Thesis/data_original/Detections/'
+DETECTIONS_PATH = '/tudelft.net/staff-umbrella/abeellabstudents/cfjol/data/Detections/'
 
 # Root directory of the project
 ROOT_DIR = os.path.abspath("../../")
diff --git a/samples/strawberry/strawberry_cluster_old.py b/samples/strawberry/strawberry_cluster_old.py
new file mode 100644
index 0000000000..2474818625
--- /dev/null
+++ b/samples/strawberry/strawberry_cluster_old.py
@@ -0,0 +1,495 @@
+import os
+import sys
+import json
+import datetime
+import numpy as np
+import skimage.draw
+from os.path import exists
+import csv
+
+# Root directory of the project
+ROOT_DIR = os.path.abspath("../../")
+
+# Main directory that contains project and data
+MAIN_DIR = os.path.abspath("../../../")
+
+# Import Mask RCNN
+sys.path.append(ROOT_DIR)  # To find local version of the library
+sys.path.append('../../mrcnn')  # ...Actually use local version
+from mrcnn.config import Config
+from mrcnn import model as modellib, utils
+
+# Path to trained weights file
+COCO_WEIGHTS_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")
+
+# Directory to save logs and model checkpoints, if not provided
+# through the command line argument --logs
+DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs")
+
+############################################################
+#  Configurations
+############################################################
+
+# Specify whether images are RGB or OCN.
+IMAGE_TYPE = "OCN"
+
+class StrawberryConfig(Config):
+    """Configuration for training on the strawberry dataset.
+    Derives from the base Config class and overrides some values.
+    """
+    # Give the configuration a recognizable name
+    NAME = "strawberry"
+
+    # We use a GPU with 12GB memory, which can fit two images.
+    # Adjust down if you use a smaller GPU.
+    IMAGES_PER_GPU = 1
+
+    # Number of classes (including background)
+    if IMAGE_TYPE == "RGB":
+      NUM_CLASSES = 1 + 1  # Background + strawberry
+    else:
+      NUM_CLASSES = 1 + 3 # Background + strawberry + flower + note
+
+    # Number of training steps per epoch
+    STEPS_PER_EPOCH = 100
+
+    # Validation steps per epoch
+    # VALIDATION_STEPS = 50
+
+    # Skip detections with < 80% confidence
+    DETECTION_MIN_CONFIDENCE = 0.8
+
+
+############################################################
+#  Dataset
+############################################################
+
+class StrawberryDataset(utils.Dataset):
+
+    def load_strawberry(self, dataset_dir, subset):
+        """Load a subset of the Strawberry dataset.
+        dataset_dir: Root directory of the dataset.
+        subset: Subset to load: train or val
+        """
+        # Add classes. For RGB, we have only one class to add.
+        self.add_class("strawberry", 1, "strawberry")
+        if IMAGE_TYPE == "OCN":
+          # For OCN, also add the flower and note classes.
+          self.add_class("flower", 2, "flower")
+          self.add_class("note", 3, "note")
+
+        # Train or validation dataset?
+        assert subset in ["train", "val"]
+        dataset_dir = os.path.join(dataset_dir, subset)
+
+        annotations = json.load(open(os.path.join(dataset_dir, "my_labels.json")))
+        annotations = list(annotations.values())  # don't need the dict keys
+
+        # Add images
+        for annotation in annotations[0]:
+            polygons = annotation['polygon']
+            # Skip empty labels or annotations with very little data
+            if len(polygons) < 20:
+                continue
+            width, height = 4000, 3000
+            image_name = annotation['file'].split("/")[-1]
+            image_path = os.path.join(dataset_dir, image_name)
+            # Skip non-existing images
+            file_exists = exists(image_path)
+            if not file_exists:
+                continue
+
+            if IMAGE_TYPE == "RGB":
+              self.add_image(
+                  "strawberry",
+                  image_id=image_name,  # use file name as a unique image id
+                  path=image_path,
+                  width=width, height=height,
+                  polygons=polygons)
+            else:
+              labels = [label.lower() for label in annotation['label']]
+              self.add_image(
+                  "strawberry",
+                  image_id=image_name,  # use file name as a unique image id
+                  path=image_path,
+                  width=width, height=height,
+                  polygons=polygons,
+                  annotations=labels)
+
+    def load_mask(self, image_id):
+        """Generate instance masks for an image.
+       Returns:
+        masks: A bool array of shape [height, width, instance count] with
+            one mask per instance.
+        class_ids: a 1D array of class IDs of the instance masks.
+        """
+        # If not a strawberry dataset image, delegate to parent class.
+        image_info = self.image_info[image_id]
+        if image_info["source"] != "strawberry":
+            return super(self.__class__, self).load_mask(image_id)
+
+        # Convert polygons to a bitmap mask of shape
+        # [height, width, instance_count]
+        info = self.image_info[image_id]
+        mask = np.zeros([info["height"], info["width"], len(info["polygons"])],
+                        dtype=np.uint8)
+        for i, polygon in enumerate(info["polygons"]):
+            # Avoid single-dimension polygons 
+            # (i.e. [x, y] instead of [[x, y], [x, y], ...])
+            try:
+                len(polygon[0])
+            except:
+                continue
+            # Get indexes of pixels inside the polygon and set them to 1
+            x = [coord[0] for coord in polygon]
+            y = [coord[1] for coord in polygon]
+            rr, cc = skimage.draw.polygon(y, x)
+            mask[rr, cc, i] = 1
+
+        if IMAGE_TYPE == "RGB":
+          # Return mask, and array of class IDs of each instance. Since we have
+          # one class ID only, we return an array of 1s
+          return mask.astype(np.bool), np.ones([mask.shape[-1]], dtype=np.int32)
+        else:
+          # Return mask, and array of class IDs of each instance.
+          class_ids = np.array([1 if a == 'strawberry' else (2 if a == 'flower' else 3) for a in info['annotations']]).astype(np.int32)
+          return mask.astype(np.bool), class_ids
+
+    def image_reference(self, image_id):
+        """Return the path of the image."""
+        info = self.image_info[image_id]
+        if info["source"] == "strawberry":
+            return info["path"]
+        else:
+            super(self.__class__, self).image_reference(image_id)
+
+
+def train(model):
+    """Train the model."""
+    # Training dataset.
+    dataset_train = StrawberryDataset()
+    dataset_train.load_strawberry(args.dataset, "train")
+    dataset_train.prepare()
+
+    # Validation dataset
+    dataset_val = StrawberryDataset()
+    dataset_val.load_strawberry(args.dataset, "val")
+    dataset_val.prepare()
+
+    # *** This training schedule is an example. Update to your needs ***
+    # Since we're using a very small dataset, and starting from
+    # COCO trained weights, we don't need to train too long. Also,
+    # no need to train all layers, just the heads should do it.
+    print("Training network heads")
+    model.train(dataset_train, dataset_val,
+                learning_rate=config.LEARNING_RATE,
+                epochs=10,
+                layers='heads')
+
+def crop(image, roi):
+    """Crop an image to a region of interest."""
+    x1, y1, x2, y2 = roi
+    return image[x1:x2, y1:y2]
+
+
+def segment(image, mask, roi):
+    """Segment the image.
+    image: RGB image [height, width, 3]
+    mask: instance segmentation mask [height, width, instance count]
+
+    Returns result image.
+    """
+    # Transpose to (nSegments, height, width)
+    mask_transpose = np.transpose(mask, [2, 0, 1])
+
+    segments = []
+
+    for mask_, roi_ in zip(mask_transpose, roi):
+      # Crop image and mask
+      image_ = crop(image, roi_)
+      mask_ = crop(mask_, roi_)
+
+      # Make a grayscale copy of the image. The grayscale copy still
+      # has 4 RGBA channels, though.
+      black = skimage.color.gray2rgba(skimage.color.rgb2gray(image_), alpha=0) * 255
+
+      # Add alpha dim to image
+      image_ = np.dstack((image_, np.full((image_.shape[0], image_.shape[1]), 255)))
+
+      # Convert (width, height) to (width, height, 4)
+      arr_new = np.ones((*mask_.shape, 4))
+      for i, x in enumerate(mask_):
+          for j, y in enumerate(x):
+              arr_new[i][j] = [y for _ in range(4)]
+      mask_ = arr_new
+
+      # Use image values on mask, black otherwise
+      res = np.where(mask_, image_, black).astype(np.uint8)
+      segments.append(res)
+    return segments
+
+
+def calculate_iou(seg, seg_bbox, mask, mask_bbox_array, track_id_array):
+  # Keep track of IoU and track id of best overlapping bbox
+  highest_iou = 0
+  best_track_id = -1
+  
+  for mask_bbox, track_id in zip(mask_bbox_array, track_id_array):
+      # If bboxes dont overlap, continue
+      if seg_bbox[2] < mask_bbox[0] or seg_bbox[0] > mask_bbox[2] or seg_bbox[3] < mask_bbox[1] or seg_bbox[1] > mask_bbox[3]:
+        continue
+
+      # Calculate intersection
+      x_range = [max(seg_bbox[0], mask_bbox[0]), min(seg_bbox[2], mask_bbox[2])]
+      y_range = [max(seg_bbox[1], mask_bbox[1]), min(seg_bbox[3], mask_bbox[3])]
+      intersection = 0
+      for x in range(x_range[0], x_range[1]):
+        for y in range(y_range[0], y_range[1]):
+          if mask[y, x] == seg[y, x]:
+            intersection += 1
+
+      if intersection == 0:
+        print('no intersection')
+        continue
+
+      # Calculate union
+      # Union = area of seg + area of mask - intersection
+      seg_union = 0
+      for x in range(seg_bbox[0], seg_bbox[2]):
+        for y in range(seg_bbox[1], seg_bbox[3]):
+          if mask[y, x] == seg[y, x]:
+            seg_union += 1
+      mask_union = 0
+      for x in range(mask_bbox[0], mask_bbox[2]):
+        for y in range(mask_bbox[1], mask_bbox[3]):
+          if mask[y, x] == seg[y, x]:
+            mask_union += 1
+      union = seg_union + mask_union - intersection     
+
+      if union == 0:
+        print('no union')
+        continue
+
+      cur = intersection / union
+      if cur > highest_iou:
+        highest_iou = cur
+        best_track_id = track_id
+    
+  return highest_iou, best_track_id
+
+
+def detect_and_segment(model, image_path=None):
+    assert image_path
+
+    #
+    # 1. Run model detection
+    #
+    print('Running model on image...')
+    print("Running on {}".format(args.image))
+    # Read image
+    image = skimage.io.imread(args.image)
+    # Detect objects
+    r = model.detect([image], verbose=1)[0]
+    print('r class ids', r['class_ids'])
+
+    #
+    # 2. Calculate accuracy
+    #
+    print('Calculating accuracy...')
+    if not len(r['masks']):
+      print('No strawberries were detected, exiting')
+      return
+    transposed_masks = np.transpose(r['masks'], [2, 0, 1])
+
+    # Get image annotation
+    if args.labels:
+      test_annotations = json.load(open(args.labels))
+    else:
+      test_annotations = json.load(open(os.path.join(MAIN_DIR, "data/Images/RGBCAM1_split/test/my_labels.json")))
+    test_annotations = list(test_annotations.values())  # don't need the dict keys
+    image_name = image_path.split("/")[-1]
+    ann = [x for x in test_annotations[0] if image_name in x['file']]
+    if len(ann) == 0:
+      raise Exception('No annotation found for image', image_name)
+    ann = ann[0]
+
+    # Convert polygons to a bitmap mask of shape
+    # [height, width, instance_count]
+    polygons = ann['polygon']
+    mask = np.zeros([3000, 4000], dtype=np.int32)
+    for polygon in polygons:
+        # Avoid single-dimension polygons 
+        # (i.e. [x, y] instead of [[x, y], [x, y], ...])
+        try:
+            len(polygon[0])
+        except:
+            continue
+        # Get indexes of pixels inside the polygon and set them to 1
+        x = [coord[0] for coord in polygon]
+        y = [coord[1] for coord in polygon]
+        rr, cc = skimage.draw.polygon(y, x)
+        mask[rr, cc] = 1
+    mask_bbox_array = ann['bbox']
+    track_id_array = ann['track_id']
+
+    iou_sum = 0
+    track_id_estimations = []
+    for seg, seg_bbox in zip(transposed_masks, r['rois']):
+        # flip x and y
+        seg_bbox = [seg_bbox[1], seg_bbox[0], seg_bbox[3], seg_bbox[2]]
+
+        # Crop predicted bbox
+        # Commented out since it doesn't seem to improve IoU
+        if False:
+          x, y = np.nonzero(seg)
+          x_min, x_max = x.min(), x.max()
+          y_min, y_max = y.min(), y.max()
+          seg_bbox = [y_min, x_min, y_max, x_max]
+
+        iou, track_id = calculate_iou(seg, seg_bbox, mask, mask_bbox_array, track_id_array)
+        iou_sum += iou
+        track_id_estimations.append(track_id)
+
+    print('average IoU', iou_sum / len(transposed_masks))
+    #
+    # 3. Generate segmentation images and crop them.
+    #
+    print('Creating segmented images...')
+    segments = segment(image, r['masks'], r['rois'])
+    for i, s in enumerate(segments):
+      x, y = np.nonzero(s[:,:,-1])
+      x_min, x_max = x.min(), x.max()
+      y_min, y_max = y.min(), y.max()
+      segments[i] = s[x_min:x_max + 1, y_min:y_max + 1, 0:4]
+
+    print(track_id_array)
+    print(track_id_estimations)
+    
+    #
+    # 4. Save output.
+    #
+    print('Outputting results...')
+    # Create output directory
+    date = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
+    path = os.path.join('output', date)
+    os.mkdir(path)
+
+    # Create output csv
+    # f = open('/Users/ceesjol/Documents/csv_output/' + date + '.csv', 'w')
+    f = open(os.path.join('output', date, 'output.csv'), 'w')
+    writer = csv.writer(f)
+    writer.writerow(["Name", "Track ID", "Pixel width"])
+
+    # Output each segment
+    for idx, (seg, track_id) in enumerate(zip(segments, track_id_estimations)):
+      # if idx == 0:
+      #   print(seg.shape[1])
+      # Image
+      image_name = args.image.split('/')[-1].split('.')[0]
+      seg_name = '_seg' + str(idx) + '_' + str(track_id)
+      file_name = image_name + seg_name + '.png'
+      skimage.io.imsave(os.path.join('output', date, file_name), seg)
+
+      # csv
+      writer.writerow([image_name, track_id, seg.shape[1]])
+
+    f.close()
+    print('Done')
+
+
+############################################################
+#  Training
+############################################################
+
+if __name__ == '__main__':
+    import argparse
+
+    # Parse command line arguments
+    parser = argparse.ArgumentParser(
+        description='Train Mask R-CNN to detect strawberries.')
+    parser.add_argument("command",
+                        metavar="<command>",
+                        help="'train' or 'segment'")
+    parser.add_argument('--dataset', required=False,
+                        metavar="/path/to/strawberry/dataset/",
+                        help='Directory of the Strawberry dataset')
+    parser.add_argument('--weights', required=True,
+                        metavar="/path/to/weights.h5",
+                        help="Path to weights .h5 file or 'coco'")
+    parser.add_argument('--logs', required=False,
+                        default=DEFAULT_LOGS_DIR,
+                        metavar="/path/to/logs/",
+                        help='Logs and checkpoints directory (default=logs/)')
+    parser.add_argument('--image', required=False,
+                        metavar="path or URL to image",
+                        help='Image to apply the segmentation on')
+    parser.add_argument('--labels', required=False,
+                        metavar="path or URL to labels JSON file",
+                        help='Labels')
+    args = parser.parse_args()
+
+    # Validate arguments
+    if args.command == "train":
+        assert args.dataset, "Argument --dataset is required for training"
+    elif args.command == "segment":
+        assert args.image,\
+               "Provide --image to apply segmentation"
+
+    print("Weights: ", args.weights)
+    print("Dataset: ", args.dataset)
+    print("Logs: ", args.logs)
+
+    # Configurations
+    if args.command == "train":
+        config = StrawberryConfig()
+    else:
+        class InferenceConfig(StrawberryConfig):
+            # Set batch size to 1 since we'll be running inference on
+            # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
+            GPU_COUNT = 1
+            IMAGES_PER_GPU = 1
+        config = InferenceConfig()
+    config.display()
+
+    # Create model
+    if args.command == "train":
+        model = modellib.MaskRCNN(mode="training", config=config,
+                                  model_dir=args.logs)
+    else:
+        model = modellib.MaskRCNN(mode="inference", config=config,
+                                  model_dir=args.logs)
+
+    # Select weights file to load
+    if args.weights.lower() == "coco":
+        weights_path = COCO_WEIGHTS_PATH
+        # Download weights file
+        if not os.path.exists(weights_path):
+            utils.download_trained_weights(weights_path)
+    elif args.weights.lower() == "last":
+        # Find last trained weights
+        weights_path = model.find_last()
+    elif args.weights.lower() == "imagenet":
+        # Start from ImageNet trained weights
+        weights_path = model.get_imagenet_weights()
+    else:
+        weights_path = args.weights
+
+    # Load weights
+    print("Loading weights ", weights_path)
+    if args.weights.lower() == "coco":
+        # Exclude the last layers because they require a matching
+        # number of classes
+        model.load_weights(weights_path, by_name=True, exclude=[
+            "mrcnn_class_logits", "mrcnn_bbox_fc",
+            "mrcnn_bbox", "mrcnn_mask"])
+    else:
+        model.load_weights(weights_path, by_name=True)
+
+    # Train or evaluate
+    if args.command == "train":
+        train(model)
+    elif args.command == "segment":
+        detect_and_segment(model, image_path=args.image)
+    else:
+        print("'{}' is not recognized. "
+              "Use 'train' or 'segment'".format(args.command))
\ No newline at end of file

From 1e5956c03743c43f8f994b18bf1fd2d703a3a9dd Mon Sep 17 00:00:00 2001
From: Cees Jol <cjol99@icloud.com>
Date: Sun, 18 Sep 2022 16:58:14 +0200
Subject: [PATCH 19/19] Cutoff by dates

---
 samples/strawberry/strawberry.py | 112 +++++++++++++++++++++----------
 1 file changed, 76 insertions(+), 36 deletions(-)

diff --git a/samples/strawberry/strawberry.py b/samples/strawberry/strawberry.py
index 6bc7e049cb..fa2f9b9d12 100644
--- a/samples/strawberry/strawberry.py
+++ b/samples/strawberry/strawberry.py
@@ -1,15 +1,20 @@
 import os
+CLUSTER = not 'Ceess-MacBook-Pro-2.local' in os.popen('hostname').read()
+
 import sys
 import json
 import datetime
 import numpy as np
 import skimage.draw
 from os.path import exists
-import csv
+# import csv
 import glob
 
-# DETECTIONS_PATH = '/Users/ceesjol/Documents/Thesis/data_original/Detections/'
-DETECTIONS_PATH = '/tudelft.net/staff-umbrella/abeellabstudents/cfjol/data/Detections/'
+if CLUSTER:
+    DETECTIONS_PATH = '/tudelft.net/staff-umbrella/abeellabstudents/cfjol/data/Detections/'
+else:
+    DETECTIONS_PATH = '/Users/ceesjol/Documents/Thesis/data_original/Detections/'
+
 
 # Root directory of the project
 ROOT_DIR = os.path.abspath("../../")
@@ -17,6 +22,19 @@
 # Main directory that contains project and data
 MAIN_DIR = os.path.abspath("../../../")
 
+# Date constants
+CAM_ID = 3
+CAM_NAME = ''
+if CAM_ID % 2 == 0:
+  CAM_NAME = f'OCNCAM{CAM_ID}'
+else:
+  CAM_NAME = f'RGBCAM{CAM_ID}'
+if CAM_ID == 3:
+    # One month for test, one month for val, almost 4 months for train
+    CUTOFF_DATE_TRAIN = datetime.datetime(2021, 8, 13)
+    CUTOFF_DATE_VAL = datetime.datetime(2021, 9, 26)
+    CUTOFF_DAYS_MARGIN = 14
+
 # Import Mask RCNN
 sys.path.append(ROOT_DIR)  # To find local version of the library
 sys.path.append('../../mrcnn')  # ...Actually use local version
@@ -49,10 +67,10 @@ class StrawberryConfig(Config):
     NUM_CLASSES = 1 + 3 # Background + strawberry + flower + note
 
     # Number of training steps per epoch
-    STEPS_PER_EPOCH = 1
+    STEPS_PER_EPOCH = 100
 
     # Validation steps per epoch
-    VALIDATION_STEPS = 1
+    VALIDATION_STEPS = 50
 
     # Skip detections with < 80% confidence
     DETECTION_MIN_CONFIDENCE = 0.8
@@ -65,18 +83,21 @@ class StrawberryConfig(Config):
 labels = []
 def get_labels():
   global labels
-  if len(labels) == 0:
-    labels = []
+  if len(labels) > 0:
+      return labels
 
-    for filepath in glob.iglob(DETECTIONS_PATH + 'v1/*.json', recursive=True):
-        print(filepath)
-        my_json = json.load(open(str(filepath)))
-        my_json = my_json["images"]
-        labels += my_json
+  print('Loading labels for {CAM_NAME}...')
 
-    len(labels)
+  for filepath in glob.iglob(DETECTIONS_PATH + '**/*.json', recursive=True):
+      if not CAM_NAME in str(filepath):
+          continue
+      print(filepath)
+      my_json = json.load(open(str(filepath)))
+      my_json = my_json["images"]
+      labels += my_json
 
   return labels
+get_labels()
 
 def get_polygon(label):
   try:
@@ -108,24 +129,39 @@ def load_strawberry(self, dataset_dir, subset):
 
         # Train or validation dataset?
         assert subset in ["train", "val"]
-        dataset_dir = os.path.join(dataset_dir, subset)
+        # dataset_dir = os.path.join(dataset_dir, subset)
 
         annotations = get_labels()
 
         # Add images
         for annotation in annotations:
-            polygons = get_polygon(annotation)
-            # Skip empty labels or annotations with very little data
-            if len(polygons) <= 2:
-                continue
-            width, height = 4000, 3000
             image_name = annotation['file'].split("/")[-1]
             image_path = os.path.join(dataset_dir, image_name)
+
             # Skip non-existing images
             file_exists = exists(image_path)
             if not file_exists:
                 continue
 
+            # Check if image date is in the right range
+            image_year = image_name.split("_")[0]
+            image_monthday = image_name.split("_")[1]
+            image_date = datetime.datetime.strptime(f'{image_year}{image_monthday}', '%Y%m%d')
+
+            if subset == "train" and not (image_date < CUTOFF_DATE_TRAIN):
+                continue
+            elif subset == "val" and not (image_date < CUTOFF_DATE_VAL and image_date >= CUTOFF_DATE_TRAIN + datetime.timedelta(days=CUTOFF_DAYS_MARGIN)):
+                continue
+            # "test" is inference only
+            # elif subset == "test" and not (image_date >= CUTOFF_DATE_VAL + datetime.timedelta(days=CUTOFF_DAYS_MARGIN)):
+            #     continue
+
+            polygons = get_polygon(annotation)
+            # Skip empty labels or annotations with very little data
+            if len(polygons) <= 2:
+                continue
+            width, height = 4000, 3000           
+
             labels = [label.lower() for label in annotation['label']]
             self.add_image(
                 "strawberry",
@@ -201,6 +237,10 @@ def train(model):
                 learning_rate=config.LEARNING_RATE,
                 epochs=1,
                 layers='heads')
+    # model.train(dataset_train, dataset_val,
+    #             learning_rate=config.LEARNING_RATE,
+    #             epochs=1,
+    #             layers='all')
 
 def crop(image, roi):
     """Crop an image to a region of interest."""
@@ -297,11 +337,11 @@ def detect_and_segment(model, image_paths, output_dir='output'):
     os.mkdir(path)
 
     # Create output csv
-    f = open(os.path.join(output_dir, date, 'output.csv'), 'w')
-    writer = csv.writer(f)
-    writer.writerow(["Track ID", "Name", "Pixel width"])
+    # f = open(os.path.join(output_dir, date, 'output.csv'), 'w')
+    # writer = csv.writer(f)
+    # writer.writerow(["Track ID", "Name", "Pixel width"])
 
-    track_ids = {}
+    # track_ids = {}
 
     for image_path in image_paths:
       # Skip JSON, DS_Store, etc.
@@ -400,19 +440,19 @@ def detect_and_segment(model, image_paths, output_dir='output'):
         skimage.io.imsave(os.path.join(output_dir, date, file_name), seg)
 
         # Store track id if it's not in track_ids, or if it's a newer image
-        if not track_id in track_ids or track_ids[track_id]['name'] < image_name:
-          track_ids[track_id] = {
-            'name': file_name,
-            'width': seg.shape[1],
-          }    
-
-    for key in track_ids.keys():
-      arr  = [key]
-      for val in track_ids[key]:
-          arr.append(track_ids[key][val])
-      writer.writerow(arr)
-
-    f.close()
+        # if not track_id in track_ids or track_ids[track_id]['name'] < image_name:
+        #   track_ids[track_id] = {
+        #     'name': file_name,
+        #     'width': seg.shape[1],
+        #   }    
+
+    # for key in track_ids.keys():
+    #   arr  = [key]
+    #   for val in track_ids[key]:
+    #       arr.append(track_ids[key][val])
+    #   writer.writerow(arr)
+
+    # f.close()
 
     print('Done')