Merge pull request #1788 from lcadalzo/tfdsv4-fix-carla-configs

tfdsv4 fix carla configs

armory/datasets/adversarial/carla_obj_det_dev/__init__.py

@@ -0,0 +1,3 @@
+"""carla_obj_det_dev dataset."""
+
+from .carla_obj_det_dev import CarlaObjDetDev

armory/datasets/adversarial/carla_obj_det_dev/carla_obj_det_dev.py

@@ -0,0 +1,225 @@
+"""carla_obj_det_dev dataset."""
+
+import collections
+import json
+import os
+from copy import deepcopy
+import numpy as np
+
+import tensorflow as tf
+import tensorflow_datasets as tfds
+
+_DESCRIPTION = """
+Synthetic multimodality (RGB, depth) dataset generated using CARLA (https://carla.org).
+"""
+
+_CITATION = """
+@inproceedings{Dosovitskiy17,
+  title = { {CARLA}: {An} Open Urban Driving Simulator},
+  author = {Alexey Dosovitskiy and German Ros and Felipe Codevilla and Antonio Lopez and Vladlen Koltun},
+  booktitle = {Proceedings of the 1st Annual Conference on Robot Learning},
+  pages = {1--16},
+  year = {2017}
+}
+"""
+
+# fmt: off
+_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_od_dev_2.0.0.tar.gz"
+# fmt: on
+
+
+class CarlaObjDetDev(tfds.core.GeneratorBasedBuilder):
+    """DatasetBuilder for carla_obj_det_dev dataset."""
+
+    VERSION = tfds.core.Version("2.0.0")
+    RELEASE_NOTES = {
+        "1.0.0": "Initial release.",
+        "1.0.1": "Correcting error to RGB and depth image pairing",
+        "2.0.0": "Eval5 update with higher resolution, HD textures, accurate annotations, and objects overlapping patch",
+    }
+
+    def _info(self) -> tfds.core.DatasetInfo:
+        """Returns the dataset metadata."""
+        features = {
+            # sequence of [RGB, depth] images
+            "image": tfds.features.Sequence(
+                tfds.features.Image(shape=(960, 1280, 3)),
+                length=2,
+            ),
+            # sequence of image features for [RGB, depth]
+            "images": tfds.features.Sequence(
+                tfds.features.FeaturesDict(
+                    {
+                        "file_name": tfds.features.Text(),
+                        "height": tf.int64,
+                        "width": tf.int64,
+                        "id": tf.int64,
+                    },
+                ),
+                length=2,
+            ),
+            # both modalities share the same categories
+            "categories": tfds.features.Sequence(
+                tfds.features.FeaturesDict(
+                    {
+                        "id": tf.int64,  # {'pedstrian':1, 'vehicles':2, 'trafficlight':3}
+                        "name": tfds.features.Text(),
+                        "supercategory": tfds.features.Text(),
+                    }
+                )
+            ),
+            # both modalities share the same objects
+            "objects": tfds.features.Sequence(
+                {
+                    "id": tf.int64,
+                    "image_id": tf.int64,
+                    "area": tf.int64,  # un-normalized area
+                    "boxes": tfds.features.BBoxFeature(),  # normalized bounding box [ymin, xmin, ymax, xmax]
+                    "labels": tfds.features.ClassLabel(num_classes=5),
+                    "is_crowd": tf.bool,
+                }
+            ),
+            # these data only apply to the "green screen patch" objects, which both modalities share
+            "patch_metadata": tfds.features.FeaturesDict(
+                {
+                    # green screen vertices in (x,y) starting from top-left moving clockwise
+                    "gs_coords": tfds.features.Tensor(shape=[4, 2], dtype=tf.int32),
+                    # binarized segmentation mask of patch.
+                    # mask[x,y] == 1 indicates patch pixel; 0 otherwise
+                    "mask": tfds.features.Image(shape=(960, 1280, 3)),
+                    "avg_patch_depth": tfds.features.Tensor(shape=(), dtype=tf.float64),
+                }
+            ),
+        }
+
+        return tfds.core.DatasetInfo(
+            builder=self,
+            description=_DESCRIPTION,
+            features=tfds.features.FeaturesDict(features),
+            citation=_CITATION,
+        )
+
+    def _split_generators(self, dl_manager: tfds.download.DownloadManager):
+        """Returns SplitGenerators."""
+        path = dl_manager.download_and_extract(_URL)
+        return {"dev": self._generate_examples(os.path.join(path, "dev"))}
+
+    def _generate_examples(self, path):
+        """yield examples"""
+
+        # For each image, gets its annotations and yield relevant data
+        depth_folder = "_out/sensor.camera.depth.2"
+        foreground_mask_folder = "_out/foreground_mask"
+        patch_metadata_folder = "_out/patch_metadata"
+
+        annotation_path = os.path.join(
+            path, "_out", "kwcoco_annotations_without_patch_and_sans_tiny_objects.json"
+        )
+
+        cocoanno = COCOAnnotation(annotation_path)
+
+        images_rgb = (
+            cocoanno.images()
+        )  # list of dictionaries of RGB image id, height, width, file_name
+
+        # sort images alphabetically
+        images_rgb = sorted(images_rgb, key=lambda x: x["file_name"].lower())
+
+        for idx, image_rgb in enumerate(images_rgb):
+
+            # Discard irrelevant fields
+            image_rgb.pop("date_captured")
+            image_rgb.pop("license")
+            image_rgb.pop("coco_url")
+            image_rgb.pop("flickr_url")
+            image_rgb.pop("video_id")
+            image_rgb.pop("frame_index")
+
+            # Pairing RGB and depth
+            fpath_rgb = image_rgb["file_name"]  # rgb image path
+            fname = fpath_rgb.split("/")[-1]
+            fname_no_ext = fname.split(".")[0]
+            fpath_depth = os.path.join(depth_folder, fname)  # depth image path
+            image_depth = deepcopy(image_rgb)
+            image_depth["file_name"] = fpath_depth
+
+            # get object annotations for each image
+            annotations = cocoanno.get_annotations(image_rgb["id"])
+
+            # For unknown reasons, when kwcoco is saved after removing tiny objects,
+            # bbox format changes from [x,y,w,h] to [x1,y1,x2,y1]
+            def build_bbox(x1, y1, x2, y2):
+                return tfds.features.BBox(
+                    ymin=y1 / image_rgb["height"],
+                    xmin=x1 / image_rgb["width"],
+                    ymax=y2 / image_rgb["height"],
+                    xmax=x2 / image_rgb["width"],
+                )
+
+            example = {
+                "image": [
+                    os.path.join(
+                        path,
+                        modality,
+                    )
+                    for modality in [fpath_rgb, fpath_depth]
+                ],
+                "images": [image_rgb, image_depth],
+                "categories": cocoanno.categories(),
+                "objects": [
+                    {
+                        "id": anno["id"],
+                        "image_id": anno["image_id"],
+                        "area": anno["area"],
+                        "boxes": build_bbox(*anno["bbox"]),
+                        "labels": anno["category_id"],
+                        "is_crowd": bool(anno["iscrowd"]),
+                    }
+                    for anno in annotations
+                ],
+                "patch_metadata": {
+                    "gs_coords": np.load(
+                        os.path.join(
+                            path, patch_metadata_folder, fname_no_ext + "_coords.npy"
+                        )
+                    ),
+                    "avg_patch_depth": np.load(
+                        os.path.join(
+                            path, patch_metadata_folder, fname_no_ext + "_avg_depth.npy"
+                        )
+                    ),
+                    "mask": os.path.join(path, foreground_mask_folder, fname),
+                },
+            }
+
+            yield idx, example
+
+
+class COCOAnnotation(object):
+    """COCO annotation helper class."""
+
+    def __init__(self, annotation_path):
+        with tf.io.gfile.GFile(annotation_path) as f:
+            data = json.load(f)
+        self._data = data
+
+        # for each images["id"], find all annotations such that annotations["image_id"] == images["id"]
+        img_id2annotations = collections.defaultdict(list)
+        for a in self._data["annotations"]:
+            img_id2annotations[a["image_id"]].append(a)
+        self._img_id2annotations = {
+            k: list(sorted(v, key=lambda a: a["id"]))
+            for k, v in img_id2annotations.items()
+        }
+
+    def categories(self):
+        """Return the category dicts, as sorted in the file."""
+        return self._data["categories"]
+
+    def images(self):
+        """Return the image dicts, as sorted in the file."""
+        return self._data["images"]
+
+    def get_annotations(self, img_id):
+        """Return all annotations associated with the image id string."""
+        return self._img_id2annotations.get(img_id, [])

armory/datasets/adversarial/carla_obj_det_dev/checksums.tsv

@@ -0,0 +1 @@
+https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_od_dev_2.0.0.tar.gz	67507963	30c7593817867eb97b3c7e1358451c576805bb4423599b09ad99f15a2ebdd5c9	carla_od_dev_2.0.0.tar.gz

armory/datasets/cached_datasets.json

@@ -1,4 +1,11 @@
 {
+    "carla_obj_det_dev": {
+        "sha256": "a7adc2400d1fafb03f6d49d10b61ac6405382bc19df446ee25e3d4afce2775a4",
+        "size": 64152715,
+        "subdir": "carla_obj_det_dev/2.0.0",
+        "url": null,
+        "version": "2.0.0"
+    },
     "carla_over_obj_det_dev": {
         "sha256": "77761f1d5c6eca40984aa40f38fab0568b9bb4a4dca696e876fccaa2dd9be56d",
         "size": 59760259,

armory/datasets/preprocessing.py

@@ -50,9 +50,17 @@ def digit(element):
     return (audio_to_canon(element["audio"]), element["label"])
 
 
+@register
+def carla_obj_det_dev(element, modality="rgb"):
+    return carla_multimodal_obj_det(element["image"], modality=modality), (
+        convert_tf_obj_det_label_to_pytorch(element["image"], element["objects"]),
+        element["patch_metadata"],
+    )
+
+
 @register
 def carla_over_obj_det_dev(element, modality="rgb"):
-    return carla_over_obj_det_image(element["image"], modality=modality), (
+    return carla_multimodal_obj_det(element["image"], modality=modality), (
         convert_tf_obj_det_label_to_pytorch(element["image"], element["objects"]),
         element["patch_metadata"],
     )
@@ -99,14 +107,6 @@ def audio_to_canon(audio, resample=None, target_dtype=tf.float32, input_type="in
     return audio
 
 
-# config = {
-#     "preprocessor": "mnist(max_frames=1)"
-#     "preprocessor_kwargs": {
-#         "max_frames": null,
-#     }
-# }
-
-
 def video_to_canon(
     video,
     resize=None,
@@ -133,7 +133,7 @@ def video_to_canon(
     return video
 
 
-def carla_over_obj_det_image(x, modality="rgb"):
+def carla_multimodal_obj_det(x, modality="rgb"):
     if modality == "rgb":
         return image_to_canon(x[0])
     elif modality == "depth":
@@ -151,8 +151,8 @@ def convert_tf_boxes_to_pytorch(x, box_array):
     Converts object detection boxes from TF format of [y1/height, x1/width, y2/height, x2/width]
     to PyTorch format of [x1, y1, x2, y2]
 
-    :param x: TF tensor of shape (nb, H, W, C)
-    :param y: TF tensor of shape (num_boxes, 4)
+    :param x: TF tensor of shape (nb, H, W, C) or (H, W, C)
+    :param box_array: TF tensor of shape (num_boxes, 4)
     :return: TF tensor of shape (num_boxes, 4)
     """
     x_shape = tf.shape(x)

scenario_configs/eval5/carla_object_detection/carla_obj_det_adversarialpatch_undefended.json

@@ -16,12 +16,14 @@
         "use_label": true
     },
     "dataset": {
-        "batch_size": 1,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "modality": "rgb",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_obj_det_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_obj_det_dev",
+            "preprocessor_kwargs": {
+                "modality": "rgb"
+            },
+            "split": "dev"
+        }
     },
     "defense": null,
     "metric": {

scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_defended.json

@@ -14,12 +14,14 @@
         "use_label": false
     },
     "dataset": {
-        "batch_size": 1,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "modality": "rgb",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_obj_det_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_obj_det_dev",
+            "preprocessor_kwargs": {
+                "modality": "rgb"
+            },
+            "split": "dev"
+        }
     },
     "defense": {
         "kwargs": {

scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_undefended.json

@@ -14,12 +14,14 @@
         "use_label": false
     },
     "dataset": {
-        "batch_size": 1,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "modality": "rgb",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_obj_det_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_obj_det_dev",
+            "preprocessor_kwargs": {
+                "modality": "rgb"
+            },
+            "split": "dev"
+        }
     },
     "defense": null,
     "metric": {

scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json

@@ -18,12 +18,14 @@
         "use_label": true
     },
     "dataset": {
-        "batch_size": 1,
-        "eval_split": "dev",
-        "framework": "numpy",
-        "modality": "both",
-        "module": "armory.data.adversarial_datasets",
-        "name": "carla_obj_det_dev"
+        "test": {
+            "batch_size": 1,
+            "name": "carla_obj_det_dev",
+            "preprocessor_kwargs": {
+                "modality": "both"
+            },
+            "split": "dev"
+        }
     },
     "defense": null,
     "metric": {