diff --git a/armory/datasets/adversarial/carla_obj_det_dev/__init__.py b/armory/datasets/adversarial/carla_obj_det_dev/__init__.py new file mode 100644 index 000000000..3d1c26be2 --- /dev/null +++ b/armory/datasets/adversarial/carla_obj_det_dev/__init__.py @@ -0,0 +1,3 @@ +"""carla_obj_det_dev dataset.""" + +from .carla_obj_det_dev import CarlaObjDetDev diff --git a/armory/datasets/adversarial/carla_obj_det_dev/carla_obj_det_dev.py b/armory/datasets/adversarial/carla_obj_det_dev/carla_obj_det_dev.py new file mode 100644 index 000000000..3adf09441 --- /dev/null +++ b/armory/datasets/adversarial/carla_obj_det_dev/carla_obj_det_dev.py @@ -0,0 +1,225 @@ +"""carla_obj_det_dev dataset.""" + +import collections +import json +import os +from copy import deepcopy +import numpy as np + +import tensorflow as tf +import tensorflow_datasets as tfds + +_DESCRIPTION = """ +Synthetic multimodality (RGB, depth) dataset generated using CARLA (https://carla.org). +""" + +_CITATION = """ +@inproceedings{Dosovitskiy17, + title = { {CARLA}: {An} Open Urban Driving Simulator}, + author = {Alexey Dosovitskiy and German Ros and Felipe Codevilla and Antonio Lopez and Vladlen Koltun}, + booktitle = {Proceedings of the 1st Annual Conference on Robot Learning}, + pages = {1--16}, + year = {2017} +} +""" + +# fmt: off +_URL = "https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_od_dev_2.0.0.tar.gz" +# fmt: on + + +class CarlaObjDetDev(tfds.core.GeneratorBasedBuilder): + """DatasetBuilder for carla_obj_det_dev dataset.""" + + VERSION = tfds.core.Version("2.0.0") + RELEASE_NOTES = { + "1.0.0": "Initial release.", + "1.0.1": "Correcting error to RGB and depth image pairing", + "2.0.0": "Eval5 update with higher resolution, HD textures, accurate annotations, and objects overlapping patch", + } + + def _info(self) -> tfds.core.DatasetInfo: + """Returns the dataset metadata.""" + features = { + # sequence of [RGB, depth] images + "image": tfds.features.Sequence( + tfds.features.Image(shape=(960, 1280, 3)), + length=2, + ), + # sequence of image features for [RGB, depth] + "images": tfds.features.Sequence( + tfds.features.FeaturesDict( + { + "file_name": tfds.features.Text(), + "height": tf.int64, + "width": tf.int64, + "id": tf.int64, + }, + ), + length=2, + ), + # both modalities share the same categories + "categories": tfds.features.Sequence( + tfds.features.FeaturesDict( + { + "id": tf.int64, # {'pedstrian':1, 'vehicles':2, 'trafficlight':3} + "name": tfds.features.Text(), + "supercategory": tfds.features.Text(), + } + ) + ), + # both modalities share the same objects + "objects": tfds.features.Sequence( + { + "id": tf.int64, + "image_id": tf.int64, + "area": tf.int64, # un-normalized area + "boxes": tfds.features.BBoxFeature(), # normalized bounding box [ymin, xmin, ymax, xmax] + "labels": tfds.features.ClassLabel(num_classes=5), + "is_crowd": tf.bool, + } + ), + # these data only apply to the "green screen patch" objects, which both modalities share + "patch_metadata": tfds.features.FeaturesDict( + { + # green screen vertices in (x,y) starting from top-left moving clockwise + "gs_coords": tfds.features.Tensor(shape=[4, 2], dtype=tf.int32), + # binarized segmentation mask of patch. + # mask[x,y] == 1 indicates patch pixel; 0 otherwise + "mask": tfds.features.Image(shape=(960, 1280, 3)), + "avg_patch_depth": tfds.features.Tensor(shape=(), dtype=tf.float64), + } + ), + } + + return tfds.core.DatasetInfo( + builder=self, + description=_DESCRIPTION, + features=tfds.features.FeaturesDict(features), + citation=_CITATION, + ) + + def _split_generators(self, dl_manager: tfds.download.DownloadManager): + """Returns SplitGenerators.""" + path = dl_manager.download_and_extract(_URL) + return {"dev": self._generate_examples(os.path.join(path, "dev"))} + + def _generate_examples(self, path): + """yield examples""" + + # For each image, gets its annotations and yield relevant data + depth_folder = "_out/sensor.camera.depth.2" + foreground_mask_folder = "_out/foreground_mask" + patch_metadata_folder = "_out/patch_metadata" + + annotation_path = os.path.join( + path, "_out", "kwcoco_annotations_without_patch_and_sans_tiny_objects.json" + ) + + cocoanno = COCOAnnotation(annotation_path) + + images_rgb = ( + cocoanno.images() + ) # list of dictionaries of RGB image id, height, width, file_name + + # sort images alphabetically + images_rgb = sorted(images_rgb, key=lambda x: x["file_name"].lower()) + + for idx, image_rgb in enumerate(images_rgb): + + # Discard irrelevant fields + image_rgb.pop("date_captured") + image_rgb.pop("license") + image_rgb.pop("coco_url") + image_rgb.pop("flickr_url") + image_rgb.pop("video_id") + image_rgb.pop("frame_index") + + # Pairing RGB and depth + fpath_rgb = image_rgb["file_name"] # rgb image path + fname = fpath_rgb.split("/")[-1] + fname_no_ext = fname.split(".")[0] + fpath_depth = os.path.join(depth_folder, fname) # depth image path + image_depth = deepcopy(image_rgb) + image_depth["file_name"] = fpath_depth + + # get object annotations for each image + annotations = cocoanno.get_annotations(image_rgb["id"]) + + # For unknown reasons, when kwcoco is saved after removing tiny objects, + # bbox format changes from [x,y,w,h] to [x1,y1,x2,y1] + def build_bbox(x1, y1, x2, y2): + return tfds.features.BBox( + ymin=y1 / image_rgb["height"], + xmin=x1 / image_rgb["width"], + ymax=y2 / image_rgb["height"], + xmax=x2 / image_rgb["width"], + ) + + example = { + "image": [ + os.path.join( + path, + modality, + ) + for modality in [fpath_rgb, fpath_depth] + ], + "images": [image_rgb, image_depth], + "categories": cocoanno.categories(), + "objects": [ + { + "id": anno["id"], + "image_id": anno["image_id"], + "area": anno["area"], + "boxes": build_bbox(*anno["bbox"]), + "labels": anno["category_id"], + "is_crowd": bool(anno["iscrowd"]), + } + for anno in annotations + ], + "patch_metadata": { + "gs_coords": np.load( + os.path.join( + path, patch_metadata_folder, fname_no_ext + "_coords.npy" + ) + ), + "avg_patch_depth": np.load( + os.path.join( + path, patch_metadata_folder, fname_no_ext + "_avg_depth.npy" + ) + ), + "mask": os.path.join(path, foreground_mask_folder, fname), + }, + } + + yield idx, example + + +class COCOAnnotation(object): + """COCO annotation helper class.""" + + def __init__(self, annotation_path): + with tf.io.gfile.GFile(annotation_path) as f: + data = json.load(f) + self._data = data + + # for each images["id"], find all annotations such that annotations["image_id"] == images["id"] + img_id2annotations = collections.defaultdict(list) + for a in self._data["annotations"]: + img_id2annotations[a["image_id"]].append(a) + self._img_id2annotations = { + k: list(sorted(v, key=lambda a: a["id"])) + for k, v in img_id2annotations.items() + } + + def categories(self): + """Return the category dicts, as sorted in the file.""" + return self._data["categories"] + + def images(self): + """Return the image dicts, as sorted in the file.""" + return self._data["images"] + + def get_annotations(self, img_id): + """Return all annotations associated with the image id string.""" + return self._img_id2annotations.get(img_id, []) diff --git a/armory/datasets/adversarial/carla_obj_det_dev/checksums.tsv b/armory/datasets/adversarial/carla_obj_det_dev/checksums.tsv new file mode 100644 index 000000000..27212e871 --- /dev/null +++ b/armory/datasets/adversarial/carla_obj_det_dev/checksums.tsv @@ -0,0 +1 @@ +https://armory-public-data.s3.us-east-2.amazonaws.com/carla/carla_od_dev_2.0.0.tar.gz 67507963 30c7593817867eb97b3c7e1358451c576805bb4423599b09ad99f15a2ebdd5c9 carla_od_dev_2.0.0.tar.gz diff --git a/armory/datasets/cached_datasets.json b/armory/datasets/cached_datasets.json index 129e5f672..f510ecc0a 100644 --- a/armory/datasets/cached_datasets.json +++ b/armory/datasets/cached_datasets.json @@ -1,4 +1,11 @@ { + "carla_obj_det_dev": { + "sha256": "a7adc2400d1fafb03f6d49d10b61ac6405382bc19df446ee25e3d4afce2775a4", + "size": 64152715, + "subdir": "carla_obj_det_dev/2.0.0", + "url": null, + "version": "2.0.0" + }, "carla_over_obj_det_dev": { "sha256": "77761f1d5c6eca40984aa40f38fab0568b9bb4a4dca696e876fccaa2dd9be56d", "size": 59760259, diff --git a/armory/datasets/preprocessing.py b/armory/datasets/preprocessing.py index b264491a8..4b8ce1a79 100644 --- a/armory/datasets/preprocessing.py +++ b/armory/datasets/preprocessing.py @@ -50,9 +50,17 @@ def digit(element): return (audio_to_canon(element["audio"]), element["label"]) +@register +def carla_obj_det_dev(element, modality="rgb"): + return carla_multimodal_obj_det(element["image"], modality=modality), ( + convert_tf_obj_det_label_to_pytorch(element["image"], element["objects"]), + element["patch_metadata"], + ) + + @register def carla_over_obj_det_dev(element, modality="rgb"): - return carla_over_obj_det_image(element["image"], modality=modality), ( + return carla_multimodal_obj_det(element["image"], modality=modality), ( convert_tf_obj_det_label_to_pytorch(element["image"], element["objects"]), element["patch_metadata"], ) @@ -99,14 +107,6 @@ def audio_to_canon(audio, resample=None, target_dtype=tf.float32, input_type="in return audio -# config = { -# "preprocessor": "mnist(max_frames=1)" -# "preprocessor_kwargs": { -# "max_frames": null, -# } -# } - - def video_to_canon( video, resize=None, @@ -133,7 +133,7 @@ def video_to_canon( return video -def carla_over_obj_det_image(x, modality="rgb"): +def carla_multimodal_obj_det(x, modality="rgb"): if modality == "rgb": return image_to_canon(x[0]) elif modality == "depth": @@ -151,8 +151,8 @@ def convert_tf_boxes_to_pytorch(x, box_array): Converts object detection boxes from TF format of [y1/height, x1/width, y2/height, x2/width] to PyTorch format of [x1, y1, x2, y2] - :param x: TF tensor of shape (nb, H, W, C) - :param y: TF tensor of shape (num_boxes, 4) + :param x: TF tensor of shape (nb, H, W, C) or (H, W, C) + :param box_array: TF tensor of shape (num_boxes, 4) :return: TF tensor of shape (num_boxes, 4) """ x_shape = tf.shape(x) diff --git a/scenario_configs/eval5/carla_object_detection/carla_obj_det_adversarialpatch_undefended.json b/scenario_configs/eval5/carla_object_detection/carla_obj_det_adversarialpatch_undefended.json index 01169ce62..2c4244cdf 100644 --- a/scenario_configs/eval5/carla_object_detection/carla_obj_det_adversarialpatch_undefended.json +++ b/scenario_configs/eval5/carla_object_detection/carla_obj_det_adversarialpatch_undefended.json @@ -16,12 +16,14 @@ "use_label": true }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "rgb", - "module": "armory.data.adversarial_datasets", - "name": "carla_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_obj_det_dev", + "preprocessor_kwargs": { + "modality": "rgb" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_defended.json b/scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_defended.json index 600fada64..0f9bfd237 100755 --- a/scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_defended.json +++ b/scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_defended.json @@ -14,12 +14,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "rgb", - "module": "armory.data.adversarial_datasets", - "name": "carla_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_obj_det_dev", + "preprocessor_kwargs": { + "modality": "rgb" + }, + "split": "dev" + } }, "defense": { "kwargs": { diff --git a/scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_undefended.json b/scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_undefended.json index 9346833c4..8b04c5005 100755 --- a/scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_undefended.json +++ b/scenario_configs/eval5/carla_object_detection/carla_obj_det_dpatch_undefended.json @@ -14,12 +14,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "rgb", - "module": "armory.data.adversarial_datasets", - "name": "carla_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_obj_det_dev", + "preprocessor_kwargs": { + "modality": "rgb" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json b/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json index 91b95702c..7de1d59d2 100644 --- a/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json +++ b/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json @@ -18,12 +18,14 @@ "use_label": true }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "both", - "module": "armory.data.adversarial_datasets", - "name": "carla_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_obj_det_dev", + "preprocessor_kwargs": { + "modality": "both" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json b/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json index 18f6c1c94..4cadc0ba4 100644 --- a/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json +++ b/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json @@ -18,12 +18,14 @@ "use_label": true }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "both", - "module": "armory.data.adversarial_datasets", - "name": "carla_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_obj_det_dev", + "preprocessor_kwargs": { + "modality": "both" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_dpatch_defended.json b/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_dpatch_defended.json index a089f9cf4..6ef18d491 100755 --- a/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_dpatch_defended.json +++ b/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_dpatch_defended.json @@ -16,12 +16,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "both", - "module": "armory.data.adversarial_datasets", - "name": "carla_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_obj_det_dev", + "preprocessor_kwargs": { + "modality": "both" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_dpatch_undefended.json b/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_dpatch_undefended.json index 533e856fd..e56d71745 100755 --- a/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_dpatch_undefended.json +++ b/scenario_configs/eval5/carla_object_detection/carla_obj_det_multimodal_dpatch_undefended.json @@ -16,12 +16,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "both", - "module": "armory.data.adversarial_datasets", - "name": "carla_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_obj_det_dev", + "preprocessor_kwargs": { + "modality": "both" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json index 9652fe09f..961665bb2 100644 --- a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json +++ b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_adversarialpatch_undefended.json @@ -16,12 +16,14 @@ "use_label": true }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "rgb", - "module": "armory.data.adversarial_datasets", - "name": "carla_over_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_over_obj_det_dev", + "preprocessor_kwargs": { + "modality": "rgb" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json index 51810a0e6..2015d2118 100644 --- a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json +++ b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_dpatch_defended.json @@ -14,12 +14,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "rgb", - "module": "armory.data.adversarial_datasets", - "name": "carla_over_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_over_obj_det_dev", + "preprocessor_kwargs": { + "modality": "rgb" + }, + "split": "dev" + } }, "defense": { "kwargs": { diff --git a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json index 0e3739246..97afa2690 100644 --- a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json +++ b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_dpatch_undefended.json @@ -14,12 +14,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "rgb", - "module": "armory.data.adversarial_datasets", - "name": "carla_over_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_over_obj_det_dev", + "preprocessor_kwargs": { + "modality": "rgb" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json index 37cb12fb7..a673e03af 100644 --- a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json +++ b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_defended.json @@ -18,12 +18,14 @@ "use_label": true }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "both", - "module": "armory.data.adversarial_datasets", - "name": "carla_over_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_over_obj_det_dev", + "preprocessor_kwargs": { + "modality": "both" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json index eec068ad9..0d75b7242 100644 --- a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json +++ b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_adversarialpatch_undefended.json @@ -18,12 +18,14 @@ "use_label": true }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "both", - "module": "armory.data.adversarial_datasets", - "name": "carla_over_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_over_obj_det_dev", + "preprocessor_kwargs": { + "modality": "both" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json index 9e4ab5226..0664827c3 100644 --- a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json +++ b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_defended.json @@ -16,12 +16,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "both", - "module": "armory.data.adversarial_datasets", - "name": "carla_over_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_over_obj_det_dev", + "preprocessor_kwargs": { + "modality": "both" + }, + "split": "dev" + } }, "defense": null, "metric": { diff --git a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json index f756e4265..72da60a2b 100644 --- a/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json +++ b/scenario_configs/eval6/carla_overhead_object_detection/carla_obj_det_multimodal_dpatch_undefended.json @@ -16,12 +16,14 @@ "use_label": false }, "dataset": { - "batch_size": 1, - "eval_split": "dev", - "framework": "numpy", - "modality": "both", - "module": "armory.data.adversarial_datasets", - "name": "carla_over_obj_det_dev" + "test": { + "batch_size": 1, + "name": "carla_over_obj_det_dev", + "preprocessor_kwargs": { + "modality": "both" + }, + "split": "dev" + } }, "defense": null, "metric": {