handling merge conflict with twosixlabs/tfdsv4

.github/workflows/3-test-docker.yml

@@ -9,8 +9,8 @@ env:
 
 
 jobs:
-  docker-torch-unit:
-    name: ☁️ Docker Pytorch Image Tests
+  docker-armory-unit:
+    name: ☁️ Docker Armory Image Tests
     runs-on: ubuntu-latest
     steps:
       - name: 🐄 checkout armory full depth with tags for scm
@@ -31,53 +31,15 @@ jobs:
 
       - name: 🚧 Build the Container
         run: |
-          python docker/build.py --framework pytorch
-
-
-      - name: 🤞 Run Image tests
-        run: |
-          IMAGE_VERSION=`armory --show-docker-version-tag`
-          docker run                                  \
-            --rm                                      \
-            --workdir /armory-repo                    \
-            twosixarmory/pytorch:${IMAGE_VERSION}    \
-              pytest                                  \
-                -c /armory-repo/pyproject.toml        \
-                -m "not docker_required and unit"     \
-                ./tests/
-
-
-  docker-tf2-unit:
-    name: ☁️ Docker TF2 Image Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: 🐄 checkout armory full depth with tags for scm
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: 🐍 Use Python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-
-      - name: ⚙️ Installing Armory
-        shell: bash
-        run: |
-          pip install .
-          armory configure --use-defaults
-
-      - name: 🚧 Build the Container
-        run: |
-          python docker/build.py --framework tf2
+          python docker/build.py --framework armory
 
       - name: 🤞 Run Image tests
         run: |
           IMAGE_VERSION=`armory --show-docker-version-tag`
           docker run                                  \
             --rm                                      \
             --workdir /armory-repo                    \
-            twosixarmory/tf2:${IMAGE_VERSION}        \
+            twosixarmory/armory:${IMAGE_VERSION}    \
               pytest                                  \
                 -c /armory-repo/pyproject.toml        \
                 -m "not docker_required and unit"     \
@@ -129,40 +91,3 @@ jobs:
                 -c /armory-repo/pyproject.toml                 \
                 -m "pytorch_deepspeech"                        \
                 ./tests/
-
-
-  docker-mot-unit:
-    name: ☁️ Docker CARLA MOT Image Tests
-    runs-on: ubuntu-latest
-    steps:
-      - name: 🐄 checkout armory full depth with tags for scm
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-
-      - name: 🐍 Use Python 3.9
-        uses: actions/setup-python@v4
-        with:
-          python-version: 3.9
-
-      - name: ⚙️ Install Armory
-        shell: bash
-        run: |
-          pip install .
-          armory configure --use-defaults
-
-      - name: 🚧 Build the Container
-        run: |
-          python docker/build.py --framework carla-mot
-
-      - name: 🤞 Run Image tests
-        run: |
-          IMAGE_VERSION=`armory --show-docker-version-tag`
-          docker run                                  \
-            --rm                                      \
-            --workdir /armory-repo                    \
-            twosixarmory/carla-mot:${IMAGE_VERSION}    \
-              pytest                                  \
-                -c /armory-repo/pyproject.toml        \
-                -m "not docker_required and unit"     \
-                ./tests/

.github/workflows/release.yml

@@ -73,10 +73,8 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - image: pytorch
+          - image: armory
           - image: pytorch-deepspeech
-          - image: tf2
-          - image: carla-mot
     steps:
       - name: 🐍 Setup Python 3.9
         uses: actions/setup-python@v4

README.md

@@ -46,7 +46,7 @@ There are three ways to interact with Armory's container system.
 
 ## armory run
 * `armory run <path/to/config.json>`
-This will run a [configuration file](/docs/configuration_files.md) end to end. Stdout
+This will run a [configuration file](//github.com/twosixlabs/armory/raw/master/docs/configuration_files.md) end to end. Stdout
 and stderror logs will be displayed to the user, and the container will be removed
 gracefully upon completion. Results from the evaluation can be found in your output
 directory.
@@ -61,21 +61,21 @@ from the evaluation can be found in the output directory. To later close the
 interactive container simply run CTRL+C from the terminal where this command was ran.
 
 ## armory launch
-* `armory launch <pytorch|tf2|pytorch-deepspeech|carla-mot>`
+* `armory launch <armory|pytorch-deepspeech>`
 This will launch a framework specific container, with appropriate mounted volumes, for
 the user to attach to for debugging purposes. A command to attach to the container will
 be returned from this call, and it can be ran in a separate terminal. To later close
 the interactive container simply run CTRL+C from the terminal where this command was
 ran.
 
-* `armory launch <pytorch|tf2|pytorch-deepspeech|carla-mot> --jupyter`.
+* `armory launch <armory|pytorch-deepspeech> --jupyter`.
 Similar to the interactive launch, this will spin up a container for a specific
 framework, but will instead return the web address of a jupyter lab server where
 debugging can be performed. To close the jupyter server simply run CTRL+C from the
 terminal where this command was ran.
 
 ## armory exec
-* `armory exec <pytorch|tf2|pytorch-deepspeech|carla-mot> -- <cmd>`
+* `armory exec <armory|pytorch-deepspeech> -- <cmd>`
 This will run a specific command within a framework specific container. A notable use
 case for this would be to run test cases using pytest. After completion of the command
 the container will be removed.
@@ -99,22 +99,22 @@ datasets, and scenarios can be used.
 
 Additionally, volumes (such as your current working directory) will be mounted from
 your system host so that you can modify code to be run, and retrieve outputs.
-For more information on these mounts, please see our [Docker documentation](/docs/docker.md#docker-volume-mounts)
+For more information on these mounts, please see our [Docker documentation](//github.com/twosixlabs/armory/raw/master/docs/docker.md#docker-volume-mounts)
 
 # Scenarios
 Armory provides several baseline threat-model scenarios for various data modalities.
 When running an armory configuration file, the robustness of a defense will be
 evaluated against that given scenario. For more information please see our
-[Scenario Documentation](/docs/scenarios.md).
+[Scenario Documentation](//github.com/twosixlabs/armory/raw/master/docs/scenarios.md).
 
 # FAQs
-Please see the [frequently asked questions](/docs/faqs.md) documentation for more information on:
+Please see the [frequently asked questions](//github.com/twosixlabs/armory/raw/master/docs/faqs.md) documentation for more information on:
 * Dataset format and preprocessing
 * Access to underlying models from wrapped classifiers.
 
 # Contributing
 Armory is an open source project and as such we welcome contributions! Please refer to
-our [contribution docs](/.github/CONTRIBUTING.md) for how to get started.
+our [contribution docs](//github.com/twosixlabs/armory/raw/master/.github/CONTRIBUTING.md) for how to get started.
 
 # Acknowledgment
 This material is based upon work supported by the Defense Advanced Research Projects

armory/__main__.py

@@ -20,16 +20,14 @@
 
 import armory
 
-from armory.logs import log
-
-from armory import paths
-from armory import arguments
+from armory import arguments, paths
 from armory.configuration import load_global_config, save_config
 from armory.eval import Evaluator
-from armory.docker import images
 from armory.utils.configuration import load_config, load_config_stdin
 from armory.utils.version import to_docker_tag
+
 import armory.logs
+from armory.logs import log
 
 
 class PortNumber(argparse.Action):
@@ -81,14 +79,15 @@ def __call__(self, parser, namespace, values, option_string=None):
 
 class DockerImage(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
-        if values in images.ALL:
+        images = armory.docker.images.IMAGE_MAP
+        if values in images.values():
             setattr(namespace, self.dest, values)
-        elif values.lower() in images.IMAGE_MAP:
-            setattr(namespace, self.dest, images.IMAGE_MAP[values])
+        elif values.lower() in images.keys():
+            setattr(namespace, self.dest, images[values])
         else:
             log.info(
                 f"WARNING: {values} not in "
-                f"{list(images.IMAGE_MAP.keys()) + list(images.IMAGE_MAP.values())}. "
+                f"{list(images.keys()) + list(images.values())}. "
                 "Attempting to load custom Docker image."
             )
             setattr(namespace, self.dest, values)
@@ -191,18 +190,18 @@ def _docker_image(parser):
         "docker_image",
         metavar="<docker image>",
         type=str,
-        help="docker image framework: 'tf2', 'pytorch', or 'pytorch-deepspeech'",
+        help="docker image framework: 'armory', or 'pytorch-deepspeech'",
         action=DockerImage,
     )
 
 
 def _docker_image_optional(parser):
     parser.add_argument(
         "--docker-image",
-        default=images.PYTORCH,
+        default=armory.docker.images.ARMORY_IMAGE_NAME,
         metavar="<docker image>",
         type=str,
-        help="docker image framework: 'tf2', 'pytorch', or 'pytorch-deepspeech'",
+        help="docker image framework: 'armory', or 'pytorch-deepspeech'",
         action=DockerImage,
     )
 
@@ -405,14 +404,14 @@ def run(command_args, prog, description) -> int:
 def _pull_docker_images(docker_client=None):
     if docker_client is None:
         docker_client = docker.from_env(version="auto")
-    for image in images.ALL:
+    for image in armory.docker.images.IMAGE_MAP.values():
         try:
             docker_client.images.get(image)
         except docker.errors.ImageNotFound:
             try:
                 log.info(f"Image {image} was not found. Downloading...")
                 repository, tag = ":".split(image)
-                images.pull_verbose(docker_client, repository, tag=tag)
+                armory.docker.images.pull_verbose(docker_client, repository, tag=tag)
             except docker.errors.NotFound:
                 log.exception(
                     f"Docker image {image} does not exist for this version. "
@@ -433,6 +432,7 @@ def download(command_args, prog, description):
         metavar="<download data config file>",
         dest="download_config",
         type=str,
+        default="armory/configs/download_data.json",
         action=DownloadConfig,
         help=f"Configuration for download of data. See {DEFAULT_SCENARIO}. Note: file must be under current working directory.",
     )

armory/datasets/cached_datasets.json

@@ -13,6 +13,20 @@
         "url": null,
         "version": "1.0.0"
     },
+    "cifar10": {
+        "sha256": "05bf2d90665560e08ca6cf6e896b503ed4b83cd0e332d3a2e5970c9329acbaee",
+        "size": 134877477,
+        "subdir": "cifar10/3.0.2",
+        "url": null,
+        "version": "3.0.2"
+    },
+    "cifar100": {
+        "sha256": "e0b5fc54b463b33e9929c1558368a9df87beabef68f341ce935ade7e6f3bd6c3",
+        "size": 133385956,
+        "subdir": "cifar100/3.0.2",
+        "url": null,
+        "version": "3.0.2"
+    },
     "digit": {
         "sha256": "805fb5e33caf2029e13f4146c9d06fdb437ac5b0f0aa9668e3201922b617c559",
         "size": 8349857,

armory/datasets/common.py

@@ -151,7 +151,7 @@ def parse_cache_filename(filename, delim=DELIM, ext=EXT):
 
 def get_cache_dir(data_dir):
     cache_dir = Path(data_dir) / CACHE_SUBDIR
-    cache_dir.mkdir(exist_ok=True)
+    cache_dir.mkdir(parents=True, exist_ok=True)
     return cache_dir
 
 

armory/datasets/config_load.py

@@ -56,13 +56,15 @@ def load_dataset(
         preprocessor = preprocessing.get(preprocessor_name)
 
     if preprocessor is not None and preprocessor_kwargs is not None:
-        preprocessing_fn = lambda x: preprocessor(x, **preprocessor_kwargs)
+        preprocessing_fn = lambda x: preprocessor(  # noqa: E731
+            x, **preprocessor_kwargs
+        )
     else:
         preprocessing_fn = preprocessor
 
     shuffle_elements = shuffle_files
 
-    armory_data_generator = generator.ArmoryDataGenerator(
+    return generator.ArmoryDataGenerator(
         info,
         ds_dict,
         split=split,
@@ -77,10 +79,3 @@ def load_dataset(
         shuffle_elements=shuffle_elements,
         key_map=None,
     )
-    return wrap_generator(armory_data_generator)
-
-
-def wrap_generator(armory_data_generator):
-    from armory.datasets import art_wrapper
-
-    return art_wrapper.WrappedDataGenerator(armory_data_generator)

armory/datasets/generator.py

@@ -155,3 +155,12 @@ def __next__(self):
 
     def __len__(self):
         return self.batches_per_epoch * self.epochs
+
+
+def wrap_generator(armory_data_generator):
+    """
+    Wrap an ArmoryDataGenerator as an ART DataGenerator
+    """
+    from armory.datasets import art_wrapper
+
+    return art_wrapper.WrappedDataGenerator(armory_data_generator)

armory/datasets/preprocessing.py

@@ -39,6 +39,12 @@ def supervised_image_classification(element):
     return (image_to_canon(element["image"]), element["label"])
 
 
+mnist = register(supervised_image_classification, "mnist")
+cifar10 = register(supervised_image_classification, "cifar10")
+cifar100 = register(supervised_image_classification, "cifar100")
+resisc45 = register(supervised_image_classification, "resisc45")
+
+
 @register
 def digit(element):
     return (audio_to_canon(element["audio"]), element["label"])
@@ -67,11 +73,6 @@ def xview(element):
     )
 
 
-mnist = register(supervised_image_classification, "mnist")
-cifar10 = register(supervised_image_classification, "cifar10")
-resisc45 = register(supervised_image_classification, "resisc45")
-
-
 def image_to_canon(image, resize=None, target_dtype=tf.float32, input_type="uint8"):
     """
     TFDS Image feature uses (height, width, channels)