linting and renaming target setting runs

mlcommons · Oct 12, 2022 · 191499f · 191499f
1 parent 1177e36
commit 191499f
Show file tree

Hide file tree

Showing 10 changed files with 53 additions and 56 deletions.
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -19,7 +19,6 @@ jobs:
       run: |
         pylint algorithmic_efficiency
         pylint baselines
-        pylint target_setting_runs
         pylint reference_algorithms
         pylint submission_runner.py
         pylint tests

diff --git a/README.md b/README.md
@@ -188,7 +188,7 @@ To run the below commands, use the versions installed via `pip install -e '.[dev
 
 To automatically fix formatting errors, run the following (*WARNING:* this will edit your code, so it is suggested to make a git commit first!):
 ```bash
-yapf -i -r -vv -p algorithmic_efficiency baselines target_setting_runs reference_algorithms tests *.py
+yapf -i -r -vv -p algorithmic_efficiency baselines reference_algorithms tests *.py
 ```
 
 To sort all import orderings, run the following:
@@ -205,7 +205,6 @@ To print out all offending pylint issues, run the following:
 ```bash
 pylint algorithmic_efficiency
 pylint baselines
-pylint target_setting_runs
 pylint reference_algorithms
 pylint submission_runner.py
 pylint tests

diff --git a/reference_algorithms/target_setting_algorithms/README.md b/reference_algorithms/target_setting_algorithms/README.md
@@ -17,8 +17,8 @@ python3 submission_runner.py \
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=criteo1tb \
-    --submission_path=target_setting_runs/jax_adamw.py \
-    --tuning_search_space=target_setting_runs/criteo1tb/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/jax_adamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/criteo1tb/tuning_search_space.json
 ```
 ```bash
 torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \
@@ -27,8 +27,8 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=criteo1tb \
-    --submission_path=target_setting_runs/pytorch_adamw.py \
-    --tuning_search_space=target_setting_runs/criteo1tb/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/pytorch_adamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/criteo1tb/tuning_search_space.json
 ```
 
 ## FastMRI
@@ -40,8 +40,8 @@ python3 submission_runner.py \
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=fastmri \
-    --submission_path=target_setting_runs/jax_nadamw.py \
-    --tuning_search_space=target_setting_runs/fastmri/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/jax_nadamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/fastmri/tuning_search_space.json
 ```
 ```bash
 torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \
@@ -50,8 +50,8 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=fastmri \
-    --submission_path=target_setting_runs/pytorch_nadamw.py \
-    --tuning_search_space=target_setting_runs/fastmri/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/pytorch_nadamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/fastmri/tuning_search_space.json
 ```
 
 ## ImageNet-Resnet
@@ -64,8 +64,8 @@ python3 submission_runner.py \
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=imagenet_resnet \
-    --submission_path=target_setting_runs/jax_nesterov.py \
-    --tuning_search_space=target_setting_runs/imagenet_resnet/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/jax_nesterov.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/imagenet_resnet/tuning_search_space.json
 ```
 ```bash
 torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \
@@ -75,8 +75,8 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=imagenet_resnet \
-    --submission_path=target_setting_runs/pytorch_nesterov.py \
-    --tuning_search_space=target_setting_runs/imagenet_resnet/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/pytorch_nesterov.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/imagenet_resnet/tuning_search_space.json
 ```
 
 ## ImageNet-ViT
@@ -89,8 +89,8 @@ python3 submission_runner.py \
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=imagenet_vit \
-    --submission_path=target_setting_runs/jax_nadamw.py \
-    --tuning_search_space=target_setting_runs/imagenet_vit/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/jax_nadamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/imagenet_vit/tuning_search_space.json
 ```
 ```bash
 torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \
@@ -100,8 +100,8 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=imagenet_vit \
-    --submission_path=target_setting_runs/pytorch_nadamw.py \
-    --tuning_search_space=target_setting_runs/imagenet_vit/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/pytorch_nadamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/imagenet_vit/tuning_search_space.json
 ```
 
 ## Librispeech-Conformer
@@ -113,8 +113,8 @@ python3 submission_runner.py \
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=librispeech_conformer \
-    --submission_path=target_setting_runs/jax_adamw.py \
-    --tuning_search_space=target_setting_runs/librispeech_conformer/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/jax_adamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/librispeech_conformer/tuning_search_space.json
 ```
 ```bash
 torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \
@@ -123,8 +123,8 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=librispeech_conformer \
-    --submission_path=target_setting_runs/pytorch_adamw.py \
-    --tuning_search_space=target_setting_runs/librispeech_conformer/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/pytorch_adamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/librispeech_conformer/tuning_search_space.json
 ```
 
 ## Librispeech-Deepspeech
@@ -136,8 +136,8 @@ python3 submission_runner.py \
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=librispeech_deepspeech \
-    --submission_path=target_setting_runs/jax_nadamw.py \
-    --tuning_search_space=target_setting_runs/librispeech_deepspeech/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/jax_nadamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/librispeech_deepspeech/tuning_search_space.json
 ```
 ```bash
 torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \
@@ -146,8 +146,8 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=librispeech_deepspeech \
-    --submission_path=target_setting_runs/pytorch_nadamw.py \
-    --tuning_search_space=target_setting_runs/librispeech_deepspeech/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/pytorch_nadamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/librispeech_deepspeech/tuning_search_space.json
 ```
 
 ## OGBG
@@ -159,8 +159,8 @@ python3 submission_runner.py \
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=ogbg \
-    --submission_path=target_setting_runs/jax_nesterov.py \
-    --tuning_search_space=target_setting_runs/ogbg/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/jax_nesterov.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/ogbg/tuning_search_space.json
 ```
 ```bash
 torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \
@@ -169,8 +169,8 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=ogbg \
-    --submission_path=target_setting_runs/pytorch_nesterov.py \
-    --tuning_search_space=target_setting_runs/ogbg/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/pytorch_nesterov.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/ogbg/tuning_search_space.json
 ```
 
 ## WMT
@@ -182,8 +182,8 @@ python3 submission_runner.py \
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=wmt \
-    --submission_path=target_setting_runs/jax_adamw.py \
-    --tuning_search_space=target_setting_runs/wmt/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/jax_adamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/wmt/tuning_search_space.json
 ```
 ```bash
 torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc_per_node=8 submission_runner.py \
@@ -192,6 +192,6 @@ torchrun --redirects 1:0,2:0,3:0,4:0,5:0,6:0,7:0 --standalone --nnodes=1 --nproc
     --experiment_dir=$ROOT_DIR \
     --experiment_name=target_setting \
     --workload=wmt \
-    --submission_path=target_setting_runs/pytorch_adamw.py \
-    --tuning_search_space=target_setting_runs/wmt/tuning_search_space.json
+    --submission_path=reference_algorithms/target_setting_runs/pytorch_adamw.py \
+    --tuning_search_space=reference_algorithms/target_setting_runs/wmt/tuning_search_space.json
 ```
diff --git a/reference_algorithms/target_setting_algorithms/jax_adamw.py b/reference_algorithms/target_setting_algorithms/jax_adamw.py
@@ -5,12 +5,12 @@
 import optax
 
 from algorithmic_efficiency import spec
-from target_setting_runs import cosine_warmup
-from target_setting_runs.data_selection import \
+from reference_algorithms.target_setting_algorithms import cosine_warmup
+from reference_algorithms.target_setting_algorithms.data_selection import \
     data_selection  # pylint: disable=unused-import
-from target_setting_runs.get_batch_size import \
+from reference_algorithms.target_setting_algorithms.get_batch_size import \
     get_batch_size  # pylint: disable=unused-import
-from target_setting_runs.jax_submission_base import \
+from reference_algorithms.target_setting_algorithms.jax_submission_base import \
     update_params  # pylint: disable=unused-import
 
 

diff --git a/reference_algorithms/target_setting_algorithms/jax_nadamw.py b/reference_algorithms/target_setting_algorithms/jax_nadamw.py
@@ -7,8 +7,6 @@
 import jax
 import jax.numpy as jnp
 import optax
-
-from algorithmic_efficiency import spec
 from target_setting_runs import cosine_warmup
 from target_setting_runs.data_selection import \
     data_selection  # pylint: disable=unused-import
@@ -17,6 +15,8 @@
 from target_setting_runs.jax_submission_base import \
     update_params  # pylint: disable=unused-import
 
+from algorithmic_efficiency import spec
+
 
 # Forked from
 # github.com/google/init2winit/blob/master/init2winit/optimizer_lib/alias.py

diff --git a/reference_algorithms/target_setting_algorithms/jax_nesterov.py b/reference_algorithms/target_setting_algorithms/jax_nesterov.py
@@ -6,15 +6,15 @@
 import jax
 import jax.numpy as jnp
 import optax
-
-from algorithmic_efficiency import spec
 from target_setting_runs.data_selection import \
     data_selection  # pylint: disable=unused-import
 from target_setting_runs.get_batch_size import \
     get_batch_size  # pylint: disable=unused-import
 from target_setting_runs.jax_submission_base import \
     update_params  # pylint: disable=unused-import
 
+from algorithmic_efficiency import spec
+
 
 def init_optimizer_state(workload: spec.Workload,
                          model_params: spec.ParameterContainer,

diff --git a/reference_algorithms/target_setting_algorithms/pytorch_adamw.py b/reference_algorithms/target_setting_algorithms/pytorch_adamw.py
@@ -1,15 +1,15 @@
 """Submission file for an AdamW optimizer with warmup+cosine LR in PyTorch."""
 
-import torch
-
-from algorithmic_efficiency import spec
 from target_setting_runs import cosine_warmup
 from target_setting_runs.data_selection import \
     data_selection  # pylint: disable=unused-import
 from target_setting_runs.get_batch_size import \
     get_batch_size  # pylint: disable=unused-import
 from target_setting_runs.pytorch_submission_base import \
     update_params  # pylint: disable=unused-import
+import torch
+
+from algorithmic_efficiency import spec
 
 
 def init_optimizer_state(workload: spec.Workload,

diff --git a/reference_algorithms/target_setting_algorithms/pytorch_nadamw.py b/reference_algorithms/target_setting_algorithms/pytorch_nadamw.py
@@ -3,17 +3,17 @@
 import math
 from typing import List
 
-import torch
-from torch import Tensor
-
-from algorithmic_efficiency import spec
 from target_setting_runs import cosine_warmup
 from target_setting_runs.data_selection import \
     data_selection  # pylint: disable=unused-import
 from target_setting_runs.get_batch_size import \
     get_batch_size  # pylint: disable=unused-import
 from target_setting_runs.pytorch_submission_base import \
     update_params  # pylint: disable=unused-import
+import torch
+from torch import Tensor
+
+from algorithmic_efficiency import spec
 
 
 # Modified from github.com/pytorch/pytorch/blob/v1.12.1/torch/optim/adamw.py

diff --git a/reference_algorithms/target_setting_algorithms/pytorch_nesterov.py b/reference_algorithms/target_setting_algorithms/pytorch_nesterov.py
@@ -1,16 +1,16 @@
 """Submission file for a SGD with Nesterov optimizer in PyTorch."""
 
-import torch
-from torch.optim.lr_scheduler import LambdaLR
-
-from algorithmic_efficiency import spec
 from target_setting_runs.data_selection import \
     data_selection  # pylint: disable=unused-import
 from target_setting_runs.get_batch_size import \
     get_batch_size  # pylint: disable=unused-import
 from target_setting_runs.jax_nesterov import create_lr_schedule_fn
 from target_setting_runs.pytorch_submission_base import \
     update_params  # pylint: disable=unused-import
+import torch
+from torch.optim.lr_scheduler import LambdaLR
+
+from algorithmic_efficiency import spec
 
 
 def init_optimizer_state(workload: spec.Workload,

diff --git a/tests/submission_runner_test.py b/tests/submission_runner_test.py
@@ -31,8 +31,7 @@ class SubmissionRunnerTest(parameterized.TestCase):
           testcase_name='mnist_jax',
           workload='mnist',
           framework='jax',
-          submission_path=(
-              f'{_MNIST_DEV_ALGO_DIR}/mnist_jax/submission.py'),
+          submission_path=(f'{_MNIST_DEV_ALGO_DIR}/mnist_jax/submission.py'),
           tuning_search_space=(
               f'{_MNIST_DEV_ALGO_DIR}/tuning_search_space.json')),
       dict(