moving reference_submissions/ to tests/reference_algorithms/

mlcommons · Oct 11, 2022 · b2f6a0e · b2f6a0e
1 parent f714190
commit b2f6a0e
Show file tree

Hide file tree

Showing 63 changed files with 23 additions and 23 deletions.
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -20,7 +20,6 @@ jobs:
         pylint algorithmic_efficiency
         pylint baselines
         pylint target_setting_runs
-        pylint reference_submissions
         pylint submission_runner.py
         pylint tests
 

diff --git a/README.md b/README.md
@@ -153,8 +153,8 @@ python3 submission_runner.py \
     --framework=jax \
     --workload=mnist \
     --experiment_dir=/home/znado \
-    --submission_path=reference_submissions/mnist/mnist_jax/submission.py \
-    --tuning_search_space=reference_submissions/mnist/tuning_search_space.json
+    --submission_path=tests/reference_algorithms/mnist/mnist_jax/submission.py \
+    --tuning_search_space=tests/reference_algorithms/mnist/tuning_search_space.json
 ```
 
 ### PyTorch
@@ -164,8 +164,8 @@ python3 submission_runner.py \
     --framework=pytorch \
     --workload=mnist \
     --experiment_dir=/home/znado \
-    --submission_path=reference_submissions/mnist/mnist_pytorch/submission.py \
-    --tuning_search_space=reference_submissions/mnist/tuning_search_space.json
+    --submission_path=tests/reference_algorithms/mnist/mnist_pytorch/submission.py \
+    --tuning_search_space=tests/reference_algorithms/mnist/tuning_search_space.json
 ```
 
 When using multiple GPUs on a single node it is recommended to use PyTorch's
@@ -197,7 +197,7 @@ To run the below commands, use the versions installed via `pip install -e '.[dev
 
 To automatically fix formatting errors, run the following (*WARNING:* this will edit your code, so it is suggested to make a git commit first!):
 ```bash
-yapf -i -r -vv -p algorithmic_efficiency baselines target_setting_runs reference_submissions tests *.py
+yapf -i -r -vv -p algorithmic_efficiency baselines target_setting_runs tests/reference_algorithms tests *.py
 ```
 
 To sort all import orderings, run the following:
@@ -215,9 +215,9 @@ To print out all offending pylint issues, run the following:
 pylint algorithmic_efficiency
 pylint baselines
 pylint target_setting_runs
-pylint reference_submissions
+pylint tests/reference_algorithms
 pylint submission_runner.py
 pylint tests
 ```
 
-You can also use `python tests/reference_submission_tests.py` to run a single model update and two model evals for each workload using the reference algorithm in `reference_submissions/`.
+You can also use `python tests/reference_submission_tests.py` to run a single model update and two model evals for each workload using the reference algorithm in `tests/reference_algorithms/`.
diff --git a/algorithmic_efficiency/workloads/librispeech_conformer/README.md b/algorithmic_efficiency/workloads/librispeech_conformer/README.md
@@ -28,12 +28,12 @@ this command will generate `spm_model.vocab` file in the folder it's run
 
 > mkdir work_dir
 > cd work_dir
-> python3 ../prepare_data.py --data_dir=../data_dir/LibriSpeech --tokenizer_vocab_path=../spm_model.vocab 
+> python3 ../prepare_data.py --data_dir=../data_dir/LibriSpeech --tokenizer_vocab_path=../spm_model.vocab
 
 the preprocessing script will generate `.npy` files for audio data, `features.csv` which have path to saved audio `.npy`
-and `trans.csv` which has path to features.csv and transcription data, individual data loaders 
+and `trans.csv` which has path to features.csv and transcription data, individual data loaders
 
-## Run workload 
+## Run workload
 
 > cd algorithmic_efficiency
-> python3 submission_runner.py     --framework=jax     --workload=librispeech_conformer     --submission_path=reference_submissions/librispeech_conformer/librispeech_jax/submission.py     --tuning_search_space=reference_submissions/librispeech_conformer/tuning_search_space.json --data_dir=algorithmic_efficiency/workloads/librispeech_conformer/work_dir/data --num_tuning_trials=1 --summary_log_dir=reference_submissions/librispeech_conformer/librispeech_jax/data_512_4_256_full_train_set/ --tokenizer_vocab_path=/home/smedapati_google_com/algorithmic-efficiency/algorithmic_efficiency/workloads/librispeech_conformer/spm_model.vocab 
+> python3 submission_runner.py     --framework=jax     --workload=librispeech_conformer     --submission_path=tests/reference_algorithms/librispeech_conformer/librispeech_jax/submission.py     --tuning_search_space=tests/reference_algorithms/librispeech_conformer/tuning_search_space.json --data_dir=algorithmic_efficiency/workloads/librispeech_conformer/work_dir/data --num_tuning_trials=1 --summary_log_dir=tests/reference_algorithms/librispeech_conformer/librispeech_jax/data_512_4_256_full_train_set/ --tokenizer_vocab_path=/home/smedapati_google_com/algorithmic-efficiency/algorithmic_efficiency/workloads/librispeech_conformer/spm_model.vocab
diff --git a/submission_runner.py b/submission_runner.py
@@ -5,9 +5,9 @@
 python3 submission_runner.py \
     --workload=mnist \
     --framework=jax \
-    --submission_path=reference_submissions/mnist/mnist_jax/submission.py \
+    --submission_path=tests/reference_algorithms/mnist/mnist_jax/submission.py \
     --tuning_ruleset=external \
-    --tuning_search_space=reference_submissions/mnist/tuning_search_space.json \
+    --tuning_search_space=tests/reference_algorithms/mnist/tuning_search_space.json \
     --num_tuning_trials=3 \
     --experiment_dir=/home/username/codes/algorithmic-efficiency/experiment_dir
 """

diff --git a/reference_submissions/__init__.py → tests/reference_algorithms/__init__.py b/reference_submissions/__init__.py → tests/reference_algorithms/__init__.py
diff --git a/reference_submissions/cifar/__init__.py → tests/reference_algorithms/cifar/__init__.py b/reference_submissions/cifar/__init__.py → tests/reference_algorithms/cifar/__init__.py
diff --git a/...e_submissions/cifar/cifar_jax/__init__.py → ...ce_algorithms/cifar/cifar_jax/__init__.py b/...e_submissions/cifar/cifar_jax/__init__.py → ...ce_algorithms/cifar/cifar_jax/__init__.py
diff --git a/...submissions/cifar/cifar_jax/submission.py → ..._algorithms/cifar/cifar_jax/submission.py b/...submissions/cifar/cifar_jax/submission.py → ..._algorithms/cifar/cifar_jax/submission.py
diff --git a/...issions/cifar/cifar_pytorch/submission.py → ...orithms/cifar/cifar_pytorch/submission.py b/...issions/cifar/cifar_pytorch/submission.py → ...orithms/cifar/cifar_pytorch/submission.py
diff --git a/...ubmissions/cifar/tuning_search_space.json → ...algorithms/cifar/tuning_search_space.json b/...ubmissions/cifar/tuning_search_space.json → ...algorithms/cifar/tuning_search_space.json
diff --git a/...sions/criteo1tb/criteo1tb_jax/__init__.py → ...ithms/criteo1tb/criteo1tb_jax/__init__.py b/...sions/criteo1tb/criteo1tb_jax/__init__.py → ...ithms/criteo1tb/criteo1tb_jax/__init__.py
diff --git a/...ons/criteo1tb/criteo1tb_jax/submission.py → ...hms/criteo1tb/criteo1tb_jax/submission.py b/...ons/criteo1tb/criteo1tb_jax/submission.py → ...hms/criteo1tb/criteo1tb_jax/submission.py
diff --git a/...s/criteo1tb/criteo1tb_pytorch/__init__.py → ...s/criteo1tb/criteo1tb_pytorch/__init__.py b/...s/criteo1tb/criteo1tb_pytorch/__init__.py → ...s/criteo1tb/criteo1tb_pytorch/__init__.py
diff --git a/...criteo1tb/criteo1tb_pytorch/submission.py → ...criteo1tb/criteo1tb_pytorch/submission.py b/...criteo1tb/criteo1tb_pytorch/submission.py → ...criteo1tb/criteo1tb_pytorch/submission.py
diff --git a/...ssions/criteo1tb/tuning_search_space.json → ...rithms/criteo1tb/tuning_search_space.json b/...ssions/criteo1tb/tuning_search_space.json → ...rithms/criteo1tb/tuning_search_space.json
diff --git a/reference_submissions/fastmri/__init__.py → .../reference_algorithms/fastmri/__init__.py b/reference_submissions/fastmri/__init__.py → .../reference_algorithms/fastmri/__init__.py
diff --git a/...bmissions/fastmri/fastmri_jax/__init__.py → ...lgorithms/fastmri/fastmri_jax/__init__.py b/...bmissions/fastmri/fastmri_jax/__init__.py → ...lgorithms/fastmri/fastmri_jax/__init__.py
diff --git a/...issions/fastmri/fastmri_jax/submission.py → ...orithms/fastmri/fastmri_jax/submission.py b/...issions/fastmri/fastmri_jax/submission.py → ...orithms/fastmri/fastmri_jax/submission.py
diff --git a/...sions/fastmri/fastmri_pytorch/__init__.py → ...ithms/fastmri/fastmri_pytorch/__init__.py b/...sions/fastmri/fastmri_pytorch/__init__.py → ...ithms/fastmri/fastmri_pytorch/__init__.py
diff --git a/...ons/fastmri/fastmri_pytorch/submission.py → ...hms/fastmri/fastmri_pytorch/submission.py b/...ons/fastmri/fastmri_pytorch/submission.py → ...hms/fastmri/fastmri_pytorch/submission.py
diff --git a/...missions/fastmri/tuning_search_space.json → ...gorithms/fastmri/tuning_search_space.json b/...missions/fastmri/tuning_search_space.json → ...gorithms/fastmri/tuning_search_space.json
diff --git a/...e_submissions/imagenet_resnet/__init__.py → ...ce_algorithms/imagenet_resnet/__init__.py b/...e_submissions/imagenet_resnet/__init__.py → ...ce_algorithms/imagenet_resnet/__init__.py
diff --git a/.../imagenet_resnet/imagenet_jax/__init__.py → .../imagenet_resnet/imagenet_jax/__init__.py b/.../imagenet_resnet/imagenet_jax/__init__.py → .../imagenet_resnet/imagenet_jax/__init__.py
diff --git a/...magenet_resnet/imagenet_jax/submission.py → ...magenet_resnet/imagenet_jax/submission.py b/...magenet_resnet/imagenet_jax/submission.py → ...magenet_resnet/imagenet_jax/submission.py
diff --git a/...genet_resnet/imagenet_pytorch/__init__.py → ...genet_resnet/imagenet_pytorch/__init__.py b/...genet_resnet/imagenet_pytorch/__init__.py → ...genet_resnet/imagenet_pytorch/__init__.py
diff --git a/...net_resnet/imagenet_pytorch/submission.py → ...net_resnet/imagenet_pytorch/submission.py b/...net_resnet/imagenet_pytorch/submission.py → ...net_resnet/imagenet_pytorch/submission.py
diff --git a/.../imagenet_resnet/tuning_search_space.json → .../imagenet_resnet/tuning_search_space.json b/.../imagenet_resnet/tuning_search_space.json → .../imagenet_resnet/tuning_search_space.json
diff --git a/...ence_submissions/imagenet_vit/__init__.py → ...rence_algorithms/imagenet_vit/__init__.py b/...ence_submissions/imagenet_vit/__init__.py → ...rence_algorithms/imagenet_vit/__init__.py
diff --git a/...ons/imagenet_vit/imagenet_jax/__init__.py → ...hms/imagenet_vit/imagenet_jax/__init__.py b/...ons/imagenet_vit/imagenet_jax/__init__.py → ...hms/imagenet_vit/imagenet_jax/__init__.py
diff --git a/...s/imagenet_vit/imagenet_jax/submission.py → ...s/imagenet_vit/imagenet_jax/submission.py b/...s/imagenet_vit/imagenet_jax/submission.py → ...s/imagenet_vit/imagenet_jax/submission.py
diff --git a/...imagenet_vit/imagenet_pytorch/__init__.py → ...imagenet_vit/imagenet_pytorch/__init__.py b/...imagenet_vit/imagenet_pytorch/__init__.py → ...imagenet_vit/imagenet_pytorch/__init__.py
diff --git a/...agenet_vit/imagenet_pytorch/submission.py → ...agenet_vit/imagenet_pytorch/submission.py b/...agenet_vit/imagenet_pytorch/submission.py → ...agenet_vit/imagenet_pytorch/submission.py
diff --git a/...ons/imagenet_vit/tuning_search_space.json → ...hms/imagenet_vit/tuning_search_space.json b/...ons/imagenet_vit/tuning_search_space.json → ...hms/imagenet_vit/tuning_search_space.json
diff --git a/...issions/librispeech_conformer/__init__.py → ...orithms/librispeech_conformer/__init__.py b/...issions/librispeech_conformer/__init__.py → ...orithms/librispeech_conformer/__init__.py
diff --git a/...ech_conformer/librispeech_jax/__init__.py → ...ech_conformer/librispeech_jax/__init__.py b/...ech_conformer/librispeech_jax/__init__.py → ...ech_conformer/librispeech_jax/__init__.py
diff --git a/...h_conformer/librispeech_jax/submission.py → ...h_conformer/librispeech_jax/submission.py b/...h_conformer/librispeech_jax/submission.py → ...h_conformer/librispeech_jax/submission.py
diff --git a/...conformer/librispeech_pytorch/__init__.py → ...conformer/librispeech_pytorch/__init__.py b/...conformer/librispeech_pytorch/__init__.py → ...conformer/librispeech_pytorch/__init__.py
diff --git a/...nformer/librispeech_pytorch/submission.py → ...nformer/librispeech_pytorch/submission.py b/...nformer/librispeech_pytorch/submission.py → ...nformer/librispeech_pytorch/submission.py
diff --git a/...speech_conformer/tuning_search_space.json → ...speech_conformer/tuning_search_space.json b/...speech_conformer/tuning_search_space.json → ...speech_conformer/tuning_search_space.json
diff --git a/...ssions/librispeech_deepspeech/__init__.py → ...rithms/librispeech_deepspeech/__init__.py b/...ssions/librispeech_deepspeech/__init__.py → ...rithms/librispeech_deepspeech/__init__.py
diff --git a/...ch_deepspeech/librispeech_jax/__init__.py → ...ch_deepspeech/librispeech_jax/__init__.py b/...ch_deepspeech/librispeech_jax/__init__.py → ...ch_deepspeech/librispeech_jax/__init__.py
diff --git a/..._deepspeech/librispeech_jax/submission.py → ..._deepspeech/librispeech_jax/submission.py b/..._deepspeech/librispeech_jax/submission.py → ..._deepspeech/librispeech_jax/submission.py
diff --git a/...peech_deepspeech/tuning_search_space.json → ...peech_deepspeech/tuning_search_space.json b/...peech_deepspeech/tuning_search_space.json → ...peech_deepspeech/tuning_search_space.json
diff --git a/reference_submissions/mnist/__init__.py → tests/reference_algorithms/mnist/__init__.py b/reference_submissions/mnist/__init__.py → tests/reference_algorithms/mnist/__init__.py
diff --git a/...e_submissions/mnist/mnist_jax/__init__.py → ...ce_algorithms/mnist/mnist_jax/__init__.py b/...e_submissions/mnist/mnist_jax/__init__.py → ...ce_algorithms/mnist/mnist_jax/__init__.py
diff --git a/...submissions/mnist/mnist_jax/submission.py → ..._algorithms/mnist/mnist_jax/submission.py b/...submissions/mnist/mnist_jax/submission.py → ..._algorithms/mnist/mnist_jax/submission.py
diff --git a/...bmissions/mnist/mnist_pytorch/__init__.py → ...lgorithms/mnist/mnist_pytorch/__init__.py b/...bmissions/mnist/mnist_pytorch/__init__.py → ...lgorithms/mnist/mnist_pytorch/__init__.py
diff --git a/...issions/mnist/mnist_pytorch/submission.py → ...orithms/mnist/mnist_pytorch/submission.py b/...issions/mnist/mnist_pytorch/submission.py → ...orithms/mnist/mnist_pytorch/submission.py
diff --git a/...ubmissions/mnist/tuning_search_space.json → ...algorithms/mnist/tuning_search_space.json b/...ubmissions/mnist/tuning_search_space.json → ...algorithms/mnist/tuning_search_space.json
diff --git a/reference_submissions/ogbg/__init__.py → tests/reference_algorithms/ogbg/__init__.py b/reference_submissions/ogbg/__init__.py → tests/reference_algorithms/ogbg/__init__.py
diff --git a/...nce_submissions/ogbg/ogbg_jax/__init__.py → ...ence_algorithms/ogbg/ogbg_jax/__init__.py b/...nce_submissions/ogbg/ogbg_jax/__init__.py → ...ence_algorithms/ogbg/ogbg_jax/__init__.py
diff --git a/...e_submissions/ogbg/ogbg_jax/submission.py → ...ce_algorithms/ogbg/ogbg_jax/submission.py b/...e_submissions/ogbg/ogbg_jax/submission.py → ...ce_algorithms/ogbg/ogbg_jax/submission.py
diff --git a/...submissions/ogbg/ogbg_pytorch/__init__.py → ..._algorithms/ogbg/ogbg_pytorch/__init__.py b/...submissions/ogbg/ogbg_pytorch/__init__.py → ..._algorithms/ogbg/ogbg_pytorch/__init__.py
diff --git a/...bmissions/ogbg/ogbg_pytorch/submission.py → ...lgorithms/ogbg/ogbg_pytorch/submission.py b/...bmissions/ogbg/ogbg_pytorch/submission.py → ...lgorithms/ogbg/ogbg_pytorch/submission.py
diff --git a/...submissions/ogbg/tuning_search_space.json → ..._algorithms/ogbg/tuning_search_space.json b/...submissions/ogbg/tuning_search_space.json → ..._algorithms/ogbg/tuning_search_space.json
diff --git a/reference_submissions/wmt/__init__.py → tests/reference_algorithms/wmt/__init__.py b/reference_submissions/wmt/__init__.py → tests/reference_algorithms/wmt/__init__.py
diff --git a/..._submissions/wmt/tuning_search_space.json → ...e_algorithms/wmt/tuning_search_space.json b/..._submissions/wmt/tuning_search_space.json → ...e_algorithms/wmt/tuning_search_space.json
diff --git a/...rence_submissions/wmt/wmt_jax/__init__.py → ...erence_algorithms/wmt/wmt_jax/__init__.py b/...rence_submissions/wmt/wmt_jax/__init__.py → ...erence_algorithms/wmt/wmt_jax/__init__.py
diff --git a/...nce_submissions/wmt/wmt_jax/submission.py → ...ence_algorithms/wmt/wmt_jax/submission.py b/...nce_submissions/wmt/wmt_jax/submission.py → ...ence_algorithms/wmt/wmt_jax/submission.py
diff --git a/...e_submissions/wmt/wmt_pytorch/__init__.py → ...ce_algorithms/wmt/wmt_pytorch/__init__.py b/...e_submissions/wmt/wmt_pytorch/__init__.py → ...ce_algorithms/wmt/wmt_pytorch/__init__.py
diff --git a/...submissions/wmt/wmt_pytorch/submission.py → ..._algorithms/wmt/wmt_pytorch/submission.py b/...submissions/wmt/wmt_pytorch/submission.py → ..._algorithms/wmt/wmt_pytorch/submission.py
diff --git a/tests/reference_submission_tests.py b/tests/reference_submission_tests.py
@@ -8,8 +8,8 @@
 
 Assumes that each reference submission is using the external tuning ruleset and
 that it is defined in:
-"reference_submissions/{workload}/{workload}_{framework}/submission.py"
-"reference_submissions/{workload}/tuning_search_space.json".
+"tests/reference_algorithms/{workload}/{workload}_{framework}/submission.py"
+"tests/reference_algorithms/{workload}/tuning_search_space.json".
 """
 import copy
 import functools
@@ -362,9 +362,9 @@ def _make_paths(repo_location, framework, workload_name):
     dataset_name = workload_name.split('_')[0]
   else:
     dataset_name = workload_name
-  workload_dir = f'{repo_location}/reference_submissions/{workload_name}'
+  workload_dir = f'{repo_location}/tests/reference_algorithms/{workload_name}'
   search_space_path = f'{workload_dir}/tuning_search_space.json'
-  submission_path = (f'reference_submissions/{workload_name}/'
+  submission_path = (f'tests/reference_algorithms/{workload_name}/'
                      f'{dataset_name}_{framework}/submission.py')
   full_submission_path = f'{repo_location}/{submission_path}'
   if not os.path.exists(full_submission_path):
@@ -388,7 +388,7 @@ def test_submission(self):
     # Example: /home/znado/algorithmic-efficiency
     repo_location = '/'.join(self_location.split('/')[:-1])
     if FLAGS.all:
-      references_dir = f'{repo_location}/reference_submissions'
+      references_dir = f'{repo_location}/tests/reference_algorithms'
       for workload_name in os.listdir(references_dir):
         for framework in ['jax', 'pytorch']:
           if framework == 'pytorch':

diff --git a/tests/submission_runner_test.py b/tests/submission_runner_test.py
@@ -29,17 +29,18 @@ class SubmissionRunnerTest(parameterized.TestCase):
           testcase_name='mnist_jax',
           workload='mnist',
           framework='jax',
-          submission_path='reference_submissions/mnist/mnist_jax/submission.py',
+          submission_path=(
+              'tests/reference_algorithms/mnist/mnist_jax/submission.py'),
           tuning_search_space=(
-              'reference_submissions/mnist/tuning_search_space.json')),
+              'tests/reference_algorithms/mnist/tuning_search_space.json')),
       dict(
           testcase_name='mnist_pytorch',
           workload='mnist',
           framework='pytorch',
           submission_path=(
-              'reference_submissions/mnist/mnist_pytorch/submission.py'),
+              'tests/reference_algorithms/mnist/mnist_pytorch/submission.py'),
           tuning_search_space=(
-              'reference_submissions/mnist/tuning_search_space.json')),
+              'tests/reference_algorithms/mnist/tuning_search_space.json')),
   )
   def test_submission(self,
                       workload,