moving back out of tests/

mlcommons · Oct 11, 2022 · ca1e283 · ca1e283
1 parent 3244026
commit ca1e283
Show file tree

Hide file tree

Showing 63 changed files with 20 additions and 19 deletions.
diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml
@@ -20,6 +20,7 @@ jobs:
         pylint algorithmic_efficiency
         pylint baselines
         pylint target_setting_runs
+        pylint reference_algorithms
         pylint submission_runner.py
         pylint tests
 

diff --git a/README.md b/README.md
@@ -153,8 +153,8 @@ python3 submission_runner.py \
     --framework=jax \
     --workload=mnist \
     --experiment_dir=/home/znado \
-    --submission_path=tests/reference_algorithms/mnist/mnist_jax/submission.py \
-    --tuning_search_space=tests/reference_algorithms/mnist/tuning_search_space.json
+    --submission_path=reference_algorithms/mnist/mnist_jax/submission.py \
+    --tuning_search_space=reference_algorithms/mnist/tuning_search_space.json
 ```
 
 ### PyTorch
@@ -164,8 +164,8 @@ python3 submission_runner.py \
     --framework=pytorch \
     --workload=mnist \
     --experiment_dir=/home/znado \
-    --submission_path=tests/reference_algorithms/mnist/mnist_pytorch/submission.py \
-    --tuning_search_space=tests/reference_algorithms/mnist/tuning_search_space.json
+    --submission_path=reference_algorithms/mnist/mnist_pytorch/submission.py \
+    --tuning_search_space=reference_algorithms/mnist/tuning_search_space.json
 ```
 
 When using multiple GPUs on a single node it is recommended to use PyTorch's
@@ -197,7 +197,7 @@ To run the below commands, use the versions installed via `pip install -e '.[dev
 
 To automatically fix formatting errors, run the following (*WARNING:* this will edit your code, so it is suggested to make a git commit first!):
 ```bash
-yapf -i -r -vv -p algorithmic_efficiency baselines target_setting_runs tests/reference_algorithms tests *.py
+yapf -i -r -vv -p algorithmic_efficiency baselines target_setting_runs reference_algorithms tests *.py
 ```
 
 To sort all import orderings, run the following:
@@ -215,9 +215,9 @@ To print out all offending pylint issues, run the following:
 pylint algorithmic_efficiency
 pylint baselines
 pylint target_setting_runs
-pylint tests/reference_algorithms
+pylint reference_algorithms
 pylint submission_runner.py
 pylint tests
 ```
 
-You can also use `python tests/reference_submission_tests.py` to run a single model update and two model evals for each workload using the reference algorithm in `tests/reference_algorithms/`.
+You can also use `python tests/reference_submission_tests.py` to run a single model update and two model evals for each workload using the reference algorithm in `reference_algorithms/`.
diff --git a/algorithmic_efficiency/workloads/librispeech_conformer/README.md b/algorithmic_efficiency/workloads/librispeech_conformer/README.md
@@ -36,4 +36,4 @@ and `trans.csv` which has path to features.csv and transcription data, individua
 ## Run workload
 
 > cd algorithmic_efficiency
-> python3 submission_runner.py     --framework=jax     --workload=librispeech_conformer     --submission_path=tests/reference_algorithms/librispeech_conformer/librispeech_jax/submission.py     --tuning_search_space=tests/reference_algorithms/librispeech_conformer/tuning_search_space.json --data_dir=algorithmic_efficiency/workloads/librispeech_conformer/work_dir/data --num_tuning_trials=1 --summary_log_dir=tests/reference_algorithms/librispeech_conformer/librispeech_jax/data_512_4_256_full_train_set/ --tokenizer_vocab_path=/home/smedapati_google_com/algorithmic-efficiency/algorithmic_efficiency/workloads/librispeech_conformer/spm_model.vocab
+> python3 submission_runner.py     --framework=jax     --workload=librispeech_conformer     --submission_path=reference_algorithms/librispeech_conformer/librispeech_jax/submission.py     --tuning_search_space=reference_algorithms/librispeech_conformer/tuning_search_space.json --data_dir=algorithmic_efficiency/workloads/librispeech_conformer/work_dir/data --num_tuning_trials=1 --summary_log_dir=reference_algorithms/librispeech_conformer/librispeech_jax/data_512_4_256_full_train_set/ --tokenizer_vocab_path=/home/smedapati_google_com/algorithmic-efficiency/algorithmic_efficiency/workloads/librispeech_conformer/spm_model.vocab
diff --git a/tests/reference_algorithms/__init__.py → reference_algorithms/__init__.py b/tests/reference_algorithms/__init__.py → reference_algorithms/__init__.py
diff --git a/tests/reference_algorithms/cifar/__init__.py → reference_algorithms/cifar/__init__.py b/tests/reference_algorithms/cifar/__init__.py → reference_algorithms/cifar/__init__.py
diff --git a/...ce_algorithms/cifar/cifar_jax/__init__.py → ...ce_algorithms/cifar/cifar_jax/__init__.py b/...ce_algorithms/cifar/cifar_jax/__init__.py → ...ce_algorithms/cifar/cifar_jax/__init__.py
diff --git a/..._algorithms/cifar/cifar_jax/submission.py → ..._algorithms/cifar/cifar_jax/submission.py b/..._algorithms/cifar/cifar_jax/submission.py → ..._algorithms/cifar/cifar_jax/submission.py
diff --git a/...orithms/cifar/cifar_pytorch/submission.py → ...orithms/cifar/cifar_pytorch/submission.py b/...orithms/cifar/cifar_pytorch/submission.py → ...orithms/cifar/cifar_pytorch/submission.py
diff --git a/...algorithms/cifar/tuning_search_space.json → ...algorithms/cifar/tuning_search_space.json b/...algorithms/cifar/tuning_search_space.json → ...algorithms/cifar/tuning_search_space.json
diff --git a/...ithms/criteo1tb/criteo1tb_jax/__init__.py → ...ithms/criteo1tb/criteo1tb_jax/__init__.py b/...ithms/criteo1tb/criteo1tb_jax/__init__.py → ...ithms/criteo1tb/criteo1tb_jax/__init__.py
diff --git a/...hms/criteo1tb/criteo1tb_jax/submission.py → ...hms/criteo1tb/criteo1tb_jax/submission.py b/...hms/criteo1tb/criteo1tb_jax/submission.py → ...hms/criteo1tb/criteo1tb_jax/submission.py
diff --git a/...s/criteo1tb/criteo1tb_pytorch/__init__.py → ...s/criteo1tb/criteo1tb_pytorch/__init__.py b/...s/criteo1tb/criteo1tb_pytorch/__init__.py → ...s/criteo1tb/criteo1tb_pytorch/__init__.py
diff --git a/...criteo1tb/criteo1tb_pytorch/submission.py → ...criteo1tb/criteo1tb_pytorch/submission.py b/...criteo1tb/criteo1tb_pytorch/submission.py → ...criteo1tb/criteo1tb_pytorch/submission.py
diff --git a/...rithms/criteo1tb/tuning_search_space.json → ...rithms/criteo1tb/tuning_search_space.json b/...rithms/criteo1tb/tuning_search_space.json → ...rithms/criteo1tb/tuning_search_space.json
diff --git a/.../reference_algorithms/fastmri/__init__.py → reference_algorithms/fastmri/__init__.py b/.../reference_algorithms/fastmri/__init__.py → reference_algorithms/fastmri/__init__.py
diff --git a/...lgorithms/fastmri/fastmri_jax/__init__.py → ...lgorithms/fastmri/fastmri_jax/__init__.py b/...lgorithms/fastmri/fastmri_jax/__init__.py → ...lgorithms/fastmri/fastmri_jax/__init__.py
diff --git a/...orithms/fastmri/fastmri_jax/submission.py → ...orithms/fastmri/fastmri_jax/submission.py b/...orithms/fastmri/fastmri_jax/submission.py → ...orithms/fastmri/fastmri_jax/submission.py
diff --git a/...ithms/fastmri/fastmri_pytorch/__init__.py → ...ithms/fastmri/fastmri_pytorch/__init__.py b/...ithms/fastmri/fastmri_pytorch/__init__.py → ...ithms/fastmri/fastmri_pytorch/__init__.py
diff --git a/...hms/fastmri/fastmri_pytorch/submission.py → ...hms/fastmri/fastmri_pytorch/submission.py b/...hms/fastmri/fastmri_pytorch/submission.py → ...hms/fastmri/fastmri_pytorch/submission.py
diff --git a/...gorithms/fastmri/tuning_search_space.json → ...gorithms/fastmri/tuning_search_space.json b/...gorithms/fastmri/tuning_search_space.json → ...gorithms/fastmri/tuning_search_space.json
diff --git a/...ce_algorithms/imagenet_resnet/__init__.py → ...ce_algorithms/imagenet_resnet/__init__.py b/...ce_algorithms/imagenet_resnet/__init__.py → ...ce_algorithms/imagenet_resnet/__init__.py
diff --git a/.../imagenet_resnet/imagenet_jax/__init__.py → .../imagenet_resnet/imagenet_jax/__init__.py b/.../imagenet_resnet/imagenet_jax/__init__.py → .../imagenet_resnet/imagenet_jax/__init__.py
diff --git a/...magenet_resnet/imagenet_jax/submission.py → ...magenet_resnet/imagenet_jax/submission.py b/...magenet_resnet/imagenet_jax/submission.py → ...magenet_resnet/imagenet_jax/submission.py
diff --git a/...genet_resnet/imagenet_pytorch/__init__.py → ...genet_resnet/imagenet_pytorch/__init__.py b/...genet_resnet/imagenet_pytorch/__init__.py → ...genet_resnet/imagenet_pytorch/__init__.py
diff --git a/...net_resnet/imagenet_pytorch/submission.py → ...net_resnet/imagenet_pytorch/submission.py b/...net_resnet/imagenet_pytorch/submission.py → ...net_resnet/imagenet_pytorch/submission.py
diff --git a/.../imagenet_resnet/tuning_search_space.json → .../imagenet_resnet/tuning_search_space.json b/.../imagenet_resnet/tuning_search_space.json → .../imagenet_resnet/tuning_search_space.json
diff --git a/...rence_algorithms/imagenet_vit/__init__.py → ...rence_algorithms/imagenet_vit/__init__.py b/...rence_algorithms/imagenet_vit/__init__.py → ...rence_algorithms/imagenet_vit/__init__.py
diff --git a/...hms/imagenet_vit/imagenet_jax/__init__.py → ...hms/imagenet_vit/imagenet_jax/__init__.py b/...hms/imagenet_vit/imagenet_jax/__init__.py → ...hms/imagenet_vit/imagenet_jax/__init__.py
diff --git a/...s/imagenet_vit/imagenet_jax/submission.py → ...s/imagenet_vit/imagenet_jax/submission.py b/...s/imagenet_vit/imagenet_jax/submission.py → ...s/imagenet_vit/imagenet_jax/submission.py
diff --git a/...imagenet_vit/imagenet_pytorch/__init__.py → ...imagenet_vit/imagenet_pytorch/__init__.py b/...imagenet_vit/imagenet_pytorch/__init__.py → ...imagenet_vit/imagenet_pytorch/__init__.py
diff --git a/...agenet_vit/imagenet_pytorch/submission.py → ...agenet_vit/imagenet_pytorch/submission.py b/...agenet_vit/imagenet_pytorch/submission.py → ...agenet_vit/imagenet_pytorch/submission.py
diff --git a/...hms/imagenet_vit/tuning_search_space.json → ...hms/imagenet_vit/tuning_search_space.json b/...hms/imagenet_vit/tuning_search_space.json → ...hms/imagenet_vit/tuning_search_space.json
diff --git a/...orithms/librispeech_conformer/__init__.py → ...orithms/librispeech_conformer/__init__.py b/...orithms/librispeech_conformer/__init__.py → ...orithms/librispeech_conformer/__init__.py
diff --git a/...ech_conformer/librispeech_jax/__init__.py → ...ech_conformer/librispeech_jax/__init__.py b/...ech_conformer/librispeech_jax/__init__.py → ...ech_conformer/librispeech_jax/__init__.py
diff --git a/...h_conformer/librispeech_jax/submission.py → ...h_conformer/librispeech_jax/submission.py b/...h_conformer/librispeech_jax/submission.py → ...h_conformer/librispeech_jax/submission.py
diff --git a/...conformer/librispeech_pytorch/__init__.py → ...conformer/librispeech_pytorch/__init__.py b/...conformer/librispeech_pytorch/__init__.py → ...conformer/librispeech_pytorch/__init__.py
diff --git a/...nformer/librispeech_pytorch/submission.py → ...nformer/librispeech_pytorch/submission.py b/...nformer/librispeech_pytorch/submission.py → ...nformer/librispeech_pytorch/submission.py
diff --git a/...speech_conformer/tuning_search_space.json → ...speech_conformer/tuning_search_space.json b/...speech_conformer/tuning_search_space.json → ...speech_conformer/tuning_search_space.json
diff --git a/...rithms/librispeech_deepspeech/__init__.py → ...rithms/librispeech_deepspeech/__init__.py b/...rithms/librispeech_deepspeech/__init__.py → ...rithms/librispeech_deepspeech/__init__.py
diff --git a/...ch_deepspeech/librispeech_jax/__init__.py → ...ch_deepspeech/librispeech_jax/__init__.py b/...ch_deepspeech/librispeech_jax/__init__.py → ...ch_deepspeech/librispeech_jax/__init__.py
diff --git a/..._deepspeech/librispeech_jax/submission.py → ..._deepspeech/librispeech_jax/submission.py b/..._deepspeech/librispeech_jax/submission.py → ..._deepspeech/librispeech_jax/submission.py
diff --git a/...peech_deepspeech/tuning_search_space.json → ...peech_deepspeech/tuning_search_space.json b/...peech_deepspeech/tuning_search_space.json → ...peech_deepspeech/tuning_search_space.json
diff --git a/tests/reference_algorithms/mnist/__init__.py → reference_algorithms/mnist/__init__.py b/tests/reference_algorithms/mnist/__init__.py → reference_algorithms/mnist/__init__.py
diff --git a/...ce_algorithms/mnist/mnist_jax/__init__.py → ...ce_algorithms/mnist/mnist_jax/__init__.py b/...ce_algorithms/mnist/mnist_jax/__init__.py → ...ce_algorithms/mnist/mnist_jax/__init__.py
diff --git a/..._algorithms/mnist/mnist_jax/submission.py → ..._algorithms/mnist/mnist_jax/submission.py b/..._algorithms/mnist/mnist_jax/submission.py → ..._algorithms/mnist/mnist_jax/submission.py
diff --git a/...lgorithms/mnist/mnist_pytorch/__init__.py → ...lgorithms/mnist/mnist_pytorch/__init__.py b/...lgorithms/mnist/mnist_pytorch/__init__.py → ...lgorithms/mnist/mnist_pytorch/__init__.py
diff --git a/...orithms/mnist/mnist_pytorch/submission.py → ...orithms/mnist/mnist_pytorch/submission.py b/...orithms/mnist/mnist_pytorch/submission.py → ...orithms/mnist/mnist_pytorch/submission.py
diff --git a/...algorithms/mnist/tuning_search_space.json → ...algorithms/mnist/tuning_search_space.json b/...algorithms/mnist/tuning_search_space.json → ...algorithms/mnist/tuning_search_space.json
diff --git a/tests/reference_algorithms/ogbg/__init__.py → reference_algorithms/ogbg/__init__.py b/tests/reference_algorithms/ogbg/__init__.py → reference_algorithms/ogbg/__init__.py
diff --git a/...ence_algorithms/ogbg/ogbg_jax/__init__.py → ...ence_algorithms/ogbg/ogbg_jax/__init__.py b/...ence_algorithms/ogbg/ogbg_jax/__init__.py → ...ence_algorithms/ogbg/ogbg_jax/__init__.py
diff --git a/...ce_algorithms/ogbg/ogbg_jax/submission.py → ...ce_algorithms/ogbg/ogbg_jax/submission.py b/...ce_algorithms/ogbg/ogbg_jax/submission.py → ...ce_algorithms/ogbg/ogbg_jax/submission.py
diff --git a/..._algorithms/ogbg/ogbg_pytorch/__init__.py → ..._algorithms/ogbg/ogbg_pytorch/__init__.py b/..._algorithms/ogbg/ogbg_pytorch/__init__.py → ..._algorithms/ogbg/ogbg_pytorch/__init__.py
diff --git a/...lgorithms/ogbg/ogbg_pytorch/submission.py → ...lgorithms/ogbg/ogbg_pytorch/submission.py b/...lgorithms/ogbg/ogbg_pytorch/submission.py → ...lgorithms/ogbg/ogbg_pytorch/submission.py
diff --git a/..._algorithms/ogbg/tuning_search_space.json → ..._algorithms/ogbg/tuning_search_space.json b/..._algorithms/ogbg/tuning_search_space.json → ..._algorithms/ogbg/tuning_search_space.json
diff --git a/tests/reference_algorithms/wmt/__init__.py → reference_algorithms/wmt/__init__.py b/tests/reference_algorithms/wmt/__init__.py → reference_algorithms/wmt/__init__.py
diff --git a/...e_algorithms/wmt/tuning_search_space.json → ...e_algorithms/wmt/tuning_search_space.json b/...e_algorithms/wmt/tuning_search_space.json → ...e_algorithms/wmt/tuning_search_space.json
diff --git a/...erence_algorithms/wmt/wmt_jax/__init__.py → reference_algorithms/wmt/wmt_jax/__init__.py b/...erence_algorithms/wmt/wmt_jax/__init__.py → reference_algorithms/wmt/wmt_jax/__init__.py
diff --git a/...ence_algorithms/wmt/wmt_jax/submission.py → ...ence_algorithms/wmt/wmt_jax/submission.py b/...ence_algorithms/wmt/wmt_jax/submission.py → ...ence_algorithms/wmt/wmt_jax/submission.py
diff --git a/...ce_algorithms/wmt/wmt_pytorch/__init__.py → ...ce_algorithms/wmt/wmt_pytorch/__init__.py b/...ce_algorithms/wmt/wmt_pytorch/__init__.py → ...ce_algorithms/wmt/wmt_pytorch/__init__.py
diff --git a/..._algorithms/wmt/wmt_pytorch/submission.py → ..._algorithms/wmt/wmt_pytorch/submission.py b/..._algorithms/wmt/wmt_pytorch/submission.py → ..._algorithms/wmt/wmt_pytorch/submission.py
diff --git a/submission_runner.py b/submission_runner.py
@@ -6,9 +6,9 @@
 python3 submission_runner.py \
     --workload=mnist \
     --framework=jax \
-    --submission_path=tests/reference_algorithms/mnist/mnist_jax/submission.py \
+    --submission_path=reference_algorithms/mnist/mnist_jax/submission.py \
     --tuning_ruleset=external \
-    --tuning_search_space=tests/reference_algorithms/mnist/tuning_search_space.json \
+    --tuning_search_space=reference_algorithms/mnist/tuning_search_space.json \
     --num_tuning_trials=3 \
     --experiment_dir=/home/username/codes/algorithmic-efficiency/experiment_dir
 

diff --git a/tests/reference_submission_tests.py b/tests/reference_submission_tests.py
@@ -8,8 +8,8 @@
 
 Assumes that each reference submission is using the external tuning ruleset and
 that it is defined in:
-"tests/reference_algorithms/{workload}/{workload}_{framework}/submission.py"
-"tests/reference_algorithms/{workload}/tuning_search_space.json".
+"reference_algorithms/{workload}/{workload}_{framework}/submission.py"
+"reference_algorithms/{workload}/tuning_search_space.json".
 """
 import copy
 import functools
@@ -362,9 +362,9 @@ def _make_paths(repo_location, framework, workload_name):
     dataset_name = workload_name.split('_')[0]
   else:
     dataset_name = workload_name
-  workload_dir = f'{repo_location}/tests/reference_algorithms/{workload_name}'
+  workload_dir = f'{repo_location}/reference_algorithms/{workload_name}'
   search_space_path = f'{workload_dir}/tuning_search_space.json'
-  submission_path = (f'tests/reference_algorithms/{workload_name}/'
+  submission_path = (f'reference_algorithms/{workload_name}/'
                      f'{dataset_name}_{framework}/submission.py')
   full_submission_path = f'{repo_location}/{submission_path}'
   if not os.path.exists(full_submission_path):
@@ -388,7 +388,7 @@ def test_submission(self):
     # Example: /home/znado/algorithmic-efficiency
     repo_location = '/'.join(self_location.split('/')[:-1])
     if FLAGS.all:
-      references_dir = f'{repo_location}/tests/reference_algorithms'
+      references_dir = f'{repo_location}/reference_algorithms'
       for workload_name in os.listdir(references_dir):
         for framework in ['jax', 'pytorch']:
           if framework == 'pytorch':

diff --git a/tests/submission_runner_test.py b/tests/submission_runner_test.py
@@ -30,17 +30,17 @@ class SubmissionRunnerTest(parameterized.TestCase):
           workload='mnist',
           framework='jax',
           submission_path=(
-              'tests/reference_algorithms/mnist/mnist_jax/submission.py'),
+              'reference_algorithms/mnist/mnist_jax/submission.py'),
           tuning_search_space=(
-              'tests/reference_algorithms/mnist/tuning_search_space.json')),
+              'reference_algorithms/mnist/tuning_search_space.json')),
       dict(
           testcase_name='mnist_pytorch',
           workload='mnist',
           framework='pytorch',
           submission_path=(
-              'tests/reference_algorithms/mnist/mnist_pytorch/submission.py'),
+              'reference_algorithms/mnist/mnist_pytorch/submission.py'),
           tuning_search_space=(
-              'tests/reference_algorithms/mnist/tuning_search_space.json')),
+              'reference_algorithms/mnist/tuning_search_space.json')),
   )
   def test_submission(self,
                       workload,