Merge pull request #378 from EducationalTestingService/make-copy-data…

…-files-package-friendly Make tests run with package and add missing test file
EducationalTestingService · Feb 24, 2020 · 822434b · 822434b
2 parents 73c16d6 + 1ce2cb9
commit 822434b
Show file tree

Hide file tree

Showing 11 changed files with 47 additions and 35 deletions.
diff --git a/.coveragerc b/.coveragerc
@@ -8,4 +8,3 @@ omit =
     */site-packages/nose/*
     */unittest2/*
     *rsmtool/notebooks/templates/report.tpl
-    *rsmtool/test_utils.py
diff --git a/.travis.yml b/.travis.yml
@@ -12,7 +12,7 @@ env:
   matrix:
   - TESTFILES="tests/test_experiment_rsmtool_1.py"
   - TESTFILES="tests/test_comparer.py tests/test_configuration_parser.py tests/test_experiment_rsmtool_2.py"
-  - TESTFILES="tests/test_analyzer.py tests/test_experiment_rsmeval.py tests/test_fairness_utils.py tests/test_prmse_utils.py tests/test_container.py"
+  - TESTFILES="tests/test_analyzer.py tests/test_experiment_rsmeval.py tests/test_fairness_utils.py tests/test_prmse_utils.py tests/test_container.py tests/test_test_utils.py"
   - TESTFILES="tests/test_experiment_rsmcompare.py tests/test_experiment_rsmsummarize.py tests/test_modeler.py tests/test_preprocessor.py tests/test_writer.py tests/test_experiment_rsmtool_3.py"
   - TESTFILES="tests/test_experiment_rsmpredict.py tests/test_reader.py tests/test_reporter.py tests/test_transformer.py tests/test_utils.py tests/test_experiment_rsmtool_4.py"
 sudo: false

diff --git a/DistributeTests.ps1 b/DistributeTests.ps1
@@ -42,6 +42,7 @@ elseif ($agentNumber -eq 3) {
     $testsToRun = $testsToRun + "tests/test_experiment_rsmeval.py"
     $testsToRun = $testsToRun + "tests/test_fairness_utils.py"
     $testsToRun = $testsToRun + "tests/test_prmse_utils.py"
+    $testsToRun = $testsToRun + "tests/test_test_utils.py"
 }
 elseif ($agentNumber -eq 4) {
     $testsToRun = $testsToRun + "tests/test_experiment_rsmcompare.py"

diff --git a/rsmtool/test_utils.py b/rsmtool/test_utils.py
@@ -929,9 +929,9 @@ def check_subgroup_outputs(output_dir, experiment_id, subgroups, file_format='cs
 
 def copy_data_files(temp_dir_name,
                     input_file_dict,
-                    given_test_dir=None):
+                    given_test_dir):
     """
-    A utility function to copy files from the ``tests/data`` directory into
+    A utility function to copy files from the given test directory into
     a specified temporary directory. Useful for tests where the
     current directory is to be used as the reference for resolving paths
     in the configuration.
@@ -944,10 +944,9 @@ def copy_data_files(temp_dir_name,
         A dictionary of files/directories to copy with keys as the
         file type and the values are their paths relative to the `tests`
         directory.
-    given_test_dir : str, optional
-        Directory where the the test experiments are located. Unless specified, the
-        rsmtool test directory is used. This can be useful when using these
-        experiments to run tests for RSMExtra.
+    given_test_dir : str
+        Directory where the the test experiments are located. This can be
+        useful when using these experiments to run tests for RSMExtra.
 
     Returns
     -------
@@ -956,9 +955,6 @@ def copy_data_files(temp_dir_name,
         input_file_dict and values showing new paths.
     """
 
-    # use the test directory from this file unless it's been overridden
-    test_dir = given_test_dir if given_test_dir else rsmtool_test_dir
-
     temp_dir = Path(temp_dir_name)
     if not temp_dir.exists():
         temp_dir.mkdir()
@@ -967,7 +963,7 @@ def copy_data_files(temp_dir_name,
     for file in input_file_dict:
         filepath = Path(input_file_dict[file])
         filename = filepath.name
-        old_filepath = test_dir / filepath
+        old_filepath = given_test_dir / filepath
         new_filepath = temp_dir / filename
         if old_filepath.is_dir():
             copytree(old_filepath, new_filepath)

diff --git a/tests/test_configuration_parser.py b/tests/test_configuration_parser.py
@@ -582,7 +582,7 @@ def test_init_with_filepath_as_positional_argument(self):
             config_dict = {'exp_id': 'my_experiment'}
             config = Configuration(config_dict, filepath)
             eq_(config._filename, 'file.json')
-            eq_(config._configdir, abspath('some/path/'))
+            eq_(config._configdir, abspath('some/path'))
             assert len(w) == 1
             assert issubclass(w[-1].category, DeprecationWarning)
 

diff --git a/tests/test_experiment_rsmcompare.py b/tests/test_experiment_rsmcompare.py
@@ -100,7 +100,8 @@ def test_run_experiment_lr_compare_with_dictionary():
     old_file_dict = {'experiment_dir': 'data/experiments/lr-self-compare-dict/lr-subgroups'}
 
     new_file_dict = copy_data_files(temp_dir.name,
-                                    old_file_dict)
+                                    old_file_dict,
+                                    rsmtool_test_dir)
 
     config_dict = {"comparison_id": "lr_self_compare_dict",
                    "experiment_dir_old": new_file_dict['experiment_dir'],
@@ -114,13 +115,11 @@ def test_run_experiment_lr_compare_with_dictionary():
                    "subgroups": ["QUESTION"]
                    }
 
-
     check_run_comparison(source,
                          experiment_id,
                          config_obj_or_dict=config_dict)
 
 
-
 @raises(ValueError)
 def test_run_comparison_wrong_input_format():
     config_list = [('experiment_id', 'AAAA'),

diff --git a/tests/test_experiment_rsmeval.py b/tests/test_experiment_rsmeval.py
@@ -133,7 +133,9 @@ def test_run_experiment_lr_eval_with_dictionary():
 
     old_file_dict = {'pred': 'data/files/predictions_scaled_with_subgroups.csv'}
 
-    new_file_dict = copy_data_files(temp_dir.name, old_file_dict)
+    new_file_dict = copy_data_files(temp_dir.name,
+                                    old_file_dict,
+                                    rsmtool_test_dir)
 
     config_dict = {"predictions_file": new_file_dict['pred'],
                    "system_score_column": "score",

diff --git a/tests/test_experiment_rsmpredict.py b/tests/test_experiment_rsmpredict.py
@@ -121,13 +121,13 @@ def test_run_experiment_lr_predict_with_object():
                    "experiment_id": "lr"
                    }
 
-
     config_parser = ConfigurationParser()
     config_parser.load_config_from_dict(config_dict,
                                         configdir=configdir)
     config_obj = config_parser.normalize_validate_and_process_config(context='rsmpredict')
 
     check_run_prediction(source,
+                         given_test_dir=rsmtool_test_dir,
                          config_obj_or_dict=config_obj)
 
 
@@ -146,15 +146,16 @@ def test_run_experiment_lr_predict_with_dictionary():
                      'experiment_dir': 'data/experiments/lr-predict-dict/existing_experiment'}
 
     new_file_dict = copy_data_files(temp_dir.name,
-                                    old_file_dict)
+                                    old_file_dict,
+                                    rsmtool_test_dir)
 
     config_dict = {"id_column": "ID",
                    "input_features_file": new_file_dict['feature_file'],
                    "experiment_dir": new_file_dict['experiment_dir'],
                    "experiment_id": "lr"}
 
-
     check_run_prediction(source,
+                         given_test_dir=rsmtool_test_dir,
                          config_obj_or_dict=config_dict)
 
 

diff --git a/tests/test_experiment_rsmsummarize.py b/tests/test_experiment_rsmsummarize.py
@@ -85,7 +85,9 @@ def test_run_experiment_lr_summary_dictionary():
 
     old_file_dict = {'experiment_dir': 'data/experiments/lr-self-summary-dict/lr-subgroups'}
 
-    new_file_dict = copy_data_files(temp_dir.name, old_file_dict)
+    new_file_dict = copy_data_files(temp_dir.name,
+                                    old_file_dict,
+                                    rsmtool_test_dir)
 
     config_dict = {"summary_id": "model_comparison",
                    "experiment_dirs": [new_file_dict['experiment_dir'],
@@ -96,7 +98,6 @@ def test_run_experiment_lr_summary_dictionary():
     check_run_summary(source, config_obj_or_dict=config_dict)
 
 
-
 def test_run_experiment_lr_summary_no_trim():
     '''
     Experiment to check the condition where no trim values can be located

diff --git a/tests/test_experiment_rsmtool_3.py b/tests/test_experiment_rsmtool_3.py
@@ -28,6 +28,7 @@
 else:
     from rsmtool.test_utils import rsmtool_test_dir
 
+
 @parameterized([
     param('lr-no-standardization', 'lr_no_standardization'),
     param('lr-exclude-test-flags', 'lr_exclude_test_flags'),
@@ -137,10 +138,10 @@ def test_run_experiment_lr_with_object_no_configdir():
                      'test': 'data/files/test.csv',
                      'features': 'data/experiments/lr-object-no-path/features.csv'}
 
-
     temp_dir = tempfile.TemporaryDirectory(prefix=getcwd())
     new_file_dict = copy_data_files(temp_dir.name,
-                                    old_file_dict)
+                                    old_file_dict,
+                                    rsmtool_test_dir)
 
     config_dict = {"train_file": new_file_dict['train'],
                    "id_column": "ID",
@@ -175,7 +176,8 @@ def test_run_experiment_lr_with_dictionary():
 
     temp_dir = tempfile.TemporaryDirectory(prefix=getcwd())
     new_file_dict = copy_data_files(temp_dir.name,
-                                    old_file_dict)
+                                    old_file_dict,
+                                    rsmtool_test_dir)
 
     config_dict = {"train_file": new_file_dict['train'],
                    "id_column": "ID",

diff --git a/tests/test_test_utils.py b/tests/test_test_utils.py
@@ -1,11 +1,21 @@
+import os
 import shutil
 
+from os.path import join
 from pathlib import Path
 
 from nose.tools import ok_, eq_
 
 from rsmtool.test_utils import copy_data_files
 
+# allow test directory to be set via an environment variable
+# which is needed for package testing
+TEST_DIR = os.environ.get('TESTDIR', None)
+if TEST_DIR:
+    rsmtool_test_dir = TEST_DIR
+else:
+    from rsmtool.test_utils import rsmtool_test_dir
+
 
 class TestCopyData():
 
@@ -19,37 +29,38 @@ def tearDown(self):
     def test_copy_data_files(self):
         file_dict = {'train': 'data/files/train.csv',
                      'features': 'data/experiments/lr/features.csv'}
-        expected_dict = {'train': 'temp_test_copy_data_file/train.csv',
-                         'features': 'temp_test_copy_data_file/features.csv'}
+        expected_dict = {'train': join('temp_test_copy_data_file', 'train.csv'),
+                         'features': join('temp_test_copy_data_file', 'features.csv')}
         self.dirs_to_remove.append('temp_test_copy_data_file')
         output_dict = copy_data_files('temp_test_copy_data_file',
-                                      file_dict)
+                                      file_dict,
+                                      rsmtool_test_dir)
         for file_type in expected_dict:
             eq_(output_dict[file_type], expected_dict[file_type])
             ok_(Path(output_dict[file_type]).exists())
             ok_(Path(output_dict[file_type]).is_file())
 
-
     def test_copy_data_files_directory(self):
         file_dict = {'exp_dir': 'data/experiments/lr-self-compare/lr-subgroups'}
-        expected_dict = {'exp_dir': 'temp_test_copy_dirs/lr-subgroups'}
+        expected_dict = {'exp_dir': join('temp_test_copy_dirs', 'lr-subgroups')}
         self.dirs_to_remove.append('temp_test_copy_dirs')
         output_dict = copy_data_files('temp_test_copy_dirs',
-                                      file_dict)
+                                      file_dict,
+                                      rsmtool_test_dir)
         for file_type in expected_dict:
             eq_(output_dict[file_type], expected_dict[file_type])
             ok_(Path(output_dict[file_type]).exists())
             ok_(Path(output_dict[file_type]).is_dir())
 
-
     def test_copy_data_files_files_and_directories(self):
         file_dict = {'exp_dir': 'data/experiments/lr-self-compare/lr-subgroups',
                      'test': 'data/files/test.csv'}
-        expected_dict = {'exp_dir': 'temp_test_copy_mixed/lr-subgroups',
-                         'test': 'temp_test_copy_mixed/test.csv'}
+        expected_dict = {'exp_dir': join('temp_test_copy_mixed', 'lr-subgroups'),
+                         'test': join('temp_test_copy_mixed', 'test.csv')}
         self.dirs_to_remove.append('temp_test_copy_mixed')
         output_dict = copy_data_files('temp_test_copy_mixed',
-                                      file_dict)
+                                      file_dict,
+                                      rsmtool_test_dir)
         for file_type in expected_dict:
             eq_(output_dict[file_type], expected_dict[file_type])
             ok_(Path(output_dict[file_type]).exists())