diff --git a/script/jenkins/test_data_api.py b/script/jenkins/test_data_api.py index 38eec4cd3..e31775b81 100644 --- a/script/jenkins/test_data_api.py +++ b/script/jenkins/test_data_api.py @@ -22,11 +22,9 @@ import datetime as dt import unittest from pathlib import Path -from sys import dont_write_bytecode import numpy as np -import pandas as pd -import xmlrunner +import pytest from pandas_datareader import wb from climada import CONFIG @@ -125,8 +123,7 @@ def test_icon_centroids_download(self): # Execute Tests if __name__ == "__main__": - TESTS = unittest.TestLoader().loadTestsFromTestCase(TestDataAvail) from sys import argv outputdir = argv[1] if len(argv) > 1 else str(Path.cwd().joinpath("tests_xml")) - xmlrunner.XMLTestRunner(output=outputdir).run(TESTS) + pytest.main([f"--junitxml={outputdir}/tests.xml", __file__]) diff --git a/script/jenkins/test_notebooks.py b/script/jenkins/test_notebooks.py index f2e4fcdbc..c7b969b0f 100644 --- a/script/jenkins/test_notebooks.py +++ b/script/jenkins/test_notebooks.py @@ -8,8 +8,7 @@ from pathlib import Path import nbformat - -import climada +import pytest BOUND_TO_FAIL = "# Note: execution of this cell will fail" """Cells containing this line will not be executed in the test""" @@ -18,160 +17,126 @@ """These notebooks are excluded from being tested""" -class NotebookTest(unittest.TestCase): - """Generic TestCase for testing the executability of notebooks - - Attributes - ---------- - wd : str - Absolute Path to the working directory, i.e., the directory of the notebook. - notebook : str - File name of the notebook. - - """ - - def __init__(self, methodName, wd=None, notebook=None): - super(NotebookTest, self).__init__(methodName) - self.wd = wd - self.notebook = notebook - - def test_notebook(self): - """Extracts code cells from the notebook and executes them one by one, using `exec`. - Magic lines and help/? calls are eliminated. - Cells containing `BOUND_TO_FAIL` are elided. - Cells doing multiprocessing are elided.""" - - cwd = Path.cwd() - try: - # cd to the notebook directory - os.chdir(self.wd) - print(f"start testing {self.notebook}") - - # read the notebook into a string - with open(self.notebook, encoding="utf8") as nb: - content = nb.read() - - # parse the string with nbformat.reads - cells = nbformat.reads(content, 4)["cells"] - - # create namespace with IPython standards - namespace = dict() - exec("from IPython.display import display", namespace) - - # run all cells - i = 0 - for c in cells: - - # skip markdown cells - if c["cell_type"] != "code": - continue - i += 1 - - # skip deliberately failing cells - if BOUND_TO_FAIL in c["source"]: - continue - - # skip multiprocessing cells - if any( - [ - tabu in c["source"].split() - for tabu in [ - "import multiprocessing", - "from multiprocessing import", - ] +# collect test cases, one for each notebook in the docs (unless they're excluded) +NOTEBOOK_DIR = Path(__file__).parent.parent.parent.joinpath("doc", "tutorial") +NOTEBOOKS = [ + (f.absolute(), f.name) + for f in sorted(NOTEBOOK_DIR.iterdir()) + if os.path.splitext(f)[1] == (".ipynb") + and not f.name in EXCLUDED_FROM_NOTEBOOK_TEST +] + + +@pytest.mark.parametrize("nb, name", NOTEBOOKS) +def test_notebook(nb, name): + """Extracts code cells from the notebook and executes them one by one, using `exec`. + Magic lines and help/? calls are eliminated. + Cells containing `BOUND_TO_FAIL` are elided. + Cells doing multiprocessing are elided.""" + notebook = nb + cwd = Path.cwd() + try: + # cd to the notebook directory + os.chdir(notebook.absolute().parent) + print(f"start testing {notebook}") + + # read the notebook into a string + with open(notebook.name, encoding="utf8") as nb: + content = nb.read() + + # parse the string with nbformat.reads + cells = nbformat.reads(content, 4)["cells"] + + # create namespace with IPython standards + namespace = dict() + exec("from IPython.display import display", namespace) + + # run all cells + i = 0 + for c in cells: + + # skip markdown cells + if c["cell_type"] != "code": + continue + i += 1 + + # skip deliberately failing cells + if BOUND_TO_FAIL in c["source"]: + continue + + # skip multiprocessing cells + if any( + [ + tabu in c["source"].split() + for tabu in [ + "import multiprocessing", + "from multiprocessing import", ] - ): - print( - "\n".join( - [ - f"\nskip multiprocessing cell {i} in {self.notebook}", - "+" + "-" * 68 + "+", - c["source"], - ] - ) - ) - continue - - # remove non python lines and help calls which require user input - # or involve pools being opened/closed - python_code = "\n".join( - [ - re.sub(r"pool=\w+", "pool=None", ln) - for ln in c["source"].split("\n") - if not ln.startswith("%") - and not ln.startswith("help(") - and not ln.startswith("ask_ok(") - and not ln.startswith("ask_ok(") - and not ln.startswith( - "pool" - ) # by convention Pool objects are called pool - and not ln.strip().endswith("?") - and not re.search( - r"(\W|^)Pool\(", ln - ) # prevent Pool object creation - ] - ) - - # execute the python code - try: - exec(python_code, namespace) - - # report failures - except Exception as e: - failure = "\n".join( + ] + ): + print( + "\n".join( [ - f"notebook {self.notebook} cell {i} failed with {e.__class__}", - f"{e}", + f"\nskip multiprocessing cell {i} in {notebook.name}", "+" + "-" * 68 + "+", c["source"], ] ) - print(f"failed {self.notebook}") - print(failure) - self.fail(failure) - - print(f"succeeded {self.notebook}") - finally: - os.chdir(cwd) - - -def main(install_dir): - import xmlrunner - - sys.path.append(str(install_dir)) - - notebook_dir = install_dir.joinpath("doc", "tutorial") - """The path to the notebook directories.""" - - # list notebooks in the NOTEBOOK_DIR - notebooks = [ - f.absolute() - for f in sorted(notebook_dir.iterdir()) - if os.path.splitext(f)[1] == (".ipynb") - and not f.name in EXCLUDED_FROM_NOTEBOOK_TEST - ] - - # build a test suite with a test for each notebook - suite = unittest.TestSuite() - for notebook in notebooks: + ) + continue + + # remove non python lines and help calls which require user input + # or involve pools being opened/closed + python_code = "\n".join( + [ + re.sub(r"pool=\w+", "pool=None", ln) + for ln in c["source"].split("\n") + if not ln.startswith("%") + and not ln.startswith("help(") + and not ln.startswith("ask_ok(") + and not ln.startswith("ask_ok(") + and not ln.startswith( + "pool" + ) # by convention Pool objects are called pool + and not ln.strip().endswith("?") + and not re.search( + r"(\W|^)Pool\(", ln + ) # prevent Pool object creation + ] + ) + + # execute the python code + try: + exec(python_code, namespace) + + # report failures + except Exception as e: + failure = "\n".join( + [ + f"notebook {notebook.name} cell {i} failed with {e.__class__}", + f"{e}", + "+" + "-" * 68 + "+", + c["source"], + ] + ) + print(f"failed {notebook}") + print(failure) + raise e - class NBTest(NotebookTest): - pass + print(f"succeeded {notebook}") + finally: + os.chdir(cwd) - test_name = "_".join(notebook.stem.split()) - setattr(NBTest, test_name, NBTest.test_notebook) - suite.addTest(NBTest(test_name, notebook.parent, notebook.name)) +def main(): # run the tests and write xml reports to tests_xml - output_dir = install_dir.joinpath("tests_xml") - xmlrunner.XMLTestRunner(output=str(output_dir)).run(suite) + pytest.main([f"--junitxml=tests_xml/tests.xml", __file__]) if __name__ == "__main__": if sys.argv[1] == "report": - install_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path.cwd() - main(install_dir) + main() else: - jd, nb = os.path.split(sys.argv[1]) - unittest.TextTestRunner(verbosity=2).run(NotebookTest("test_notebook", jd, nb)) + nb = sys.argv[1] + test_notebook(Path(nb), nb) diff --git a/setup.py b/setup.py index 94514cf74..4ed63cb94 100644 --- a/setup.py +++ b/setup.py @@ -95,7 +95,6 @@ "xarray", "xlrd", "xlsxwriter", - "xmlrunner", ], extras_require={ "doc": DEPS_DOC,