Skip to content

Commit

Permalink
dismiss xmlrunner
Browse files Browse the repository at this point in the history 
use pytest.mark.parametrize in lieu of a dynamic TestSuite for notebooks
  • Loading branch information
@emanuel-schmid
emanuel-schmid committed Jan 10, 2025
1 parent f796d1e commit db52d41
Show file tree
Hide file tree
Showing 3 changed files with 110 additions and 149 deletions.
7 changes: 2 additions & 5 deletions script/jenkins/test_data_api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,9 @@
import datetime as dt
import unittest
from pathlib import Path
from sys import dont_write_bytecode

import numpy as np
import pandas as pd
import xmlrunner
import pytest
from pandas_datareader import wb

from climada import CONFIG
Expand Down Expand Up @@ -125,8 +123,7 @@ def test_icon_centroids_download(self):

# Execute Tests
if __name__ == "__main__":
TESTS = unittest.TestLoader().loadTestsFromTestCase(TestDataAvail)
from sys import argv

outputdir = argv[1] if len(argv) > 1 else str(Path.cwd().joinpath("tests_xml"))
xmlrunner.XMLTestRunner(output=outputdir).run(TESTS)
pytest.main([f"--junitxml={outputdir}/tests.xml", __file__])
251 changes: 108 additions & 143 deletions script/jenkins/test_notebooks.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@
from pathlib import Path

import nbformat

import climada
import pytest

BOUND_TO_FAIL = "# Note: execution of this cell will fail"
"""Cells containing this line will not be executed in the test"""
Expand All @@ -18,160 +17,126 @@
"""These notebooks are excluded from being tested"""


class NotebookTest(unittest.TestCase):
"""Generic TestCase for testing the executability of notebooks
Attributes
----------
wd : str
Absolute Path to the working directory, i.e., the directory of the notebook.
notebook : str
File name of the notebook.
"""

def __init__(self, methodName, wd=None, notebook=None):
super(NotebookTest, self).__init__(methodName)
self.wd = wd
self.notebook = notebook

def test_notebook(self):
"""Extracts code cells from the notebook and executes them one by one, using `exec`.
Magic lines and help/? calls are eliminated.
Cells containing `BOUND_TO_FAIL` are elided.
Cells doing multiprocessing are elided."""

cwd = Path.cwd()
try:
# cd to the notebook directory
os.chdir(self.wd)
print(f"start testing {self.notebook}")

# read the notebook into a string
with open(self.notebook, encoding="utf8") as nb:
content = nb.read()

# parse the string with nbformat.reads
cells = nbformat.reads(content, 4)["cells"]

# create namespace with IPython standards
namespace = dict()
exec("from IPython.display import display", namespace)

# run all cells
i = 0
for c in cells:

# skip markdown cells
if c["cell_type"] != "code":
continue
i += 1

# skip deliberately failing cells
if BOUND_TO_FAIL in c["source"]:
continue

# skip multiprocessing cells
if any(
[
tabu in c["source"].split()
for tabu in [
"import multiprocessing",
"from multiprocessing import",
]
# collect test cases, one for each notebook in the docs (unless they're excluded)
NOTEBOOK_DIR = Path(__file__).parent.parent.parent.joinpath("doc", "tutorial")
NOTEBOOKS = [
(f.absolute(), f.name)
for f in sorted(NOTEBOOK_DIR.iterdir())
if os.path.splitext(f)[1] == (".ipynb")
and not f.name in EXCLUDED_FROM_NOTEBOOK_TEST
]


@pytest.mark.parametrize("nb, name", NOTEBOOKS)
def test_notebook(nb, name):
"""Extracts code cells from the notebook and executes them one by one, using `exec`.
Magic lines and help/? calls are eliminated.
Cells containing `BOUND_TO_FAIL` are elided.
Cells doing multiprocessing are elided."""
notebook = nb
cwd = Path.cwd()
try:
# cd to the notebook directory
os.chdir(notebook.absolute().parent)
print(f"start testing {notebook}")

# read the notebook into a string
with open(notebook.name, encoding="utf8") as nb:
content = nb.read()

# parse the string with nbformat.reads
cells = nbformat.reads(content, 4)["cells"]

# create namespace with IPython standards
namespace = dict()
exec("from IPython.display import display", namespace)

# run all cells
i = 0
for c in cells:

# skip markdown cells
if c["cell_type"] != "code":
continue
i += 1

# skip deliberately failing cells
if BOUND_TO_FAIL in c["source"]:
continue

# skip multiprocessing cells
if any(
[
tabu in c["source"].split()
for tabu in [
"import multiprocessing",
"from multiprocessing import",
]
):
print(
"\n".join(
[
f"\nskip multiprocessing cell {i} in {self.notebook}",
"+" + "-" * 68 + "+",
c["source"],
]
)
)
continue

# remove non python lines and help calls which require user input
# or involve pools being opened/closed
python_code = "\n".join(
[
re.sub(r"pool=\w+", "pool=None", ln)
for ln in c["source"].split("\n")
if not ln.startswith("%")
and not ln.startswith("help(")
and not ln.startswith("ask_ok(")
and not ln.startswith("ask_ok(")
and not ln.startswith(
"pool"
) # by convention Pool objects are called pool
and not ln.strip().endswith("?")
and not re.search(
r"(\W|^)Pool\(", ln
) # prevent Pool object creation
]
)

# execute the python code
try:
exec(python_code, namespace)

# report failures
except Exception as e:
failure = "\n".join(
]
):
print(
"\n".join(
[
f"notebook {self.notebook} cell {i} failed with {e.__class__}",
f"{e}",
f"\nskip multiprocessing cell {i} in {notebook.name}",
"+" + "-" * 68 + "+",
c["source"],
]
)
print(f"failed {self.notebook}")
print(failure)
self.fail(failure)

print(f"succeeded {self.notebook}")
finally:
os.chdir(cwd)


def main(install_dir):
import xmlrunner

sys.path.append(str(install_dir))

notebook_dir = install_dir.joinpath("doc", "tutorial")
"""The path to the notebook directories."""

# list notebooks in the NOTEBOOK_DIR
notebooks = [
f.absolute()
for f in sorted(notebook_dir.iterdir())
if os.path.splitext(f)[1] == (".ipynb")
and not f.name in EXCLUDED_FROM_NOTEBOOK_TEST
]

# build a test suite with a test for each notebook
suite = unittest.TestSuite()
for notebook in notebooks:
)
continue

# remove non python lines and help calls which require user input
# or involve pools being opened/closed
python_code = "\n".join(
[
re.sub(r"pool=\w+", "pool=None", ln)
for ln in c["source"].split("\n")
if not ln.startswith("%")
and not ln.startswith("help(")
and not ln.startswith("ask_ok(")
and not ln.startswith("ask_ok(")
and not ln.startswith(
"pool"
) # by convention Pool objects are called pool
and not ln.strip().endswith("?")
and not re.search(
r"(\W|^)Pool\(", ln
) # prevent Pool object creation
]
)

# execute the python code
try:
exec(python_code, namespace)

# report failures
except Exception as e:
failure = "\n".join(
[
f"notebook {notebook.name} cell {i} failed with {e.__class__}",
f"{e}",
"+" + "-" * 68 + "+",
c["source"],
]
)
print(f"failed {notebook}")
print(failure)
raise e

class NBTest(NotebookTest):
pass
print(f"succeeded {notebook}")
finally:
os.chdir(cwd)

test_name = "_".join(notebook.stem.split())
setattr(NBTest, test_name, NBTest.test_notebook)
suite.addTest(NBTest(test_name, notebook.parent, notebook.name))

def main():
# run the tests and write xml reports to tests_xml
output_dir = install_dir.joinpath("tests_xml")
xmlrunner.XMLTestRunner(output=str(output_dir)).run(suite)
pytest.main([f"--junitxml=tests_xml/tests.xml", __file__])


if __name__ == "__main__":
if sys.argv[1] == "report":
install_dir = Path(sys.argv[2]) if len(sys.argv) > 2 else Path.cwd()
main(install_dir)
main()

else:
jd, nb = os.path.split(sys.argv[1])
unittest.TextTestRunner(verbosity=2).run(NotebookTest("test_notebook", jd, nb))
nb = sys.argv[1]
test_notebook(Path(nb), nb)
1 change: 0 additions & 1 deletion setup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,6 @@
"xarray",
"xlrd",
"xlsxwriter",
"xmlrunner",
],
extras_require={
"doc": DEPS_DOC,
Expand Down

0 comments on commit db52d41

Please sign in to comment.