From d6e445df8610dc43309b8608eaec1941f2c9b49a Mon Sep 17 00:00:00 2001 From: michaeldeistler Date: Tue, 29 Oct 2024 17:32:37 +0100 Subject: [PATCH 01/37] Add runtime tests --- .github/workflows/tests.yml | 2 +- tests/test_runtime.py | 124 ++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 tests/test_runtime.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3eb90b0a..ff5ef5bc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,4 +39,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov - pytest tests/ --cov=jaxley --cov-report=xml + pytest tests/ -m "not runtime" --cov=jaxley --cov-report=xml diff --git a/tests/test_runtime.py b/tests/test_runtime.py new file mode 100644 index 00000000..bcb445bf --- /dev/null +++ b/tests/test_runtime.py @@ -0,0 +1,124 @@ +# This file is part of Jaxley, a differentiable neuroscience simulator. Jaxley is +# licensed under the Apache License Version 2.0, see + +import os +import time + +import numpy as np +import pytest +from jax import jit + +import jaxley as jx +from jaxley.channels import HH +from jaxley.connect import sparse_connect +from jaxley.synapses import IonotropicSynapse + + +def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): + _ = np.random.seed(1) # For sparse connectivity matrix. + + if artificial: + comp = jx.Compartment() + branch = jx.Branch(comp, 2) + depth = 3 + parents = [-1] + [b // 2 for b in range(0, 2**depth - 2)] + cell = jx.Cell(branch, parents=parents) + else: + dirname = os.path.dirname(__file__) + fname = os.path.join(dirname, "swc_files", "morph.swc") + cell = jx.read_swc(fname, nseg=4) + net = jx.Network([cell for _ in range(num_cells)]) + + # Channels. + net.insert(HH()) + + # Synapses. + if connect: + sparse_connect( + net.cell("all"), net.cell("all"), IonotropicSynapse(), connection_prob + ) + + # Recordings. + net[0, 1, 0].record(verbose=False) + + # Trainables. + net.make_trainable("radius", verbose=False) + params = net.get_parameters() + + net.to_jax() + return net, params + + +@pytest.mark.runtime +@pytest.mark.parametrize( + "num_cells, artificial, connect, connection_prob, voltage_solver, identifier", + ( + # Test a single SWC cell with both solvers. + pytest.param(1, False, False, 0.0, "jaxley.stone", 0), + pytest.param(1, False, False, 0.0, "jax.sparse", 1), + # Test a network of SWC cells with both solvers. + pytest.param(10, False, True, 0.1, "jaxley.stone", 2), + pytest.param(10, False, True, 0.1, "jax.sparse", 3), + # Test a larger network of smaller neurons with both solvers. + pytest.param(1000, True, True, 0.001, "jaxley.stone", 4), + pytest.param(1000, True, True, 0.001, "jax.sparse", 5), + ), +) +def test_runtime( + num_cells: int, + artificial: bool, + connect: bool, + connection_prob: float, + voltage_solver: str, + identifier: int, +): + delta_t = 0.025 + t_max = 100.0 + + net, params = build_net( + num_cells, + artificial=artificial, + connect=connect, + connection_prob=connection_prob, + ) + + def simulate(params): + return jx.integrate( + net, + params=params, + t_max=t_max, + delta_t=delta_t, + voltage_solver=voltage_solver, + ) + + jitted_simulate = jit(simulate) + + start_time = time.time() + _ = jitted_simulate(params).block_until_ready() + compile_time = time.time() - start_time + + params[0]["radius"] = params[0]["radius"].at[0].set(0.5) + start_time = time.time() + _ = jitted_simulate(params).block_until_ready() + run_time = time.time() - start_time + + compile_times = { + 0: 16.858529806137085, + 1: 0.8063809871673584, + 2: 5.4792890548706055, + 3: 6.175129175186157, + 4: 2.755805015563965, + 5: 13.303060293197632, + } + run_times = { + 0: 0.08291006088256836, + 1: 0.596994161605835, + 2: 0.8518729209899902, + 3: 5.746302127838135, + 4: 1.3585789203643799, + 5: 12.48916506767273, + } + + tolerance = 1.2 + assert compile_time < compile_times[identifier] * tolerance + assert run_time < run_times[identifier] * tolerance From e3ab3cbcd179416791ddd4b7912e5526297cb4c2 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Mon, 4 Nov 2024 16:03:23 +0100 Subject: [PATCH 02/37] enh: update regression tests. works with correct permissions for actions. tested in different repo --- .github/workflows/regression_tests.yml | 61 +++++++++++ .../workflows/update_regression_baseline.yml | 37 +++++++ tests/regression_test_baselines.json | 17 +++ tests/regression_test_results.json | 17 +++ ...ions for actions. tested in different repo | 9 ++ tests/regression_test_runner.py | 75 +++++++++++++ tests/test_regression.py | 103 ++++++++++++++++++ 7 files changed, 319 insertions(+) create mode 100644 .github/workflows/regression_tests.yml create mode 100644 .github/workflows/update_regression_baseline.yml create mode 100644 tests/regression_test_baselines.json create mode 100644 tests/regression_test_results.json create mode 100644 tests/regression_test_results.json~enh: update regression tests. works with correct permissions for actions. tested in different repo create mode 100644 tests/regression_test_runner.py create mode 100644 tests/test_regression.py diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml new file mode 100644 index 00000000..ebfd4b64 --- /dev/null +++ b/.github/workflows/regression_tests.yml @@ -0,0 +1,61 @@ +# .github/workflows/regression_tests.yml +name: Regression Tests + +on: + pull_request: + branches: + - main + +jobs: + regression_tests: + name: regression_tests + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + with: + lfs: true + fetch-depth: 0 # This ensures we can checkout main branch too + + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + architecture: 'x64' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run benchmarks on PR branch + run: | + python tests/regression_test_runner.py + + - name: Compare to main + if: github.event.pull_request.base.ref == 'main' + run: | + # Check if regression test results exist in main branch + git checkout main + if [ -f 'tests/regression_test_results.json' ]; then + python tests/regression_test_runner.py > regression_test_report.txt + else + echo "No regression test results found in main branch" > regression_test_report.txt + fi + + git checkout - + + - name: Comment PR + if: github.event.pull_request.base.ref == 'main' + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const TestReport = fs.readFileSync('regression_test_report.txt', 'utf8'); + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `## Regression Test Results\n\`\`\`\n${TestReport}\n\`\`\`` + }); \ No newline at end of file diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml new file mode 100644 index 00000000..19c92f2c --- /dev/null +++ b/.github/workflows/update_regression_baseline.yml @@ -0,0 +1,37 @@ +# .github/workflows/regression_tests.yml +name: Regression Tests + +on: + workflow_dispatch: + +jobs: + regression_tests: + name: regression_tests + runs-on: ubuntu-20.04 + + steps: + - uses: actions/checkout@v3 + with: + lfs: true + fetch-depth: 0 # This ensures we can checkout main branch too + + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + architecture: 'x64' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e ".[dev]" + + - name: Run benchmarks on PR branch + run: | + python tests/regression_test_runner.py + + + - name: Save new regression baseline + uses: actions/upload-artifact@v3 + with: + name: regression-test-results + path: tests/regression_test_results.json \ No newline at end of file diff --git a/tests/regression_test_baselines.json b/tests/regression_test_baselines.json new file mode 100644 index 00000000..a04bc97d --- /dev/null +++ b/tests/regression_test_baselines.json @@ -0,0 +1,17 @@ +{ + "ec3a4fad11d2bfb1bc5f8f10529cb06f2ff9919b377e9c0a3419c7f7f237f06e": { + "test_name": "test_runtime", + "input_kwargs": { + "num_cells": 1, + "artificial": false, + "connect": false, + "connection_prob": 0.0, + "voltage_solver": "jaxley.stone" + }, + "runtimes": { + "build_time": 1.0473196506500244, + "compile_time": 29.728327592213947, + "run_time": 0.15101234118143717 + } + } +} \ No newline at end of file diff --git a/tests/regression_test_results.json b/tests/regression_test_results.json new file mode 100644 index 00000000..f0da8ba3 --- /dev/null +++ b/tests/regression_test_results.json @@ -0,0 +1,17 @@ +{ + "ec3a4fad11d2bfb1bc5f8f10529cb06f2ff9919b377e9c0a3419c7f7f237f06e": { + "test_name": "test_runtime", + "input_kwargs": { + "num_cells": 1, + "artificial": false, + "connect": false, + "connection_prob": 0.0, + "voltage_solver": "jaxley.stone" + }, + "runtimes": { + "build_time": 2.8589820861816406, + "compile_time": 31.13769817352295, + "run_time": 0.12563538551330566 + } + } +} \ No newline at end of file diff --git a/tests/regression_test_results.json~enh: update regression tests. works with correct permissions for actions. tested in different repo b/tests/regression_test_results.json~enh: update regression tests. works with correct permissions for actions. tested in different repo new file mode 100644 index 00000000..40925f57 --- /dev/null +++ b/tests/regression_test_results.json~enh: update regression tests. works with correct permissions for actions. tested in different repo @@ -0,0 +1,9 @@ +{ + "module.test_runtime1": 1.0000774711370468, + "module.test_runtime2": 1.0000772399362177, + "module.test_runtime3": 1.000094958813861, + "module.test_runtime4": 1.0000748871825635, + "module.test_runtime5": 1.0000761719420552, + "module.test_runtime6": 1.0001780251041055, + "module.test_runtime7": 1.0000949839595705 +} \ No newline at end of file diff --git a/tests/regression_test_runner.py b/tests/regression_test_runner.py new file mode 100644 index 00000000..bc0e5f5b --- /dev/null +++ b/tests/regression_test_runner.py @@ -0,0 +1,75 @@ +# tests/benchmark_runner.py +import timeit +import json +from pathlib import Path +import importlib.util + +def load_module(file_path): + """Dynamically load a Python file as a module.""" + spec = importlib.util.spec_from_file_location("module", file_path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + +def get_test_functions(module): + """Get all functions in the module that start with test.""" + is_test = lambda value: callable(value) and value.__name__.startswith("test") + return [value for value in vars(module).values() if is_test(value)] + +def run_benchmarks(functions_to_test): + """Run benchmarks for the specified functions.""" + results = {} + for func in functions_to_test: + print(f"Running benchmark for {func.__name__}...") + # Run each function 1000 times and take the average + time_taken = timeit.timeit(lambda: func(), number=1) + # Use the function's qualified name (module.function) as the key + results[f"{func.__module__}.{func.__name__}"] = time_taken + return results + +def save_results(results, output_file): + """Save benchmark results to JSON file.""" + with open(output_file, 'w') as f: + json.dump(results, f, indent=2) + +def compare_results(base_results, new_results): + """Compare two sets of benchmark results and generate a diff report.""" + report = [] + for func_name in new_results: + new_time = new_results[func_name] + base_time = base_results.get(func_name) + + if base_time is None: + report.append(f"🆕 {func_name}: {new_time:.6f}s (new function)") + continue + + diff_pct = ((new_time - base_time) / base_time) * 100 + if diff_pct > 0: + emoji = "🔴" + elif diff_pct < 0: + emoji = "🟢" + else: + emoji = "⚪" + + report.append( + f"{emoji} {func_name}: {new_time:.6f}s " + f"({diff_pct:+.1f}% vs {base_time:.6f}s)" + ) + + return "\n".join(report) + +if __name__ == "__main__": + regression_test_file = 'tests/test_regression.py' + output_file = 'tests/regression_test_results.json' + base_results_file = 'tests/regression_test_results.json' if Path('tests/regression_test_results.json').exists() else None + + # Load the regression test module + test_module = load_module(regression_test_file) + test_functions = get_test_functions(test_module) + results = run_benchmarks(test_functions) + save_results(results, output_file) + + if base_results_file: + with open(base_results_file) as f: + base_results = json.load(f) + print(compare_results(base_results, results)) \ No newline at end of file diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 00000000..b3364555 --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,103 @@ +# This file is part of Jaxley, a differentiable neuroscience simulator. Jaxley is +# licensed under the Apache License Version 2.0, see + +import os +import time + +import numpy as np +import pytest +from jax import jit + +import jaxley as jx +from jaxley.channels import HH +from jaxley.connect import sparse_connect +from jaxley.synapses import IonotropicSynapse + +# mark all tests as runtime tests in this file +pytestmark = pytest.mark.runtime + +def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): + _ = np.random.seed(1) # For sparse connectivity matrix. + + if artificial: + comp = jx.Compartment() + branch = jx.Branch(comp, 2) + depth = 3 + parents = [-1] + [b // 2 for b in range(0, 2**depth - 2)] + cell = jx.Cell(branch, parents=parents) + else: + dirname = os.path.dirname(__file__) + fname = os.path.join(dirname, "swc_files", "morph.swc") + cell = jx.read_swc(fname, nseg=4) + net = jx.Network([cell for _ in range(num_cells)]) + + # Channels. + net.insert(HH()) + + # Synapses. + if connect: + sparse_connect( + net.cell("all"), net.cell("all"), IonotropicSynapse(), connection_prob + ) + + # Recordings. + net[0, 1, 0].record(verbose=False) + + # Trainables. + net.make_trainable("radius", verbose=False) + params = net.get_parameters() + + net.to_jax() + return net, params + +def setup_runtime( + num_cells: int, + artificial: bool, + connect: bool, + connection_prob: float, + voltage_solver: str, + identifier: int, +): + delta_t = 0.025 + t_max = 100.0 + + net, params = build_net( + num_cells, + artificial=artificial, + connect=connect, + connection_prob=connection_prob, + ) + + def simulate(params): + return jx.integrate( + net, + params=params, + t_max=t_max, + delta_t=delta_t, + voltage_solver=voltage_solver, + ) + + jitted_simulate = jit(simulate) + + _ = jitted_simulate(params).block_until_ready() + + params[0]["radius"] = params[0]["radius"].at[0].set(0.5) + _ = jitted_simulate(params).block_until_ready() + +def test_runtime1(): + setup_runtime(1, False, False, 0.0, "jaxley.stone", 0) + +def test_runtime2(): + setup_runtime(1, False, False, 0.0, "jax.sparse", 1) + +def test_runtime3(): + setup_runtime(10, False, True, 0.1, "jaxley.stone", 2) + +def test_runtime4(): + setup_runtime(10, False, True, 0.1, "jax.sparse", 3) + +def test_runtime5(): + setup_runtime(1000, True, True, 0.001, "jaxley.stone", 4) + +def test_runtime6(): + setup_runtime(1000, True, True, 0.001, "jax.sparse", 5) \ No newline at end of file From 9e18551889203398db89d2daafd104e7e9147e7d Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Mon, 4 Nov 2024 16:05:54 +0100 Subject: [PATCH 03/37] fix: change order of saving the file --- tests/regression_test_runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/regression_test_runner.py b/tests/regression_test_runner.py index bc0e5f5b..9466a850 100644 --- a/tests/regression_test_runner.py +++ b/tests/regression_test_runner.py @@ -67,9 +67,11 @@ def compare_results(base_results, new_results): test_module = load_module(regression_test_file) test_functions = get_test_functions(test_module) results = run_benchmarks(test_functions) - save_results(results, output_file) if base_results_file: with open(base_results_file) as f: base_results = json.load(f) - print(compare_results(base_results, results)) \ No newline at end of file + print(compare_results(base_results, results)) + + # save new results + save_results(results, output_file) \ No newline at end of file From 7235b10c17b93fe391997c3a0db84bc49876acd0 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Mon, 4 Nov 2024 16:10:58 +0100 Subject: [PATCH 04/37] fix: black and remove pytest markervariable --- tests/regression_test_runner.py | 34 +++++++++++++++++++++------------ tests/test_regression.py | 12 ++++++++++-- 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/tests/regression_test_runner.py b/tests/regression_test_runner.py index 9466a850..d1437117 100644 --- a/tests/regression_test_runner.py +++ b/tests/regression_test_runner.py @@ -4,6 +4,7 @@ from pathlib import Path import importlib.util + def load_module(file_path): """Dynamically load a Python file as a module.""" spec = importlib.util.spec_from_file_location("module", file_path) @@ -11,11 +12,13 @@ def load_module(file_path): spec.loader.exec_module(module) return module + def get_test_functions(module): """Get all functions in the module that start with test.""" is_test = lambda value: callable(value) and value.__name__.startswith("test") return [value for value in vars(module).values() if is_test(value)] + def run_benchmarks(functions_to_test): """Run benchmarks for the specified functions.""" results = {} @@ -27,22 +30,24 @@ def run_benchmarks(functions_to_test): results[f"{func.__module__}.{func.__name__}"] = time_taken return results + def save_results(results, output_file): """Save benchmark results to JSON file.""" - with open(output_file, 'w') as f: + with open(output_file, "w") as f: json.dump(results, f, indent=2) + def compare_results(base_results, new_results): """Compare two sets of benchmark results and generate a diff report.""" report = [] for func_name in new_results: new_time = new_results[func_name] base_time = base_results.get(func_name) - + if base_time is None: report.append(f"🆕 {func_name}: {new_time:.6f}s (new function)") continue - + diff_pct = ((new_time - base_time) / base_time) * 100 if diff_pct > 0: emoji = "🔴" @@ -50,28 +55,33 @@ def compare_results(base_results, new_results): emoji = "🟢" else: emoji = "⚪" - + report.append( f"{emoji} {func_name}: {new_time:.6f}s " f"({diff_pct:+.1f}% vs {base_time:.6f}s)" ) - + return "\n".join(report) + if __name__ == "__main__": - regression_test_file = 'tests/test_regression.py' - output_file = 'tests/regression_test_results.json' - base_results_file = 'tests/regression_test_results.json' if Path('tests/regression_test_results.json').exists() else None - + regression_test_file = "tests/test_regression.py" + output_file = "tests/regression_test_results.json" + base_results_file = ( + "tests/regression_test_results.json" + if Path("tests/regression_test_results.json").exists() + else None + ) + # Load the regression test module test_module = load_module(regression_test_file) test_functions = get_test_functions(test_module) results = run_benchmarks(test_functions) - + if base_results_file: with open(base_results_file) as f: base_results = json.load(f) print(compare_results(base_results, results)) - + # save new results - save_results(results, output_file) \ No newline at end of file + save_results(results, output_file) diff --git a/tests/test_regression.py b/tests/test_regression.py index b3364555..af8bff03 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -14,7 +14,8 @@ from jaxley.synapses import IonotropicSynapse # mark all tests as runtime tests in this file -pytestmark = pytest.mark.runtime +pytest.mark.runtime + def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): _ = np.random.seed(1) # For sparse connectivity matrix. @@ -50,6 +51,7 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): net.to_jax() return net, params + def setup_runtime( num_cells: int, artificial: bool, @@ -84,20 +86,26 @@ def simulate(params): params[0]["radius"] = params[0]["radius"].at[0].set(0.5) _ = jitted_simulate(params).block_until_ready() + def test_runtime1(): setup_runtime(1, False, False, 0.0, "jaxley.stone", 0) + def test_runtime2(): setup_runtime(1, False, False, 0.0, "jax.sparse", 1) + def test_runtime3(): setup_runtime(10, False, True, 0.1, "jaxley.stone", 2) + def test_runtime4(): setup_runtime(10, False, True, 0.1, "jax.sparse", 3) + def test_runtime5(): setup_runtime(1000, True, True, 0.001, "jaxley.stone", 4) + def test_runtime6(): - setup_runtime(1000, True, True, 0.001, "jax.sparse", 5) \ No newline at end of file + setup_runtime(1000, True, True, 0.001, "jax.sparse", 5) From d061b57884cf0d741bcf36f006b42514672ebbcc Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Mon, 4 Nov 2024 16:13:19 +0100 Subject: [PATCH 05/37] fix: catch testmarker --- tests/regression_test_runner.py | 13 ++++++++++--- tests/test_regression.py | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/tests/regression_test_runner.py b/tests/regression_test_runner.py index d1437117..f28cf8bc 100644 --- a/tests/regression_test_runner.py +++ b/tests/regression_test_runner.py @@ -1,8 +1,10 @@ # tests/benchmark_runner.py -import timeit +import importlib.util import json +import timeit from pathlib import Path -import importlib.util + +from pytest import MarkDecorator def load_module(file_path): @@ -15,7 +17,12 @@ def load_module(file_path): def get_test_functions(module): """Get all functions in the module that start with test.""" - is_test = lambda value: callable(value) and value.__name__.startswith("test") + + def is_test(value): + if not isinstance(value, MarkDecorator): + return callable(value) and value.__name__.startswith("test") + return False + return [value for value in vars(module).values() if is_test(value)] diff --git a/tests/test_regression.py b/tests/test_regression.py index af8bff03..e5347fa9 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -14,7 +14,7 @@ from jaxley.synapses import IonotropicSynapse # mark all tests as runtime tests in this file -pytest.mark.runtime +pytestmark = pytest.mark.runtime def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): From 269c9dc4760bcaa49903d31a88c9950622722dd1 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Mon, 4 Nov 2024 16:16:12 +0100 Subject: [PATCH 06/37] fix: printout w.o. base results --- tests/regression_test_runner.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/regression_test_runner.py b/tests/regression_test_runner.py index f28cf8bc..12c66d1e 100644 --- a/tests/regression_test_runner.py +++ b/tests/regression_test_runner.py @@ -88,7 +88,9 @@ def compare_results(base_results, new_results): if base_results_file: with open(base_results_file) as f: base_results = json.load(f) - print(compare_results(base_results, results)) + else: + base_results = {} + print(compare_results(base_results, results)) # save new results save_results(results, output_file) From 2599e2e65c9249f77c355bd0104f92134a167b4e Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Mon, 4 Nov 2024 16:28:00 +0100 Subject: [PATCH 07/37] fix: dont save new results --- tests/regression_test_runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/regression_test_runner.py b/tests/regression_test_runner.py index 12c66d1e..63ea6d18 100644 --- a/tests/regression_test_runner.py +++ b/tests/regression_test_runner.py @@ -93,4 +93,4 @@ def compare_results(base_results, new_results): print(compare_results(base_results, results)) # save new results - save_results(results, output_file) + # save_results(results, output_file) From c3dcaa71d22dacb291d8691e6dabdf7ae3234b60 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Mon, 4 Nov 2024 16:51:23 +0100 Subject: [PATCH 08/37] fix: change regression test workflow --- .github/workflows/regression_tests.yml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index ebfd4b64..c71b8cf6 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -35,15 +35,14 @@ jobs: if: github.event.pull_request.base.ref == 'main' run: | # Check if regression test results exist in main branch - git checkout main - if [ -f 'tests/regression_test_results.json' ]; then - python tests/regression_test_runner.py > regression_test_report.txt + if [ -f 'git cat-file -e main:tests/regression_test_results.json' ]; then + git checkout main tests/regression_test_results.json else - echo "No regression test results found in main branch" > regression_test_report.txt + echo "No regression test results found in main branch" fi - + python tests/regression_test_runner.py > regression_test_report.txt git checkout - - + - name: Comment PR if: github.event.pull_request.base.ref == 'main' uses: actions/github-script@v7 From 842206dbcd400b78c5ac4eaf8a0cf5e475d665af Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Mon, 4 Nov 2024 17:10:58 +0100 Subject: [PATCH 09/37] enh: update regresssion tests --- .github/workflows/regression_tests.yml | 10 +++------- .github/workflows/update_regression_baseline.yml | 1 - 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index c71b8cf6..317dc3e9 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -26,12 +26,8 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[dev]" - - - name: Run benchmarks on PR branch - run: | - python tests/regression_test_runner.py - - - name: Compare to main + + - name: Run benchmarks and compare to baseline if: github.event.pull_request.base.ref == 'main' run: | # Check if regression test results exist in main branch @@ -41,7 +37,7 @@ jobs: echo "No regression test results found in main branch" fi python tests/regression_test_runner.py > regression_test_report.txt - git checkout - + git checkout . - name: Comment PR if: github.event.pull_request.base.ref == 'main' diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 19c92f2c..122f6d40 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -29,7 +29,6 @@ jobs: run: | python tests/regression_test_runner.py - - name: Save new regression baseline uses: actions/upload-artifact@v3 with: From 884b09a203c58686bd25cc43f0618c9d34e5c5d9 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Tue, 5 Nov 2024 07:54:18 +0100 Subject: [PATCH 10/37] fix: fix old idea b4 scrapping it --- tests/regression_test_runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/regression_test_runner.py b/tests/regression_test_runner.py index 63ea6d18..19c792ed 100644 --- a/tests/regression_test_runner.py +++ b/tests/regression_test_runner.py @@ -30,7 +30,6 @@ def run_benchmarks(functions_to_test): """Run benchmarks for the specified functions.""" results = {} for func in functions_to_test: - print(f"Running benchmark for {func.__name__}...") # Run each function 1000 times and take the average time_taken = timeit.timeit(lambda: func(), number=1) # Use the function's qualified name (module.function) as the key From e9d16d115e026bad2d9a59de92fdec0fedd6892b Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Wed, 6 Nov 2024 13:44:38 +0100 Subject: [PATCH 11/37] enh: better regression tests. works locally, need to update baseline --- .github/workflows/regression_tests.yml | 36 ++-- .../workflows/update_regression_baseline.yml | 17 +- tests/regression_test_runner.py | 95 --------- tests/test_regression.py | 196 ++++++++++++++---- 4 files changed, 185 insertions(+), 159 deletions(-) delete mode 100644 tests/regression_test_runner.py diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 317dc3e9..69dfb87b 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -31,26 +31,26 @@ jobs: if: github.event.pull_request.base.ref == 'main' run: | # Check if regression test results exist in main branch - if [ -f 'git cat-file -e main:tests/regression_test_results.json' ]; then - git checkout main tests/regression_test_results.json + if [ -f 'git cat-file -e main:tests/regression_test_baselines.json' ]; then + git checkout main tests/regression_test_baselines.json else echo "No regression test results found in main branch" fi - python tests/regression_test_runner.py > regression_test_report.txt - git checkout . + PYTHONHASHSEED=0 pytest tests/test_regression.py + git checkout - - name: Comment PR - if: github.event.pull_request.base.ref == 'main' - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const fs = require('fs'); - const TestReport = fs.readFileSync('regression_test_report.txt', 'utf8'); + # - name: Comment PR + # if: github.event.pull_request.base.ref == 'main' + # uses: actions/github-script@v7 + # with: + # github-token: ${{ secrets.GITHUB_TOKEN }} + # script: | + # const fs = require('fs'); + # const TestReport = fs.readFileSync('regression_test_report.txt', 'utf8'); - await github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `## Regression Test Results\n\`\`\`\n${TestReport}\n\`\`\`` - }); \ No newline at end of file + # await github.rest.issues.createComment({ + # issue_number: context.issue.number, + # owner: context.repo.owner, + # repo: context.repo.repo, + # body: `## Regression Test Results\n\`\`\`\n${TestReport}\n\`\`\`` + # }); \ No newline at end of file diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 122f6d40..afe5202d 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -2,7 +2,9 @@ name: Regression Tests on: - workflow_dispatch: + pull_request: + branches: + - main jobs: regression_tests: @@ -24,13 +26,8 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[dev]" - - - name: Run benchmarks on PR branch + + - name: Update baseline + if: github.event.pull_request.base.ref == 'main' run: | - python tests/regression_test_runner.py - - - name: Save new regression baseline - uses: actions/upload-artifact@v3 - with: - name: regression-test-results - path: tests/regression_test_results.json \ No newline at end of file + PYTHONHASHSEED=0 UPDATE_BASELINE=1 pytest tests/test_regression.py \ No newline at end of file diff --git a/tests/regression_test_runner.py b/tests/regression_test_runner.py deleted file mode 100644 index 19c792ed..00000000 --- a/tests/regression_test_runner.py +++ /dev/null @@ -1,95 +0,0 @@ -# tests/benchmark_runner.py -import importlib.util -import json -import timeit -from pathlib import Path - -from pytest import MarkDecorator - - -def load_module(file_path): - """Dynamically load a Python file as a module.""" - spec = importlib.util.spec_from_file_location("module", file_path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module - - -def get_test_functions(module): - """Get all functions in the module that start with test.""" - - def is_test(value): - if not isinstance(value, MarkDecorator): - return callable(value) and value.__name__.startswith("test") - return False - - return [value for value in vars(module).values() if is_test(value)] - - -def run_benchmarks(functions_to_test): - """Run benchmarks for the specified functions.""" - results = {} - for func in functions_to_test: - # Run each function 1000 times and take the average - time_taken = timeit.timeit(lambda: func(), number=1) - # Use the function's qualified name (module.function) as the key - results[f"{func.__module__}.{func.__name__}"] = time_taken - return results - - -def save_results(results, output_file): - """Save benchmark results to JSON file.""" - with open(output_file, "w") as f: - json.dump(results, f, indent=2) - - -def compare_results(base_results, new_results): - """Compare two sets of benchmark results and generate a diff report.""" - report = [] - for func_name in new_results: - new_time = new_results[func_name] - base_time = base_results.get(func_name) - - if base_time is None: - report.append(f"🆕 {func_name}: {new_time:.6f}s (new function)") - continue - - diff_pct = ((new_time - base_time) / base_time) * 100 - if diff_pct > 0: - emoji = "🔴" - elif diff_pct < 0: - emoji = "🟢" - else: - emoji = "⚪" - - report.append( - f"{emoji} {func_name}: {new_time:.6f}s " - f"({diff_pct:+.1f}% vs {base_time:.6f}s)" - ) - - return "\n".join(report) - - -if __name__ == "__main__": - regression_test_file = "tests/test_regression.py" - output_file = "tests/regression_test_results.json" - base_results_file = ( - "tests/regression_test_results.json" - if Path("tests/regression_test_results.json").exists() - else None - ) - - # Load the regression test module - test_module = load_module(regression_test_file) - test_functions = get_test_functions(test_module) - results = run_benchmarks(test_functions) - - if base_results_file: - with open(base_results_file) as f: - base_results = json.load(f) - else: - base_results = {} - print(compare_results(base_results, results)) - - # save new results - # save_results(results, output_file) diff --git a/tests/test_regression.py b/tests/test_regression.py index e5347fa9..0f0ae2ef 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -12,9 +12,134 @@ from jaxley.channels import HH from jaxley.connect import sparse_connect from jaxley.synapses import IonotropicSynapse +from functools import wraps +import json -# mark all tests as runtime tests in this file -pytestmark = pytest.mark.runtime + +# due to the use of hash functions, `test_regression.py` has to be run with the +# environment variable PYTHONHASHSEED=0, i.e. `:~$PYTHONHASHSEED=0 pytest `tests/test_regression.py`. +# For details see https://stackoverflow.com/questions/30585108/disable-hash-randomization-from-within-python-program + + +# Every runtime test needs to have the following structure: +# +# @compare_to_baseline() +# def test_runtime_of_x(**kwargs) -> Dict: +# t1 = time.time() +# time.sleep(0.1) +# # do something +# t2 = time.time() +# # do something else +# t3 = time.time() +# return {"dt1": t2-t1, dt2: t3-t2} + +pytestmark = pytest.mark.runtime # mark all tests as runtime tests in this file +UPDATE_BASELINE = ( + os.environ["UPDATE_BASELINE"] if "UPDATE_BASELINE" in os.environ else 0 +) +tolerance = 0.2 + +BASELINES = {} +if os.path.exists("tests/regression_test_baselines.json"): + with open("tests/regression_test_baselines.json") as f: + BASELINES = json.load(f) + + +def generate_unique_key(d): + return str(hash(json.dumps(d, sort_keys=True))) + + +def append_to_json(fpath, test_name, input_kwargs, runtimes): + header = {"test_name": test_name, "input_kwargs": input_kwargs} + data = {generate_unique_key(header): {**header, "runtimes": runtimes}} + + # Save data to a JSON file + if os.path.exists(fpath): + with open(fpath, "r") as f: + result_data = json.load(f) + result_data.update(data) + else: + result_data = data + with open(fpath, "w") as f: + json.dump(result_data, f, indent=2) + + +class compare_to_baseline: + def __init__(self, baseline_iters=1, test_iters=1): + self.baseline_iters = baseline_iters + self.test_iters = test_iters + + def __call__(self, func): + @wraps(func) # ensures kwargs exposed to pytest + def test_wrapper(**kwargs): + header = {"test_name": func.__name__, "input_kwargs": kwargs} + key = generate_unique_key(header) + + runs = [] + num_iters = self.baseline_iters if UPDATE_BASELINE else self.test_iters + for _ in range(num_iters): + runtimes = func(**kwargs) + runs.append(runtimes) + runtimes = {k: np.mean([d[k] for d in runs]) for k in runs[0]} + + fpath = ( + "tests/regression_test_results.json" + if not UPDATE_BASELINE + else "tests/regression_test_baselines.json" + ) + append_to_json(fpath, header["test_name"], header["input_kwargs"], runtimes) + + if not UPDATE_BASELINE: + assert key in BASELINES, f"No basline found for {header}" + func_baselines = BASELINES[key]["runtimes"] + for key, baseline in func_baselines.items(): + diff = ( + float("nan") + if np.isclose(baseline, 0) + else (runtimes[key] - baseline) / baseline + ) + assert runtimes[key] < baseline * ( + 1 + tolerance + ), f"{key} is {diff:.2%} slower than the baseline." + + return test_wrapper + + +def generate_report(base_results, new_results): + """Compare two sets of benchmark results and generate a diff report.""" + report = [] + for key in new_results: + new_data = new_results[key] + base_data = base_results.get(key) + kwargs = ", ".join([f"{k}={v}" for k, v in new_data["input_kwargs"].items()]) + func_name = new_data["test_name"] + func_signature = f"{func_name}({kwargs})" + + new_runtimes = new_data["runtimes"] + base_runtimes = ( + {k: None for k in new_data.keys()} + if base_data is None + else base_data["runtimes"] + ) + + report.append(func_signature) + for key, new_time in new_runtimes.items(): + base_time = base_runtimes.get(key) + diff = None if base_time is None else ((new_time - base_time) / base_time) + + emoji = "🆕" if diff is None else "⚪" + emoji = "🔴" if diff > tolerance else emoji + emoji = "🟢" if diff < 0 else emoji + + time_str = ( + f"({new_time:.6f}s)" + if diff is None + else f"({diff:+.1f}% vs {base_time:.6f}s)" + ) + report.append(f"{emoji} {key}: {time_str}.") + report.append("") + + return "\n".join(report) def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): @@ -52,24 +177,31 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): return net, params -def setup_runtime( +@pytest.mark.parametrize( + "num_cells, artificial, connect, connection_prob, voltage_solver", + ( + # Test a single SWC cell with both solvers. + pytest.param(1, False, False, 0.0, "jaxley.stone"), + pytest.param(1, False, False, 0.0, "jax.sparse"), + # Test a network of SWC cells with both solvers. + pytest.param(10, False, True, 0.1, "jaxley.stone"), + pytest.param(10, False, True, 0.1, "jax.sparse"), + # Test a larger network of smaller neurons with both solvers. + pytest.param(1000, True, True, 0.001, "jaxley.stone"), + pytest.param(1000, True, True, 0.001, "jax.sparse"), + ), +) +@compare_to_baseline(baseline_iters=3) +def test_runtime( num_cells: int, artificial: bool, connect: bool, connection_prob: float, voltage_solver: str, - identifier: int, ): delta_t = 0.025 t_max = 100.0 - net, params = build_net( - num_cells, - artificial=artificial, - connect=connect, - connection_prob=connection_prob, - ) - def simulate(params): return jx.integrate( net, @@ -79,33 +211,25 @@ def simulate(params): voltage_solver=voltage_solver, ) + runtimes = {} + + start_time = time.time() + net, params = build_net( + num_cells, + artificial=artificial, + connect=connect, + connection_prob=connection_prob, + ) + runtimes["build_time"] = time.time() - start_time + jitted_simulate = jit(simulate) + start_time = time.time() _ = jitted_simulate(params).block_until_ready() - + runtimes["compile_time"] = time.time() - start_time params[0]["radius"] = params[0]["radius"].at[0].set(0.5) - _ = jitted_simulate(params).block_until_ready() - -def test_runtime1(): - setup_runtime(1, False, False, 0.0, "jaxley.stone", 0) - - -def test_runtime2(): - setup_runtime(1, False, False, 0.0, "jax.sparse", 1) - - -def test_runtime3(): - setup_runtime(10, False, True, 0.1, "jaxley.stone", 2) - - -def test_runtime4(): - setup_runtime(10, False, True, 0.1, "jax.sparse", 3) - - -def test_runtime5(): - setup_runtime(1000, True, True, 0.001, "jaxley.stone", 4) - - -def test_runtime6(): - setup_runtime(1000, True, True, 0.001, "jax.sparse", 5) + start_time = time.time() + _ = jitted_simulate(params).block_until_ready() + runtimes["run_time"] = time.time() - start_time + return runtimes From 63ad4f7392480c26be8e87eca8ebcc01377a87be Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Wed, 6 Nov 2024 13:46:33 +0100 Subject: [PATCH 12/37] fix: update_baselines updated --- .../workflows/update_regression_baseline.yml | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index afe5202d..712a609b 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -1,15 +1,16 @@ # .github/workflows/regression_tests.yml -name: Regression Tests +name: Update Regression Tests on: - pull_request: - branches: - - main + workflow_dispatch: jobs: regression_tests: name: regression_tests runs-on: ubuntu-20.04 + permissions: + contents: write + pull-requests: write steps: - uses: actions/checkout@v3 @@ -30,4 +31,17 @@ jobs: - name: Update baseline if: github.event.pull_request.base.ref == 'main' run: | - PYTHONHASHSEED=0 UPDATE_BASELINE=1 pytest tests/test_regression.py \ No newline at end of file + git config --global user.name 'GitHub Action' + git config --global user.email 'action@github.com' + PYTHONHASHSEED=0 UPDATE_BASELINE=1 pytest tests/test_regression.py + + - name: Commit and push if baseline changed + if: github.event.pull_request.base.ref == 'main' + run: | + if git diff --quiet tests/regression_test_baselines.json; then + echo "No changes to baseline file" + else + git add tests/regression_test_baselines.json + git commit -m "Update regression test baselines" + git push origin HEAD:${{ github.head_ref }} + fi \ No newline at end of file From 8b01ba6f45d7d029ded87a6441aecfaa70ef50cc Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Wed, 6 Nov 2024 13:52:21 +0100 Subject: [PATCH 13/37] fix: run isort --- tests/test_regression.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_regression.py b/tests/test_regression.py index 0f0ae2ef..8951eb34 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -1,8 +1,10 @@ # This file is part of Jaxley, a differentiable neuroscience simulator. Jaxley is # licensed under the Apache License Version 2.0, see +import json import os import time +from functools import wraps import numpy as np import pytest @@ -12,9 +14,6 @@ from jaxley.channels import HH from jaxley.connect import sparse_connect from jaxley.synapses import IonotropicSynapse -from functools import wraps -import json - # due to the use of hash functions, `test_regression.py` has to be run with the # environment variable PYTHONHASHSEED=0, i.e. `:~$PYTHONHASHSEED=0 pytest `tests/test_regression.py`. From 318371e2d791239539202d9fa85dc03bd859a9fd Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 15:11:04 +0100 Subject: [PATCH 14/37] rm: old runtime tests --- tests/test_runtime.py | 124 ------------------------------------------ 1 file changed, 124 deletions(-) delete mode 100644 tests/test_runtime.py diff --git a/tests/test_runtime.py b/tests/test_runtime.py deleted file mode 100644 index bcb445bf..00000000 --- a/tests/test_runtime.py +++ /dev/null @@ -1,124 +0,0 @@ -# This file is part of Jaxley, a differentiable neuroscience simulator. Jaxley is -# licensed under the Apache License Version 2.0, see - -import os -import time - -import numpy as np -import pytest -from jax import jit - -import jaxley as jx -from jaxley.channels import HH -from jaxley.connect import sparse_connect -from jaxley.synapses import IonotropicSynapse - - -def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): - _ = np.random.seed(1) # For sparse connectivity matrix. - - if artificial: - comp = jx.Compartment() - branch = jx.Branch(comp, 2) - depth = 3 - parents = [-1] + [b // 2 for b in range(0, 2**depth - 2)] - cell = jx.Cell(branch, parents=parents) - else: - dirname = os.path.dirname(__file__) - fname = os.path.join(dirname, "swc_files", "morph.swc") - cell = jx.read_swc(fname, nseg=4) - net = jx.Network([cell for _ in range(num_cells)]) - - # Channels. - net.insert(HH()) - - # Synapses. - if connect: - sparse_connect( - net.cell("all"), net.cell("all"), IonotropicSynapse(), connection_prob - ) - - # Recordings. - net[0, 1, 0].record(verbose=False) - - # Trainables. - net.make_trainable("radius", verbose=False) - params = net.get_parameters() - - net.to_jax() - return net, params - - -@pytest.mark.runtime -@pytest.mark.parametrize( - "num_cells, artificial, connect, connection_prob, voltage_solver, identifier", - ( - # Test a single SWC cell with both solvers. - pytest.param(1, False, False, 0.0, "jaxley.stone", 0), - pytest.param(1, False, False, 0.0, "jax.sparse", 1), - # Test a network of SWC cells with both solvers. - pytest.param(10, False, True, 0.1, "jaxley.stone", 2), - pytest.param(10, False, True, 0.1, "jax.sparse", 3), - # Test a larger network of smaller neurons with both solvers. - pytest.param(1000, True, True, 0.001, "jaxley.stone", 4), - pytest.param(1000, True, True, 0.001, "jax.sparse", 5), - ), -) -def test_runtime( - num_cells: int, - artificial: bool, - connect: bool, - connection_prob: float, - voltage_solver: str, - identifier: int, -): - delta_t = 0.025 - t_max = 100.0 - - net, params = build_net( - num_cells, - artificial=artificial, - connect=connect, - connection_prob=connection_prob, - ) - - def simulate(params): - return jx.integrate( - net, - params=params, - t_max=t_max, - delta_t=delta_t, - voltage_solver=voltage_solver, - ) - - jitted_simulate = jit(simulate) - - start_time = time.time() - _ = jitted_simulate(params).block_until_ready() - compile_time = time.time() - start_time - - params[0]["radius"] = params[0]["radius"].at[0].set(0.5) - start_time = time.time() - _ = jitted_simulate(params).block_until_ready() - run_time = time.time() - start_time - - compile_times = { - 0: 16.858529806137085, - 1: 0.8063809871673584, - 2: 5.4792890548706055, - 3: 6.175129175186157, - 4: 2.755805015563965, - 5: 13.303060293197632, - } - run_times = { - 0: 0.08291006088256836, - 1: 0.596994161605835, - 2: 0.8518729209899902, - 3: 5.746302127838135, - 4: 1.3585789203643799, - 5: 12.48916506767273, - } - - tolerance = 1.2 - assert compile_time < compile_times[identifier] * tolerance - assert run_time < run_times[identifier] * tolerance From 364fd67e21844e7c5e3db7de64d338fae5c6dd1e Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 16:01:34 +0100 Subject: [PATCH 15/37] wip: deterministic hashes, added reporting --- .github/workflows/regression_tests.yml | 2 +- .../workflows/update_regression_baseline.yml | 2 +- pyproject.toml | 1 + tests/conftest.py | 35 ++++++ tests/test_regression.py | 113 +++++++++--------- 5 files changed, 95 insertions(+), 58 deletions(-) diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 69dfb87b..df7cdeb2 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -36,7 +36,7 @@ jobs: else echo "No regression test results found in main branch" fi - PYTHONHASHSEED=0 pytest tests/test_regression.py + pytest tests/test_regression.py git checkout # - name: Comment PR diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 712a609b..d8fddff4 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -33,7 +33,7 @@ jobs: run: | git config --global user.name 'GitHub Action' git config --global user.email 'action@github.com' - PYTHONHASHSEED=0 UPDATE_BASELINE=1 pytest tests/test_regression.py + UPDATE_BASELINE=1 pytest tests/test_regression.py - name: Commit and push if baseline changed if: github.event.pull_request.base.ref == 'main' diff --git a/pyproject.toml b/pyproject.toml index d37e2bda..d1072768 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -69,6 +69,7 @@ dev = [ [tool.pytest.ini_options] markers = [ "slow: marks tests as slow (T > 10s)", + "regression: marks regression tests", ] [tool.isort] diff --git a/tests/conftest.py b/tests/conftest.py index dad1c4a5..354594d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ # This file is part of Jaxley, a differentiable neuroscience simulator. Jaxley is # licensed under the Apache License Version 2.0, see +import json import os from copy import deepcopy from typing import Optional @@ -9,6 +10,7 @@ import jaxley as jx from jaxley.synapses import IonotropicSynapse +from tests.test_regression import generate_regression_report @pytest.fixture(scope="session") @@ -202,3 +204,36 @@ def get_or_compute_swc2jaxley_params( yield get_or_compute_swc2jaxley_params params = {} + + +@pytest.fixture(scope="session", autouse=True) +def print_session_report(request): + """Cleanup a testing directory once we are finished.""" + + def print_regression_report(): + dirname = os.path.dirname(__file__) + baseline_fname = os.path.join(dirname, "regression_test_baselines.json") + results_fname = os.path.join(dirname, "regression_test_results.json") + + baselines = {} + if os.path.exists(baseline_fname): + with open(baseline_fname) as f: + baselines = json.load(f) + + results = {} + if os.path.exists(results_fname): + with open(results_fname) as f: + results = json.load(f) + + # the following allows to print the report to the console despite pytest + # capturing the output and without specifying the "-s" flag + capmanager = request.config.pluginmanager.getplugin("capturemanager") + with capmanager.global_and_fixture_disabled(): + print("\n\n\nRegression Test Report\n----------------------\n") + if not baselines: + print( + "No baselines found. Run `git checkout main;UPDATE_BASELINE=1 pytest tests/test_regression.py; git checkout -`" + ) + print(generate_regression_report(baselines, results)) + + request.addfinalizer(print_regression_report) diff --git a/tests/test_regression.py b/tests/test_regression.py index 8951eb34..26426260 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -1,6 +1,7 @@ # This file is part of Jaxley, a differentiable neuroscience simulator. Jaxley is # licensed under the Apache License Version 2.0, see +import hashlib import json import os import time @@ -15,11 +16,6 @@ from jaxley.connect import sparse_connect from jaxley.synapses import IonotropicSynapse -# due to the use of hash functions, `test_regression.py` has to be run with the -# environment variable PYTHONHASHSEED=0, i.e. `:~$PYTHONHASHSEED=0 pytest `tests/test_regression.py`. -# For details see https://stackoverflow.com/questions/30585108/disable-hash-randomization-from-within-python-program - - # Every runtime test needs to have the following structure: # # @compare_to_baseline() @@ -30,22 +26,64 @@ # t2 = time.time() # # do something else # t3 = time.time() -# return {"dt1": t2-t1, dt2: t3-t2} +# return {"sth": t2-t1, sth_else: t3-t2} + -pytestmark = pytest.mark.runtime # mark all tests as runtime tests in this file +pytestmark = pytest.mark.regression # mark all tests as regression tests in this file UPDATE_BASELINE = ( os.environ["UPDATE_BASELINE"] if "UPDATE_BASELINE" in os.environ else 0 ) tolerance = 0.2 -BASELINES = {} +baselines = {} if os.path.exists("tests/regression_test_baselines.json"): with open("tests/regression_test_baselines.json") as f: - BASELINES = json.load(f) + baselines = json.load(f) + + +def generate_regression_report(base_results, new_results): + """Compare two sets of benchmark results and generate a diff report.""" + report = [] + for key in new_results: + new_data = new_results[key] + base_data = base_results.get(key) + kwargs = ", ".join([f"{k}={v}" for k, v in new_data["input_kwargs"].items()]) + func_name = new_data["test_name"] + func_signature = f"{func_name}({kwargs})" + + new_runtimes = new_data["runtimes"] + base_runtimes = ( + {k: None for k in new_data.keys()} + if base_data is None + else base_data["runtimes"] + ) + + report.append(func_signature) + for key, new_time in new_runtimes.items(): + base_time = base_runtimes.get(key) + diff = None if base_time is None else ((new_time - base_time) / base_time) + + emoji = "🆕" if diff is None else "⚪" + emoji = "🔴" if diff > tolerance else emoji + emoji = "🟢" if diff < 0 else emoji + + time_str = ( + f"({new_time:.6f}s)" + if diff is None + else f"({diff:+.1f}% vs {base_time:.6f}s)" + ) + report.append(f"{emoji} {key}: {time_str}.") + report.append("") + + return "\n".join(report) def generate_unique_key(d): - return str(hash(json.dumps(d, sort_keys=True))) + # Generate a unique key for each test case. Makes it possible to compare tests + # with different input_kwargs. + hash_obj = hashlib.sha256(bytes(json.dumps(d, sort_keys=True), encoding="utf-8")) + hash = hash_obj.hexdigest() + return str(hash) def append_to_json(fpath, test_name, input_kwargs, runtimes): @@ -89,8 +127,8 @@ def test_wrapper(**kwargs): append_to_json(fpath, header["test_name"], header["input_kwargs"], runtimes) if not UPDATE_BASELINE: - assert key in BASELINES, f"No basline found for {header}" - func_baselines = BASELINES[key]["runtimes"] + assert key in baselines, f"No basline found for {header}" + func_baselines = baselines[key]["runtimes"] for key, baseline in func_baselines.items(): diff = ( float("nan") @@ -104,43 +142,6 @@ def test_wrapper(**kwargs): return test_wrapper -def generate_report(base_results, new_results): - """Compare two sets of benchmark results and generate a diff report.""" - report = [] - for key in new_results: - new_data = new_results[key] - base_data = base_results.get(key) - kwargs = ", ".join([f"{k}={v}" for k, v in new_data["input_kwargs"].items()]) - func_name = new_data["test_name"] - func_signature = f"{func_name}({kwargs})" - - new_runtimes = new_data["runtimes"] - base_runtimes = ( - {k: None for k in new_data.keys()} - if base_data is None - else base_data["runtimes"] - ) - - report.append(func_signature) - for key, new_time in new_runtimes.items(): - base_time = base_runtimes.get(key) - diff = None if base_time is None else ((new_time - base_time) / base_time) - - emoji = "🆕" if diff is None else "⚪" - emoji = "🔴" if diff > tolerance else emoji - emoji = "🟢" if diff < 0 else emoji - - time_str = ( - f"({new_time:.6f}s)" - if diff is None - else f"({diff:+.1f}% vs {base_time:.6f}s)" - ) - report.append(f"{emoji} {key}: {time_str}.") - report.append("") - - return "\n".join(report) - - def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): _ = np.random.seed(1) # For sparse connectivity matrix. @@ -181,13 +182,13 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): ( # Test a single SWC cell with both solvers. pytest.param(1, False, False, 0.0, "jaxley.stone"), - pytest.param(1, False, False, 0.0, "jax.sparse"), - # Test a network of SWC cells with both solvers. - pytest.param(10, False, True, 0.1, "jaxley.stone"), - pytest.param(10, False, True, 0.1, "jax.sparse"), - # Test a larger network of smaller neurons with both solvers. - pytest.param(1000, True, True, 0.001, "jaxley.stone"), - pytest.param(1000, True, True, 0.001, "jax.sparse"), + # pytest.param(1, False, False, 0.0, "jax.sparse"), + # # Test a network of SWC cells with both solvers. + # pytest.param(10, False, True, 0.1, "jaxley.stone"), + # pytest.param(10, False, True, 0.1, "jax.sparse"), + # # Test a larger network of smaller neurons with both solvers. + # pytest.param(1000, True, True, 0.001, "jaxley.stone"), + # pytest.param(1000, True, True, 0.001, "jax.sparse"), ), ) @compare_to_baseline(baseline_iters=3) From 9c06bd156579aaa129b7d13440d8a217fbebb25c Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 16:03:00 +0100 Subject: [PATCH 16/37] fix: add regression test output files to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 6162a95b..d5638eb6 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,8 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +tests/regression_test_results.json +tests/regression_test_baselines.json # Translations *.mo From 6d3d0a55947dbfa1718c0fa2f9583be3ac3ff19d Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 17:06:58 +0100 Subject: [PATCH 17/37] wip: small changes --- .../workflows/update_regression_baseline.yml | 2 +- tests/test_regression.py | 36 ++++++++++--------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index d8fddff4..91b2ad62 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -33,7 +33,7 @@ jobs: run: | git config --global user.name 'GitHub Action' git config --global user.email 'action@github.com' - UPDATE_BASELINE=1 pytest tests/test_regression.py + NEW_BASELINE=1 pytest tests/test_regression.py - name: Commit and push if baseline changed if: github.event.pull_request.base.ref == 'main' diff --git a/tests/test_regression.py b/tests/test_regression.py index 26426260..511c68c4 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -30,14 +30,16 @@ pytestmark = pytest.mark.regression # mark all tests as regression tests in this file -UPDATE_BASELINE = ( - os.environ["UPDATE_BASELINE"] if "UPDATE_BASELINE" in os.environ else 0 -) +NEW_BASELINE = os.environ["NEW_BASELINE"] if "NEW_BASELINE" in os.environ else 0 +dirname = os.path.dirname(__file__) +fpath_baselines = os.path.join(dirname, "regression_test_baselines.json") +fpath_results = os.path.join(dirname, "regression_test_results.json") + tolerance = 0.2 baselines = {} -if os.path.exists("tests/regression_test_baselines.json"): - with open("tests/regression_test_baselines.json") as f: +if os.path.exists(fpath_baselines): + with open(fpath_baselines) as f: baselines = json.load(f) @@ -63,9 +65,15 @@ def generate_regression_report(base_results, new_results): base_time = base_runtimes.get(key) diff = None if base_time is None else ((new_time - base_time) / base_time) - emoji = "🆕" if diff is None else "⚪" - emoji = "🔴" if diff > tolerance else emoji - emoji = "🟢" if diff < 0 else emoji + emoji = "" + if diff is None: + emoji = "🆕" + elif diff > tolerance: + emoji = "🔴" + elif diff < 0: + emoji = "🟢" + else: + emoji = "⚪" time_str = ( f"({new_time:.6f}s)" @@ -113,20 +121,16 @@ def test_wrapper(**kwargs): key = generate_unique_key(header) runs = [] - num_iters = self.baseline_iters if UPDATE_BASELINE else self.test_iters + num_iters = self.baseline_iters if NEW_BASELINE else self.test_iters for _ in range(num_iters): runtimes = func(**kwargs) runs.append(runtimes) runtimes = {k: np.mean([d[k] for d in runs]) for k in runs[0]} - fpath = ( - "tests/regression_test_results.json" - if not UPDATE_BASELINE - else "tests/regression_test_baselines.json" - ) + fpath = fpath_results if not NEW_BASELINE else fpath_baselines append_to_json(fpath, header["test_name"], header["input_kwargs"], runtimes) - if not UPDATE_BASELINE: + if not NEW_BASELINE: assert key in baselines, f"No basline found for {header}" func_baselines = baselines[key]["runtimes"] for key, baseline in func_baselines.items(): @@ -232,4 +236,4 @@ def simulate(params): start_time = time.time() _ = jitted_simulate(params).block_until_ready() runtimes["run_time"] = time.time() - start_time - return runtimes + return runtimes # @compare_to_baseline decorator will compare this to the baseline From 2855ff757f98752ddda0e1422f47143e981a31a0 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 20:03:09 +0100 Subject: [PATCH 18/37] fix: use marker and update tests to exclude regressions --- .github/workflows/regression_tests.yml | 2 +- .github/workflows/tests.yml | 2 +- .github/workflows/update_regression_baseline.yml | 2 +- tests/conftest.py | 2 +- tests/test_regression.py | 14 +++++++------- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index df7cdeb2..9a08ca77 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -36,7 +36,7 @@ jobs: else echo "No regression test results found in main branch" fi - pytest tests/test_regression.py + pytest -m regression git checkout # - name: Comment PR diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ff5ef5bc..1750f290 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -39,4 +39,4 @@ jobs: - name: Test with pytest run: | pip install pytest pytest-cov - pytest tests/ -m "not runtime" --cov=jaxley --cov-report=xml + pytest tests/ -m "not regression" --cov=jaxley --cov-report=xml diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 91b2ad62..427e9213 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -33,7 +33,7 @@ jobs: run: | git config --global user.name 'GitHub Action' git config --global user.email 'action@github.com' - NEW_BASELINE=1 pytest tests/test_regression.py + NEW_BASELINE=1 pytest -m regression - name: Commit and push if baseline changed if: github.event.pull_request.base.ref == 'main' diff --git a/tests/conftest.py b/tests/conftest.py index 354594d7..15f01295 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -232,7 +232,7 @@ def print_regression_report(): print("\n\n\nRegression Test Report\n----------------------\n") if not baselines: print( - "No baselines found. Run `git checkout main;UPDATE_BASELINE=1 pytest tests/test_regression.py; git checkout -`" + "No baselines found. Run `git checkout main;UPDATE_BASELINE=1 pytest -m regression; git checkout -`" ) print(generate_regression_report(baselines, results)) diff --git a/tests/test_regression.py b/tests/test_regression.py index 511c68c4..932649c3 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -186,13 +186,13 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): ( # Test a single SWC cell with both solvers. pytest.param(1, False, False, 0.0, "jaxley.stone"), - # pytest.param(1, False, False, 0.0, "jax.sparse"), - # # Test a network of SWC cells with both solvers. - # pytest.param(10, False, True, 0.1, "jaxley.stone"), - # pytest.param(10, False, True, 0.1, "jax.sparse"), - # # Test a larger network of smaller neurons with both solvers. - # pytest.param(1000, True, True, 0.001, "jaxley.stone"), - # pytest.param(1000, True, True, 0.001, "jax.sparse"), + pytest.param(1, False, False, 0.0, "jax.sparse"), + # Test a network of SWC cells with both solvers. + pytest.param(10, False, True, 0.1, "jaxley.stone"), + pytest.param(10, False, True, 0.1, "jax.sparse"), + # Test a larger network of smaller neurons with both solvers. + pytest.param(1000, True, True, 0.001, "jaxley.stone"), + pytest.param(1000, True, True, 0.001, "jax.sparse"), ), ) @compare_to_baseline(baseline_iters=3) From 61c577c3e3f09cb95c590b7c4c3d08ed8084f8c8 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 20:29:58 +0100 Subject: [PATCH 19/37] enh: add comment trigger for baseline_updates --- .github/workflows/regression_tests.yml | 18 +----------- .../workflows/update_regression_baseline.yml | 28 ++++++++++++------- tests/test_regression.py | 2 +- 3 files changed, 20 insertions(+), 28 deletions(-) diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 9a08ca77..c7d5f8ce 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -37,20 +37,4 @@ jobs: echo "No regression test results found in main branch" fi pytest -m regression - git checkout - - # - name: Comment PR - # if: github.event.pull_request.base.ref == 'main' - # uses: actions/github-script@v7 - # with: - # github-token: ${{ secrets.GITHUB_TOKEN }} - # script: | - # const fs = require('fs'); - # const TestReport = fs.readFileSync('regression_test_report.txt', 'utf8'); - - # await github.rest.issues.createComment({ - # issue_number: context.issue.number, - # owner: context.repo.owner, - # repo: context.repo.repo, - # body: `## Regression Test Results\n\`\`\`\n${TestReport}\n\`\`\`` - # }); \ No newline at end of file + git checkout \ No newline at end of file diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 427e9213..6c35f789 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -1,12 +1,15 @@ # .github/workflows/regression_tests.yml -name: Update Regression Tests +name: Update Regression Baseline on: - workflow_dispatch: + issue_comment: + types: [created] + jobs: regression_tests: name: regression_tests + if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_baseline') runs-on: ubuntu-20.04 permissions: contents: write @@ -27,6 +30,15 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[dev]" + + - name: Get PR branch + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: Checkout PR branch + uses: actions/checkout@v3 + with: + ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Update baseline if: github.event.pull_request.base.ref == 'main' @@ -35,13 +47,9 @@ jobs: git config --global user.email 'action@github.com' NEW_BASELINE=1 pytest -m regression - - name: Commit and push if baseline changed + - name: Commit and push if: github.event.pull_request.base.ref == 'main' run: | - if git diff --quiet tests/regression_test_baselines.json; then - echo "No changes to baseline file" - else - git add tests/regression_test_baselines.json - git commit -m "Update regression test baselines" - git push origin HEAD:${{ github.head_ref }} - fi \ No newline at end of file + git add tests/regression_test_baselines.json + git commit -m "Update regression test baselines" + git push origin HEAD:${{ steps.comment-branch.outputs.head_ref }} \ No newline at end of file diff --git a/tests/test_regression.py b/tests/test_regression.py index 932649c3..ced9d75c 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -110,7 +110,7 @@ def append_to_json(fpath, test_name, input_kwargs, runtimes): class compare_to_baseline: - def __init__(self, baseline_iters=1, test_iters=1): + def __init__(self, baseline_iters=3, test_iters=1): self.baseline_iters = baseline_iters self.test_iters = test_iters From a20bbd2aecbba06503ed668e0196f781ad32f45f Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 20:38:12 +0100 Subject: [PATCH 20/37] wip: baseline trigger added on pr to get the first baselines --- .github/workflows/regression_tests.yml | 14 ++++----- .../workflows/update_regression_baseline.yml | 31 +++++++++---------- 2 files changed, 22 insertions(+), 23 deletions(-) diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index c7d5f8ce..9eeda934 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -30,11 +30,11 @@ jobs: - name: Run benchmarks and compare to baseline if: github.event.pull_request.base.ref == 'main' run: | - # Check if regression test results exist in main branch - if [ -f 'git cat-file -e main:tests/regression_test_baselines.json' ]; then - git checkout main tests/regression_test_baselines.json - else - echo "No regression test results found in main branch" - fi + # # Check if regression test results exist in main branch + # if [ -f 'git cat-file -e main:tests/regression_test_baselines.json' ]; then + # git checkout main tests/regression_test_baselines.json + # else + # echo "No regression test results found in main branch" + # fi pytest -m regression - git checkout \ No newline at end of file + # git checkout \ No newline at end of file diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 6c35f789..e058956c 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -1,23 +1,31 @@ -# .github/workflows/regression_tests.yml +# .github/workflows/update_regression_tests.yml name: Update Regression Baseline on: - issue_comment: - types: [created] - + # issue_comment: + # types: [created] + pull_request: + branches: + - main jobs: - regression_tests: + update_regression_tests: name: regression_tests - if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_baseline') + # if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_baseline') runs-on: ubuntu-20.04 permissions: contents: write pull-requests: write steps: - - uses: actions/checkout@v3 + - name: Get PR branch + uses: xt0rted/pull-request-comment-branch@v1 + id: comment-branch + + - name: Checkout PR branch + uses: actions/checkout@v3 with: + ref: ${{ steps.comment-branch.outputs.head_ref }} lfs: true fetch-depth: 0 # This ensures we can checkout main branch too @@ -30,15 +38,6 @@ jobs: run: | python -m pip install --upgrade pip pip install -e ".[dev]" - - - name: Get PR branch - uses: xt0rted/pull-request-comment-branch@v1 - id: comment-branch - - - name: Checkout PR branch - uses: actions/checkout@v3 - with: - ref: ${{ steps.comment-branch.outputs.head_ref }} - name: Update baseline if: github.event.pull_request.base.ref == 'main' From 3b4ab437a8967e84b86b21464728518222c9cb0a Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:03:00 +0100 Subject: [PATCH 21/37] fix: tmp fixes --- .github/workflows/update_regression_baseline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index e058956c..2530bad2 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout PR branch uses: actions/checkout@v3 with: - ref: ${{ steps.comment-branch.outputs.head_ref }} + # ref: ${{ steps.comment-branch.outputs.head_sha }} lfs: true fetch-depth: 0 # This ensures we can checkout main branch too @@ -51,4 +51,4 @@ jobs: run: | git add tests/regression_test_baselines.json git commit -m "Update regression test baselines" - git push origin HEAD:${{ steps.comment-branch.outputs.head_ref }} \ No newline at end of file + git push origin HEAD:${{ steps.comment-branch.outputs.head_sha }} \ No newline at end of file From d577fb4126c5332ec0e5762d0eca9ed77cfaea6d Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:06:17 +0100 Subject: [PATCH 22/37] fix: dummy workflow --- .github/workflows/update_regression_baseline.yml | 2 +- tests/test_regression.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 2530bad2..3c3129e4 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -10,7 +10,7 @@ on: jobs: update_regression_tests: - name: regression_tests + name: update_regression_tests # if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_baseline') runs-on: ubuntu-20.04 permissions: diff --git a/tests/test_regression.py b/tests/test_regression.py index ced9d75c..511c68c4 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -110,7 +110,7 @@ def append_to_json(fpath, test_name, input_kwargs, runtimes): class compare_to_baseline: - def __init__(self, baseline_iters=3, test_iters=1): + def __init__(self, baseline_iters=1, test_iters=1): self.baseline_iters = baseline_iters self.test_iters = test_iters @@ -186,13 +186,13 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): ( # Test a single SWC cell with both solvers. pytest.param(1, False, False, 0.0, "jaxley.stone"), - pytest.param(1, False, False, 0.0, "jax.sparse"), - # Test a network of SWC cells with both solvers. - pytest.param(10, False, True, 0.1, "jaxley.stone"), - pytest.param(10, False, True, 0.1, "jax.sparse"), - # Test a larger network of smaller neurons with both solvers. - pytest.param(1000, True, True, 0.001, "jaxley.stone"), - pytest.param(1000, True, True, 0.001, "jax.sparse"), + # pytest.param(1, False, False, 0.0, "jax.sparse"), + # # Test a network of SWC cells with both solvers. + # pytest.param(10, False, True, 0.1, "jaxley.stone"), + # pytest.param(10, False, True, 0.1, "jax.sparse"), + # # Test a larger network of smaller neurons with both solvers. + # pytest.param(1000, True, True, 0.001, "jaxley.stone"), + # pytest.param(1000, True, True, 0.001, "jax.sparse"), ), ) @compare_to_baseline(baseline_iters=3) From 4ea4ef8778b73dab2747c87da91be37775482919 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:11:17 +0100 Subject: [PATCH 23/37] fix: test workflow --- .github/workflows/regression_tests.yml | 6 +++--- .github/workflows/tests.yml | 12 ++++++------ .github/workflows/update_regression_baseline.yml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index 9eeda934..c6daa60b 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -2,9 +2,9 @@ name: Regression Tests on: - pull_request: - branches: - - main +# pull_request: +# branches: +# - main jobs: regression_tests: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1750f290..5b5ebf07 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -1,12 +1,12 @@ name: Tests on: - push: - branches: - - main - pull_request: - branches: - - main +# push: +# branches: +# - main +# pull_request: +# branches: +# - main jobs: build: diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 3c3129e4..75b6f1e4 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout PR branch uses: actions/checkout@v3 with: - # ref: ${{ steps.comment-branch.outputs.head_sha }} + ref: ${{ steps.comment-branch.outputs.head_sha }} lfs: true fetch-depth: 0 # This ensures we can checkout main branch too From 5f0310bf72d05512db52b2d61aa2ad72d5f60f88 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:12:52 +0100 Subject: [PATCH 24/37] rm: rm baselines and results --- tests/regression_test_baselines.json | 17 ----------------- tests/regression_test_results.json | 17 ----------------- ...ssions for actions. tested in different repo | 9 --------- 3 files changed, 43 deletions(-) delete mode 100644 tests/regression_test_baselines.json delete mode 100644 tests/regression_test_results.json delete mode 100644 tests/regression_test_results.json~enh: update regression tests. works with correct permissions for actions. tested in different repo diff --git a/tests/regression_test_baselines.json b/tests/regression_test_baselines.json deleted file mode 100644 index a04bc97d..00000000 --- a/tests/regression_test_baselines.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "ec3a4fad11d2bfb1bc5f8f10529cb06f2ff9919b377e9c0a3419c7f7f237f06e": { - "test_name": "test_runtime", - "input_kwargs": { - "num_cells": 1, - "artificial": false, - "connect": false, - "connection_prob": 0.0, - "voltage_solver": "jaxley.stone" - }, - "runtimes": { - "build_time": 1.0473196506500244, - "compile_time": 29.728327592213947, - "run_time": 0.15101234118143717 - } - } -} \ No newline at end of file diff --git a/tests/regression_test_results.json b/tests/regression_test_results.json deleted file mode 100644 index f0da8ba3..00000000 --- a/tests/regression_test_results.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "ec3a4fad11d2bfb1bc5f8f10529cb06f2ff9919b377e9c0a3419c7f7f237f06e": { - "test_name": "test_runtime", - "input_kwargs": { - "num_cells": 1, - "artificial": false, - "connect": false, - "connection_prob": 0.0, - "voltage_solver": "jaxley.stone" - }, - "runtimes": { - "build_time": 2.8589820861816406, - "compile_time": 31.13769817352295, - "run_time": 0.12563538551330566 - } - } -} \ No newline at end of file diff --git a/tests/regression_test_results.json~enh: update regression tests. works with correct permissions for actions. tested in different repo b/tests/regression_test_results.json~enh: update regression tests. works with correct permissions for actions. tested in different repo deleted file mode 100644 index 40925f57..00000000 --- a/tests/regression_test_results.json~enh: update regression tests. works with correct permissions for actions. tested in different repo +++ /dev/null @@ -1,9 +0,0 @@ -{ - "module.test_runtime1": 1.0000774711370468, - "module.test_runtime2": 1.0000772399362177, - "module.test_runtime3": 1.000094958813861, - "module.test_runtime4": 1.0000748871825635, - "module.test_runtime5": 1.0000761719420552, - "module.test_runtime6": 1.0001780251041055, - "module.test_runtime7": 1.0000949839595705 -} \ No newline at end of file From 0deec7822517a703ca965d17f433bdaa5ac8ea07 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:16:23 +0100 Subject: [PATCH 25/37] fix: use head_ref instead of head_sha --- .github/workflows/update_regression_baseline.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 75b6f1e4..aa16af69 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -25,7 +25,7 @@ jobs: - name: Checkout PR branch uses: actions/checkout@v3 with: - ref: ${{ steps.comment-branch.outputs.head_sha }} + ref: ${{ steps.comment-branch.outputs.head_ref }} # using head_sha vs. head_ref makes this work for forks lfs: true fetch-depth: 0 # This ensures we can checkout main branch too @@ -51,4 +51,4 @@ jobs: run: | git add tests/regression_test_baselines.json git commit -m "Update regression test baselines" - git push origin HEAD:${{ steps.comment-branch.outputs.head_sha }} \ No newline at end of file + git push origin HEAD:${{ steps.comment-branch.outputs.head_ref }} \ No newline at end of file From 82a72779275073285587334a069cf45c281eca71 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:17:45 +0100 Subject: [PATCH 26/37] fix: rm comment_pr_branch checkout --- .github/workflows/update_regression_baseline.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index aa16af69..7af7c354 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -18,14 +18,14 @@ jobs: pull-requests: write steps: - - name: Get PR branch - uses: xt0rted/pull-request-comment-branch@v1 - id: comment-branch + # - name: Get PR branch + # uses: xt0rted/pull-request-comment-branch@v1 + # id: comment-branch - name: Checkout PR branch uses: actions/checkout@v3 with: - ref: ${{ steps.comment-branch.outputs.head_ref }} # using head_sha vs. head_ref makes this work for forks + # ref: ${{ steps.comment-branch.outputs.head_ref }} # using head_sha vs. head_ref makes this work for forks lfs: true fetch-depth: 0 # This ensures we can checkout main branch too From be804890113b298550cbd926b3968dde879b2479 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:25:32 +0100 Subject: [PATCH 27/37] fix: force git add due to gitignore --- .github/workflows/update_regression_baseline.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 7af7c354..6bbf645d 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -2,7 +2,7 @@ name: Update Regression Baseline on: - # issue_comment: + # issue_comment: # event runs on the default branch # types: [created] pull_request: branches: @@ -11,8 +11,8 @@ on: jobs: update_regression_tests: name: update_regression_tests - # if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_baseline') runs-on: ubuntu-20.04 + # if: github.event.issue.pull_request && contains(github.event.comment.body, '/update_baseline') permissions: contents: write pull-requests: write @@ -25,7 +25,7 @@ jobs: - name: Checkout PR branch uses: actions/checkout@v3 with: - # ref: ${{ steps.comment-branch.outputs.head_ref }} # using head_sha vs. head_ref makes this work for forks + # ref: ${{ steps.comment-branch.outputs.head_sha }} # using head_sha vs. head_ref makes this work for forks lfs: true fetch-depth: 0 # This ensures we can checkout main branch too @@ -49,6 +49,6 @@ jobs: - name: Commit and push if: github.event.pull_request.base.ref == 'main' run: | - git add tests/regression_test_baselines.json + git add -f tests/regression_test_baselines.json # since it's in .gitignore git commit -m "Update regression test baselines" - git push origin HEAD:${{ steps.comment-branch.outputs.head_ref }} \ No newline at end of file + git push origin HEAD:${{ steps.comment-branch.outputs.head_sha }} \ No newline at end of file From baa98e7d31c8aabba96ebe4d7f9ac624de2571fa Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:31:44 +0100 Subject: [PATCH 28/37] fix: new test --- .github/workflows/update_regression_baseline.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 6bbf645d..54a3d8cb 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -51,4 +51,5 @@ jobs: run: | git add -f tests/regression_test_baselines.json # since it's in .gitignore git commit -m "Update regression test baselines" - git push origin HEAD:${{ steps.comment-branch.outputs.head_sha }} \ No newline at end of file + # git push origin HEAD:${{ steps.comment-branch.outputs.head_sha }} + git push origin HEAD:${{ github.head_ref }} \ No newline at end of file From c82862d428ad4bd0a1ecc1208694657f551b055d Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:37:48 +0100 Subject: [PATCH 29/37] add: add first new baseline, now that the workflow runs --- tests/test_regression.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/test_regression.py b/tests/test_regression.py index 511c68c4..ced9d75c 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -110,7 +110,7 @@ def append_to_json(fpath, test_name, input_kwargs, runtimes): class compare_to_baseline: - def __init__(self, baseline_iters=1, test_iters=1): + def __init__(self, baseline_iters=3, test_iters=1): self.baseline_iters = baseline_iters self.test_iters = test_iters @@ -186,13 +186,13 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): ( # Test a single SWC cell with both solvers. pytest.param(1, False, False, 0.0, "jaxley.stone"), - # pytest.param(1, False, False, 0.0, "jax.sparse"), - # # Test a network of SWC cells with both solvers. - # pytest.param(10, False, True, 0.1, "jaxley.stone"), - # pytest.param(10, False, True, 0.1, "jax.sparse"), - # # Test a larger network of smaller neurons with both solvers. - # pytest.param(1000, True, True, 0.001, "jaxley.stone"), - # pytest.param(1000, True, True, 0.001, "jax.sparse"), + pytest.param(1, False, False, 0.0, "jax.sparse"), + # Test a network of SWC cells with both solvers. + pytest.param(10, False, True, 0.1, "jaxley.stone"), + pytest.param(10, False, True, 0.1, "jax.sparse"), + # Test a larger network of smaller neurons with both solvers. + pytest.param(1000, True, True, 0.001, "jaxley.stone"), + pytest.param(1000, True, True, 0.001, "jax.sparse"), ), ) @compare_to_baseline(baseline_iters=3) From 297d91c3b8e22daf32d675aab1c1ee5146623e41 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:50:38 +0100 Subject: [PATCH 30/37] fix: test sth --- .github/workflows/update_regression_baseline.yml | 4 ++-- tests/test_regression.py | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 54a3d8cb..ae382308 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -42,8 +42,8 @@ jobs: - name: Update baseline if: github.event.pull_request.base.ref == 'main' run: | - git config --global user.name 'GitHub Action' - git config --global user.email 'action@github.com' + git config --global user.name '${{ github.event.pull_request.user.login }}' + git config --global user.email '${{ github.event.pull_request.user.login }}@users.noreply.github.com' NEW_BASELINE=1 pytest -m regression - name: Commit and push diff --git a/tests/test_regression.py b/tests/test_regression.py index ced9d75c..511c68c4 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -110,7 +110,7 @@ def append_to_json(fpath, test_name, input_kwargs, runtimes): class compare_to_baseline: - def __init__(self, baseline_iters=3, test_iters=1): + def __init__(self, baseline_iters=1, test_iters=1): self.baseline_iters = baseline_iters self.test_iters = test_iters @@ -186,13 +186,13 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): ( # Test a single SWC cell with both solvers. pytest.param(1, False, False, 0.0, "jaxley.stone"), - pytest.param(1, False, False, 0.0, "jax.sparse"), - # Test a network of SWC cells with both solvers. - pytest.param(10, False, True, 0.1, "jaxley.stone"), - pytest.param(10, False, True, 0.1, "jax.sparse"), - # Test a larger network of smaller neurons with both solvers. - pytest.param(1000, True, True, 0.001, "jaxley.stone"), - pytest.param(1000, True, True, 0.001, "jax.sparse"), + # pytest.param(1, False, False, 0.0, "jax.sparse"), + # # Test a network of SWC cells with both solvers. + # pytest.param(10, False, True, 0.1, "jaxley.stone"), + # pytest.param(10, False, True, 0.1, "jax.sparse"), + # # Test a larger network of smaller neurons with both solvers. + # pytest.param(1000, True, True, 0.001, "jaxley.stone"), + # pytest.param(1000, True, True, 0.001, "jax.sparse"), ), ) @compare_to_baseline(baseline_iters=3) From 12ee256f56c0efcd3ce515107ff264d6dc0f312a Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Thu, 21 Nov 2024 21:54:22 +0100 Subject: [PATCH 31/37] fix: test sth else --- .github/workflows/update_regression_baseline.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index ae382308..0c40c676 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -16,6 +16,8 @@ jobs: permissions: contents: write pull-requests: write + env: + username: ${{ github.actor }} steps: # - name: Get PR branch @@ -42,8 +44,8 @@ jobs: - name: Update baseline if: github.event.pull_request.base.ref == 'main' run: | - git config --global user.name '${{ github.event.pull_request.user.login }}' - git config --global user.email '${{ github.event.pull_request.user.login }}@users.noreply.github.com' + git config --global user.name '$username' + git config --global user.email '$username@users.noreply.github.com' NEW_BASELINE=1 pytest -m regression - name: Commit and push From b4fa9b099b419833743d3f8e6f1f864cbe25e95c Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Fri, 22 Nov 2024 10:56:43 +0100 Subject: [PATCH 32/37] fix: retrigger for full baselines --- .github/workflows/update_regression_baseline.yml | 2 +- tests/test_regression.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 0c40c676..7ee7c31e 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -17,7 +17,7 @@ jobs: contents: write pull-requests: write env: - username: ${{ github.actor }} + username: ${{ github.event.pull_request.user.login }} steps: # - name: Get PR branch diff --git a/tests/test_regression.py b/tests/test_regression.py index 511c68c4..932649c3 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -186,13 +186,13 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): ( # Test a single SWC cell with both solvers. pytest.param(1, False, False, 0.0, "jaxley.stone"), - # pytest.param(1, False, False, 0.0, "jax.sparse"), - # # Test a network of SWC cells with both solvers. - # pytest.param(10, False, True, 0.1, "jaxley.stone"), - # pytest.param(10, False, True, 0.1, "jax.sparse"), - # # Test a larger network of smaller neurons with both solvers. - # pytest.param(1000, True, True, 0.001, "jaxley.stone"), - # pytest.param(1000, True, True, 0.001, "jax.sparse"), + pytest.param(1, False, False, 0.0, "jax.sparse"), + # Test a network of SWC cells with both solvers. + pytest.param(10, False, True, 0.1, "jaxley.stone"), + pytest.param(10, False, True, 0.1, "jax.sparse"), + # Test a larger network of smaller neurons with both solvers. + pytest.param(1000, True, True, 0.001, "jaxley.stone"), + pytest.param(1000, True, True, 0.001, "jax.sparse"), ), ) @compare_to_baseline(baseline_iters=3) From d7ff6a6ec9b1b9af170ef54df4de494995a6a793 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Fri, 22 Nov 2024 11:30:44 +0100 Subject: [PATCH 33/37] fix: test commenting of new baselines --- .github/workflows/regression_tests.yml | 6 +-- .../workflows/update_regression_baseline.yml | 20 +++++++- tests/test_regression.py | 50 ++++++++++++------- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/.github/workflows/regression_tests.yml b/.github/workflows/regression_tests.yml index c6daa60b..78fabd09 100644 --- a/.github/workflows/regression_tests.yml +++ b/.github/workflows/regression_tests.yml @@ -2,9 +2,9 @@ name: Regression Tests on: -# pull_request: -# branches: -# - main + # pull_request: + # branches: + # - main jobs: regression_tests: diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 7ee7c31e..687d96f5 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -17,7 +17,7 @@ jobs: contents: write pull-requests: write env: - username: ${{ github.event.pull_request.user.login }} + username: ${{ github.event.pull_request.user.login }} # ${{ github.event.comment.user.login }} steps: # - name: Get PR branch @@ -46,7 +46,25 @@ jobs: run: | git config --global user.name '$username' git config --global user.email '$username@users.noreply.github.com' + mv tests/regression_test_baselines.json tests/regression_test_baselines.json.bak NEW_BASELINE=1 pytest -m regression + + - name: Add comment to PR + uses: actions/github-script@v7 + if: always() + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const fs = require('fs'); + const TestReport = fs.readFileSync('regression_test_report.txt', 'utf8'); + + await github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: `## New Baselines \n\`\`\`\n${TestReport}\n\`\`\`` + }); + - name: Commit and push if: github.event.pull_request.base.ref == 'main' diff --git a/tests/test_regression.py b/tests/test_regression.py index 932649c3..1e487705 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -28,6 +28,13 @@ # t3 = time.time() # return {"sth": t2-t1, sth_else: t3-t2} +def load_json(fpath): + dct = {} + if os.path.exists(fpath): + with open(fpath, "r") as f: + dct = json.load(f) + return dct + pytestmark = pytest.mark.regression # mark all tests as regression tests in this file NEW_BASELINE = os.environ["NEW_BASELINE"] if "NEW_BASELINE" in os.environ else 0 @@ -37,10 +44,8 @@ tolerance = 0.2 -baselines = {} -if os.path.exists(fpath_baselines): - with open(fpath_baselines) as f: - baselines = json.load(f) +baselines = load_json(fpath_baselines) + def generate_regression_report(base_results, new_results): @@ -99,12 +104,9 @@ def append_to_json(fpath, test_name, input_kwargs, runtimes): data = {generate_unique_key(header): {**header, "runtimes": runtimes}} # Save data to a JSON file - if os.path.exists(fpath): - with open(fpath, "r") as f: - result_data = json.load(f) - result_data.update(data) - else: - result_data = data + result_data = load_json(fpath) + result_data.update(data) + with open(fpath, "w") as f: json.dump(result_data, f, indent=2) @@ -120,6 +122,8 @@ def test_wrapper(**kwargs): header = {"test_name": func.__name__, "input_kwargs": kwargs} key = generate_unique_key(header) + new_data, old_data = None, None + runs = [] num_iters = self.baseline_iters if NEW_BASELINE else self.test_iters for _ in range(num_iters): @@ -143,6 +147,18 @@ def test_wrapper(**kwargs): 1 + tolerance ), f"{key} is {diff:.2%} slower than the baseline." + # save report + if NEW_BASELINE: + new_data = load_json(fpath_baselines) + old_data = load_json(fpath_baselines + ".bak") + else: + new_data = load_json(fpath_results) + old_data = load_json(fpath_baselines) + report = generate_regression_report(old_data, new_data) + + with open(dirname + "/regression_test_report.txt", "w") as f: + f.write(report) + return test_wrapper @@ -186,13 +202,13 @@ def build_net(num_cells, artificial=True, connect=True, connection_prob=0.0): ( # Test a single SWC cell with both solvers. pytest.param(1, False, False, 0.0, "jaxley.stone"), - pytest.param(1, False, False, 0.0, "jax.sparse"), - # Test a network of SWC cells with both solvers. - pytest.param(10, False, True, 0.1, "jaxley.stone"), - pytest.param(10, False, True, 0.1, "jax.sparse"), - # Test a larger network of smaller neurons with both solvers. - pytest.param(1000, True, True, 0.001, "jaxley.stone"), - pytest.param(1000, True, True, 0.001, "jax.sparse"), + # pytest.param(1, False, False, 0.0, "jax.sparse"), + # # Test a network of SWC cells with both solvers. + # pytest.param(10, False, True, 0.1, "jaxley.stone"), + # pytest.param(10, False, True, 0.1, "jax.sparse"), + # # Test a larger network of smaller neurons with both solvers. + # pytest.param(1000, True, True, 0.001, "jaxley.stone"), + # pytest.param(1000, True, True, 0.001, "jax.sparse"), ), ) @compare_to_baseline(baseline_iters=3) From b1cd30a94969ddf6dc8e88398efe0074c5cc9f8c Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Fri, 22 Nov 2024 11:34:23 +0100 Subject: [PATCH 34/37] fix: fix path --- .../workflows/update_regression_baseline.yml | 2 +- tests/test_regression.py | 40 ++++++++++--------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 687d96f5..44a1f742 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -56,7 +56,7 @@ jobs: github-token: ${{ secrets.GITHUB_TOKEN }} script: | const fs = require('fs'); - const TestReport = fs.readFileSync('regression_test_report.txt', 'utf8'); + const TestReport = fs.readFileSync('tests/regression_test_report.txt', 'utf8'); await github.rest.issues.createComment({ issue_number: context.issue.number, diff --git a/tests/test_regression.py b/tests/test_regression.py index 1e487705..1b14a5f3 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -219,37 +219,41 @@ def test_runtime( connection_prob: float, voltage_solver: str, ): + import time delta_t = 0.025 t_max = 100.0 - def simulate(params): - return jx.integrate( - net, - params=params, - t_max=t_max, - delta_t=delta_t, - voltage_solver=voltage_solver, - ) + # def simulate(params): + # return jx.integrate( + # net, + # params=params, + # t_max=t_max, + # delta_t=delta_t, + # voltage_solver=voltage_solver, + # ) runtimes = {} start_time = time.time() - net, params = build_net( - num_cells, - artificial=artificial, - connect=connect, - connection_prob=connection_prob, - ) + # net, params = build_net( + # num_cells, + # artificial=artificial, + # connect=connect, + # connection_prob=connection_prob, + # ) + time.sleep(0.1) runtimes["build_time"] = time.time() - start_time - jitted_simulate = jit(simulate) + # jitted_simulate = jit(simulate) start_time = time.time() - _ = jitted_simulate(params).block_until_ready() + time.sleep(0.31) + # _ = jitted_simulate(params).block_until_ready() runtimes["compile_time"] = time.time() - start_time - params[0]["radius"] = params[0]["radius"].at[0].set(0.5) + # params[0]["radius"] = params[0]["radius"].at[0].set(0.5) start_time = time.time() - _ = jitted_simulate(params).block_until_ready() + # _ = jitted_simulate(params).block_until_ready() + time.sleep(0.21) runtimes["run_time"] = time.time() - start_time return runtimes # @compare_to_baseline decorator will compare this to the baseline From cb1b676b3e017d3cbbb0888ccc67ab802d7da08b Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Fri, 22 Nov 2024 11:44:04 +0100 Subject: [PATCH 35/37] fix: rm mv old baseline since there is none --- .github/workflows/update_regression_baseline.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index 44a1f742..a461e767 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -17,7 +17,7 @@ jobs: contents: write pull-requests: write env: - username: ${{ github.event.pull_request.user.login }} # ${{ github.event.comment.user.login }} + username: ${{ github.event.pull_request.user.login }} # ${{ github.actor }} steps: # - name: Get PR branch @@ -46,10 +46,10 @@ jobs: run: | git config --global user.name '$username' git config --global user.email '$username@users.noreply.github.com' - mv tests/regression_test_baselines.json tests/regression_test_baselines.json.bak + # mv tests/regression_test_baselines.json tests/regression_test_baselines.json.bak NEW_BASELINE=1 pytest -m regression - - name: Add comment to PR + - name: Add Baseline update report to PR comment uses: actions/github-script@v7 if: always() with: From 780f9846f7ccb1fac3355489e4f02e0db8d8b9e6 Mon Sep 17 00:00:00 2001 From: $username <$username@users.noreply.github.com> Date: Fri, 22 Nov 2024 10:45:12 +0000 Subject: [PATCH 36/37] Update regression test baselines --- tests/regression_test_baselines.json | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 tests/regression_test_baselines.json diff --git a/tests/regression_test_baselines.json b/tests/regression_test_baselines.json new file mode 100644 index 00000000..cdf9e30b --- /dev/null +++ b/tests/regression_test_baselines.json @@ -0,0 +1,17 @@ +{ + "ec3a4fad11d2bfb1bc5f8f10529cb06f2ff9919b377e9c0a3419c7f7f237f06e": { + "test_name": "test_runtime", + "input_kwargs": { + "num_cells": 1, + "artificial": false, + "connect": false, + "connection_prob": 0.0, + "voltage_solver": "jaxley.stone" + }, + "runtimes": { + "build_time": 0.10014088948567708, + "compile_time": 0.3103648026784261, + "run_time": 0.2102543512980143 + } + } +} \ No newline at end of file From 1be99228d5ac678aae1225638d82b4512f4d3f82 Mon Sep 17 00:00:00 2001 From: jnsbck-uni Date: Fri, 22 Nov 2024 11:46:44 +0100 Subject: [PATCH 37/37] wip: test mv old baseline --- .github/workflows/update_regression_baseline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update_regression_baseline.yml b/.github/workflows/update_regression_baseline.yml index a461e767..98ada263 100644 --- a/.github/workflows/update_regression_baseline.yml +++ b/.github/workflows/update_regression_baseline.yml @@ -46,7 +46,7 @@ jobs: run: | git config --global user.name '$username' git config --global user.email '$username@users.noreply.github.com' - # mv tests/regression_test_baselines.json tests/regression_test_baselines.json.bak + mv tests/regression_test_baselines.json tests/regression_test_baselines.json.bak NEW_BASELINE=1 pytest -m regression - name: Add Baseline update report to PR comment