From 7db1bfd06953755dd86b8412cd0728d06f3dcde2 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 12 Feb 2024 06:11:51 +0000 Subject: [PATCH 01/15] Add PySR integration test --- pytest/test_all.py | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/pytest/test_all.py b/pytest/test_all.py index af7111e6..a0e45d56 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -17,3 +17,39 @@ def test_issue_394(): assert jl.f is f assert jl.y is y assert jl.seval("f(x)") == 4 + + +def test_integration_pysr(): + "Integration tests for PySR" + import subprocess + import sys + import tempfile + + with tempfile.TemporaryDirectory() as tempdir: + subprocess.run([sys.executable, "-m", "virtualenv", tempdir], check=True) + # Install this package + subprocess.run([sys.executable, "-m", "pip", "install", "."], check=True) + # Install PySR with no requirement on JuliaCall + subprocess.run( + [sys.executable, "-m", "pip", "install", "--no-deps", "pysr"], check=True + ) + # Install PySR test requirements + subprocess.run( + [ + sys.executable, + "-m", + "pip", + "install", + "sympy", + "pandas", + "scikit_learn", + "click", + "setuptools", + "typing_extensions", + "pytest", + "nbval", + ], + check=True, + ) + # Run PySR main test suite + subprocess.run([sys.executable, "-m", "pysr", "test", "main"], check=True) From 5f5d3f4a80e331890f9b44731400e98281d54f5f Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 12 Feb 2024 06:15:57 +0000 Subject: [PATCH 02/15] Add missing virtualenv install --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d678f218..3677215f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -66,7 +66,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest pytest-cov + pip install flake8 pytest pytest-cov virtualenv cp pysrc/juliacall/juliapkg-dev.json pysrc/juliacall/juliapkg.json pip install -e . - name: Lint with flake8 From 977296df1e7bae6a1fee7447282066c036a8b6aa Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Mon, 12 Feb 2024 06:27:14 +0000 Subject: [PATCH 03/15] Use correct python executable for virtualenv --- pytest/test_all.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/pytest/test_all.py b/pytest/test_all.py index a0e45d56..67dc76c4 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -21,22 +21,33 @@ def test_issue_394(): def test_integration_pysr(): "Integration tests for PySR" + import os + import platform import subprocess import sys import tempfile with tempfile.TemporaryDirectory() as tempdir: subprocess.run([sys.executable, "-m", "virtualenv", tempdir], check=True) + + virtualenv_path = os.path.join( + tempdir, "Scripts" if platform.system() == "Windows" else "bin" + ) + virtualenv_executable = os.path.join(virtualenv_path, "python") + + assert os.path.exists(virtualenv_executable) + # Install this package - subprocess.run([sys.executable, "-m", "pip", "install", "."], check=True) + subprocess.run([virtualenv_executable, "-m", "pip", "install", "."], check=True) # Install PySR with no requirement on JuliaCall subprocess.run( - [sys.executable, "-m", "pip", "install", "--no-deps", "pysr"], check=True + [virtualenv_executable, "-m", "pip", "install", "--no-deps", "pysr"], + check=True, ) # Install PySR test requirements subprocess.run( [ - sys.executable, + virtualenv_executable, "-m", "pip", "install", @@ -52,4 +63,6 @@ def test_integration_pysr(): check=True, ) # Run PySR main test suite - subprocess.run([sys.executable, "-m", "pysr", "test", "main"], check=True) + subprocess.run( + [virtualenv_executable, "-m", "pysr", "test", "main"], check=True + ) From c0dcd736467a497a2f083b0a7bb96c0c84f3306e Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 10:45:01 +0800 Subject: [PATCH 04/15] test: move pysr test to integration folder --- pytest/integration/pysr.py | 45 ++++++++++++++++++++++++++++++++++ pytest/test_all.py | 49 -------------------------------------- 2 files changed, 45 insertions(+), 49 deletions(-) create mode 100644 pytest/integration/pysr.py diff --git a/pytest/integration/pysr.py b/pytest/integration/pysr.py new file mode 100644 index 00000000..321a53ce --- /dev/null +++ b/pytest/integration/pysr.py @@ -0,0 +1,45 @@ +def test_integration_pysr(): + "Integration tests for PySR" + import os + import platform + import subprocess + import sys + import tempfile + + with tempfile.TemporaryDirectory() as tempdir: + subprocess.run([sys.executable, "-m", "virtualenv", tempdir], check=True) + + virtualenv_path = os.path.join( + tempdir, "Scripts" if platform.system() == "Windows" else "bin" + ) + virtualenv_executable = os.path.join(virtualenv_path, "python") + + assert os.path.exists(virtualenv_executable) + + # Install this package + subprocess.run([virtualenv_executable, "-m", "pip", "install", "."], check=True) + # Install PySR with no requirement on JuliaCall + subprocess.run( + [virtualenv_executable, "-m", "pip", "install", "--no-deps", "pysr"], + check=True, + ) + # Install PySR test requirements + subprocess.run( + [ + virtualenv_executable, + "-m", + "pip", + "install", + "sympy", + "pandas", + "scikit_learn", + "click", + "setuptools", + "pytest", + ], + check=True, + ) + # Run PySR main test suite + subprocess.run( + [virtualenv_executable, "-m", "pysr", "test", "main"], check=True + ) diff --git a/pytest/test_all.py b/pytest/test_all.py index 0ecb406c..a895398a 100644 --- a/pytest/test_all.py +++ b/pytest/test_all.py @@ -164,52 +164,3 @@ def test_call_nogil(yld, raw): t2 = time() - t0 # executing the tasks should take about 1 second because they happen in parallel assert 0.9 < t2 < 1.5 - - -def test_integration_pysr(): - "Integration tests for PySR" - import os - import platform - import subprocess - import sys - import tempfile - - with tempfile.TemporaryDirectory() as tempdir: - subprocess.run([sys.executable, "-m", "virtualenv", tempdir], check=True) - - virtualenv_path = os.path.join( - tempdir, "Scripts" if platform.system() == "Windows" else "bin" - ) - virtualenv_executable = os.path.join(virtualenv_path, "python") - - assert os.path.exists(virtualenv_executable) - - # Install this package - subprocess.run([virtualenv_executable, "-m", "pip", "install", "."], check=True) - # Install PySR with no requirement on JuliaCall - subprocess.run( - [virtualenv_executable, "-m", "pip", "install", "--no-deps", "pysr"], - check=True, - ) - # Install PySR test requirements - subprocess.run( - [ - virtualenv_executable, - "-m", - "pip", - "install", - "sympy", - "pandas", - "scikit_learn", - "click", - "setuptools", - "typing_extensions", - "pytest", - "nbval", - ], - check=True, - ) - # Run PySR main test suite - subprocess.run( - [virtualenv_executable, "-m", "pysr", "test", "main"], check=True - ) From cd610773ad06193b50fa6c6adcf6a4c10906df1f Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 10:54:43 +0800 Subject: [PATCH 05/15] ci: run integration tests separately --- .github/workflows/tests.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 859b8090..c3055f62 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,6 +58,11 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] pyversion: ["3.x", "3.8"] + test-type: ['unit'] + include: + - os: ubuntu-latest + pyversion: '3.x' + test-type: 'integration' steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.pyversion }} @@ -82,7 +87,11 @@ jobs: flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Run tests run: | - pytest -s --nbval --cov=pysrc ./pytest/ + if [ "${{ matrix.test-type }}" == "integration" ]; then + pytest -s --nbval --cov=pysrc ./pytest/integration/ + else + pytest -s --nbval --cov=pysrc ./pytest/ -k "not integration" + fi env: PYTHON_JULIACALL_THREADS: '2' - name: Upload coverage to Codecov From bb793b3064b3907c47e7b9d0c90daf0d790402bb Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 10:59:34 +0800 Subject: [PATCH 06/15] ci: skip code coverage for integration tests --- .github/workflows/tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c3055f62..b053c669 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -88,7 +88,7 @@ jobs: - name: Run tests run: | if [ "${{ matrix.test-type }}" == "integration" ]; then - pytest -s --nbval --cov=pysrc ./pytest/integration/ + pytest -s ./pytest/integration/ else pytest -s --nbval --cov=pysrc ./pytest/ -k "not integration" fi @@ -96,5 +96,6 @@ jobs: PYTHON_JULIACALL_THREADS: '2' - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 + if: ${{ matrix.test-type == 'unit' }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} From b2791271944936e7539a4826bcc640970b59a421 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 11:00:55 +0800 Subject: [PATCH 07/15] ci: improve naming scheme of CI --- .github/workflows/tests.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index b053c669..ba2c52a4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ on: jobs: julia: - name: Test (${{ matrix.os }}, julia ${{ matrix.jlversion }}) + name: test-${{ matrix.test-type }}-${{ matrix.os }}-jl${{ matrix.jlversion }} runs-on: ${{ matrix.os }} strategy: fail-fast: true @@ -20,6 +20,7 @@ jobs: arch: [x64] # x86 unsupported by MicroMamba os: [ubuntu-latest, windows-latest, macos-latest] jlversion: ['1','1.6'] + test-type: ['unit'] steps: - uses: actions/checkout@v2 - name: Set up Julia ${{ matrix.jlversion }} @@ -48,10 +49,11 @@ jobs: uses: julia-actions/julia-processcoverage@v1 - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 + if: ${{ matrix.test-type == 'unit' }} env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} python: - name: Test (${{ matrix.os }}, python ${{ matrix.pyversion }}) + name: test-${{ matrix.test-type }}-${{ matrix.os }}-py${{ matrix.pyversion }} runs-on: ${{ matrix.os }} strategy: fail-fast: true From b5749e1eb66cd16114681c0c25734bf84450af70 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 11:03:01 +0800 Subject: [PATCH 08/15] test: fix naming scheme --- pytest/integration/{pysr.py => test_pysr.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename pytest/integration/{pysr.py => test_pysr.py} (100%) diff --git a/pytest/integration/pysr.py b/pytest/integration/test_pysr.py similarity index 100% rename from pytest/integration/pysr.py rename to pytest/integration/test_pysr.py From a7cf3819b6723ac71b50982a61c1c1dd4d5753a1 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 11:03:52 +0800 Subject: [PATCH 09/15] ci: improve naming scheme of CI --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ba2c52a4..709005fb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,7 +12,7 @@ on: jobs: julia: - name: test-${{ matrix.test-type }}-${{ matrix.os }}-jl${{ matrix.jlversion }} + name: julia-${{ matrix.test-type }}-${{ matrix.os }}-jl${{ matrix.jlversion }} runs-on: ${{ matrix.os }} strategy: fail-fast: true @@ -53,7 +53,7 @@ jobs: env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} python: - name: test-${{ matrix.test-type }}-${{ matrix.os }}-py${{ matrix.pyversion }} + name: python-${{ matrix.test-type }}-${{ matrix.os }}-py${{ matrix.pyversion }} runs-on: ${{ matrix.os }} strategy: fail-fast: true From 3391801c689ad82c69c60bd25b6f08620a5137b9 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 11:29:23 +0800 Subject: [PATCH 10/15] test: ensure we can see the integration test output --- pytest/integration/test_pysr.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pytest/integration/test_pysr.py b/pytest/integration/test_pysr.py index 321a53ce..7b48d3df 100644 --- a/pytest/integration/test_pysr.py +++ b/pytest/integration/test_pysr.py @@ -7,7 +7,8 @@ def test_integration_pysr(): import tempfile with tempfile.TemporaryDirectory() as tempdir: - subprocess.run([sys.executable, "-m", "virtualenv", tempdir], check=True) + run_kws = dict(check=True, capture_output=True) + subprocess.run([sys.executable, "-m", "virtualenv", tempdir], **run_kws) virtualenv_path = os.path.join( tempdir, "Scripts" if platform.system() == "Windows" else "bin" @@ -17,11 +18,11 @@ def test_integration_pysr(): assert os.path.exists(virtualenv_executable) # Install this package - subprocess.run([virtualenv_executable, "-m", "pip", "install", "."], check=True) + subprocess.run([virtualenv_executable, "-m", "pip", "install", "."], **run_kws) # Install PySR with no requirement on JuliaCall subprocess.run( [virtualenv_executable, "-m", "pip", "install", "--no-deps", "pysr"], - check=True, + **run_kws, ) # Install PySR test requirements subprocess.run( @@ -37,9 +38,9 @@ def test_integration_pysr(): "setuptools", "pytest", ], - check=True, + **run_kws, ) # Run PySR main test suite subprocess.run( - [virtualenv_executable, "-m", "pysr", "test", "main"], check=True + [virtualenv_executable, "-m", "pysr", "test", "main"], **run_kws, ) From 96bfd1d60ce84825446560b4492e52c8bfb107c4 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 12:06:43 +0800 Subject: [PATCH 11/15] Revert "test: ensure we can see the integration test output" This reverts commit 3391801c689ad82c69c60bd25b6f08620a5137b9. --- pytest/integration/test_pysr.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pytest/integration/test_pysr.py b/pytest/integration/test_pysr.py index 7b48d3df..321a53ce 100644 --- a/pytest/integration/test_pysr.py +++ b/pytest/integration/test_pysr.py @@ -7,8 +7,7 @@ def test_integration_pysr(): import tempfile with tempfile.TemporaryDirectory() as tempdir: - run_kws = dict(check=True, capture_output=True) - subprocess.run([sys.executable, "-m", "virtualenv", tempdir], **run_kws) + subprocess.run([sys.executable, "-m", "virtualenv", tempdir], check=True) virtualenv_path = os.path.join( tempdir, "Scripts" if platform.system() == "Windows" else "bin" @@ -18,11 +17,11 @@ def test_integration_pysr(): assert os.path.exists(virtualenv_executable) # Install this package - subprocess.run([virtualenv_executable, "-m", "pip", "install", "."], **run_kws) + subprocess.run([virtualenv_executable, "-m", "pip", "install", "."], check=True) # Install PySR with no requirement on JuliaCall subprocess.run( [virtualenv_executable, "-m", "pip", "install", "--no-deps", "pysr"], - **run_kws, + check=True, ) # Install PySR test requirements subprocess.run( @@ -38,9 +37,9 @@ def test_integration_pysr(): "setuptools", "pytest", ], - **run_kws, + check=True, ) # Run PySR main test suite subprocess.run( - [virtualenv_executable, "-m", "pysr", "test", "main"], **run_kws, + [virtualenv_executable, "-m", "pysr", "test", "main"], check=True ) From 3504ec6575cc4f5f2ec69341d942c50945f92f4f Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 12:18:23 +0800 Subject: [PATCH 12/15] test: simplify pysr integration test --- .github/workflows/tests.yml | 10 +++++- pytest/integration/test_pysr.py | 56 +++++++++------------------------ 2 files changed, 23 insertions(+), 43 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 709005fb..d6efde49 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -78,9 +78,17 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest pytest-cov nbval numpy virtualenv + pip install flake8 pytest pytest-cov nbval numpy cp pysrc/juliacall/juliapkg-dev.json pysrc/juliacall/juliapkg.json pip install -e . + - name: Install integration test dependencies + if: ${{ matrix.test-type == 'integration' }} + run: | + pip install sympy pandas scikit_learn click setuptools + # Note that the `--no-deps` flag is needed for ensuring + # we avoid installing a newer julicall. The initial `pip install` + # is for the non-juliacall dependencies + pip install --no-deps pysr - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names diff --git a/pytest/integration/test_pysr.py b/pytest/integration/test_pysr.py index 321a53ce..3f8e3b6c 100644 --- a/pytest/integration/test_pysr.py +++ b/pytest/integration/test_pysr.py @@ -1,45 +1,17 @@ def test_integration_pysr(): - "Integration tests for PySR" - import os - import platform - import subprocess - import sys - import tempfile + "Simple PySR search" + import pysr + import numpy as np - with tempfile.TemporaryDirectory() as tempdir: - subprocess.run([sys.executable, "-m", "virtualenv", tempdir], check=True) + rng = np.random.RandomState(0) + X = rng.randn(100, 5) + y = np.cos(X[:, 0] * 2.1 - 0.5) + X[:, 1] * 0.7 + model = pysr.PySRRegressor( + niterations=30, + unary_operators=["cos"], + binary_operators=["*", "+", "-"], + early_stop_condition=1e-5, + ) + model.fit(X, y) + assert model.equations_.iloc[-1]["loss"] < 1e-5 - virtualenv_path = os.path.join( - tempdir, "Scripts" if platform.system() == "Windows" else "bin" - ) - virtualenv_executable = os.path.join(virtualenv_path, "python") - - assert os.path.exists(virtualenv_executable) - - # Install this package - subprocess.run([virtualenv_executable, "-m", "pip", "install", "."], check=True) - # Install PySR with no requirement on JuliaCall - subprocess.run( - [virtualenv_executable, "-m", "pip", "install", "--no-deps", "pysr"], - check=True, - ) - # Install PySR test requirements - subprocess.run( - [ - virtualenv_executable, - "-m", - "pip", - "install", - "sympy", - "pandas", - "scikit_learn", - "click", - "setuptools", - "pytest", - ], - check=True, - ) - # Run PySR main test suite - subprocess.run( - [virtualenv_executable, "-m", "pysr", "test", "main"], check=True - ) From c271a97e31a79ae404ea097a3622021ec68a4961 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 12:24:03 +0800 Subject: [PATCH 13/15] ci: make conditional compatible with windows --- .github/workflows/tests.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d6efde49..2367704c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -55,6 +55,8 @@ jobs: python: name: python-${{ matrix.test-type }}-${{ matrix.os }}-py${{ matrix.pyversion }} runs-on: ${{ matrix.os }} + env: + PYTHON_JULIACALL_THREADS: '2' strategy: fail-fast: true matrix: @@ -95,15 +97,12 @@ jobs: flake8 . --count --select=E9,F63,F7,F82 --ignore=F821 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Run tests - run: | - if [ "${{ matrix.test-type }}" == "integration" ]; then - pytest -s ./pytest/integration/ - else - pytest -s --nbval --cov=pysrc ./pytest/ -k "not integration" - fi - env: - PYTHON_JULIACALL_THREADS: '2' + - name: Run unit tests + if: ${{ matrix.test-type == 'unit' }} + run: pytest -s --nbval --cov=pysrc ./pytest/ -k "not integration" + - name: Run integration tests + if: ${{ matrix.test-type == 'integration' }} + run: pytest -s ./pytest/integration/ - name: Upload coverage to Codecov uses: codecov/codecov-action@v2 if: ${{ matrix.test-type == 'unit' }} From 4e3df8be26f42c3402b95cdd1319974c108a0cd3 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 12:33:23 +0800 Subject: [PATCH 14/15] ci: time test --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2367704c..520d8992 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -89,7 +89,7 @@ jobs: pip install sympy pandas scikit_learn click setuptools # Note that the `--no-deps` flag is needed for ensuring # we avoid installing a newer julicall. The initial `pip install` - # is for the non-juliacall dependencies + # is for the non-juliacall dependencies. pip install --no-deps pysr - name: Lint with flake8 run: | From fa272c6a663727c0c4cd130fe24b209c58630517 Mon Sep 17 00:00:00 2001 From: MilesCranmer Date: Fri, 23 Aug 2024 14:59:33 +0800 Subject: [PATCH 15/15] test: save to temp file --- pytest/integration/test_pysr.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pytest/integration/test_pysr.py b/pytest/integration/test_pysr.py index 3f8e3b6c..d71a1470 100644 --- a/pytest/integration/test_pysr.py +++ b/pytest/integration/test_pysr.py @@ -11,6 +11,8 @@ def test_integration_pysr(): unary_operators=["cos"], binary_operators=["*", "+", "-"], early_stop_condition=1e-5, + temp_equation_file=True, + progress=False, ) model.fit(X, y) assert model.equations_.iloc[-1]["loss"] < 1e-5