Merge pull request #327 from 21cmfast/release

Release v3.3.0

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.2.0
+current_version = 3.3.0
 commit = False
 tag = False
 

.flake8

@@ -6,10 +6,14 @@ ignore =
     W503
     F403
     F401
-    N803  # Naming upper/lowercase -- too hard right now.
-    N806  # Naming upper/lowercase -- too hard right now.
-    N802  # Naming upper/lowercase -- too hard right now.
-    D401  # Docstring in imperative mood. This should *not* be the case for @property's, but can't ignore them atm.
+    # Naming upper/lowercase -- too hard right now.
+    N803
+    # Naming upper/lowercase -- too hard right now.
+    N806
+    # Naming upper/lowercase -- too hard right now.
+    N802
+    # Docstring in imperative mood. This should *not* be the case for @property's, but can't ignore them atm.
+    D401
 max-line-length = 88
 max-complexity = 40
 docstring-convention=numpy

.github/workflows/publish-to-pypi.yaml

@@ -38,14 +38,14 @@ jobs:
       # already exist, and thus fail.
       - name: Publish distribution 📦 to Test PyPI
         if: ${{ !contains(github.ref, 'production') }}
-        uses: pypa/gh-action-pypi-publish@master
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.test_pypi_password }}
           repository_url: https://test.pypi.org/legacy/
       # deploy to actual PyPI only when a tag is pushed (which should happen only
       # automatically on production)
       - name: Publish distribution 📦 to PyPI
         if: startsWith(github.ref, 'refs/tags')
-        uses: pypa/gh-action-pypi-publish@master
+        uses: pypa/gh-action-pypi-publish@release/v1
         with:
           password: ${{ secrets.pypi_password }}

.github/workflows/test_suite.yaml

@@ -93,12 +93,13 @@ jobs:
 
       - name: Run Tests
         run: |
-          python -m pytest -l --cov=py21cmfast --cov-config=.coveragerc -vv --cov-report xml:./coverage.xml --durations=25 ${{ env.extra_pytest_args }}
+          python -m pytest -n 2 -l --cov=py21cmfast --cov-config=.coveragerc -vv --cov-report xml:./coverage.xml --durations=25 ${{ env.extra_pytest_args }}
 
       - uses: codecov/codecov-action@v2
         if: matrix.os == 'ubuntu-latest' && success() && !contains(github.event.pull_request.labels.*.name, 'auto-pr')
         with:
           fail_ci_if_error: true
           verbose: true
+          token: ${{ secrets.CODECOV_TOKEN }}
 
 # CC=$HOME/miniconda/envs/$ENV_NAME/bin/gcc

.gitignore

@@ -52,3 +52,5 @@ c2r*
 r2c*
 plots/
 .vscode/
+src/21cmfast/_version.py
+src/py21cmfast/_version.py

.pre-commit-config.yaml

@@ -2,7 +2,7 @@ exclude: '(^docs/conf.py|^src/py21cmfast/_data/|changethelog.py)'
 
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks
-      rev: v4.2.0
+      rev: v4.4.0
       hooks:
           - id: trailing-whitespace
           - id: check-added-large-files
@@ -16,43 +16,42 @@ repos:
           - id: requirements-txt-fixer
           - id: mixed-line-ending
             args: ['--fix=no']
-    - repo: https://gitlab.com/pycqa/flake8
-      rev: 3.9.2  # pick a git hash / tag to point to
+    - repo: https://github.com/pycqa/flake8
+      rev: 6.0.0  # pick a git hash / tag to point to
       hooks:
         - id: flake8
           additional_dependencies:
             - flake8-quotes
-            - flake8-comprehensions
+#            - flake8-comprehensions  # no support for flake8 v6
             - flake8-builtins
-            - flake8-eradicate
+            # - flake8-eradicate  # no support for flake8 v6
             - pep8-naming
             - flake8-pytest
             - flake8-docstrings
             - flake8-rst-docstrings
             - flake8-rst
-            - flake8-copyright
-            - flake8-ownership
+            # - flake8-copyright  # no support flor flake8 v6
             - flake8-print
 
     - repo: https://github.com/psf/black
-      rev: 22.3.0
+      rev: 23.1.0
       hooks:
           - id: black
 
-    - repo: https://github.com/pre-commit/mirrors-isort
-      rev: v5.10.1
+    - repo: https://github.com/PyCQA/isort
+      rev: 5.12.0
       hooks:
       - id: isort
 
     - repo: https://github.com/pre-commit/pygrep-hooks
-      rev: v1.9.0
+      rev: v1.10.0
       hooks:
       - id: rst-backticks
       - id: rst-directive-colons
       - id: rst-inline-touching-normal
 
     - repo: https://github.com/asottile/pyupgrade
-      rev: v2.32.0
+      rev: v3.3.1
       hooks:
       - id: pyupgrade
         args: [--py36-plus]

CHANGELOG.rst

@@ -4,6 +4,47 @@ Changelog
 dev-version
 -----------
 
+v3.3.0 [17 May 2023]
+----------------------
+
+Internals
+---------
+
+* Refactored setting up of inputs to high-level functions so that there is less code
+  repetition.
+
+Fixed
+-----
+
+* Running with ``R_BUBBLE_MAX`` too large auto-fixes it to be ``BOX_LEN`` (#112)
+* Bug in calling ``clear_cache``.
+* Inconsistency in the way that the very highest redshift of an evolution is handled
+  between low-level code (eg. ``spin_temperature()``) and high-level code (eg. ``run_coeval()``).
+
+
+Added
+-----
+
+* New ``validate_all_inputs`` function that cross-references the four main input structs
+  and ensures all the parameters make sense together. Mostly for internal use.
+* Ability to save/read directly from an open HDF5 File (#170)
+* An implementation of cloud-in-cell to more accurately redistribute the perturbed mass
+  across all neighbouring cells instead of the previous nearest cell approach
+* Changed PhotonConsEndCalibz from z = 5 -> z = 3.5 to handle later reionisation
+  scenarios in line with current observations (#305)
+* Add in an initialisation check for the photon conservation to address some issues
+  arising for early EOR histories (#311)
+* Added ``NON_CUBIC_FACTOR`` to ``UserParams`` to allow for non-cubic coeval boxes (#289)
+
+v3.2.1 [13 Sep 2022]
+----------------------
+
+Changed
+~~~~~~~
+
+* Included log10_mturnovers(_mini) in lightcone class. Only useful when USE_MINI_HALOS
+
+
 v3.2.0 [11 Jul 2022]
 ----------------------
 

INSTALLATION.rst

@@ -42,11 +42,52 @@ which supports ``-fopenmp``. As long as these packages install into the standard
 a standard installation of ``21cmFAST`` will be automatically possible (see below).
 If they are installed to a place not on the ``LD_LIBRARY``/``INCLUDE`` paths, then you
 must use the compilation options (see below) to specify where they are.
+For example, you can check if the header file for ``fftw3`` is
+in its default location ``/usr/include/`` by running::
+
+    cd /usr/include/
+    find fftw3.h
+
+or::
+
+    locate fftw3.h
 
 .. note:: there exists the option of installing ``gsl``, ``fftw`` and ``gcc`` using ``conda``.
           This is discussed below in the context of MacOSX, where it is often the
           easiest way to get the dependencies, but it is equally applicable to linux.
 
+Ubuntu
+^^^^^^
+If you are installing 21cmFAST just as a user, the very simplest method is ``conda``
+-- with this method you simply need ``conda install -c conda-forge 21cmFAST``, and all
+dependencies will be automatically installed. However, if you are going to use
+``pip`` to install the package directly from the repository, there is
+a [bug in pip](https://stackoverflow.com/questions/71340058/conda-does-not-look-for-libpthread-and-libpthread-nonshared-at-the-right-place-w)
+that means it cannot find conda-installed shared libraries properly. In that case, it is much
+easier to install the basic dependencies (``gcc``, ``gsl`` and ``fftw3``) with your
+system's package manager. ``gcc`` is by default available in Ubuntu.
+To check if ``gcc`` is installed, run ``gcc --version`` in your terminal.
+Install ``fftw3`` and ``gsl`` on your system with  ``sudo apt-get install libfftw3-dev libgsl-dev``.
+
+
+In your ``21cmfast`` environment, now install the ``21cmFAST`` package using::
+
+    cd /path/to/21cmFAST/
+    pip install .
+
+If there is an issue during installation, add ``DEBUG=all`` or ``--DEBUG`` which may provide additional
+information.
+
+.. note:: If there is an error during compilation that the ``fftw3`` library cannot be found,
+          check where the ``fftw3`` library is actually located using ``locate libfftw3.so``.
+          For example, it may be located in ``/usr/lib/x86_64-linux-gnu/``. Then, provide this path
+          to the installation command with the ``LIB`` flag. For more details see the note in the
+          MacOSX section below.
+
+.. note:: You may choose to install ``gsl`` as an anaconda package as well, however, in that case,
+          you need to add both ``INC`` paths in the installation command e.g.:
+          ``GSL_INC=/path/to/conda/env/include FFTW_INC=/usr/include``
+
 MacOSX
 ~~~~~~
 On MacOSX, obtaining ``gsl`` and ``fftw`` is typically more difficult, and in addition,

VERSION

@@ -1 +1 @@
-3.2.0
+3.3.0

build_cffi.py

@@ -11,9 +11,9 @@
 # Set compilation arguments dependent on environment... a bit buggy
 # =================================================================
 if "DEBUG" in os.environ:
-    extra_compile_args = ["-fopenmp", "-w", "-g", "-O0"]
+    extra_compile_args = ["-fopenmp", "-w", "-g", "-O0", "--verbose"]
 else:
-    extra_compile_args = ["-fopenmp", "-Ofast", "-w"]
+    extra_compile_args = ["-fopenmp", "-Ofast", "-w", "--verbose"]
 
 # Set the C-code logging level.
 # If DEBUG is set, we default to the highest level, but if not,

ci/macos-latest-env.yml

@@ -5,6 +5,7 @@ dependencies:
   - pytest>=5.0
   - pytest-cov
   - pytest-remotedata>=0.3.2
+  - pytest-xdist
   - click
   - numpy
   - pyyaml

ci/ubuntu-latest-env.yml

@@ -5,6 +5,7 @@ dependencies:
   - pytest>=5.0
   - pytest-cov
   - pytest-remotedata>=0.3.2
+  - pytest-xdist
   - click
   - numpy
   - pyyaml

devel/estimated_mem_usage_plot.py

@@ -0,0 +1,125 @@
+"""Simple example script for plotting estimated memory usage."""
+import matplotlib.pyplot as plt
+import numpy as np
+
+from py21cmfast import AstroParams, CosmoParams, FlagOptions, UserParams
+from py21cmfast._memory import estimate_memory_lightcone
+
+user_params = UserParams(
+    {
+        "HII_DIM": 250,
+        "BOX_LEN": 2000.0,
+        "DIM": 1000,
+        "N_THREADS": 8,
+        "USE_RELATIVE_VELOCITIES": True,
+        "POWER_SPECTRUM": 5,
+        "USE_FFTW_WISDOM": True,
+        "PERTURB_ON_HIGH_RES": True,
+        "USE_INTERPOLATION_TABLES": True,
+        "FAST_FCOLL_TABLES": True,
+    }
+)
+flag_options = FlagOptions(
+    {
+        "INHOMO_RECO": True,
+        "USE_MASS_DEPENDENT_ZETA": True,
+        "USE_TS_FLUCT": True,
+        "USE_MINI_HALOS": True,
+    },
+    USE_VELS_AUX=True,
+)
+
+
+h2dim = np.array(list(range(200, 1500, 100)))
+
+
+mems4 = []
+for h2 in range(200, 1500, 100):
+    user_params.update(HII_DIM=h2, DIM=4 * h2)
+    mem = estimate_memory_lightcone(
+        max_redshift=1420 / 50 - 1,
+        redshift=1420 / 200 - 1,
+        user_params=user_params,
+        flag_options=flag_options,
+        cosmo_params=CosmoParams(),
+        astro_params=AstroParams(),
+    )
+    mems4.append(mem)
+peaks4 = np.array([m["peak_memory"] / (1024**3) for m in mems4])
+
+mems3 = []
+for h2 in range(200, 1500, 100):
+    user_params.update(HII_DIM=h2, DIM=3 * h2)
+    mem = estimate_memory_lightcone(
+        max_redshift=1420 / 50 - 1,
+        redshift=1420 / 200 - 1,
+        user_params=user_params,
+        flag_options=flag_options,
+        cosmo_params=CosmoParams(),
+        astro_params=AstroParams(),
+    )
+    mems3.append(mem)
+peaks3 = np.array([m["peak_memory"] / (1024**3) for m in mems3])
+
+mems2 = []
+for h2 in range(200, 1500, 100):
+    user_params.update(HII_DIM=h2, DIM=2 * h2)
+    mem = estimate_memory_lightcone(
+        max_redshift=1420 / 50 - 1,
+        redshift=1420 / 200 - 1,
+        user_params=user_params,
+        flag_options=flag_options,
+        cosmo_params=CosmoParams(),
+        astro_params=AstroParams(),
+    )
+    mems2.append(mem)
+peaks2 = np.array([m["peak_memory"] / (1024**3) for m in mems2])
+
+
+user_params.update(HII_DIM=1300, DIM=2600)  # what we actually did.
+mem = estimate_memory_lightcone(
+    max_redshift=1420 / 50 - 1,
+    redshift=1420 / 200 - 1,
+    user_params=user_params,
+    flag_options=flag_options,
+    cosmo_params=CosmoParams(),
+    astro_params=AstroParams(),
+)
+
+
+plt.plot(h2dim, peaks2, label="DIM=2*HII_DIM")
+plt.plot(h2dim, peaks3, label="DIM=3*HII_DIM")
+plt.plot(h2dim, peaks4, label="DIM=4*HII_DIM")
+
+plt.scatter(
+    1300,
+    mem["peak_memory"] / 1024**3,
+    marker="*",
+    color="k",
+    label="Proposed Sims",
+    s=100,
+    zorder=3,
+)
+
+plt.scatter(
+    1300, 2924.847, marker="+", color="r", label="Measured Peak RAM", s=75, zorder=3
+)
+
+theory = h2dim**3
+plt.plot(
+    h2dim,
+    theory * (peaks3.max() / theory.max()) * 1.2,
+    label="HII_DIM^3 scaling",
+    color="grey",
+    ls="--",
+)
+
+plt.axhline(4000, linestyle="dashed", color="k")
+plt.text(h2dim[0], 4300, "Avail RAM on Bridges2-EM")
+plt.legend()
+plt.yscale("log")
+plt.xscale("log")
+plt.ylabel("Peak memory (GB)")
+plt.xlabel("HII_DIM")
+
+plt.savefig("peak_memory_usage.png")