Modernize testing, load configurations more dynamically, use pooch

.flake8

@@ -0,0 +1,24 @@
+[flake8]
+exclude =
+	.git,
+	docs,
+	build,
+	.eggs,
+	tests/mini-esgf-data
+max-line-length = 88
+max-complexity = 12
+ignore =
+	C901
+	E203
+	E231
+	E266
+	E402
+	E501
+	F401
+	F403
+	F632
+	W503
+	W504
+	F841
+	F541
+	F821

.github/workflows/main.yml

@@ -9,12 +9,34 @@ on:
       - master
 
 jobs:
+
+  lint:
+    name: Lint (Python${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - name: Set up Python${{ matrix.python-version }}
+        uses: actions/setup-python@39cd14951b08e74b54015e9e001cdefcf80e669f # v5.1.1
+        with:
+          python-version: "3.x"
+      - name: Install linting libraries
+        run: |
+          python -m pip install flake8 black
+      - name: Lint with flake8
+        run: |
+          flake8 roocs_utils tests
+      - name: Check formatting with black
+        run: |
+          black --check roocs_utils tests
+
   build:
     name: Test (Python${{ matrix.python-version }})
+    needs: lint
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
     steps:
       - name: Checkout Repository
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -25,14 +47,6 @@ jobs:
       - name: Install roocs-utils
         run: |
           python -m pip install --editable ".[dev]"
-      - name: Lint with flake8
-        run: |
-          flake8 roocs_utils tests
-        if: matrix.python-version == 3.8
-      - name: Check formatting with black
-        run: |
-          black --check --target-version py38 roocs_utils tests
-        if: matrix.python-version == 3.8
       - name: Test with pytest
         run: |
           pytest -v tests

.gitignore

@@ -6,6 +6,10 @@ __pycache__/
 # C extensions
 *.so
 
+# Documentation
+docs/apidoc/modules.rst
+docs/apidoc/roocs_utils*.rst
+
 # Distribution / packaging
 .Python
 env/

.pre-commit-config.yaml

@@ -4,33 +4,27 @@ default_language_version:
 
 repos:
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.16.0
+    rev: v3.19.0
     hooks:
       - id: pyupgrade
-        args: ['--py38-plus']
+        args: ['--py39-plus']
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
-        exclude: setup.cfg
       - id: end-of-file-fixer
-        exclude: setup.cfg
       - id: check-yaml
       - id: debug-statements
   - repo: https://github.com/ambv/black
-    rev: 24.4.2
+    rev: 24.10.0
     hooks:
       - id: black
-        args: ["--target-version", "py38"]
+        args: ["--target-version", "py39"]
   - repo: https://github.com/pycqa/flake8
-    rev: 7.1.0
+    rev: 7.1.1
     hooks:
       - id: flake8
-        args: ['--config=setup.cfg']
-  - repo: https://github.com/asottile/reorder_python_imports
-    rev: v3.13.0
-    hooks:
-      - id: reorder-python-imports
+        args: ['--config=.flake8']
   - repo: meta
     hooks:
       - id: check-hooks-apply

.readthedocs.yml

@@ -8,6 +8,9 @@ version: 2
 # Build documentation in the docs/ directory with Sphinx
 sphinx:
   configuration: docs/conf.py
+  # fail_on_warning might generate hard to fix error, in this case it can be
+  # disabled but this also means those errors will fail silently, choose wisely.
+  fail_on_warning: true
 
 # Optionally build your docs in additional formats such as PDF and ePub
 # formats:
@@ -17,6 +20,9 @@ build:
   os: ubuntu-22.04
   tools:
     python: "mambaforge-22.9"
+  jobs:
+    pre_build:
+      - sphinx-apidoc -o docs/apidoc --private --module-first roocs_utils
 
 conda:
   environment: environment.yml

AUTHORS.rst

@@ -17,3 +17,4 @@ Contributors
 ------------
 
 * Martin Schupfner <[email protected]> `@sol1105 <https://github.com/sol1105>`_
+* Trevor James Smith <[email protected]> `@Zeistperre <https://github.com/Zeistperre>`_

CONTRIBUTING.rst

@@ -16,7 +16,6 @@ Report bugs at https://github.com/roocs/roocs-utils/issues.
 
 If you are reporting a bug, please include:
 
-
 * Your operating system name and version.
 * Any details about your local setup that might be helpful in troubleshooting.
 * Detailed steps to reproduce the bug.
@@ -60,27 +59,23 @@ Ready to contribute? Here's how to set up ``roocs-utils`` for local development.
 
 
 #. Fork the ``roocs-utils`` repo on GitHub.
-#.
-    Clone your fork locally:
+#. Clone your fork locally:
 
     $ git clone [email protected]:your_name_here/roocs-utils.git
 
-#.
-    Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:
+#. Install your local copy into a virtualenv. Assuming you have virtualenvwrapper installed, this is how you set up your fork for local development:
 
     $ mkvirtualenv roocs-utils
     $ cd roocs-utils/
     $ python setup.py develop
 
-#.
-    Create a branch for local development:
+#. Create a branch for local development:
 
     $ git checkout -b name-of-your-bugfix-or-feature
 
     Now you can make your changes locally.
 
-#.
-    When you are done making changes, check that your changes pass flake8 and the
+#. When you are done making changes, check that your changes pass flake8 and the
     tests, including testing other Python versions with tox:
 
     $ flake8 roocs-utils tests
@@ -89,15 +84,13 @@ Ready to contribute? Here's how to set up ``roocs-utils`` for local development.
 
     To get flake8 and tox, just pip install them into your virtualenv.
 
-#.
-    Commit your changes and push your branch to GitHub:
+#. Commit your changes and push your branch to GitHub:
 
     $ git add .
     $ git commit -m "Your detailed description of your changes."
     $ git push origin name-of-your-bugfix-or-feature
 
-#.
-    Submit a pull request through the GitHub website.
+#. Submit a pull request through the GitHub website.
 
 Pull Request Guidelines
 -----------------------

HISTORY.rst

@@ -1,6 +1,20 @@
 Version History
 ===============
 
+v0.7.0 (unreleased)
+-------------------
+
+This is a major release to help synchronize the package with the other `roocs` packages in advance of
+an eventual merge with `CLISOPS`. This will be the final release of `roocs-utils` as a standalone package.
+
+Breaking Changes
+^^^^^^^^^^^^^^^^
+* The package has been refactored to use `pyproject.toml` with the `flit-core` backend for packaging.
+* The testing framework has been updated to use `pooch` for testing data caching, with on-the-fly data download and caching.
+* `pre-commit` hooks have been updated and simplified.
+* `pytest-xdist` has been added to the `dev` dependencies (distributed testing is disabled by default).
+* `GitPython` has been removed from the dependencies in favour of `pooch` for testing data caching.
+
 v0.6.9 (2024-07-15)
 -------------------
 
@@ -41,7 +55,7 @@ v0.6.5 (2023-11-09)
 Other Changes
 ^^^^^^^^^^^^^
 * Updated ``realization`` dimension in common coords (#108).
-* Linting.
+* Code linting.
 * Added Python 3.11 to tests.
 * Updated requirements for cf_xarray.
 
@@ -93,14 +107,14 @@ Bug Fixes
 Breaking Changes
 ^^^^^^^^^^^^^^^^
 * Intake catalog maker removed, now in it's own package: `roocs/catalog-maker <https://github.com/roocs/catalog-maker>`_
-* Change to input parameter classes::
-  * Added: ``roocs_utils.parameter.time_components_parameter.TimeComponentsParameter``
-  * Modified input types required for classes::
-    * ``roocs_utils.parameter.time_parameter.TimeParameter``
-    * ``roocs_utils.parameter.level_parameter.LevelParameter``
-  * They both now require their inputs to be one of::
-    * ``roocs_utils.parameter.param_utils.Interval`` - to specify a range/interval
-    * ``roocs_utils.parameter.param_utils.Series`` - to specify a series of values
+* Change to input parameter classes:
+    * Added: ``roocs_utils.parameter.time_components_parameter.TimeComponentsParameter``
+    * Modified input types required for classes:
+        * ``roocs_utils.parameter.time_parameter.TimeParameter``
+        * ``roocs_utils.parameter.level_parameter.LevelParameter``
+    * They both now require their inputs to be one of:
+        * ``roocs_utils.parameter.param_utils.Interval`` - to specify a range/interval
+        * ``roocs_utils.parameter.param_utils.Series`` - to specify a series of values
 
 New Features
 ^^^^^^^^^^^^
@@ -134,12 +148,12 @@ Breaking Changes
 ^^^^^^^^^^^^^^^^
 * Inventory maker now removed and replaced by intake catalog maker which writes a csv file with the dataset entries and a yaml description file.
 * In ``etc/roocs.ini`` the option ``use_inventory`` has been replaced by ``use_catalog`` and the inventory maker options have been replaced with equivalent catalog options. However, the option to include file paths or not no longer exists.
-* The catalog maker now uses a database backend and creates a csv file so there are 3 new dependencies for the catalog maker: pandas and abcunit-backend and psycopg2.
-This means a database backend must be specified and the paths for the pickle files in ``etc/roocs.ini`` are no longer necessary. For more information see the README.
+* The catalog maker now uses a database backend and creates a csv file so there are three new dependencies for the catalog maker: `pandas` and `abcunit-backend` and `psycopg2`.
+  This means a database backend must be specified and the paths for the pickle files in ``etc/roocs.ini`` are no longer necessary. For more information see the README.
 
 Other Changes
 ^^^^^^^^^^^^^
-* oyaml removed as a dependency
+* `oyaml` removed as a dependency
 
 v0.3.0 (2021-03-30)
 -------------------
@@ -154,7 +168,8 @@ v0.2.1 (2021-02-19)
 
 Bug Fixes
 ^^^^^^^^^
-* clean up imports ... remove pandas dependency.
+* Cleaned up imports.
+* Removed `pandas` dependency.
 
 v0.2.0 (2021-02-18)
 -------------------
@@ -165,7 +180,7 @@ Breaking Changes
 * oyaml>=0.9 - new dependency for inventory
 * Interface to inventory maker changed. Detailed instructions for use added in README.
 * Adjusted file name template. Underscore removed before ``__derive__time_range``
-* New dev dependency: GitPython==3.1.12
+* New dev dependency: `GitPython==3.1.12`
 
 New Features
 ^^^^^^^^^^^^
@@ -205,9 +220,7 @@ Fixing formatting of doc strings and imports
 
 Breaking Changes
 ^^^^^^^^^^^^^^^^
-* Use of ``roocs_utils.parameter.parameterise.parameterise``:
-import should now be ``from roocs_utils.parameter import parameterise``
-and usage should be, for example ``parameters = parameterise(collection=ds, time=time, area=area, level=level)``
+* Use of ``roocs_utils.parameter.parameterise.parameterise``: import should now be ``from roocs_utils.parameter import parameterise`` and usage should be, for example ``parameters = parameterise(collection=ds, time=time, area=area, level=level)``
 
 New Features
 ^^^^^^^^^^^^
@@ -241,9 +254,7 @@ Breaking Changes
 New Features
 ^^^^^^^^^^^^
 * ``parameterise`` function added in ``roocs_utils.parameter`` to use in all roocs packages.
-* ``ROOCS_CONFIG`` environment variable can be used to override default config in ``etc/roocs.ini``.
-  To use a local config file set ``ROOCS_CONFIG`` as the file path to this file. Several file paths can be provided
-  separated by a ``:``
+* ``ROOCS_CONFIG`` environment variable can be used to override default config in ``etc/roocs.ini``. To use a local config file set ``ROOCS_CONFIG`` as the file path to this file. Several file paths can be provided separated by a ``:``
 * Inventory functionality added - this can be used to create an inventory of datasets. See ``README`` for more info.
 * ``project_utils`` added with the following functions to get the project name of a dataset and the base directory for
   that project.
@@ -252,8 +263,7 @@ New Features
 
 Bug Fixes
 ^^^^^^^^^
-* ``xarray_utils.xarray_utils.get_main_variable`` updated to exclude common coordinates from the search for the
-  main variable. This fixes a bug where coordinates such as ``lon_bounds`` would be returned as the main variable.
+* ``xarray_utils.xarray_utils.get_main_variable`` updated to exclude common coordinates from the search for the main variable. This fixes a bug where coordinates such as ``lon_bounds`` would be returned as the main variable.
 
 Other Changes
 ^^^^^^^^^^^^^

LICENSE

@@ -1,26 +1,28 @@
-Copyright 2018 United Kingdom Research and Innovation
+BSD 3-Clause License
 
-Authors: Eleanor Smith
+Copyright (c) 2018-2024, United Kingdom Research and Innovation
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
+
 1. Redistributions of source code must retain the above copyright notice, this
    list of conditions and the following disclaimer.
+
 2. Redistributions in binary form must reproduce the above copyright notice,
    this list of conditions and the following disclaimer in the documentation
    and/or other materials provided with the distribution.
-3. Neither the name of the copyright holder nor the names of its contributors
-   may be used to endorse or promote products derived from this software
-   without specific prior written permission.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
 
 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.