diff --git a/.github/labeler.yml b/.github/labeler.yml new file mode 100644 index 0000000..fc43219 --- /dev/null +++ b/.github/labeler.yml @@ -0,0 +1,34 @@ +dependencies: +- changed-files: + - any-glob-to-any-file: + - "requirements.txt" + - "setup.py" + - "pyproject.toml" + - "Pipfile" + - "Pipfile.lock" + - "requirements/*.txt" + - "requirements/*.in" + +documentation: +- changed-files: + - any-glob-to-any-file: + - "docs/**/*" + - "*.md" + - "*.rst" + +config: +- changed-files: + - any-glob-to-any-file: + - ".pre-commit-config.yaml" + - "config/**/*" + - "settings/**/*" + - "*.ini" + - "*.cfg" + - "*.conf" + +CI/CD: +- changed-files: + - any-glob-to-any-file: + - ".github/**/*" + - "tox.ini" + - ".coveragerc" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..702daa3 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,50 @@ +## Description + + + +## Type of Change + +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) +- [ ] Documentation update + +## Issue Number + + + + + +## Code Compatibility + +- [ ] I have performed a self-review of my code + +### Code Performance and Testing + +- [ ] I have added tests that prove my fix is effective or that my feature works +- [ ] I ran the complete Pytest test suite locally, and they pass +- [ ] I have tested the changes on a single GPU +- [ ] I have tested the changes on multiple GPUs / multi-node setups +- [ ] I have run the Benchmark Profiler against the old version of the code + + + +### Dependencies + +- [ ] I have ensured that the code is still pip-installable after the changes and runs +- [ ] I have not introduced new dependencies in the inference portion of the model +- [ ] I have tested that new dependencies themselves are pip-installable. + + + +### Documentation + +- [ ] My code follows the style guidelines of this project +- [ ] I have updated the documentation and docstrings to reflect the changes +- [ ] I have added comments to my code, particularly in hard-to-understand areas + + + +## Additional Notes + + diff --git a/.github/release.yml b/.github/release.yml new file mode 100644 index 0000000..c530195 --- /dev/null +++ b/.github/release.yml @@ -0,0 +1,23 @@ +# .github/release.yml +# https://docs.github.com/en/repositories/releasing-projects-on-github/automatically-generated-release-notes + +changelog: + exclude: + labels: + - ignore-for-release + - no-changelog + authors: + - pre-commit-ci + categories: + - title: Breaking Changes 🛠 + labels: + - "breaking change" + - title: Exciting New Features 🎉 + labels: + - enhancement + - title: Config Changes 📑 + labels: + - config + - title: Other Changes 🔗 + labels: + - "*" diff --git a/.github/workflows/downstream-ci-hpc.yml b/.github/workflows/downstream-ci-hpc.yml new file mode 100644 index 0000000..7a0a6ec --- /dev/null +++ b/.github/workflows/downstream-ci-hpc.yml @@ -0,0 +1,52 @@ +name: Test downstream dependent packages on HPC + +on: + # Trigger the workflow on push to main or develop, except tag creation + push: + branches: + - 'main' + - 'develop' + tags-ignore: + - '**' + paths-ignore: + - "docs/**" + - "CHANGELOG.md" + - "README.md" + + # Trigger the workflow on pull request + pull_request: + paths-ignore: + - "docs/**" + - "CHANGELOG.md" + - "README.md" + + # Trigger the workflow manually + workflow_dispatch: ~ + + # Trigger after public PR approved for CI + pull_request_target: + types: [labeled] + paths-ignore: + - "docs/**" + - "CHANGELOG.md" + - "README.md" + +jobs: + # Run CI including downstream packages on self-hosted runners + downstream-ci: + name: downstream-ci + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main + with: + anemoi-inference: ecmwf/anemoi-inference@${{ github.event.pull_request.head.sha || github.sha }} + codecov_upload: true + secrets: inherit + + # Build downstream packages on HPC + downstream-ci-hpc: + name: downstream-ci-hpc + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + with: + anemoi-inference: ecmwf/anemoi-inference@${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit diff --git a/.github/workflows/pr-conventional-commit.yml b/.github/workflows/pr-conventional-commit.yml new file mode 100644 index 0000000..1f18a42 --- /dev/null +++ b/.github/workflows/pr-conventional-commit.yml @@ -0,0 +1,21 @@ +name: "Ensure Conventional Commit in PR title" + +on: + pull_request_target: + types: + - opened + - edited + - synchronize + - reopened + +permissions: + pull-requests: read + +jobs: + main: + name: Validate PR title + runs-on: ubuntu-latest + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/pr-label-conventional-commits.yml b/.github/workflows/pr-label-conventional-commits.yml new file mode 100644 index 0000000..6771ea1 --- /dev/null +++ b/.github/workflows/pr-label-conventional-commits.yml @@ -0,0 +1,43 @@ +name: "Pull Request Labeler Conventional Commits" + +on: + pull_request: + branches: [main] + types: + [opened, reopened, labeled, unlabeled] + +jobs: + assign-labels: + runs-on: ubuntu-latest + name: Assign labels in pull request + if: github.event.pull_request.merged == false + steps: + - uses: actions/checkout@v3 + - name: Assign labels from Conventional Commits + id: action-assign-labels + uses: mauroalderete/action-assign-labels@v1 + with: + pull-request-number: ${{ github.event.pull_request.number }} + github-token: ${{ secrets.GITHUB_TOKEN }} + conventional-commits: | + conventional-commits: + - type: 'fix' + nouns: ['FIX', 'Fix', 'fix', 'FIXED', 'Fixed', 'fixed'] + labels: ['bug'] + - type: 'feature' + nouns: ['FEATURE', 'Feature', 'feature', 'FEAT', 'Feat', 'feat'] + labels: ['enhancement'] + - type: 'breaking_change' + nouns: ['BREAKING CHANGE', 'BREAKING', 'MAJOR'] + labels: ['breaking change'] + - type: 'documentation' + nouns: ['doc','docs','docu','document','documentation'] + labels: ['documentation'] + - type: 'build' + nouns: ['build','rebuild','ci'] + labels: ['CI/CD'] + - type: 'config' + nouns: ['config', 'conf', 'cofiguration'] + labels: ['config'] + maintain-labels-not-matched: false + apply-changes: true diff --git a/.github/workflows/pr-label-file-based.yml b/.github/workflows/pr-label-file-based.yml new file mode 100644 index 0000000..d5d5a20 --- /dev/null +++ b/.github/workflows/pr-label-file-based.yml @@ -0,0 +1,13 @@ +name: "Pull Request Labeler File-based" +on: +- pull_request_target + +jobs: + labeler: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Assign labels from file changes + - uses: actions/labeler@v5 diff --git a/.github/workflows/pr-label-public.yml b/.github/workflows/pr-label-public.yml new file mode 100644 index 0000000..ed3649a --- /dev/null +++ b/.github/workflows/pr-label-public.yml @@ -0,0 +1,10 @@ +# Manage labels of pull requests that originate from forks +name: Label PRs from public forks + +on: + pull_request_target: + types: [opened, synchronize] + +jobs: + label: + uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2 diff --git a/.github/workflows/push-to-private.yml b/.github/workflows/push-to-private.yml new file mode 100644 index 0000000..3e5ef35 --- /dev/null +++ b/.github/workflows/push-to-private.yml @@ -0,0 +1,34 @@ +name: Push to private repository + +on: + push: + branches: + - main + +jobs: + push_changes: + if: ${{ !contains(github.repository, 'private') }} + runs-on: ubuntu-latest + + steps: + - name: Checkout source repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + fetch-tags: true + + - name: Set up Git configuration + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Setup SSH key + uses: webfactory/ssh-agent@v0.5.0 + with: + ssh-private-key: ${{ secrets.KEY_TO_PRIVATE }} + + - name: Push changes to private repository + run: | + git remote add private git@github.com:${{ github.repository }}-private.git + git push --set-upstream private main + diff --git a/.github/workflows/python-publish.yml b/.github/workflows/python-publish.yml index 9046e94..db64f44 100644 --- a/.github/workflows/python-publish.yml +++ b/.github/workflows/python-publish.yml @@ -1,27 +1,14 @@ +--- # This workflow will upload a Python Package using Twine when a release is created # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries -name: Upload Python Package +name: Upload Python Package to PyPI on: release: types: [created] jobs: - quality: - uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-precommit-run.yml@v2 - with: - skip-hooks: "no-commit-to-branch" - - checks: - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - uses: ecmwf-actions/reusable-workflows/.github/workflows/qa-pytest-pyproject.yml@v2 - with: - python-version: ${{ matrix.python-version }} - deploy: - needs: [checks, quality] uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2 secrets: inherit diff --git a/.github/workflows/python-pull-request.yml b/.github/workflows/python-pull-request.yml index c2be6a4..7210897 100644 --- a/.github/workflows/python-pull-request.yml +++ b/.github/workflows/python-pull-request.yml @@ -1,12 +1,13 @@ -# This workflow will upload a Python Package using Twine when a release is created -# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries - -name: Code Quality checks for PRs +name: Code Quality checks and Testing on: - push: pull_request: types: [opened, synchronize, reopened] + push: + branches: + - main + schedule: + - cron: "9 2 * * 0" # at 9:02 on sunday jobs: quality: diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..a4f606c --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,25 @@ +name: Run Release Please +on: + push: + branches: + - main + - hotfix + +permissions: + contents: write + pull-requests: write + +jobs: + release-please: + runs-on: ubuntu-latest + steps: + - uses: googleapis/release-please-action@v4 + with: + # this assumes that you have created a personal access token + # (PAT) and configured it as a GitHub action secret named + # `MY_RELEASE_PLEASE_TOKEN` (this secret name is not important). + token: ${{ secrets.RELEASE_PLEASE_TOKEN }} + # optional. customize path to .release-please-config.json + config-file: .release-please-config.json + # Currently releases are done from main + target-branch: ${{ github.ref_name }} diff --git a/.gitignore b/.gitignore index 68c44aa..1f2b9c7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,12 @@ -# Byte-compiled / optimized / DLL files +# Python __pycache__/ *.py[cod] *$py.class - -# C extensions *.so +.Python +__pypackages__/ # Distribution / packaging -.Python build/ develop-eggs/ dist/ @@ -26,17 +25,7 @@ share/python-wheels/ *.egg MANIFEST -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports +# Testing and coverage htmlcov/ .tox/ .nox/ @@ -51,77 +40,15 @@ coverage.xml .pytest_cache/ cover/ -# Translations +# Documentation +docs/_build/ +/site *.mo *.pot -# Django stuff: -*.log -local_settings.py -db.sqlite3 -db.sqlite3-journal - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -.pybuilder/ -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# IPython -profile_default/ -ipython_config.py - -# pyenv -# For a library or package, you might want to ignore these files since the code is -# intended to run in multiple environments; otherwise, check them in: -# .python-version - -# pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not -# install all needed dependencies. -#Pipfile.lock - -# poetry -# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. -# This is especially recommended for binary packages to ensure reproducibility, and is more -# commonly ignored for libraries. -# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control -#poetry.lock - -# pdm -# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. -#pdm.lock -# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it -# in version control. -# https://pdm.fming.dev/#use-with-ide -.pdm.toml - -# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm -__pypackages__/ - -# Celery stuff -celerybeat-schedule -celerybeat.pid - -# SageMath parsed files -*.sage.py - # Environments -.envrc .env +.envrc .venv env/ venv/ @@ -129,69 +56,80 @@ ENV/ env.bak/ venv.bak/ -# Spyder project settings +# IDEs +.idea/ .spyderproject .spyproject - -# Rope project settings .ropeproject +.vscode/ +*.code-workspace +*.sublime-project +*.sublime-workspace -# mkdocs documentation -/site +# Jupyter Notebook +.ipynb_checkpoints -# mypy +# Type checking .mypy_cache/ .dmypy.json dmypy.json - -# Pyre type checker .pyre/ - -# pytype static type analyzer .pytype/ -# Cython debug symbols -cython_debug/ - -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ - +# Data files *.grib *.onnx *.ckpt -*.swp *.npy +*.npz +*.zarr/ +*.nc +*.h5 +*.hdf5 +*.pkl +*.parquet +*.csv +*.xlsx +*.xls + +# ML artifacts +wandb/ +mlruns/ +lightning_logs/ +*.ckpt +*.pt +*.pth +runs/ +checkpoints/ + +# Temporary and system files +*.swp *.download +*.out +*.sync +*.dot +*.tmp +*.log +*.log.* +.DS_Store +~* +tmp/ +temp/ +logs/ +_dev/ +outputs/ +*tmp_data/ + +# Project specific ? ?.* foo bar -*.grib -*.nc -*.npz -*.json -*.zarr/ ~$images.pptx test.py +test.ipynb cutout.png -*.out - -_build/ -? -?.* -~* -*.sync _version.py -*.ckpt -*.tmp -/outputs/ -# Top level files only -/*.yaml -*.gif -*.zarr/ -/*-plots/ -/definitions*/ +*.to_upload +tempCodeRunnerFile.python +Untitled-*.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 8ee3f11..96a1f26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,7 +58,7 @@ repos: rev: v0.0.14 hooks: - id: rstfmt - exclude: '(cli|schemas)/.*' # Because we use argparse + exclude: 'cli/.*' # Because we use argparse - repo: https://github.com/tox-dev/pyproject-fmt rev: "v2.5.0" hooks: diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 26768c7..06c8ab4 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -9,7 +9,7 @@ sphinx: configuration: docs/conf.py python: - install: + install: - method: pip path: . extra_requirements: diff --git a/.release-please-config.json b/.release-please-config.json new file mode 100644 index 0000000..43f05bb --- /dev/null +++ b/.release-please-config.json @@ -0,0 +1,20 @@ +{ + "release-type": "python", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "separate-pull-requests": true, + "always-update": true, + "changelog-type": "github", + "include-component-in-tag": false, + "include-v-in-tag": false, + "draft": true, + "pull-request-title-pattern": "chore${scope}: Preparing Next Release for ${component} ${version}", + "pull-request-header": ":robot: Automated Release PR\n\nThis PR was created by `release-please` to prepare the next release. Once merged:\n\n1. A new version tag will be created\n2. A GitHub release will be published\n3. The changelog will be updated\n\nChanges to be included in the next release:", + "pull-request-footer": "> [!IMPORTANT]\n> :warning: Merging this PR will:\n> - Create a new release\n> - Trigger deployment pipelines\n> - Update package versions\n\n **Before merging:**\n - Ensure all tests pass\n - Review the changelog carefully\n - Get required approvals\n\n [Release-please documentation](https://github.com/googleapis/release-please)", + "packages": { + ".": { + "package-name": "anemoi-inference" + } + }, + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json" +} diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 48cb41e..861af19 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -1,6 +1,6 @@ ## How to Contribute -Please see the [read the docs](https://anemoi-training.readthedocs.io/en/latest/dev/contributing.html). +Please see the [read the docs](https://anemoi.readthedocs.io/en/latest/dev/contributing.html). ## Contributors @@ -10,4 +10,4 @@ Thank you to all the wonderful people who have contributed to Anemoi. Contributi ## Contributing Organisations -Significant contributions have been made by the following organisations: [DWD](https://www.dwd.de/), [MET Norway](https://www.met.no/), [MeteoSwiss](https://www.meteoswiss.admin.ch/), [RMI](https://www.meteo.be/) & [ECMWF](https://www.ecmwf.int/) +Significant contributions have been made by the following organisations: [DWD](https://www.dwd.de/), [FMI](https://www.ilmatieteenlaitos.fi/), [KNMI](https://www.knmi.nl), [MET Norway](https://www.met.no/), [MeteoSwiss](https://www.meteoswiss.admin.ch/), [RMI](https://www.meteo.be/) & [ECMWF](https://www.ecmwf.int/) diff --git a/README.md b/README.md index c2206cf..90ffb76 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,15 @@ -# anemoi-inference +# anemoi-utils **DISCLAIMER** This project is **BETA** and will be **Experimental** for the foreseeable future. Interfaces and functionality are likely to change, and the project itself may be scrapped. **DO NOT** use this software in any project/software that is operational. -Miscellanous tools for training data-driven weather forecasts. + ## Documentation -The documentation can be found at https://anemoi-inference.readthedocs.io/. +The documentation can be found at https://NaN.readthedocs.io/. ## Install @@ -22,7 +22,7 @@ $ pip install anemoi-inference ## License ``` -Copyright 2024, Anemoi contributors. +Copyright 2024-2025, Anemoi Contributors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License.