diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..a948ff311 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +{ + "name": "DeveloperEnv", + "image": "ghcr.io/jageo/autoplex/autoplex:python-3.10", + "hostRequirements": { + "cpus": 4, + "memory": "16gb", + "storage": "32gb" + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-python.python", + "ms-azuretools.vscode-docker"] + }, + }, + "postCreateCommand": "pip cache purge && uv pip install -e .[strict,docs] && uv pip install ase==3.23.0 && pre-commit install", +} diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..edd292292 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,48 @@ +name: Build and Push Docker Image + +on: + workflow_dispatch: + release: + types: [ created ] # Runs only when a new release is created + +jobs: + build: + runs-on: ubuntu-latest + if: github.repository_owner == 'JaGeo' && github.ref == 'refs/heads/main' + permissions: + contents: read + packages: write + attestations: write + id-token: write + strategy: + matrix: + python-version: [ "3.10", "3.11", "3.12" ] + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: lowercase github.repository + run: | + echo "IMAGE_NAME=`echo ${{github.repository}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV} + + # Build Docker image with a custom Python version + - name: Build Docker image (Python ${{ matrix.python-version }}) + run: | + docker build \ + --build-arg PYTHON_VERSION=${{ matrix.python-version }} \ + -t ghcr.io/${{ env.IMAGE_NAME }}/autoplex:python-${{ matrix.python-version }} . + # Push Docker image to GHCR + - name: Push Docker image to GHCR (Python ${{ matrix.python-version }}) + run: | + docker push ghcr.io/${{ env.IMAGE_NAME }}/autoplex:python-${{ matrix.python-version }} diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8b589a683..78f908f14 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -4,8 +4,9 @@ name: Testing Linux on: + workflow_dispatch: push: - branches: '*' + branches: ['*'] pull_request: branches: [ main ] @@ -16,55 +17,35 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11"] - split: [1] #, 2, 3, 4] # Number of splits + python-version: ["3.10", "3.11"] + split: [1, 2, 3, 4] # Number of splits steps: - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip cache purge - python -m pip install --upgrade pip - python -m pip install flake8 pytest pytest-mock pytest-split pytest-cov - python -m pip install types-setuptools - python -m pip cache purge - python -m pip install .[strict] - python -m pip cache purge - python -m pip install ase==3.23.0 - - name: Print installed package versions - run: | - pip show torch torchdata - - name: Install Buildcell - run: | - curl -O https://www.mtg.msm.cam.ac.uk/files/airss-0.9.3.tgz; tar -xf airss-0.9.3.tgz; rm airss-0.9.3.tgz; cd airss; make ; make install ; make neat; cd .. - - name: Add Buildcell to PATH - run: echo "$GITHUB_WORKSPACE/airss/bin" >> $GITHUB_PATH - - name: Install Julia - run: curl -fsSL https://install.julialang.org | sh -s -- -y - - name: Set up Julia environment (Needed for ACEpotentials.jl interface) - run: | - julia -e 'using Pkg; Pkg.Registry.add("General"); Pkg.Registry.add(Pkg.Registry.RegistrySpec(url="https://github.com/ACEsuit/ACEregistry")); Pkg.add("ACEpotentials"); Pkg.add("DataFrames"); Pkg.add("CSV")' - - name: Linting with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest and coverage + - name: Run tests using Docker image for Python ${{ matrix.python-version }} run: | - # run the line below locally to update tests durations file - # pytest --cov=autoplex --cov-append --splits 1 --group 1 --durations-path ./tests/test_data/.pytest-split-durations --store-durations - pytest --cov=autoplex --cov-report term-missing --cov-append --splits 1 --group ${{ matrix.split }} -vv --durations-path ./tests/test_data/.pytest-split-durations + docker pull ghcr.io/jageo/autoplex/autoplex:python-${{ matrix.python-version }} + docker run --rm \ + -v ${{ github.workspace }}:/workspace \ + -w /workspace \ + ghcr.io/jageo/autoplex/autoplex:python-${{ matrix.python-version }} \ + bash -c " + python -m pip install --upgrade pip && \ + python -m uv pip install pytest pytest-mock pytest-split pytest-cov types-setuptools && \ + python -m uv pip install .[strict] && \ + python -m uv pip install ase==3.23.0 && \ + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics && \ + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics && \ + pytest --cov=autoplex --cov-report term-missing --cov-append --splits 4 --group ${{ matrix.split }} -vv --durations-path /workspace/tests/test_data/.pytest-split-durations + " - name: Upload coverage uses: actions/upload-artifact@v3 with: - name: coverage-${{ matrix.split }} - path: .coverage - + name: coverage-${{ matrix.python-version }}-${{ matrix.split }} + include-hidden-files: true + overwrite: false + path: ./.coverage + coverage: needs: build runs-on: ubuntu-latest @@ -88,20 +69,24 @@ jobs: docs: runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} # enables conda/mamba env activation by reading bash profile steps: - - uses: actions/checkout@v3 - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - cache: pip - cache-dependency-path: pyproject.toml - - - name: Install dependencies + - name: Check out repo + uses: actions/checkout@v4 + - name: Set up micromamba + uses: mamba-org/setup-micromamba@main + - name: Create mamba environment run: | - python -m pip install --upgrade pip - pip install .[docs,strict] - + micromamba create -n autoplex_docs python=3.10 --yes + - name: Install uv + run: micromamba run -n autoplex_docs pip install uv + - name: Install autoplex and dependencies + run: | + micromamba activate autoplex_docs + uv pip install --upgrade pip + uv pip install .[docs,strict] - name: Build - run: sphinx-build -W docs _build + run: micromamba run -n autoplex_docs sphinx-build -W docs _build diff --git a/.gitignore b/.gitignore index 87e0bfc35..94daf6a92 100644 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,9 @@ target/ # Pycharm .idea/* +# VSCode +.vscode/ + # Jupyter Notebook .ipynb_checkpoints diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..3e92e9d7f --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,27 @@ +# Guidelines for contributions to autoplex +- Please write unit tests; this is a requirement for any added code to be accepted. (Automated testing will be performed using `pytest`; you can look into the `tests` folder for examples). +- Please ensure high coverage of the code based on the tests (you can test this with `coverage`). +- Please use numpy docstrings (use an IDE and switch on this docstring type; you can check examples in our code base; the docstring should be useful for other people) +- Please ensure that type hints are added for each variable, function, class, and method (this helps code readability, especially if someone else wants to build on your code). +- Please write the code in a way that gives users the option to change parameters (this is mainly applicable, for example, fitting protocols/flows). In other words, please avoid hardcoding settings or physical properties. +Reasonable default values should be set, but the user needs to have the opportunity to modify them if they wish. + +## General code structure +- We are currently aiming to follow the code structure below for each submodule (This is an initial idea; of course, this could change depending on the needs in the future) + - autoplex/submodule/job.py (any jobs defined will be inside this module) + - autoplex/submodule/flows.py (workflows defined will be hosted in this module) + - autoplex/submodule/utils.py (all functions that act as utilities for defining flow or job, for example, a small subtask to calculate some metric or plotting, will be hosted in this module) + +## Formatting requirements +- Variable names should be descriptive and should use snake case (`variable_name`, not `VariableName`). +- If you define a `Maker`, please use python class naming convention (e.g., `PhononMaker`, `RssMaker`). + +## Commit guidelines +1. `pip install pre-commit`. +2. Next, run `pre-commit install` (this will install all the hooks from pre-commit-config.yaml) +3. Step 1 and 2 needs to be done only once in the local repository +4. Proceed with modifying the code and adding commits as usual. This should automatically run the linters. +5. To manually run the pre-commit hooks on all files, just use `pre-commit run --all-files` +6. To run pre-commit on a specific file, use `pre-commit run --files path/to/your/modified/module/` + +Please check out atomate2 for example code (https://github.com/materialsproject/atomate2) diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..8e5856155 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +# Use an official micromamba image as the base image +ARG PYTHON_VERSION=3.10 + +FROM mambaorg/micromamba:1.5.10 + + +# Set environment variables for micromamba +ENV MAMBA_DOCKERFILE_ACTIVATE=1 +ENV MAMBA_ROOT_PREFIX=/opt/conda + +# Switch to root to install all dependencies (using non-root user causes permission issues) +USER root + +# Make arg accessible to the rest of the Dockerfile +ARG PYTHON_VERSION + +# Install system dependencies +RUN apt-get update && apt-get install -y \ + build-essential \ + bash \ + bc \ + unzip \ + wget \ + gfortran \ + liblapack-dev \ + libblas-dev \ + cmake \ + make \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* + +# Install Python +RUN micromamba install -y -n base -c conda-forge \ python=${PYTHON_VERSION} && \ + micromamba clean --all --yes + +# Install testing dependencies +RUN python -m pip install --upgrade pip \ + && pip install uv \ + && uv pip install flake8 pre-commit pytest pytest-mock pytest-split pytest-cov types-setuptools + +# Install Julia +RUN curl -fsSL https://julialang-s3.julialang.org/bin/linux/x64/1.9/julia-1.9.2-linux-x86_64.tar.gz | tar -xz -C /opt \ + && ln -s /opt/julia-1.9.2/bin/julia /usr/local/bin/julia + + +# Set up Julia environment (ACEpotentials.jl interface) +RUN julia -e 'using Pkg; Pkg.Registry.add("General"); Pkg.Registry.add(Pkg.Registry.RegistrySpec(url="https://github.com/ACEsuit/ACEregistry")); Pkg.add("ACEpotentials"); Pkg.add("DataFrames"); Pkg.add("CSV")' + +# Install Buildcell (airss) +RUN curl -fsSL https://www.mtg.msm.cam.ac.uk/files/airss-0.9.3.tgz -o /opt/airss-0.9.3.tgz \ + && tar -xf /opt/airss-0.9.3.tgz -C /opt \ + && rm /opt/airss-0.9.3.tgz \ + && cd /opt/airss \ + && make \ + && make install \ + && make neat + +# Add Buildcell to PATH +ENV PATH="${PATH}:/opt/airss/bin" diff --git a/docs/dev/contributing.md b/docs/dev/contributing.md index 3d0896f70..e75758cf1 100644 --- a/docs/dev/contributing.md +++ b/docs/dev/contributing.md @@ -1,27 +1,2 @@ -# Guidelines for contributions -- Please write unit tests; this is a requirement for any added code to be accepted. (Automated testing will be performed using `pytest`; you can look into the `tests` folder for examples). -- Please ensure high coverage of the code based on the tests (you can test this with `coverage`). -- Please use numpy docstrings (use an IDE and switch on this docstring type; you can check examples in our code base; the docstring should be useful for other people) -- Please ensure that type hints are added for each variable, function, class, and method (this helps code readability, especially if someone else wants to build on your code). -- Please write the code in a way that gives users the option to change parameters (this is mainly applicable, for example, fitting protocols/flows). In other words, please avoid hardcoding settings or physical properties. -Reasonable default values should be set, but the user needs to have the opportunity to modify them if they wish. - -## General code structure -- We are currently aiming to follow the code structure below for each submodule (This is an initial idea; of course, this could change depending on the needs in the future) - - autoplex/submodule/job.py (any jobs defined will be inside this module) - - autoplex/submodule/flows.py (workflows defined will be hosted in this module) - - autoplex/submodule/utils.py (all functions that act as utilities for defining flow or job, for example, a small subtask to calculate some metric or plotting, will be hosted in this module) - -## Formatting requirements -- Variable names should be descriptive and should use snake case (`variable_name`, not `VariableName`). -- If you define a `Maker`, please use python class naming convention (e.g., `PhononMaker`, `RssMaker`). - -## Commit guidelines -1. `pip install pre-commit`. -2. Next, run `pre-commit install` (this will install all the hooks from pre-commit-config.yaml) -3. Step 1 and 2 needs to be done only once in the local repository -4. Proceed with modifying the code and adding commits as usual. This should automatically run the linters. -5. To manually run the pre-commit hooks on all files, just use `pre-commit run --all-files` -6. To run pre-commit on a specific file, use `pre-commit run --files path/to/your/modified/module/` - -Please check out atomate2 for example code (https://github.com/materialsproject/atomate2) \ No newline at end of file +```{include} ../../CONTRIBUTING.md +``` \ No newline at end of file diff --git a/docs/dev/dev_install.md b/docs/dev/dev_install.md index 1d3cd7a4b..cd9e71efc 100644 --- a/docs/dev/dev_install.md +++ b/docs/dev/dev_install.md @@ -7,7 +7,13 @@ git clone https://github.com/JaGeo/autoplex.git cd autoplex pip install -e .[docs,strict,dev] ``` -This will install autoplex will all dependencies for tests, pre-commit and docs building. +This will install autoplex will all dependencies for tests, pre-commit and docs building. +However, note that non-python programs like `buildcell` and `julia` needed for ACE potential fitting will not be installed with above command. One needs to install these +seperately. + +Alternatively, one can use the `devcontainer` provided to have your developer environment setup automatically in your IDE. It has been tested to work in [VSCode](https://code.visualstudio.com/docs/devcontainers/containers#_quick-start-open-an-existing-folder-in-a-container) and [PyCharm](https://blog.jetbrains.com/pycharm/2023/06/2023-2-eap-4/). +Only prerequisite is one has [docker](https://docs.docker.com/get-started/get-docker/) installed on the system as it uses the published docker images to create this developer env. + ## Running unit tests diff --git a/pyproject.toml b/pyproject.toml index e6f59958c..98dff972e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,11 +25,9 @@ classifiers = [ requires-python = ">=3.9" dependencies = [ "pymatgen>=2024.9.17.1", - "atomate2[strict]@git+https://github.com/QuantumChemist/atomate2.git@adjust_get_supercell_size", - #"FireWorks>=2.0.3", + "atomate2[strict,strict-forcefields]@git+https://github.com/QuantumChemist/atomate2.git@adjust_get_supercell_size", "ase==3.23.0", "matgl==1.1.3", - #"quippy-ase==0.9.14", "numpy==1.26.4", "mace-torch==0.3.4", "lightning-utilities==0.11.2", @@ -65,19 +63,12 @@ docs = [ strict = [ "pymatgen>=2024.8.9", - "atomate2[strict]@git+https://github.com/QuantumChemist/atomate2.git@adjust_get_supercell_size", - #"FireWorks==2.0.3", - # last known working ASE version: https://gitlab.com/ase/ase@2bab58f4e - #"ase@git+https://gitlab.com/ase/ase.git@2bab58f4e", - #"quippy-ase==0.9.14", + "atomate2[strict,strict-forcefields]@git+https://github.com/QuantumChemist/atomate2.git@adjust_get_supercell_size", "ase==3.23.0", "mace-torch==0.3.4", "lightning-utilities==0.11.2", "numpy==1.26.4", "typing", - #"jobflow==0.1.13", - #"pydantic==1.10.9", - #"pymatgen-analysis-defects==2023.8.22", "dgl==2.1.0", "torchdata==0.7.1", ] @@ -178,7 +169,12 @@ exclude_lines = [ "show_plot", ] -# omit everything in test directory from coverage +[tool.coverage.paths] +source = [ + "autoplex/", + "/workspace/autoplex/", +] + [tool.coverage.run] include = ["autoplex/*"]