Docker compose 1.x.y (#53)

Co-authored-by: Rodney Garnett <[email protected]>

.circleci/config.yml

@@ -1,35 +1,113 @@
 version: 2.1
 jobs:
   build:
-    parameters:
-      python-version:
-        type: string
+    environment:
+      MIN_PYTHON_VERSION: "3.8"
+      MAX_PYTHON_VERSION: "3.13"
+      XDMOD_10_5_IMAGE: tools-ext-01.ccr.xdmod.org/xdmod-10.5.0-x86_64:rockylinux8.5-0.3
+      XDMOD_11_0_IMAGE: tools-ext-01.ccr.xdmod.org/xdmod:x86_64-rockylinux8.9.20231119-v11.0.0-1.0-03
     docker:
-      - image: cimg/python:<< parameters.python-version >>
+      - image: cimg/base:current
     steps:
       - checkout
+      - setup_remote_docker
       - run:
-          name: Install dependencies
+          name: Spin up containers and test different Python versions against different XDMoD web server versions
           command: |
-            python -m pip install --upgrade pip
-            python -m pip install flake8 flake8-commas flake8-quotes pytest
-            python -m pip install -e .
-      - run:
-          name: Lint with Flake8
-          command: python3 -m flake8 . --max-complexity=10 --show-source --exclude __init__.py
-      - run:
-          name: Test with pytest
-          command: python3 -m pytest tests/unit
-workflows:
-  full-build:
-    jobs:
-      - build:
-          matrix:
-            parameters:
-              python-version:
-                - "3.13"
-                - "3.12"
-                - "3.11"
-                - "3.10"
-                - "3.9"
-                - "3.8"
+            set -x
+            docker compose -f .circleci/docker-compose.yml up -d
+            declare -a python_containers=$(yq '.services | keys | .[] | select(. == "python-*")' .circleci/docker-compose.yml)
+            declare -a xdmod_containers=$(yq '.services | keys | .[] | select(. == "xdmod-*")' .circleci/docker-compose.yml)
+            # Copy the xdmod-data source code to the Python containers, lint
+            # with Flake 8, and install the package and its testing
+            # dependencies.
+            for python_container in $python_containers; do
+              docker cp . $python_container:/home/circleci/project
+              docker exec $python_container bash -c 'sudo chown -R circleci:circleci /home/circleci/project'
+              docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install --upgrade pip'
+              docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install --upgrade flake8 flake8-commas flake8-quotes'
+              docker exec -w /home/circleci/project $python_container bash -c 'python3 -m flake8 . --max-complexity=10 --show-source --exclude __init__.py'
+              docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install -e .'
+              docker exec -w /home/circleci/project $python_container bash -c 'python3 -m pip install --upgrade python-dotenv pytest coverage'
+            done
+            # Set up XDMoD web server containers.
+            for xdmod_container in $xdmod_containers; do
+              # Generate OpenSSL key and certificate.
+              docker exec $xdmod_container bash -c "openssl genrsa -rand /proc/cpuinfo:/proc/filesystems:/proc/interrupts:/proc/ioports:/proc/uptime 2048 > /etc/pki/tls/private/$xdmod_container.key"
+              docker exec $xdmod_container bash -c "openssl req -new -key /etc/pki/tls/private/$xdmod_container.key -x509 -sha256 -days 365 -set_serial $RANDOM -extensions v3_req -out /etc/pki/tls/certs/$xdmod_container.crt -subj '/C=XX/L=Default City/O=Default Company Ltd/CN=$xdmod_container' -addext 'subjectAltName=DNS:$xdmod_container'"
+              # Update the server hostnames and certificates so the Python
+              # containers can make requests to them.
+              docker exec $xdmod_container bash -c "sed -i \"s/localhost/$xdmod_container/g\" /etc/httpd/conf.d/xdmod.conf"
+              if [[ "$xdmod_container" =~ 'xdmod-*-dev' ]]; then
+                if [ "$xdmod_container" = 'xdmod-main-dev' ]; then
+                  branch='main'
+                else
+                  branch="xdmod$(echo $xdmod_container | sed 's/xdmod-\(.*\)-dev/\1/' | sed 's/-/./')"
+                fi
+                # Install and run the latest development version of the XDMoD
+                # web server.
+                docker exec $xdmod_container bash -c 'git clone --depth=1 --branch=$branch https://github.com/ubccr/xdmod.git /root/xdmod'
+                docker exec -w /root/xdmod $xdmod_container bash -c 'composer install'
+                docker exec -w /root/xdmod $xdmod_container bash -c '/root/bin/buildrpm xdmod'
+                docker exec -w /root/xdmod $xdmod_container bash -c 'XDMOD_TEST_MODE=upgrade ./tests/ci/bootstrap.sh'
+                docker exec -w /root/xdmod $xdmod_container bash -c './tests/ci/validate.sh'
+              elif [[ "$xdmod_container" =~ xdmod-* ]]; then
+                # Run the XDMoD web server.
+                docker exec $xdmod_container bash -c '/root/bin/services start'
+              fi
+              # Copy the 10,000 users file into the container and shred it.
+              # We use this file so we can test filters with more than 10,000
+              # values and date ranges that span multiple quarters.
+              docker cp tests/ci/artifacts/10000users.log $xdmod_container:.
+              docker exec $xdmod_container xdmod-shredder -r frearson -f slurm -i 10000users.log
+              # Ingest and aggregate.
+              date=$(date --utc +%Y-%m-%d)
+              docker exec $xdmod_container xdmod-ingestor --ingest
+              docker exec $xdmod_container xdmod-ingestor --aggregate=job --last-modified-start-date $date
+              # Copy certificate (for doing requests) from the XDMoD container.
+              docker cp $xdmod_container:/etc/pki/tls/certs/$xdmod_container.crt .
+              # Copy certificate to one of the Python containers and get an
+              # XDMoD API token for the XDMoD container.
+              docker cp $xdmod_container.crt $python_container:/home/circleci/project
+              rest_token=$(docker exec \
+                -e CURL_CA_BUNDLE="/home/circleci/project/$xdmod_container.crt" \
+                $python_container \
+                bash -c "curl \
+                  -sS \
+                  -X POST \
+                  -c xdmod.cookie \
+                  -d 'username=normaluser&password=normaluser' \
+                  https://$xdmod_container/rest/auth/login \
+                  | jq -r '.results.token'"
+              )
+              docker exec $python_container bash -c 'echo -n "XDMOD_API_TOKEN="' > ${xdmod_container}-token
+              docker exec \
+                -e CURL_CA_BUNDLE="/home/circleci/project/$xdmod_container.crt" \
+                $python_container \
+                bash -c "curl \
+                  -sS \
+                  -X POST \
+                  -b xdmod.cookie \
+                  https://$xdmod_container/rest/users/current/api/token?token=$rest_token \
+                  | jq -r '.data.token'" \
+                  >> ${xdmod_container}-token
+            done
+            # Run the tests against each XDMoD web server.
+            for python_container in $python_containers; do
+              for xdmod_container in $xdmod_containers; do
+                # Copy certificate (for doing requests) to the Python
+                # container.
+                docker cp $xdmod_container.crt $python_container:/home/circleci/project
+                # Copy XDMoD API token to the Python container.
+                docker cp ${xdmod_container}-token $python_container:/home/circleci/.xdmod-data-token
+                # Run tests in the Python container.
+                docker exec \
+                  -e CURL_CA_BUNDLE="/home/circleci/project/$xdmod_container.crt" \
+                  -e XDMOD_HOST="https://$xdmod_container" \
+                  -e XDMOD_VERSION="$xdmod_container" \
+                  $python_container \
+                  bash -c 'python3 -m coverage run --branch --append -m pytest -vvs -o log_cli=true tests/'
+              done
+              # Make sure 100% test coverage.
+              docker exec $python_container bash -c 'python3 -m coverage report -m --fail-under=100'
+            done

.circleci/docker-compose.yml

@@ -0,0 +1,27 @@
+services:
+  # Test the minimum and maximum supported Python versions:
+  python-min:
+    container_name: python-min
+    image: cimg/python:${MIN_PYTHON_VERSION}
+    tty: true
+  python-max:
+    container_name: python-max
+    image: cimg/python:${MAX_PYTHON_VERSION}
+    tty: true
+  # Test the supported and development versions of the XDMoD web server:
+  xdmod-main-dev:
+    container_name: xdmod-main-dev
+    image: ${XDMOD_11_0_IMAGE}
+    tty: true
+  xdmod-11-0-dev:
+    container_name: xdmod-11-0-dev
+    image: ${XDMOD_11_0_IMAGE}
+    tty: true
+  xdmod-11-0:
+    container_name: xdmod-11-0
+    image: ${XDMOD_11_0_IMAGE}
+    tty: true
+  xdmod-10-5:
+    container_name: xdmod-10-5
+    image: ${XDMOD_10_5_IMAGE}
+    tty: true

.github/PULL_REQUEST_TEMPLATE.md

@@ -30,6 +30,5 @@
 - [ ] `xdmod_data/__version__.py` has been updated to the next development version
 - [ ] The milestone is set correctly on the pull request
 - [ ] The appropriate labels have been added to the pull request
-- [ ] Running the automated tests (see `docs/developing.md`) produces no errors
 - [ ] Updates have been made to the `xdmod-notebooks` repository as necessary, and the notebooks all run successfully
 - [ ] The changes in this PR have been ported/backported to other branches as needed

.gitignore

@@ -1,132 +1,7 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
 build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-pip-wheel-metadata/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
 .coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
+dist/
 docs/_build/
-
-# PyBuilder
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-.python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# Vi swap files
-.*.swp
+*.egg-info/
+__pycache__/
+.*.sw?

CHANGELOG.md

@@ -6,6 +6,7 @@
 - Implement 100% test coverage ([\#45](https://github.com/ubccr/xdmod-data/pull/45)).
 - Update Flake8 rules ([\#46](https://github.com/ubccr/xdmod-data/pull/46)).
 - Change to using CircleCI instead of GitHub Actions for continuous integration ([\#50](https://github.com/ubccr/xdmod-data/pull/50)).
+- Set up Docker Compose for CI testing ([\#53](https://github.com/ubccr/xdmod-data/pull/53)).
 
 ## v1.0.2 (2024-10-31)
 

docs/developing.md

@@ -1,64 +1,5 @@
 # Instructions for developers
 
-## Testing the code
-1. Start up a virtual environment, e.g.:
-    ```
-    python3 -m venv ~/xdmod-data-test-env
-    source ~/xdmod-data-test-env/bin/activate
-    ```
-    Your command prompt should now start with `(xdmod-data-test-env)`.
-1. Install your local development copy of `xdmod-data` in editable mode:
-    ```
-    python3 -m pip install --force-reinstall -e /path/to/your/xdmod-data
-    ```
-1. Install `python-dotenv`, `pytest`, and `coverage`:
-    ```
-    python3 -m pip install --upgrade python-dotenv pytest coverage
-    ```
-1. Create an empty file in your home directory at `~/.xdmod-data-token` with permissions set to 600.
-1. With an [ACCESS XDMoD](https://xdmod.access-ci.org) account with "User" as the Top Role, create an API token if you do not already have one (sign in and click My Profile -> API Token).
-1. Add the following line to the file `~/.xdmod-data-token`, replacing `<token>` with your token.
-    ```
-    XDMOD_API_TOKEN=<token>
-    ```
-1. Change directories to your local development copy of `xdmod-data`.
-1. Run the following command and make sure all the tests pass:
-    ```
-    coverage run -m pytest -vvs -o log_cli=true tests/
-    ```
-1. Run the following command and make sure the code is 100% covered by tests:
-    ```
-    coverage report -m
-    ```
-1. Downgrade to the minimum version of the dependencies. Replace the version numbers below with the values from `setup.cfg`.
-    ```
-    python3 -m pip install --force-reinstall numpy==1.23.0 pandas==1.5.0 plotly==5.8.0 requests==2.19.0
-    ```
-1. Run the following command again and make sure all the tests pass (Deprecation warnings in `urllib3` are OK).
-    ```
-    coverage run -m pytest -vvs -o log_cli=true tests/
-    ```
-1. Run the following command and make sure the code is 100% covered by tests:
-    ```
-    coverage report -m
-    ```
-
-## Linting the code
-1. Start up a virtual environment, e.g.:
-    ```
-    python3 -m venv ~/xdmod-data-test-env
-    source ~/xdmod-data-test-env/bin/activate
-    ```
-    Your command prompt should now start with `(xdmod-data-test-env)`.
-1. Install Flake8 and additional packages
-    ```
-    python3 -m pip install flake8 flake8-quotes flake8-commas
-    ```
-1. Run Flake8
-    ```
-    flake8 --max-complexity=10 --show-source --exclude __init__.py /path/to/your/xdmod-data
-    ```
-
 ## Releasing a new version
 1. Make a new branch of `xdmod-data` and:
     1. Make sure the version number is updated in `xdmod_data/__version__.py`.