From 783d4f5b62e6c5e06cdfe4c3eb71d5946f265a3a Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Wed, 31 Jul 2024 10:22:15 -0400 Subject: [PATCH] Initial commit --- .github/workflows/slides.yml | 38 ++++ .gitignore | 166 ++++++++++++++++ LICENSE | 21 ++ README.md | 5 + pyproject.toml | 23 +++ slides.md | 368 +++++++++++++++++++++++++++++++++++ src/cicdexample/__init__.py | 7 + tests/test_basic.py | 30 +++ 8 files changed, 658 insertions(+) create mode 100644 .github/workflows/slides.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100644 pyproject.toml create mode 100644 slides.md create mode 100644 src/cicdexample/__init__.py create mode 100644 tests/test_basic.py diff --git a/.github/workflows/slides.yml b/.github/workflows/slides.yml new file mode 100644 index 0000000..e6f0b20 --- /dev/null +++ b/.github/workflows/slides.yml @@ -0,0 +1,38 @@ +name: Deploy Slides + +on: push + +permissions: + contents: read + pages: write + id-token: write + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Check out repo + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" + - name: Install markdown slides + run: pip install git+https://gitlab.com/da_doomer/markdown-slides.git + - name: Generate slides + run: mdslides slides.md --output_dir slides + - name: Upload pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./slides + + deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1c4b06d --- /dev/null +++ b/.gitignore @@ -0,0 +1,166 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# Generated slides +slides + +# macOS stuff +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..2e792a6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Andres Rios Tascon + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..4d2a056 --- /dev/null +++ b/README.md @@ -0,0 +1,5 @@ +# Continuous Integration and Continuous Delivery (CI/CD) with GitHub Workflows + +This repository contains the material used for the CI/CD workshop at the Princeton's Software Engineering Summer School 2024. The material is based on the CI/CD portion of the [Software Engineering for Scientific Computing Course](https://henryiii.github.io/se-for-sci/content/intro.html) and the [CI/CD Course](https://hsf-training.github.io/hsf-training-cicd-github/) by the [HEP Software Foundation](https://hepsoftwarefoundation.org/). + +The live slides can be found [here](https://ariostas-talks.github.io/2024-07-31-cicd). \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a607774 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,23 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "cicdexample" +version = "0.0.1" +description = "Example Python package" +readme = {file = "README.md", content-type = "text/markdown"} +license = {file = "LICENSE"} +requires-python = ">=3.8" +dependencies = [ + "numpy", +] +[project.optional-dependencies] +test = ["pytest"] + + +[tool.pytest.ini_options] +addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] +xfail_strict = true +filterwarnings = ["error"] +testpaths = ["tests"] diff --git a/slides.md b/slides.md new file mode 100644 index 0000000..a57b3b1 --- /dev/null +++ b/slides.md @@ -0,0 +1,368 @@ +[comment]: # (This presentation was made with markdown-slides) +[comment]: # (Can be found here: https://gitlab.com/da_doomer/markdown-slides) +[comment]: # (Compile this presentation with the command below) +[comment]: # (mdslides slides.md) + +[comment]: # (Set the theme:) +[comment]: # (THEME = white) +[comment]: # (CODE_THEME = github) + +[comment]: # (controls: true) +[comment]: # (keyboard: true) +[comment]: # (markdown: { smartypants: true }) +[comment]: # (hash: false) +[comment]: # (respondToHashChanges: false) + +## Continuous Integration and Continuous Delivery (CI/CD) with GitHub Workflows + +
+ +### Andres Rios Tascon + +Software Engineering Summer School 2024 + +[comment]: # (!!!) + +## What is CI and CD + +- Continuous Integration:
+ Frequent merging of changes into main branch + +- Continuous Deployment:
+ Automatic release of new software versions + +[comment]: # (!!!) + +## What is CI for? + +- Running tests (dynamic and static) + - Check if new changes break any existing functionality + - Check if code follows formatting guidelines + + +[comment]: # (||| data-auto-animate) + +## What is CI for? + +- Building documentation
+ - Run Doxygen/Sphinx/etc generate documentation in pdf or html form + - Check if examples in the documentation work + +[comment]: # (||| data-auto-animate) + +## What is CI for? + +- Build static websites + - Build and publish a website to GitHub Pages/ReadTheDocs/etc + +[comment]: # (||| data-auto-animate) + +## What is CI for? + +- Generating pull requests for updates and other maintenance + - Update dependencies + - Fix typos and coding style + +[comment]: # (||| data-auto-animate) + +## What is CI for? + +- Whatever your project needs! + +[comment]: # (!!! data-auto-animate) + +## What are the benefits? + +- Consistent, controlled environment between runs +- Runs every PR/commit/tag/whatever you choose +- Can't be skipped or forgotten, no contributor setup +- Can run lots of OS's, Python versions, compilers, etc + +[comment]: # (!!!) + +## Some major CI services + +- **Travis CI**: Very popular for years, but not anymore. +- **Jenkins**: A self-host only OSS solution. +- **Appveyor**: The original Windows CI service. +- **Circle CI**: The first more “modern” design. +- **GitLab CI**: For years, this was one of the best services. Still very good. +- **Azure Pipelines**: Very modular design is easy to upgrade and maintain. +- **GitHub Actions (GHA)**: Extremely simple and popular. Actions are easy to write and share. + +[comment]: # (!!!) + +### Today we will be focusing on GitHub Actions + +
+ +We first need to briefly discuss exit codes and YAML. + +[comment]: # (!!!) + +### Exit codes + +- Every time you run a command in a shell there is an exit code that indicates if it ran successfully, or if there was an error. +- An exit code of `0` indicates that the command ran successfully, other numbers indicate an error. +- Sometimes different numbers correspond to different errors. + +[comment]: # (||| data-auto-animate) + +### Exit codes + +```bash [1-3|4-7|8-12] +> mkdir test +> echo $? +0 +> mkdir test +mkdir: test: File exists +> echo $? +1 +> mkdir -z test +mkdir: illegal option -- z +usage: mkdir [-pv] [-m mode] directory_name ... +> echo $? +64 +``` + +[comment]: # (||| data-auto-animate) + +### Exit codes + +- CI workflows will typically stop once they encounter a non-zero exit code. +- Sometimes you may need to run a command that might fail, but you want the workflow to proceed. +- Some scripts and binaries don't respect this standard and return non-zero exit codes even when successful. + +[comment]: # (||| data-auto-animate) + +### Exit codes + +Error codes can be ignored using logical or (||) +```bash +> mkdir -z test || echo "ignore" +ignore +``` + +It is also useful to use logical and (&&) to run a command only if another one is successful. +``` +> && +``` + +[comment]: # (!!! data-auto-animate) + +### YAML + +YAML (YAML Ain’t Markup Language, originally Yet Another Markup Language) is a human-readable data serialization language. + +- Easy to read and use +- Very commonly used in CI configuration files. +- File extension is .yml or .yaml + +[comment]: # (||| data-auto-animate) + +### YAML + +Defining scalar values: + +```yaml +number-value: 42 +boolean-value: true # can also be on or yes +string-value: "Hello world" +another-string: String without quotes +``` + +[comment]: # (||| data-auto-animate) + +### YAML + +Defining lists: + +```yaml +colors: + - red + - green + - blue + +more_colors: [black, white] +``` + +[comment]: # (||| data-auto-animate) + +### YAML + +Defining dictionaries: + +```yaml +person: + name: John Smith + age: 33 + occupation: accountant + +same_person: {name: John Smith, age: 33, occupation: accountant} +``` + +[comment]: # (||| data-auto-animate) + +### YAML + +Defining multi-line strings: + +```yaml +some-text: > + Multiple + lines + of + text +same-text: "Multiple lines of text\n" +``` + +[comment]: # (||| data-auto-animate) + +### YAML + +Defining multi-line strings: + +```yaml +some-text: | + Multiple + lines + of + text +same-text: "Multiple\nlines\nof\ntext\n" +``` + +[comment]: # (!!! data-auto-animate) + +### Setting up GitHub workflows + +- Each workflow is configured by a yaml file placed in `.github/workflows` +- Can be set to trigger by a wide variety of events +- Can run your own commands or use actions written by you or third-parties + +[comment]: # (||| data-auto-animate) + +### Setting up GitHub workflows + +This is the basic structure of a workflow file. + +```yaml +on: + +jobs: + job_1: + name: + runs-on: + steps: + - run: + - name: + run: + + job_2: + name: + runs-on: + steps: + - uses: +``` + +[comment]: # (!!! data-auto-animate) + +### CI/CD hands-on workshop + +Go to the following link: + +https://github.com/ariostas-talks/2024-07-31-cicd + +[comment]: # (!!! data-auto-animate) + +### Exercise + +Create a new workflow that generates these slides and publishes them to GitHub pages. Things you need: + +- `actions/checkout` +- `actions/setup-python` +- `markdown-slides` from https://gitlab.com/da_doomer/markdown-slides +- `actions/upload-pages-artifact` +- `actions/deploy-pages` +- Repo > Settings > Pages > Source > GitHub Actions + +[comment]: # (||| data-auto-animate) + +### Exercise + +Start by checking out the repo. + +```yaml +- name: Check out repo + uses: actions/checkout@v4 +``` + +[comment]: # (||| data-auto-animate) + +### Exercise + +Then set up Python. + +```yaml +- name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.11" +``` + +[comment]: # (||| data-auto-animate) + +### Exercise + +Install `markdown-slides`. + +```yaml +- name: Install markdown slides + run: pip install git+https://gitlab.com/da_doomer/markdown-slides.git +``` + +[comment]: # (||| data-auto-animate) + +### Exercise + +Generate the slides. + +```yaml +- name: Generate slides + run: mdslides slides.md --output_dir slides +``` + +[comment]: # (||| data-auto-animate) + +### Exercise + +Upload pages artifact + +```yaml +- name: Upload pages artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ./slides +``` + +[comment]: # (||| data-auto-animate) + +### Exercise + +Deploy to GitHub Pages + +```yaml +deploy: + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + runs-on: ubuntu-latest + needs: build + steps: + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v3 +``` + +[comment]: # (!!! data-auto-animate) + +There's many more things to learn, but with this basic knowledge there is lots of things you can do! diff --git a/src/cicdexample/__init__.py b/src/cicdexample/__init__.py new file mode 100644 index 0000000..ca1ea1e --- /dev/null +++ b/src/cicdexample/__init__.py @@ -0,0 +1,7 @@ +import numpy as np + +def square_plus_one(x): + return x**2 + 1 + +def exp_minus_one(x): + return np.expm1(x) \ No newline at end of file diff --git a/tests/test_basic.py b/tests/test_basic.py new file mode 100644 index 0000000..2343859 --- /dev/null +++ b/tests/test_basic.py @@ -0,0 +1,30 @@ +from pytest import approx +import numpy as np +from cicdexample import square_plus_one, exp_minus_one + +def test_square_plus_one(): + assert square_plus_one(0) == 1 + assert square_plus_one(1) == 2 + assert square_plus_one(2) == 5 + + l = np.arange(100) + l2p1 = square_plus_one(l) + assert all(c == (i**2)+1 for i,c in enumerate(l2p1)) + + l = np.arange(10, step=0.2) + l2p1 = square_plus_one(l) + assert all(c == approx(((i*0.2)**2)+1) for i,c in enumerate(l2p1)) + + +def test_exp_minus_one(): + assert exp_minus_one(0) == approx(0) + assert exp_minus_one(1) == approx(np.e - 1) + assert exp_minus_one(2) == approx(np.e**2 - 1) + + l = np.arange(10) + elm1 = exp_minus_one(l) + assert all(c == approx(np.exp(i)-1) for i,c in enumerate(elm1)) + + l = np.arange(10, step=0.2) + elm1 = exp_minus_one(l) + assert all(c == approx(np.exp(i*0.2)-1) for i,c in enumerate(elm1))