From 2ff23e9387d4e4275fa22efed31a7b51479cb3ca Mon Sep 17 00:00:00 2001 From: bot Date: Mon, 2 Sep 2024 06:51:25 +0000 Subject: [PATCH] [automation] transform lesson to sandpaper --- .editorconfig | 25 ++ .github/workflows/README.md | 198 ++++++++++ .github/workflows/pr-close-signal.yaml | 23 ++ .github/workflows/pr-comment.yaml | 185 ++++++++++ .github/workflows/pr-post-remove-branch.yaml | 32 ++ .github/workflows/pr-preflight.yaml | 39 ++ .github/workflows/pr-receive.yaml | 131 +++++++ .github/workflows/sandpaper-main.yaml | 61 +++ .github/workflows/sandpaper-version.txt | 1 + .github/workflows/update-cache.yaml | 125 +++++++ .github/workflows/update-workflows.yaml | 66 ++++ .github/workflows/workbench-beta-phase.yml | 60 +++ .gitignore | 54 +++ CODE_OF_CONDUCT.md | 13 + CONTRIBUTING.md | 123 +++++++ LICENSE.md | 79 ++++ README.md | 31 +- _extras/guide.md | 33 -- config.yaml | 90 +++++ episodes/01-singularity-gettingstarted.md | 168 +++++---- episodes/02-singularity-cache.md | 96 +++-- episodes/03-singularity-shell.md | 130 ++++--- episodes/04-singularity-files.md | 116 +++--- episodes/05-singularity-docker.md | 219 ++++++----- episodes/06-singularity-images-prep.md | 266 +++++++------ episodes/07-singularity-images-building.md | 246 +++++++------ episodes/08-singularity-mpi.md | 348 ++++++++++-------- {fig => episodes/fig}/SingularityInDocker.png | Bin .../files}/osu_latency.slurm.template | 0 index.md | 62 ++-- instructors/instructor-notes.md | 29 ++ {_extras => learners}/discuss.md | 5 +- learners/reference.md | 11 + setup.md => learners/setup.md | 5 +- profiles/learner-profiles.md | 5 + reference.md | 9 - site/README.md | 2 + 37 files changed, 2300 insertions(+), 786 deletions(-) create mode 100644 .editorconfig create mode 100644 .github/workflows/README.md create mode 100644 .github/workflows/pr-close-signal.yaml create mode 100644 .github/workflows/pr-comment.yaml create mode 100644 .github/workflows/pr-post-remove-branch.yaml create mode 100644 .github/workflows/pr-preflight.yaml create mode 100644 .github/workflows/pr-receive.yaml create mode 100644 .github/workflows/sandpaper-main.yaml create mode 100644 .github/workflows/sandpaper-version.txt create mode 100644 .github/workflows/update-cache.yaml create mode 100644 .github/workflows/update-workflows.yaml create mode 100644 .github/workflows/workbench-beta-phase.yml create mode 100644 .gitignore create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE.md delete mode 100644 _extras/guide.md create mode 100644 config.yaml rename {fig => episodes/fig}/SingularityInDocker.png (100%) rename {files => episodes/files}/osu_latency.slurm.template (100%) create mode 100644 instructors/instructor-notes.md rename {_extras => learners}/discuss.md (58%) create mode 100644 learners/reference.md rename setup.md => learners/setup.md (97%) create mode 100644 profiles/learner-profiles.md delete mode 100644 reference.md create mode 100644 site/README.md diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..1d7d564 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,25 @@ +root = true + +[*] +charset = utf-8 +insert_final_newline = true +trim_trailing_whitespace = true + +[*.md] +indent_size = 2 +indent_style = space +max_line_length = 100 # Please keep this in sync with bin/lesson_check.py! + +[*.r] +max_line_length = 80 + +[*.py] +indent_size = 4 +indent_style = space +max_line_length = 79 + +[*.sh] +end_of_line = lf + +[Makefile] +indent_style = tab diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..d6edf88 --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,198 @@ +# Carpentries Workflows + +This directory contains workflows to be used for Lessons using the {sandpaper} +lesson infrastructure. Two of these workflows require R (`sandpaper-main.yaml` +and `pr-recieve.yaml`) and the rest are bots to handle pull request management. + +These workflows will likely change as {sandpaper} evolves, so it is important to +keep them up-to-date. To do this in your lesson you can do the following in your +R console: + +```r +# Install/Update sandpaper +options(repos = c(carpentries = "https://carpentries.r-universe.dev/", + CRAN = "https://cloud.r-project.org")) +install.packages("sandpaper") + +# update the workflows in your lesson +library("sandpaper") +update_github_workflows() +``` + +Inside this folder, you will find a file called `sandpaper-version.txt`, which +will contain a version number for sandpaper. This will be used in the future to +alert you if a workflow update is needed. + +What follows are the descriptions of the workflow files: + +## Deployment + +### 01 Build and Deploy (sandpaper-main.yaml) + +This is the main driver that will only act on the main branch of the repository. +This workflow does the following: + + 1. checks out the lesson + 2. provisions the following resources + - R + - pandoc + - lesson infrastructure (stored in a cache) + - lesson dependencies if needed (stored in a cache) + 3. builds the lesson via `sandpaper:::ci_deploy()` + +#### Caching + +This workflow has two caches; one cache is for the lesson infrastructure and +the other is for the the lesson dependencies if the lesson contains rendered +content. These caches are invalidated by new versions of the infrastructure and +the `renv.lock` file, respectively. If there is a problem with the cache, +manual invaliation is necessary. You will need maintain access to the repository +and you can either go to the actions tab and [click on the caches button to find +and invalidate the failing cache](https://github.blog/changelog/2022-10-20-manage-caches-in-your-actions-workflows-from-web-interface/) +or by setting the `CACHE_VERSION` secret to the current date (which will +invalidate all of the caches). + +## Updates + +### Setup Information + +These workflows run on a schedule and at the maintainer's request. Because they +create pull requests that update workflows/require the downstream actions to run, +they need a special repository/organization secret token called +`SANDPAPER_WORKFLOW` and it must have the `public_repo` and `workflow` scope. + +This can be an individual user token, OR it can be a trusted bot account. If you +have a repository in one of the official Carpentries accounts, then you do not +need to worry about this token being present because the Carpentries Core Team +will take care of supplying this token. + +If you want to use your personal account: you can go to + +to create a token. Once you have created your token, you should copy it to your +clipboard and then go to your repository's settings > secrets > actions and +create or edit the `SANDPAPER_WORKFLOW` secret, pasting in the generated token. + +If you do not specify your token correctly, the runs will not fail and they will +give you instructions to provide the token for your repository. + +### 02 Maintain: Update Workflow Files (update-workflow.yaml) + +The {sandpaper} repository was designed to do as much as possible to separate +the tools from the content. For local builds, this is absolutely true, but +there is a minor issue when it comes to workflow files: they must live inside +the repository. + +This workflow ensures that the workflow files are up-to-date. The way it work is +to download the update-workflows.sh script from GitHub and run it. The script +will do the following: + +1. check the recorded version of sandpaper against the current version on github +2. update the files if there is a difference in versions + +After the files are updated, if there are any changes, they are pushed to a +branch called `update/workflows` and a pull request is created. Maintainers are +encouraged to review the changes and accept the pull request if the outputs +are okay. + +This update is run ~~weekly or~~ on demand. + +### 03 Maintain: Update Package Cache (update-cache.yaml) + +For lessons that have generated content, we use {renv} to ensure that the output +is stable. This is controlled by a single lockfile which documents the packages +needed for the lesson and the version numbers. This workflow is skipped in +lessons that do not have generated content. + +Because the lessons need to remain current with the package ecosystem, it's a +good idea to make sure these packages can be updated periodically. The +update cache workflow will do this by checking for updates, applying them in a +branch called `updates/packages` and creating a pull request with _only the +lockfile changed_. + +From here, the markdown documents will be rebuilt and you can inspect what has +changed based on how the packages have updated. + +## Pull Request and Review Management + +Because our lessons execute code, pull requests are a secruity risk for any +lesson and thus have security measures associted with them. **Do not merge any +pull requests that do not pass checks and do not have bots commented on them.** + +This series of workflows all go together and are described in the following +diagram and the below sections: + +![Graph representation of a pull request](https://carpentries.github.io/sandpaper/articles/img/pr-flow.dot.svg) + +### Pre Flight Pull Request Validation (pr-preflight.yaml) + +This workflow runs every time a pull request is created and its purpose is to +validate that the pull request is okay to run. This means the following things: + +1. The pull request does not contain modified workflow files +2. If the pull request contains modified workflow files, it does not contain + modified content files (such as a situation where @carpentries-bot will + make an automated pull request) +3. The pull request does not contain an invalid commit hash (e.g. from a fork + that was made before a lesson was transitioned from styles to use the + workbench). + +Once the checks are finished, a comment is issued to the pull request, which +will allow maintainers to determine if it is safe to run the +"Receive Pull Request" workflow from new contributors. + +### Recieve Pull Request (pr-recieve.yaml) + +**Note of caution:** This workflow runs arbitrary code by anyone who creates a +pull request. GitHub has safeguarded the token used in this workflow to have no +priviledges in the repository, but we have taken precautions to protect against +spoofing. + +This workflow is triggered with every push to a pull request. If this workflow +is already running and a new push is sent to the pull request, the workflow +running from the previous push will be cancelled and a new workflow run will be +started. + +The first step of this workflow is to check if it is valid (e.g. that no +workflow files have been modified). If there are workflow files that have been +modified, a comment is made that indicates that the workflow is not run. If +both a workflow file and lesson content is modified, an error will occurr. + +The second step (if valid) is to build the generated content from the pull +request. This builds the content and uploads three artifacts: + +1. The pull request number (pr) +2. A summary of changes after the rendering process (diff) +3. The rendered files (build) + +Because this workflow builds generated content, it follows the same general +process as the `sandpaper-main` workflow with the same caching mechanisms. + +The artifacts produced are used by the next workflow. + +### Comment on Pull Request (pr-comment.yaml) + +This workflow is triggered if the `pr-recieve.yaml` workflow is successful. +The steps in this workflow are: + +1. Test if the workflow is valid and comment the validity of the workflow to the + pull request. +2. If it is valid: create an orphan branch with two commits: the current state + of the repository and the proposed changes. +3. If it is valid: update the pull request comment with the summary of changes + +Importantly: if the pull request is invalid, the branch is not created so any +malicious code is not published. + +From here, the maintainer can request changes from the author and eventually +either merge or reject the PR. When this happens, if the PR was valid, the +preview branch needs to be deleted. + +### Send Close PR Signal (pr-close-signal.yaml) + +Triggered any time a pull request is closed. This emits an artifact that is the +pull request number for the next action + +### Remove Pull Request Branch (pr-post-remove-branch.yaml) + +Tiggered by `pr-close-signal.yaml`. This removes the temporary branch associated with +the pull request (if it was created). diff --git a/.github/workflows/pr-close-signal.yaml b/.github/workflows/pr-close-signal.yaml new file mode 100644 index 0000000..9b129d5 --- /dev/null +++ b/.github/workflows/pr-close-signal.yaml @@ -0,0 +1,23 @@ +name: "Bot: Send Close Pull Request Signal" + +on: + pull_request: + types: + [closed] + +jobs: + send-close-signal: + name: "Send closing signal" + runs-on: ubuntu-latest + if: ${{ github.event.action == 'closed' }} + steps: + - name: "Create PRtifact" + run: | + mkdir -p ./pr + printf ${{ github.event.number }} > ./pr/NUM + - name: Upload Diff + uses: actions/upload-artifact@v3 + with: + name: pr + path: ./pr + diff --git a/.github/workflows/pr-comment.yaml b/.github/workflows/pr-comment.yaml new file mode 100644 index 0000000..bb2eb03 --- /dev/null +++ b/.github/workflows/pr-comment.yaml @@ -0,0 +1,185 @@ +name: "Bot: Comment on the Pull Request" + +# read-write repo token +# access to secrets +on: + workflow_run: + workflows: ["Receive Pull Request"] + types: + - completed + +concurrency: + group: pr-${{ github.event.workflow_run.pull_requests[0].number }} + cancel-in-progress: true + + +jobs: + # Pull requests are valid if: + # - they match the sha of the workflow run head commit + # - they are open + # - no .github files were committed + test-pr: + name: "Test if pull request is valid" + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + outputs: + is_valid: ${{ steps.check-pr.outputs.VALID }} + payload: ${{ steps.check-pr.outputs.payload }} + number: ${{ steps.get-pr.outputs.NUM }} + msg: ${{ steps.check-pr.outputs.MSG }} + steps: + - name: 'Download PR artifact' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'pr' + + - name: "Get PR Number" + if: ${{ steps.dl.outputs.success == 'true' }} + id: get-pr + run: | + unzip pr.zip + echo "NUM=$(<./NR)" >> $GITHUB_OUTPUT + + - name: "Fail if PR number was not present" + id: bad-pr + if: ${{ steps.dl.outputs.success != 'true' }} + run: | + echo '::error::A pull request number was not recorded. The pull request that triggered this workflow is likely malicious.' + exit 1 + - name: "Get Invalid Hashes File" + id: hash + run: | + echo "json<> $GITHUB_OUTPUT + - name: "Check PR" + id: check-pr + if: ${{ steps.dl.outputs.success == 'true' }} + uses: carpentries/actions/check-valid-pr@main + with: + pr: ${{ steps.get-pr.outputs.NUM }} + sha: ${{ github.event.workflow_run.head_sha }} + headroom: 3 # if it's within the last three commits, we can keep going, because it's likely rapid-fire + invalid: ${{ fromJSON(steps.hash.outputs.json)[github.repository] }} + fail_on_error: true + + # Create an orphan branch on this repository with two commits + # - the current HEAD of the md-outputs branch + # - the output from running the current HEAD of the pull request through + # the md generator + create-branch: + name: "Create Git Branch" + needs: test-pr + runs-on: ubuntu-latest + if: ${{ needs.test-pr.outputs.is_valid == 'true' }} + env: + NR: ${{ needs.test-pr.outputs.number }} + permissions: + contents: write + steps: + - name: 'Checkout md outputs' + uses: actions/checkout@v3 + with: + ref: md-outputs + path: built + fetch-depth: 1 + + - name: 'Download built markdown' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'built' + + - if: ${{ steps.dl.outputs.success == 'true' }} + run: unzip built.zip + + - name: "Create orphan and push" + if: ${{ steps.dl.outputs.success == 'true' }} + run: | + cd built/ + git config --local user.email "actions@github.com" + git config --local user.name "GitHub Actions" + CURR_HEAD=$(git rev-parse HEAD) + git checkout --orphan md-outputs-PR-${NR} + git add -A + git commit -m "source commit: ${CURR_HEAD}" + ls -A | grep -v '^.git$' | xargs -I _ rm -r '_' + cd .. + unzip -o -d built built.zip + cd built + git add -A + git commit --allow-empty -m "differences for PR #${NR}" + git push -u --force --set-upstream origin md-outputs-PR-${NR} + + # Comment on the Pull Request with a link to the branch and the diff + comment-pr: + name: "Comment on Pull Request" + needs: [test-pr, create-branch] + runs-on: ubuntu-latest + if: ${{ needs.test-pr.outputs.is_valid == 'true' }} + env: + NR: ${{ needs.test-pr.outputs.number }} + permissions: + pull-requests: write + steps: + - name: 'Download comment artifact' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'diff' + + - if: ${{ steps.dl.outputs.success == 'true' }} + run: unzip ${{ github.workspace }}/diff.zip + + - name: "Comment on PR" + id: comment-diff + if: ${{ steps.dl.outputs.success == 'true' }} + uses: carpentries/actions/comment-diff@main + with: + pr: ${{ env.NR }} + path: ${{ github.workspace }}/diff.md + + # Comment if the PR is open and matches the SHA, but the workflow files have + # changed + comment-changed-workflow: + name: "Comment if workflow files have changed" + needs: test-pr + runs-on: ubuntu-latest + if: ${{ always() && needs.test-pr.outputs.is_valid == 'false' }} + env: + NR: ${{ github.event.workflow_run.pull_requests[0].number }} + body: ${{ needs.test-pr.outputs.msg }} + permissions: + pull-requests: write + steps: + - name: 'Check for spoofing' + id: dl + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: 'built' + + - name: 'Alert if spoofed' + id: spoof + if: ${{ steps.dl.outputs.success == 'true' }} + run: | + echo 'body<> $GITHUB_ENV + echo '' >> $GITHUB_ENV + echo '## :x: DANGER :x:' >> $GITHUB_ENV + echo 'This pull request has modified workflows that created output. Close this now.' >> $GITHUB_ENV + echo '' >> $GITHUB_ENV + echo 'EOF' >> $GITHUB_ENV + + - name: "Comment on PR" + id: comment-diff + uses: carpentries/actions/comment-diff@main + with: + pr: ${{ env.NR }} + body: ${{ env.body }} + diff --git a/.github/workflows/pr-post-remove-branch.yaml b/.github/workflows/pr-post-remove-branch.yaml new file mode 100644 index 0000000..62c2e98 --- /dev/null +++ b/.github/workflows/pr-post-remove-branch.yaml @@ -0,0 +1,32 @@ +name: "Bot: Remove Temporary PR Branch" + +on: + workflow_run: + workflows: ["Bot: Send Close Pull Request Signal"] + types: + - completed + +jobs: + delete: + name: "Delete branch from Pull Request" + runs-on: ubuntu-latest + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + permissions: + contents: write + steps: + - name: 'Download artifact' + uses: carpentries/actions/download-workflow-artifact@main + with: + run: ${{ github.event.workflow_run.id }} + name: pr + - name: "Get PR Number" + id: get-pr + run: | + unzip pr.zip + echo "NUM=$(<./NUM)" >> $GITHUB_OUTPUT + - name: 'Remove branch' + uses: carpentries/actions/remove-branch@main + with: + pr: ${{ steps.get-pr.outputs.NUM }} diff --git a/.github/workflows/pr-preflight.yaml b/.github/workflows/pr-preflight.yaml new file mode 100644 index 0000000..d0d7420 --- /dev/null +++ b/.github/workflows/pr-preflight.yaml @@ -0,0 +1,39 @@ +name: "Pull Request Preflight Check" + +on: + pull_request_target: + branches: + ["main"] + types: + ["opened", "synchronize", "reopened"] + +jobs: + test-pr: + name: "Test if pull request is valid" + if: ${{ github.event.action != 'closed' }} + runs-on: ubuntu-latest + outputs: + is_valid: ${{ steps.check-pr.outputs.VALID }} + permissions: + pull-requests: write + steps: + - name: "Get Invalid Hashes File" + id: hash + run: | + echo "json<> $GITHUB_OUTPUT + - name: "Check PR" + id: check-pr + uses: carpentries/actions/check-valid-pr@main + with: + pr: ${{ github.event.number }} + invalid: ${{ fromJSON(steps.hash.outputs.json)[github.repository] }} + fail_on_error: true + - name: "Comment result of validation" + id: comment-diff + if: ${{ always() }} + uses: carpentries/actions/comment-diff@main + with: + pr: ${{ github.event.number }} + body: ${{ steps.check-pr.outputs.MSG }} diff --git a/.github/workflows/pr-receive.yaml b/.github/workflows/pr-receive.yaml new file mode 100644 index 0000000..371ef54 --- /dev/null +++ b/.github/workflows/pr-receive.yaml @@ -0,0 +1,131 @@ +name: "Receive Pull Request" + +on: + pull_request: + types: + [opened, synchronize, reopened] + +concurrency: + group: ${{ github.ref }} + cancel-in-progress: true + +jobs: + test-pr: + name: "Record PR number" + if: ${{ github.event.action != 'closed' }} + runs-on: ubuntu-latest + outputs: + is_valid: ${{ steps.check-pr.outputs.VALID }} + steps: + - name: "Record PR number" + id: record + if: ${{ always() }} + run: | + echo ${{ github.event.number }} > ${{ github.workspace }}/NR # 2022-03-02: artifact name fixed to be NR + - name: "Upload PR number" + id: upload + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: pr + path: ${{ github.workspace }}/NR + - name: "Get Invalid Hashes File" + id: hash + run: | + echo "json<> $GITHUB_OUTPUT + - name: "echo output" + run: | + echo "${{ steps.hash.outputs.json }}" + - name: "Check PR" + id: check-pr + uses: carpentries/actions/check-valid-pr@main + with: + pr: ${{ github.event.number }} + invalid: ${{ fromJSON(steps.hash.outputs.json)[github.repository] }} + + build-md-source: + name: "Build markdown source files if valid" + needs: test-pr + runs-on: ubuntu-latest + if: ${{ needs.test-pr.outputs.is_valid == 'true' }} + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + RENV_PATHS_ROOT: ~/.local/share/renv/ + CHIVE: ${{ github.workspace }}/site/chive + PR: ${{ github.workspace }}/site/pr + MD: ${{ github.workspace }}/site/built + steps: + - name: "Check Out Main Branch" + uses: actions/checkout@v3 + + - name: "Check Out Staging Branch" + uses: actions/checkout@v3 + with: + ref: md-outputs + path: ${{ env.MD }} + + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: "Set up Pandoc" + uses: r-lib/actions/setup-pandoc@v2 + + - name: "Setup Lesson Engine" + uses: carpentries/actions/setup-sandpaper@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Setup Package Cache" + uses: carpentries/actions/setup-lesson-deps@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Validate and Build Markdown" + id: build-site + run: | + sandpaper::package_cache_trigger(TRUE) + sandpaper::validate_lesson(path = '${{ github.workspace }}') + sandpaper:::build_markdown(path = '${{ github.workspace }}', quiet = FALSE) + shell: Rscript {0} + + - name: "Generate Artifacts" + id: generate-artifacts + run: | + sandpaper:::ci_bundle_pr_artifacts( + repo = '${{ github.repository }}', + pr_number = '${{ github.event.number }}', + path_md = '${{ env.MD }}', + path_pr = '${{ env.PR }}', + path_archive = '${{ env.CHIVE }}', + branch = 'md-outputs' + ) + shell: Rscript {0} + + - name: "Upload PR" + uses: actions/upload-artifact@v3 + with: + name: pr + path: ${{ env.PR }} + + - name: "Upload Diff" + uses: actions/upload-artifact@v3 + with: + name: diff + path: ${{ env.CHIVE }} + retention-days: 1 + + - name: "Upload Build" + uses: actions/upload-artifact@v3 + with: + name: built + path: ${{ env.MD }} + retention-days: 1 + + - name: "Teardown" + run: sandpaper::reset_site() + shell: Rscript {0} diff --git a/.github/workflows/sandpaper-main.yaml b/.github/workflows/sandpaper-main.yaml new file mode 100644 index 0000000..e17707a --- /dev/null +++ b/.github/workflows/sandpaper-main.yaml @@ -0,0 +1,61 @@ +name: "01 Build and Deploy Site" + +on: + push: + branches: + - main + - master + schedule: + - cron: '0 0 * * 2' + workflow_dispatch: + inputs: + name: + description: 'Who triggered this build?' + required: true + default: 'Maintainer (via GitHub)' + reset: + description: 'Reset cached markdown files' + required: false + default: false + type: boolean +jobs: + full-build: + name: "Build Full Site" + runs-on: ubuntu-latest + permissions: + checks: write + contents: write + pages: write + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + RENV_PATHS_ROOT: ~/.local/share/renv/ + steps: + + - name: "Checkout Lesson" + uses: actions/checkout@v3 + + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: "Set up Pandoc" + uses: r-lib/actions/setup-pandoc@v2 + + - name: "Setup Lesson Engine" + uses: carpentries/actions/setup-sandpaper@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Setup Package Cache" + uses: carpentries/actions/setup-lesson-deps@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: "Deploy Site" + run: | + reset <- "${{ github.event.inputs.reset }}" == "true" + sandpaper::package_cache_trigger(TRUE) + sandpaper:::ci_deploy(reset = reset) + shell: Rscript {0} diff --git a/.github/workflows/sandpaper-version.txt b/.github/workflows/sandpaper-version.txt new file mode 100644 index 0000000..201a22c --- /dev/null +++ b/.github/workflows/sandpaper-version.txt @@ -0,0 +1 @@ +0.16.2 diff --git a/.github/workflows/update-cache.yaml b/.github/workflows/update-cache.yaml new file mode 100644 index 0000000..676d742 --- /dev/null +++ b/.github/workflows/update-cache.yaml @@ -0,0 +1,125 @@ +name: "03 Maintain: Update Package Cache" + +on: + workflow_dispatch: + inputs: + name: + description: 'Who triggered this build (enter github username to tag yourself)?' + required: true + default: 'monthly run' + schedule: + # Run every tuesday + - cron: '0 0 * * 2' + +jobs: + preflight: + name: "Preflight Check" + runs-on: ubuntu-latest + outputs: + ok: ${{ steps.check.outputs.ok }} + steps: + - id: check + run: | + if [[ ${{ github.event_name }} == 'workflow_dispatch' ]]; then + echo "ok=true" >> $GITHUB_OUTPUT + echo "Running on request" + # using single brackets here to avoid 08 being interpreted as octal + # https://github.com/carpentries/sandpaper/issues/250 + elif [ `date +%d` -le 7 ]; then + # If the Tuesday lands in the first week of the month, run it + echo "ok=true" >> $GITHUB_OUTPUT + echo "Running on schedule" + else + echo "ok=false" >> $GITHUB_OUTPUT + echo "Not Running Today" + fi + + check_renv: + name: "Check if We Need {renv}" + runs-on: ubuntu-latest + needs: preflight + if: ${{ needs.preflight.outputs.ok == 'true'}} + outputs: + needed: ${{ steps.renv.outputs.exists }} + steps: + - name: "Checkout Lesson" + uses: actions/checkout@v3 + - id: renv + run: | + if [[ -d renv ]]; then + echo "exists=true" >> $GITHUB_OUTPUT + fi + + check_token: + name: "Check SANDPAPER_WORKFLOW token" + runs-on: ubuntu-latest + needs: check_renv + if: ${{ needs.check_renv.outputs.needed == 'true' }} + outputs: + workflow: ${{ steps.validate.outputs.wf }} + repo: ${{ steps.validate.outputs.repo }} + steps: + - name: "validate token" + id: validate + uses: carpentries/actions/check-valid-credentials@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + + update_cache: + name: "Update Package Cache" + needs: check_token + if: ${{ needs.check_token.outputs.repo== 'true' }} + runs-on: ubuntu-latest + env: + GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }} + RENV_PATHS_ROOT: ~/.local/share/renv/ + steps: + + - name: "Checkout Lesson" + uses: actions/checkout@v3 + + - name: "Set up R" + uses: r-lib/actions/setup-r@v2 + with: + use-public-rspm: true + install-r: false + + - name: "Update {renv} deps and determine if a PR is needed" + id: update + uses: carpentries/actions/update-lockfile@main + with: + cache-version: ${{ secrets.CACHE_VERSION }} + + - name: Create Pull Request + id: cpr + if: ${{ steps.update.outputs.n > 0 }} + uses: carpentries/create-pull-request@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + delete-branch: true + branch: "update/packages" + commit-message: "[actions] update ${{ steps.update.outputs.n }} packages" + title: "Update ${{ steps.update.outputs.n }} packages" + body: | + :robot: This is an automated build + + This will update ${{ steps.update.outputs.n }} packages in your lesson with the following versions: + + ``` + ${{ steps.update.outputs.report }} + ``` + + :stopwatch: In a few minutes, a comment will appear that will show you how the output has changed based on these updates. + + If you want to inspect these changes locally, you can use the following code to check out a new branch: + + ```bash + git fetch origin update/packages + git checkout update/packages + ``` + + - Auto-generated by [create-pull-request][1] on ${{ steps.update.outputs.date }} + + [1]: https://github.com/carpentries/create-pull-request/tree/main + labels: "type: package cache" + draft: false diff --git a/.github/workflows/update-workflows.yaml b/.github/workflows/update-workflows.yaml new file mode 100644 index 0000000..288bcd1 --- /dev/null +++ b/.github/workflows/update-workflows.yaml @@ -0,0 +1,66 @@ +name: "02 Maintain: Update Workflow Files" + +on: + workflow_dispatch: + inputs: + name: + description: 'Who triggered this build (enter github username to tag yourself)?' + required: true + default: 'weekly run' + clean: + description: 'Workflow files/file extensions to clean (no wildcards, enter "" for none)' + required: false + default: '.yaml' + schedule: + # Run every Tuesday + - cron: '0 0 * * 2' + +jobs: + check_token: + name: "Check SANDPAPER_WORKFLOW token" + runs-on: ubuntu-latest + outputs: + workflow: ${{ steps.validate.outputs.wf }} + repo: ${{ steps.validate.outputs.repo }} + steps: + - name: "validate token" + id: validate + uses: carpentries/actions/check-valid-credentials@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + + update_workflow: + name: "Update Workflow" + runs-on: ubuntu-latest + needs: check_token + if: ${{ needs.check_token.outputs.workflow == 'true' }} + steps: + - name: "Checkout Repository" + uses: actions/checkout@v3 + + - name: Update Workflows + id: update + uses: carpentries/actions/update-workflows@main + with: + clean: ${{ github.event.inputs.clean }} + + - name: Create Pull Request + id: cpr + if: "${{ steps.update.outputs.new }}" + uses: carpentries/create-pull-request@main + with: + token: ${{ secrets.SANDPAPER_WORKFLOW }} + delete-branch: true + branch: "update/workflows" + commit-message: "[actions] update sandpaper workflow to version ${{ steps.update.outputs.new }}" + title: "Update Workflows to Version ${{ steps.update.outputs.new }}" + body: | + :robot: This is an automated build + + Update Workflows from sandpaper version ${{ steps.update.outputs.old }} -> ${{ steps.update.outputs.new }} + + - Auto-generated by [create-pull-request][1] on ${{ steps.update.outputs.date }} + + [1]: https://github.com/carpentries/create-pull-request/tree/main + labels: "type: template and tools" + draft: false diff --git a/.github/workflows/workbench-beta-phase.yml b/.github/workflows/workbench-beta-phase.yml new file mode 100644 index 0000000..2faa25d --- /dev/null +++ b/.github/workflows/workbench-beta-phase.yml @@ -0,0 +1,60 @@ +name: "Deploy to AWS" + +on: + workflow_run: + workflows: ["01 Build and Deploy Site"] + types: + - completed + workflow_dispatch: + +jobs: + preflight: + name: "Preflight Check" + runs-on: ubuntu-latest + outputs: + ok: ${{ steps.check.outputs.ok }} + folder: ${{ steps.check.outputs.folder }} + steps: + - id: check + run: | + if [[ -z "${{ secrets.DISTRIBUTION }}" || -z "${{ secrets.AWS_ACCESS_KEY_ID }}" || -z "${{ secrets.AWS_SECRET_ACCESS_KEY }}" ]]; then + echo ":information_source: No site configured" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo 'To deploy the preview on AWS, you need the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `DISTRIBUTION` secrets set up' >> $GITHUB_STEP_SUMMARY + else + echo "::set-output name=folder::"$(sed -E 's^.+/(.+)^\1^' <<< ${{ github.repository }}) + echo "::set-output name=ok::true" + fi + + full-build: + name: "Deploy to AWS" + needs: [preflight] + if: ${{ needs.preflight.outputs.ok }} + runs-on: ubuntu-latest + steps: + + - name: "Checkout site folder" + uses: actions/checkout@v3 + with: + ref: 'gh-pages' + path: 'source' + + - name: "Deploy to Bucket" + uses: jakejarvis/s3-sync-action@v0.5.1 + with: + args: --acl public-read --follow-symlinks --delete --exclude '.git/*' + env: + AWS_S3_BUCKET: preview.carpentries.org + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + SOURCE_DIR: 'source' + DEST_DIR: ${{ needs.preflight.outputs.folder }} + + - name: "Invalidate CloudFront" + uses: chetan/invalidate-cloudfront-action@master + env: + PATHS: /* + AWS_REGION: 'us-east-1' + DISTRIBUTION: ${{ secrets.DISTRIBUTION }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c4a48f2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,54 @@ +# sandpaper files +episodes/*html +site/* +!site/README.md + +# History files +.Rhistory +.Rapp.history +# Session Data files +.RData +# User-specific files +.Ruserdata +# Example code in package build process +*-Ex.R +# Output files from R CMD build +/*.tar.gz +# Output files from R CMD check +/*.Rcheck/ +# RStudio files +.Rproj.user/ +# produced vignettes +vignettes/*.html +vignettes/*.pdf +# OAuth2 token, see https://github.com/hadley/httr/releases/tag/v0.3 +.httr-oauth +# knitr and R markdown default cache directories +*_cache/ +/cache/ +# Temporary files created by R markdown +*.utf8.md +*.knit.md +# R Environment Variables +.Renviron +# pkgdown site +docs/ +# translation temp files +po/*~ +# renv detritus +renv/sandbox/ +*.pyc +*~ +.DS_Store +.ipynb_checkpoints +.sass-cache +.jekyll-cache/ +__pycache__ +_site +.Rproj.user +.bundle/ +.vendor/ +vendor/ +.docker-vendor/ +Gemfile.lock +.*history diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f19b804 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,13 @@ +--- +title: "Contributor Code of Conduct" +--- + +As contributors and maintainers of this project, +we pledge to follow the [The Carpentries Code of Conduct][coc]. + +Instances of abusive, harassing, or otherwise unacceptable behavior +may be reported by following our [reporting guidelines][coc-reporting]. + + +[coc-reporting]: https://docs.carpentries.org/topic_folders/policies/incident-reporting.html +[coc]: https://docs.carpentries.org/topic_folders/policies/code-of-conduct.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..6c2b81c --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,123 @@ +## Contributing + +[The Carpentries][cp-site] ([Software Carpentry][swc-site], [Data +Carpentry][dc-site], and [Library Carpentry][lc-site]) are open source +projects, and we welcome contributions of all kinds: new lessons, fixes to +existing material, bug reports, and reviews of proposed changes are all +welcome. + +### Contributor Agreement + +By contributing, you agree that we may redistribute your work under [our +license](LICENSE.md). In exchange, we will address your issues and/or assess +your change proposal as promptly as we can, and help you become a member of our +community. Everyone involved in [The Carpentries][cp-site] agrees to abide by +our [code of conduct](CODE_OF_CONDUCT.md). + +### How to Contribute + +The easiest way to get started is to file an issue to tell us about a spelling +mistake, some awkward wording, or a factual error. This is a good way to +introduce yourself and to meet some of our community members. + +1. If you do not have a [GitHub][github] account, you can [send us comments by + email][contact]. However, we will be able to respond more quickly if you use + one of the other methods described below. + +2. If you have a [GitHub][github] account, or are willing to [create + one][github-join], but do not know how to use Git, you can report problems + or suggest improvements by [creating an issue][repo-issues]. This allows us + to assign the item to someone and to respond to it in a threaded discussion. + +3. If you are comfortable with Git, and would like to add or change material, + you can submit a pull request (PR). Instructions for doing this are + [included below](#using-github). For inspiration about changes that need to + be made, check out the [list of open issues][issues] across the Carpentries. + +Note: if you want to build the website locally, please refer to [The Workbench +documentation][template-doc]. + +### Where to Contribute + +1. If you wish to change this lesson, add issues and pull requests here. +2. If you wish to change the template used for workshop websites, please refer + to [The Workbench documentation][template-doc]. + + +### What to Contribute + +There are many ways to contribute, from writing new exercises and improving +existing ones to updating or filling in the documentation and submitting [bug +reports][issues] about things that do not work, are not clear, or are missing. +If you are looking for ideas, please see [the list of issues for this +repository][repo-issues], or the issues for [Data Carpentry][dc-issues], +[Library Carpentry][lc-issues], and [Software Carpentry][swc-issues] projects. + +Comments on issues and reviews of pull requests are just as welcome: we are +smarter together than we are on our own. **Reviews from novices and newcomers +are particularly valuable**: it's easy for people who have been using these +lessons for a while to forget how impenetrable some of this material can be, so +fresh eyes are always welcome. + +### What *Not* to Contribute + +Our lessons already contain more material than we can cover in a typical +workshop, so we are usually *not* looking for more concepts or tools to add to +them. As a rule, if you want to introduce a new idea, you must (a) estimate how +long it will take to teach and (b) explain what you would take out to make room +for it. The first encourages contributors to be honest about requirements; the +second, to think hard about priorities. + +We are also not looking for exercises or other material that only run on one +platform. Our workshops typically contain a mixture of Windows, macOS, and +Linux users; in order to be usable, our lessons must run equally well on all +three. + +### Using GitHub + +If you choose to contribute via GitHub, you may want to look at [How to +Contribute to an Open Source Project on GitHub][how-contribute]. In brief, we +use [GitHub flow][github-flow] to manage changes: + +1. Create a new branch in your desktop copy of this repository for each + significant change. +2. Commit the change in that branch. +3. Push that branch to your fork of this repository on GitHub. +4. Submit a pull request from that branch to the [upstream repository][repo]. +5. If you receive feedback, make changes on your desktop and push to your + branch on GitHub: the pull request will update automatically. + +NB: The published copy of the lesson is usually in the `main` branch. + +Each lesson has a team of maintainers who review issues and pull requests or +encourage others to do so. The maintainers are community volunteers, and have +final say over what gets merged into the lesson. + +### Other Resources + +The Carpentries is a global organisation with volunteers and learners all over +the world. We share values of inclusivity and a passion for sharing knowledge, +teaching and learning. There are several ways to connect with The Carpentries +community listed at including via social +media, slack, newsletters, and email lists. You can also [reach us by +email][contact]. + +[repo]: https://example.com/FIXME +[repo-issues]: https://example.com/FIXME/issues +[contact]: mailto:team@carpentries.org +[cp-site]: https://carpentries.org/ +[dc-issues]: https://github.com/issues?q=user%3Adatacarpentry +[dc-lessons]: https://datacarpentry.org/lessons/ +[dc-site]: https://datacarpentry.org/ +[discuss-list]: https://carpentries.topicbox.com/groups/discuss +[github]: https://github.com +[github-flow]: https://guides.github.com/introduction/flow/ +[github-join]: https://github.com/join +[how-contribute]: https://egghead.io/courses/how-to-contribute-to-an-open-source-project-on-github +[issues]: https://carpentries.org/help-wanted-issues/ +[lc-issues]: https://github.com/issues?q=user%3ALibraryCarpentry +[swc-issues]: https://github.com/issues?q=user%3Aswcarpentry +[swc-lessons]: https://software-carpentry.org/lessons/ +[swc-site]: https://software-carpentry.org/ +[lc-site]: https://librarycarpentry.org/ +[template-doc]: https://carpentries.github.io/workbench/ diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..7632871 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,79 @@ +--- +title: "Licenses" +--- + +## Instructional Material + +All Carpentries (Software Carpentry, Data Carpentry, and Library Carpentry) +instructional material is made available under the [Creative Commons +Attribution license][cc-by-human]. The following is a human-readable summary of +(and not a substitute for) the [full legal text of the CC BY 4.0 +license][cc-by-legal]. + +You are free: + +- to **Share**---copy and redistribute the material in any medium or format +- to **Adapt**---remix, transform, and build upon the material + +for any purpose, even commercially. + +The licensor cannot revoke these freedoms as long as you follow the license +terms. + +Under the following terms: + +- **Attribution**---You must give appropriate credit (mentioning that your work + is derived from work that is Copyright (c) The Carpentries and, where + practical, linking to ), provide a [link to the + license][cc-by-human], and indicate if changes were made. You may do so in + any reasonable manner, but not in any way that suggests the licensor endorses + you or your use. + +- **No additional restrictions**---You may not apply legal terms or + technological measures that legally restrict others from doing anything the + license permits. With the understanding that: + +Notices: + +* You do not have to comply with the license for elements of the material in + the public domain or where your use is permitted by an applicable exception + or limitation. +* No warranties are given. The license may not give you all of the permissions + necessary for your intended use. For example, other rights such as publicity, + privacy, or moral rights may limit how you use the material. + +## Software + +Except where otherwise noted, the example programs and other software provided +by The Carpentries are made available under the [OSI][osi]-approved [MIT +license][mit-license]. + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +## Trademark + +"The Carpentries", "Software Carpentry", "Data Carpentry", and "Library +Carpentry" and their respective logos are registered trademarks of [Community +Initiatives][ci]. + +[cc-by-human]: https://creativecommons.org/licenses/by/4.0/ +[cc-by-legal]: https://creativecommons.org/licenses/by/4.0/legalcode +[mit-license]: https://opensource.org/licenses/mit-license.html +[ci]: https://communityin.org/ +[osi]: https://opensource.org diff --git a/README.md b/README.md index eaea1b3..dc42b3c 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ +> **ATTENTION** This is an experimental test of [The Carpentries Workbench](https://carpentries.github.io/workbench) lesson infrastructure. +> It was automatically converted from the source lesson via [the lesson transition script](https://github.com/carpentries/lesson-transition/). +> +> If anything seems off, please contact Zhian Kamvar [zkamvar@carpentries.org](mailto:zkamvar@carpentries.org) + # Reproducible computational environments using containers: Introduction to Singularity [![Create a Slack Account with us](https://img.shields.io/badge/Create_Slack_Account-The_Carpentries-071159.svg)](https://swc-slack-invite.herokuapp.com/) @@ -6,11 +11,11 @@ This lesson provides an introduction to the [Singularity container platform](htt It covers the basics of using Singularity and creating containers: - - What is Singularity? - - Installing/running Singularity on the command line - - Running containers - - Creating Singularity images - - Running an MPI parallel application from a Singularity container +- What is Singularity? +- Installing/running Singularity on the command line +- Running containers +- Creating Singularity images +- Running an MPI parallel application from a Singularity container ## Contributing @@ -21,19 +26,18 @@ We'd like to ask you to familiarize yourself with our [Contribution Guide](CONTR the [more detailed guidelines][lesson-example] on proper formatting, ways to render the lesson locally, and even how to write new episodes. -Please see the current list of [issues][issues] for ideas for contributing to this +Please see the current list of [issues] for ideas for contributing to this repository. For making your contribution, we use the GitHub flow, which is -nicely explained in the chapter [Contributing to a Project](http://git-scm.com/book/en/v2/GitHub-Contributing-to-a-Project) in Pro Git +nicely explained in the chapter [Contributing to a Project](https://git-scm.com/book/en/v2/GitHub-Contributing-to-a-Project) in Pro Git by Scott Chacon. -Look for the tag ![good_first_issue](https://img.shields.io/badge/-good%20first%20issue-gold.svg). This indicates that the maintainers will welcome a pull request fixing this issue. - +Look for the tag ![good\_first\_issue](https://img.shields.io/badge/-good%20first%20issue-gold.svg). This indicates that the maintainers will welcome a pull request fixing this issue. ## Maintainer(s) Current maintainers of this lesson are -* [Jeremy Cohen](https://github.com/jcohen02) -* [Andy Turner](https://github.com/aturner-epcc) +- [Jeremy Cohen](https://github.com/jcohen02) +- [Andy Turner](https://github.com/aturner-epcc) ## Authors @@ -43,7 +47,8 @@ A list of contributors to the lesson can be found in [AUTHORS](AUTHORS) To cite this lesson, please consult with [CITATION](CITATION) -[cdh]: https://cdh.carpentries.org -[community-lessons]: https://carpentries.org/community-lessons [lesson-example]: https://carpentries.github.io/lesson-example [issues]: https://github.com/carpentries-incubator/singularity-introduction/issues + + + diff --git a/_extras/guide.md b/_extras/guide.md deleted file mode 100644 index afa8bcc..0000000 --- a/_extras/guide.md +++ /dev/null @@ -1,33 +0,0 @@ ---- -layout: page -title: "Instructor Notes" ---- - -## Resouces for Instructors - -## Workshop Structure - -_[Instructors, please add notes here reporting on your experiences of teaching this module, either standalone, or as part of a wider workshop.]_ - - - **Containers course covering Docker and Singularity:** This Singularity module is regularly taught alongside the [Introduction to Docker](https://github.com/carpentries-incubator/docker-introduction) module as part of a 2-day course "_Reproducible computational environments using containers_" run through the [ARCHER2 training programme](https://www.archer2.ac.uk/training/) in the UK. [See an example](https://www.archer2.ac.uk/training/courses/221207-containers/) of this course run in December 2022. - - This course has been run both online and in person. Experience suggests that this Singularity module requires between 5 and 6 hours of time to run effectively. The main aspect that takes a significant amount of time is the material at the end of the module looking at building a Singularity image containing an MPI code and then running this in parallel on an HPC platform. The variation in timing depends on how much experience the learners already have with running parallel jobs on HPC platforms and how much they wish to go into the details of the processes of running parallel jobs. For some groups of learners, the MPI use case is not something that is relevant and they may request to cover this material without the section on running parallel MPI jobs. In this case, the material can comfortably be compelted in 4 hours. - - - -## Technical tips and tricks - - - **HPC access:** Many learners will be keen to learn Singularity so that they can make use of it on a remote High Performance Computing (HPC) cluster. It is therefore strongly recommended that workshop organizers provide course attendees with access to an HPC platform that has Singularity pre-installed for undertaking this module. Where necessary, it is also recommended that guest accounts are set up and learners are asked to test access to the platform before the workshop. - - - **Use of the Singularity Docker container:** Singularity is a Linux tool. The optimal approach to building Singularity images, a key aspect of the material in this module, requires that the learner have a platform with Singularity installed, on which they have admin/root access. Since it's likely that many learners undertaking this module will not be using Linux as the main operating system on the computer on which they are undertaking the training, we need an alternative option. To address this, we use Docker to run the [Singularity Docker container](https://quay.io/repository/singularity/singularity). This ensures that learners have access to a local Singularity deployment (running within a Docker container) on which they have root access and can build Singularity images. The layers of indirection that this requires can prove confusing and we are looking at alternatives but from experience of teaching the module so far, this has proved to be the most reasonable solution at present. - -#### Pre-workshop Planning / Installation / Setup - -As highlighted above, this module is designed to support learners who wish to use Singularity on an HPC cluster. Elements of the module are therefore designed to be run on a High Performance Computing cluster (i.e. clusters that run SLURM, SGE, or other job scheduling software). It is possible for learners to undertake large parts of the module on their own computer. However, since a key use case for Singularity containers is their use on HPC infrastructure, it is recommended that an HPC platform be used in the teaching of this module. In such a case, if Singularity is not already on the cluster, admin rights would be required to install Singularity cluster-wide. Practically, it is likely that this will require a support ticket to be raised with cluster administrators and may require some time for them to investigate the software if they are unfamiliar with Singularity. - -## Common problems - -Some installs of Singularity require the use of `--bind` for compatibility between the container and the host system, if the host system does not have a directory that is a default or required directory in the container that directory will need to be bound elsewhere in order to work correctly: (i.e. `--bind /data:/mnt`) - - - diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..bb8f75b --- /dev/null +++ b/config.yaml @@ -0,0 +1,90 @@ +#------------------------------------------------------------ +# Values for this lesson. +#------------------------------------------------------------ + +# Which carpentry is this (swc, dc, lc, or cp)? +# swc: Software Carpentry +# dc: Data Carpentry +# lc: Library Carpentry +# cp: Carpentries (to use for instructor training for instance) +# incubator: The Carpentries Incubator +carpentry: 'incubator' + +# Overall title for pages. +title: 'Reproducible computational environments using containers: Introduction to Singularity' + +# Date the lesson was created (YYYY-MM-DD, this is empty by default) +created: + +# Comma-separated list of keywords for the lesson +keywords: 'software, data, lesson, The Carpentries' + +# Life cycle stage of the lesson +# possible values: pre-alpha, alpha, beta, stable +life_cycle: 'alpha' + +# License of the lesson materials (recommended CC-BY 4.0) +license: 'CC-BY 4.0' + +# Link to the source repository for this lesson +source: 'https://github.com/fishtree-attempt/singularity-introduction/' + +# Default branch of your lesson +branch: 'main' + +# Who to contact if there are any issues +contact: 'jeremy.cohen@imperial.ac.uk' + +# Navigation ------------------------------------------------ +# +# Use the following menu items to specify the order of +# individual pages in each dropdown section. Leave blank to +# include all pages in the folder. +# +# Example ------------- +# +# episodes: +# - introduction.md +# - first-steps.md +# +# learners: +# - setup.md +# +# instructors: +# - instructor-notes.md +# +# profiles: +# - one-learner.md +# - another-learner.md + +# Order of episodes in your lesson +episodes: +- 01-singularity-gettingstarted.md +- 02-singularity-cache.md +- 03-singularity-shell.md +- 04-singularity-files.md +- 05-singularity-docker.md +- 06-singularity-images-prep.md +- 07-singularity-images-building.md +- 08-singularity-mpi.md + +# Information for Learners +learners: + +# Information for Instructors +instructors: + +# Learner Profiles +profiles: + +# Customisation --------------------------------------------- +# +# This space below is where custom yaml items (e.g. pinning +# sandpaper and varnish versions) should live + + +carpentry_description: Lesson Description +url: https://preview.carpentries.org/singularity-introduction +analytics: carpentries +lang: en +workbench-beta: yes diff --git a/episodes/01-singularity-gettingstarted.md b/episodes/01-singularity-gettingstarted.md index 0a38b1b..9d8c31b 100644 --- a/episodes/01-singularity-gettingstarted.md +++ b/episodes/01-singularity-gettingstarted.md @@ -1,28 +1,33 @@ --- -title: "Singularity: Getting started" -start: true +title: 'Singularity: Getting started' +start: yes teaching: 30 exercises: 20 -questions: -- "What is Singularity and why might I want to use it?" -objectives: -- "Understand what Singularity is and when you might want to use it." -- "Undertake your first run of a simple Singularity container." -keypoints: -- "Singularity is another container platform and it is often used in cluster/HPC/research environments." -- "Singularity has a different security model to other container platforms, one of the key reasons that it is well suited to HPC and cluster environments." -- "Singularity has its own container image format (SIF)." -- "The `singularity` command can be used to pull images from Singularity Hub and run a container from an image file." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Understand what Singularity is and when you might want to use it. +- Undertake your first run of a simple Singularity container. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- What is Singularity and why might I want to use it? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + The episodes in this lesson will introduce you to the [Singularity](https://sylabs.io/singularity/) container platform and demonstrate how to set up and use Singularity. This material is split into 2 parts: *Part I: Basic usage, working with images* - 1. **Singularity: Getting started**: This introductory episode - + +1. **Singularity: Getting started**: This introductory episode + Working with Singularity containers: +
  1. The singularity cache: Why, where and how does Singularity cache images locally?
  2. Running commands within a Singularity container: How to run commands within a Singularity container.
    3. @@ -30,125 +35,146 @@ Working with Singularity containers:
  3. Using Docker images with Singularity: How to run Singularity containers from Docker images.
*Part II: Creating images, running parallel codes* -
    -
  1. Preparing to build Singularity images: Getting started with the Docker Singularity container.
    2. -
  2. Building Singularity images: Explaining how to build and share your own Singularity images.
    3. -
  3. Running MPI parallel jobs using Singularity containers: Explaining how to run MPI parallel codes from within Singularity containers.
    4. +
    1. Preparing to build Singularity images: Getting started with the Docker Singularity container.
    2. Building Singularity images: Explaining how to build and share your own Singularity images.
    3. Running MPI parallel jobs using Singularity containers: Explaining how to run MPI parallel codes from within Singularity containers.
    -> ## Work in progress... -> This lesson is new material that is under ongoing development. We will introduce Singularity and demonstrate how to work with it. As the tools and best practices continue to develop, elements of this material are likely to evolve. We welcome any comments or suggestions on how the material can be improved or extended. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout -# Singularity - Part I +## Work in progress... + +This lesson is new material that is under ongoing development. We will introduce Singularity and demonstrate how to work with it. As the tools and best practices continue to develop, elements of this material are likely to evolve. We welcome any comments or suggestions on how the material can be improved or extended. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +## Singularity - Part I ## What is Singularity? [Singularity](https://sylabs.io/singularity/) is a container platform that allows software engineers and researchers to easily share their work with others by packaging and deploying their software applications in a portable and reproducible manner. When you download a Singularity container image, you essentially receive a virtual computer disk that contains all of the necessary software, libraries and configuration to run one or more applications or undertake a particular task, e.g. to support a specific research project. This saves you the time and effort of installing and configuring software on your own system or setting up a new computer from scratch, as you can simply run a Singularity container from the image and have a virtual environment that is identical to the one used by the person who created the image. Container platforms like Singularity provide a convenient and consistent way to access and run software and tools. Singularity is increasingly widely used in the research community for supporting research projects as it allows users to isolate their software environments from the host operating system and can simplify tasks such as running multiple experiments simultaneously. -You may be familiar with Docker, another container platform that is now used widely. If you are, you will see that in some ways, Singularity is similar to Docker. However, in others, particularly in the system's architecture, it is fundamentally different. These differences mean that Singularity is particularly well-suited to running on distributed, High Performance Computing (HPC) infrastructure, as well as a Linux laptop or desktop! +You may be familiar with Docker, another container platform that is now used widely. If you are, you will see that in some ways, Singularity is similar to Docker. However, in others, particularly in the system's architecture, it is fundamentally different. These differences mean that Singularity is particularly well-suited to running on distributed, High Performance Computing (HPC) infrastructure, as well as a Linux laptop or desktop! -_Later in this material, when we come to look at building Singularity images ourselves, we will make use of Docker to provide an environment in which we can run Singularity with administrative privileges. In this context, some basic knowledge of Docker is strongly recommended. If you are covering this module independently, or as part of a course that hasn't covered Docker, you can find an introduction to Docker in the "[Reproducible Computational Environments Using Containers: Introduction to Docker](https://carpentries-incubator.github.io/docker-introduction/index.html)" lesson._ +*Later in this material, when we come to look at building Singularity images ourselves, we will make use of Docker to provide an environment in which we can run Singularity with administrative privileges. In this context, some basic knowledge of Docker is strongly recommended. If you are covering this module independently, or as part of a course that hasn't covered Docker, you can find an introduction to Docker in the "[Reproducible Computational Environments Using Containers: Introduction to Docker](https://carpentries-incubator.github.io/docker-introduction/index.html)" lesson.* System administrators will not, generally, install Docker on shared computing platforms such as lab desktops, research clusters or HPC platforms because the design of Docker presents potential security issues for shared platforms with multiple users. Singularity, on the other hand, can be run by end-users entirely within "user space", that is, no special administrative privileges need to be assigned to a user in order for them to run and interact with containers on a platform where Singularity has been installed. ## Getting started with Singularity -Initially developed within the research community, Singularity is open source and the [repository](https://github.com/hpcng/singularity) is currently available in the "[The Next Generation of High Performance Computing](https://github.com/hpcng)" GitHub organisation. Part I of this Singularity material is intended to be undertaken on a remote platform where Singularity has been pre-installed. -_If you're attending a taught version of this course, you will be provided with access details for a remote platform made available to you for use for Part I of the Singularity material. This platform will have the Singularity software pre-installed._ +Initially developed within the research community, Singularity is open source and the [repository](https://github.com/hpcng/singularity) is currently available in the "[The Next Generation of High Performance Computing](https://github.com/hpcng)" GitHub organisation. Part I of this Singularity material is intended to be undertaken on a remote platform where Singularity has been pre-installed. + +*If you're attending a taught version of this course, you will be provided with access details for a remote platform made available to you for use for Part I of the Singularity material. This platform will have the Singularity software pre-installed.* + +::::::::::::::::::::::::::::::::::::::::: callout + +## Installing Singularity on your own laptop/desktop + +If you have a Linux system on which you have administrator access and you would like to install Singularity on this system, some information is provided at the start of [Part II of the Singularity material](06-singularity-images-prep.md). -> ## Installing Singularity on your own laptop/desktop -> If you have a Linux system on which you have administrator access and you would like to install Singularity on this system, some information is provided at the start of [Part II of the Singularity material]({{ page.root }}/06-singularity-images-prep/index.html#installing-singularity-on-your-local-system-optional-advanced-task). -{: .callout} + +:::::::::::::::::::::::::::::::::::::::::::::::::: Sign in to the remote platform, with Singularity installed, that you've been provided with access to. Check that the `singularity` command is available in your terminal: -> ## Loading a module -> HPC systems often use *modules* to provide access to software on the system so you may need to use the command: -> ~~~ -> $ module load singularity -> ~~~ -> {: .language-bash} -> before you can use the `singularity` command on the system. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Loading a module + +HPC systems often use *modules* to provide access to software on the system so you may need to use the command: + +```bash +$ module load singularity +``` + +before you can use the `singularity` command on the system. -~~~ + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +```bash $ singularity --version -~~~ -{: .language-bash} +``` -~~~ +```output singularity version 3.5.3 -~~~ -{: .output} +``` Depending on the version of Singularity installed on your system, you may see a different version. At the time of writing, `v3.5.3` is the latest release of Singularity. ## Images and containers -We'll start with a brief note on the terminology used in this section of the course. We refer to both **_images_** and **_containers_**. What is the distinction between these two terms? +We'll start with a brief note on the terminology used in this section of the course. We refer to both ***images*** and ***containers***. What is the distinction between these two terms? -**_Images_** are bundles of files including an operating system, software and potentially data and other application-related files. They may sometimes be referred to as a _disk image_ or _container image_ and they may be stored in different ways, perhaps as a single file, or as a group of files. Either way, we refer to this file, or collection of files, as an image. +***Images*** are bundles of files including an operating system, software and potentially data and other application-related files. They may sometimes be referred to as a *disk image* or *container image* and they may be stored in different ways, perhaps as a single file, or as a group of files. Either way, we refer to this file, or collection of files, as an image. -A **_container_** is a virtual environment that is based on an image. That is, the files, applications, tools, etc that are available within a running container are determined by the image that the container is started from. It may be possible to start multiple container instances from an image. You could, perhaps, consider an image to be a form of template from which running container instances can be started. +A ***container*** is a virtual environment that is based on an image. That is, the files, applications, tools, etc that are available within a running container are determined by the image that the container is started from. It may be possible to start multiple container instances from an image. You could, perhaps, consider an image to be a form of template from which running container instances can be started. ## Getting an image and running a Singularity container -If you recall from learning about Docker, Docker images are formed of a set of _layers_ that make up the complete image. When you pull a Docker image from Docker Hub, you see the different layers being downloaded to your system. They are stored in your local Docker repository on your system and you can see details of the available images using the `docker` command. +If you recall from learning about Docker, Docker images are formed of a set of *layers* that make up the complete image. When you pull a Docker image from Docker Hub, you see the different layers being downloaded to your system. They are stored in your local Docker repository on your system and you can see details of the available images using the `docker` command. Singularity images are a little different. Singularity uses the [Singularity Image Format (SIF)](https://github.com/sylabs/sif) and images are provided as single `SIF` files (with a `.sif` filename extension). Singularity images can be pulled from [Singularity Hub](https://singularity-hub.org/), a registry for container images. Singularity is also capable of running containers based on images pulled from [Docker Hub](https://hub.docker.com/) and some other sources. We'll look at accessing containers from Docker Hub later in the Singularity material. -> ## Singularity Hub -> Note that in addition to providing a repository that you can pull images from, [Singularity Hub](https://singularity-hub.org/) can also build Singularity images for you from a _**recipe**_ - a configuration file defining the steps to build an image. We'll look at recipes and building images later. -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Singularity Hub + +Note that in addition to providing a repository that you can pull images from, [Singularity Hub](https://singularity-hub.org/) can also build Singularity images for you from a ***recipe*** - a configuration file defining the steps to build an image. We'll look at recipes and building images later. -Let's begin by creating a `test` directory, changing into it and _pulling_ a test _Hello World_ image from Singularity Hub: -~~~ +:::::::::::::::::::::::::::::::::::::::::::::::::: + +Let's begin by creating a `test` directory, changing into it and *pulling* a test *Hello World* image from Singularity Hub: + +```bash $ mkdir test $ cd test $ singularity pull hello-world.sif shub://vsoch/hello-world -~~~ -{: .language-bash} +``` -~~~ +```output INFO: Downloading shub image 59.75 MiB / 59.75 MiB [===============================================================================================================] 100.00% 52.03 MiB/s 1s -~~~ -{: .output} +``` What just happened?! We pulled a SIF image from Singularity Hub using the `singularity pull` command and directed it to store the image file using the name `hello-world.sif` in the current directory. If you run the `ls` command, you should see that the `hello-world.sif` file is now present in the current directory. This is our image and we can now run a container based on this image: -~~~ +```bash $ singularity run hello-world.sif -~~~ -{: .language-bash} +``` -~~~ +```output RaawwWWWWWRRRR!! Avocado! -~~~ -{: .output} - -The above command ran the _hello-world_ container from the image we downloaded from Singularity Hub and the resulting output was shown. +``` +The above command ran the *hello-world* container from the image we downloaded from Singularity Hub and the resulting output was shown. How did the container determine what to do when we ran it?! What did running the container actually do to result in the displayed output? When you run a container from a Singularity image without using any additional command line arguments, the container runs the default run script that is embedded within the image. This is a shell script that can be used to run commands, tools or applications stored within the image on container startup. We can inspect the image's run script using the `singularity inspect` command: -~~~ +```bash $ singularity inspect -r hello-world.sif -~~~ -{: .language-bash} +``` -~~~ +```output #!/bin/sh exec /bin/bash /rawr.sh -~~~ -{: .output} +``` This shows us the script within the `hello-world.sif` image configured to run by default when we use the `singularity run` command. That concludes this introductory Singularity episode. The next episode looks in more detail at running containers. + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Singularity is another container platform and it is often used in cluster/HPC/research environments. +- Singularity has a different security model to other container platforms, one of the key reasons that it is well suited to HPC and cluster environments. +- Singularity has its own container image format (SIF). +- The `singularity` command can be used to pull images from Singularity Hub and run a container from an image file. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/02-singularity-cache.md b/episodes/02-singularity-cache.md index ab2f0fd..b09b480 100644 --- a/episodes/02-singularity-cache.md +++ b/episodes/02-singularity-cache.md @@ -1,74 +1,92 @@ --- -title: "The Singularity cache" +title: The Singularity cache teaching: 10 exercises: 0 -questions: -- "Why does Singularity use a local cache?" -- "Where does Singularity store images?" -objectives: -- "Learn about Singularity's image cache." -- "Learn how to manage Singularity images stored locally." -keypoints: -- "Singularity caches downloaded images so that an unchanged image isn't downloaded again when it is requested using the `singularity pull` command." -- "You can free up space in the cache by removing all locally cached images or by specifying individual images to remove." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Learn about Singularity's image cache. +- Learn how to manage Singularity images stored locally. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- Why does Singularity use a local cache? +- Where does Singularity store images? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + ## Singularity's image cache -While Singularity doesn't have a local image repository in the same way as Docker, it does cache downloaded image files. As we saw in the previous episode, images are simply `.sif` files stored on your local disk. +While Singularity doesn't have a local image repository in the same way as Docker, it does cache downloaded image files. As we saw in the previous episode, images are simply `.sif` files stored on your local disk. If you delete a local `.sif` image that you have pulled from a remote image repository and then pull it again, if the image is unchanged from the version you previously pulled, you will be given a copy of the image file from your local cache rather than the image being downloaded again from the remote source. This removes unnecessary network transfers and is particularly useful for large images which may take some time to transfer over the network. To demonstrate this, remove the `hello-world.sif` file stored in your `test` directory and then issue the `pull` command again: -~~~ +```bash $ rm hello-world.sif $ singularity pull hello-world.sif shub://vsoch/hello-world -~~~ -{: .language-bash} +``` -~~~ +```output INFO: Use image from cache -~~~ -{: .output} +``` As we can see in the above output, the image has been returned from the cache and we don't see the output that we saw previously showing the image being downloaded from Singularity Hub. How do we know what is stored in the local cache? We can find out using the `singularity cache` command: -~~~ +```bash $ singularity cache list -~~~ -{: .language-bash} +``` -~~~ +```output There are 1 container file(s) using 62.65 MB and 0 oci blob file(s) using 0.00 kB of space Total space used: 62.65 MB -~~~ -{: .output} +``` -This tells us how many container files are stored in the cache and how much disk space the cache is using but it doesn't tell us _what_ is actually being stored. To find out more information we can add the `-v` verbose flag to the `list` command: +This tells us how many container files are stored in the cache and how much disk space the cache is using but it doesn't tell us *what* is actually being stored. To find out more information we can add the `-v` verbose flag to the `list` command: -~~~ +```bash $ singularity cache list -v -~~~ -{: .language-bash} +``` -~~~ +```output NAME DATE CREATED SIZE TYPE hello-world_latest.sif 2020-04-03 13:20:44 62.65 MB shub There are 1 container file(s) using 62.65 MB and 0 oci blob file(s) using 0.00 kB of space Total space used: 62.65 MB -~~~ -{: .output} +``` + +This provides us with some more useful information about the actual images stored in the cache. In the `TYPE` column we can see that our image type is `shub` because it's a `SIF` image that has been pulled from Singularity Hub. + +::::::::::::::::::::::::::::::::::::::::: callout + +## Cleaning the Singularity image cache + +We can remove images from the cache using the `singularity cache clean` command. Running the command without any options will display a warning and ask you to confirm that you want to remove everything from your cache. + +You can also remove specific images or all images of a particular type. Look at the output of `singularity cache clean --help` for more information. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::::: callout + +## Cache location + +By default, Singularity uses `$HOME/.singularity/cache` as the location for the cache. You can change the location of the cache by setting the `SINGULARITY_CACHEDIR` environment variable to the cache location you want to use. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Singularity caches downloaded images so that an unchanged image isn't downloaded again when it is requested using the `singularity pull` command. +- You can free up space in the cache by removing all locally cached images or by specifying individual images to remove. -This provides us with some more useful information about the actual images stored in the cache. In the `TYPE` column we can see that our image type is `shub` because it's a `SIF` image that has been pulled from Singularity Hub. +:::::::::::::::::::::::::::::::::::::::::::::::::: -> ## Cleaning the Singularity image cache -> We can remove images from the cache using the `singularity cache clean` command. Running the command without any options will display a warning and ask you to confirm that you want to remove everything from your cache. -> -> You can also remove specific images or all images of a particular type. Look at the output of `singularity cache clean --help` for more information. -{: .callout} -> ## Cache location -> By default, Singularity uses `$HOME/.singularity/cache` as the location for the cache. You can change the location of the cache by setting the `SINGULARITY_CACHEDIR` environment variable to the cache location you want to use. -{: .callout} diff --git a/episodes/03-singularity-shell.md b/episodes/03-singularity-shell.md index 7e6def6..646270b 100644 --- a/episodes/03-singularity-shell.md +++ b/episodes/03-singularity-shell.md @@ -1,55 +1,62 @@ --- -title: "Using Singularity containers to run commands" +title: Using Singularity containers to run commands teaching: 10 exercises: 5 -questions: -- "How do I run different commands within a container?" -- "How do I access an interactive shell within a container?" -objectives: -- "Learn how to run different commands when starting a container." -- "Learn how to open an interactive shell within a container environment." -keypoints: -- "The `singularity exec` is an alternative to `singularity run` that allows you to start a container running a specific command." -- "The `singularity shell` command can be used to start a container and run an interactive shell within it." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Learn how to run different commands when starting a container. +- Learn how to open an interactive shell within a container environment. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How do I run different commands within a container? +- How do I access an interactive shell within a container? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + ## Running specific commands within a container We saw earlier that we can use the `singularity inspect` command to see the run script that a container is configured to run by default. What if we want to run a different command within a container? If we know the path of an executable that we want to run within a container, we can use the `singularity exec` command. For example, using the `hello-world.sif` container that we've already pulled from Singularity Hub, we can run the following within the `test` directory where the `hello-world.sif` file is located: -~~~ +```bash $ singularity exec hello-world.sif /bin/echo Hello World! -~~~ -{: .language-bash} +``` -~~~ +```output Hello World! -~~~ -{: .output} +``` Here we see that a container has been started from the `hello-world.sif` image and the `/bin/echo` command has been run within the container, passing the input `Hello World!`. The command has echoed the provided input to the console and the container has terminated. Note that the use of `singularity exec` has overriden any run script set within the image metadata and the command that we specified as an argument to `singularity exec` has been run instead. -> ## Basic exercise: Running a different command within the "hello-world" container -> -> Can you run a container based on the `hello-world.sif` image that **prints the current date and time**? -> -> > ## Solution -> > -> > ~~~ -> > $ singularity exec hello-world.sif /bin/date -> > ~~~ -> > {: .language-bash} -> > -> > ~~~ -> > Fri Jun 26 15:17:44 BST 2020 -> > ~~~ -> > {: .output} -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::::: challenge + +## Basic exercise: Running a different command within the "hello-world" container + +Can you run a container based on the `hello-world.sif` image that **prints the current date and time**? + +::::::::::::::: solution + +## Solution + +```bash +$ singularity exec hello-world.sif /bin/date +``` + +```output +Fri Jun 26 15:17:44 BST 2020 +``` + +::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::::::::::::::
    #### **The difference between `singularity run` and `singularity exec`** @@ -58,47 +65,56 @@ Above we used the `singularity exec` command. In earlier episodes of this course we used `singularity run`. To clarify, the difference between these two commands is: - - `singularity run`: This will run the default command set for containers - based on the specfied image. This default command is set within - the image metadata when the image is built (we'll see more about this - in later episodes). You do not specify a command to run when using - `singularity run`, you simply specify the image file name. As we saw - earlier, you can use the `singularity inspect` command to see what command - is run by default when starting a new container based on an image. +- `singularity run`: This will run the default command set for containers + based on the specfied image. This default command is set within + the image metadata when the image is built (we'll see more about this + in later episodes). You do not specify a command to run when using + `singularity run`, you simply specify the image file name. As we saw + earlier, you can use the `singularity inspect` command to see what command + is run by default when starting a new container based on an image. - - `singularity exec`: This will start a container based on the specified - image and run the command provided on the command line following - `singularity exec `. This will override any default - command specified within the image metadata that would otherwise be - run if you used `singularity run`. +- `singularity exec`: This will start a container based on the specified + image and run the command provided on the command line following + `singularity exec `. This will override any default + command specified within the image metadata that would otherwise be + run if you used `singularity run`. ## Opening an interactive shell within a container If you want to open an interactive shell within a container, Singularity provides the `singularity shell` command. Again, using the `hello-world.sif` image, and within our `test` directory, we can run a shell within a container from the hello-world image: -~~~ +```bash $ singularity shell hello-world.sif -~~~ -{: .language-bash} +``` -~~~ +```output Singularity> whoami [] Singularity> ls hello-world.sif Singularity> -~~~ -{: .output} +``` As shown above, we have opened a shell in a new container started from the `hello-world.sif` image. Note that the shell prompt has changed to show we are now within the Singularity container. -> ## Discussion: Running a shell inside a Singularity container -> -> Q: What do you notice about the output of the above commands entered within the Singularity container shell? -> -> Q: Does this differ from what you might see within a Docker container? -{: .discussion} +:::::::::::::::::::::::::::::::::::::: discussion + +## Discussion: Running a shell inside a Singularity container + +Q: What do you notice about the output of the above commands entered within the Singularity container shell? + +Q: Does this differ from what you might see within a Docker container? + + +:::::::::::::::::::::::::::::::::::::::::::::::::: Use the `exit` command to exit from the container shell. +:::::::::::::::::::::::::::::::::::::::: keypoints + +- The `singularity exec` is an alternative to `singularity run` that allows you to start a container running a specific command. +- The `singularity shell` command can be used to start a container and run an interactive shell within it. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + diff --git a/episodes/04-singularity-files.md b/episodes/04-singularity-files.md index 1cc63ec..8c709ef 100644 --- a/episodes/04-singularity-files.md +++ b/episodes/04-singularity-files.md @@ -1,35 +1,38 @@ --- -title: "Files in Singularity containers" +title: Files in Singularity containers teaching: 10 exercises: 10 -questions: -- "How do I make data available in a Singularity container?" -- "What data is made available by default in a Singularity container?" -objectives: -- "Understand that some data from the host system is usually made available by default within a container" -- "Learn more about how Singularity handles users and binds directories from the host filesystem." -keypoints: -- "Your current directory and home directory are usually available by default in a container." -- "You have the same username and permissions in a container as on the host system." -- "You can specify additional host system directories to be available in the container." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Understand that some data from the host system is usually made available by default within a container +- Learn more about how Singularity handles users and binds directories from the host filesystem. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How do I make data available in a Singularity container? +- What data is made available by default in a Singularity container? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + The way in which user accounts and access permissions are handeld in Singularity containers is very different from that in Docker (where you effectively always have superuser/root access). When running a Singularity container, you only have the same permissions to access files as the user you are running as on the host system. In this episode we'll look at working with files in the context of Singularity containers and how this links with Singularity's approach to users and permissions within containers. ## Users within a Singularity container -The first thing to note is that when you ran `whoami` within the container shell you started at the end of the previous episode, you should have seen the username that you were signed in as on the host system when you ran the container. +The first thing to note is that when you ran `whoami` within the container shell you started at the end of the previous episode, you should have seen the username that you were signed in as on the host system when you ran the container. For example, if my username were `jc1000`, I'd expect to see the following: -~~~ +```bash $ singularity shell hello-world.sif Singularity> whoami jc1000 -~~~ -{: .language-bash} +``` But hang on! I downloaded the standard, public version of the `hello-world.sif` image from Singularity Hub. I haven't customised it in any way. How is it configured with my own user details?! @@ -41,15 +44,15 @@ This means that the host system can effectively ensure that you cannot access/mo ## Files and directories within a Singularity container -Singularity also _binds_ some _directories_ from the host system where you are running the `singularity` command into the container that you're starting. Note that this bind process is not copying files into the running container, it is making an existing directory on the host system visible and accessible within the container environment. If you write files to this directory within the running container, when the container shuts down, those changes will persist in the relevant location on the host system. +Singularity also *binds* some *directories* from the host system where you are running the `singularity` command into the container that you're starting. Note that this bind process is not copying files into the running container, it is making an existing directory on the host system visible and accessible within the container environment. If you write files to this directory within the running container, when the container shuts down, those changes will persist in the relevant location on the host system. There is a default configuration of which files and directories are bound into the container but ultimate control of how things are set up on the system where you are running Singularity is determined by the system administrator. As a result, this section provides an overview but you may find that things are a little different on the system that you're running on. -One directory that is likely to be accessible within a container that you start is your _home directory_. You may also find that the directory from which you issued the `singularity` command (the _current working directory_) is also mapped. +One directory that is likely to be accessible within a container that you start is your *home directory*. You may also find that the directory from which you issued the `singularity` command (the *current working directory*) is also mapped. The mapping of file content and directories from a host system into a Singularity container is illustrated in the example below showing a subset of the directories on the host Linux system and in a Singularity container: -~~~ +```output Host system: Singularity container: ------------- ---------------------- / / @@ -64,28 +67,35 @@ Host system: Singularity co ├── sbin └────────>└── jc1000 └── ... └── ... -~~~ -{: .output} - -> ## Questions and exercises: Files in Singularity containers -> -> **Q1:** What do you notice about the ownership of files in a container started from the hello-world image? (e.g. take a look at the ownership of files in the root directory (`/`)) -> -> **Exercise 1:** In this container, try editing (for example using the editor `vi` which should be avaiable in the container) the `/rawr.sh` file. What do you notice? -> -> _If you're not familiar with `vi` there are many quick reference pages online showing the main commands for using the editor, for example [this one](http://web.mit.edu/merolish/Public/vi-ref.pdf)._ -> -> **Exercise 2:** In your home directory within the container shell, try and create a simple text file. Is it possible to do this? If so, why? If not, why not?! If you can successfully create a file, what happens to it when you exit the shell and the container shuts down? -> -> > ## Answers -> > -> > **A1:** Use the `ls -l` command to see a detailed file listing including file ownership and permission details. You should see that most of the files in the `/` directory are owned by `root`, as you'd probably expect on any Linux system. If you look at the files in your home directory, they should be owned by you. -> > -> > **A Ex1:** We've already seen from the previous answer that the files in `/` are owned by `root` so we wouldn't expect to be able to edit them if we're not the root user. However, if you tried to edit `/rawr.sh` you probably saw that the file was read only and, if you tried for example to delete the file you would have seen an error similar to the following: `cannot remove '/rawr.sh': Read-only file system`. This tells us something else about the filesystem. It's not just that we don't have permission to delete the file, the filesystem itself is read-only so even the `root` user wouldn't be able to edit/delete this file. We'll look at this in more detail shortly. -> > -> > **A Ex2:** Within your home directory, you _should_ be able to successfully create a file. Since you're seeing your home directory on the host system which has been bound into the container, when you exit and the container shuts down, the file that you created within the container should still be present when you look at your home directory on the host system. -> {: .solution} -{: .challenge} +``` + +::::::::::::::::::::::::::::::::::::::: challenge + +## Questions and exercises: Files in Singularity containers + +**Q1:** What do you notice about the ownership of files in a container started from the hello-world image? (e.g. take a look at the ownership of files in the root directory (`/`)) + +**Exercise 1:** In this container, try editing (for example using the editor `vi` which should be avaiable in the container) the `/rawr.sh` file. What do you notice? + +*If you're not familiar with `vi` there are many quick reference pages online showing the main commands for using the editor, for example [this one](https://web.mit.edu/merolish/Public/vi-ref.pdf).* + +**Exercise 2:** In your home directory within the container shell, try and create a simple text file. Is it possible to do this? If so, why? If not, why not?! If you can successfully create a file, what happens to it when you exit the shell and the container shuts down? + +::::::::::::::: solution + +## Answers + +**A1:** Use the `ls -l` command to see a detailed file listing including file ownership and permission details. You should see that most of the files in the `/` directory are owned by `root`, as you'd probably expect on any Linux system. If you look at the files in your home directory, they should be owned by you. + +**A Ex1:** We've already seen from the previous answer that the files in `/` are owned by `root` so we wouldn't expect to be able to edit them if we're not the root user. However, if you tried to edit `/rawr.sh` you probably saw that the file was read only and, if you tried for example to delete the file you would have seen an error similar to the following: `cannot remove '/rawr.sh': Read-only file system`. This tells us something else about the filesystem. It's not just that we don't have permission to delete the file, the filesystem itself is read-only so even the `root` user wouldn't be able to edit/delete this file. We'll look at this in more detail shortly. + +**A Ex2:** Within your home directory, you *should* be able to successfully create a file. Since you're seeing your home directory on the host system which has been bound into the container, when you exit and the container shuts down, the file that you created within the container should still be present when you look at your home directory on the host system. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: ## Binding additional host system directories to the container @@ -96,12 +106,12 @@ You will sometimes need to bind additional host system directories into a contai The `-B` option to the `singularity` command is used to specify additonal binds. For example, to bind the `/work/z19/shared` directory into a container you could use (note this directory is unlikely to exist on the host system you are using so you'll need to test this using a different directory): -``` +```bash $ singularity shell -B /work/z19/shared hello-world.sif Singularity> ls /work/z19/shared ``` -{: .language-bash} -``` + +```output CP2K-regtest cube eleanor image256x192.pgm kevin pblas q-e-qe-6.7 ebe evince.simg image512x384.pgm low_priority.slurm pblas.tar.gz q-qe Q1529568 edge192x128.pgm extrae image768x1152.pgm mkdir petsc regtest-ls-rtp_forCray @@ -109,16 +119,15 @@ adrianj edge256x192.pgm gnuplot-5.4.1.tar.gz image768x768.pgm moose.job antlr-2.7.7.tar.gz edge512x384.pgm hj job-defmpi-cpe-21.03-robust mrb4cab petsc-hypre-cpe21.03 xios-2.5 cdo-archer2.sif edge768x768.pgm image192x128.pgm jsindt paraver petsc-hypre-cpe21.03-gcc10.2.0 ``` -{: .output} Note that, by default, a bind is mounted at the same path in the container as on the host system. You can also specify where a host directory is mounted in the container by separating the host path from the container path by a colon (`:`) in the option: -``` +```bash $ singularity shell -B /work/z19/shared:/shared-data hello-world.sif Singularity> ls /shared-data ``` -{: .language-bash} -``` + +```output CP2K-regtest cube eleanor image256x192.pgm kevin pblas q-e-qe-6.7 ebe evince.simg image512x384.pgm low_priority.slurm pblas.tar.gz q-qe Q1529568 edge192x128.pgm extrae image768x1152.pgm mkdir petsc regtest-ls-rtp_forCray @@ -126,7 +135,6 @@ adrianj edge256x192.pgm gnuplot-5.4.1.tar.gz image768x768.pgm moose.job antlr-2.7.7.tar.gz edge512x384.pgm hj job-defmpi-cpe-21.03-robust mrb4cab petsc-hypre-cpe21.03 xios-2.5 cdo-archer2.sif edge768x768.pgm image192x128.pgm jsindt paraver petsc-hypre-cpe21.03-gcc10.2.0 ``` -{: .output} You can also specify multiple binds to `-B` by separating them by commas (`,`). @@ -134,4 +142,14 @@ You can also copy data into a container image at build time if there is some sta ## References -\[1\] Gregory M. Kurzer, Containers for Science, Reproducibility and Mobility: Singularity P2. Intel HPC Developer Conference, 2017. Available at: https://www.intel.com/content/dam/www/public/us/en/documents/presentation/hpc-containers-singularity-advanced.pdf +\[1\] Gregory M. Kurzer, Containers for Science, Reproducibility and Mobility: Singularity P2. Intel HPC Developer Conference, 2017. Available at: [https://www.intel.com/content/dam/www/public/us/en/documents/presentation/hpc-containers-singularity-advanced.pdf](https://www.intel.com/content/dam/www/public/us/en/documents/presentation/hpc-containers-singularity-advanced.pdf) + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Your current directory and home directory are usually available by default in a container. +- You have the same username and permissions in a container as on the host system. +- You can specify additional host system directories to be available in the container. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/05-singularity-docker.md b/episodes/05-singularity-docker.md index 3a06f7e..39157eb 100644 --- a/episodes/05-singularity-docker.md +++ b/episodes/05-singularity-docker.md @@ -1,29 +1,34 @@ --- -title: "Using Docker images with Singularity" +title: Using Docker images with Singularity teaching: 5 exercises: 10 -questions: -- "How do I use Docker images with Singularity?" -objectives: -- "Learn how to run Singularity containers based on Docker images." -keypoints: -- "Singularity can start a container from a Docker image which can be pulled directly from Docker Hub." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Learn how to run Singularity containers based on Docker images. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How do I use Docker images with Singularity? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + ## Using Docker images with Singularity Singularity can also start containers directly from Docker images, opening up access to a huge number of existing container images available on [Docker Hub](https://hub.docker.com/) and other registries. -While Singularity doesn't actually run a container using the Docker image (it first converts it to a format suitable for use by Singularity), the approach used provides a seamless experience for the end user. When you direct Singularity to run a container based on pull a Docker image, Singularity pulls the slices or _layers_ that make up the Docker image and converts them into a single-file Singularity SIF image. +While Singularity doesn't actually run a container using the Docker image (it first converts it to a format suitable for use by Singularity), the approach used provides a seamless experience for the end user. When you direct Singularity to run a container based on pull a Docker image, Singularity pulls the slices or *layers* that make up the Docker image and converts them into a single-file Singularity SIF image. -For example, moving on from the simple _Hello World_ examples that we've looked at so far, let's pull one of the [official Docker Python images](https://hub.docker.com/_/python). We'll use the image with the tag `3.9.6-slim-buster` which has Python 3.9.6 installed on Debian's [Buster](https://www.debian.org/releases/buster/) (v10) Linux distribution: +For example, moving on from the simple *Hello World* examples that we've looked at so far, let's pull one of the [official Docker Python images](https://hub.docker.com/_/python). We'll use the image with the tag `3.9.6-slim-buster` which has Python 3.9.6 installed on Debian's [Buster](https://www.debian.org/releases/buster/) (v10) Linux distribution: -~~~ +```bash $ singularity pull python-3.9.6.sif docker://python:3.9.6-slim-buster -~~~ -{: .language-bash} +``` -~~~ +```output INFO: Converting OCI blobs to SIF format INFO: Starting build... Getting image source signatures @@ -41,100 +46,118 @@ Storing signatures 2021/07/27 17:23:40 info unpack layer: sha256:248d7d56b4a792ca7bdfe866fde773a9cf2028f973216160323684ceabb36451 2021/07/27 17:23:40 info unpack layer: sha256:478d2dfa1a8d7fc4d9957aca29ae4f4187bc2e5365400a842aaefce8b01c2658 INFO: Creating SIF file... -~~~ -{: .output} +``` -Note how we see singularity saying that it's "_Converting OCI blobs to SIF format_". We then see the layers of the Docker image being downloaded and unpacked and written into a single SIF file. Once the process is complete, we should see the python-3.9.6.sif image file in the current directory. +Note how we see singularity saying that it's "*Converting OCI blobs to SIF format*". We then see the layers of the Docker image being downloaded and unpacked and written into a single SIF file. Once the process is complete, we should see the python-3.9.6.sif image file in the current directory. We can now run a container from this image as we would with any other singularity image. -> ## Running the Python 3.9.6 image that we just pulled from Docker Hub -> -> Try running the Python 3.9.6 image. What happens? -> -> Try running some simple Python statements... -> -> > ## Running the Python 3.9.6 image -> > -> > ~~~ -> > $ singularity run python-3.9.6.sif -> > ~~~ -> > {: .language-bash} -> > -> > This should put you straight into a Python interactive shell within the running container: -> > -> > ~~~ -> > Python 3.9.6 (default, Jul 22 2021, 15:24:21) -> > [GCC 8.3.0] on linux -> > Type "help", "copyright", "credits" or "license" for more information. -> > >>> -> > ~~~ -> > Now try running some simple Python statements: -> > ~~~ -> > >>> import math -> > >>> math.pi -> > 3.141592653589793 -> > >>> -> > ~~~ -> > {: .language-python} -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::::: challenge + +## Running the Python 3.9.6 image that we just pulled from Docker Hub + +Try running the Python 3.9.6 image. What happens? + +Try running some simple Python statements... + +::::::::::::::: solution + +## Running the Python 3.9.6 image + +```bash +$ singularity run python-3.9.6.sif +``` + +This should put you straight into a Python interactive shell within the running container: + +``` +Python 3.9.6 (default, Jul 22 2021, 15:24:21) +[GCC 8.3.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> +``` + +Now try running some simple Python statements: + +```python +>>> import math +>>> math.pi +3.141592653589793 +>>> +``` + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: In addition to running a container and having it run the default run script, you could also start a container running a shell in case you want to undertake any configuration prior to running Python. This is covered in the following exercise: -> ## Open a shell within a Python container -> -> Try to run a shell within a singularity container based on the `python-3.9.6.sif` image. That is, run a container that opens a shell rather than the default Python interactive console as we saw above. -> See if you can find more than one way to achieve this. -> -> Within the shell, try starting the Python interactive console and running some Python commands. -> -> > ## Solution -> > -> > Recall from the earlier material that we can use the `singularity shell` command to open a shell within a container. To open a regular shell within a container based on the `python-3.9.6.sif` image, we can therefore simply run: -> > ~~~ -> > $ singularity shell python-3.9.6.sif -> > ~~~ -> > {: .language-bash} -> > -> > ~~~ -> > Singularity> echo $SHELL -> > /bin/bash -> > Singularity> cat /etc/issue -> > Debian GNU/Linux 10 \n \l -> > -> > Singularity> python -> > Python 3.9.6 (default, Jul 22 2021, 15:24:21) -> > [GCC 8.3.0] on linux -> > Type "help", "copyright", "credits" or "license" for more information. -> > >>> print('Hello World!') -> > Hello World! -> > >>> exit() -> > -> > Singularity> exit -> > $ -> > ~~~ -> > {: .output} -> > -> > It is also possible to use the `singularity exec` command to run an executable within a container. We could, therefore, use the `exec` command to run `/bin/bash`: -> > -> > ~~~ -> > $ singularity exec python-3.9.6.sif /bin/bash -> > ~~~ -> > {: .language-bash} -> > -> > ~~~ -> > Singularity> echo $SHELL -> > /bin/bash -> > ~~~ -> > {: .output} -> > -> > You can run the Python console from your container shell simply by running the `python` command. -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::::: challenge + +## Open a shell within a Python container + +Try to run a shell within a singularity container based on the `python-3.9.6.sif` image. That is, run a container that opens a shell rather than the default Python interactive console as we saw above. +See if you can find more than one way to achieve this. + +Within the shell, try starting the Python interactive console and running some Python commands. + +::::::::::::::: solution + +## Solution + +Recall from the earlier material that we can use the `singularity shell` command to open a shell within a container. To open a regular shell within a container based on the `python-3.9.6.sif` image, we can therefore simply run: + +```bash +$ singularity shell python-3.9.6.sif +``` + +```output +Singularity> echo $SHELL +/bin/bash +Singularity> cat /etc/issue +Debian GNU/Linux 10 \n \l + +Singularity> python +Python 3.9.6 (default, Jul 22 2021, 15:24:21) +[GCC 8.3.0] on linux +Type "help", "copyright", "credits" or "license" for more information. +>>> print('Hello World!') +Hello World! +>>> exit() + +Singularity> exit +$ +``` + +It is also possible to use the `singularity exec` command to run an executable within a container. We could, therefore, use the `exec` command to run `/bin/bash`: + +```bash +$ singularity exec python-3.9.6.sif /bin/bash +``` + +```output +Singularity> echo $SHELL +/bin/bash +``` + +You can run the Python console from your container shell simply by running the `python` command. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: This concludes the fifth episode and Part I of the Singularity material. Part II contains a further three episodes where we'll look at creating your own images and then more advanced use of containers for running MPI parallel applications. ## References -\[1\] Gregory M. Kurzer, Containers for Science, Reproducibility and Mobility: Singularity P2. Intel HPC Developer Conference, 2017. Available at: https://www.intel.com/content/dam/www/public/us/en/documents/presentation/hpc-containers-singularity-advanced.pdf +\[1\] Gregory M. Kurzer, Containers for Science, Reproducibility and Mobility: Singularity P2. Intel HPC Developer Conference, 2017. Available at: [https://www.intel.com/content/dam/www/public/us/en/documents/presentation/hpc-containers-singularity-advanced.pdf](https://www.intel.com/content/dam/www/public/us/en/documents/presentation/hpc-containers-singularity-advanced.pdf) + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Singularity can start a container from a Docker image which can be pulled directly from Docker Hub. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/06-singularity-images-prep.md b/episodes/06-singularity-images-prep.md index 46fa776..d8ba3a9 100644 --- a/episodes/06-singularity-images-prep.md +++ b/episodes/06-singularity-images-prep.md @@ -1,19 +1,23 @@ --- -title: "Preparing to build Singularity images" +title: Preparing to build Singularity images teaching: 15 exercises: 20 -questions: -- "What environment do I need to build a Singularity image and how do I set it up?" -objectives: -- "Understand how to the Docker Singularity image provides an environment for building Singularity images." -- "Understand different ways to run containers based on the Docker Singularity image." -keypoints: -- "A Docker image is provided to run Singularity - this avoids the need to have a local Singularity installation on your system." -- "The Docker Singularity image can be used to build containers on Linux, macOS and Windows." -- "You can also run Singularity containers within the Docker Singularity image." --- -# Singularity - Part II +::::::::::::::::::::::::::::::::::::::: objectives + +- Understand how to the Docker Singularity image provides an environment for building Singularity images. +- Understand different ways to run containers based on the Docker Singularity image. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- What environment do I need to build a Singularity image and how do I set it up? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +## Singularity - Part II ## Brief recap @@ -25,118 +29,144 @@ In this first of three episodes in Part II of the Singularity material, we'll lo ## Preparing to use Singularity for building images -So far you've been able to work with Singularity from your own user account as a non-privileged user. This part of the Singularity material requires that you use Singularity in an environment where you have administrative (root) access. While it is possible to build Singularity containers without root access, it is highly recommended that you do this as the _root_ user, as highlighted in [this section](https://sylabs.io/guides/3.5/user-guide/build_a_container.html#creating-writable-sandbox-directories) of the Singularity documentation. Bear in mind that the system that you use to build containers doesn't have to be the system where you intend to run the containers. If, for example, you are intending to build a container that you can subsequently run on a Linux-based cluster, you could build the container on your own Linux-based desktop or laptop computer. You could then transfer the built image directly to the target platform or upload it to an image repository and pull it onto the target platform from this repository. +So far you've been able to work with Singularity from your own user account as a non-privileged user. This part of the Singularity material requires that you use Singularity in an environment where you have administrative (root) access. While it is possible to build Singularity containers without root access, it is highly recommended that you do this as the *root* user, as highlighted in [this section](https://sylabs.io/guides/3.5/user-guide/build_a_container.html#creating-writable-sandbox-directories) of the Singularity documentation. Bear in mind that the system that you use to build containers doesn't have to be the system where you intend to run the containers. If, for example, you are intending to build a container that you can subsequently run on a Linux-based cluster, you could build the container on your own Linux-based desktop or laptop computer. You could then transfer the built image directly to the target platform or upload it to an image repository and pull it onto the target platform from this repository. There are **three** different options for accessing a suitable environment to undertake the material in this part of the course: - 1. Run Singularity from within a Docker container - this will enable you to have the required privileges to build images - 1. Install Singularity locally on a system where you have administrative access - 1. Use Singularity on a system where it is already pre-installed and you have administrative (root) access - -We'll focus on the first option in this part of the course - _running singularity from within a Docker container_. If you would like to install Singularity directly on your system, see the box below for some further pointers. However, please note that the installation process is an advanced task that is beyond the scope of this course so we won't be covering this. - -> ## Installing Singularity on your local system (optional) \[Advanced task\] -> -> If you are running Linux and would like to install Singularity locally on your system, the source code is provided via the [The Next Generation of High Performance Computing (HPCng) community](https://github.com/hpcng)'s [Singularity repository](https://github.com/hpcng/singularity). See the releases [here](https://github.com/hpcng/singularity/releases). You will need to install various dependencies on your system and then build Singularity from source code. -> -> _If you are not familiar with building applications from source code, it is strongly recommended that you use the Docker Singularity image, as described below in the "Getting started with the Docker Singularity image" section rather than attempting to build and install Singularity yourself. The installation process is an advanced task that is beyond the scope of this session._ -> -> However, if you have Linux systems knowledge and would like to attempt a local install of Singularity, you can find details in the [INSTALL.md](https://github.com/hpcng/singularity/blob/master/INSTALL.md) file within the Singularity repository that explains how to install the prerequisites and build and install the software. Singularity is written in the [Go](https://golang.org/) programming language and Go is the main dependency that you'll need to install on your system. The process of installing Go and any other requirements is detailed in the INSTALL.md file. -> -{: .callout} - -> ## Note -> If you do not have access to a system with Docker installed, or a Linux system where you can build and install Singularity but you have administrative privileges on another system, you could look at installing a virtualisation tool such as [VirtualBox](https://www.virtualbox.org/) on which you could run a Linux Virtual Machine (VM) image. Within the Linux VM image, you will be able to install Singularity. Again this is beyond the scope of the course. -> -> If you are not able to access/run Singularity yourself on a system where you have administrative privileges, you can still follow through this material as it is being taught (or read through it in your own time if you're not participating in a taught version of the course) since it will be helpful to have an understanding of how Singularity images can be built. -> -> You could also attempt to follow this section of the lesson without using root and instead using the `singularity` command's [`--fakeroot`](https://sylabs.io/guides/3.5/user-guide/fakeroot.html) option. However, you may encounter issues with permissions when trying to build images and run your containers and this is why running the commands as root is strongly recommended and is the approach described in this lesson. -{: .callout} +1. Run Singularity from within a Docker container - this will enable you to have the required privileges to build images +2. Install Singularity locally on a system where you have administrative access +3. Use Singularity on a system where it is already pre-installed and you have administrative (root) access + +We'll focus on the first option in this part of the course - *running singularity from within a Docker container*. If you would like to install Singularity directly on your system, see the box below for some further pointers. However, please note that the installation process is an advanced task that is beyond the scope of this course so we won't be covering this. + +::::::::::::::::::::::::::::::::::::::::: callout + +## Installing Singularity on your local system (optional) \[Advanced task\] + +If you are running Linux and would like to install Singularity locally on your system, the source code is provided via the [The Next Generation of High Performance Computing (HPCng) community](https://github.com/hpcng)'s [Singularity repository](https://github.com/hpcng/singularity). See the releases [here](https://github.com/hpcng/singularity/releases). You will need to install various dependencies on your system and then build Singularity from source code. + +*If you are not familiar with building applications from source code, it is strongly recommended that you use the Docker Singularity image, as described below in the "Getting started with the Docker Singularity image" section rather than attempting to build and install Singularity yourself. The installation process is an advanced task that is beyond the scope of this session.* + +However, if you have Linux systems knowledge and would like to attempt a local install of Singularity, you can find details in the [INSTALL.md](https://github.com/hpcng/singularity/blob/master/INSTALL.md) file within the Singularity repository that explains how to install the prerequisites and build and install the software. Singularity is written in the [Go](https://golang.org/) programming language and Go is the main dependency that you'll need to install on your system. The process of installing Go and any other requirements is detailed in the INSTALL.md file. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::::: callout + +## Note + +If you do not have access to a system with Docker installed, or a Linux system where you can build and install Singularity but you have administrative privileges on another system, you could look at installing a virtualisation tool such as [VirtualBox](https://www.virtualbox.org/) on which you could run a Linux Virtual Machine (VM) image. Within the Linux VM image, you will be able to install Singularity. Again this is beyond the scope of the course. + +If you are not able to access/run Singularity yourself on a system where you have administrative privileges, you can still follow through this material as it is being taught (or read through it in your own time if you're not participating in a taught version of the course) since it will be helpful to have an understanding of how Singularity images can be built. + +You could also attempt to follow this section of the lesson without using root and instead using the `singularity` command's [`--fakeroot`](https://sylabs.io/guides/3.5/user-guide/fakeroot.html) option. However, you may encounter issues with permissions when trying to build images and run your containers and this is why running the commands as root is strongly recommended and is the approach described in this lesson. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: ## Getting started with the Docker Singularity image The [Singularity Docker image](https://quay.io/repository/singularity/singularity) is available from [Quay.io](https://quay.io/). -> ## Familiarise yourself with the Docker Singularity image -> - Using your previously acquired Docker knowledge, get the Singularity image for `v3.5.3` and ensure that you can run a Docker container using this image. For this exercise, we recommend using the image with the `v3.5.3-slim` tag since it's a much smaller image. -> -> - Create a directory (e.g. `$HOME/singularity_data`) on your host machine that you can use for storage of _definition files_ (we'll introduce these shortly) and generated image files. -> -> This directory should be bind mounted into the Docker container at the location `/home/singularity` every time you run it - this will give you a location in which to store built images so that they are available on the host system once the container exits. (take a look at the `-v` switch to the `docker run` command) -> -> _Hint: To be able to build an image using the Docker Singularity container, you'll need to add the `--privileged` switch to your docker command line._ -> -> _Hint: If you want to run a shell within the Docker Singularity container, you'll need to override the entrypoint to tell the container to run `/bin/bash` - take a look at Docker's `--entrypoint` switch._ -> -> -> Questions / Exercises: -> -> 1. Can you run a container from the Docker Singularity image? What is happening when you run the container? -> 1. Can you run an interactive `/bin/sh` shell in the Docker Singularity container? -> 1. Can you run an interactive Singularity shell in a Singularity container, within the Docker Singularity container?! -> -> > ## Running a container from the image -> > Answers: -> > 1. _Can you run a container from the Docker Singularity image? What is happening when you run the container?_ -> > -> > The name/tag of the Docker Singularity image we'll be using is: `quay.io/singularity/singularity:v3.5.3-slim` -> > -> > Having a bound directory from the host system accessible within your running Docker Singularity container will give you somewhere to place created Singularity images so that they are accessible on the host system after the container exits. Begin by changing into the directory that you created above for storing your definiton files and built images (e.g. `$HOME/singularity_data`). -> > -> > Running a Docker container from the image and binding the current directory to `/home/singularity` within the container can be achieved as follows: -> > -> > ``` -> > docker run --privileged --rm -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim -> > ``` -> > Note that the image is configured to run the `singularity` command by default. So, when you run a container from it with no arguments, you see the singularity help output as if you had Singularity installed locally and had typed `singularity` on the command line. -> > -> > To run a Singularity command, such as `singularity cache list`, within the docker container directly from the host system's terminal you'd enter: -> > -> > ``` -> > docker run --privileged --rm -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim cache list -> > ``` -> > -> > The following diagram shows how the Docker Singularity image is being used to run a container on your host system and how a Singularity container can, in turn, be started within the Docker container: -> > -> > ![](/fig/SingularityInDocker.png) -> > -> > 1. _Can you run an interactive shell in the Docker Singularity container?_ -> > -> > To start a shell within the Singularity Docker container where you can then run the `singularity` command directly: -> > ``` -> > docker run -it --entrypoint=/bin/sh --privileged --rm -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim -> > ``` -> > Here we use the `--entrypoint` switch to the `docker` command to override the default behaviour when starting the container and instead of running the `singularity` command directly, we run a 'sh' shell. We also add the `-it` switch to provide an interactive terminal connection. -> > -> > -> > 1. _Can you run an interactive Singularity shell in a Singularity container, within the Docker Singularity container?!_ -> > -> > As shown in the diagram above, you can do this. It is necessary to run `singularity shell ` within the Docker Singularity container. You would use a command similar to the following (assuming that `my_test_image.sif` is in the current directory where you run this command): -> > -> > ``` -> > docker run --rm -it --privileged -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim shell --contain /home/singularity/my_test_image.sif -> > ``` -> > -> > You may notice there's a flag being passed to singularity shell (`--contain` - `-c` is the short form and also works). What is this doing? When running a singularity container, you may remember that we highlighted that some key files/directories from the host system are mapped into containers by default when you start them. The configuration in the Docker Singularity container attempts to mount the file `/etc/localtime` into the Singularity container but there is not a timezone configuration present in the Docker Singularity container and this file doesn't exist resulting in an error. `--contain` prevents the default mounting of some key files/directories into the container and prevents this error from occurring. Later in this material, there's an example of how to rectify the issue by creating a timezone configuration in the Docker Singularity container so that the `--contain` switch is no longer needed. -> > -> > _Summary / Comments:_ -> > -> > You may choose to: -> > - open a shell within the Docker image so you can work at a command prompt and run the `singularity` command directly -> > - use the `docker run` command to run a new container instance every time you want to run the `singularity` command (the Docker Singularity image is configured with the `singularity` command as its entrypoint). -> > -> > Either option is fine for this section of the material. -> > -> > To make things easier to read in the remainder of the material, command examples will use the `singularity` command directly, e.g. `singularity cache list`. If you're running a shell in the Docker Singularity container, you can enter the commands as they appear. If you're using the container's default run behaviour and running a container instance for each run of the command, you'll need to replace `singularity` with `docker run --privileged -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim` or similar. -> > -> > This can be a little cumbersome to work with. However, if you're using Linux or macOS on your host system, you can add a _command alias_ to alias the command `singularity` on your host system to run the Docker Singularity container, e.g. (for bash shells - syntax for other shells varies): -> > -> > ``` -> > alias singularity='docker run --privileged -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim' -> > ``` -> > -> > This means you'll only have to type `singularity` at the command line as shown in the examples throughout this section of the material -> > -> > -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::::: challenge + +## Familiarise yourself with the Docker Singularity image + +- Using your previously acquired Docker knowledge, get the Singularity image for `v3.5.3` and ensure that you can run a Docker container using this image. For this exercise, we recommend using the image with the `v3.5.3-slim` tag since it's a much smaller image. + +- Create a directory (e.g. `$HOME/singularity_data`) on your host machine that you can use for storage of *definition files* (we'll introduce these shortly) and generated image files. + + This directory should be bind mounted into the Docker container at the location `/home/singularity` every time you run it - this will give you a location in which to store built images so that they are available on the host system once the container exits. (take a look at the `-v` switch to the `docker run` command) + +*Hint: To be able to build an image using the Docker Singularity container, you'll need to add the `--privileged` switch to your docker command line.* + +*Hint: If you want to run a shell within the Docker Singularity container, you'll need to override the entrypoint to tell the container to run `/bin/bash` - take a look at Docker's `--entrypoint` switch.* + +Questions / Exercises: + +1. Can you run a container from the Docker Singularity image? What is happening when you run the container? +2. Can you run an interactive `/bin/sh` shell in the Docker Singularity container? +3. Can you run an interactive Singularity shell in a Singularity container, within the Docker Singularity container?! + +::::::::::::::: solution + +## Running a container from the image + +Answers: + +1. *Can you run a container from the Docker Singularity image? What is happening when you run the container?* + + The name/tag of the Docker Singularity image we'll be using is: `quay.io/singularity/singularity:v3.5.3-slim` + + Having a bound directory from the host system accessible within your running Docker Singularity container will give you somewhere to place created Singularity images so that they are accessible on the host system after the container exits. Begin by changing into the directory that you created above for storing your definiton files and built images (e.g. `$HOME/singularity_data`). + + Running a Docker container from the image and binding the current directory to `/home/singularity` within the container can be achieved as follows: + + ``` + docker run --privileged --rm -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim + ``` + + Note that the image is configured to run the `singularity` command by default. So, when you run a container from it with no arguments, you see the singularity help output as if you had Singularity installed locally and had typed `singularity` on the command line. + + To run a Singularity command, such as `singularity cache list`, within the docker container directly from the host system's terminal you'd enter: + + ``` + docker run --privileged --rm -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim cache list + ``` + + The following diagram shows how the Docker Singularity image is being used to run a container on your host system and how a Singularity container can, in turn, be started within the Docker container: + + ![](/fig/SingularityInDocker.png) + +2. *Can you run an interactive shell in the Docker Singularity container?* + + To start a shell within the Singularity Docker container where you can then run the `singularity` command directly: + + ``` + docker run -it --entrypoint=/bin/sh --privileged --rm -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim + ``` + + Here we use the `--entrypoint` switch to the `docker` command to override the default behaviour when starting the container and instead of running the `singularity` command directly, we run a 'sh' shell. We also add the `-it` switch to provide an interactive terminal connection. + +3. *Can you run an interactive Singularity shell in a Singularity container, within the Docker Singularity container?!* + + As shown in the diagram above, you can do this. It is necessary to run `singularity shell ` within the Docker Singularity container. You would use a command similar to the following (assuming that `my_test_image.sif` is in the current directory where you run this command): + + ``` + docker run --rm -it --privileged -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim shell --contain /home/singularity/my_test_image.sif + ``` + + You may notice there's a flag being passed to singularity shell (`--contain` - `-c` is the short form and also works). What is this doing? When running a singularity container, you may remember that we highlighted that some key files/directories from the host system are mapped into containers by default when you start them. The configuration in the Docker Singularity container attempts to mount the file `/etc/localtime` into the Singularity container but there is not a timezone configuration present in the Docker Singularity container and this file doesn't exist resulting in an error. `--contain` prevents the default mounting of some key files/directories into the container and prevents this error from occurring. Later in this material, there's an example of how to rectify the issue by creating a timezone configuration in the Docker Singularity container so that the `--contain` switch is no longer needed. + +*Summary / Comments:* + +You may choose to: + +- open a shell within the Docker image so you can work at a command prompt and run the `singularity` command directly +- use the `docker run` command to run a new container instance every time you want to run the `singularity` command (the Docker Singularity image is configured with the `singularity` command as its entrypoint). + +Either option is fine for this section of the material. + +To make things easier to read in the remainder of the material, command examples will use the `singularity` command directly, e.g. `singularity cache list`. If you're running a shell in the Docker Singularity container, you can enter the commands as they appear. If you're using the container's default run behaviour and running a container instance for each run of the command, you'll need to replace `singularity` with `docker run --privileged -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim` or similar. + +This can be a little cumbersome to work with. However, if you're using Linux or macOS on your host system, you can add a *command alias* to alias the command `singularity` on your host system to run the Docker Singularity container, e.g. (for bash shells - syntax for other shells varies): + +``` +alias singularity='docker run --privileged -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim' +``` + +This means you'll only have to type `singularity` at the command line as shown in the examples throughout this section of the material + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- A Docker image is provided to run Singularity - this avoids the need to have a local Singularity installation on your system. +- The Docker Singularity image can be used to build containers on Linux, macOS and Windows. +- You can also run Singularity containers within the Docker Singularity image. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/07-singularity-images-building.md b/episodes/07-singularity-images-building.md index 42f93c5..28b7dd8 100644 --- a/episodes/07-singularity-images-building.md +++ b/episodes/07-singularity-images-building.md @@ -1,18 +1,22 @@ --- -title: "Building Singularity images" +title: Building Singularity images teaching: 30 exercises: 30 -questions: -- "How do I create my own Singularity images?" -objectives: -- "Understand the different Singularity container file formats." -- "Understand how to build and share your own Singularity containers." -keypoints: -- "Singularity definition files are used to define the build process and configuration for an image." -- "Singularity's Docker container provides a way to build images on a platform where Singularity is not installed but Docker is available." -- "Existing images from remote registries such as Docker Hub and Singularity Hub can be used as a base for creating new Singularity images." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Understand the different Singularity container file formats. +- Understand how to build and share your own Singularity containers. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How do I create my own Singularity images? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + ## Building Singularity images ### Introduction @@ -25,31 +29,41 @@ Similarly to Docker and many other modern software tools, Singularity follows th There are various approaches to building Singularity images. We highlight two different approaches here and focus on one of them: - - _Building within a sandbox:_ You can build a container interactively within a sandbox environment. This means you get a shell within the container environment and install and configure packages and code as you wish before exiting the sandbox and converting it into a container image. -- _Building from a [Singularity Definition File](https://sylabs.io/guides/3.5/user-guide/build_a_container.html#creating-writable-sandbox-directories)_: This is Singularity's equivalent to building a Docker container from a `Dockerfile` and we'll discuss this approach in this section. +- *Building within a sandbox:* You can build a container interactively within a sandbox environment. This means you get a shell within the container environment and install and configure packages and code as you wish before exiting the sandbox and converting it into a container image. +- *Building from a [Singularity Definition File](https://sylabs.io/guides/3.5/user-guide/build_a_container.html#creating-writable-sandbox-directories)*: This is Singularity's equivalent to building a Docker container from a `Dockerfile` and we'll discuss this approach in this section. You can take a look at Singularity's "[Build a Container](https://sylabs.io/guides/3.5/user-guide/build_a_container.html)" documentation for more details on different approaches to building containers. -> ## Why look at Singularity Definition Files? -> Why do you think we might be looking at the _definition file approach_ here rather than the _sandbox approach_? -> -> > ## Discussion -> > The sandbox approach is great for prototyping and testing out an image configuration but it doesn't provide the best support for our ultimate goal of _reproducibility_. If you spend time sitting at your terminal in front of a shell typing different commands to add configuration, maybe you realise you made a mistake so you undo one piece of configuration and change it. This goes on until you have your completed, working configuration but there's no explicit record of exactly what you did to create that configuration. -> > -> > Say your container image file gets deleted by accident, or someone else wants to create an equivalent image to test something. How will they do this and know for sure that they have the same configuration that you had? -> > With a definition file, the configuration steps are explicitly defined and can be easily stored (and re-run). -> > -> > Definition files are small text files while container files may be very large, multi-gigabyte files that are difficult and time consuming to move around. This makes definition files ideal for storing in a version control system along with their revisions. -> {: .solution} -{: .challenge} +::::::::::::::::::::::::::::::::::::::: challenge + +## Why look at Singularity Definition Files? + +Why do you think we might be looking at the *definition file approach* here rather than the *sandbox approach*? + +::::::::::::::: solution + +## Discussion + +The sandbox approach is great for prototyping and testing out an image configuration but it doesn't provide the best support for our ultimate goal of *reproducibility*. If you spend time sitting at your terminal in front of a shell typing different commands to add configuration, maybe you realise you made a mistake so you undo one piece of configuration and change it. This goes on until you have your completed, working configuration but there's no explicit record of exactly what you did to create that configuration. + +Say your container image file gets deleted by accident, or someone else wants to create an equivalent image to test something. How will they do this and know for sure that they have the same configuration that you had? +With a definition file, the configuration steps are explicitly defined and can be easily stored (and re-run). + +Definition files are small text files while container files may be very large, multi-gigabyte files that are difficult and time consuming to move around. This makes definition files ideal for storing in a version control system along with their revisions. + + + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: ### Creating a Singularity Definition File -A Singularity Definition File is a text file that contains a series of statements that are used to create a container image. In line with the _configuration as code_ approach mentioned above, the definition file can be stored in your code repository alongside your application code and used to create a reproducible image. This means that for a given commit in your repository, the version of the definition file present at that commit can be used to reproduce a container with a known state. It was pointed out earlier in the course, when covering Docker, that this property also applies for Dockerfiles. +A Singularity Definition File is a text file that contains a series of statements that are used to create a container image. In line with the *configuration as code* approach mentioned above, the definition file can be stored in your code repository alongside your application code and used to create a reproducible image. This means that for a given commit in your repository, the version of the definition file present at that commit can be used to reproduce a container with a known state. It was pointed out earlier in the course, when covering Docker, that this property also applies for Dockerfiles. We'll now look at a very simple example of a definition file: -~~~ +```bash Bootstrap: docker From: ubuntu:20.04 @@ -58,62 +72,56 @@ From: ubuntu:20.04 %runscript python -c 'print("Hello World! Hello from our custom Singularity image!")' -~~~ -{: .language-bash} +``` A definition file has a number of optional sections, specified using the `%` prefix, that are used to define or undertake different configuration during different stages of the image build process. You can find full details in Singularity's [Definition Files documentation](https://sylabs.io/guides/3.5/user-guide/definition_files.html). In our very simple example here, we only use the `%post` and `%runscript` sections. Let's step through this definition file and look at the lines in more detail: -~~~ +```bash Bootstrap: docker From: ubuntu:20.04 -~~~ -{: .language-bash} +``` -These first two lines define where to _bootstrap_ our image from. Why can't we just put some application binaries into a blank image? Any applications or tools that we want to run will need to interact with standard system libraries and potentially a wide range of other libraries and tools. These need to be available within the image and we therefore need some sort of operating system as the basis for our image. The most straightforward way to achieve this is to start from an existing base image containing an operating system. In this case, we're going to start from a minimal Ubuntu 20.04 Linux Docker image. Note that we're using a Docker image as the basis for creating a Singularity image. This demonstrates the flexibility in being able to start from different types of images when creating a new Singularity image. +These first two lines define where to *bootstrap* our image from. Why can't we just put some application binaries into a blank image? Any applications or tools that we want to run will need to interact with standard system libraries and potentially a wide range of other libraries and tools. These need to be available within the image and we therefore need some sort of operating system as the basis for our image. The most straightforward way to achieve this is to start from an existing base image containing an operating system. In this case, we're going to start from a minimal Ubuntu 20.04 Linux Docker image. Note that we're using a Docker image as the basis for creating a Singularity image. This demonstrates the flexibility in being able to start from different types of images when creating a new Singularity image. The `Bootstrap: docker` line is similar to prefixing an image path with `docker://` when using, for example, the `singularity pull` command. A range of [different bootstrap options](https://sylabs.io/guides/3.5/user-guide/definition_files.html#preferred-bootstrap-agents) are supported. `From: ubuntu:20.04` says that we want to use the `ubuntu` image with the tag `20.04` from Docker Hub. Next we have the `%post` section of the definition file: -~~~ +```bash %post apt-get -y update && apt-get install -y python3 -~~~ -{: .language-bash} +``` -In this section of the file we can do tasks such as package installation, pulling data files from remote locations and undertaking local configuration within the image. The commands that appear in this section are standard shell commands and they are run _within_ the context of our new container image. So, in the case of this example, these commands are being run within the context of a minimal Ubuntu 20.04 image that initially has only a very small set of core packages installed. +In this section of the file we can do tasks such as package installation, pulling data files from remote locations and undertaking local configuration within the image. The commands that appear in this section are standard shell commands and they are run *within* the context of our new container image. So, in the case of this example, these commands are being run within the context of a minimal Ubuntu 20.04 image that initially has only a very small set of core packages installed. Here we use Ubuntu's package manager to update our package indexes and then install the `python3` package along with any required dependencies. The `-y` switches are used to accept, by default, interactive prompts that might appear asking you to confirm package updates or installation. This is required because our definition file should be able to run in an unattended, non-interactive environment. Finally we have the `%runscript` section: -~~~ +```bash %runscript python3 -c 'print("Hello World! Hello from our custom Singularity image!")' -~~~ -{: .language-bash} +``` This section is used to define a script that should be run when a container is started based on this image using the `singularity run` command. In this simple example we use `python3` to print out some text to the console. -We can now save the contents of the simple defintion file shown above to a file and build an image based on it. In the case of this example, the definition file has been named `my_test_image.def`. (Note that the instructions here assume you've bound the image output directory you created to the `/home/singularity` directory in your Docker Singularity container, as explained in the "[_Getting started with the Docker Singularity image_](#getting-started-with-the-docker-singularity-image)" section above.): +We can now save the contents of the simple defintion file shown above to a file and build an image based on it. In the case of this example, the definition file has been named `my_test_image.def`. (Note that the instructions here assume you've bound the image output directory you created to the `/home/singularity` directory in your Docker Singularity container, as explained in the "[*Getting started with the Docker Singularity image*](#getting-started-with-the-docker-singularity-image)" section above.): -~~~ +```bash $ singularity build /home/singularity/my_test_image.sif /home/singularity/my_test_image.def -~~~ -{: .language-bash} +``` Recall from the details at the start of this section that if you are running your command from the host system command line, running an instance of a Docker container for each run of the command, your command will look something like this: -~~~ +```bash $ docker run --privileged --rm -v ${PWD}:/home/singularity quay.io/singularity/singularity:v3.5.3-slim build /home/singularity/my_test_image.sif /home/singularity/my_test_image.def -~~~ -{: .language-bash} +``` The above command requests the building of an image based on the `my_test_image.def` file with the resulting image saved to the `my_test_image.sif` file. Note that you will need to prefix the command with `sudo` if you're running a locally installed version of Singularity and not running via Docker because it is necessary to have administrative privileges to build the image. You should see output similar to the following: -~~~ +```output INFO: Starting build... Getting image source signatures Copying blob d51af753c3d3 skipped: already exists @@ -144,91 +152,96 @@ INFO: Adding runscript INFO: Creating SIF file... INFO: Build complete: my_test_image.sif $ -~~~ -{: .output} +``` You should now have a `my_test_image.sif` file in the current directory. Note that in the above output, where it says `INFO: Starting build...` there is a series of `skipped: already exists` messages for the `Copying blob` lines. This is because the Docker image slices for the Ubuntu 20.04 image have previously been downloaded and are cached on the system where this example is being run. On your system, if the image is not already cached, you will see the slices being downloaded from Docker Hub when these lines of output appear. -> ## Permissions of the created image file -> -> You may find that the created Singularity image file on your host filesystem is owned by the `root` user and not your user. In this case, you won't be able to change the ownership/permissions of the file directly if you don't have root access. -> -> However, the image file will be readable by you and you should be able to take a copy of the file under a new name which you will then own. You will then be able to modify the permissions of this copy of the image and delete the original root-owned file since the default permissions should allow this. -> -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Permissions of the created image file + +You may find that the created Singularity image file on your host filesystem is owned by the `root` user and not your user. In this case, you won't be able to change the ownership/permissions of the file directly if you don't have root access. + +However, the image file will be readable by you and you should be able to take a copy of the file under a new name which you will then own. You will then be able to modify the permissions of this copy of the image and delete the original root-owned file since the default permissions should allow this. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::::: callout -> ## Cluster platform configuration for running Singularity containers -> -> _**Note to instructors:** Add details into this box of any custom configuration that needs to be done on the cluster platform or other remote system that you're providing access to for the purpose of undertaking this course. If `singularity` does not require any custom configuration by the user on the host platform, you can remove this box._ -> -{: .callout} +## Cluster platform configuration for running Singularity containers -It is recommended that you move the created `.sif` file to a platform with an installation of Singularity, rather than attempting to run the image using the Docker container. However, if you do wish to try using the Docker container, see the notes below on "_Using singularity run from within the Docker container_" for further information. +***Note to instructors:** Add details into this box of any custom configuration that needs to be done on the cluster platform or other remote system that you're providing access to for the purpose of undertaking this course. If `singularity` does not require any custom configuration by the user on the host platform, you can remove this box.* + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +It is recommended that you move the created `.sif` file to a platform with an installation of Singularity, rather than attempting to run the image using the Docker container. However, if you do wish to try using the Docker container, see the notes below on "*Using singularity run from within the Docker container*" for further information. If you have access to a remote platform with Singularity installed on it, you should now move your created `.sif` image file to this platform. You could, for example, do this using the command line secure copy command `scp`. -> ## Using `scp` (secure copy) to copy files between systems -> -> `scp` is a widely used tool that uses the SSH protocol to securely copy files between systems. As such, the syntax is similar to that of SSH. -> -> For example, if you want to copy the `my_image.sif` file from the current directory on your local system to your home directory (e.g. `/home/myuser/`) on a remote system (e.g. _hpc.myinstitution.ac.uk_) where an SSH private key is required for login, you would use a command similar to the following: -> -> ``` -> scp -i /path/to/keyfile/id_mykey ./my_image.sif myuser@hpc.myinstitution.ac.uk:/home/myuser/ -> ``` -> Note that if you leave off the `/home/myuser` and just end the command with the `:`, the file will, by default, be copied to your home directory. -> -{: .callout} +::::::::::::::::::::::::::::::::::::::::: callout + +## Using `scp` (secure copy) to copy files between systems + +`scp` is a widely used tool that uses the SSH protocol to securely copy files between systems. As such, the syntax is similar to that of SSH. + +For example, if you want to copy the `my_image.sif` file from the current directory on your local system to your home directory (e.g. `/home/myuser/`) on a remote system (e.g. *hpc.myinstitution.ac.uk*) where an SSH private key is required for login, you would use a command similar to the following: + +``` +scp -i /path/to/keyfile/id_mykey ./my_image.sif myuser@hpc.myinstitution.ac.uk:/home/myuser/ +``` + +Note that if you leave off the `/home/myuser` and just end the command with the `:`, the file will, by default, be copied to your home directory. + +:::::::::::::::::::::::::::::::::::::::::::::::::: We can now attempt to run a container from the image that we built: -~~~ +```bash $ singularity run my_test_image.sif -~~~ -{: .language-bash} +``` If everything worked successfully, you should see the message printed by Python: -~~~ +```output Hello World! Hello from our custom Singularity image! -~~~ -{: .output} - -> ## Using `singularity run` from within the Docker container -> -> It is strongly recommended that you don't use the Docker container for running Singularity images, only for creating them, since the Singularity command runs within the container as the root user. -> -> However, for the purposes of this simple example, and potentially for testing/debugging purposes it is useful to know how to run a Singularity container within the Docker Singularity container. You may recall from the [Running a container from the image](/06-singularity-images-prep/index.html#running-a-container-from-the-image) section in the previous episode that we used the `--contain` switch with the `singularity` command. If you don't use this switch, it is likely that you will get an error relating to `/etc/localtime` similar to the following: -> -> ~~~ -> WARNING: skipping mount of /etc/localtime: no such file or directory -> FATAL: container creation failed: mount /etc/localtime->/etc/localtime error: while mounting /etc/localtime: mount source /etc/localtime doesn't exist -> ~~~ -> {: .output} -> -> This occurs because the `/etc/localtime` file that provides timezone configuration is not present within the Docker container. If you want to use the Docker container to test that your newly created image runs, you can use the `--contain` switch, or you can open a shell in the Docker container and add a timezone configuration as described in the [Alpine Linux documentation](https://wiki.alpinelinux.org/wiki/Setting_the_timezone): -> -> ~~~ -> $ apk add tzdata -> $ cp /usr/share/zoneinfo/Europe/London /etc/localtime -> ~~~ -> {: .language-bash} -> -> The `singularity run` command should now work successfully without needing to use `--contain`. Bear in mind that once you exit the Docker Singularity container shell and shutdown the container, this configuration will not persist. -{: .callout} +``` + +::::::::::::::::::::::::::::::::::::::::: callout + +## Using `singularity run` from within the Docker container + +It is strongly recommended that you don't use the Docker container for running Singularity images, only for creating them, since the Singularity command runs within the container as the root user. +However, for the purposes of this simple example, and potentially for testing/debugging purposes it is useful to know how to run a Singularity container within the Docker Singularity container. You may recall from the [Running a container from the image](06-singularity-images-prep.md) section in the previous episode that we used the `--contain` switch with the `singularity` command. If you don't use this switch, it is likely that you will get an error relating to `/etc/localtime` similar to the following: + +```output +WARNING: skipping mount of /etc/localtime: no such file or directory +FATAL: container creation failed: mount /etc/localtime->/etc/localtime error: while mounting /etc/localtime: mount source /etc/localtime doesn't exist +``` + +This occurs because the `/etc/localtime` file that provides timezone configuration is not present within the Docker container. If you want to use the Docker container to test that your newly created image runs, you can use the `--contain` switch, or you can open a shell in the Docker container and add a timezone configuration as described in the [Alpine Linux documentation](https://wiki.alpinelinux.org/wiki/Setting_the_timezone): + +```bash +$ apk add tzdata +$ cp /usr/share/zoneinfo/Europe/London /etc/localtime +``` + +The `singularity run` command should now work successfully without needing to use `--contain`. Bear in mind that once you exit the Docker Singularity container shell and shutdown the container, this configuration will not persist. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: ### More advanced definition files -Here we've looked at a very simple example of how to create an image. At this stage, you might want to have a go at creating your own definition file for some code of your own or an application that you work with regularly. There are several definition file sections that were _not_ used in the above example, these are: +Here we've looked at a very simple example of how to create an image. At this stage, you might want to have a go at creating your own definition file for some code of your own or an application that you work with regularly. There are several definition file sections that were *not* used in the above example, these are: - - `%setup` - - `%files` - - `%environment` - - `%startscript` - - `%test` - - `%labels` - - `%help` +- `%setup` +- `%files` +- `%environment` +- `%startscript` +- `%test` +- `%labels` +- `%help` The [`Sections` part of the definition file documentation](https://sylabs.io/guides/3.5/user-guide/definition_files.html#sections) details all the sections and provides an example definition file that makes use of all the sections. @@ -236,7 +249,16 @@ The [`Sections` part of the definition file documentation](https://sylabs.io/gui Singularity has a wide range of features. You can find full details in the [Singularity User Guide](https://sylabs.io/guides/3.5/user-guide/index.html) and we highlight a couple of key features here that may be of use/interest: -**Remote Builder Capabilities:** If you have access to a platform with Singularity installed but you don't have root access to create containers, you may be able to use the [Remote Builder](https://cloud.sylabs.io/builder) functionality to offload the process of building an image to remote cloud resources. You'll need to register for a _cloud token_ via the link on the [Remote Builder](https://cloud.sylabs.io/builder) page. +**Remote Builder Capabilities:** If you have access to a platform with Singularity installed but you don't have root access to create containers, you may be able to use the [Remote Builder](https://cloud.sylabs.io/builder) functionality to offload the process of building an image to remote cloud resources. You'll need to register for a *cloud token* via the link on the [Remote Builder](https://cloud.sylabs.io/builder) page. **Signing containers:** If you do want to share container image (`.sif`) files directly with colleagues or collaborators, how can the people you send an image to be sure that they have received the file without it being tampered with or suffering from corruption during transfer/storage? And how can you be sure that the same goes for any container image file you receive from others? Singularity supports signing containers. This allows a digital signature to be linked to an image file. This signature can be used to verify that an image file has been signed by the holder of a specific key and that the file is unchanged from when it was signed. You can find full details of how to use this functionality in the Singularity documentation on [Signing and Verifying Containers](https://sylabs.io/guides/3.0/user-guide/signNverify.html). +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Singularity definition files are used to define the build process and configuration for an image. +- Singularity's Docker container provides a way to build images on a platform where Singularity is not installed but Docker is available. +- Existing images from remote registries such as Docker Hub and Singularity Hub can be used as a base for creating new Singularity images. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/episodes/08-singularity-mpi.md b/episodes/08-singularity-mpi.md index 083c8d2..9111d12 100644 --- a/episodes/08-singularity-mpi.md +++ b/episodes/08-singularity-mpi.md @@ -1,19 +1,22 @@ --- -title: "Running MPI parallel jobs using Singularity containers" +title: Running MPI parallel jobs using Singularity containers teaching: 30 exercises: 40 -questions: -# - "Can I run MPI parallel codes from Singularity containers on a local/institutional/national HPC platform?" -- "How do I set up and run an MPI job from a Singularity container?" -objectives: -- "Learn how MPI applications within Singularity containers can be run on HPC platforms" -- "Understand the challenges and related performance implications when running MPI jobs via Singularity" -keypoints: -- "Singularity images containing MPI applications can be built on one platform and then run on another (e.g. an HPC cluster) if the two platforms have compatible MPI implementations." -- "When running an MPI application within a Singularity container, use the MPI executable on the host system to launch a Singularity container for each process." -- "Think about parallel application performance requirements and how where you build/run your image may affect that." --- +::::::::::::::::::::::::::::::::::::::: objectives + +- Learn how MPI applications within Singularity containers can be run on HPC platforms +- Understand the challenges and related performance implications when running MPI jobs via Singularity + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::: questions + +- How do I set up and run an MPI job from a Singularity container? + +:::::::::::::::::::::::::::::::::::::::::::::::::: + ## Running MPI parallel codes with Singularity containers ### MPI overview @@ -34,7 +37,7 @@ As described in Singularity's [MPI documentation](https://sylabs.io/guides/3.5/u #### **Container portability and performance on HPC platforms** -While building a container on a local system that is intended for use on a remote HPC platform does provide some level of portability, if you're after the best possible performance, it can present some issues. The version of MPI in the container will need to be built and configured to support the hardware on your target platform if the best possible performance is to be achieved. Where a platform has specialist hardware with proprietary drivers, building on a different platform with different hardware present means that building with the right driver support for optimal performance is not likely to be possible. This is especially true if the version of MPI available is different (but compatible). Singularity's [MPI documentation](https://sylabs.io/guides/3.5/user-guide/mpi.html) highlights two different models for working with MPI codes. The _[hybrid model](https://sylabs.io/guides/3.5/user-guide/mpi.html#hybrid-model)_ that we'll be looking at here involves using the MPI executable from the MPI installation on the host system to launch singularity and run the application within the container. The application in the container is linked against and uses the MPI installation within the container which, in turn, communicates with the MPI daemon process running on the host system. In the following section we'll look at building a Singularity image containing a small MPI application that can then be run using the hybrid model. +While building a container on a local system that is intended for use on a remote HPC platform does provide some level of portability, if you're after the best possible performance, it can present some issues. The version of MPI in the container will need to be built and configured to support the hardware on your target platform if the best possible performance is to be achieved. Where a platform has specialist hardware with proprietary drivers, building on a different platform with different hardware present means that building with the right driver support for optimal performance is not likely to be possible. This is especially true if the version of MPI available is different (but compatible). Singularity's [MPI documentation](https://sylabs.io/guides/3.5/user-guide/mpi.html) highlights two different models for working with MPI codes. The *[hybrid model](https://sylabs.io/guides/3.5/user-guide/mpi.html#hybrid-model)* that we'll be looking at here involves using the MPI executable from the MPI installation on the host system to launch singularity and run the application within the container. The application in the container is linked against and uses the MPI installation within the container which, in turn, communicates with the MPI daemon process running on the host system. In the following section we'll look at building a Singularity image containing a small MPI application that can then be run using the hybrid model. ### Building and running a Singularity image for an MPI code @@ -50,7 +53,7 @@ Begin by creating a directory and, within that directory, downloading and saving In the same directory, save the following definition file content to a `.def` file, e.g. `osu_benchmarks.def`: -~~~ +```output Bootstrap: docker From: ubuntu:20.04 @@ -78,62 +81,64 @@ From: ubuntu:20.04 %runscript echo "Rank ${PMI_RANK} - About to run: ${OSU_DIR}/$*" exec ${OSU_DIR}/$* -~~~ -{: .output} +``` A quick overview of what the above definition file is doing: - - The image is being bootstrapped from the `ubuntu:20.04` Docker image. - - In the `%files` section: The OSU Micro-Benchmarks and MPICH tar files are copied from the current directory into the `/root` directory within the image. - - In the `%environment` section: Set a couple of environment variables that will be available within all containers run from the generated image. - - In the `%post` section: - - Ubuntu's `apt-get` package manager is used to update the package directory and then install the compilers and other libraries required for the MPICH build. - - The MPICH .tar.gz file is extracted and the configure, build and install steps are run. Note the use of the `--with-device` option to configure MPICH to use the correct driver to support improved communication performance on a high performance cluster. - - The OSU Micro-Benchmarks .tar.gz file is extracted and the configure, build and install steps are run to build the benchmark code from source. - - In the `%runscript` section: A runscript is set up that will echo the rank number of the current process and then run the command provided as a command line argument. - -_Note that base path of the the executable to run (`$OSU_DIR`) is hardcoded in the run script_. The command line parameter that you provide when running a container instance based on the image is then added to this base path. Example command line parameters include: `startup/osu_hello`, `collective/osu_allgather`, `pt2pt/osu_latency`, `one-sided/osu_put_latency`. - -> ## Build and test the OSU Micro-Benchmarks image -> -> Using the above definition file, build a Singularity image named `osu_benchmarks.sif`. -> -> Once the image has finished building, test it by running the `osu_hello` benchmark that is found in the `startup` benchmark folder. -> -> _NOTE: If you're not using the Singularity Docker image to build your Singularity image, you will need to edit the path to the .tar.gz file in the `%files` section of the definition file._ -> -> > ## Solution -> > -> > You should be able to build an image from the definition file as follows: -> > -> > ~~~ -> > $ singularity build osu_benchmarks.sif osu_benchmarks.def -> > ~~~ -> > {: .language-bash} -> > -> > _Note that if you're running the Singularity Docker container directly from the command line to undertake your build, you'll need to provide the full path to the `.def` file **within** the container_ - it is likely that this will be different to the file path on your host system. For example, if you've bind mounted the directory on your local system containing the file to `/home/singularity` within the container, the full path to the `.def` file will be `/home/singularity/osu_benchmarks.def`. -> > -> > Assuming the image builds successfully, you can then try running the container locally and also transfer the SIF file to a cluster platform that you have access to (that has Singularity installed) and run it there. -> > -> > Let's begin with a single-process run of `startup/osu_hello` on _your local system_ (where you built the container) to ensure that we can run the container as expected. We'll use the MPI installation _within_ the container for this test. _Note that when we run a parallel job on an HPC cluster platform, we use the MPI installation on the cluster to coordinate the run so things are a little different..._ -> > -> > Start a shell in the Singularity container based on your image and then run a single process job via `mpirun`: -> > -> > ~~~ -> > $ singularity shell --contain /home/singularity/osu_benchmarks.sif -> > Singularity> mpirun -np 1 $OSU_DIR/startup/osu_hello -> > ~~~ -> > {: .language-bash} -> > -> > You should see output similar to the following: -> > -> > ~~~ -> > # OSU MPI Hello World Test v5.7.1 -> > This is a test with 1 processes -> > ~~~ -> > {: .output} -> {: .solution} -{: .challenge} +- The image is being bootstrapped from the `ubuntu:20.04` Docker image. +- In the `%files` section: The OSU Micro-Benchmarks and MPICH tar files are copied from the current directory into the `/root` directory within the image. +- In the `%environment` section: Set a couple of environment variables that will be available within all containers run from the generated image. +- In the `%post` section: + - Ubuntu's `apt-get` package manager is used to update the package directory and then install the compilers and other libraries required for the MPICH build. + - The MPICH .tar.gz file is extracted and the configure, build and install steps are run. Note the use of the `--with-device` option to configure MPICH to use the correct driver to support improved communication performance on a high performance cluster. + - The OSU Micro-Benchmarks .tar.gz file is extracted and the configure, build and install steps are run to build the benchmark code from source. +- In the `%runscript` section: A runscript is set up that will echo the rank number of the current process and then run the command provided as a command line argument. + +*Note that base path of the the executable to run (`$OSU_DIR`) is hardcoded in the run script*. The command line parameter that you provide when running a container instance based on the image is then added to this base path. Example command line parameters include: `startup/osu_hello`, `collective/osu_allgather`, `pt2pt/osu_latency`, `one-sided/osu_put_latency`. + +::::::::::::::::::::::::::::::::::::::: challenge + +## Build and test the OSU Micro-Benchmarks image + +Using the above definition file, build a Singularity image named `osu_benchmarks.sif`. + +Once the image has finished building, test it by running the `osu_hello` benchmark that is found in the `startup` benchmark folder. + +*NOTE: If you're not using the Singularity Docker image to build your Singularity image, you will need to edit the path to the .tar.gz file in the `%files` section of the definition file.* + +::::::::::::::: solution + +## Solution + +You should be able to build an image from the definition file as follows: + +```bash +$ singularity build osu_benchmarks.sif osu_benchmarks.def +``` + +*Note that if you're running the Singularity Docker container directly from the command line to undertake your build, you'll need to provide the full path to the `.def` file **within** the container* - it is likely that this will be different to the file path on your host system. For example, if you've bind mounted the directory on your local system containing the file to `/home/singularity` within the container, the full path to the `.def` file will be `/home/singularity/osu_benchmarks.def`. + +Assuming the image builds successfully, you can then try running the container locally and also transfer the SIF file to a cluster platform that you have access to (that has Singularity installed) and run it there. + +Let's begin with a single-process run of `startup/osu_hello` on *your local system* (where you built the container) to ensure that we can run the container as expected. We'll use the MPI installation *within* the container for this test. *Note that when we run a parallel job on an HPC cluster platform, we use the MPI installation on the cluster to coordinate the run so things are a little different...* + +Start a shell in the Singularity container based on your image and then run a single process job via `mpirun`: + +```bash +$ singularity shell --contain /home/singularity/osu_benchmarks.sif +Singularity> mpirun -np 1 $OSU_DIR/startup/osu_hello +``` + +You should see output similar to the following: + +```output +# OSU MPI Hello World Test v5.7.1 +This is a test with 1 processes +``` + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: #### **Running Singularity containers via MPI** @@ -149,109 +154,132 @@ Ultimately, this means that our running MPI code is linking to the MPI libraries We can now try running a 2-process MPI run of a point to point benchmark `osu_latency`. If your local system has both MPI and Singularity installed and has multiple cores, you can run this test on that system. Alternatively you can run on a cluster. Note that you may need to submit this command via a job submission script submitted to a job scheduler if you're running on a cluster. If you're attending a taught version of this course, some information will be provided below in relation to the cluster that you've been provided with access to. +::::::::::::::::::::::::::::::::::::::: challenge + +## Undertake a parallel run of the `osu_latency` benchmark (general example) + +Move the `osu_benchmarks.sif` Singularity image onto the cluster (or other suitable) platform where you're going to undertake your benchmark run. + +You should be able to run the benchmark using a command similar to the one shown below. However, if you are running on a cluster, you may need to write and submit a job submission script at this point to initiate running of the benchmark. + +```bash +$ mpirun -np 2 singularity run osu_benchmarks.sif pt2pt/osu_latency +``` + +::::::::::::::: solution + +## Expected output and discussion + +As you can see in the mpirun command shown above, we have called `mpirun` on the host system and are passing to MPI the `singularity` executable for which the parameters are the image file and any parameters we want to pass to the image's run script, in this case the path/name of the benchmark executable to run. + +The following shows an example of the output you should expect to see. You should have latency values shown for message sizes up to 4MB. + +```output +Rank 1 - About to run: /.../mpi/pt2pt/osu_latency +Rank 0 - About to run: /.../mpi/pt2pt/osu_latency +# OSU MPI Latency Test v5.6.2 +# Size Latency (us) +0 0.38 +1 0.34 +... +``` + +::::::::::::::::::::::::: -> ## Undertake a parallel run of the `osu_latency` benchmark (general example) -> -> Move the `osu_benchmarks.sif` Singularity image onto the cluster (or other suitable) platform where you're going to undertake your benchmark run. -> -> You should be able to run the benchmark using a command similar to the one shown below. However, if you are running on a cluster, you may need to write and submit a job submission script at this point to initiate running of the benchmark. -> -> ~~~ -> $ mpirun -np 2 singularity run osu_benchmarks.sif pt2pt/osu_latency -> ~~~ -> {: .language-bash} -> -> > ## Expected output and discussion -> > -> > As you can see in the mpirun command shown above, we have called `mpirun` on the host system and are passing to MPI the `singularity` executable for which the parameters are the image file and any parameters we want to pass to the image's run script, in this case the path/name of the benchmark executable to run. -> > -> > The following shows an example of the output you should expect to see. You should have latency values shown for message sizes up to 4MB. -> > -> >~~~ -> > Rank 1 - About to run: /.../mpi/pt2pt/osu_latency -> > Rank 0 - About to run: /.../mpi/pt2pt/osu_latency -> > # OSU MPI Latency Test v5.6.2 -> > # Size Latency (us) -> > 0 0.38 -> > 1 0.34 -> > ... -> > ~~~ -> > {: .output} -> {: .solution} -{: .challenge} - - -> ## Undertake a parallel run of the `osu_latency` benchmark (taught course cluster example) -> _**Note to instructors:** Add details into this box relating to running the above example on your chosen cluster platform. The example SLURM script provided here is based on the UK's ARCHER2 HPC platform, you should replace the template file in the `files` directory of the repository with a submission script suited to your platform, if one is required._ -> -> This version of the exercise, for undertaking a parallel run of the osu_latency benchmark with your Singularity container that contains an MPI build, is specific to this run of the course. -> -> The information provided here is specifically tailored to the HPC platform that you've been given access to for this taught version of the course. -> -> Move the `osu_benchmarks.sif` Singularity image onto the cluster where you're going to undertake your benchmark run. You should use `scp` or a similar utility to copy the file. -> -> The platform you've been provided with access to uses `Slurm` to schedule jobs to run on the platform. You now need to create a `Slurm` job submission script to run the benchmark. -> -> Download this [template script]({{site.url}}{{site.baseurl}}/files/osu_latency.slurm.template) and edit it to suit your configuration. -> -> Submit the modified job submission script to the `Slurm` scheduler using the `sbatch` command. -> -> ~~~ -> $ sbatch osu_latency.slurm -> ~~~ -> {: .language-bash} -> -> > ## Expected output and discussion -> > -> > As you will have seen in the commands using the provided template job submission script, we have called `mpirun` on the host system and are passing to MPI the `singularity` executable for which the parameters are the image file and any parameters we want to pass to the image's run script. In this case, the parameters are the path/name of the benchmark executable to run. -> > -> > The following shows an example of the output you should expect to see. You should have latency values shown for message sizes up to 4MB. -> > -> >~~~ -> > INFO: Convert SIF file to sandbox... -> > INFO: Convert SIF file to sandbox... -> > Rank 1 - About to run: /.../mpi/pt2pt/osu_latency -> > Rank 0 - About to run: /.../mpi/pt2pt/osu_latency -> > # OSU MPI Latency Test v5.6.2 -> > # Size Latency (us) -> > 0 1.49 -> > 1 1.50 -> > 2 1.50 -> > ... -> > 4194304 915.44 -> > INFO: Cleaning up image... -> > INFO: Cleaning up image... -> > ~~~ -> > {: .output} -> {: .solution} -{: .challenge} +:::::::::::::::::::::::::::::::::::::::::::::::::: + +::::::::::::::::::::::::::::::::::::::: challenge + +## Undertake a parallel run of the `osu_latency` benchmark (taught course cluster example) + +***Note to instructors:** Add details into this box relating to running the above example on your chosen cluster platform. The example SLURM script provided here is based on the UK's ARCHER2 HPC platform, you should replace the template file in the `files` directory of the repository with a submission script suited to your platform, if one is required.* + +This version of the exercise, for undertaking a parallel run of the osu\_latency benchmark with your Singularity container that contains an MPI build, is specific to this run of the course. + +The information provided here is specifically tailored to the HPC platform that you've been given access to for this taught version of the course. + +Move the `osu_benchmarks.sif` Singularity image onto the cluster where you're going to undertake your benchmark run. You should use `scp` or a similar utility to copy the file. + +The platform you've been provided with access to uses `Slurm` to schedule jobs to run on the platform. You now need to create a `Slurm` job submission script to run the benchmark. + +Download this [template script]({{site.url}}files/osu_latency.slurm.template) and edit it to suit your configuration. + +Submit the modified job submission script to the `Slurm` scheduler using the `sbatch` command. + +```bash +$ sbatch osu_latency.slurm +``` + +::::::::::::::: solution + +## Expected output and discussion + +As you will have seen in the commands using the provided template job submission script, we have called `mpirun` on the host system and are passing to MPI the `singularity` executable for which the parameters are the image file and any parameters we want to pass to the image's run script. In this case, the parameters are the path/name of the benchmark executable to run. + +The following shows an example of the output you should expect to see. You should have latency values shown for message sizes up to 4MB. + +```output +INFO: Convert SIF file to sandbox... +INFO: Convert SIF file to sandbox... +Rank 1 - About to run: /.../mpi/pt2pt/osu_latency +Rank 0 - About to run: /.../mpi/pt2pt/osu_latency +# OSU MPI Latency Test v5.6.2 +# Size Latency (us) +0 1.49 +1 1.50 +2 1.50 +... +4194304 915.44 +INFO: Cleaning up image... +INFO: Cleaning up image... +``` + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: This has demonstrated that we can successfully run a parallel MPI executable from within a Singularity container. However, in this case, the two processes will almost certainly have run on the same physical node so this is not testing the performance of the interconnects between nodes. You could now try running a larger-scale test. You can also try running a benchmark that uses multiple processes, for example try `collective/osu_gather`. -> ## Investigate performance when using a container image built on a local system and run on a cluster -> -> To get an idea of any difference in performance between the code within your Singularity image and the same code built natively on the target HPC platform, try building the OSU benchmarks from source, locally on the cluster. Then try running the same benchmark(s) that you ran via the singularity container. Have a look at the outputs you get when running `collective/osu_gather` or one of the other collective benchmarks to get an idea of whether there is a performance difference and how significant it is. -> -> Try running with enough processes that the processes are spread across different physical nodes so that you're making use of the cluster's network interconnects. -> -> What do you see? -> -> > ## Discussion -> > You may find that performance is significantly better with the version of the code built directly on the HPC platform. Alternatively, performance may be similar between the two versions. -> > -> > How big is the performance difference between the two builds of the code? -> > -> > What might account for any difference in performance between the two builds of the code? -> > -> {: .solution} -{: .challenge} - -If performance is an issue for you with codes that you'd like to run via Singularity, you are advised to take a look at using the _[bind model](https://sylabs.io/guides/3.5/user-guide/mpi.html#bind-model)_ for building/running MPI applications through Singularity. +::::::::::::::::::::::::::::::::::::::: challenge + +## Investigate performance when using a container image built on a local system and run on a cluster + +To get an idea of any difference in performance between the code within your Singularity image and the same code built natively on the target HPC platform, try building the OSU benchmarks from source, locally on the cluster. Then try running the same benchmark(s) that you ran via the singularity container. Have a look at the outputs you get when running `collective/osu_gather` or one of the other collective benchmarks to get an idea of whether there is a performance difference and how significant it is. + +Try running with enough processes that the processes are spread across different physical nodes so that you're making use of the cluster's network interconnects. + +What do you see? + +::::::::::::::: solution + +## Discussion + +You may find that performance is significantly better with the version of the code built directly on the HPC platform. Alternatively, performance may be similar between the two versions. + +How big is the performance difference between the two builds of the code? + +What might account for any difference in performance between the two builds of the code? + +::::::::::::::::::::::::: + +:::::::::::::::::::::::::::::::::::::::::::::::::: + +If performance is an issue for you with codes that you'd like to run via Singularity, you are advised to take a look at using the *[bind model](https://sylabs.io/guides/3.5/user-guide/mpi.html#bind-model)* for building/running MPI applications through Singularity. ## Singularity wrap-up This concludes the 8 episodes of the course covering Singularity. We hope you found this information useful and that it has inspired you to use Singularity to help enhance the way you build/work with research software. As a new set of material, we appreciate that there are likely to be improvements that can be made to enhance the quality of this material. We welcome your thoughts, suggestions and feedback on improvements that could be made to help others making use of these lessons. + +:::::::::::::::::::::::::::::::::::::::: keypoints + +- Singularity images containing MPI applications can be built on one platform and then run on another (e.g. an HPC cluster) if the two platforms have compatible MPI implementations. +- When running an MPI application within a Singularity container, use the MPI executable on the host system to launch a Singularity container for each process. +- Think about parallel application performance requirements and how where you build/run your image may affect that. + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + diff --git a/fig/SingularityInDocker.png b/episodes/fig/SingularityInDocker.png similarity index 100% rename from fig/SingularityInDocker.png rename to episodes/fig/SingularityInDocker.png diff --git a/files/osu_latency.slurm.template b/episodes/files/osu_latency.slurm.template similarity index 100% rename from files/osu_latency.slurm.template rename to episodes/files/osu_latency.slurm.template diff --git a/index.md b/index.md index 75f0474..417e00e 100644 --- a/index.md +++ b/index.md @@ -1,10 +1,14 @@ --- -layout: lesson -root: . # Is the only page that doesn't follow the pattern /:path/index.html -permalink: index.html # Is the only page that doesn't follow the pattern /:path/index.html +permalink: index.html +site: sandpaper::sandpaper_site --- -This lesson provides an introduction to using the [Singularity container platform](https://github.com/hpcng/singularity). Singularity is particularly suited to running containers on infrastructure where users don't have administrative privileges, for example shared infrastructure such as High Performance Computing (HPC) clusters. +> **ATTENTION** This is an experimental test of [The Carpentries Workbench](https://carpentries.github.io/workbench) lesson infrastructure. +> It was automatically converted from the source lesson via [the lesson transition script](https://github.com/carpentries/lesson-transition/). +> +> If anything seems off, please contact Zhian Kamvar [zkamvar@carpentries.org](mailto:zkamvar@carpentries.org) + +This lesson provides an introduction to using the [Singularity container platform](https://github.com/hpcng/singularity). Singularity is particularly suited to running containers on infrastructure where users don't have administrative privileges, for example shared infrastructure such as High Performance Computing (HPC) clusters. This lesson will introduce Singularity from scratch showing you how to run a simple container and building up to creating your own containers and running parallel scientific workloads on HPC infrastructure. @@ -12,24 +16,32 @@ This lesson will introduce Singularity from scratch showing you how to run a sim {% comment %} This is a comment in Liquid {% endcomment %} -> ## Prerequisites -> There are two core elements to this lesson - _running containers_ and _building containers_. The prerequisites are slightly different for each and are explained below. -> -> **Running containers:** (episodes 1-5 and 8) -> - Access to a local or remote platform with Singularity pre-installed and accessible to you as a user (i.e. no administrator/root access required). -> - If you are attending a taught version of this material, it is expected that the course organisers will provide access to a platform (e.g. an institutional HPC cluster) that you can use for these sections of the material. -> - The platform you will be using should also have MPI installed (required for episode 8). -> -> **Building containers:** (episodes 6 and 7) -> Building containers requires access to a platform with an installation of Singularity on which you also have administrative access. If you run Linux and are comfortable with following the [Singularity installation instructions](https://sylabs.io/guides/3.5/admin-guide/installation.html), then installing Singularity directly on your system is an option. However, we strongly recommend using the [Docker Singularity container](https://quay.io/repository/singularity/singularity?tab=tags) for this section of the material. Details are provided on how to use the container in the relevant section of the lesson material. To support building containers, the prerequisite is therefore: -> -> - Access to a system with Docker installed on which you can run the Docker Singularity container. -> -> OR -> -> - Access to a local or remote Linux-based system on which you have administrator (root) access and can install the Singularity software. -> -> **Please note that the version of Singularity used in this part of the course is _version 3.5.3_ which was the latest stable release at the time of writing.** If you are installing Singularity on your own system for use in the course, you are recommneded to install version 3.5.3. -{: .prereq} - -{% include links.md %} +:::::::::::::::::::::::::::::::::::::::::: prereq + +## Prerequisites + +There are two core elements to this lesson - *running containers* and *building containers*. The prerequisites are slightly different for each and are explained below. + +**Running containers:** (episodes 1-5 and 8) + +- Access to a local or remote platform with Singularity pre-installed and accessible to you as a user (i.e. no administrator/root access required). + - If you are attending a taught version of this material, it is expected that the course organisers will provide access to a platform (e.g. an institutional HPC cluster) that you can use for these sections of the material. +- The platform you will be using should also have MPI installed (required for episode 8). + +**Building containers:** (episodes 6 and 7) +Building containers requires access to a platform with an installation of Singularity on which you also have administrative access. If you run Linux and are comfortable with following the [Singularity installation instructions](https://sylabs.io/guides/3.5/admin-guide/installation.html), then installing Singularity directly on your system is an option. However, we strongly recommend using the [Docker Singularity container](https://quay.io/repository/singularity/singularity?tab=tags) for this section of the material. Details are provided on how to use the container in the relevant section of the lesson material. To support building containers, the prerequisite is therefore: + +- Access to a system with Docker installed on which you can run the Docker Singularity container. + + OR + +- Access to a local or remote Linux-based system on which you have administrator (root) access and can install the Singularity software. + +**Please note that the version of Singularity used in this part of the course is *version 3.5.3* which was the latest stable release at the time of writing.** If you are installing Singularity on your own system for use in the course, you are recommneded to install version 3.5.3. + + +:::::::::::::::::::::::::::::::::::::::::::::::::: + + + + diff --git a/instructors/instructor-notes.md b/instructors/instructor-notes.md new file mode 100644 index 0000000..865bee6 --- /dev/null +++ b/instructors/instructor-notes.md @@ -0,0 +1,29 @@ +--- +title: Instructor Notes +--- + +## Resouces for Instructors + +## Workshop Structure + +*[Instructors, please add notes here reporting on your experiences of teaching this module, either standalone, or as part of a wider workshop.]* + +- **Containers course covering Docker and Singularity:** This Singularity module is regularly taught alongside the [Introduction to Docker](https://github.com/carpentries-incubator/docker-introduction) module as part of a 2-day course "*Reproducible computational environments using containers*" run through the [ARCHER2 training programme](https://www.archer2.ac.uk/training/) in the UK. [See an example](https://www.archer2.ac.uk/training/courses/221207-containers/) of this course run in December 2022. + +This course has been run both online and in person. Experience suggests that this Singularity module requires between 5 and 6 hours of time to run effectively. The main aspect that takes a significant amount of time is the material at the end of the module looking at building a Singularity image containing an MPI code and then running this in parallel on an HPC platform. The variation in timing depends on how much experience the learners already have with running parallel jobs on HPC platforms and how much they wish to go into the details of the processes of running parallel jobs. For some groups of learners, the MPI use case is not something that is relevant and they may request to cover this material without the section on running parallel MPI jobs. In this case, the material can comfortably be compelted in 4 hours. + +## Technical tips and tricks + +- **HPC access:** Many learners will be keen to learn Singularity so that they can make use of it on a remote High Performance Computing (HPC) cluster. It is therefore strongly recommended that workshop organizers provide course attendees with access to an HPC platform that has Singularity pre-installed for undertaking this module. Where necessary, it is also recommended that guest accounts are set up and learners are asked to test access to the platform before the workshop. + +- **Use of the Singularity Docker container:** Singularity is a Linux tool. The optimal approach to building Singularity images, a key aspect of the material in this module, requires that the learner have a platform with Singularity installed, on which they have admin/root access. Since it's likely that many learners undertaking this module will not be using Linux as the main operating system on the computer on which they are undertaking the training, we need an alternative option. To address this, we use Docker to run the [Singularity Docker container](https://quay.io/repository/singularity/singularity). This ensures that learners have access to a local Singularity deployment (running within a Docker container) on which they have root access and can build Singularity images. The layers of indirection that this requires can prove confusing and we are looking at alternatives but from experience of teaching the module so far, this has proved to be the most reasonable solution at present. + +#### Pre-workshop Planning / Installation / Setup + +As highlighted above, this module is designed to support learners who wish to use Singularity on an HPC cluster. Elements of the module are therefore designed to be run on a High Performance Computing cluster (i.e. clusters that run SLURM, SGE, or other job scheduling software). It is possible for learners to undertake large parts of the module on their own computer. However, since a key use case for Singularity containers is their use on HPC infrastructure, it is recommended that an HPC platform be used in the teaching of this module. In such a case, if Singularity is not already on the cluster, admin rights would be required to install Singularity cluster-wide. Practically, it is likely that this will require a support ticket to be raised with cluster administrators and may require some time for them to investigate the software if they are unfamiliar with Singularity. + +## Common problems + +Some installs of Singularity require the use of `--bind` for compatibility between the container and the host system, if the host system does not have a directory that is a default or required directory in the container that directory will need to be bound elsewhere in order to work correctly: (i.e. `--bind /data:/mnt`) + + diff --git a/_extras/discuss.md b/learners/discuss.md similarity index 58% rename from _extras/discuss.md rename to learners/discuss.md index bfc33c5..515e3ba 100644 --- a/_extras/discuss.md +++ b/learners/discuss.md @@ -1,6 +1,9 @@ --- title: Discussion --- + FIXME -{% include links.md %} + + + diff --git a/learners/reference.md b/learners/reference.md new file mode 100644 index 0000000..cf6f94d --- /dev/null +++ b/learners/reference.md @@ -0,0 +1,11 @@ +--- +title: 'Glossary' +--- + +## Glossary + +FIXME + + + + diff --git a/setup.md b/learners/setup.md similarity index 97% rename from setup.md rename to learners/setup.md index 303f4ad..3d975e7 100644 --- a/setup.md +++ b/learners/setup.md @@ -1,10 +1,13 @@ --- title: Setup --- + If you are attending a taught version of this lesson, it is likely that the course organisers will provide access to a platform with Singularity and MPI pre-installed for undertaking parts of the lesson. You may be required to undertake an account registration process in order to gain access this platform. The course organisers will provide details in advance. For building containers, you'll need access to a platform with [Docker](https://www.docker.com/) installed, where either your user is in the `docker` group (i.e. you can run and commit containers and run other docker commands as your own user without having to prefix docker commands with `sudo`), or where your user is configured for sudo access on the system and you can run Docker commands when prefixing them with `sudo`. Beyond any account registration that may be required and the prerequisites described on the main lesson page, there is no further lesson setup to complete. -{% include links.md %} + + + diff --git a/profiles/learner-profiles.md b/profiles/learner-profiles.md new file mode 100644 index 0000000..434e335 --- /dev/null +++ b/profiles/learner-profiles.md @@ -0,0 +1,5 @@ +--- +title: FIXME +--- + +This is a placeholder file. Please add content here. diff --git a/reference.md b/reference.md deleted file mode 100644 index 8c82616..0000000 --- a/reference.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -layout: reference ---- - -## Glossary - -FIXME - -{% include links.md %} diff --git a/site/README.md b/site/README.md new file mode 100644 index 0000000..42997e3 --- /dev/null +++ b/site/README.md @@ -0,0 +1,2 @@ +This directory contains rendered lesson materials. Please do not edit files +here.