Merge branch 'develop' into gcp-batch-backend-execution-actor-test

.github/workflows/integration_tests.yml

@@ -93,9 +93,9 @@ jobs:
             friendly_name: Centaur Engine Upgrade Local with MySQL 5.7
           - build_type: referenceDiskManifestBuilderApp
             friendly_name: Reference Disk Manifest Builder App
-#          - build_type: centaurSlurm
-#            build_mysql: 5.7
-#            friendly_name: "Centaur Slurm with MySQL 5.7"
+          - build_type: centaurSlurm
+            build_mysql: 5.7
+            friendly_name: "Centaur Slurm with MySQL 5.7"
           - build_type: centaurBlob
             build_mysql: 5.7
             friendly_name: Centaur Blob

docs/Ecosystem.md

@@ -3,4 +3,3 @@
 Many community projects have been created to support the use of the core Cromwell engine. Check them out to make your Cromwell experience even better!
 
 * [Cromshell](https://github.com/broadinstitute/cromshell): Shell script for interacting with cromwell servers created at the [Broad Institute](https://github.com/broadinstitute).
-* [Oliver](https://github.com/stjudecloud/oliver): An opinionated Cromwell orchestration manager created by the [St. Jude Cloud team](https://github.com/stjudecloud).

src/ci/bin/test_slurm.inc.sh

@@ -31,6 +31,21 @@ cromwell::build::slurm::setup_slurm_environment() {
     # Create various directories used by slurm
     sudo mkdir -p /var/run/munge
     sudo mkdir -p /var/spool/slurmd
+    sudo chown slurm:slurm /var/spool/slurmd
+
+    # Set up an AppArmor profile for Apptainer to allow non-root users to use it.
+    # This became required in Ubuntu 24.04
+    sudo tee /etc/apparmor.d/apptainer << 'EOF'
+abi <abi/4.0>,
+include <tunables/global>
+
+profile apptainer /usr/bin/apptainer{,-suid} flags=(unconfined) {
+  userns,
+}
+EOF
+
+    # Reload to get new profile created above
+    sudo systemctl reload apparmor
 
     # A mash of configure-until-it-runs. Feel free to PR suggestions/fixes.
     # https://slurm.schedmd.com/tutorials.html
@@ -40,13 +55,14 @@ cromwell::build::slurm::setup_slurm_environment() {
     cat <<SLURM_CONF | sudo tee /etc/slurm/slurm.conf >/dev/null
 ClusterName=localhost
 ControlMachine=localhost
-NodeName=localhost
-PartitionName=localpartition Nodes=localhost Default=YES
+NodeName=localhost CPUs=4 Sockets=1 CoresPerSocket=2 ThreadsPerCore=2
+PartitionName=localpartition Nodes=localhost Default=YES Oversubscribe=Force
 ProctrackType=proctrack/pgid
 ReturnToService=1
-SelectType=select/cons_res
+SelectType=select/cons_tres
 SelectTypeParameters=CR_CPU
 SlurmctldDebug=3
+StateSaveLocation=/var/spool/slurmd
 SLURM_CONF
 
     # Start the slurm master

src/ci/resources/slurm_application.conf

@@ -14,15 +14,15 @@ backend {
         # https://slurm.schedmd.com/sbatch.html
         submit-docker = """
             set -euo pipefail
-            CACHE_DIR=$HOME/.singularity/cache
+            CACHE_DIR=$HOME/.apptainer/cache
             mkdir -p $CACHE_DIR
-            LOCK_FILE=$CACHE_DIR/singularity_pull_flock
+            LOCK_FILE=$CACHE_DIR/apptainer_pull_flock
             DOCKER_NAME=$(sed -e 's/[^A-Za-z0-9._-]/_/g' <<< ${docker})
             IMAGE=$CACHE_DIR/$DOCKER_NAME.sif
             (
               flock --verbose --exclusive --timeout 900 9 || exit 1
               if [ ! -e "$IMAGE" ]; then
-                singularity build $IMAGE docker://${docker}
+                apptainer build $IMAGE docker://${docker}
               fi
             ) 9>$LOCK_FILE
 
@@ -31,7 +31,7 @@ backend {
               -D ${cwd} \
               -o ${cwd}/execution/stdout \
               -e ${cwd}/execution/stderr \
-              --wrap "singularity exec --containall --bind ${cwd}:${docker_cwd} $IMAGE ${job_shell} ${docker_script}"
+              --wrap "apptainer exec --containall --bind ${cwd}:${docker_cwd} $IMAGE ${job_shell} ${docker_script}"
         """
 
         # https://slurm.schedmd.com/scancel.html