Merge pull request #87 from lanl/ci_upgrade

Fixup CI after deployment change

.gitlab-ci.yml

@@ -6,39 +6,35 @@ stages:
   - build_n_test
 
 variables:
-  GIT_SUBMODULE_STRATEGY: recursive
-  SPINER_GCC_VERSION: "9.3.0"
+  XCAP_PROJECT_DIR: "/usr/projects/xcap/oss"
+  TESTING_CLUSTER: darwin
+  SPINER_GCC_VERSION: "10.3.0"
   SPINER_CUDA_VERSION: "11.6.0"
-  SPINER_SPACK_SPEC: "spiner@main+python+test%gcc@${SPINER_GCC_VERSION}"
+  SPINER_OPENMPI_VERSION: "4.1.1"
+  SPINER_SPACK_SPEC: "spiner@main+python+test%gcc@=${SPINER_GCC_VERSION}"
   COLOR_CYAN: "\e[1;36m"
   COLOR_PLAIN: "\e[0m"
-  # set to different spack upstream for testing
-  SPINER_TEMP_SPACK_DIR: ""
-  # uncomment for debugging the CI and persist the Spack installation
-  # SPACK_USER_CACHE_PATH: "${CI_PROJECT_DIR}/spack-local"
-  # DEVEL_SPACK_CHECKOUT: "${CI_PROJECT_DIR}/spack"
-
-before_script:
-  - echo "Running on $(hostname)"
-  - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
-  - export SPACK_DISABLE_LOCAL_CONFIG=true
-  - export SPACK_USER_CACHE_PATH=${SPACK_USER_CACHE_PATH:-/tmp/spack-local}
-  - export DEVEL_SPACK_CHECKOUT=${DEVEL_SPACK_CHECKOUT:-/tmp/spack}
-  - export SPINER_SPACK_DIR=${SPINER_TEMP_SPACK_DIR:=$XCAP_OSS_SPACK_DIR}
-  - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_CHECKOUT}"
-  - |
-    rsync -rpl \
-        --exclude=".git" \
-        --exclude="etc/spack/repos.yaml" \
-        --exclude="etc/spack/packages.yaml" \
-        --include="etc/spack/**" \
-        --include="lib/spack/**" \
-        --exclude-from=${SPINER_SPACK_DIR}/.gitignore \
-        ${SPINER_SPACK_DIR}/ ${DEVEL_SPACK_CHECKOUT}/
-  - source ${DEVEL_SPACK_CHECKOUT}/share/spack/setup-env.sh
-  - spack --version
-  - spack compiler list
-  - section end spack_init
+  # uncomment to have the CI Spack installation for debugging
+  # PROJECT_TMP_CI_DIR: "${CI_PROJECT_DIR}"
+  # uncomment & set to different spack upstream for testing
+  # PROJECT_TMP_SPACK_DIR: "deployment/CI/spack-v0.20.1-8"
+  # uncomment for spack debug output
+  # PROJECT_SPACK_DEBUG_FLAG: "-d"
+
+workflow:
+  rules:
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+    - if: $CI_PIPELINE_SOURCE == "pipeline"
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_TAG
+
+default:
+  tags:
+    - darwin-slurm-shared
+  id_tokens:
+    SITE_ID_TOKEN:
+      aud: https://asc-git.lanl.gov
 
 ##########################
 # Build Matrix Variables #
@@ -50,73 +46,91 @@ before_script:
 .buildtype_release: &buildtype_release
   BUILD_TYPE: "Release"
 
+######################
+# Platform Variables #
+######################
+
 .skylake: &skylake
     SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
 
 .a100: &a100
     SCHEDULER_PARAMETERS: "--nodes=1 --partition=shared-gpu-ampere"
 
-#################
-# General Setup #
-#################
-
-.job: &job_def
-  id_tokens:
-    SITE_ID_TOKEN:
-      aud: https://asc-git.lanl.gov
-  tags:
-    - darwin-slurm-shared
-  variables:
-    SCHEDULER_PARAMETERS: "--nodes=1 --partition=skylake-gold,skylake-platinum"
-  only:
-    refs:
-      - merge_requests
-      - pipelines
-  except:
-    refs:
-
 #######################
 # Job Script Template #
 #######################
 
 .test:
   stage: build_n_test
-  extends:
-    - .job
+  before_script:
+    - echo "Running on $(hostname)"
+    - section() { echo $'\e[0K'"section_$1:$(date +%s):$2"$'\r\e[0K'"${3+${COLOR_CYAN}$3${COLOR_PLAIN}}"; }
+    - export PYTHONNOUSERSITE=1
+    - export SPACK_DISABLE_LOCAL_CONFIG=true
+    - export SPACK_SKIP_MODULES=true
+    - export TMP_USER_PROJECT_DIR="/tmp/${USER}/${CI_PROJECT_NAME}/${CI_JOB_NAME}"
+    - export PROJECT_TMP_CI_DIR=${PROJECT_TMP_CI_DIR:-${TMP_USER_PROJECT_DIR}}
+    - export SPACK_USER_CACHE_PATH="${PROJECT_TMP_CI_DIR}/spack-local"
+    - export DEVEL_SPACK_ROOT="${PROJECT_TMP_CI_DIR}/spack"
+    - export DEVEL_SPACK_MIRROR="${CI_PROJECT_DIR}/spack-mirror"
+    - rm -rf ${SPACK_USER_CACHE_PATH} ${DEVEL_SPACK_ROOT}
+    - |
+      if [[ -z "${PROJECT_TMP_SPACK_DIR}" ]]; then
+        export PROJECT_SPACK_ROOT=${XCAP_PROJECT_DIR}/spack
+      else
+        export PROJECT_SPACK_ROOT="${XCAP_PROJECT_DIR}/${PROJECT_TMP_SPACK_DIR}-${TESTING_CLUSTER}"
+      fi
+    - section start "spack_init[collapsed=true]" "Creating Spack instance at ${DEVEL_SPACK_ROOT}"
+    - echo "Upstream ${PROJECT_SPACK_ROOT}"
+    - mkdir -m 2770 -p ${PROJECT_TMP_CI_DIR}
+    - |
+      rsync -rpl \
+          --exclude=".git" \
+          --exclude={"*__pycache__*","*.pyc"} \
+          --include="etc/spack/**" \
+          --include="lib/spack/**" \
+          --exclude-from=${PROJECT_SPACK_ROOT}/.gitignore \
+          ${PROJECT_SPACK_ROOT}/ ${DEVEL_SPACK_ROOT}/
+    - sed -i "s;xcap-admin;xcap;g" "${DEVEL_SPACK_ROOT}/etc/spack/packages.yaml"
+    - source ${DEVEL_SPACK_ROOT}/share/spack/setup-env.sh
+    - spack --version
+    - spack compiler list
+    - section end spack_init
   script:
     - module load gcc/${SPINER_GCC_VERSION}
     - module load cuda/${SPINER_CUDA_VERSION}
     - |
       if [[ ${CI_JOB_NAME} =~ "a100" ]];
       then
-        export SPINER_SPACK_SPEC="${SPINER_SPACK_SPEC}+hdf5+mpi+kokkos ^kokkos+wrapper+cuda cuda_arch=80";
+        module load openmpi/${SPINER_OPENMPI_VERSION}-gcc_${SPINER_GCC_VERSION}
+        export SPINER_SPACK_SPEC="${SPINER_SPACK_SPEC}+hdf5+mpi+kokkos ^kokkos+wrapper+cuda cuda_arch=80 ^openmpi@${SPINER_OPENMPI_VERSION}";
       fi
     - |
       section start "spack_build[collapsed=true]" "Building via Spack"
-      mkdir -p spack_build_env
-      spack env create -d spack_build_env
-      spack env activate -d spack_build_env
+      spack env create spack_build_env
+      spack env activate spack_build_env
+      spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
       spack repo add spack-repo
       spack repo list
       spack config add upstreams:default:install_tree:${SPINER_SPACK_DIR}/opt/spack/
       spack dev-build -q -j $(nproc) ${SPINER_SPACK_SPEC}
       spack env deactivate
       section end spack_build
     - section start "spack_env[collapsed=true]" "Creating Spack environment"
-    - mkdir -p spack_env
-    - spack env create -d spack_env
-    - spack env activate -d spack_env
+    - spack env create spack_env
+    - spack env activate spack_env
+    - spack repo add ${DEVEL_SPACK_ROOT}/var/spack/repos/xcap_deployment
     - spack repo add spack-repo
     - spack repo list
-    - spack config add upstreams:default:install_tree:${SPINER_SPACK_DIR}/opt/spack/
+    - spack config add upstreams:default:install_tree:${PROJECT_SPACK_ROOT}/opt/spack/
     - spack spec -I ${SPINER_SPACK_SPEC}
     - spack add ${SPINER_SPACK_SPEC}
     - spack install -j $(nproc) --show-log-on-error --no-checksum --yes-to-all -u cmake
     - section end spack_env
     - mkdir -p build
     - cd build
     - |
-      cmake --log-level=DEBUG \
+      cmake \
             -DSPINER_BUILD_TESTS=ON \
             -DCMAKE_INSTALL_PREFIX=${CI_PROJECT_DIR}/install \
             -DSPINER_USE_HDF=ON \
@@ -140,13 +154,11 @@ before_script:
 ########
 
 gnu_skylake:
-  <<: *job_def
   extends: .test
   variables:
     <<: *skylake
 
 gnu_a100:
-  <<: *job_def
   extends: .test
   variables:
     <<: *a100