Starting a LLNL-Pascal config

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/compilers.yaml

@@ -0,0 +1,67 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+compilers:
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/tce/packages/xl/xl-2022.08.19/bin/xlc
+      cxx: /usr/tce/packages/xl/xl-2022.08.19/bin/xlC
+      f77: /usr/tce/packages/xl/xl-2022.08.19/bin/xlf
+      fc: /usr/tce/packages/xl/xl-2022.08.19/bin/xlf
+    flags:
+      cflags: -g -O2
+      cxxflags: -g -O2 -std=c++14
+      fflags: -g -O2
+    operating_system: rhel7
+    target: ppc64le
+    modules: [cuda/11.7.0, xl/2022.08.19-cuda-11.7.0]
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/tce/packages/xl/xl-2022.08.19/bin/xlc
+      cxx: /usr/tce/packages/xl/xl-2022.08.19/bin/xlC
+      f77: /usr/tce/packages/xl/xl-2022.08.19/bin/xlf
+      fc: /usr/tce/packages/xl/xl-2022.08.19/bin/xlf
+    flags:
+      cflags: -g -O2
+      cxxflags: -g -O2 -std=c++11
+      fflags: -g -O2
+    operating_system: rhel7
+    target: ppc64le
+    modules: [cuda/10.1.243, xl/2022.08.19] # TODO: Make these available to ramble or remove them entirely
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/tce/packages/xl/xl-2022.08.19-cuda-11.8.0/bin/xlc
+      cxx: /usr/tce/packages/xl/xl-2022.08.19-cuda-11.8.0/bin/xlC
+      f77: /usr/tce/packages/xl/xl-2022.08.19-cuda-11.8.0/bin/xlf
+      fc: /usr/tce/packages/xl/xl-2022.08.19-cuda-11.8.0/bin/xlf
+    flags: # TODO: Fix spack concretization bug
+      cflags: -g -O2
+      cxxflags: -g -O2 -std=c++11
+      fflags: -g -O2
+    operating_system: rhel7
+    target: ppc64le
+    modules: [cuda/11.8.0, xl/2022.08.19-cuda-11.8.0] # TODO: Make these available to ramble or remove them entirely
+    environment: {}
+    extra_rpaths: []
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gcc
+      cxx: /usr/tce/packages/gcc/gcc-8.3.1/bin/g++
+      f77: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran
+      fc: /usr/tce/packages/gcc/gcc-8.3.1/bin/gfortran
+    flags: {}
+    operating_system: rhel7
+    target: ppc64le
+    modules: []
+    environment: {}
+    extra_rpaths: []

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/auxiliary_software_files/packages.yaml

@@ -0,0 +1,73 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+packages:
+  tar:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+    buildable: false
+  cmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cmake/cmake-3.23.1
+    buildable: false
+  gmake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tcetmp/packages/gmake/gmake-4.2.1
+    buildable: false
+  automake:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  autoconf:
+    externals:
+    - spec: [email protected]
+      prefix: /usr
+  curand:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-10.1.243
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0
+    buildable: false
+  cusparse:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-10.1.243
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0
+    buildable: false
+  cuda:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-10.1.243
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0
+    buildable: false
+  blas:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-10.1.243
+    - spec: [email protected]
+      prefix: /usr/tce/packages/cuda/cuda-11.8.0
+    buildable: false
+  lapack:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tcetmp/packages/lapack/lapack-3.9.0-xl-2020.03.18
+    buildable: false
+  mpi:
+    externals:
+    - spec: [email protected]
+      prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-11.8.0
+      extra_attributes:
+        ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm"
+    - spec: [email protected]
+      prefix: /usr/tce/packages/spectrum-mpi/spectrum-mpi-rolling-release-xl-2022.08.19-cuda-10.1.243
+      extra_attributes:
+        ldflags: "-lmpiprofilesupport -lmpi_ibm_usempi -lmpi_ibm_mpifh -lmpi_ibm"
+    buildable: false

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/spack.yaml

@@ -0,0 +1,21 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+spack:
+  packages:
+    default-compiler:
+      spack_spec: XXXX-cuda{default_cuda_version}
+    default-mpi:
+      spack_spec:
+    compiler-gcc:
+      spack_spec: [email protected]
+    compiler-cuda10:
+      spack_spec: XXXXX-cuda{cuda10_version}
+    cublas-cuda-10:
+      spack_spec: cublas@{cuda10_version}
+    cublas-cuda-default:
+      spack_spec: cublas@{default_cuda_version}
+    lapack:
+      spack_spec: XXXX

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/system_definition.yaml

@@ -0,0 +1,32 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+system_definition:
+  name: Pascal
+  site: LLNL
+  system: Penguin-broadwell-P100-OmniPath
+  integrator:
+    vendor: Penguin
+    name:
+  processor:
+    vendor: Intel
+    name: Xeon_E5-2695_v4
+    ISA: x86_64
+    uArch: broadwell
+  accelerator:
+    vendor: NVIDIA
+    name: P100
+    ISA: PTX
+    uArch: sm_56
+  interconnect:
+    vendor: Cornelis
+    name: OmniPath
+  system-tested:
+    site: LLNL
+    name: pascal
+    installation-year: 2018
+    description:
+  top500-system-instances:
+    -

configs/LLNL-Pascal-Penguin-broadwell-P100-OmniPath/variables.yaml

@@ -0,0 +1,14 @@
+# Copyright 2023 Lawrence Livermore National Security, LLC and other
+# Benchpark Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+variables:
+  gtl_flag: ''  # to be overwritten by tests that need GTL
+  rocm_arch: 'gfx90a'
+  batch_time: '120m'
+  mpi_command: 'flux run -N {n_nodes} -n {n_ranks}'
+  batch_submit: 'flux batch {execute_experiment}'
+  batch_nodes: '# flux: -N {n_nodes}'
+  batch_ranks: '# flux: -n {n_ranks}'
+  batch_timeout: '# flux: -t {batch_time}'