From 14cde3e0a52de181a91f1c0e135f699e495d4d2d Mon Sep 17 00:00:00 2001 From: "Olga T. Pearce" Date: Fri, 19 Jan 2024 13:54:22 -0600 Subject: [PATCH 01/16] Adding a Fugaku system config --- .../spack.yaml | 15 ++++++++++ .../system_definition.yaml | 30 +++++++++++++++++++ .../variables.yaml | 12 ++++++++ 3 files changed, 57 insertions(+) create mode 100644 configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml create mode 100644 configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml create mode 100644 configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml new file mode 100644 index 000000000..57d07c043 --- /dev/null +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml @@ -0,0 +1,15 @@ +# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +spack: + packages: + default-compiler: + spack_spec: + default-mpi: + spack_spec: + blas: + spack_spec: + lapack: + spack_spec: diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml new file mode 100644 index 000000000..3ed2979f2 --- /dev/null +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml @@ -0,0 +1,30 @@ +# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +system_definition: + name: RIKEN-Fugaku-Fujitsu-A64FX-TofuD + integrator: + vendor: Fujitsu + name: + processor: + vendor: ARM + name: A64FX + ISA: Armv8.2-A-SVE + uArch: A64FX + accelerator: + vendor: + name: + ISA: + uArch: + interconnect: + vendor: Fujitsu + name: TofuD + system-tested: + site: RIKEN + name: Fugaku + installation-year: 2020 + description: top500 + top500-system-instances: + - Fugaku (RIKEN) diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml new file mode 100644 index 000000000..e4674cde0 --- /dev/null +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml @@ -0,0 +1,12 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + +variables: + batch_time: '02:00' + mpi_command: 'srun -N {n_nodes} -n {n_ranks}' + batch_submit: 'sbatch {execute_experiment}' + batch_nodes: '#SBATCH -N {n_nodes}' + batch_ranks: '#SBATCH -n {n_ranks}' + batch_timeout: '#SBATCH -t {batch_time}:00' From 1aa97f63dada59f6a526b774191a41cf39d261a6 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Fri, 19 Jan 2024 14:47:14 -0600 Subject: [PATCH 02/16] license --- configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml index 57d07c043..e237becf0 100644 --- a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml @@ -1,4 +1,4 @@ -# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other +# Copyright 2023 Lawrence Livermore National Security, LLC and other # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: Apache-2.0 From c495c0b8f0c2b6843a8b19c5e85ec1b064bfbc64 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Fri, 19 Jan 2024 14:47:31 -0600 Subject: [PATCH 03/16] Update system_definition.yaml --- configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml index 3ed2979f2..70ff51891 100644 --- a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml @@ -1,4 +1,4 @@ -# Copyright 2023-2024 Lawrence Livermore National Security, LLC and other +# Copyright 2023 Lawrence Livermore National Security, LLC and other # Benchpark Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: Apache-2.0 From a43a3eb8c3e87a118284066dd7d1601e097f36c5 Mon Sep 17 00:00:00 2001 From: "Olga T. Pearce" Date: Sat, 20 Jan 2024 13:26:05 -0600 Subject: [PATCH 04/16] adding site --- .../RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml index 70ff51891..5f6fe8686 100644 --- a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml @@ -4,7 +4,9 @@ # SPDX-License-Identifier: Apache-2.0 system_definition: - name: RIKEN-Fugaku-Fujitsu-A64FX-TofuD + name: Fugaku + site: RIKEN + system: Fujitsu-A64FX-TofuD integrator: vendor: Fujitsu name: From fe9cf4932fa9c746d0b771d6114abe8bfe4d9f14 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 29 Jan 2024 15:38:21 -0800 Subject: [PATCH 05/16] fix code to increase content width of sphinx docs --- docs/_static/css/custom.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index 8aa6c288f..b07bdb1b5 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,3 +1,3 @@ .wy-nav-content { - max-width: 1200px !important; + max-width: none; } From b131fe6123cf27fac86c165f3c8cd0329a61ee38 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Tue, 30 Jan 2024 13:27:48 +0900 Subject: [PATCH 06/16] Initial compilers for Fugaku --- configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml | 12 ++++++++---- .../system_definition.yaml | 4 ++-- .../RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml | 12 ++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml index e237becf0..433fbd4e2 100644 --- a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml @@ -6,10 +6,14 @@ spack: packages: default-compiler: - spack_spec: + spack_spec: fcc # + compiler-gcc: + spack_spec: gcc + compiler-intel: + spack_spec: intel default-mpi: - spack_spec: + spack_spec: fujitsu-mpi # mpifcc, mpich-tofu blas: - spack_spec: + spack_spec: blas lapack: - spack_spec: + spack_spec: lapack diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml index 5f6fe8686..99b530e9f 100644 --- a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml @@ -14,7 +14,7 @@ system_definition: vendor: ARM name: A64FX ISA: Armv8.2-A-SVE - uArch: A64FX + uArch: aarch accelerator: vendor: name: @@ -26,7 +26,7 @@ system_definition: system-tested: site: RIKEN name: Fugaku - installation-year: 2020 + installation-year: 2019 description: top500 top500-system-instances: - Fugaku (RIKEN) diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml index e4674cde0..fec0aeeb3 100644 --- a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml @@ -4,9 +4,9 @@ # SPDX-License-Identifier: Apache-2.0 variables: - batch_time: '02:00' - mpi_command: 'srun -N {n_nodes} -n {n_ranks}' - batch_submit: 'sbatch {execute_experiment}' - batch_nodes: '#SBATCH -N {n_nodes}' - batch_ranks: '#SBATCH -n {n_ranks}' - batch_timeout: '#SBATCH -t {batch_time}:00' + batch_time: '' + mpi_command: 'mpirun -n {n_ranks} --oversubscribe' + batch_submit: '{execute_experiment}' + batch_nodes: '' + batch_ranks: '' + batch_timeout: '' From 3d64465a529fe7421c109acf5986e08b9db7eec2 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Tue, 30 Jan 2024 13:38:23 +0900 Subject: [PATCH 07/16] thread counts for fugaku --- experiments/amg2023/openmp/ramble.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experiments/amg2023/openmp/ramble.yaml b/experiments/amg2023/openmp/ramble.yaml index a6612e3d1..8cc2584dc 100644 --- a/experiments/amg2023/openmp/ramble.yaml +++ b/experiments/amg2023/openmp/ramble.yaml @@ -34,7 +34,7 @@ ramble: experiment_setup: '' processes_per_node: ['8', '4'] n_nodes: ['1', '2'] - threads_per_node_core: ['8', '10', '13'] #TODO: Specify n_threads according to available n_nodes and n_ranks + threads_per_node_core: ['4', '6', '12'] #TODO: Specify n_threads according to available n_nodes and n_ranks omp_num_threads: '{threads_per_node_core} * {n_nodes}' experiments: amg2023_omp_problem1_{n_nodes}_{omp_num_threads}_{px}_{py}_{pz}_{nx}_{ny}_{nz}: From 897c70ca32afc7ba10093436c6ead7f105ce65b8 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Tue, 30 Jan 2024 14:21:02 +0900 Subject: [PATCH 08/16] adding PJM scheduler instructions https://matsci.dev/blog/2023/fugaku/ --- configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml index fec0aeeb3..d9e3aa4d4 100644 --- a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml +++ b/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml @@ -4,8 +4,13 @@ # SPDX-License-Identifier: Apache-2.0 variables: - batch_time: '' + batch_time: '02:00' mpi_command: 'mpirun -n {n_ranks} --oversubscribe' + batch_submit: 'qsub {execute_experiment}' + batch_nodes: '#PJM -L node={n_nodes}' + #batch_ranks: '#PJM --mpi max_proc_per_node=48' ## -n {n_ranks} + batch_timeout: '#PJM -L "elapse={batch_time}:00"' + batch_submit: '{execute_experiment}' batch_nodes: '' batch_ranks: '' From 59d5aca738715aa6f0edbe85163661fef1393c84 Mon Sep 17 00:00:00 2001 From: Stephanie Brink Date: Mon, 29 Jan 2024 23:05:42 -0800 Subject: [PATCH 09/16] add updated table of system definitions with fugaku --- docs/tables/current-system-definitions.csv | 40 +++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/tables/current-system-definitions.csv b/docs/tables/current-system-definitions.csv index f4972971b..9768e5de4 100644 --- a/docs/tables/current-system-definitions.csv +++ b/docs/tables/current-system-definitions.csv @@ -1,20 +1,20 @@ -,Magma,Sierra,Tioga,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,x86_64 -**site**,LLNL,LLNL,LLNL,,, -**system**,Penguin-icelake-OmniPath,IBM-power9-V100-Infiniband,HPECray-zen3-MI250X-Slingshot,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,x86_64 -**integrator.vendor**,PenguinComputing,IBM,HPECray,AWS,HPECray, -**integrator.name**,RelionCluster,AC922,EX235a,ParallelCluster3.7.2-Hpc7a,EX235a, -**processor.vendor**,Intel,IBM,AMD,AMD,AMD, -**processor.name**,XeonPlatinum924248C,POWER9,EPYC-Zen3,EPYC-Zen4,EPYC-Zen3, -**processor.ISA**,x86_64,ppc64le,x86_64,x86_64,x86_64,x86_64 -**processor.uArch**,icelake,power9,zen3,zen4,zen3, -**accelerator.vendor**,,NVIDIA,AMD,,AMD, -**accelerator.name**,,V100,MI250X,,MI250X, -**accelerator.ISA**,,PTX,GCN,,GCN, -**accelerator.uArch**,,sm_70,gfx90a,,gfx90a, -**interconnect.vendor**,Intel,Mellanox,HPECray,AWS,HPECray, -**interconnect.name**,OmniPath,EDR-Infiniband,Slingshot11,EFA,Slingshot11, -**system-tested.site**,LLNL,LLNL,LLNL,AWS,LLNL, -**system-tested.name**,magma,lassen,tioga,,tioga, -**system-tested.installation-year**,2019,2018,2022,,2022, -**system-tested.description**,`top500 `_,`top500 `_,`top500 `_,`aws/hpc7a `_,`top500 `_, -**top500-system-instances**,Magma (LLNL),Sierra (LLNL),"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", +,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Fugaku,Tioga,x86_64 +**site**,LLNL,,,LLNL,RIKEN,LLNL, +**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,Fujitsu-A64FX-TofuD,HPECray-zen3-MI250X-Slingshot,x86_64 +**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,Fujitsu,HPECray, +**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,,EX235a, +**processor.vendor**,IBM,AMD,AMD,Intel,ARM,AMD, +**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,A64FX,EPYC-Zen3, +**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64 +**processor.uArch**,power9,zen4,zen3,icelake,A64FX,zen3, +**accelerator.vendor**,NVIDIA,,AMD,,,AMD, +**accelerator.name**,V100,,MI250X,,,MI250X, +**accelerator.ISA**,PTX,,GCN,,,GCN, +**accelerator.uArch**,sm_70,,gfx90a,,,gfx90a, +**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,Fujitsu,HPECray, +**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,TofuD,Slingshot11, +**system-tested.site**,LLNL,AWS,LLNL,LLNL,RIKEN,LLNL, +**system-tested.name**,lassen,,tioga,magma,Fugaku,tioga, +**system-tested.installation-year**,2018,,2022,2019,2020,2022, +**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,`top500 `_, +**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),Fugaku (RIKEN),"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", From c0e1fae38bea982dbe35465465604f30b4dbe0d3 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Wed, 21 Feb 2024 16:48:54 +1100 Subject: [PATCH 10/16] renaming RIKEN to RCCS --- .../RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml | 0 .../RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml | 0 .../RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename configs/{ => RCCS-Fugaku-Fujitsu-A64FX-TofuD}/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml (100%) rename configs/{ => RCCS-Fugaku-Fujitsu-A64FX-TofuD}/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml (100%) rename configs/{ => RCCS-Fugaku-Fujitsu-A64FX-TofuD}/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml (100%) diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml similarity index 100% rename from configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml rename to configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml similarity index 100% rename from configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml rename to configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml diff --git a/configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml similarity index 100% rename from configs/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml rename to configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml From a1482f151904978e66684de7431d215833843ac6 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Wed, 21 Feb 2024 16:57:56 +1100 Subject: [PATCH 11/16] Removing extra dir --- .../{RIKEN-Fugaku-Fujitsu-A64FX-TofuD => }/spack.yaml | 0 .../{RIKEN-Fugaku-Fujitsu-A64FX-TofuD => }/system_definition.yaml | 0 .../{RIKEN-Fugaku-Fujitsu-A64FX-TofuD => }/variables.yaml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/{RIKEN-Fugaku-Fujitsu-A64FX-TofuD => }/spack.yaml (100%) rename configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/{RIKEN-Fugaku-Fujitsu-A64FX-TofuD => }/system_definition.yaml (100%) rename configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/{RIKEN-Fugaku-Fujitsu-A64FX-TofuD => }/variables.yaml (100%) diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml similarity index 100% rename from configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml rename to configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml similarity index 100% rename from configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml rename to configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml similarity index 100% rename from configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/RIKEN-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml rename to configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml From 354a4708c6b5fc6fdf0918c385528aa2cb5afb39 Mon Sep 17 00:00:00 2001 From: jdomke <28772296+jdomke@users.noreply.github.com> Date: Wed, 21 Feb 2024 17:07:07 +1100 Subject: [PATCH 12/16] initial draft for fugaku config (#142) * initial draft for fugaku config * Add integrator name * Removing extra dir --------- Co-authored-by: domke <673751-domke@users.noreply.gitlab.com> Co-authored-by: pearce8 --- .../auxiliary_software_files/compilers.yaml | 131 ++++ .../auxiliary_software_files/packages.yaml | 581 ++++++++++++++++++ .../spack.yaml | 20 +- .../system_definition.yaml | 18 +- .../variables.yaml | 14 +- 5 files changed, 738 insertions(+), 26 deletions(-) create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml create mode 100644 configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml new file mode 100644 index 000000000..c0c3480a6 --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml @@ -0,0 +1,131 @@ +compilers: +- compiler: + spec: fj@4.10.0 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.38/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.38/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.38/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.38/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.38/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.38/lib64 + extra_rpaths: [] +- compiler: + spec: fj@4.8.1 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.36/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.36/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.36/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.36/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.36/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.36/lib64 + extra_rpaths: [] +- compiler: + spec: fj@4.8.0 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.35/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.35/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.35/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.35/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.35/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.35/lib64 + extra_rpaths: [] +- compiler: + spec: fj@4.7.0 + modules: [] + paths: + cc: /opt/FJSVxtclanga/tcsds-1.2.34/bin/fcc + cxx: /opt/FJSVxtclanga/tcsds-1.2.34/bin/FCC + f77: /opt/FJSVxtclanga/tcsds-1.2.34/bin/frt + fc: /opt/FJSVxtclanga/tcsds-1.2.34/bin/frt + flags: {} + operating_system: rhel8 + target: aarch64 + environment: + set: + fcc_ENV: -Nclang + FCC_ENV: -Nclang + prepend_path: + PATH: /opt/FJSVxtclanga/tcsds-1.2.34/bin + LD_LIBRARY_PATH: /opt/FJSVxtclanga/tcsds-1.2.34/lib64 + extra_rpaths: [] +- compiler: + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + operating_system: rhel8 + target: aarch64 + modules: [] + environment: + unset: [] + extra_rpaths: [] + flags: {} + spec: gcc@8.5.0 +- compiler: + paths: + cc: /usr/bin/gcc + cxx: /usr/bin/g++ + f77: /usr/bin/gfortran + fc: /usr/bin/gfortran + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + unset: [] + extra_rpaths: [] + flags: {} + spec: gcc@8.4.1 +- compiler: + spec: gcc@12.2.0 + paths: + cc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin/gcc + cxx: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin//g++ + f77: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin/gfortran + fc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-skylake_avx512/gcc-8.4.1/gcc-12.2.0-bjidm56oz6mejysl3wvjr3mbn7sqk3ec/bin/gfortran + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: {} + extra_rpaths: [] +- compiler: + spec: gcc@12.2.0 + paths: + cc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/gcc + cxx: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/g++ + f77: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/gfortran + fc: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/gcc-8.5.0/gcc-12.2.0-sxcx7kmt3qiktffgzzvrj2wmup3g32bc/bin/gfortran + flags: {} + operating_system: rhel8 + target: aarch64 + modules: [] + environment: {} + extra_rpaths: [] diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml new file mode 100644 index 000000000..be32cb59c --- /dev/null +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml @@ -0,0 +1,581 @@ +packages: + all: + compiler: [fj, gcc] + providers: + mpi: [fujitsu-mpi, openmpi, mpich] + blas: [fujitsu-ssl2, openblas] + lapack: [fujitsu-ssl2, openblas] + scalapack: [fujitsu-ssl2, netlib-scalapack] + fftw-api: [fujitsu-fftw, fftw, rist-fftw] + permissions: + write: group + htslib: + version: [1.12] + python: + externals: + - spec: "python@3.10.8%fj +ssl arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/fj-4.8.1/python-3.10.8-7q66snjvhvy7im57hncbgpirmddrb5sk + - spec: "python@3.10.8%fj +ssl+tkinter arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-a64fx/fj-4.8.1/python-3.10.8-5q3ncyl2my7oomopsmukduqo36u6pnkg + - spec: "python@3.10.8%gcc +ssl arch=linux-rhel8-cascadelake" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-cascadelake/gcc-12.2.0/python-3.10.8-yt6afcnywa36aebxovs3ldscknyzlva3 + - spec: "python@3.10.8%gcc +ssl+tkinter arch=linux-rhel8-cascadelake" + prefix: /vol0004/apps/oss/spack-v0.19/opt/spack/linux-rhel8-cascadelake/gcc-12.2.0/python-3.10.8-a5u7ucknpzdipu2dd3d7hf2ybkdyprly + openssh: + permissions: + write: user + fujitsu-mpi: + version: [head, 4.8.1, 4.8.0, 4.7.0] + externals: + - spec: "fujitsu-mpi@head%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-mpi-latest + - spec: "fujitsu-mpi@4.8.1%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-mpi-1.2.36 + - spec: "fujitsu-mpi@4.8.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-mpi-1.2.35 + - spec: "fujitsu-mpi@4.7.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-1.2.34 + - spec: "fujitsu-mpi@head%gcc@12.2.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/mpigcc/fjmpi-gcc12 + - spec: "fujitsu-mpi@head%gcc@10.4.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/mpigcc/fjmpi-gcc10 + - spec: "fujitsu-mpi@head%gcc@8.5.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/mpigcc/fjmpi-gcc8 + buildable: False + fujitsu-ssl2: + version: [head, 4.8.1, 4.8.0, 4.7.0] + externals: + - spec: "fujitsu-ssl2@head%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-ssl2-latest + - spec: "fujitsu-ssl2@4.8.1%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-ssl2-1.2.36 + - spec: "fujitsu-ssl2@4.8.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-ssl2-1.2.35 + - spec: "fujitsu-ssl2@4.7.0%fj arch=linux-rhel8-a64fx" + prefix: /opt/FJSVxtclanga/tcsds-1.2.34 + buildable: False + rist-fftw: + externals: + - spec: "rist-fftw@3.3.9-272-g63d6bd70 arch=linux-rhel8-a64fx" + prefix: /vol0004/share/rist/fftw/gcc-10.3.0/3.3.9-272-g63d6bd70 + buildable: False + ntchem: + externals: + - spec: "ntchem@13.0.0%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/ntchem/aarch64/NTChem2013/13.0.0/tcsds-1.2.36/release_mpiomp + - spec: "ntchem@12.2.0%fj arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/ntchem/aarch64/NTChem2013/12.2.0/tcsds-1.2.34/release_mpiomp + buildable: False + abinitmp: + externals: + - spec: "abinitmp@1-22%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/SPACK-Feb2023-ABINIT-MP-VER1-REV22 + - spec: "abinitmp@2-4%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/SPACK-Feb2023-ABINIT-MP-VER2-REV4 + - spec: "abinitmp@2-8%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/SPACK-Jan2024-ABINIT-MP-VER2-REV8 + buildable: False + fds: + externals: + - spec: "fds@6.7.7%fj@4.8.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/FDS/fds-FDS6.7.7/Build/mpi_fugaku + - spec: "fds@6.7.9%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/FDS/fds-FDS6.7.9/Build/mpi_fugaku + buildable: False + ffvhc-ace: + externals: + - spec: "ffvhc-ace@0.1%fj@4.8.1 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/FFVHC-ACE/0.1/bin + buildable: False + quantum-espresso: + externals: + - spec: "quantum-espresso@6.5%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.5/bin + - spec: "quantum-espresso@6.6%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.6/bin + - spec: "quantum-espresso@6.7%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.7/bin + - spec: "quantum-espresso@6.8%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-6.8/bin + - spec: "quantum-espresso@7.0%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.0/bin + - spec: "quantum-espresso@7.1%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.1/bin + - spec: "quantum-espresso@7.2%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.2/bin + - spec: "quantum-espresso@7.3%fj@4.10.0 arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/opt/qe-7.3/bin + buildable: False + autoconf: + externals: + - spec: "autoconf@2.69 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "autoconf@2.69 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "autoconf@2.69 arch=linux-rhel8-skylake_avx512" + prefix: /usr + automake: + externals: + - spec: "automake@1.16.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "automake@1.16.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "automake@1.16.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + binutils: + externals: + - spec: "binutils@2.30 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "binutils@2.30 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "binutils@2.30 arch=linux-rhel8-skylake_avx512" + prefix: /usr + bzip2: + externals: + - spec: "bzip2@1.0.6 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "bzip2@1.0.6 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "bzip2@1.0.6 arch=linux-rhel8-skylake_avx512" + prefix: /usr + curl: + externals: + - spec: "curl@7.61.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "curl@7.61.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "curl@7.61.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + dbus: + externals: + - spec: "dbus@1.12.8 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "dbus@1.12.8 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "dbus@1.12.8 arch=linux-rhel8-skylake_avx512" + prefix: /usr + elfutils: + externals: + - spec: "elfutils@0.186 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "elfutils@0.182 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "elfutils@0.182 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "elfutils@0.182 arch=linux-rhel8-skylake_avx512" + prefix: /usr + expat: + externals: + - spec: "expat@2.2.5 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "expat@2.2.5 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "expat@2.2.5 arch=linux-rhel8-skylake_avx512" + prefix: /usr + findutils: + externals: + - spec: "findutils@4.6.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "findutils@4.6.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "findutils@4.6.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + fontconfig: + externals: + - spec: "fontconfig@2.13.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "fontconfig@2.13.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "fontconfig@2.13.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + freetype: + externals: + - spec: "freetype@2.9.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "freetype@2.9.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "freetype@2.9.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gmake: + externals: + - spec: "gmake@4.2.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gmake@4.2.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gmake@4.2.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gdbm: + externals: + - spec: "gdbm@1.18 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gdbm@1.18 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gdbm@1.18 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gettext: + externals: + - spec: "gettext@0.19.8.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gettext@0.19.8.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gettext@0.19.8.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gmp: + externals: + - spec: "gmp@6.1.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gmp@6.1.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gmp@6.1.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + gnutls: + externals: + - spec: "gnutls@3.6.16 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gnutls@3.6.14 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "gnutls@3.6.14 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "gnutls@3.6.14 arch=linux-rhel8-skylake_avx512" + prefix: /usr + hwloc: + externals: + - spec: "hwloc@2.2.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "hwloc@2.2.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "hwloc@2.2.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + jansson: + externals: + - spec: "jansson@2.14 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "jansson@2.11 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "jansson@2.11 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libaio: + externals: + - spec: "libaio@0.3.112 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libaio@0.3.112 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libaio@0.3.112 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libcap: + externals: + - spec: "libcap@2.48 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libcap@2.26 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libcap@2.26 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libdrm: + externals: + - spec: "libdrm@2.4.108 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libdrm@2.4.103 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libdrm@2.4.103 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libdrm@2.4.103 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libedit: + externals: + - spec: "libedit@3.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libedit@3.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libedit@3.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libevent: + externals: + - spec: "libevent@2.1.8 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libevent@2.1.8 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libevent@2.1.8 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libfabric: + externals: + - spec: "libfabric@1.14.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libfabric@1.11.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libfabric@1.11.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libffi: + externals: + - spec: "libffi@3.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libffi@3.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libffi@3.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libglvnd: + externals: + - spec: "libglvnd@1.3.4 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libglvnd@1.3.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libglvnd@1.3.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libibumad: + externals: + - spec: "libibumad@37.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libibumad@32.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libibumad@54mlnx1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libibumad@54mlnx1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libpciaccess: + externals: + - spec: "libpciaccess@0.14 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libpciaccess@0.14 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libpciaccess@0.14 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libpng: + externals: + - spec: "libpng@1.6.34 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libpng@1.6.34 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libpng@1.6.34 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libtasn1: + externals: + - spec: "libtasn1@4.13 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libtasn1@4.13 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libtasn1@4.13 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libtirpc: + externals: + - spec: "libtirpc@1.1.4 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libtirpc@1.1.4 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libtirpc@1.1.4 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libtool: + externals: + - spec: "libtool@2.4.6 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libtool@2.4.6 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libtool@2.4.6 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libuuid: + externals: + - spec: "libuuid@2.32.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libuuid@2.32.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libuuid@2.32.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libxcb: + externals: + - spec: "libxcb@1.13.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libxcb@1.13.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libxcb@1.13.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libxkbcommon: + externals: + - spec: "libxkbcommon@0.9.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libxkbcommon@0.9.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libxkbcommon@0.9.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + libxml2: + externals: + - spec: "libxml2@2.9.7 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "libxml2@2.9.7 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "libxml2@2.9.7 arch=linux-rhel8-skylake_avx512" + prefix: /usr + lz4: + externals: + - spec: "lz4@1.8.3 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "lz4@1.8.3 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "lz4@1.8.3 arch=linux-rhel8-skylake_avx512" + prefix: /usr + m4: + externals: + - spec: "m4@1.4.18 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "m4@1.4.18 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "m4@1.4.18 arch=linux-rhel8-skylake_avx512" + prefix: /usr + ncurses: + externals: + - spec: "ncurses@6.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "ncurses@6.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "ncurses@6.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + nettle: + externals: + - spec: "nettle@3.4.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "nettle@3.4.1 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "nettle@3.4.1 arch=linux-rhel8-skylake_avx512" + prefix: /usr + nspr: + externals: + - spec: "nspr@4.32.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "nspr@4.25.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "nspr@4.25.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "nspr@4.25.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + numactl: + externals: + - spec: "numactl@2.0.12 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "numactl@2.0.12 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "numactl@2.0.12 arch=linux-rhel8-skylake_avx512" + prefix: /usr + opengl: + buildable: False + externals: + - spec: opengl@4.5.0 + prefix: /usr + openssl: + buildable: False + externals: + - spec: "openssl@1.1.1k arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "openssl@1.1.1g arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "openssl@1.1.1k arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "openssl@1.1.1k arch=linux-rhel8-skylake_avx512" + prefix: /usr + papi: + externals: + - spec: "papi@5.6.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "papi@5.6.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "papi@5.6.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + pcre: + externals: + - spec: "pcre@8.42 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "pcre@8.42 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "pcre@8.42 arch=linux-rhel8-skylake_avx512" + prefix: /usr + perl: + externals: + - spec: "perl@5.26.3 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "perl@5.26.3 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "perl@5.26.3 arch=linux-rhel8-skylake_avx512" + prefix: /usr + pkgconf: + externals: + - spec: "pkgconf@1.4.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "pkgconf@1.4.2 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "pkgconf@1.4.2 arch=linux-rhel8-skylake_avx512" + prefix: /usr + popt: + externals: + - spec: "popt@1.18 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "popt@1.18 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "popt@1.18 arch=linux-rhel8-skylake_avx512" + prefix: /usr + readline: + externals: + - spec: "readline@7.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "readline@7.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "readline@7.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + sqlite: + externals: + - spec: "sqlite@3.26.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "sqlite@3.26.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "sqlite@3.26.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + tcl: + externals: + - spec: "tcl@8.6.8 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "tcl@8.6.8 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "tcl@8.6.8 arch=linux-rhel8-skylake_avx512" + prefix: /usr + ucx: + externals: + - spec: "ucx@1.11.2 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "ucx@1.9.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "ucx@1.11.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "ucx@1.11.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + valgrind: + externals: + - spec: "valgrind@3.18.1 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "valgrind@3.16.0 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "valgrind@3.16.0 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "valgrind@3.16.0 arch=linux-rhel8-skylake_avx512" + prefix: /usr + xz: + externals: + - spec: "xz@5.2.4 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "xz@5.2.4 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "xz@5.2.4 arch=linux-rhel8-skylake_avx512" + prefix: /usr + zlib: + externals: + - spec: "zlib@1.2.11 arch=linux-rhel8-a64fx" + prefix: /usr + - spec: "zlib@1.2.11 arch=linux-rhel8-cascadelake" + prefix: /usr + - spec: "zlib@1.2.11 arch=linux-rhel8-skylake_avx512" + prefix: /usr + buildable: False + # pmlib: had problems with spack. so far binary packages only. 2023/3/20 mikami + pmlib: + externals: + - spec: "pmlib@9.0-clang-precise arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-clang-precise + - spec: "pmlib@9.0-clang-power arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-clang-power + - spec: "pmlib@9.0-trad-power arch=linux-rhel8-a64fx" + prefix: /vol0004/apps/oss/pmlib-v9.0/9.0-trad-power + buildable: False diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml index 433fbd4e2..bf2cf20c8 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/spack.yaml @@ -6,14 +6,18 @@ spack: packages: default-compiler: - spack_spec: fcc # - compiler-gcc: - spack_spec: gcc - compiler-intel: - spack_spec: intel + spack_spec: fj@{default_fujitsu_version} default-mpi: - spack_spec: fujitsu-mpi # mpifcc, mpich-tofu + spack_spec: fujitsu-mpi@head%fj@{default_fujitsu_version} + compiler-gcc: + spack_spec: gcc@12.2.0 + compiler-fujitsu: + spack_spec: fj@{default_fujitsu_version} blas: - spack_spec: blas + spack_spec: fujitsu-ssl2@head%fj@{default_fujitsu_version} lapack: - spack_spec: lapack + spack_spec: fujitsu-ssl2@head%fj@{default_fujitsu_version} + mpi-gcc: + spack_spec: fujitsu-mpi@head%gcc@12.2.0 + mpi-fujitsu: + spack_spec: fujitsu-mpi@head%fj@{default_fujitsu_version} \ No newline at end of file diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml index 99b530e9f..afdbd6d63 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/system_definition.yaml @@ -5,16 +5,16 @@ system_definition: name: Fugaku - site: RIKEN + site: RIKEN Center for Computational Science system: Fujitsu-A64FX-TofuD integrator: vendor: Fujitsu - name: + name: FX1000 processor: - vendor: ARM + vendor: Fujitsu name: A64FX ISA: Armv8.2-A-SVE - uArch: aarch + uArch: aarch64 accelerator: vendor: name: @@ -22,11 +22,11 @@ system_definition: uArch: interconnect: vendor: Fujitsu - name: TofuD + name: TofuInterconnectD system-tested: - site: RIKEN + site: R-CCS name: Fugaku - installation-year: 2019 - description: top500 + installation-year: 2020 + description: top500 top500-system-instances: - - Fugaku (RIKEN) + - Fugaku (R-CCS) \ No newline at end of file diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml index d9e3aa4d4..046ce5155 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/variables.yaml @@ -5,13 +5,9 @@ variables: batch_time: '02:00' - mpi_command: 'mpirun -n {n_ranks} --oversubscribe' - batch_submit: 'qsub {execute_experiment}' - batch_nodes: '#PJM -L node={n_nodes}' - #batch_ranks: '#PJM --mpi max_proc_per_node=48' ## -n {n_ranks} + mpi_command: 'mpiexec' + batch_submit: 'pjsub {execute_experiment}' + batch_nodes: '#PJM -L "node={n_nodes}"' + batch_ranks: '#PJM --mpi proc={n_ranks}' batch_timeout: '#PJM -L "elapse={batch_time}:00"' - - batch_submit: '{execute_experiment}' - batch_nodes: '' - batch_ranks: '' - batch_timeout: '' + default_fujitsu_version: '4.8.1' \ No newline at end of file From 08ea49d56ab4bea3c177665fbfb08db3c9b938bc Mon Sep 17 00:00:00 2001 From: pearce8 Date: Wed, 21 Feb 2024 22:47:40 +1100 Subject: [PATCH 13/16] license --- .../auxiliary_software_files/compilers.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml index c0c3480a6..25129f761 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/compilers.yaml @@ -1,3 +1,8 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + compilers: - compiler: spec: fj@4.10.0 From 0e2af341e34c09e46e6b3792fc8b9ff233567bc0 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Wed, 21 Feb 2024 22:49:52 +1100 Subject: [PATCH 14/16] column width --- docs/_static/css/custom.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index b07bdb1b5..8aa6c288f 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,3 +1,3 @@ .wy-nav-content { - max-width: none; + max-width: 1200px !important; } From e516391cec8066bc16dc63395bf681065b82afc9 Mon Sep 17 00:00:00 2001 From: pearce8 Date: Wed, 21 Feb 2024 23:29:21 +1100 Subject: [PATCH 15/16] license --- .../auxiliary_software_files/packages.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml index be32cb59c..931188040 100644 --- a/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml +++ b/configs/RCCS-Fugaku-Fujitsu-A64FX-TofuD/auxiliary_software_files/packages.yaml @@ -1,3 +1,8 @@ +# Copyright 2023 Lawrence Livermore National Security, LLC and other +# Benchpark Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: Apache-2.0 + packages: all: compiler: [fj, gcc] From e83de75eb69ecc914fa3f461d3254dfa1638451e Mon Sep 17 00:00:00 2001 From: pearce8 Date: Thu, 22 Feb 2024 14:01:22 +1100 Subject: [PATCH 16/16] updating table for docs --- docs/tables/current-system-definitions.csv | 40 +++++++++++----------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/docs/tables/current-system-definitions.csv b/docs/tables/current-system-definitions.csv index 509468e64..e891916b6 100644 --- a/docs/tables/current-system-definitions.csv +++ b/docs/tables/current-system-definitions.csv @@ -1,20 +1,20 @@ -,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Fugaku,Tioga,x86_64 -**site**,LLNL,,,LLNL,RIKEN,LLNL, -**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,Fujitsu-A64FX-TofuD,HPECray-zen3-MI250X-Slingshot,x86_64 -**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,Fujitsu,HPECray, -**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,,EX235a, -**processor.vendor**,IBM,AMD,AMD,Intel,ARM,AMD, -**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,A64FX,EPYC-Zen3, -**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64 -**processor.uArch**,power9,zen4,zen3,icelake,A64FX,zen3, -**accelerator.vendor**,NVIDIA,,AMD,,,AMD, -**accelerator.name**,V100,,MI250X,,,MI250X, -**accelerator.ISA**,PTX,,GCN,,,GCN, -**accelerator.uArch**,sm_70,,gfx90a,,,gfx90a, -**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,Fujitsu,HPECray, -**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,TofuD,Slingshot11, -**system-tested.site**,LLNL,AWS,LLNL,LLNL,RIKEN,LLNL, -**system-tested.name**,lassen,,tioga,magma,Fugaku,tioga, -**system-tested.installation-year**,2018,,2022,2019,2020,2022, -**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,`top500 `_, -**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),Fugaku (RIKEN),"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)", \ No newline at end of file +,Sierra,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Magma,Fugaku,Pascal,Tioga,x86_64 +**site**,LLNL,,,LLNL,RIKEN Center for Computational Science,LLNL,LLNL, +**system**,IBM-power9-V100-Infiniband,AWS_PCluster_Hpc7a-zen4-EFA,HPECray-zen3-MI250X-Slingshot,Penguin-icelake-OmniPath,Fujitsu-A64FX-TofuD,Penguin-broadwell-P100-OmniPath,HPECray-zen3-MI250X-Slingshot,x86_64 +**integrator.vendor**,IBM,AWS,HPECray,PenguinComputing,Fujitsu,Penguin,HPECray, +**integrator.name**,AC922,ParallelCluster3.7.2-Hpc7a,EX235a,RelionCluster,FX1000,,EX235a, +**processor.vendor**,IBM,AMD,AMD,Intel,Fujitsu,Intel,AMD, +**processor.name**,POWER9,EPYC-Zen4,EPYC-Zen3,XeonPlatinum924248C,A64FX,Xeon_E5-2695_v4,EPYC-Zen3, +**processor.ISA**,ppc64le,x86_64,x86_64,x86_64,Armv8.2-A-SVE,x86_64,x86_64,x86_64 +**processor.uArch**,power9,zen4,zen3,icelake,aarch64,broadwell,zen3, +**accelerator.vendor**,NVIDIA,,AMD,,,NVIDIA,AMD, +**accelerator.name**,V100,,MI250X,,,P100,MI250X, +**accelerator.ISA**,PTX,,GCN,,,PTX,GCN, +**accelerator.uArch**,sm_70,,gfx90a,,,sm_56,gfx90a, +**interconnect.vendor**,Mellanox,AWS,HPECray,Intel,Fujitsu,Cornelis,HPECray, +**interconnect.name**,EDR-Infiniband,EFA,Slingshot11,OmniPath,TofuInterconnectD,OmniPath,Slingshot11, +**system-tested.site**,LLNL,AWS,LLNL,LLNL,R-CCS,LLNL,LLNL, +**system-tested.name**,lassen,,tioga,magma,Fugaku,pascal,tioga, +**system-tested.installation-year**,2018,,2022,2019,2020,2018,2022, +**system-tested.description**,`top500 `_,`aws/hpc7a `_,`top500 `_,`top500 `_,`top500 `_,,`top500 `_, +**top500-system-instances**,Sierra (LLNL),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",Magma (LLNL),Fugaku (R-CCS),,"Frontier (ORNL), Lumi (CSC), Tioga (LLNL)",