diff --git a/docs/Installation/InstallationOnClusters.md b/docs/Installation/InstallationOnClusters.md index bc32e7fdbaaa..64cbf3b4096e 100644 --- a/docs/Installation/InstallationOnClusters.md +++ b/docs/Installation/InstallationOnClusters.md @@ -62,20 +62,6 @@ with the `idev` command. For unknown reasons, incremental builds work poorly on frontera. Running `make` will often unnecessarily recompile SpECTRE libraries. -## Wheeler at Caltech - -Follow the general instructions using `wheeler` for `SYSTEM_TO_RUN_ON`, except -you do not need to install any dependencies, so you can skip steps 5 and 6. You -can optionally compile using LLVM/Clang by sourcing `wheeler_clang.sh` instead -of `wheeler_gcc.sh` - -If you are running jobs on a Wheeler interactive compute -node, make sure that when you allocate the interactive node using -`srun`, use the `-c ` option to `srun`, and not the `-n -` option. If you use the `-n ` -option and pass the number of cores for NUMBER_OF_TASKS, then you will -get multiple MPI ranks on your node and the run will hang. - ## CaltechHPC at Caltech Cluster documentation: https://www.hpc.caltech.edu diff --git a/support/Environments/minerva_gcc.sh b/support/Environments/minerva_gcc.sh deleted file mode 100755 index 795a3796da9c..000000000000 --- a/support/Environments/minerva_gcc.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/env sh - -# Distributed under the MIT License. -# See LICENSE.txt for details. - -# Instructions to **compile** spectre on Minerva: -# -# 1. We recommend you compile spectre on a compute node to avoid disrupting the -# login node for other users. This is how you can request a compute node on -# the `devel` queue for a few hours: -# ```sh -# srun -p devel --nodes=1 --ntasks-per-node=16 --time=08:00:00 --pty bash -i -# ``` -# You will be dropped right into a shell on the compute node as soon as the -# scheduler has allocated one for you so you can proceed with these -# instructions. -# 2. Clone the spectre repository, if you haven't already. A good place is in -# your `/home` directory on Minerva: -# ```sh -# cd /home/yourname -# git clone git@github.com:yourname/spectre.git -# ``` -# 3. Set the `$SPECTRE_HOME` environment variable to the location of the spectre -# repository, e.g. `/home/yourname/spectre`: -# ```sh -# export SPECTRE_HOME=path/to/spectre -# ``` -# 4. Source this script and setup modules: -# ```sh -# source $SPECTRE_HOME/support/Environments/minerva_gcc.sh` -# spectre_setup_modules -# ``` -# Note: Add steps 3 and 4 to your `.bashrc` file if you don't want to repeat -# them every time you log in. The `spectre_setup_modules` function only -# adjusts your `MODULEPATH` to make the installed modules visible but doesn't -# load any, so it's safe to add to your `.bashrc`. -# 5. Create a build directory, if you don't have one already. A good place is in -# your `/work` directory on Minerva: -# ```sh -# cd /work/yourname -# mkdir spectre-build -# ``` -# Note: Add a timestamp or descriptive labels to the name of the build -# directory, since you may create more build directories later, e.g. -# `build_YYYY-MM-DD` or `build-clang-Debug`. -# 6. Run `cmake` to configure the build directory: -# ```sh -# cd path/to/build/directory -# module purge -# spectre_run_cmake -# ``` -# Note: Remember to `module purge` to work in a clean environment, unless you -# have reasons not to. -# 7. Compile! With the build directory configured, this script sourced and -# modules set up, you can skip the previous steps from now on. -# ```sh -# module purge -# spectre_load_modules -# make -j16 SPECTRE_EXECUTABLE -# ``` -# Replace `SPECTRE_EXECUTABLE` with the target you want to build, e.g. -# `unit-tests` or `SolvePoisson3D`. -# -# Instructions to **run** spectre executables on Minerva: -# -# 1. Create a run directory. A good place is in your `/scratch` directory on -# Minerva. Make sure to choose a descriptive name, e.g. -# `/scratch/yourname/spectre/your_project/00_the_run`. -# 2. Copy `$SPECTRE_HOME/support/SubmitScripts/Minerva.sh` to the run directory -# and edit it as the comments in that file instruct. -# 3. Submit the job to Minerva's queue: -# ```sh -# sbatch Minerva.sh -# ``` - -spectre_setup_modules() { - export MODULEPATH="\ -/home/SPACK2021/share/spack/modules/linux-centos7-haswell:$MODULEPATH" - export MODULEPATH="\ -/home/nfischer/spack/share/spack/modules/linux-centos7-haswell:$MODULEPATH" -} - -spectre_load_modules() { - module load gcc-10.2.0-gcc-10.2.0-vaerku7 - module load binutils-2.36.1-gcc-10.2.0-wtzd7wm - source /home/nfischer/spack/var/spack/environments/spectre_2021-03-18/loads - export CHARM_ROOT="\ -/home/nfischer/spack/opt/spack/linux-centos7-haswell/gcc-10.2.0/\ -charmpp-6.10.2-2waqdh24tz4yt5ozy5clhx6ahxjgivwz" - # Libsharp is installed separately with `-fPIC` since the Spack package - # doesn't support that option (yet) - export LIBSHARP_ROOT="/work/nfischer/spectre/libsharp_2021-03-18/auto" -} - -spectre_unload_modules() { - echo "Unloading a subset of modules is not supported." - echo "Run 'module purge' to unload all modules." -} - -spectre_run_cmake() { - if [ -z ${SPECTRE_HOME} ]; then - echo "You must set SPECTRE_HOME to the cloned SpECTRE directory" - return 1 - fi - spectre_load_modules - cmake \ - -D CMAKE_C_COMPILER=gcc \ - -D CMAKE_CXX_COMPILER=g++ \ - -D CMAKE_Fortran_COMPILER=gfortran \ - -D CHARM_ROOT=$CHARM_ROOT \ - -D LIBSHARP_ROOT=$LIBSHARP_ROOT \ - -D CMAKE_BUILD_TYPE=Release \ - -D DEBUG_SYMBOLS=OFF \ - -D BUILD_SHARED_LIBS=ON \ - -D MEMORY_ALLOCATOR=SYSTEM \ - -D BUILD_PYTHON_BINDINGS=ON \ - -D BOOTSTRAP_PY_DEPS=ON \ - -Wno-dev "$@" $SPECTRE_HOME -} diff --git a/support/Environments/wheeler_clang.sh b/support/Environments/wheeler_clang.sh deleted file mode 100755 index 0dfc879e18a0..000000000000 --- a/support/Environments/wheeler_clang.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/env sh - -# Distributed under the MIT License. -# See LICENSE.txt for details. - -spectre_setup_modules() { - echo "All modules on Wheeler are provided by the system" -} - -spectre_unload_modules() { - module unload gcc/9.3.0 - module unload blaze/3.8 - module unload boost/1.65.0-gcc-6.4.0 - module unload brigand/master - module unload catch/3.5.1 - module unload gsl/2.1 - module unload libsharp/1.0.0 - module unload libxsmm/1.16.1 - module unload openblas/0.2.18 - module unload papi/5.5.1 - module unload yaml-cpp/master - module unload impi/2017.1 - module unload cmake/3.18.2 - module unload ccache/4.7.3 - module unload ninja/1.10.0 - module unload doxygen/1.8.13 - module unload git/2.8.4 - module unload llvm/13.0.1 - module unload charm/7.0.0-intelmpi-smp - module unload envs/spectre-python - module unload pybind11/2.6.1 - module unload hdf5/1.12.2 -} - -spectre_load_modules() { - module load gcc/9.3.0 - module load blaze/3.8 - module load boost/1.65.0-gcc-6.4.0 - module load brigand/master - module load catch/3.5.1 - module load gsl/2.1 - module load libsharp/1.0.0 - module load libxsmm/1.16.1 - module load openblas/0.2.18 - module load papi/5.5.1 - module load yaml-cpp/master - module load impi/2017.1 - module load cmake/3.18.2 - module load ccache/4.7.3 - module load ninja/1.10.0 - module load doxygen/1.8.13 - module load git/2.8.4 - module load llvm/13.0.1 - module load charm/7.0.0-intelmpi-smp - module load envs/spectre-python - module load pybind11/2.6.1 - module load hdf5/1.12.2 -} - -spectre_run_cmake() { - if [ -z ${SPECTRE_HOME} ]; then - echo "You must set SPECTRE_HOME to the cloned SpECTRE directory" - return 1 - fi - spectre_load_modules - cmake -D CHARM_ROOT=$CHARM_ROOT \ - -D CMAKE_BUILD_TYPE=Release \ - -D CMAKE_C_COMPILER=clang \ - -D CMAKE_CXX_COMPILER=clang++ \ - -D CMAKE_Fortran_COMPILER=gfortran \ - -D MEMORY_ALLOCATOR=SYSTEM \ - -D BUILD_PYTHON_BINDINGS=ON \ - -D MACHINE=Wheeler \ - "$@" \ - $SPECTRE_HOME -} diff --git a/support/Environments/wheeler_gcc.sh b/support/Environments/wheeler_gcc.sh deleted file mode 100755 index 4ed296d2c4e3..000000000000 --- a/support/Environments/wheeler_gcc.sh +++ /dev/null @@ -1,81 +0,0 @@ -#!/bin/env sh - -# Distributed under the MIT License. -# See LICENSE.txt for details. - -spectre_setup_modules() { - echo "All modules on Wheeler are provided by the system" -} - -spectre_unload_modules() { - module unload gcc/9.3.0 - module unload blaze/3.8 - module unload boost/1.65.0-gcc-6.4.0 - module unload brigand/master - module unload catch/3.5.1 - module unload gsl/2.1 - module unload libsharp/1.0.0 - module unload libxsmm/1.16.1 - module unload openblas/0.2.18 - module unload papi/5.5.1 - module unload yaml-cpp/master - module unload impi/2017.1 - module unload cmake/3.18.2 - module unload ccache/4.7.3 - module unload ninja/1.10.0 - module unload doxygen/1.8.13 - module unload git/2.8.4 - module unload lcov/1.13 - module unload charm/7.0.0-intelmpi-smp - module unload pybind11/2.6.1 - module unload hdf5/1.12.2 - module unload libbacktrace/1.0 - module unload spec-exporter/2023-07 - module unload paraview/5.11.1-osmesa - module unload envs/spectre-python -} - -spectre_load_modules() { - module load gcc/9.3.0 - module load blaze/3.8 - module load boost/1.65.0-gcc-6.4.0 - module load brigand/master - module load catch/3.5.1 - module load gsl/2.1 - module load libsharp/1.0.0 - module load libxsmm/1.16.1 - module load openblas/0.2.18 - module load papi/5.5.1 - module load yaml-cpp/master - module load impi/2017.1 - module load cmake/3.18.2 - module load ccache/4.7.3 - module load ninja/1.10.0 - module load doxygen/1.8.13 - module load git/2.8.4 - module load lcov/1.13 - module load charm/7.0.0-intelmpi-smp - module load pybind11/2.6.1 - module load hdf5/1.12.2 - module load libbacktrace/1.0 - module load spec-exporter/2023-07 - module load paraview/5.11.1-osmesa - # Load after ParaView to give priority to the packages in the environment - module load envs/spectre-python -} - -spectre_run_cmake() { - if [ -z ${SPECTRE_HOME} ]; then - echo "You must set SPECTRE_HOME to the cloned SpECTRE directory" - return 1 - fi - spectre_load_modules - cmake -D CHARM_ROOT=$CHARM_ROOT \ - -D CMAKE_BUILD_TYPE=Release \ - -D CMAKE_Fortran_COMPILER=gfortran \ - -D MEMORY_ALLOCATOR=SYSTEM \ - -D BUILD_PYTHON_BINDINGS=ON \ - -D MACHINE=Wheeler \ - "$@" \ - $SPECTRE_HOME -} diff --git a/support/Machines/Wheeler.yaml b/support/Machines/Wheeler.yaml deleted file mode 100644 index c56a360b439e..000000000000 --- a/support/Machines/Wheeler.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# Distributed under the MIT License. -# See LICENSE.txt for details. - -Machine: - Name: Wheeler - Description: | - Supercomputer at Caltech. - More information: - https://github.com/sxs-collaboration/WelcomeToSXS/wiki/Wheeler - # Use one of the 24 available cores for communication by default - DefaultProcsPerNode: 23 - DefaultQueue: "productionQ" - DefaultTimeLimit: "1-00:00:00" - LaunchCommandSingleNode: [] diff --git a/support/SubmitScripts/Minerva.sh b/support/SubmitScripts/Minerva.sh deleted file mode 100644 index 0c3459de2182..000000000000 --- a/support/SubmitScripts/Minerva.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -#SBATCH -J spectre -#SBATCH --nodes 2 -#SBATCH -t 24:00:00 -#SBATCH -p nr -#SBATCH -o spectre.out -#SBATCH -e spectre.out -#SBATCH --ntasks-per-node 16 -#SBATCH --no-requeue - -# Distributed under the MIT License. -# See LICENSE.txt for details. - -# To run a job on Minerva: -# -# 1. Set the -J, --nodes, and -t options above, which correspond to job name, -# number of nodes, and wall time limit in HH:MM:SS, respectively. -# 2. Set the build directory, run directory, executable name, -# and input file below. -# Note: The executable will not be copied from the build directory, so if you -# update your build directory this file will use the updated executable. -# 3. Optionally, if you need more control over how SpECTRE is launched on -# Minerva you can edit the launch command at the end of this file directly. -# 4. Submit the job to the queue: -# ```sh -# sbatch Minerva.sh -# ``` - -# Replace these paths with the path to your build directory and to the -# directory where you want the output to appear, i.e. the run directory -export SPECTRE_BUILD_DIR=/work/nfischer/spectre/build_2021-03-18-Release -export SPECTRE_RUN_DIR=${PWD} - -# Choose the executable and input file to run -# To use an input file in the current directory, set -# SPECTRE_INPUT_FILE to `${PWD}/InputFileName.yaml` or just `InputFileName.yaml` -export SPECTRE_EXECUTABLE=SolveXcts -export SPECTRE_INPUT_FILE=Schwarzschild.yaml - -# --- You probably don't need to edit anything below this line --- - -mkdir -p ${SPECTRE_RUN_DIR} && cd ${SPECTRE_RUN_DIR} - -# Copy the input file into the run directory to preserve it -cp ${SPECTRE_INPUT_FILE} ${SPECTRE_RUN_DIR}/ - -# Set up the environment -export MODULEPATH="\ -/home/SPACK2021/share/spack/modules/linux-centos7-haswell:$MODULEPATH" -export MODULEPATH="\ -/home/nfischer/spack/share/spack/modules/linux-centos7-haswell:$MODULEPATH" -module purge -module load gcc-10.2.0-gcc-10.2.0-vaerku7 -module load binutils-2.36.1-gcc-10.2.0-wtzd7wm -source /home/nfischer/spack/var/spack/environments/spectre_2021-03-18/loads - -# Set permissions for files created with this script -umask 0022 - -# Add SpECTRE executables to the path -export PATH=${SPECTRE_BUILD_DIR}/bin:$PATH - -# Run the executable on the available nodes. Notes: -# - We place only one proc per node because we're running with SMP. -# - We run on 15 procs per 16-core-node because Charm++ uses one thread per node -# for communication. -# - IntelMPI, OpenMPI, srun and charmrun have different interfaces for similar -# things. IntelMPI uses `-n NUM_NODES` and `-ppn PROCS_PER_NODE`, and OpenMPI -# uses `-np NUM_NODES` and `--map-by ppr:PROCS_PER_NODE:node`. We are -# currently using IntelMPI on Minerva. -mpirun -n ${SLURM_JOB_NUM_NODES} -ppn 1 \ - ${SPECTRE_EXECUTABLE} +ppn 15 +pemap 0-14 +commap 15 \ - --input-file ${SPECTRE_INPUT_FILE} diff --git a/support/SubmitScripts/Wheeler.sh b/support/SubmitScripts/Wheeler.sh deleted file mode 100755 index 9c050628675b..000000000000 --- a/support/SubmitScripts/Wheeler.sh +++ /dev/null @@ -1,23 +0,0 @@ -{% extends "SubmitTemplateBase.sh" %} - -# Distributed under the MIT License. -# See LICENSE.txt for details. - -# Wheeler is a supercomputer at Caltech. -# More information: -# https://github.com/sxs-collaboration/WelcomeToSXS/wiki/Wheeler - -{% block head %} -{{ super() -}} -#SBATCH --nodes {{ num_nodes | default(1) }} -#SBATCH --ntasks-per-node 1 -#SBATCH --cpus-per-task 24 -#SBATCH -A sxs -#SBATCH -p {{ queue | default("productionQ") }} -#SBATCH -t {{ time_limit | default("1-00:00:00") }} -{% endblock %} - -# We had an issue calling 'module list' on the 'unlimited' queue -{% block list_modules %} -/usr/bin/modulecmd bash list -{% endblock %} diff --git a/tests/Unit/Parallel/Test_AlgorithmReduction.cpp b/tests/Unit/Parallel/Test_AlgorithmReduction.cpp index 62ab86c17ebf..3bf8fdf6d391 100644 --- a/tests/Unit/Parallel/Test_AlgorithmReduction.cpp +++ b/tests/Unit/Parallel/Test_AlgorithmReduction.cpp @@ -42,10 +42,8 @@ class DataBox; } // namespace db struct TestMetavariables; -// The reason we use a 46 element array is that on Wheeler, the SXS -// supercomputer at Caltech, there are 23 worker threads per node and we want to -// be able to test on two nodes to make sure multinode communication is working -// correctly. +// The reason we use a 46 element array is that we want to be able to test on +// two nodes to make sure multinode communication is working correctly. static constexpr int number_of_1d_array_elements = 46; // [reduce_sum_int_action]