diff --git a/doc/Sphinx/Use/install_linux_GPU.rst b/doc/Sphinx/Use/install_linux_GPU.rst new file mode 100644 index 000000000..197cd499b --- /dev/null +++ b/doc/Sphinx/Use/install_linux_GPU.rst @@ -0,0 +1,74 @@ + +Install on Linux for GPU +----------------------------- + +First, make sure you have a recent version of CMAKE, and the other libraries +to compile Smilei on CPU as usual. In particular, for this example, you +need GCC <= 12. + +Make a directory to store all the nvidia tools. We call it $NVDIR: + +.. code:: bash + + cd $NVDIR + +The first step is download ``nvhpc`` and install it: + +.. code:: bash + + wget https://developer.download.nvidia.com/hpc-sdk/23.11/nvhpc_2023_2311_Linux_x86_64_cuda_12.3.tar.gz + tar xpzf nvhpc_2023_2311_Linux_x86_64_cuda_12.3.tar.gz + nvhpc_2023_2311_Linux_x86_64_cuda_12.3/install # install in $NVDIR when asked + + rm nvhpc_2023_2311_Linux_x86_64_cuda_12.3.tar.gz + +Then compile ``hdf5``: + +.. code:: bash + + export PATH=$NVDIR/Linux_x86_64/23.11/compilers/bin:$PATH + export PATH=$NVDIR/Linux_x86_64/23.11/comm_libs/mpi/bin:$PATH + + wget https://github.com/HDFGroup/hdf5/releases/download/hdf5-1_14_2/hdf5-1_14_2.tar.gz + tar xpzf hdf5-1_14_2.tar.gz + + cd hdfsrc/ + mkdir build + cd build + cmake -DCMAKE_C_COMPILER=`which mpicc` -DCMAKE_INSTALL_PREFIX=$NVDIR/hdfsrc/install -DHDF5_ENABLE_PARALLEL=ON .. + make + make install + +Create a file ``nvidia_env.sh`` containing the following commands: + +.. code:: bash + + export BUILD_DIR=build_nvidia + + export PATH=$NVDIR/Linux_x86_64/23.11/compilers/bin:$PATH + export PATH=$NVDIR/Linux_x86_64/23.11/comm_libs/mpi/bin:$PATH + + export HDF5_ROOT_DIR=$NVDIR/hdfsrc/install/ + export LD_LIBRARY_PATH=$HDF5_ROOT_DIR/lib + + export LDFLAGS="-acc=gpu -gpu=ccnative -cudalib=curand " + export CXXFLAGS="-acc=gpu -gpu=ccnative,fastmath -std=c++14 -lcurand -Minfo=accel -w -D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1 -I$NVDIR/Linux_x86_64/23.11/math_libs/include/" + export GPU_COMPILER_FLAGS="-O3 --std c++14 -arch=sm_86 --expt-relaxed-constexpr --compiler-bindir $NVDIR -I$NVDIR/Linux_x86_64/23.11/comm_libs/12.3/openmpi4/openmpi-4.1.5/include/ -I$NVDIR/hdfsrc/install/include/" + + export SMILEICXX_DEPS=g++ + + export SLURM_LOCALID=0 + +To compile Smilei: + +.. code:: bash + + source nvidia_env.sh + make config="gpu_nvidia" + +To run: + +.. code:: bash + + source nvidia_env.sh + smilei namelist.py diff --git a/doc/Sphinx/Use/installation.rst b/doc/Sphinx/Use/installation.rst index 4027459d8..791e96dc1 100755 --- a/doc/Sphinx/Use/installation.rst +++ b/doc/Sphinx/Use/installation.rst @@ -190,6 +190,9 @@ Typically ``CXXFLAGS += -ta=tesla:cc80`` for ``nvhpc`` <23.4 and * The hdf5 module should be compiled with the nvidia/cray compiler; openmpi as well, but depending on the nvhpc module it might not be needed as it can be included in the nvhpc module. +For testing purposes, Smilei can be run on a linux PC with a good +GPU. As a guidance, read these :doc:`instructions for GPU on linux`. + ---- .. _vectorization_flags: diff --git a/doc/Sphinx/site.rst b/doc/Sphinx/site.rst index b6fb3d84a..ff3af981b 100755 --- a/doc/Sphinx/site.rst +++ b/doc/Sphinx/site.rst @@ -23,6 +23,7 @@ Unreferenced pages /Use/install_linux /Use/install_macos /Use/install_supercomputer + /Use/install_linux_GPU /Use/laser_offset /Use/maxwell-juttner /Use/particle_initialization diff --git a/makefile b/makefile index 6c8b2f241..9063a58a0 100755 --- a/makefile +++ b/makefile @@ -289,7 +289,7 @@ ifneq ($(strip $(my_config)),) $(error "Unused parameters in config : $(my_config)") endif -SMILEICXX.DEPS ?= $(SMILEICXX) +SMILEICXX_DEPS ?= $(SMILEICXX) #----------------------------------------------------- # Rules for building the excutable smilei diff --git a/scripts/compile_tools/machine/adastra b/scripts/compile_tools/machine/adastra index 43d544487..7aab184ce 100644 --- a/scripts/compile_tools/machine/adastra +++ b/scripts/compile_tools/machine/adastra @@ -36,7 +36,7 @@ ################################################################################ SMILEICXX := CC -SMILEICXX.DEPS := $(SMILEICXX) +SMILEICXX_DEPS := $(SMILEICXX) # GPU_COMPILER := hipcc GPU_COMPILER := $(SMILEICXX) diff --git a/scripts/compile_tools/machine/cp_laptop b/scripts/compile_tools/machine/cp_laptop deleted file mode 100644 index c5b5a30b1..000000000 --- a/scripts/compile_tools/machine/cp_laptop +++ /dev/null @@ -1,20 +0,0 @@ -# - -# Compile command: make -j 10 machine="cp_laptop" config="gpu_nvidia detailed_timers verbose" - - - - -SMILEICXX.DEPS = g++ -I/local/home/cprouveur/tools/hdfsrc/install/include/ -#GPU_COMPILER = nvcc - -CXXFLAGS += -w -CXXFLAGS += -acc=gpu -gpu=cc86,fastmath -std=c++14 -lcurand -Minfo=accel # what is offloaded/copied -# CXXFLAGS += -Minfo=all # very verbose output -CXXFLAGS += -D__GCC_ATOMIC_TEST_AND_SET_TRUEVAL=1 - - -GPU_COMPILER_FLAGS += -O3 --std c++14 -arch=sm_86 -I/opt/nvidia/hpc_sdk/Linux_x86_64/23.11/comm_libs/12.3/openmpi4/openmpi-4.1.5/include -I/local/home/cprouveur/tools/hdfsrc/install/include/ -GPU_COMPILER_FLAGS += --expt-relaxed-constexpr - -LDFLAGS += -acc=gpu -gpu=cc86 -cudalib=curand # ccnative also works diff --git a/scripts/compile_tools/machine/jean_zay_gpu_A100 b/scripts/compile_tools/machine/jean_zay_gpu_A100 index 92b3f2fb7..c31251888 100644 --- a/scripts/compile_tools/machine/jean_zay_gpu_A100 +++ b/scripts/compile_tools/machine/jean_zay_gpu_A100 @@ -6,7 +6,7 @@ # http://www.idris.fr/jean-zay # -SMILEICXX.DEPS = g++ +SMILEICXX_DEPS = g++ #GPU_COMPILER = nvcc CXXFLAGS += -w diff --git a/scripts/compile_tools/machine/jean_zay_gpu_V100 b/scripts/compile_tools/machine/jean_zay_gpu_V100 index 91d68a4ff..7fa7ce513 100644 --- a/scripts/compile_tools/machine/jean_zay_gpu_V100 +++ b/scripts/compile_tools/machine/jean_zay_gpu_V100 @@ -6,7 +6,7 @@ # http://www.idris.fr/jean-zay # -SMILEICXX.DEPS = g++ +SMILEICXX_DEPS = g++ #GPU_COMPILER = nvcc CXXFLAGS += -w diff --git a/scripts/compile_tools/machine/juanjo_cpu b/scripts/compile_tools/machine/juanjo_cpu deleted file mode 100644 index 17cae34bf..000000000 --- a/scripts/compile_tools/machine/juanjo_cpu +++ /dev/null @@ -1,8 +0,0 @@ -CXXFLAGS = -acc -Mcudalib=curand -ta=tesla:cc35 -Minfo=accel -LDFLAGS += -ta=tesla:cc35 -L/usr/local/openmpi/lib/ -L/opt/nvidia/hpc_sdk/Linux_x86_64/21.2/cuda/11.2/lib64/ -lcudart -GPU_COMPILER_FLAGS += -arch=sm_35 -I/usr/local/openmpi/include/ -I/opt/nvidia/hpc_sdk/Linux_x86_64/21.2/cuda/11.2/include/ -CXXFLAGS += -I/usr/local/hdf5/include/ -I/usr/local/openmpi/include/ - - - - diff --git a/scripts/compile_tools/machine/ruche_gpu2 b/scripts/compile_tools/machine/ruche_gpu2 index 0a004f782..a9406d60d 100644 --- a/scripts/compile_tools/machine/ruche_gpu2 +++ b/scripts/compile_tools/machine/ruche_gpu2 @@ -7,7 +7,7 @@ # Compile command: make -j 40 machine="ruche_gpu2" config="gpu_nvidia noopenmp detailed_timers verbose" -SMILEICXX.DEPS = g++ #nvcc -I/gpfs/softs/spack_0.17/opt/spack/linux-centos7-haswell/nvhpc-23.7/openmpi-4.1.5-ckfuippq6gf6qsilwitd7d2zyd5bng32/include/ -I/gpfs/softs/spack_0.17/opt/spack/linux-centos7-haswell/nvhpc-23.7/hdf5-1.12.0-3em63nl4p5tmv37offfmuvz2uswvgwzv/include/ +SMILEICXX_DEPS = g++ #nvcc -I/gpfs/softs/spack_0.17/opt/spack/linux-centos7-haswell/nvhpc-23.7/openmpi-4.1.5-ckfuippq6gf6qsilwitd7d2zyd5bng32/include/ -I/gpfs/softs/spack_0.17/opt/spack/linux-centos7-haswell/nvhpc-23.7/hdf5-1.12.0-3em63nl4p5tmv37offfmuvz2uswvgwzv/include/ GPU_COMPILER = nvcc -I/gpfs/softs/spack_0.17/opt/spack/linux-centos7-haswell/nvhpc-23.7/openmpi-4.1.5-ckfuippq6gf6qsilwitd7d2zyd5bng32/include/ -I/gpfs/softs/spack_0.17/opt/spack/linux-centos7-haswell/nvhpc-23.7/hdf5-1.12.0-3em63nl4p5tmv37offfmuvz2uswvgwzv/include/