-
Notifications
You must be signed in to change notification settings - Fork 37
177 lines (166 loc) · 6.78 KB
/
ci-short.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
name: CI short
on: [pull_request]
# Cancel "duplicated" workflows triggered by pushes to internal
# branches with associated PRs.
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
env:
CTEST_OUTPUT_ON_FAILURE: 1
CMAKE_BUILD_PARALLEL_LEVEL: 5 # num threads for build
MACHINE_CFG: cmake/machinecfg/CI.cmake
OMPI_MCA_mpi_common_cuda_event_max: 1000
# CUDA IPC within docker repeated seem to cause issue on the CI machine
OMPI_MCA_btl_smcuda_use_cuda_ipc: 0
# https://github.com/open-mpi/ompi/issues/4948#issuecomment-395468231
OMPI_MCA_btl_vader_single_copy_mechanism: none
jobs:
style:
runs-on: [self-hosted, A100]
container:
image: ghcr.io/parthenon-hpc-lab/cuda11.6-mpi-hdf5-ascent
# map to local user id on CI machine to allow writing to build cache
options: --user 1001 --cap-add CAP_SYS_PTRACE --shm-size="8g" --ulimit memlock=134217728
steps:
- uses: actions/checkout@v3
with:
submodules: 'true'
- name: cpplint
run: python ./tst/style/cpplint.py --counting=detailed --recursive src example tst
- name: copyright
run: |
cmake -DCMAKE_CXX_FLAGS=-Werror -Bbuild-copyright-check
cmake --build build-copyright-check -t check-copyright
- uses: actions/upload-artifact@v3
with:
name: configure-log-style
path: build-copyright-check/CMakeFiles/CMakeOutput.log
retention-days: 3
unit:
strategy:
matrix:
device: ['cuda', 'host']
runs-on: [self-hosted, A100]
container:
image: ghcr.io/parthenon-hpc-lab/cuda11.6-mpi-hdf5-ascent
# map to local user id on CI machine to allow writing to build cache
options: --user 1001 --cap-add CAP_SYS_PTRACE --shm-size="8g" --ulimit memlock=134217728
steps:
- uses: actions/checkout@v3
with:
submodules: 'true'
- name: Configure
run: |
cmake -B build \
-DCMAKE_BUILD_TYPE=Debug \
-DMACHINE_VARIANT=${{ matrix.device }}-mpi
- name: Build
run: cmake --build build
- name: Test
run: |
cd build
# Pick GPU with most available memory
export CUDA_VISIBLE_DEVICES=$(nvidia-smi --query-gpu=memory.free,index --format=csv,nounits,noheader | sort -nr | head -1 | awk '{ print $NF }')
ctest -LE 'performance|regression'
- uses: actions/upload-artifact@v3
with:
name: configure-log-unit-${{ matrix.device }}
path: build/CMakeFiles/CMakeOutput.log
retention-days: 3
integration:
strategy:
matrix:
device: ['cuda', 'host']
runs-on: [self-hosted, A100]
container:
image: ghcr.io/parthenon-hpc-lab/cuda11.6-mpi-hdf5-ascent
# map to local user id on CI machine to allow writing to build cache
options: --user 1001 --cap-add CAP_SYS_PTRACE --shm-size="8g" --ulimit memlock=134217728
steps:
- uses: actions/checkout@v3
with:
submodules: 'true'
- name: Configure
run: |
cmake -B build \
-DCMAKE_BUILD_TYPE=Release \
-DMACHINE_VARIANT=${{ matrix.device }}-mpi
# Test example with "variables" and output
- name: advection
run: |
cmake --build build -t advection-example
cd build
# Pick GPU with most available memory
export CUDA_VISIBLE_DEVICES=$(nvidia-smi --query-gpu=memory.free,index --format=csv,nounits,noheader | sort -nr | head -1 | awk '{ print $NF }')
ctest -R regression_mpi_test:output_hdf5
# Test example with swarms
- name: particle-leapfrog
run: |
cmake --build build -t particle-leapfrog
cd build
# Pick GPU with most available memory
export CUDA_VISIBLE_DEVICES=$(nvidia-smi --query-gpu=memory.free,index --format=csv,nounits,noheader | sort -nr | head -1 | awk '{ print $NF }')
ctest -R regression_mpi_test:particle_leapfrog
# Now testing if there are no hidden memcopies between host and device.
# Using a static grid (i.e., not AMR) as additional transfers are expected
# during loadbalance and refinement, but not for a static grid. We also need to
# turn off sparse, since sparse results in additional transfers.
# Also delaying start as there are explicit copies during initialization, e.g.,
# when the Variable caches are created.
- name: host-device-copy
if: ${{ matrix.device }} == 'cuda'
run: |
cd build
nsys profile --delay=5 --duration=5 --stats=true -s none -t cuda example/advection/advection-example \
-i ../tst/regression/test_suites/advection_performance/parthinput.advection_performance \
parthenon/mesh/nx1=128 parthenon/mesh/nx2=128 parthenon/mesh/nx3=128 \
parthenon/meshblock/nx1=64 parthenon/meshblock/nx2=64 parthenon/meshblock/nx3=64 \
parthenon/sparse/enable_sparse=false parthenon/time/nlim=200000 parthenon/time/tlim=200 2>&1 | tee profile.txt
if grep HtoD profile.txt; then exit 1; fi
if grep DtoH profile.txt; then exit 1; fi
- uses: actions/upload-artifact@v3
with:
name: configure-log-integration-${{ matrix.device }}
path: |
build/CMakeFiles/CMakeOutput.log
build/profile.txt
retention-days: 3
integration-amdgpu:
runs-on: [self-hosted, navi1030]
container:
image: ghcr.io/parthenon-hpc-lab/rocm5.4.3-mpi-hdf5
# Map to local user id on CI machine to allow writing to build cache and
# forward device handles to access AMD GPU within container
options: --user 1000 -w /home/ci --device /dev/kfd --device /dev/dri --security-opt seccomp=unconfined
env:
CMAKE_GENERATOR: Ninja
CMAKE_BUILD_PARALLEL_LEVEL: 8 # num threads for build
steps:
- uses: actions/checkout@v3
with:
submodules: 'true'
- name: Configure
run: |
cmake -B build \
-DMACHINE_CFG=${PWD}/cmake/machinecfg/GitHubActions.cmake \
-DCMAKE_BUILD_TYPE=Release \
-DMACHINE_VARIANT=hip-mpi \
-DCMAKE_CXX_COMPILER=hipcc
# Test example with "variables" and output
- name: advection
run: |
cmake --build build -t advection-example
cd build
ctest -R regression_mpi_test:output_hdf5
# Test example with swarms
- name: particle-leapfrog
run: |
cmake --build build -t particle-leapfrog
cd build
ctest -R regression_mpi_test:particle_leapfrog
- uses: actions/upload-artifact@v3
with:
name: configure-log-integration-amdgpu
path: |
build/CMakeFiles/CMakeOutput.log
retention-days: 3