-
Notifications
You must be signed in to change notification settings - Fork 6
393 lines (352 loc) · 15.1 KB
/
build_test.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
# Build and test Kokkos FFT using Docker and Singularity images. Pre-generated
# images are pulled from Github registry; they are updated only if the current
# PR or commit modified the Docker files.
name: CI
on:
pull_request:
branches:
- main
env:
# Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
BUILD_TYPE: Release
# Force the use of BuildKit for Docker
DOCKER_BUILDKIT: 1
jobs:
# run linter on the code
lint:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: DoozyX/[email protected]
with:
source: 'common/ fft/ examples/'
exclude: ''
extensions: 'hpp,cpp'
clangFormatVersion: 12
# check if Docker images should be recreated
check_docker_files:
uses: ./.github/workflows/__check_docker_files.yaml
with:
event_name: ${{ github.event_name }}
# build Docker images if necessary
build_base:
needs: check_docker_files
if: ${{ needs.check_docker_files.outputs.docker_files_have_changed == 'true' }}
uses: ./.github/workflows/__build_base.yaml
with:
image_suffix: ${{ needs.check_docker_files.outputs.image_suffix }}
image_tag: ${{ needs.check_docker_files.outputs.image_tag }}
# build project
build:
runs-on: ubuntu-latest
needs:
- check_docker_files
- build_base
# run this job even if build_base did not run
if: ${{ ! cancelled() && (needs.build_base.result == 'success' || needs.build_base.result == 'skipped') }}
strategy:
matrix:
backend:
- name: openmp
image: gcc
compiler:
c: gcc
cxx: g++
cmake_flags:
kokkos: -DKokkos_ENABLE_OPENMP=ON
kokkos_fft: ""
- name: threads
image: gcc
compiler:
c: gcc
cxx: g++
cmake_flags:
kokkos: -DKokkos_ENABLE_THREADS=ON
kokkos_fft: ""
- name: serial
image: gcc
compiler:
c: gcc
cxx: g++
cmake_flags:
kokkos: -DKokkos_ENABLE_SERIAL=ON
kokkos_fft: ""
- name: cuda
image: nvcc
compiler:
c: gcc
cxx: g++
cmake_flags:
kokkos: -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON
kokkos_fft: ""
- name: hip
image: rocm
compiler:
c: hipcc
cxx: hipcc
cmake_flags:
kokkos: -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON
kokkos_fft: -DKokkosFFT_ENABLE_ROCFFT=ON
- name: rocm
image: rocm
compiler:
c: hipcc
cxx: hipcc
cmake_flags:
kokkos: -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON
kokkos_fft: ""
- name: sycl
image: intel
compiler:
c: icx
cxx: icpx
cmake_flags:
# building for Intel PVC was unsuccessful without the proper
# device, so for now, we simply generate generic Intel GPU code
kokkos: -DKokkos_ENABLE_SYCL=ON -DKokkos_ARCH_INTEL_GEN=ON
kokkos_fft: ""
target:
- name: native
cmake_flags: ""
- name: host_device
cmake_flags: -DKokkosFFT_ENABLE_HOST_AND_DEVICE=ON
exclude:
- backend:
name: openmp
target:
name: host_device
- backend:
name: threads
target:
name: host_device
- backend:
name: serial
target:
name: host_device
steps:
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/[email protected]
with:
tool-cache: true
large-packages: false
- name: Checkout built branch
uses: actions/checkout@v4
with:
submodules: recursive
- name: Configure
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake -B build \
-DCMAKE_INSTALL_PREFIX=/work/install \
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
-DCMAKE_CXX_STANDARD=17 \
-DKokkosFFT_ENABLE_EXAMPLES=ON \
-DKokkosFFT_ENABLE_TESTS=ON \
-DKokkosFFT_ENABLE_BENCHMARK=ON \
-DKokkosFFT_ENABLE_INTERNAL_KOKKOS=ON \
${{ matrix.backend.cmake_flags.kokkos }} \
${{ matrix.backend.cmake_flags.kokkos_fft }} \
${{ matrix.target.cmake_flags }}
- name: Build
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --build build -j $(( $(nproc) * 2 + 1 ))
- name: Prepare artifacts
# this is mandatory to preserve execution rights
run: tar -cvf tests_${{ matrix.backend.name }}.tar build/
if: ${{ matrix.target.name == 'native' }}
- name: Save artifacts
# use v3 as more recent versions cannot run on Ruche
uses: actions/upload-artifact@v3
with:
name: tests_${{ matrix.backend.name }}
path: tests_${{ matrix.backend.name }}.tar
if: ${{ matrix.target.name == 'native' }}
# use the project in another project
install:
runs-on: ubuntu-latest
needs:
- check_docker_files
- build_base
- build
# run this job even if build_base did not run
if: ${{ ! cancelled() && (needs.build_base.result == 'success' || needs.build_base.result == 'skipped') }}
strategy:
matrix:
backend:
- name: openmp
image: gcc
compiler:
c: gcc
cxx: g++
cmake_flags:
kokkos: -DKokkos_ENABLE_OPENMP=ON
kokkos_fft: ""
- name: threads
image: gcc
compiler:
c: gcc
cxx: g++
cmake_flags:
kokkos: -DKokkos_ENABLE_THREADS=ON
kokkos_fft: ""
- name: serial
image: gcc
compiler:
c: gcc
cxx: g++
cmake_flags:
kokkos: -DKokkos_ENABLE_SERIAL=ON
kokkos_fft: ""
- name: cuda
image: nvcc
compiler:
c: gcc
cxx: g++
cmake_flags:
kokkos: -DKokkos_ENABLE_CUDA=ON -DKokkos_ARCH_AMPERE80=ON
kokkos_fft: ""
- name: hip
image: rocm
compiler:
c: hipcc
cxx: hipcc
cmake_flags:
kokkos: -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON
kokkos_fft: -DKokkosFFT_ENABLE_ROCFFT=ON
- name: rocm
image: rocm
compiler:
c: hipcc
cxx: hipcc
cmake_flags:
kokkos: -DKokkos_ENABLE_HIP=ON -DKokkos_ARCH_VEGA90A=ON
kokkos_fft: ""
- name: sycl
image: intel
compiler:
c: icx
cxx: icpx
cmake_flags:
# building for Intel PVC was unsuccessful without the proper
# device, so for now, we simply generate generic Intel GPU code
kokkos: -DKokkos_ENABLE_SYCL=ON -DKokkos_ARCH_INTEL_GEN=ON
kokkos_fft: ""
steps:
- name: Checkout built branch
uses: actions/checkout@v4
with:
submodules: recursive
- name: Prepare as subdirectory
run: |
# prepare kokkos
git clone ./tpls/kokkos ./install_test/as_subdirectory/kokkos
# prepare kokkos-fft
git clone . ./install_test/as_subdirectory/kokkos-fft
- name: Configure and build test project as subdirectory
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake -B install_test/as_subdirectory/build \
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
-DCMAKE_CXX_STANDARD=17 \
${{ matrix.backend.cmake_flags.kokkos }} \
${{ matrix.backend.cmake_flags.kokkos_fft }} \
install_test/as_subdirectory
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --build install_test/as_subdirectory/build
- name: Configure, build and install Kokkos as library
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake -B build_kokkos \
-DCMAKE_INSTALL_PREFIX=$PWD/install \
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
-DCMAKE_CXX_STANDARD=17 \
${{ matrix.backend.cmake_flags.kokkos }} \
tpls/kokkos
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --build build_kokkos -j $(( $(nproc) * 2 + 1 ))
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --install build_kokkos
- name: Configure, build and install KokkosFFT as library
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake -B build_kokkos_fft \
-DCMAKE_INSTALL_PREFIX=$PWD/install\
-DCMAKE_PREFIX_PATH=$PWD/install \
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
-DCMAKE_CXX_STANDARD=17 \
${{ matrix.backend.cmake_flags.kokkos_fft }}
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --build build_kokkos_fft -j $(( $(nproc) * 2 + 1 ))
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --install build_kokkos_fft
- name: Configure and build test project as library
run: |
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake -B install_test/as_library/build \
-DCMAKE_BUILD_TYPE=${{ env.BUILD_TYPE }} \
-DCMAKE_PREFIX_PATH=$PWD/install \
-DCMAKE_C_COMPILER=${{ matrix.backend.compiler.c }} \
-DCMAKE_CXX_COMPILER=${{ matrix.backend.compiler.cxx }} \
-DCMAKE_CXX_STANDARD=17 \
install_test/as_library
docker run -v ${{ github.workspace }}:/work ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
cmake --build install_test/as_library/build -j $(( $(nproc) * 2 + 1 ))
# test the project
test:
runs-on: ${{ matrix.backend.runner }}
needs:
- check_docker_files
- build
# run this job even if build_base did not run
if: ${{ ! cancelled() && needs.build.result == 'success' }}
strategy:
matrix:
backend:
# run CUDA tests on Ruche supercomputer
- name: cuda
image: nvcc
runner: [self-hosted, cuda]
# run OpenMP tests on Azure server
- name: openmp
image: gcc
runner: ubuntu-latest
# run Threads tests on Azure server
- name: threads
image: gcc
runner: ubuntu-latest
# run Serial tests on Azure server
- name: serial
image: gcc
runner: ubuntu-latest
steps:
- name: Get artifacts
# use v3 as more recent versions cannot run on Ruche
uses: actions/download-artifact@v3
with:
name: tests_${{ matrix.backend.name }}
- name: Deploy artifacts
run: tar -xvf tests_${{ matrix.backend.name }}.tar
- name: Pull Singularity image
# pulling the image in advance seems necessary as sometimes invoking `singularity run` on the image URL fails because it cannot find ghcr.io
run: singularity pull oras://ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }}
if: ${{ matrix.backend.name == 'cuda' }}
- name: Run CUDA tests within Slurm job and Singularity image
run: |
srun --nodes=1 --time=01:00:00 -p gpua100 --gres=gpu:1 \
singularity run --nv --bind $PWD/build:/work/build -H /work/build base_${{ matrix.backend.image }}_singularity_${{ needs.check_docker_files.outputs.image_suffix }}_${{ needs.check_docker_files.outputs.image_tag }}.sif \
ctest --output-on-failure
if: ${{ matrix.backend.name == 'cuda' }}
- name: Run OpenMP/Threads/Serial tests within Docker image
run: |
docker run -v $PWD/build:/work/build -w /work/build ghcr.io/cexa-project/kokkos-fft/base_${{ matrix.backend.image }}_${{ needs.check_docker_files.outputs.image_suffix }}:${{ needs.check_docker_files.outputs.image_tag }} \
ctest --output-on-failure
if: ${{ matrix.backend.name == 'openmp' || matrix.backend.name == 'threads' || matrix.backend.name == 'serial' }}