-
Notifications
You must be signed in to change notification settings - Fork 92
348 lines (276 loc) · 11 KB
/
kafka_api_ci_tests.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
name: '[Kafka API] CI Tests'
on:
pull_request:
push:
branches:
- main
env:
KAFKA_SRC_LINK: https://archive.apache.org/dist/kafka/3.3.1/kafka_2.13-3.3.1.tgz
CPU_CORE_NUM: 2
LIBRDKAFKA_TAG: v2.4.0
BUILD_SUB_DIR: builds/sub-build
jobs:
kafka-api-tests:
runs-on: ${{ matrix.os }}
env:
OS_VERSION: ${{ matrix.os }}
BUILD_CXX: ${{ matrix.build-cxx }}
BUILD_TYPE: ${{ matrix.build-type }}
CXX_STANDARD: ${{ matrix.cxx-standard }}
ENABLE_UT_STUBS: ${{ matrix.enable-ut-stubs }}
CHECK_OPTION: ${{ matrix.check-option }}
GENERATE_DOC: ${{ matrix.generate-doc }}
WITH_INSTALLATION: ${{ matrix.with-installation }}
TEST_LABELS: ${{ matrix.test-labels }}
strategy:
matrix:
include:
- os: macos-12
build-cxx: clang++
test-labels: unit|integration
enable-ut-stubs: true
- os: macos-12
build-cxx: clang++
test-labels: robustness
- os: macos-12
build-cxx: clang++
check-option: clang-tidy
- os: ubuntu-22.04
build-cxx: g++
build-type: Debug
test-labels: unit|integration
enable-ut-stubs: true
- os: ubuntu-24.04
build-cxx: g++
build-type: Release
test-labels: robustness
- os: ubuntu-24.04
build-cxx: g++
build-type: Release
cxx-standard: 14
test-labels: unit|integration
- os: ubuntu-24.04
build-cxx: g++
check-option: asan
test-labels: unit|integration
- os: ubuntu-24.04
build-cxx: g++
check-option: tsan
test-labels: unit|integration
- os: ubuntu-24.04
build-cxx: g++
check-option: ubsan
test-labels: unit|integration
- os: ubuntu-24.04
build-cxx: clang++
test-labels: unit|integration
generate-doc: true
with-installation: true
- os: ubuntu-24.04
build-cxx: clang++
check-option: clang-tidy
enable-ut-stubs: true
- os: ubuntu-22.04
build-cxx: g++
test-labels: unit|integration
- os: ubuntu-22.04
build-cxx: clang++
test-labels: robustness
steps:
- uses: actions/checkout@v3
- name: Preparation
if: ${{!contains(matrix.os, 'windows')}}
run: |
if [[ ${OS_VERSION} == 'ubuntu'* ]]; then
sed -e 's/azure.archive.ubuntu.com/us.archive.ubuntu.com/g' -e t -e d /etc/apt/sources.list | sudo tee /etc/apt/sources.list.d/nonazure.list
sudo apt-get update
elif [[ ${OS_VERSION} == 'macos'* ]]; then
echo "CPU_CORE_NUM=3" >> $GITHUB_ENV
fi
- name: Install Dependencies
run: |
mkdir -p ${BUILD_SUB_DIR}
cd ${BUILD_SUB_DIR}
# 1. Install clang/clang-tidy
if [[ ${CHECK_OPTION} == 'clang-tidy' ]]; then
if [[ ${OS_VERSION} == 'ubuntu'* ]]; then
sudo apt install -y clang-tidy
elif [[ ${OS_VERSION} == 'macos'* ]]; then
brew install llvm
export PATH=/usr/local/opt/llvm/bin/:$PATH
fi
fi
# 2. Install googletest (v1.11.0)
wget -nv https://github.com/google/googletest/archive/refs/tags/release-1.11.0.tar.gz
tar -xzf release-1.11.0.tar.gz
cd googletest-release-1.11.0
env CXX=${BUILD_CXX} cmake ./
make -j${CPU_CORE_NUM} && sudo make install
cd ../
# 3. Install boost lib
if [[ ${OS_VERSION} == 'ubuntu'* ]]; then
sudo apt install -y libboost-all-dev
elif [[ ${OS_VERSION} == 'macos'* ]]; then
brew install boost
fi
# 4. Install librdkafka
git clone https://github.com/edenhill/librdkafka.git
cd librdkafka
git reset --hard ${LIBRDKAFKA_TAG}
./configure --cxx=${BUILD_CXX}
make -j${CPU_CORE_NUM} && sudo make install
cd ../
# 5. Install rapidjson (for `addons/KafkaMetrics.h`)
if [[ ${OS_VERSION} == 'ubuntu'* ]]; then
sudo apt install -y rapidjson-dev
else
wget -nv https://github.com/Tencent/rapidjson/archive/refs/tags/v1.1.0.tar.gz
tar -xzf v1.1.0.tar.gz
fi
# 6. Install tools to generate document
if [ ${GENERATE_DOC} ]; then
sudo apt install -y python3-markdown
sudo apt install -y doxygen
fi
# 6. Install Jre, Kafka for integration/regression tests
if [[ ${TEST_LABELS} == *'integration'* ]] || [[ ${TEST_LABELS} == *'robustness'* ]]; then
# Install Jre
if [[ ${OS_VERSION} == 'ubuntu-22.04' ]]; then
sudo apt install -y openjdk-18-jre
fi
# Install kafka
time wget -nv ${KAFKA_SRC_LINK}
tar -xzf `basename ${KAFKA_SRC_LINK}`
export PATH=`pwd`/`basename ${KAFKA_SRC_LINK} .tgz`/bin:$PATH
fi
# 7. Install gtest-parallel for integration test
if [[ ${TEST_LABELS} == *'integration'* ]]; then
wget https://github.com/google/gtest-parallel/archive/refs/heads/master.zip
unzip master.zip
export PATH=`pwd`/gtest-parallel-master:$PATH
fi
# Save the PATH to environment file
echo "PATH=${PATH}" >> $GITHUB_ENV
# Print dev env
cmake --version | head -n 1
g++ --version | head -n 1
clang++ --version | head -n 1
- name: Config
run: |
cd ${BUILD_SUB_DIR}
# Restore the PATH from environment file
export PATH=${{ env.PATH }}
if [ ${CXX_STANDARD} ]; then
export CMAKE_CXX_STANDARD="-DCMAKE_CXX_STANDARD=${CXX_STANDARD}"
fi
if [ ${BUILD_TYPE} ]; then
export CMAKE_BUILD_TYPE="-DCMAKE_BUILD_TYPE=${BUILD_TYPE}"
else
export CMAKE_BUILD_TYPE=""
fi
if [[ ${ENABLE_UT_STUBS} ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_ENABLE_UT_STUBS=ON"
fi
if [[ ${CHECK_OPTION} == 'clang-tidy' ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_CLANG_TIDY=ON"
fi
if [[ ${CHECK_OPTION} == 'asan' ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_USE_ASAN=ON"
fi
if [[ ${CHECK_OPTION} == 'tsan' ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_USE_TSAN=ON"
fi
if [[ ${CHECK_OPTION} == *"ubsan" ]]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_USE_UBSAN=ON"
fi
if [ ${GENERATE_DOC} ]; then
export BUILD_OPTION="${BUILD_OPTION} -DBUILD_OPTION_GEN_DOC=ON"
fi
if [ -d "rapidjson-1.1.0" ]; then
export RAPIDJSON_INCLUDE_DIRS=`pwd`/rapidjson-1.1.0/include
fi
env CXX=${BUILD_CXX} cmake ../.. ${CMAKE_CXX_STANDARD} ${CMAKE_BUILD_TYPE} ${BUILD_OPTION}
- name: Build
run: |
cd ${BUILD_SUB_DIR}
make -j${CPU_CORE_NUM} VERBOSE=1
- name: Install
run: |
if [ ${WITH_INSTALLATION} ]; then
cd ${BUILD_SUB_DIR}
sudo make install
fi
- name: Test
if: matrix.test-labels
timeout-minutes: 15
run: |
cd ${BUILD_SUB_DIR}
# Restore the PATH from environment file
export PATH=${{ env.PATH }}
# Start Kafka cluster
rm -f test.env
../../scripts/start-local-kafka-cluster.py --zookeeper-port 42181 --broker-ports 40091 40092 40093 --temp-dir ./tmp &
# Wait server to be ready
for i in {1..60}; do cat test.env 2>/dev/null && break || sleep 1; done
# Set the environment variables: KAFKA_BROKER_LIST, KAFKA_BROKER_PIDS
source test.env
# Run tests
ctest -VV -L "${TEST_LABELS}"
# Stop Kafka cluster for integration test & regression test
if [[ ${TEST_LABELS} == *'integration'* ]] || [[ ${TEST_LABELS} == *'robustness'* ]]; then
# Stop kafka cluster
kafka-server-stop.sh
zookeeper-server-stop.sh
fi
kafka-api-tests-windows:
runs-on: windows-latest
env:
TEST_LABELS: ${{ matrix.test-labels }}
strategy:
matrix:
include:
- test-labels: ''
steps:
- uses: actions/checkout@v3
- name: Install Dependencies
run: |
mkdir -p $Env:BUILD_SUB_DIR
cd $Env:BUILD_SUB_DIR
# Install librdkafka
vcpkg install librdkafka
# Install googletest
vcpkg install gtest
cp -v "C:\VCPKG\INSTALLED\x64-windows\lib\manual-link\gtest_main*" "C:\VCPKG\INSTALLED\x64-windows\lib\"
cp -v "C:\VCPKG\INSTALLED\x64-windows\lib\manual-link\gtest_main*" "C:\VCPKG\INSTALLED\x64-windows\lib\"
# Install boost headers/libraries
vcpkg install boost-optional
vcpkg install boost-algorithm
vcpkg install boost-program-options
ls "C:\VCPKG\INSTALLED\x64-windows\lib"
cp -v "C:\VCPKG\INSTALLED\x64-windows\lib\boost_program_options-vc144-mt-x64-1_85.lib" "C:\VCPKG\INSTALLED\x64-windows\lib\boost_program_options.lib"
# Install rapidjson
vcpkg install rapidjson
vcpkg integrate install
- name: Config
run: |
cd $Env:BUILD_SUB_DIR
$Env:GTEST_ROOT='C:\VCPKG\INSTALLED\x64-windows\'
$Env:BOOST_ROOT='C:\VCPKG\INSTALLED\x64-windows\'
$Env:LIBRDKAFKA_INCLUDE_DIR='C:\VCPKG\INSTALLED\x64-windows\include\'
$Env:LIBRDKAFKA_LIBRARY_DIR='C:\VCPKG\INSTALLED\x64-windows\lib\'
$Env:RAPIDJSON_INCLUDE_DIRS='C:\VCPKG\INSTALLED\x64-windows\include\'
cmake -B ./ -A x64 -S ../.. "-DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake"
- name: Build
run: |
cd $Env:BUILD_SUB_DIR
cmake --build ./
- name: Test
if: matrix.test-labels
timeout-minutes: 15
run: |
cd $Env:BUILD_SUB_DIR
tree "tests"
# Install kafka
Invoke-WebRequest -Uri $Env:KAFKA_SRC_LINK -OutFile kafka_2.13-2.8.1.tgz
tar xvzf kafka_2.13-2.8.1.tgz
ctest -VV -L $Env:TEST_LABELS