Skip to content

Merge branch 'main' into dont-swallow-errors #1

Merge branch 'main' into dont-swallow-errors

Merge branch 'main' into dont-swallow-errors #1

Workflow file for this run

name: facebook/rocksdb/pr-jobs
on: [push, pull_request]
jobs:
# NOTE: multiple workflows would be recommended, but the current GHA UI in
# PRs doesn't make it clear when there's an overall error with a workflow,
# making it easy to overlook something broken. Grouping everything into one
# workflow minimizes the problem because it will be suspicious if there are
# no GHA results.
#
# The if: ${{ github.repository_owner == 'facebook' }} lines prevent the
# jobs from attempting to run on repo forks, because of a few problems:
# * runs-on labels are repository (owner) specific, so the job might wait
# for days waiting for a runner that simply isn't available.
# * Pushes to branches on forks for pull requests (the normal process) would
# run the workflow jobs twice: once in the pull-from fork and once for the PR
# destination repo. This is wasteful and dumb.
# * It is not known how to avoid copy-pasting the line to each job,
# increasing the risk of misconfiguration, especially on forks that might
# want to run with this GHA setup.
#
# DEBUGGING WITH SSH: Temporarily add this as a job step, either before the
# step of interest without the "if:" line or after the failing step with the
# "if:" line. Then use ssh command printed in CI output.
# - name: Setup tmate session # TEMPORARY!
# if: ${{ failure() }}
# uses: mxschmitt/action-tmate@v3
# with:
# limit-access-to-actor: true
# ======================== Fast Initial Checks ====================== #
check-format-and-targets:
if: ${{ github.repository_owner == 'facebook' }}
runs-on: ubuntu-latest
steps:
- uses: actions/[email protected]
with:
fetch-depth: 0 # Need full checkout to determine merge base
fetch-tags: true
- uses: "./.github/actions/setup-upstream"
- name: Setup Python
uses: actions/setup-python@v5
- name: Install Dependencies
run: python -m pip install --upgrade pip
- name: Install argparse
run: pip install argparse
- name: Download clang-format-diff.py
run: wget https://raw.githubusercontent.com/llvm/llvm-project/release/12.x/clang/tools/clang-format/clang-format-diff.py
- name: Check format
run: VERBOSE_CHECK=1 make check-format
- name: Compare buckify output
run: make check-buck-targets
- name: Simple source code checks
run: make check-sources
# ========================= Linux With Tests ======================== #
build-linux:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: make V=1 J=32 -j32 check
- uses: "./.github/actions/post-steps"
build-linux-cmake-mingw:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 4-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: update-alternatives --set x86_64-w64-mingw32-g++ /usr/bin/x86_64-w64-mingw32-g++-posix
- name: Build cmake-mingw
run: |-
export PATH=$JAVA_HOME/bin:$PATH
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
mkdir build && cd build && cmake -DJNI=1 -DWITH_GFLAGS=OFF .. -DCMAKE_C_COMPILER=x86_64-w64-mingw32-gcc -DCMAKE_CXX_COMPILER=x86_64-w64-mingw32-g++ -DCMAKE_SYSTEM_NAME=Windows && make -j4 rocksdb rocksdbjni
- uses: "./.github/actions/post-steps"
build-linux-cmake-with-folly:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- uses: "./.github/actions/setup-folly"
- uses: "./.github/actions/build-folly"
- run: "(mkdir build && cd build && cmake -DUSE_FOLLY=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)"
- uses: "./.github/actions/post-steps"
build-linux-cmake-with-folly-lite-no-test:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- uses: "./.github/actions/setup-folly"
- run: "(mkdir build && cd build && cmake -DUSE_FOLLY_LITE=1 -DWITH_GFLAGS=1 .. && make V=1 -j20)"
- uses: "./.github/actions/post-steps"
build-linux-gcc-7-with-folly:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- uses: "./.github/actions/setup-folly"
- uses: "./.github/actions/build-folly"
- run: USE_FOLLY=1 LIB_MODE=static CC=gcc-7 CXX=g++-7 V=1 make -j32 check
- uses: "./.github/actions/post-steps"
build-linux-gcc-7-with-folly-lite-no-test:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- uses: "./.github/actions/setup-folly"
- run: USE_FOLLY_LITE=1 CC=gcc-7 CXX=g++-7 V=1 make -j32 all
- uses: "./.github/actions/post-steps"
build-linux-cmake-with-folly-coroutines:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
env:
CC: gcc-10
CXX: g++-10
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- uses: "./.github/actions/setup-folly"
- uses: "./.github/actions/build-folly"
- run: "(mkdir build && cd build && cmake -DUSE_COROUTINES=1 -DWITH_GFLAGS=1 -DROCKSDB_BUILD_SHARED=0 .. && make V=1 -j20 && ctest -j20)"
- uses: "./.github/actions/post-steps"
build-linux-cmake-with-benchmark:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: mkdir build && cd build && cmake -DWITH_GFLAGS=1 -DWITH_BENCHMARK=1 .. && make V=1 -j20 && ctest -j20
- uses: "./.github/actions/post-steps"
build-linux-encrypted_env-no_compression:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: ENCRYPTED_ENV=1 ROCKSDB_DISABLE_SNAPPY=1 ROCKSDB_DISABLE_ZLIB=1 ROCKSDB_DISABLE_BZIP=1 ROCKSDB_DISABLE_LZ4=1 ROCKSDB_DISABLE_ZSTD=1 make V=1 J=32 -j32 check
- run: "./sst_dump --help | grep -E -q 'Supported compression types: kNoCompression$' # Verify no compiled in compression\n"
- uses: "./.github/actions/post-steps"
# ======================== Linux No Test Runs ======================= #
build-linux-release:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- run: make V=1 -j32 LIB_MODE=shared release
- run: ls librocksdb.so
- run: "./db_stress --version"
- run: make clean
- run: make V=1 -j32 release
- run: ls librocksdb.a
- run: "./db_stress --version"
- run: make clean
- run: apt-get remove -y libgflags-dev
- run: make V=1 -j32 LIB_MODE=shared release
- run: ls librocksdb.so
- run: if ./db_stress --version; then false; else true; fi
- run: make clean
- run: make V=1 -j32 release
- run: ls librocksdb.a
- run: if ./db_stress --version; then false; else true; fi
- uses: "./.github/actions/post-steps"
build-linux-release-rtti:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 8-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
- run: "./db_stress --version"
- run: make clean
- run: apt-get remove -y libgflags-dev
- run: USE_RTTI=1 DEBUG_LEVEL=0 make V=1 -j16 static_lib tools db_bench
- run: if ./db_stress --version; then false; else true; fi
build-examples:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 4-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- name: Build examples
run: make V=1 -j4 static_lib && cd examples && make V=1 -j4
- uses: "./.github/actions/post-steps"
build-fuzzers:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 4-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- name: Build rocksdb lib
run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j4 static_lib
- name: Build fuzzers
run: cd fuzz && make sst_file_writer_fuzzer db_fuzzer db_map_fuzzer
- uses: "./.github/actions/post-steps"
build-linux-clang-no_test_run:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 8-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- run: CC=clang CXX=clang++ USE_CLANG=1 PORTABLE=1 make V=1 -j16 all
- uses: "./.github/actions/post-steps"
build-linux-clang-13-no_test_run:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: CC=clang-13 CXX=clang++-13 USE_CLANG=1 make -j32 all microbench
- uses: "./.github/actions/post-steps"
build-linux-gcc-8-no_test_run:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: CC=gcc-8 CXX=g++-8 V=1 make -j32 all
- uses: "./.github/actions/post-steps"
build-linux-gcc-10-cxx20-no_test_run:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: CC=gcc-10 CXX=g++-10 V=1 ROCKSDB_CXX_STANDARD=c++20 make -j32 all
- uses: "./.github/actions/post-steps"
build-linux-gcc-11-no_test_run:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: LIB_MODE=static CC=gcc-11 CXX=g++-11 V=1 make -j32 all microbench
- uses: "./.github/actions/post-steps"
# ======================== Linux Other Checks ======================= #
build-linux-clang10-clang-analyze:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 CLANG_ANALYZER="/usr/bin/clang++-10" CLANG_SCAN_BUILD=scan-build-10 USE_CLANG=1 make V=1 -j32 analyze
- uses: "./.github/actions/post-steps"
- name: compress test report
run: tar -cvzf scan_build_report.tar.gz scan_build_report
if: failure()
- uses: actions/[email protected]
with:
path: scan_build_report.tar.gz
build-linux-unity-and-headers:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 4-core-ubuntu
container:
image: gcc:latest
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- run: apt-get update -y && apt-get install -y libgflags-dev
- name: Unity build
run: make V=1 -j8 unity_test
- run: make V=1 -j8 -k check-headers
- uses: "./.github/actions/post-steps"
build-linux-mini-crashtest:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 4-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: ulimit -S -n `ulimit -H -n` && make V=1 -j8 CRASH_TEST_EXT_ARGS='--duration=960 --max_key=2500000 --use_io_uring=0' blackbox_crash_test_with_atomic_flush
- uses: "./.github/actions/post-steps"
# ======================= Linux with Sanitizers ===================== #
build-linux-clang10-asan:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 32-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: COMPILE_WITH_ASAN=1 CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
- uses: "./.github/actions/post-steps"
build-linux-clang10-ubsan:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: COMPILE_WITH_UBSAN=1 OPT="-fsanitize-blacklist=.circleci/ubsan_suppression_list.txt" CC=clang-10 CXX=clang++-10 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 ubsan_check
- uses: "./.github/actions/post-steps"
build-linux-clang10-mini-tsan:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 32-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: COMPILE_WITH_TSAN=1 CC=clang-13 CXX=clang++-13 ROCKSDB_DISABLE_ALIGNED_NEW=1 USE_CLANG=1 make V=1 -j32 check
- uses: "./.github/actions/post-steps"
build-linux-static_lib-alt_namespace-status_checked:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 16-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- run: ASSERT_STATUS_CHECKED=1 TEST_UINT128_COMPAT=1 ROCKSDB_MODIFY_NPHASH=1 LIB_MODE=static OPT="-DROCKSDB_NAMESPACE=alternative_rocksdb_ns" make V=1 -j24 check
- uses: "./.github/actions/post-steps"
# ========================= MacOS with Tests ======================== #
build-macos:
if: ${{ github.repository_owner == 'facebook' }}
runs-on: macos-13
env:
ROCKSDB_DISABLE_JEMALLOC: 1
steps:
- uses: actions/[email protected]
- uses: maxim-lobanov/[email protected]
with:
xcode-version: 14.3.1
- uses: "./.github/actions/increase-max-open-files-on-macos"
- uses: "./.github/actions/install-gflags-on-macos"
- uses: "./.github/actions/pre-steps-macos"
- name: Run all tests
run: ulimit -S -n `ulimit -H -n` && make V=1 J=16 -j16 all
- uses: "./.github/actions/post-steps"
build-macos-cmake:
if: ${{ github.repository_owner == 'facebook' }}
runs-on: macos-13
strategy:
matrix:
run_even_tests: [true, false]
steps:
- uses: actions/[email protected]
- uses: maxim-lobanov/[email protected]
with:
xcode-version: 14.3.1
- uses: "./.github/actions/increase-max-open-files-on-macos"
- uses: "./.github/actions/install-gflags-on-macos"
- uses: "./.github/actions/pre-steps-macos"
- name: cmake generate project file
run: ulimit -S -n `ulimit -H -n` && mkdir build && cd build && cmake -DWITH_GFLAGS=1 ..
- name: Build tests
run: cd build && make V=1 -j16
- name: Run even tests
run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 0,,2
if: ${{ matrix.run_even_tests }}
- name: Run odd tests
run: ulimit -S -n `ulimit -H -n` && cd build && ctest -j16 -I 1,,2
if: ${{ ! matrix.run_even_tests }}
- uses: "./.github/actions/post-steps"
# ======================== Windows with Tests ======================= #
# NOTE: some windows jobs are in "nightly" to save resources
build-windows-vs2019:
if: ${{ github.repository_owner == 'facebook' }}
runs-on: windows-2019
env:
CMAKE_GENERATOR: Visual Studio 16 2019
CMAKE_PORTABLE: 1
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/windows-build-steps"
# ============================ Java Jobs ============================ #
build-linux-java:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 4-core-ubuntu
container:
image: zjay437/rocksdb:0.6
options: --shm-size=16gb
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/pre-steps"
- name: Set Java Environment
run: |-
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
- name: Test RocksDBJava
run: make V=1 J=8 -j8 jtest
- uses: "./.github/actions/post-steps"
build-linux-java-static:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 4-core-ubuntu
container:
image: evolvedbinary/rocksjava:centos6_x64-be
options: --shm-size=16gb
steps:
# The docker image is based on such an old OS that it has a GLIBC
# incompatibility with actions/checkout and other actions. Thus we
# implement a manual checkout step.
- name: Checkout
env:
GH_TOKEN: ${{ github.token }}
run: |
chown `whoami` . || true
git clone --no-checkout https://oath2:[email protected]/${{ github.repository }}.git .
git -c protocol.version=2 fetch --update-head-ok --no-tags --prune --no-recurse-submodules --depth=1 origin +${{ github.sha }}:${{ github.ref }}
git checkout --progress --force ${{ github.ref }}
git log -1 --format='%H'
- uses: "./.github/actions/pre-steps"
- name: Set Java Environment
run: |-
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
- name: Build RocksDBJava Static Library
run: scl enable devtoolset-7 'make V=1 J=8 -j8 rocksdbjavastatic'
# NOTE: post-steps skipped because of compatibility issues with docker image
build-macos-java:
if: ${{ github.repository_owner == 'facebook' }}
runs-on: macos-13
env:
JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home"
ROCKSDB_DISABLE_JEMALLOC: 1
steps:
- uses: actions/[email protected]
- uses: maxim-lobanov/[email protected]
with:
xcode-version: 14.3.1
- uses: "./.github/actions/increase-max-open-files-on-macos"
- uses: "./.github/actions/install-gflags-on-macos"
- uses: "./.github/actions/install-jdk8-on-macos"
- uses: "./.github/actions/pre-steps-macos"
- name: Set Java Environment
run: |-
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
- name: Test RocksDBJava
run: make V=1 J=16 -j16 jtest
- uses: "./.github/actions/post-steps"
build-macos-java-static:
if: ${{ github.repository_owner == 'facebook' }}
runs-on: macos-13
env:
JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home"
steps:
- uses: actions/[email protected]
- uses: maxim-lobanov/[email protected]
with:
xcode-version: 14.3.1
- uses: "./.github/actions/increase-max-open-files-on-macos"
- uses: "./.github/actions/install-gflags-on-macos"
- uses: "./.github/actions/install-jdk8-on-macos"
- uses: "./.github/actions/pre-steps-macos"
- name: Set Java Environment
run: |-
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
- name: Build RocksDBJava x86 and ARM Static Libraries
run: make V=1 J=16 -j16 rocksdbjavastaticosx
- uses: "./.github/actions/post-steps"
build-macos-java-static-universal:
if: ${{ github.repository_owner == 'facebook' }}
runs-on: macos-13
env:
JAVA_HOME: "/Library/Java/JavaVirtualMachines/liberica-jdk-8.jdk/Contents/Home"
steps:
- uses: actions/[email protected]
- uses: maxim-lobanov/[email protected]
with:
xcode-version: 14.3.1
- uses: "./.github/actions/increase-max-open-files-on-macos"
- uses: "./.github/actions/install-gflags-on-macos"
- uses: "./.github/actions/install-jdk8-on-macos"
- uses: "./.github/actions/pre-steps-macos"
- name: Set Java Environment
run: |-
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
- name: Build RocksDBJava Universal Binary Static Library
run: make V=1 J=16 -j16 rocksdbjavastaticosx_ub
- uses: "./.github/actions/post-steps"
build-linux-java-pmd:
if: ${{ github.repository_owner == 'facebook' }}
runs-on:
labels: 4-core-ubuntu
steps:
- uses: actions/[email protected]
- uses: "./.github/actions/install-maven"
- uses: "./.github/actions/pre-steps"
- name: Set Java Environment
run: |-
echo "JAVA_HOME=${JAVA_HOME}"
which java && java -version
which javac && javac -version
- name: PMD RocksDBJava
run: make V=1 J=8 -j8 jpmd
- uses: "./.github/actions/post-pmd-steps"