introducing alternative coverage measurements using gcov #3
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: build and test | |
on: | |
push: | |
branches: [ main, "pr/**" ] | |
pull_request: | |
branches: [ main ] | |
workflow_dispatch: | |
merge_group: | |
env: | |
CARGO_TERM_COLOR: always | |
CARGO_NET_GIT_FETCH_WITH_CLI: true | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.ref }} | |
cancel-in-progress: true | |
jobs: | |
common: | |
strategy: | |
fail-fast: false | |
matrix: | |
os: [ ubuntu-latest, windows-latest, macOS-latest ] | |
runs-on: ${{ matrix.os }} | |
steps: | |
- name: Install mimetype | |
if: runner.os == 'Linux' | |
run: sudo apt-get install libfile-mimeinfo-perl | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: nightly | |
override: true | |
- name: Install mimetype | |
if: runner.os == 'Linux' | |
run: sudo apt-get install libfile-mimeinfo-perl | |
- name: install mdbook | |
uses: baptiste0928/[email protected] | |
with: | |
crate: mdbook | |
- name: install linkcheck | |
uses: baptiste0928/[email protected] | |
with: | |
crate: mdbook-linkcheck | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
with: { shared-key: "ubuntu" } | |
if: runner.os == 'Linux' | |
- uses: Swatinem/rust-cache@v2 | |
if: runner.os != 'Linux' | |
- name: Check for binary blobs | |
if: runner.os == 'Linux' | |
run: ./scripts/check_for_blobs.sh | |
- name: default nightly | |
run: rustup default nightly | |
- name: Build libafl debug | |
run: cargo build -p libafl | |
- name: Test the book | |
# TODO: fix books test fail with updated windows-rs | |
if: runner.os != 'Windows' | |
run: cd docs && mdbook test -L ../target/debug/deps | |
- name: Run tests | |
run: cargo test | |
- name: Test libafl no_std | |
run: cd libafl && cargo test --no-default-features | |
- name: Test libafl_bolts no_std no_alloc | |
run: cd libafl_bolts && cargo test --no-default-features | |
- name: Test libafl_targets no_std | |
run: cd libafl_targets && cargo test --no-default-features | |
llvm-tester: | |
runs-on: ubuntu-22.04 | |
continue-on-error: true | |
strategy: | |
matrix: | |
llvm-version: [ "16", "17" ] # Add 18 when KyleMayes/install-llvm-action enables it | |
steps: | |
- name: Remove Dotnet & Haskell | |
run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: stable | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
with: { shared-key: "llvm-tester" } | |
- name: Install LLVM and Clang | |
uses: KyleMayes/install-llvm-action@v2 | |
with: | |
version: "${{matrix.llvm-version}}" | |
- name: Build and test with llvm-${{ matrix.llvm-version }} | |
run: pwd && ls & cd libafl_cc && cargo build --release | |
ubuntu-doc-build: | |
runs-on: ubuntu-22.04 | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/ubuntu-prepare | |
- uses: Swatinem/rust-cache@v2 | |
# ---- doc check ---- | |
- name: Build Docs | |
run: RUSTFLAGS="--cfg docsrs" cargo +nightly doc --all-features --no-deps | |
ubuntu-doc-test: | |
runs-on: ubuntu-22.04 | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/ubuntu-prepare | |
- uses: Swatinem/rust-cache@v2 | |
# ---- doc check ---- | |
- name: Test Docs | |
run: RUSTFLAGS="--cfg docsrs" cargo +nightly test --doc --all-features | |
ubuntu-miri: | |
runs-on: ubuntu-22.04 | |
needs: ubuntu | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/ubuntu-prepare | |
- uses: Swatinem/rust-cache@v2 | |
- name: Add nightly rustfmt and clippy | |
run: rustup toolchain install nightly --component miri --allow-downgrade | |
# --- miri undefined behavior test -- | |
- name: Run miri tests | |
run: RUST_BACKTRACE=1 MIRIFLAGS="-Zmiri-disable-isolation" cargo +nightly miri test | |
ubuntu: | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Remove Dotnet & Haskell | |
run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: stable | |
- name: Remove existing clang and LLVM | |
run: sudo apt purge llvm* clang* lld* lldb* opt* | |
- name: Install and cache deps | |
run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev | |
- name: Add nightly rustfmt and clippy | |
run: rustup toolchain install nightly --component rustfmt --component clippy --component miri --allow-downgrade | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
with: { shared-key: "ubuntu" } | |
- name: Install LLVM and Clang | |
uses: KyleMayes/install-llvm-action@v1 | |
with: | |
directory: ${{ runner.temp }}/llvm | |
version: 17 | |
# ---- format check ---- | |
# pcguard edges and pcguard hitcounts are not compatible and we need to build them seperately | |
- name: Check pcguard edges | |
run: cargo check --features=sancov_pcguard_edges | |
- name: Run clang-format style check for C/C++ programs. | |
run: clang-format -n -Werror --style=file $(find . -type f \( -name '*.cpp' -o -iname '*.hpp' -o -name '*.cc' -o -name '*.cxx' -o -name '*.cc' -o -name '*.h' \) | grep -v '/target/' | grep -v 'libpng-1\.6\.37' | grep -v 'stb_image\.h' | grep -v 'dlmalloc\.c' | grep -v 'QEMU-Nyx') | |
- name: run shellcheck | |
run: shellcheck ./scripts/*.sh | |
# ---- build normal and examples ---- | |
- name: Run a normal build | |
run: cargo build --verbose | |
- name: Build examples | |
run: cargo build --examples --verbose | |
ubuntu-clippy: | |
runs-on: ubuntu-22.04 | |
steps: | |
- name: Remove Dotnet & Haskell | |
run: rm -rf /usr/share/dotnet && rm -rf /opt/ghc | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: stable | |
- name: Install and cache deps | |
run: sudo apt update && sudo apt install ninja-build shellcheck libgtk-3-dev gcc-arm-linux-gnueabi g++-arm-linux-gnueabi libslirp-dev libz3-dev | |
- name: Add nightly rustfmt and clippy | |
run: rustup toolchain install nightly --component clippy --allow-downgrade && rustup default nightly | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
with: { shared-key: "ubuntu" } | |
- name: Install LLVM and Clang | |
uses: KyleMayes/install-llvm-action@v1 | |
with: | |
directory: ${{ runner.temp }}/llvm | |
version: 17 | |
- name: Run clippy | |
run: ./scripts/clippy.sh | |
# --- test embedding the libafl_libfuzzer_runtime library | |
# Fix me plz | |
# - name: Test Build libafl_libfuzzer with embed | |
# run: cargo +nightly test --features=embed-runtime --manifest-path libafl_libfuzzer/Cargo.toml | |
ubuntu-check-nightly: | |
runs-on: ubuntu-22.04 | |
needs: ubuntu | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/ubuntu-prepare | |
- uses: Swatinem/rust-cache@v2 | |
with: { shared-key: "ubuntu" } | |
# ---- build and feature check ---- | |
# cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs | |
- name: Check nightly features | |
run: cargo +nightly check --features=agpl && cargo +nightly check --features=nautilus | |
ubuntu-check: | |
runs-on: ubuntu-22.04 | |
needs: ubuntu | |
strategy: | |
matrix: | |
instance_idx: [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17" ] | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/ubuntu-prepare | |
- uses: Swatinem/rust-cache@v2 | |
with: { shared-key: "ubuntu" } | |
# ---- build and feature check ---- | |
# cargo-hack's --feature-powerset would be nice here but libafl has a too many knobs | |
- name: Check each feature | |
# Skipping `python` as it has to be built with the `maturin` tool | |
# `agpl`, `nautilus` require nightly | |
# `sancov_pcguard_edges` is tested seperatelyc | |
run: python3 ./scripts/parallellize_cargo_check.py ${{ matrix.instance_idx }} | |
ubuntu-concolic: | |
runs-on: ubuntu-latest | |
needs: ubuntu | |
steps: | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: stable | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
with: { shared-key: "ubuntu" } | |
- name: Install smoke test deps | |
run: sudo ./libafl_concolic/test/smoke_test_ubuntu_deps.sh | |
- name: Run smoke test | |
run: ./libafl_concolic/test/smoke_test.sh | |
python-bindings: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: stable | |
- name: Remove existing clang and LLVM | |
run: sudo apt purge llvm* clang* | |
- name: Install LLVM and Clang | |
uses: KyleMayes/install-llvm-action@v1 | |
with: | |
directory: ${{ runner.temp }}/llvm | |
version: 17 | |
- name: Install deps | |
run: sudo apt-get install -y ninja-build python3-dev python3-pip python3-venv libz3-dev | |
- name: Install maturin | |
run: python3 -m pip install maturin | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
- name: Run a maturin build | |
run: export LLVM_CONFIG=llvm-config-16 && cd ./bindings/pylibafl && python3 -m venv .env && . .env/bin/activate && pip install --upgrade --force-reinstall . && ./test.sh | |
- name: Run python test | |
run: . ./bindings/pylibafl/.env/bin/activate # && cd ./fuzzers/python_qemu/ && python3 fuzzer.py 2>&1 | grep "Bye" | |
cargo-fmt: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: nightly | |
override: true | |
components: rustfmt | |
- uses: actions/checkout@v3 | |
- name: Format Check | |
run: cargo fmt -- --check | |
fuzzers-preflight: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Fuzzer in CI Check | |
run: ./scripts/check_tested_fuzzers.sh | |
fuzzers: | |
needs: | |
- ubuntu | |
- fuzzers-preflight | |
strategy: | |
matrix: | |
os: [ ubuntu-latest ] | |
fuzzer: | |
- ./fuzzers/cargo_fuzz | |
- ./fuzzers/fuzzbench_fork_qemu | |
- ./fuzzers/libfuzzer_stb_image_sugar | |
- ./fuzzers/nyx_libxml2_standalone | |
- ./fuzzers/baby_fuzzer_gramatron | |
- ./fuzzers/tinyinst_simple | |
- ./fuzzers/baby_fuzzer_with_forkexecutor | |
- ./fuzzers/baby_no_std | |
- ./fuzzers/baby_fuzzer_swap_differential | |
- ./fuzzers/baby_fuzzer_grimoire | |
- ./fuzzers/baby_fuzzer | |
- ./fuzzers/libfuzzer_libpng_launcher | |
- ./fuzzers/libfuzzer_libpng_accounting | |
- ./fuzzers/forkserver_libafl_cc | |
- ./fuzzers/libfuzzer_libpng_tcp_manager | |
- ./fuzzers/backtrace_baby_fuzzers | |
- ./fuzzers/fuzzbench_qemu | |
- ./fuzzers/nyx_libxml2_parallel | |
- ./fuzzers/frida_gdiplus | |
- ./fuzzers/libfuzzer_stb_image_concolic | |
- ./fuzzers/nautilus_sync | |
- ./fuzzers/push_harness | |
- ./fuzzers/libfuzzer_libpng_centralized | |
- ./fuzzers/baby_fuzzer_nautilus | |
- ./fuzzers/fuzzbench_text | |
- ./fuzzers/libfuzzer_libpng_cmin | |
- ./fuzzers/forkserver_simple | |
- ./fuzzers/baby_fuzzer_unicode | |
- ./fuzzers/libfuzzer_libpng_norestart | |
- ./fuzzers/baby_fuzzer_multi | |
- ./fuzzers/libafl_atheris | |
- ./fuzzers/frida_libpng | |
- ./fuzzers/fuzzbench_ctx | |
- ./fuzzers/fuzzbench_forkserver_cmplog | |
- ./fuzzers/push_stage_harness | |
- ./fuzzers/libfuzzer_libmozjpeg | |
- ./fuzzers/libfuzzer_libpng_aflpp_ui | |
- ./fuzzers/libfuzzer_libpng | |
- ./fuzzers/baby_fuzzer_wasm | |
- ./fuzzers/fuzzbench | |
- ./fuzzers/libfuzzer_stb_image | |
- ./fuzzers/fuzzbench_forkserver | |
# - ./fuzzers/libfuzzer_windows_asan | |
# - ./fuzzers/dynamic_analysis | |
- ./fuzzers/baby_fuzzer_minimizing | |
- ./fuzzers/frida_executable_libpng | |
- ./fuzzers/tutorial | |
- ./fuzzers/baby_fuzzer_tokens | |
- ./fuzzers/backtrace_baby_fuzzers/rust_code_with_inprocess_executor | |
- ./fuzzers/backtrace_baby_fuzzers/c_code_with_fork_executor | |
- ./fuzzers/backtrace_baby_fuzzers/command_executor | |
- ./fuzzers/backtrace_baby_fuzzers/forkserver_executor | |
- ./fuzzers/backtrace_baby_fuzzers/c_code_with_inprocess_executor | |
- ./fuzzers/backtrace_baby_fuzzers/rust_code_with_fork_executor | |
runs-on: ${{ matrix.os }} | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/fuzzer-tester-prepare | |
- name: Build and run example fuzzers (Linux) | |
if: runner.os == 'Linux' | |
shell: bash | |
run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }} | |
changes: | |
runs-on: ubuntu-latest | |
permissions: | |
pull-requests: read | |
outputs: | |
qemu: ${{ steps.filter.outputs.qemu }} | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: dorny/paths-filter@v3 | |
id: filter | |
with: | |
filters: | | |
qemu: | |
- 'libafl_qemu/**' | |
- 'fuzzers/*qemu*/**' | |
fuzzers-qemu: | |
needs: changes | |
if: ${{ needs.changes.outputs.qemu == 'true' }} | |
strategy: | |
matrix: | |
os: [ubuntu-latest] | |
fuzzer: | |
- ./fuzzers/qemu_cmin | |
- ./fuzzers/qemu_systemmode | |
- ./fuzzers/qemu_coverage | |
- ./fuzzers/qemu_launcher | |
runs-on: [ self-hosted, qemu ] | |
container: registry.gitlab.com/qemu-project/qemu/qemu/ubuntu2204:latest | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/qemu-fuzzer-tester-prepare | |
- name: Build and run example QEMU fuzzers (Linux) | |
if: runner.os == 'Linux' | |
shell: bash | |
run: RUN_ON_CI=1 LLVM_CONFIG=llvm-config ./scripts/test_fuzzer.sh ${{ matrix.fuzzer }} | |
nostd-build: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: nightly | |
override: true | |
components: rustfmt, rust-src | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
- name: Add targets | |
run: rustup target add arm-linux-androideabi && rustup target add thumbv6m-none-eabi | |
- name: Build aarch64-unknown-none | |
run: cd ./fuzzers/baby_no_std && cargo +nightly build -Zbuild-std=core,alloc --target aarch64-unknown-none -v --release && cd ../.. | |
- name: run x86_64 until panic! | |
run: cd ./fuzzers/baby_no_std && cargo +nightly run || test $? -ne 0 || exit 1 | |
- name: no_std tests | |
run: cd ./libafl && cargo test --no-default-features | |
nostd-clippy: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: nightly | |
override: true | |
components: rustfmt, clippy, rust-src | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
- name: Add targets | |
run: rustup target add arm-linux-androideabi && rustup target add thumbv6m-none-eabi | |
- name: libafl armv6m-none-eabi (32 bit no_std) clippy | |
run: cd ./libafl && cargo clippy --target thumbv6m-none-eabi --no-default-features | |
- name: Build no_std no_alloc bolts | |
run: cd ./libafl_bolts && cargo +nightly build -Zbuild-std=core --target aarch64-unknown-none --no-default-features -v --release && cd ../ | |
build-docker: | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Build docker | |
run: docker build -t libafl . | |
windows-frida-libpng: | |
runs-on: windows-latest | |
needs: | |
- common | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/windows-tester-prepare | |
- name: Build fuzzers/frida_libpng | |
run: cd fuzzers/frida_libpng/ && cargo make test | |
windows-frida-libfuzzer-stb-image: | |
runs-on: windows-latest | |
needs: | |
- common | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/windows-tester-prepare | |
- name: Build fuzzers/libfuzzer_stb_image | |
run: cd fuzzers/libfuzzer_stb_image && cargo build --release | |
windows-frida-gdiplus: | |
runs-on: windows-latest | |
needs: | |
- common | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/windows-tester-prepare | |
- name: Build fuzzers/frida_gdiplus | |
run: cd fuzzers/frida_gdiplus/ && cargo make test && cargo make test_cmplog | |
windows-tinyinst-simple: | |
runs-on: windows-latest | |
needs: | |
- common | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: ./.github/workflows/windows-tester-prepare | |
- name: install cxx bridge | |
run: cargo install cxxbridge-cmd | |
- name: Build fuzzers/tinyinst_simple | |
run: cd fuzzers/tinyinst_simple/ && cargo make test | |
windows-clippy: | |
runs-on: windows-latest | |
needs: | |
- common | |
steps: | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: stable | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
- name: Run clippy | |
uses: actions-rs/cargo@v1 | |
with: | |
command: clippy | |
macos: | |
runs-on: macOS-latest | |
steps: | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: stable | |
- name: Add nightly rustfmt and clippy | |
run: rustup toolchain install nightly --component rustfmt --component clippy --allow-downgrade && rustup default nightly | |
- name: Install deps | |
run: brew install z3 gtk+3 | |
- name: Install cxxbridge | |
run: cargo install cxxbridge-cmd | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
- name: MacOS Build | |
run: cargo build --verbose | |
- name: Increase map sizes | |
run: ./scripts/shmem_limits_macos.sh | |
- name: Run Tests | |
run: cargo test | |
- name: Clippy | |
run: cargo +nightly clippy --tests --all | |
ios: | |
runs-on: macOS-latest | |
steps: | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: stable | |
- name: install ios | |
run: rustup target add aarch64-apple-ios | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
- name: Build iOS | |
run: cargo build --target aarch64-apple-ios && cd libafl_frida && cargo build --target aarch64-apple-ios && cd .. | |
android: | |
runs-on: ubuntu-22.04 | |
steps: | |
- uses: actions-rs/toolchain@v1 | |
with: | |
profile: minimal | |
toolchain: stable | |
- uses: nttld/setup-ndk@v1 | |
with: | |
ndk-version: r25b | |
- name: install android | |
run: rustup target add aarch64-linux-android | |
- name: install cargo ndk | |
run: cargo install cargo-ndk | |
- uses: actions/checkout@v3 | |
- uses: Swatinem/rust-cache@v2 | |
- name: Build Android | |
run: cd libafl && cargo ndk -t arm64-v8a build --release | |
#run: cargo build --target aarch64-linux-android | |
# TODO: Figure out how to properly build stuff with clang | |
#- name: Add clang path to $PATH env | |
# if: runner.os == 'Windows' | |
# run: echo "C:\msys64\mingw64\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 | |
#- name: Try if clang works | |
# run: clang -v | |
#- name: Windows Test | |
# run: C:\Rust\.cargo\bin\cargo.exe test --verbose | |
freebsd: | |
runs-on: ubuntu-22.04 | |
name: Simple build in FreeBSD | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Test in FreeBSD | |
id: test | |
uses: vmactions/freebsd-vm@v1 | |
with: | |
usesh: true | |
sync: rsync | |
copyback: false | |
mem: 2048 | |
release: 13.2 | |
prepare: | | |
pkg install -y curl bash sudo llvm16 | |
curl https://sh.rustup.rs -sSf | sh -s -- -y | |
run: | | |
freebsd-version | |
. "$HOME/.cargo/env" | |
rustup toolchain install nightly | |
export LLVM_CONFIG=/usr/local/bin/llvm-config16 | |
pwd | |
ls -lah | |
echo "local/bin" | |
ls -lah /usr/local/bin/ | |
which llvm-config | |
chmod +x ./scripts/clippy.sh | |
bash ./scripts/shmem_limits_fbsd.sh | |
bash ./scripts/clippy.sh | |
cargo test |