diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 7c24db311ad..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,621 +0,0 @@ -version: 2.1 - -# If you only want circle to run on direct commits to main, you can uncomment this out -# and uncomment the filters: *filter-only-main down below too -# -aliases: - - &filter-only-main - branches: - only: - - main - - &filter-not-gh-pages-not-main - branches: - ignore: - - gh-pages - - main - -orbs: - win: circleci/windows@2.2.0 - -commands: - # Setup container. - debian-based-image-build-setup: - parameters: - mode_32: - type: boolean - default: false - install_clang: - type: boolean - default: false - install_clang_llvm_org: - type: integer - default: -1 # Off - steps: - - run: - name: Update Apt Data - command: | - sudo apt-get update || ( apt-get update && apt-get install --no-install-recommends -y sudo ; ) - - run: - name: Install tools - command: | - sudo apt-get install --no-install-recommends -y git-restore-mtime zstd tar - - when: - condition: << parameters.mode_32 >> - steps: - - run: - name: Add 32-bit Arch - command: | - sudo dpkg --add-architecture i386 && sudo apt-get update - - when: - condition: << parameters.install_clang >> - steps: - - run: - name: Install Clang - command: | - sudo apt-get install -y --no-install-recommends clang - # Clang from llvm.org - # - # add-apt-repository in the install script needs to be rewritten: - # * -y to not ask for input - # * seemingly the first run does not insert the source, so need to run twice - # - # We symlink the clang-X binary, there is no update-alternatives it seems. - # For simplicity assume there is only one version installed. - # - # 1. 0 = default version - - when: - condition: - equal: [0, << parameters.install_clang_llvm_org >> ] - steps: - - run: - name: Install Clang (apt.llvm.org) - # Link the numbered version so it's easier to use. - command: | - sudo apt-get install -y --no-install-recommends lsb-release wget software-properties-common gnupg - sudo /bin/bash -c "$(wget -O - https://apt.llvm.org/llvm.sh | sed -e 's/^add-apt-repository.*$/& -y\n& -y/')" - ls /usr/bin/clang* - sudo ln -s /usr/bin/clang-?? /usr/bin/clang - sudo ln -s /usr/bin/clang++-?? /usr/bin/clang++ - # 2. >0 = specific version - - when: - condition: - and: - - not: - equal: [-1, << parameters.install_clang_llvm_org >> ] - - not: - equal: [0, << parameters.install_clang_llvm_org >> ] - steps: - - run: - name: Install Clang (apt.llvm.org) - # Link the numbered version so it's easier to use. - command: | - sudo apt-get install -y --no-install-recommends lsb-release wget software-properties-common gnupg - sudo /bin/bash -c "$(wget -O - https://apt.llvm.org/llvm.sh | sed -e 's/^add-apt-repository.*$/& -y\n& -y/')" "llvm.sh" << parameters.install_clang_llvm_org >> - ls /usr/bin/clang* - sudo ln -s /usr/bin/clang-?? /usr/bin/clang - sudo ln -s /usr/bin/clang++-?? /usr/bin/clang++ - - # Setup. Includes boost. - base-build-setup: - parameters: - save_boost_cache: - type: boolean - default: false - setup_toolchain_extra: - type: string - default: "" - steps: - # It is OK to generically restore boost, even if it is not used. - - restore_cache: - name: Restoring Boost DL Cache - keys: - - boost-1-71-0-v5 - - restore_cache: - name: Restoring Protobuf DL Cache - keys: - - protobuf3-v3 - - run: - name: Setup - environment: - TOOLCHAIN_TMP: /tmp/toolchain - working_directory: repo - command: | - sudo -E ./setup_oss_toolchain.sh << parameters.setup_toolchain_extra >> - # Only save the cache when asked, e.g., hopefully when it was populated. - - when: - condition: << parameters.save_boost_cache >> - steps: - - save_cache: - name: "Saving Boost DL Cache" - paths: - - /tmp/toolchain/dl_cache/boost_cache - key: boost-1-71-0-v5 - - save_cache: - name: "Saving Protobuf DL Cache" - paths: - - /tmp/toolchain/dl_cache/protobuf - key: protobuf3-v3 - - # For testing, need additional dependencies not provided by the image. - - # Note: CircleCI does not allow to drop existing caches (in case they are - # outdated, for example). The expectation is that the "cache key" - # changes, for example via the checksum of a dependency file. - # - # This does not apply to our setup. So we use an environment variable - # defined in the project settings. If a cache needs to be invalidated, - # update the variable value. - # - # However, at the same time the project settings values cannot be used - # directly. So use a file as an intermediate. - # - # See: - # * [https://support.circleci.com/hc/en-us/articles/115015426888-Clear-project-dependency-cache] - # * [https://devops.stackexchange.com/questions/9147/how-to-get-other-than-no-value-when-interpolating-environment-some-var] - test-build-setup: - steps: - - run: - name: Setup for tests - command: | - sudo apt-get install -y --no-install-recommends default-jdk-headless zip unzip - - - restore_cache: - name: Restoring SDK Cache - keys: - - v3-android-build-tools - - # We have to emulate cache behavior. Skip downloads if files exist. - - run: - name: Check/Install (SDK) - command: | - if [ -e sdk/build-tools/29.0.2/dx ] ; then echo "Found SDK." ; exit 0 ; fi - - # Check whether cache file exists, then check dx again. - if [ -f "sdk.tar.zstd" ] ; then - zstd --decompress --stdout sdk.tar.zstd | tar xf - - if [ -e sdk/build-tools/29.0.2/dx ] ; then echo "Unpacked SDK." ; exit 0 ; fi - fi - - # Download. - echo "SDK missing, downloading..." - rm -rf sdk 2>/dev/null && mkdir -p sdk/cmdline-tools - echo "Downloading SDK to $(pwd)/sdk" - wget https://dl.google.com/android/repository/commandlinetools-linux-6609375_latest.zip - unzip commandlinetools-linux-6609375_latest.zip -d sdk/cmdline-tools/ - pushd sdk/cmdline-tools/tools/bin >/dev/null - echo 'y' | ./sdkmanager --install 'build-tools;29.0.2' - ./sdkmanager --install 'platforms;android-29' - popd >/dev/null - - # Compress for cache. For reason see description of build cache. - # We do not need the emulator and it is large. Delete. - rm -rf sdk/emulator 2>/dev/null - tar cf - sdk | zstd -T0 -3 > sdk.tar.zstd - - - save_cache: - name: Saving SDK Cache - paths: - - sdk.tar.zstd - key: v3-android-build-tools - - # Level of indirection because there seems to be no way to set parameters - # in the steps. - setup-build-and-test-w-make-impl: - description: "Prepare, build and test ReDex" - parameters: - setup_toolchain_extra: - type: string - default: "" - configure_extra: - default: "" - type: string - save_boost_cache: - type: boolean - default: false - run_tests: - type: boolean - default: true - job_name: - type: string - steps: - - base-build-setup: - save_boost_cache: << parameters.save_boost_cache >> - setup_toolchain_extra: << parameters.setup_toolchain_extra >> - - - test-build-setup - - - run: - name: Create Build Cache File - command: | - echo "<< parameters.job_name >>.<< parameters.setup_toolchain_extra >>.<< parameters.configure_extra >>" > /tmp/build.txt - - - restore_cache: - name: Restoring Build Cache - keys: - # Always distinguish caches for the job, arch and branch. Look for the current revision first, - # then "newest" cache. - - &build-cache v2-build-cache-{{ checksum "/tmp/build.txt" }}-{{ arch }}-{{ .Branch }}-{{ .Revision }} - - v2-build-cache-{{ checksum "/tmp/build.txt" }}-{{ arch }}-{{ .Branch }}- - - v1-build-cache-{{ checksum "/tmp/build.txt" }}-{{ arch }}-{{ .Branch }}- - - - run: - name: Create build directory - command: | - if [ -f "build.tar.zstd" ] ; then zstd --decompress --stdout build.tar.zstd | tar xf - ; else mkdir -p build ; fi - rm -f build.tar.zstd - - - run: - name: Autoreconf - working_directory: repo - command: | - autoreconf -ivf - - - run: - name: Configure - working_directory: build - command: | - ../repo/configure --enable-protobuf --with-android-sdk="$(pwd)/../sdk" << parameters.configure_extra >> - - - run: - name: Build - working_directory: build - # The old Travis limit. Hopefully ensures GCC does - # not die. - command: | - make -j4 V=0 - - # We do not cache tests. They are fast to build and very large. - # It seems better to compress manually, as the CircleCI saver seems to - # be single-threaded. zstd -3 seems about as good as what CircleCI is - # doing, is very fast, and might speed up the following step, where - # we don't seem to be able to turn off compression explicitly. - # - # Think about not caching redex-all. It is very large, likely - # changes and linking is not that slow. - - run: - name: Compress build directory - command: | - tar cf - build | zstd -T0 -3 > build.tar.zstd - - - save_cache: - name: Saving Build Cache - key: *build-cache - paths: - - build.tar.zstd - - - run: - name: Build tests - working_directory: build - command: | - make -j4 check TESTS= V=0 - - - when: - condition: << parameters.run_tests >> - steps: - - run: - name: Run tests - working_directory: build - command: | - mkdir -p /tmp/test-results - export GTEST_OUTPUT=xml:/tmp/test-results/ - make -j4 check V=0 - - store_test_results: - path: /tmp/test-results - - setup-build-and-test-w-make: - description: "Prepare, build and test ReDex" - parameters: - save_boost_cache: - type: boolean - default: false - # Whether to build 32-bit. - mode_32: - type: boolean - default: false - job_name: - type: string - steps: - - when: - condition: << parameters.mode_32 >> - steps: - - setup-build-and-test-w-make-impl: - save_boost_cache: << parameters.save_boost_cache >> - setup_toolchain_extra: "32" - # Would be nice to properly use environment, but that's only at - # the jobs or run level. - configure_extra: "--host=i686-linux-gnu CFLAGS=-m32 CXXFLAGS=-m32 LDFLAGS=-m32" - # Some tests segfault in 32-bit mode. - run_tests: false - job_name: << parameters.job_name >> - # "else" not supported. - - when: - condition: - not: << parameters.mode_32 >> - steps: - - setup-build-and-test-w-make-impl: - save_boost_cache: << parameters.save_boost_cache >> - job_name: << parameters.job_name >> - - build_debian: - description: "(Modern) Debian build and test" - parameters: - save_boost_cache: - type: boolean - default: false - install_clang: - type: boolean - default: false - install_clang_llvm_org: - type: integer - default: -1 - mode_32: - type: boolean - default: false - job_name: - type: string - steps: - - checkout: - path: repo - - debian-based-image-build-setup: - install_clang: << parameters.install_clang>> - install_clang_llvm_org: << parameters.install_clang_llvm_org >> - mode_32: << parameters.mode_32>> - - run: - name: Run git-restore-mtime - working_directory: repo - command: | - git restore-mtime - - setup-build-and-test-w-make: - save_boost_cache: << parameters.save_boost_cache >> - mode_32: << parameters.mode_32 >> - job_name: << parameters.job_name >>-<< parameters.install_clang_llvm_org >> - - setup-build-and-test-windows: - description: "Prepare, build and test ReDex on Windows with MSYS2" - steps: - - run: - name: Install MSYS2 - shell: bash.exe - command: choco install msys2 -y - - run: - name: Update MSYS2 package DB - shell: c:/tools/msys64/msys2_shell.cmd -defterm -no-start -msys2 -full-path -here -c - command: pacman -Syuu --noconfirm - - run: - name: Setup - shell: c:/tools/msys64/msys2_shell.cmd -defterm -no-start -msys2 -full-path -here -c - command: pacman -S --needed --noconfirm make mingw-w64-x86_64-boost mingw-w64-x86_64-cmake mingw-w64-x86_64-gcc mingw-w64-x86_64-jsoncpp mingw-w64-x86_64-make zip unzip mingw-w64-x86_64-python - - - run: - name: Build - shell: c:/tools/msys64/msys2_shell.cmd -defterm -no-start -mingw64 -full-path -here -c - command: mkdir build && cd build && cmake -G "MSYS Makefiles" .. && make -j 4 V=0 - - - run: - name: Minimal Test - # Using bash because somehow piping does not seem to work correctly with cmd. - shell: bash.exe - command: 'build/redex-all --show-passes | grep -E "Registered passes: [1-9][0-9]*"' - - - run: - name: Package - shell: c:/tools/msys64/msys2_shell.cmd -defterm -no-start -mingw64 -full-path -here -c - command: cd build && make -j 4 package V=0 - -jobs: - ########## - # UBUNTU # - ########## - - build-20_04: - docker: - - image: circleci/buildpack-deps:focal - # GCC is hungry. - resource_class: large - steps: - - build_debian: - job_name: ubuntu_20.04 - - build-22_04: - docker: - - image: cimg/base:current-22.04 - # GCC is hungry. - resource_class: large - steps: - - build_debian: - job_name: ubuntu_22.04 - - ########## - # DEBIAN # - ########## - - # Should be Debian 10 at time of writing. - build-deb_oldoldstable: - docker: - - image: buildpack-deps:oldoldstable-scm - # GCC is hungry. - resource_class: large - steps: - - build_debian: - job_name: debian_10 - - build-deb_stable: - docker: - - image: buildpack-deps:stable-scm - # GCC is hungry. - resource_class: large - steps: - - build_debian: - job_name: debian_12 - - build-deb_stable-w-clang: - docker: - - image: buildpack-deps:stable-scm - resource_class: large - environment: - CC: clang - CXX: clang++ - steps: - - build_debian: - install_clang: true - job_name: debian_12_clang - - build-deb_stable-32: - docker: - - image: buildpack-deps:stable-scm - # GCC is hungry. - resource_class: large - steps: - - build_debian: - mode_32: true - job_name: debian_12_32 - - build-deb_stable-w-clang-llvm-org: - docker: - - image: buildpack-deps:stable-scm - resource_class: large - environment: - CC: clang - CXX: clang++ - parameters: - clang_version: - type: integer - default: 0 - steps: - - build_debian: - install_clang_llvm_org: << parameters.clang_version>> - job_name: debian_12_clang_upstream - - # Note: buildpack-deps:testing is the same as stable at time of writing, - # so jump to unstable directly. - - build-deb_unstable: - docker: - - image: buildpack-deps:unstable-scm - # GCC is hungry. - resource_class: large - steps: - - build_debian: - job_name: debian_unstable - - build-deb_unstable-w-clang: - docker: - - image: buildpack-deps:unstable-scm - resource_class: large - environment: - CC: clang - CXX: clang++ - steps: - - build_debian: - install_clang: true - job_name: debian_unstable_clang - - build-deb_unstable-32: - docker: - - image: buildpack-deps:unstable-scm - # GCC is hungry. - resource_class: large - steps: - - build_debian: - mode_32: true - job_name: debian_unstable_32 - - ########### - # WINDOWS # - ########### - - build-windows: - executor: - name: win/default - # GCC is hungry. - size: "large" - - steps: - - checkout - - setup-build-and-test-windows - - build-windows-artifacts: - executor: - name: win/default - # GCC is hungry. - size: "large" - - steps: - - checkout - - setup-build-and-test-windows - - # Artifacts syntax does not allow wildcards. Copy result (without knowing details) - # into a temp dir instead. - - run: - name: Copy package for CircleCI - shell: c:/tools/msys64/msys2_shell.cmd -defterm -no-start -mingw64 -full-path -here -c - command: mkdir artifacts && mv build/Redex*.zip artifacts/ - - - store_artifacts: - path: .\artifacts - -# TODO: branch filtering is deprecated, but workflow filtering documentation -# for 2.1 does not exist and 2.0 requires a schedule. - -workflows: - version: 2.1 - build_main: - jobs: - # Current Ubuntu LTS. - - build-20_04: - filters: *filter-only-main - # Oldest officially supported Debian build. - - build-deb_oldoldstable: - filters: *filter-only-main - # For better format strings checking. - - build-deb_stable-32: - filters: *filter-only-main - # For newer compilers. - - build-deb_unstable: - filters: *filter-only-main - # At least one Clang check. - - build-deb_stable-w-clang-llvm-org: - filters: *filter-only-main - # For compatibility. - - build-windows: - filters: *filter-only-main - - build_other: - jobs: - # Note: only filter gh-pages for 'build_other' so that pull - # requests/stable get checked. - - build-20_04: - filters: *filter-not-gh-pages-not-main - - build-deb_stable: - filters: *filter-not-gh-pages-not-main - - build-deb_stable-32: - filters: *filter-not-gh-pages-not-main - # Only use the default (supposed stable) version on diffs. - - build-deb_stable-w-clang-llvm-org: - filters: *filter-not-gh-pages-not-main - - build-deb_oldoldstable: - filters: *filter-not-gh-pages-not-main - - build-deb_unstable: - filters: *filter-not-gh-pages-not-main - - build-windows: - filters: *filter-not-gh-pages-not-main - - nightly: - triggers: - - schedule: - cron: "0 0 * * *" - filters: *filter-only-main - jobs: - - build-20_04 - - build-22_04 - - build-deb_oldoldstable - - build-deb_stable - - build-deb_stable-w-clang - - build-deb_stable-32 - - build-deb_unstable - - build-windows-artifacts - # All llvm.org versions. Keep this up-to-date. Ignore the duplicate. - - build-deb_stable-w-clang-llvm-org: - matrix: - parameters: - clang_version: [0, 15, 17] diff --git a/.github/actions/setup-build-and-test-w-make-impl/action.yml b/.github/actions/setup-build-and-test-w-make-impl/action.yml index 51c58467ba4..82b4f556715 100644 --- a/.github/actions/setup-build-and-test-w-make-impl/action.yml +++ b/.github/actions/setup-build-and-test-w-make-impl/action.yml @@ -42,6 +42,7 @@ runs: if [ -f "build.tar.zstd" ] ; then echo "Decompressing build cache" zstd --decompress --stdout build.tar.zstd | tar xf - + ls -l build.tar.zstd rm build.tar.zstd find build | wc -l fi @@ -59,8 +60,19 @@ runs: ../repo/configure --enable-protobuf --with-android-sdk="$(pwd)/../sdk" ${{ inputs.configure_extra }} working-directory: build shell: bash + - name: Print dates + run: | + ls -l ../repo/proto/ + ls -l protores/ + echo '====' + ls -l libredex.la + ls -l ../repo/libredex/*.cpp + ls -l libredex/ + exit 1 + working-directory: build + shell: bash - name: Build - run: make -j4 V=0 + run: make -j4 -d working-directory: build shell: bash - name: Compress build directory