From fa7ca182a9b0c65f567116b67108da14182e5338 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Wed, 11 Sep 2024 13:08:14 +0200 Subject: [PATCH 1/5] ci: Print inner env --- ci/test/03_test_script.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/test/03_test_script.sh b/ci/test/03_test_script.sh index 7ee424ac9bb0a..1c1b5fa545b0c 100755 --- a/ci/test/03_test_script.sh +++ b/ci/test/03_test_script.sh @@ -30,6 +30,10 @@ df -h # Tests that run natively guess the host export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")} +echo "=== BEGIN env ===" +env +echo "=== END env ===" + ( # compact->outputs[i].file_size is uninitialized memory, so reading it is UB. # The statistic bytes_written is only used for logging, which is disabled in From aaaa7cf8bad0e125a67e996162271eb26bef4fb6 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Wed, 11 Sep 2024 14:56:04 +0200 Subject: [PATCH 2/5] cirrus: Drop CCACHE_NOHASHDIR Now that the build path is constant again after commit fa193f5dfc9 normalized all folders, this can be dropped. --- .cirrus.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index f5874744b5295..94ddeaf1024a0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -6,7 +6,6 @@ env: # Global defaults CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling processes and setting this variable avoids killing the CI script itself on error CCACHE_MAXSIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" - CCACHE_NOHASHDIR: "1" # Debug info might contain a stale path if the build dir changes, but this is fine # A self-hosted machine(s) can be used via Cirrus CI. It can be configured with # multiple users to run tasks in parallel. No sudo permission is required. From fa146904e19e345714fd40fd2f0fbad8118957b9 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Wed, 11 Sep 2024 14:21:57 +0200 Subject: [PATCH 3/5] ci: Bump default CCACHE_MAXSIZE to 500M This also allows to drop individually hardcoded values, which are impossible to overwrite and hard to maintain. --- .cirrus.yml | 1 - ci/test/00_setup_env.sh | 2 +- ci/test/00_setup_env_mac_native.sh | 1 - ci/test/00_setup_env_native_asan.sh | 1 - ci/test/00_setup_env_native_fuzz.sh | 1 - ci/test/00_setup_env_native_fuzz_with_msan.sh | 1 - ci/test/00_setup_env_native_fuzz_with_valgrind.sh | 1 - ci/test/00_setup_env_native_msan.sh | 1 - ci/test/00_setup_env_native_tidy.sh | 1 - 9 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 94ddeaf1024a0..085a2939284b0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,7 +4,6 @@ env: # Global defaults MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling processes and setting this variable avoids killing the CI script itself on error - CCACHE_MAXSIZE: "200M" CCACHE_DIR: "/tmp/ccache_dir" # A self-hosted machine(s) can be used via Cirrus CI. It can be configured with diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 944655d8ad3a1..fb0099e4dccfe 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -53,7 +53,7 @@ export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} # See man 7 debconf export DEBIAN_FRONTEND=noninteractive -export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-100M} +export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:-500M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # The cache dir. diff --git a/ci/test/00_setup_env_mac_native.sh b/ci/test/00_setup_env_mac_native.sh index 76668d97f206e..087846a8a5eab 100755 --- a/ci/test/00_setup_env_mac_native.sh +++ b/ci/test/00_setup_env_mac_native.sh @@ -15,5 +15,4 @@ export BITCOIN_CONFIG="-DBUILD_GUI=ON -DWITH_ZMQ=ON -DWITH_MINIUPNPC=ON -DWITH_N export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" -export CCACHE_MAXSIZE=400M export RUN_FUZZ_TESTS=true diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 0ec30f23af116..e27563f3237bf 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -32,4 +32,3 @@ export BITCOIN_CONFIG="\ -DAPPEND_CXXFLAGS='-std=c++23' \ -DAPPEND_CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' \ " -export CCACHE_MAXSIZE=300M diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index e1a353056df8f..1aa24870454ca 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -23,5 +23,4 @@ export BITCOIN_CONFIG="\ -DCMAKE_C_FLAGS='-ftrivial-auto-var-init=pattern' \ -DCMAKE_CXX_FLAGS='-ftrivial-auto-var-init=pattern' \ " -export CCACHE_MAXSIZE=200M export LLVM_SYMBOLIZER_PATH="/usr/bin/llvm-symbolizer-18" diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 7cea4d73afc6d..cfdbc8c0142a7 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -31,4 +31,3 @@ export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" export RUN_FUZZ_TESTS=true -export CCACHE_MAXSIZE=250M diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 02903b5199590..867615c0151b3 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -21,4 +21,3 @@ export BITCOIN_CONFIG="\ -DCMAKE_C_COMPILER=clang-16 \ -DCMAKE_CXX_COMPILER=clang++-16 \ " -export CCACHE_MAXSIZE=200M diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 2c85ba31d160f..c6b3d68be60b6 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -28,4 +28,3 @@ export BITCOIN_CONFIG="\ " export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" -export CCACHE_MAXSIZE=250M diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 5105455df84f4..4a6f5bb3e22ca 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -25,4 +25,3 @@ export BITCOIN_CONFIG="\ -DCMAKE_C_FLAGS_RELWITHDEBINFO='-O0 -g0' \ -DCMAKE_CXX_FLAGS_RELWITHDEBINFO='-O0 -g0' \ " -export CCACHE_MAXSIZE=200M From fa252da0b9cc6c7e795366ce4a1ddc4c198dff15 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Wed, 11 Sep 2024 14:42:07 +0200 Subject: [PATCH 4/5] ci: Remove hardcoded CCACHE_DIR in cirrus This makes it easier to overwrite the value. Also, drop the dot in the CCACHE_DIR fallback value, because the folder in the scratch dir does not need and probably should not be hidden. --- .cirrus.yml | 1 - ci/test/00_setup_env.sh | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 085a2939284b0..bea1144aa2aff 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,7 +4,6 @@ env: # Global defaults MAKEJOBS: "-j10" TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache CI_FAILFAST_TEST_LEAVE_DANGLING: "1" # Cirrus CI does not care about dangling processes and setting this variable avoids killing the CI script itself on error - CCACHE_DIR: "/tmp/ccache_dir" # A self-hosted machine(s) can be used via Cirrus CI. It can be configured with # multiple users to run tasks in parallel. No sudo permission is required. diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index fb0099e4dccfe..d488a86024899 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -58,7 +58,7 @@ export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} # The cache dir. # This folder exists only on the ci guest, and on the ci host as a volume. -export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} +export CCACHE_DIR="${CCACHE_DIR:-$BASE_SCRATCH_DIR/ccache}" # Folder where the build result is put (bin and lib). export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out} # The folder for previous release binaries. From fa99e4521b6fc0e7f6636d40bc0d6a7325227374 Mon Sep 17 00:00:00 2001 From: MarcoFalke <*~=`'#}+{/-|&$^_@721217.xyz> Date: Wed, 11 Sep 2024 15:31:23 +0200 Subject: [PATCH 5/5] ci: Allow CCACHE_DIR bind mount This may be useful. For example, to store the directory in a specific place, instead of having to use a volume. Possibly, but not limited to sharing a cache: https://ccache.dev/manual/4.10.1.html#_sharing_a_local_cache --- ci/test/02_run_container.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ci/test/02_run_container.sh b/ci/test/02_run_container.sh index afd447c347f73..1727f9296b196 100755 --- a/ci/test/02_run_container.sh +++ b/ci/test/02_run_container.sh @@ -48,6 +48,14 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then CI_PREVIOUS_RELEASES_MOUNT="type=bind,src=${PREVIOUS_RELEASES_DIR},dst=$PREVIOUS_RELEASES_DIR" fi + if [ "$DANGER_CI_ON_HOST_CCACHE_FOLDER" ]; then + if [ ! -d "${CCACHE_DIR}" ]; then + echo "Error: Directory '${CCACHE_DIR}' must be created in advance." + exit 1 + fi + CI_CCACHE_MOUNT="type=bind,src=${CCACHE_DIR},dst=${CCACHE_DIR}" + fi + docker network create --ipv6 --subnet 1111:1111::/112 ci-ip6net || true if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then