integration-test: Set rust-lld as a linker only on macOS

.github/workflows/ci.yml

@@ -202,19 +202,69 @@ jobs:
           - target: x86_64-unknown-linux-gnu
             # We don't use ubuntu-latest because we care about the apt packages available.
             os: ubuntu-22.04
+          - target: x86_64-unknown-linux-musl
+            os: ubuntu-22.04
+            container:
+              image: docker.io/alpine:3.20
+              options: --privileged -v /sys/fs/bpf:/sys/fs/bpf -v /sys/kernel:/sys/kernel
     runs-on: ${{ matrix.os }}
+    container: ${{ matrix.container }}
     steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
+      - name: Install prerequisites
+        if: runner.os == 'Linux' && contains(matrix.container.image, 'alpine')
+        # bash is used in the `rust-toolchain` action.
+        #
+        # clang and make are needed for building the C eBPF programs, which are
+        # part of the integration tests.
+        #
+        # curl, file and jq are used in the steps below.
+        #
+        # dpkg is used to unpack the kernel images from .deb packages, which we
+        # use for virtualized tests. lynx is used to download them.
+        #
+        # gcc is needed as a linker, it also provides the runtime library, both
+        # needed by Rust.
+        #
+        # git is needed for the `checkout` action.
+        #
+        # libstdc++ is a dependency of llvm-sys, which is a dependency of
+        # bpf-linker.
+        #
+        # musl-dev provides CRT objects, which are linked by Rust.
+        #
+        # QEMU is used to run virtualized integration tests.
+        #
+        # sudo is needed by integration tests.
+        run: |
+          set -euxo pipefail
+          apk add \
+            bash \
+            clang \
+            curl \
+            dpkg \
+            file \
+            gcc \
+            git \
+            jq \
+            libstdc++-dev \
+            lynx \
+            make \
+            musl-dev \
+            qemu-system-arm \
+            qemu-system-x86_64 \
+            sudo
 
       - name: Install prerequisites
-        if: runner.os == 'Linux'
+        if: runner.os == 'Linux' && matrix.container == ''
         # ubuntu-22.04 comes with clang 13-15[0]; support for signed and 64bit
         # enum values was added in clang 15[1] which isn't in `$PATH`.
         #
         # gcc-multilib provides at least <asm/types.h> which is referenced by libbpf.
         #
+        # lynx is used to download kernel images.
+        #
+        # QEMU is used to run virtualized integration tests.
+        #
         # [0] https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md
         #
         # [1] https://github.com/llvm/llvm-project/commit/dc1c43d
@@ -224,6 +274,26 @@ jobs:
           sudo apt -y install gcc-multilib lynx qemu-system-{arm,x86}
           echo /usr/lib/llvm-15/bin >> $GITHUB_PATH
 
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      # When running in a container as root, git throws the following error:
+      #
+      # fatal: detected dubious ownership in repository at '/__w/aya/aya'
+      #
+      # Which makes a lot of sense, not running regular git commands as root is
+      # a good thing. However, using a container image with a regular user
+      # results in permission errors thrown by the runner binary.[0] It's most
+      # likely because of a host volume mount which is owned by root. It would
+      # be great to make it work, but it's not going to be trivial and might
+      # require changes in the runner code itself.
+      #
+      # [0] https://github.com/aya-rs/aya/actions/runs/12034434029/job/33550963904
+      - name: Mark the directory as safe for git
+        if: matrix.container != ''
+        run: git config --global --add safe.directory /__w/aya/aya
+
       - name: Install prerequisites
         if: runner.os == 'macOS'
         # The xargs shipped on macOS always exits 0 with -P0, so we need GNU findutils.

test/integration-test/src/tests/iter.rs

@@ -1,4 +1,4 @@
-use std::io::BufRead;
+use std::{env, io::BufRead, path::Path};
 
 use aya::{programs::Iter, Btf, Ebpf};
 use test_log::test;
@@ -20,11 +20,15 @@ fn iter_task() {
     let line_init = lines.next().unwrap().unwrap();
 
     assert_eq!(line_title, "tgid     pid      name");
-    let expected_values = ["1        1        init", "1        1        systemd"];
-    assert!(
-        expected_values.contains(&line_init.as_str()),
-        "Unexpected line_init value: '{}', expected one of: {:?}",
-        line_init,
-        expected_values
-    );
+    // It's hard to predict what's the PID of the first process in a container.
+    // Use this assertion only on non-containerized systems.
+    if !Path::new("/.dockerenv").exists() && env::var_os("container").is_none() {
+        let expected_values = ["1        1        init", "1        1        systemd"];
+        assert!(
+            expected_values.contains(&line_init.as_str()),
+            "Unexpected line_init value: '{}', expected one of: {:?}",
+            line_init,
+            expected_values
+        );
+    }
 }

xtask/src/run.rs

@@ -54,12 +54,60 @@ pub fn build<F>(target: Option<&str>, f: F) -> Result<Vec<(String, PathBuf)>>
 where
     F: FnOnce(&mut Command) -> &mut Command,
 {
-    // Always use rust-lld and -Zbuild-std in case we're cross-compiling.
     let mut cmd = Command::new("cargo");
     cmd.args(["build", "--message-format=json"]);
     if let Some(target) = target {
-        let config = format!("target.{target}.linker = \"rust-lld\"");
-        cmd.args(["--target", target, "--config", &config]);
+        cmd.args(["--target", target]);
+        // During a cross build, it's important to pick the correct linker.
+        //
+        // On Linux, C compiler driver should be always used as the linker.
+        // The C compiler eventually ends up calling the linker binary (e.g.
+        // ld, lld), but before doing that, it figures out the appropiate
+        // linker flags, which include the system library paths. Calling linker
+        // binaries directly results in them not being able to find system
+        // libraries (like libc or runtime library), which can manifest in
+        // errors like `unable to find library -lgcc_s`.
+        //
+        // The issue was discussed with the Rust maintainers[0] and the
+        // consensus is to always use `-C linker` to specify the C compiler
+        /// (e.g. `-C linker=gcc`, `-C linker=clang`). Choice of a specific
+        // linker (like ldd or mold) can be done with `-C link-arg=-fuse-ld=`.
+        //
+        // However, the same doesn't hold true for macOS. Cross toolchains for
+        // Linux targets on macOS hosts, provided by rustup, are self-contained,
+        // come with libc, runtime library and don't depend on any system
+        // libraries. Therefore, direct usage of rust-lld through
+        // `-C linker=rust-lld` works fine, because rust-lld is able to find
+        // libc and runtime in rustup's toolchain.
+        //
+        // Using system-wide compiler (clang) on macOS would take the opposite
+        // effect than on Linux. The system compiler would be the one not being
+        // able to find the Linux-compatible libc and runtime
+        //
+        // To sum it up, this is the way of determining the linker we follow:
+        //
+        // - On Linux, use a C compiler for the cross target.
+        // - On macOS, use rust-lld directly.
+        //
+        // The first point is already covered by the configuration in
+        // `.cargo/config.toml`, which uses cross GCC compilers as linkers for
+        // popular non-x86_64 targets (e.g. aarch64-linux-musl-gcc). People
+        // who want to use a different compiler (e.g. clang), can overwrite
+        // RUSTFLAGS. Set of flags like `-C linker=clang
+        // -C link-arg=--target=aarch64-unknown-linux-musl
+        // -C link-arg=-fuse-ld=lld` should result in the build which uses only
+        // LLVM and has no dependency on GCC. mold can be used with
+        // `-C link-arg=-fuse-ld=mold`
+        //
+        // To cover the macOS case, we explicitly set the linker to rust-lld,
+        // ignoring the default configuration from `.cargo/config.toml`.
+        //
+        // [0] https://github.com/rust-lang/rust/issues/130062
+        #[cfg(target_os = "macos")]
+        {
+            let config = format!("target.{target}.linker = \"rust-lld\"");
+            cmd.args(["--config", &config]);
+        }
     }
     f(&mut cmd);