From 54cac689eb684a7b6ee94b14c135b358dad35309 Mon Sep 17 00:00:00 2001 From: Manu Bretelle Date: Mon, 12 Feb 2024 16:18:29 -0800 Subject: [PATCH 1/4] build: Build s390x on x86_64 hosts When running on self-hosted, build s390x from x86_64 hosts. Signed-off-by: Manu Bretelle --- .github/scripts/matrix.py | 11 +++++++++++ .github/workflows/kernel-build-test.yml | 6 +++++- .github/workflows/test.yml | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.github/scripts/matrix.py b/.github/scripts/matrix.py index 8b33f5a..100cf3d 100644 --- a/.github/scripts/matrix.py +++ b/.github/scripts/matrix.py @@ -74,6 +74,16 @@ def runs_on(self) -> List[str]: return ["self-hosted", self.arch.value] return [DEFAULT_RUNNER] + @property + def build_runs_on(self) -> List[str]: + if is_managed_repo(): + # Build s390x on x86_64 + return [ + "self-hosted", + self.arch.value == "s390x" and Arch.X86_64.value or self.arch.value, + ] + return [DEFAULT_RUNNER] + @property def tests(self) -> Dict[str, Any]: tests_list = [ @@ -103,6 +113,7 @@ def to_dict(self) -> Dict[str, Any]: "build_release": self.build_release, "runs_on": self.runs_on, "tests": self.tests, + "build_runs_on": self.build_runs_on, } diff --git a/.github/workflows/kernel-build-test.yml b/.github/workflows/kernel-build-test.yml index ce1baa4..6296b36 100644 --- a/.github/workflows/kernel-build-test.yml +++ b/.github/workflows/kernel-build-test.yml @@ -19,6 +19,10 @@ on: required: true type: string description: The runners to run the test on. This is a json string representing an array of labels. + build_runs_on: + required: true + type: string + description: The runners to run the builds on. This is a json string representing an array of labels. llvm-version: required: true type: string @@ -61,7 +65,7 @@ jobs: arch: ${{ inputs.arch }} toolchain_full: ${{ inputs.toolchain_full }} toolchain: ${{ inputs.toolchain }} - runs_on: ${{ inputs.runs_on }} + runs_on: ${{ inputs.build_runs_on }} llvm-version: ${{ inputs.llvm-version }} kernel: ${{ inputs.kernel }} download_sources: ${{ inputs.download_sources }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ff9d36c..4b9c0a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,7 @@ jobs: toolchain_full: ${{ matrix.toolchain.fullname }} toolchain: ${{ matrix.toolchain.name }} runs_on: ${{ toJSON(matrix.runs_on) }} + build_runs_on: ${{ toJSON(matrix.build_runs_on) }} llvm-version: ${{ matrix.toolchain.version }} kernel: ${{ matrix.kernel }} tests: ${{ toJSON(matrix.tests) }} From d9360124fc3864a1b28018e87207b1233c6a3ece Mon Sep 17 00:00:00 2001 From: Manu Bretelle Date: Mon, 12 Feb 2024 18:24:09 -0800 Subject: [PATCH 2/4] tar-artifacts: Support for cross-compilation When running `make -s image_name` we need to pass `ARCH` variable. There is no good way (that I know of) to infer linux's expected value from `uname -m` value. Here, I copied libbpf/ci's helpers to get the translation done. Signed-off-by: Manu Bretelle --- .github/scripts/tar-artifact.sh | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/scripts/tar-artifact.sh b/.github/scripts/tar-artifact.sh index cd88f9e..8886cd6 100644 --- a/.github/scripts/tar-artifact.sh +++ b/.github/scripts/tar-artifact.sh @@ -5,6 +5,28 @@ set -eux arch="${1}" toolchain="${2}" +# Convert a platform (as returned by uname -m) to the kernel +# arch (as expected by ARCH= env). +platform_to_kernel_arch() { + case $1 in + s390x) + echo "s390" + ;; + aarch64) + echo "arm64" + ;; + riscv64) + echo "riscv" + ;; + x86_64) + echo "x86" + ;; + *) + echo "$1" + ;; + esac +} + # Remove intermediate object files that we have no use for. Ideally # we'd just exclude them from tar below, but it does not provide # options to express the precise constraints. @@ -14,7 +36,7 @@ find selftests/ -name "*.o" -a ! -name "*.bpf.o" -print0 | \ # Strip debug information, which is excessively large (consuming # bandwidth) while not actually being used (the kernel does not use # DWARF to symbolize stacktraces). -strip --strip-debug "${KBUILD_OUTPUT}"/vmlinux +"${arch}"-linux-gnu-strip --strip-debug "${KBUILD_OUTPUT}"/vmlinux additional_file_list=() if [ "${GITHUB_REPOSITORY}" == "kernel-patches/vmtest" ]; then @@ -29,7 +51,7 @@ if [ "${GITHUB_REPOSITORY}" == "kernel-patches/vmtest" ]; then ) fi -image_name=$(make -s image_name) +image_name=$(make ARCH="$(platform_to_kernel_arch "${arch}")" -s image_name) # zstd is installed by default in the runner images. tar -cf - \ From d53f68698c26324ad1923f8ae8dc4192639532fd Mon Sep 17 00:00:00 2001 From: Manu Bretelle Date: Mon, 12 Feb 2024 18:33:00 -0800 Subject: [PATCH 3/4] patch: Force to use target bpftool when cross-compiling Signed-off-by: Manu Bretelle --- ci/diffs/0099-selftest-cross-compile.diff | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 ci/diffs/0099-selftest-cross-compile.diff diff --git a/ci/diffs/0099-selftest-cross-compile.diff b/ci/diffs/0099-selftest-cross-compile.diff new file mode 100644 index 0000000..7816509 --- /dev/null +++ b/ci/diffs/0099-selftest-cross-compile.diff @@ -0,0 +1,14 @@ +diff --git a/tools/testing/selftests/bpf/Makefile b/tools/testing/selftests/bpf/Makefile +index a38a3001527c..a403a9d5c529 100644 +--- a/tools/testing/selftests/bpf/Makefile ++++ b/tools/testing/selftests/bpf/Makefile +@@ -171,7 +171,8 @@ INCLUDE_DIR := $(SCRATCH_DIR)/include + BPFOBJ := $(BUILD_DIR)/libbpf/libbpf.a + ifneq ($(CROSS_COMPILE),) + HOST_BUILD_DIR := $(BUILD_DIR)/host +-HOST_SCRATCH_DIR := $(OUTPUT)/host-tools ++#HOST_SCRATCH_DIR := $(OUTPUT)/host-tools ++HOST_SCRATCH_DIR := $(SCRATCH_DIR) + HOST_INCLUDE_DIR := $(HOST_SCRATCH_DIR)/include + else + HOST_BUILD_DIR := $(BUILD_DIR) From 296a41600ce589f2b61a35d6008cf8672f9e4d96 Mon Sep 17 00:00:00 2001 From: Manu Bretelle Date: Tue, 13 Feb 2024 10:49:34 -0800 Subject: [PATCH 4/4] actions: upgrade to v4 v3 actions are using nodejs16 which is being deprecated. Bump to v4 to kill the warning and also benefit from https://github.blog/2024-02-12-get-started-with-v4-of-github-actions-artifacts/ Signed-off-by: Manu Bretelle --- .github/actions/veristat_baseline_compare/action.yml | 6 +++--- .github/workflows/kernel-build.yml | 6 +++--- .github/workflows/kernel-test.yml | 4 ++-- .github/workflows/kernel-veristat.yml | 4 ++-- .github/workflows/lint.yml | 8 ++++---- .github/workflows/test.yml | 2 +- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/actions/veristat_baseline_compare/action.yml b/.github/actions/veristat_baseline_compare/action.yml index 7ea487a..9e25a2b 100644 --- a/.github/actions/veristat_baseline_compare/action.yml +++ b/.github/actions/veristat_baseline_compare/action.yml @@ -10,7 +10,7 @@ inputs: runs: using: "composite" steps: - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: ${{ inputs.baseline_name }} if-no-files-found: error @@ -20,7 +20,7 @@ runs: # - get baseline log from cache # - compare it to current run - if: ${{ github.event_name == 'pull_request' }} - uses: actions/cache/restore@v3 + uses: actions/cache/restore@v4 with: key: ${{ inputs.baseline_name }} restore-keys: | @@ -43,7 +43,7 @@ runs: "${{ github.workspace }}/${{ inputs.baseline_name }}" - if: ${{ github.event_name == 'push' }} - uses: actions/cache/save@v3 + uses: actions/cache/save@v4 with: key: ${{ inputs.baseline_name }}-${{ github.run_id }} path: '${{ github.workspace }}/${{ inputs.baseline_name }}' diff --git a/.github/workflows/kernel-build.yml b/.github/workflows/kernel-build.yml index 641a131..3a7b043 100644 --- a/.github/workflows/kernel-build.yml +++ b/.github/workflows/kernel-build.yml @@ -50,7 +50,7 @@ jobs: REPO_PATH: "" KBUILD_OUTPUT: kbuild-output/ steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # We fetch an actual bit of history here to facilitate incremental # builds (which may check out some earlier upstream change). with: @@ -72,7 +72,7 @@ jobs: run: | bash .github/scripts/get-commit-metadata.sh - name: Pull recent KBUILD_OUTPUT contents - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.KBUILD_OUTPUT }} key: kbuild-output-${{ inputs.arch }}-${{ inputs.toolchain_full }}-${{ steps.get-commit-metadata.outputs.branch }}-${{ steps.get-commit-metadata.outputs.timestamp }}-${{ steps.get-commit-metadata.outputs.commit }} @@ -134,7 +134,7 @@ jobs: # Only on pushed changes are build artifacts actually cached, because # of github.com/actions/cache's cache isolation logic. rm -rf "${KBUILD_OUTPUT}" - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }}${{ inputs.release && '-release' || '' }} if-no-files-found: error diff --git a/.github/workflows/kernel-test.yml b/.github/workflows/kernel-test.yml index 78fd5fa..38d2d66 100644 --- a/.github/workflows/kernel-test.yml +++ b/.github/workflows/kernel-test.yml @@ -46,8 +46,8 @@ jobs: # booleans are weird in GH. CONTINUE_ON_ERROR: ${{ inputs.continue_on_error }} steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: name: vmlinux-${{ inputs.arch }}-${{ inputs.toolchain_full }} path: . diff --git a/.github/workflows/kernel-veristat.yml b/.github/workflows/kernel-veristat.yml index 70ba0af..03a3c4d 100644 --- a/.github/workflows/kernel-veristat.yml +++ b/.github/workflows/kernel-veristat.yml @@ -39,8 +39,8 @@ jobs: KBUILD_OUTPUT: kbuild-output/ ARCH_AND_TOOL: ${{ inputs.arch }}-${{ inputs.toolchain }} steps: - - uses: actions/checkout@v3 - - uses: actions/download-artifact@v3 + - uses: actions/checkout@v4 + - uses: actions/download-artifact@v4 with: name: vmlinux-${{ env.ARCH_AND_TOOL }} path: . diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 69f06e9..1c910fd 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run ShellCheck uses: ludeeus/action-shellcheck@master env: @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run black uses: psf/black@stable with: @@ -48,7 +48,7 @@ jobs: repository: ['bpf', 'vmtest', 'bar'] steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: run script run: | python3 .github/scripts/matrix.py @@ -59,7 +59,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Run unittests run: python3 -m unittest scripts/tests/*.py working-directory: .github diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4b9c0a2..3c81e02 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: outputs: build-matrix: ${{ steps.set-matrix-impl.outputs.build_matrix }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - id: set-matrix-impl run: | python3 .github/scripts/matrix.py