From 1e22aaea3cb28bfff10755a5860ac42a66bacba8 Mon Sep 17 00:00:00 2001 From: Rick Newton-Rogers Date: Thu, 28 Nov 2024 09:05:45 +0000 Subject: [PATCH] WIP --- .github/workflows/pull_request.yml | 2 +- .github/workflows/swift_test_matrix.yml | 119 ++++++++++++++++++ .github/workflows/unit_tests.yml | 117 ++++++++++++++++++ scripts/generate_matrix.sh | 153 ++++++++++++++++++++++++ 4 files changed, 390 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/swift_test_matrix.yml create mode 100644 .github/workflows/unit_tests.yml create mode 100755 scripts/generate_matrix.sh diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 53a3e0ac..502bb30c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -22,7 +22,7 @@ jobs: unit-tests: name: Unit tests - uses: apple/swift-nio/.github/workflows/unit_tests.yml@main + uses: ./.github/workflows/unit_tests.yml with: linux_5_9_arguments_override: "--explicit-target-dependency-import-check error" linux_5_10_arguments_override: "--explicit-target-dependency-import-check error" diff --git a/.github/workflows/swift_test_matrix.yml b/.github/workflows/swift_test_matrix.yml new file mode 100644 index 00000000..cfdcd894 --- /dev/null +++ b/.github/workflows/swift_test_matrix.yml @@ -0,0 +1,119 @@ +name: Matrix + +on: + workflow_call: + inputs: + name: + type: string + description: "The name of the workflow used for the concurrency group." + required: true + matrix_path: + type: string + description: "The path of the test matrix definition." + default: "" + matrix_string: + type: string + description: "The test matrix definition." + default: "" + +# We are cancelling previously triggered workflow runs +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.name }} + cancel-in-progress: true + +jobs: + generate-matrix: + name: Prepare matrices + runs-on: ubuntu-latest + outputs: + linux-matrix: ${{ steps.load-matrix.outputs.linux-matrix }} + windows-matrix: ${{ steps.load-matrix.outputs.windows-matrix }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Mark the workspace as safe + # https://github.com/actions/checkout/issues/766 + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + - id: file_check + run: | + if [ ! -z '${{ inputs.matrix_string }}' ]; then + echo "matrix_string_populated=true" >> $GITHUB_ENV + else + echo "matrix_string_populated=false" >> $GITHUB_ENV + fi + - id: cat-matrix + if: ${{ env.matrix_string_populated == 'false' }} + run: | + jq -c '{linux: .linux}' ${{ inputs.matrix_path }} >| __linux_coverage_matrix.json + jq -c '{windows: .windows}' ${{ inputs.matrix_path }} >| __windows_coverage_matrix.json + - id: write-matrix + if: ${{ env.matrix_string_populated == 'true' }} + run: | + echo '${{ inputs.matrix_string }}' | jq -c '{linux: .linux}' >| __linux_coverage_matrix.json + echo '${{ inputs.matrix_string }}' | jq -c '{windows: .windows}' >| __windows_coverage_matrix.json + cat __linux_coverage_matrix.json + cat __windows_coverage_matrix.json + - id: load-matrix + run: | + { + echo 'linux-matrix<> "$GITHUB_OUTPUT" + + { + echo 'windows-matrix<> "$GITHUB_OUTPUT" + + linux: + name: Linux (${{ matrix.linux.name }}) + needs: generate-matrix + runs-on: ${{ matrix.linux.runner }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs.linux-matrix) }} + container: + image: ${{ matrix.linux.image }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Mark the workspace as safe + # https://github.com/actions/checkout/issues/766 + run: git config --global --add safe.directory ${GITHUB_WORKSPACE} + - name: Pre-build + if: ${{ matrix.linux.setup_command }} + run: | + ${{ matrix.linux.setup_command }} + - name: Run matrix job + run: | + ${{ matrix.linux.command }} ${{ matrix.linux.command_arguments }} + + windows: + name: Windows (${{ matrix.windows.name }}) + needs: generate-matrix + runs-on: ${{ matrix.windows.runner }} + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.generate-matrix.outputs.windows-matrix) }} + steps: + - name: Pull Docker image + run: docker pull ${{ matrix.windows.image }} + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + submodules: true + - name: Run matrix job + run: | + if [[ ! -z "${{ matrix.windows.pre_run_command }}" ]]; then + pre_run_command_expression="powershell Invoke-Expression ${{ matrix.windows.pre_run_command }} &" + else + pre_run_command_expression="" + fi + docker run -v ${{ github.workspace }}:C:\source ${{ matrix.windows.image }} cmd /s /c "swift --version & cd C:\source\ & $pre_run_command_expression powershell Invoke-Expression ${{ matrix.windows.command }} ${{ matrix.windows.command_arguments }}" diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml new file mode 100644 index 00000000..d3524313 --- /dev/null +++ b/.github/workflows/unit_tests.yml @@ -0,0 +1,117 @@ +name: Unit tests + +on: + workflow_call: + inputs: + linux_5_9_enabled: + type: boolean + description: "Boolean to enable the Linux 5.9 Swift version matrix job. Defaults to true." + default: true + linux_5_9_arguments_override: + type: string + description: "The arguments passed to swift test in the Linux 5.9 Swift version matrix job." + default: "" + linux_5_10_enabled: + type: boolean + description: "Boolean to enable the Linux 5.10 Swift version matrix job. Defaults to true." + default: true + linux_5_10_arguments_override: + type: string + description: "The arguments passed to swift test in the Linux 5.10 Swift version matrix job." + default: "" + linux_6_0_enabled: + type: boolean + description: "Boolean to enable the Linux 6.0 Swift version matrix job. Defaults to true." + default: true + linux_6_0_arguments_override: + type: string + description: "The arguments passed to swift test in the Linux 6.0 Swift version matrix job." + default: "" + linux_nightly_6_0_enabled: + type: boolean + description: "Boolean to enable the Linux nightly 6.0 Swift version matrix job. Defaults to true." + default: true + linux_nightly_6_0_arguments_override: + type: string + description: "The arguments passed to swift test in the Linux nightly 6.0 Swift version matrix job." + default: "" + linux_nightly_main_enabled: + type: boolean + description: "Boolean to enable the Linux nightly main Swift version matrix job. Defaults to true." + default: true + linux_nightly_main_arguments_override: + type: string + description: "The arguments passed to swift test in the Linux nightly main Swift version matrix job." + default: "" + + windows_6_0_enabled: + type: boolean + description: "Boolean to enable the Windows 6.0 Swift version matrix job. Defaults to true." + default: false + windows_6_0_arguments_override: + type: string + description: "The arguments passed to swift test in the Windows 6.0 Swift version matrix job." + default: "" + windows_nightly_6_0_enabled: + type: boolean + description: "Boolean to enable the Windows nightly 6.0 Swift version matrix job. Defaults to true." + default: false + windows_nightly_6_0_arguments_override: + type: string + description: "The arguments passed to swift test in the Windows nightly 6.0 Swift version matrix job." + default: "" + windows_nightly_main_enabled: + type: boolean + description: "Boolean to enable the Windows nightly main Swift version matrix job. Defaults to true." + default: false + windows_nightly_main_arguments_override: + type: string + description: "The arguments passed to swift test in the Windows nightly main Swift version matrix job." + default: "" + +jobs: + construct-matrix: + name: Construct unit test matrix + runs-on: ubuntu-latest + outputs: + unit-test-matrix: '${{ steps.generate-matrix.outputs.unit-test-matrix }}' + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + persist-credentials: false + - id: generate-matrix + run: | + { + echo 'unit-test-matrix<> "$GITHUB_OUTPUT" + env: + MATRIX_LINUX_COMMAND: "swift test" + MATRIX_LINUX_5_9_ENABLED: ${{ inputs.linux_5_9_enabled }} + MATRIX_LINUX_5_9_COMMAND_ARGUMENTS: ${{ inputs.linux_5_9_arguments_override }} + MATRIX_LINUX_5_10_ENABLED: ${{ inputs.linux_5_10_enabled }} + MATRIX_LINUX_5_10_COMMAND_ARGUMENTS: ${{ inputs.linux_5_10_arguments_override }} + MATRIX_LINUX_6_0_ENABLED: ${{ inputs.linux_6_0_enabled }} + MATRIX_LINUX_6_0_COMMAND_ARGUMENTS: ${{ inputs.linux_6_0_arguments_override }} + MATRIX_LINUX_NIGHTLY_6_0_ENABLED: ${{ inputs.linux_nightly_6_0_enabled }} + MATRIX_LINUX_NIGHTLY_6_0_COMMAND_ARGUMENTS: ${{ inputs.linux_nightly_6_0_arguments_override }} + MATRIX_LINUX_NIGHTLY_MAIN_ENABLED: ${{ inputs.linux_nightly_main_enabled }} + MATRIX_LINUX_NIGHTLY_MAIN_COMMAND_ARGUMENTS: ${{ inputs.linux_nightly_main_arguments_override }} + MATRIX_WINDOWS_COMMAND: "swift test" + MATRIX_WINDOWS_6_0_ENABLED: ${{ inputs.windows_6_0_enabled }} + MATRIX_WINDOWS_6_0_COMMAND_ARGUMENTS: ${{ inputs.windows_6_0_arguments_override }} + MATRIX_WINDOWS_NIGHTLY_6_0_ENABLED: ${{ inputs.windows_nightly_6_0_enabled }} + MATRIX_WINDOWS_NIGHTLY_6_0_COMMAND_ARGUMENTS: ${{ inputs.windows_nightly_6_0_arguments_override }} + MATRIX_WINDOWS_NIGHTLY_MAIN_ENABLED: ${{ inputs.windows_nightly_main_enabled }} + MATRIX_WINDOWS_NIGHTLY_MAIN_COMMAND_ARGUMENTS: ${{ inputs.windows_nightly_main_arguments_override }} + + unit-tests: + name: Unit tests + needs: construct-matrix + # TODO: use global reference + uses: ./.github/workflows/swift_test_matrix.yml + with: + name: "Unit tests" + matrix_string: '${{ needs.construct-matrix.outputs.unit-test-matrix }}' diff --git a/scripts/generate_matrix.sh b/scripts/generate_matrix.sh new file mode 100755 index 00000000..b4e98296 --- /dev/null +++ b/scripts/generate_matrix.sh @@ -0,0 +1,153 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2024 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +# Parameters +linux_command="$MATRIX_LINUX_COMMAND" # required if any Linux pipeline is enabled +linux_setup_command="$MATRIX_LINUX_SETUP_COMMAND" +linux_5_9_enabled="${MATRIX_LINUX_5_9_ENABLED:true}" +linux_5_9_command_arguments="$MATRIX_LINUX_5_9_COMMAND_ARGUMENTS" +linux_5_10_enabled="${MATRIX_LINUX_5_10_ENABLED:true}" +linux_5_10_command_arguments="$MATRIX_LINUX_5_10_COMMAND_ARGUMENTS" +linux_6_0_enabled="${MATRIX_LINUX_6_0_ENABLED:true}" +linux_6_0_command_arguments="$MATRIX_LINUX_6_0_COMMAND_ARGUMENTS" +linux_nightly_6_0_enabled="${MATRIX_LINUX_NIGHTLY_6_0_ENABLED:true}" +linux_nightly_6_0_command_arguments="$MATRIX_LINUX_NIGHTLY_6_0_COMMAND_ARGUMENTS" +linux_nightly_main_enabled="${MATRIX_LINUX_NIGHTLY_MAIN_ENABLED:true}" +linux_nightly_main_command_arguments="$MATRIX_LINUX_NIGHTLY_MAIN_COMMAND_ARGUMENTS" + +windows_command="$MATRIX_WINDOWS_COMMAND" # required if any Windows pipeline is enabled +windows_setup_command="$MATRIX_WINDOWS_SETUP_COMMAND" +windows_6_0_enabled="${MATRIX_WINDOWS_6_0_ENABLED:false}" +windows_6_0_command_arguments="$MATRIX_WINDOWS_6_0_COMMAND_ARGUMENTS" +windows_nightly_6_0_enabled="${MATRIX_WINDOWS_NIGHTLY_6_0_ENABLED:false}" +windows_nightly_6_0_command_arguments="$MATRIX_WINDOWS_NIGHTLY_6_0_COMMAND_ARGUMENTS" +windows_nightly_main_enabled="${MATRIX_WINDOWS_NIGHTLY_MAIN_ENABLED:false}" +windows_nightly_main_command_arguments="$MATRIX_WINDOWS_NIGHTLY_MAIN_COMMAND_ARGUMENTS" + +# Defaults +linux_runner="ubuntu-latest" +linux_5_9_container_image="swift:5.9-jammy" +linux_5_10_container_image="swift:5.10-jammy" +linux_6_0_container_image="swift:6.0-jammy" +linux_nightly_6_0_container_image="swiftlang/swift:nightly-6.0-jammy" +linux_nightly_main_container_image="swiftlang/swift:nightly-main-jammy" + +windows_6_0_runner="windows-2022" +windows_6_0_container_image="swift:6.0-windowsservercore-ltsc2022" +windows_nightly_6_0_runner="windows-2019" +windows_nightly_6_0_container_image="swiftlang/swift:nightly-6.0-windowsservercore-1809" +windows_nightly_main_runner="windows-2019" +windows_nightly_main_container_image="swiftlang/swift:nightly-main-windowsservercore-1809" + +# Create matrix from inputs +matrix='{"linux": [], "windows": [], "macos": []}' + +## Linux +if [[ "$linux_5_9_enabled" == "true" || "$linux_5_10_enabled" == "true" || "$linux_6_0_enabled" == "true" || \ + "$linux_nightly_6_0_enabled" == "true" || "$linux_nightly_main_enabled" == "true" ]]; then + if [[ -z "$linux_command" ]]; then + echo "No linux command defined"; exit 1 + fi +fi + + +if [[ "$linux_5_9_enabled" == "true" ]]; then + matrix=$(echo "$matrix" | jq -c \ + --arg setup_command "$linux_setup_command" \ + --arg command "$linux_command" \ + --arg command_arguments "$linux_5_9_command_arguments" \ + --arg container_image "$linux_5_9_container_image" \ + --arg runner "$linux_runner" \ + '.linux[.linux| length] |= . + { "name": "5.9", "image": $container_image, "swift_version": "5.9", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}') +fi + +if [[ "$linux_5_10_enabled" == "true" ]]; then + matrix=$(echo "$matrix" | jq -c \ + --arg setup_command "$linux_setup_command" \ + --arg command "$linux_command" \ + --arg command_arguments "$linux_5_10_command_arguments" \ + --arg container_image "$linux_5_10_container_image" \ + --arg runner "$linux_runner" \ + '.linux[.linux| length] |= . + { "name": "5.10", "image": $container_image, "swift_version": "5.10", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}') +fi + +if [[ "$linux_6_0_enabled" == "true" ]]; then + matrix=$(echo "$matrix" | jq -c \ + --arg setup_command "$linux_setup_command" \ + --arg command "$linux_command" \ + --arg command_arguments "$linux_6_0_command_arguments" \ + --arg container_image "$linux_6_0_container_image" \ + --arg runner "$linux_runner" \ + '.linux[.linux| length] |= . + { "name": "6.0", "image": $container_image, "swift_version": "6.0", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}') +fi + +if [[ "$linux_nightly_6_0_enabled" == "true" ]]; then + matrix=$(echo "$matrix" | jq -c \ + --arg setup_command "$linux_setup_command" \ + --arg command "$linux_command" \ + --arg command_arguments "$linux_nightly_6_0_command_arguments" \ + --arg container_image "$linux_nightly_6_0_container_image" \ + --arg runner "$linux_runner" \ + '.linux[.linux| length] |= . + { "name": "nightly-6.0", "image": $container_image, "swift_version": "nightly-6.0", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}') +fi + +if [[ "$linux_nightly_main_enabled" == "true" ]]; then + matrix=$(echo "$matrix" | jq -c \ + --arg setup_command "$linux_setup_command" \ + --arg command "$linux_command" \ + --arg command_arguments "$linux_nightly_main_command_arguments" \ + --arg container_image "$linux_nightly_main_container_image" \ + --arg runner "$linux_runner" \ + '.linux[.linux| length] |= . + { "name": "nightly-main", "image": $container_image, "swift_version": "nightly-main", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner}') +fi + +## Windows +if [[ "$windows_6_0_enabled" == "true" || "$windows_nightly_6_0_enabled" == "true" || "$windows_nightly_main_enabled" == "true" ]]; then + if [[ -z "$windows_command" ]]; then + echo "No windows command defined"; exit 1 + fi +fi + +if [[ "$windows_6_0_enabled" == "true" ]]; then + matrix=$(echo "$matrix" | jq -c \ + --arg setup_command "$windows_setup_command" \ + --arg command "$windows_command" \ + --arg command_arguments "$windows_6_0_command_arguments" \ + --arg container_image "$windows_6_0_container_image" \ + --arg runner "$windows_6_0_runner" \ + '.windows[.windows| length] |= . + { "name": "6.0", "image": $container_image, "swift_version": "6.0", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }') +fi + +if [[ "$windows_nightly_6_0_enabled" == "true" ]]; then + matrix=$(echo "$matrix" | jq -c \ + --arg setup_command "$windows_setup_command" \ + --arg command "$windows_command" \ + --arg command_arguments "$windows_nightly_6_0_command_arguments" \ + --arg container_image "$windows_nightly_6_0_container_image" \ + --arg runner "$windows_nightly_6_0_runner" \ + '.windows[.windows| length] |= . + { "name": "nightly-6.0", "image": $container_image, "swift_version": "nightly-6.0", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }') +fi + +if [[ "$windows_nightly_main_enabled" == "true" ]]; then + matrix=$(echo "$matrix" | jq -c \ + --arg setup_command "$windows_setup_command" \ + --arg command "$windows_command" \ + --arg command_arguments "$windows_nightly_main_command_arguments" \ + --arg container_image "$windows_nightly_main_container_image" \ + --arg runner "$windows_nightly_main_runner" \ + '.windows[.windows| length] |= . + { "name": "nightly-main", "image": $container_image, "swift_version": "nightly-main", "command": $command, "command_arguments": $command_arguments, "setup_command": $setup_command, "runner": $runner }') +fi + +echo "$matrix" | jq -c \ No newline at end of file