From 7214bb1bf10e47229f49e694f7f3dc75d7c64f71 Mon Sep 17 00:00:00 2001 From: Gianni Trevisiol Date: Wed, 22 Nov 2023 15:28:45 -0800 Subject: [PATCH] Create a new CI/CD workflow file that will be used specifically validate release branches. (#3048) * draft * wip * wip * wip * wip * wip * wip * wip - long ver * wip * doc * doc * Removed pref-profile test * comments * doc * doc * test * wip * update id * Add prefix to perf.yml * doc --- .github/workflows/cicd-release-validation.yml | 542 ++++++++++++++++++ .github/workflows/perf.yml | 2 +- docs/ReleaseProcess.md | 7 +- 3 files changed, 546 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/cicd-release-validation.yml diff --git a/.github/workflows/cicd-release-validation.yml b/.github/workflows/cicd-release-validation.yml new file mode 100644 index 0000000000..4475edfb72 --- /dev/null +++ b/.github/workflows/cicd-release-validation.yml @@ -0,0 +1,542 @@ +# Copyright (c) Microsoft Corporation +# SPDX-License-Identifier: MIT + +# This is the top-level workflow that runs on release branches. +# It invokes other jobs to perform builds and run tests. +# All jobs run in parallel, using build artifacts to synchronize jobs. +# +# If you add or remove any tests that use reusable-test.yml on a pull request, +# you must update codecov.yml to match. + +name: CI/CD - Release validation + +on: + # Trigger the workflow when a release-tag is created. + push: + tags: + - 'v0.*' # To be updated on GA, and after that on every major release. + + # Permit manual runs of the workflow. + workflow_dispatch: + inputs: + # Request a target release branch to run CI/CD on. + branch: + description: 'Target release branch to run CI/CD on.' + required: true + default: 'release/' + +concurrency: + group: release-validation-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + id-token: write # Required to log in to Azure. + contents: read + checks: read # Required by reusable-test.yml to check build status. + security-events: write # Required by codeql task. + issues: write # Required to create issues. + +jobs: + # Perform the regular build. + regular: + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-build.yml + with: + build_artifact: Build-x64 + generate_release_package: true + build_msi: true + build_nuget: true + build_options: /p:ReleaseJIT='True' + configurations: '["Debug", "FuzzerDebug", "Release"]' + + # Perform the native-only build. + regular_native-only: + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-build.yml + with: + build_artifact: Build-x64-native-only + build_msi: true + build_nuget: true + configurations: '["NativeOnlyDebug", "NativeOnlyRelease"]' + + # Run the unit tests in GitHub. + unit_tests: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: unit_tests + pre_test: appverif -enable Exceptions Handles Heaps Leak Locks Memory SRWLock Threadpool TLS DangerousAPIs DirtyStacks TimeRollOver -for unit_tests.exe + # Exclude [processes] test that CodeCoverage can't work with. + test_command: .\unit_tests.exe -d yes ~[processes] + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: true + gather_dumps: true + capture_etw: true + leak_detection: true + + # Run the netebpfext unit tests in GitHub. + netebpf_ext_unit_tests: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: netebpf_ext_unit_tests + pre_test: appverif -enable Exceptions Handles Heaps Leak Locks Memory SRWLock Threadpool TLS DangerousAPIs DirtyStacks TimeRollOver -for unit_tests.exe + test_command: .\netebpfext_unit.exe -d yes + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: true + gather_dumps: true + capture_etw: true + leak_detection: true + + # Run the bpf2c tests in GitHub. + bpf2c: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + test_command: .\bpf2c_tests.exe -d yes + name: bpf2c + build_artifact: Build-x64 + environment: windows-2022 + vs_dev: true + code_coverage: true + gather_dumps: true + capture_etw: true + + # Run the bpf2c conformance tests in GitHub. + bpf2c_conformance: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + pre_test: Invoke-WebRequest https://github.com/Alan-Jowett/bpf_conformance/releases/download/v0.0.4/bpf_conformance_runner.exe -OutFile bpf_conformance_runner.exe + test_command: .\bpf_conformance_runner.exe --test_file_directory %SOURCE_ROOT%\external\ebpf-verifier\external\bpf_conformance\tests --exclude_regex lock* --plugin_path bpf2c_plugin.exe --debug true --plugin_options "--include %SOURCE_ROOT%\include" + name: bpf2c_conformance + build_artifact: Build-x64 + environment: windows-2022 + vs_dev: true + code_coverage: true + gather_dumps: true + capture_etw: true + + # Run the driver tests on self-hosted runners. + driver_ws2019: + needs: regular + # Only run this on repos that have self-host runners. + if: github.repository == 'microsoft/ebpf-for-windows' && (github.event_name == 'workflow_dispatch') + uses: ./.github/workflows/reusable-test.yml + with: + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "file" + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "CI/CD" + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true + name: driver_ws2019 + build_artifact: Build-x64 + environment: ebpf_cicd_tests_ws2019 + # driver test copies dumps to testlog folder. + gather_dumps: false + # driver tests manually gather code coverage + code_coverage: false + + # Run the driver tests on self-hosted runners. + driver_ws2022: + needs: regular + # Only run this on repos that have self-host runners. + if: github.repository == 'microsoft/ebpf-for-windows' && (github.event_name == 'workflow_dispatch') + uses: ./.github/workflows/reusable-test.yml + with: + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "file" + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "CI/CD" + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true + name: driver_ws2022 + build_artifact: Build-x64 + environment: ebpf_cicd_tests_ws2022 + # driver test copies dumps to testlog folder. + gather_dumps: false + # driver tests manually gather code coverage + code_coverage: false + + # Run the native-only driver tests on self-hosted runners. + driver_native_only_ws2019: + needs: regular_native-only + # Only run this on repos that have self-host runners. + if: github.repository == 'microsoft/ebpf-for-windows' && (github.event_name == 'workflow_dispatch') + uses: ./.github/workflows/reusable-test.yml + with: + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "file" + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "CI/CD" + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true + name: driver_native_only_ws2019 + build_artifact: Build-x64-native-only + environment: ebpf_cicd_tests_ws2019 + # driver test copies dumps to testlog folder. + gather_dumps: false + # driver tests manually gather code coverage + code_coverage: false + configurations: '["NativeOnlyDebug", "NativeOnlyRelease"]' + + driver_native_only_ws2022: + needs: regular_native-only + # Only run this on repos that have self-host runners. + if: github.repository == 'microsoft/ebpf-for-windows' && (github.event_name == 'workflow_dispatch') + uses: ./.github/workflows/reusable-test.yml + with: + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "file" + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "CI/CD" + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true + name: driver_native_only_ws2022 + build_artifact: Build-x64-native-only + environment: ebpf_cicd_tests_ws2022 + # driver test copies dumps to testlog folder. + gather_dumps: false + # driver tests manually gather code coverage + code_coverage: false + configurations: '["NativeOnlyDebug", "NativeOnlyRelease"]' + + ossar: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/ossar-scan.yml + with: + build_artifact: Build-x64 + + # Build with C++ static analyzer. + analyze: + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-build.yml + with: + build_artifact: Build-x64-Analyze + # Analysis on external projects is conditional, as on small CI/CD VMs the compiler can run OOM + build_options: /p:Analysis='True' /p:AnalysisOnExternal='False' + + # Build with C++ address sanitizer. + sanitize: + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-build.yml + with: + build_artifact: Build-x64-Sanitize + build_options: /p:AddressSanitizer='True' + + # Run Cilium regression tests in GitHub. + cilium_tests: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: cilium_tests + test_command: .\cilium_tests.exe -d yes + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + gather_dumps: true + + # Run the quick stress tests in GitHub. + stress: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: stress + # Until there is a dedicated stress test, re-use the perf test. + test_command: .\ebpf_performance.exe -d yes + build_artifact: Build-x64 + environment: windows-2022 + # No code coverage on stress. + code_coverage: false + gather_dumps: true + + # Run the unit tests in GitHub with address sanitizer. + sanitize_unit_tests: + needs: sanitize + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: unit_tests + # Exclude [processes] test that ASAN can't work with. + test_command: .\unit_tests.exe -d yes ~[processes] + build_artifact: Build-x64-Sanitize + environment: windows-2022 + code_coverage: false + gather_dumps: true + capture_etw: true + + # Run the fault injection simulator in GitHub. + fault_injection: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: fault_injection + test_command: .\unit_tests.exe + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: true + gather_dumps: true + fault_injection: true + leak_detection: true + + # Run the low memory simulator for netebpfext_unit tests. + fault_injection_netebpfext_unit: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: netebpfext_fault_injection + test_command: .\netebpfext_unit.exe + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: true + gather_dumps: true + fault_injection: true + leak_detection: true + + # Run a fast multi-threaded stress test pass against the usersim user-mode 'mock' framework. + # Added as a 'per-PR' test to catch usersim regressions and/or run-time usage issues. + quick_user_mode_multi_threaded_stress_test: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: quick_user_mode_multi_threaded_stress + test_command: .\ebpf_stress_tests_um -tt=8 -td=2 + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + leak_detection: false + gather_dumps: true + capture_etw: true + + codeql: + uses: ./.github/workflows/reusable-build.yml + with: + build_artifact: Build-x64-CodeQl + build_codeql: true + + # Run the complete fault injection simulator in GitHub (this takes a long time to run). + fault_injection_full: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: fault_injection_full + test_command: .\unit_tests.exe + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + gather_dumps: true + fault_injection: true + leak_detection: true + + # Run the complete fault injection simulator for netebpfext in GitHub (this takes a long time to run). + netebpfext_fault_injection_full: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: netebpfext_fault_injection_full + test_command: .\netebpfext_unit.exe + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + gather_dumps: true + fault_injection: true + + # Run multi-threaded stress tests against the user mode 'mock' framework. + user_mode_multi_threaded_stress_test: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: user_mode_multi_threaded_stress + test_command: .\ebpf_stress_tests_um -tt=8 -td=10 + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + leak_detection: false + gather_dumps: true + capture_etw: true + + # Run multi-threaded stress tests with 'restart extension' disabled (default behavior) + # against the kernel mode eBPF sub-system. + km_mt_stress_tests: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: km_mt_stress_tests + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "memory" + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Stress" + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true + build_artifact: Build-x64 + environment: ebpf_cicd_tests_ws2019 + code_coverage: false + # For this test, we only want kernel mode dumps and not user mode dumps. + gather_dumps: false + + # Run multi-threaded stress tests with 'restart extension' enabled + # against the kernel mode eBPF sub-system. + km_mt_stress_tests_restart_extension: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: km_mt_stress_tests_restart_extension + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "memory" + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Stress" -Options @("RestartExtension") + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true + build_artifact: Build-x64 + environment: ebpf_cicd_tests_ws2019 + code_coverage: false + # For this test, we only want kernel mode dumps and not user mode dumps. + gather_dumps: false + + ###################################################### + # Run indeterministic tests multiple times (3 times). + ###################################################### + # FUZZERS ############################################ + bpf2c_fuzzer: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: bpf2c_fuzzer + test_command: .\bpf2c_fuzzer.exe bpf2c_fuzzer_corpus -use_value_profile=1 -max_total_time=900 -artifact_prefix=Artifacts\ + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + gather_dumps: true + configurations: '["FuzzerDebug"]' + + verifier_fuzzer: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: verifier_fuzzer + test_command: .\verifier_fuzzer.exe verifier_corpus -use_value_profile=1 -max_total_time=900 -artifact_prefix=Artifacts\ + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + gather_dumps: true + configurations: '["FuzzerDebug"]' + + execution_context_fuzzer: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: execution_context_fuzzer + test_command: .\execution_context_fuzzer.exe execution_context_fuzzer_corpus -use_value_profile=1 -runs=3000 -artifact_prefix=Artifacts\ + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + gather_dumps: true + configurations: '["FuzzerDebug"]' + + core_helper_fuzzer: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: core_helper_fuzzer + test_command: .\core_helper_fuzzer core_helper_corpus -max_len=139 -runs=3000 -use_value_profile=1 -artifact_prefix=Artifacts\ + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + gather_dumps: true + configurations: '["FuzzerDebug"]' + + netebpfext_fuzzer: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: netebpfext_fuzzer + test_command: .\netebpfext_fuzzer netebpfext_corpus -max_len=12 -runs=3000 -use_value_profile=1 -artifact_prefix=Artifacts\ + build_artifact: Build-x64 + environment: windows-2022 + code_coverage: false + gather_dumps: true + configurations: '["FuzzerDebug"]' + +# PREFORMANCE ############################################ +# Iteration 1 + performance-1: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: km_performance + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "file" + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Performance" + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true + build_artifact: Build-x64 + environment: ebpf_cicd_perf_ws2022 + configurations: '["Release"]' + + upload_perf_results-1: + needs: performance-1 + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/upload-perf-results.yml + with: + name: upload_perf_results + result_artifact: km_performance-x64-Release-1 + secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + +# Iteration 2 + performance-2: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: km_performance + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "file" + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Performance" + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true + build_artifact: Build-x64 + environment: ebpf_cicd_perf_ws2022 + configurations: '["Release"]' + + upload_perf_results-2: + needs: performance-2 + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/upload-perf-results.yml + with: + name: upload_perf_results + result_artifact: km_performance-x64-Release-1 + secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + +# Iteration 3 + performance-3: + needs: regular + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/reusable-test.yml + with: + name: km_performance + pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "file" + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Performance" + post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true + build_artifact: Build-x64 + environment: ebpf_cicd_perf_ws2022 + configurations: '["Release"]' + + upload_perf_results-3: + needs: performance-3 + if: github.event_name == 'workflow_dispatch' + uses: ./.github/workflows/upload-perf-results.yml + with: + name: upload_perf_results + result_artifact: km_performance-x64-Release-3 + secrets: + AZURE_CLIENT_ID: ${{ secrets.AZURE_CLIENT_ID }} + AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }} + AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} \ No newline at end of file diff --git a/.github/workflows/perf.yml b/.github/workflows/perf.yml index 6e16ca8443..b5e3ced0bc 100644 --- a/.github/workflows/perf.yml +++ b/.github/workflows/perf.yml @@ -12,7 +12,7 @@ on: concurrency: # Cancel any CI/CD workflow currently in progress for the same PR. # Allow running concurrently with any other commits. - group: ${{ github.workflow }}-${{ github.ref }} + group: perf-${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true permissions: diff --git a/docs/ReleaseProcess.md b/docs/ReleaseProcess.md index ee70ba34ac..dfe1ffba18 100644 --- a/docs/ReleaseProcess.md +++ b/docs/ReleaseProcess.md @@ -46,8 +46,6 @@ eBPF for Windows, and how to service it later. - `eBPF-for-Windows.X.Y.Z.nupkg` 1. Submit the PR for review (from its draft state), and wait for it to be approved and merged into the main repo's "`main`" branch. 1. On the main `ebpf-for-windows` repo, create a new release branch from `main` **corresponding to the previous PR's commit**, name it "`release/X.Y`" (replace "X.Y" with the version number being released). -1. Request the Admin of the main `ebpf-for-windows` repo to protect and apply release policies to the "`release/X.Y`" branch. -1. Wait for the main `ebpf-for-windows` repo's Admin to complete the previous step. 1. Publish the release as per the "[Publishing a Release](ReleaseProcess.md#publishing-a-release)" process. ## Servicing a release @@ -82,7 +80,6 @@ Servicing a release has two main scenarios: 1. Create a **Draft** pull-request for your working branch into the main repo's "`release/X.Y`" branch, and title the PR as *"Release v`X.Y.Z`"* (replace "`X.Y.Z`" with the version number being released). 1. Wait for the CI/CD pipeline for the PR to complete successfully. 1. Submit the PR for review (from its draft state), and wait for it to be approved and merged into the main repo's "`release/X.Y`" branch. -1. Create a tag for the PR's commit number, on the main repo's "`release/X.Y`" branch, with the version number being released, i.e., "`vX.Y.Z`". 1. Publish the release as per the "[Publishing a Release](ReleaseProcess.md#publishing-a-release)" process. @@ -112,7 +109,9 @@ As a result of creating new release or servicing an existing one, the following ### Publishing the Release to GitHub -1. Create a tag for the PR's commit number, with the version number being released, i.e. "`vX.Y.Z`". +1. On the `microsoft/ebpf-for-windows` repo's `main` branch, create a tag for the release-PR's commit. The tag should reflect the version number being released and adhere to the following notation: "`vX.Y.Z`". +1. The tag creation will automatically trigger the "`CI/CD - Release validation`" workflow for the PR: wait for it to complete successfully. +1. Once the CI/CD pipeline completes successfully, wait for the `sign-off` label to be added from the Triage meeting, on the automated "release-issue" associated to the release to be published. 1. Go to the repo on GitHub and click on "``" and click on right the "`Create a new release`" link. 1. Click on the "`Choose a tag`" combo box and select the tag with new "`vX.Y.Z`" version number, as created earlier. 1. Fill in the release title as "`vX.Y.Z`" (replace "`X.Y.Z`" with the version number being released).