From 5e84f5c914520c4bf40647fb0436283df23a0999 Mon Sep 17 00:00:00 2001 From: Gianni Trevisiol Date: Tue, 12 Mar 2024 15:58:51 -0700 Subject: [PATCH 01/10] draft (#3333) --- docs/InstallEbpf.md | 56 +- installer/Product.wxs | 845 +----------------- installer/ebpf-for-windows.wixproj | 96 -- scripts/check_msi_installation.ps1 | 2 +- ...k_msi_installation_files_regular_debug.txt | 243 +---- 5 files changed, 36 insertions(+), 1206 deletions(-) diff --git a/docs/InstallEbpf.md b/docs/InstallEbpf.md index 94047cf158..48c53dc770 100644 --- a/docs/InstallEbpf.md +++ b/docs/InstallEbpf.md @@ -16,49 +16,40 @@ Do the following from within the VM: * **Runtime Components** (mandatory): this feature adds the eBPF runtime and core components, which are also required by the other components. If you select only this feature, only [native code generation](NativeCodeGeneration.md) is enabled. * **JIT** (optional): this sub-feature adds support for JIT-compiled eBPF programs and (in a Debug build only) interpreted eBPF programs. - * **Development** (optional): this feature adds headers and libraries used for development. If you only want to use eBPF for development - rather than running programs, you can [use the NuGet package](GettingStarted.md#using-ebpf-in-development) - instead of the MSI. - * **Testing** (optional): this feature adds tests for the eBPF runtime for use by eBPF runtime developers. An **command line install/uninstall** is also supported, through the direct use of `C:\Windows\system32\msiexec.exe` from an *administrative Command Prompt*: -- The installation folder can be customized by assigning the desired path to the `INSTALLFOLDER` parameter (path with spaces must be put between double quotes), i.e.: +* The installation folder can be customized by assigning the desired path to the `INSTALLFOLDER` parameter (path with spaces must be put between double quotes), i.e.: + ```bash INSTALLFOLDER="C:\Program Files\ebpf-for-windows" ``` -- The following feature-components are available for customization, and must be assigned as comma-separated values to the `ADDLOCAL` parameter: +* The following feature-components are available for customization, and must be assigned as comma-separated values to the `ADDLOCAL` parameter: - - `eBPF_Runtime_Components` (**mandatory**): runtime components (installed in `[Installation folder]\*`, `[Installation folder]\drivers`). - - `eBPF_Runtime_Components_JIT` (optional): JIT compiler service (installed in `[Installation folder]\JIT`). - - `eBPF_Development` (optional): header files, scripts and libraries for development (installed in `[Installation folder]\include`,`[Installation folder]\lib`,`[Installation folder]\scripts`). - - `eBPF_Testing` (optional): header files for development (installed in `[Installation folder]\testing`). + * `eBPF_Runtime_Components` (**mandatory**): runtime components (installed in `[Installation folder]\*`, `[Installation folder]\drivers`). + * `eBPF_Runtime_Components_JIT` (optional): JIT compiler service (installed in `[Installation folder]\JIT`). e.g., (full featured): ```bash - ADDLOCAL=eBPF_Runtime_Components,eBPF_Runtime_Components_JIT,eBPF_Development,eBPF_Testing + ADDLOCAL=eBPF_Runtime_Components,eBPF_Runtime_Components_JIT ``` - Below are some examples of CLI installations/uninstallation, using "`C:\Program Files\ebpf-for-windows`" as the installation folder: * Installation: > **Note**: add the "`/qn`" switch for **unattended install**. ```bash - # Debug MSI - fully-featured installation - C:\Windows\system32\msiexec.exe /i eBPF-for-Windows.x.x.x.msi INSTALLFOLDER="C:\Program Files\ebpf-for-windows" ADDLOCAL=eBPF_Runtime_Components,eBPF_Runtime_Components_JIT,eBPF_Development,eBPF_Testing + # Debug MSI - fully-featured installation, including the JIT compiler (available on pre-release versions only) + C:\Windows\system32\msiexec.exe /i eBPF-for-Windows.x.x.x.msi INSTALLFOLDER="C:\Program Files\ebpf-for-windows" ADDLOCAL=eBPF_Runtime_Components,eBPF_Runtime_Components_JIT - # Debug MSI - fully-featured installation, no JIT compiler - C:\Windows\system32\msiexec.exe /i eBPF-for-Windows.x.x.x.msi INSTALLFOLDER="C:\Program Files\ebpf-for-windows" ADDLOCAL=eBPF_Runtime_Components,eBPF_Development,eBPF_Testing + # Debug MSI - minimal installation (only runtime components) + C:\Windows\system32\msiexec.exe /i eBPF-for-Windows.x.x.x.msi INSTALLFOLDER="C:\Program Files\ebpf-for-windows" ADDLOCAL=eBPF_Runtime_Components # Release MSI - fully-featured installation, including the JIT compiler (available on pre-release versions only) - C:\Windows\system32\msiexec.exe /i eBPF-for-Windows.x.x.x.msi INSTALLFOLDER="C:\Program Files\ebpf-for-windows" ADDLOCAL=eBPF_Runtime_Components,eBPF_Runtime_Components_JIT,eBPF_Development,eBPF_Testing - - # Release MSI - fully-featured installation (no JIT compiler on future post-release versions) - C:\Windows\system32\msiexec.exe /i eBPF-for-Windows.x.x.x.msi INSTALLFOLDER="C:\Program Files\ebpf-for-windows" ADDLOCAL=eBPF_Runtime_Components,eBPF_Development,eBPF_Testing + C:\Windows\system32\msiexec.exe /i eBPF-for-Windows.x.x.x.msi INSTALLFOLDER="C:\Program Files\ebpf-for-windows" ADDLOCAL=eBPF_Runtime_Components,eBPF_Runtime_Components_JIT # Release MSI - minimal installation (only runtime components) C:\Windows\system32\msiexec.exe /i eBPF-for-Windows.x.x.x.msi INSTALLFOLDER="C:\Program Files\ebpf-for-windows" ADDLOCAL=eBPF_Runtime_Components @@ -79,27 +70,33 @@ C:\Windows\system32\msiexec.exe /i eBPF-for-Windows.x.x.x.msi /l ``` ### Method 2 (Install files you built yourself) + This method uses a machine that has already built the binaries for `x64/Debug` or `x64/Release`. 1. Deploy the binaries to `C:\Temp` in your VM, as follows (from within a "*Developer PowerShell for VS 2022*"): - - If you **built the binaries from inside the VM**, then from your `ebpf-for-windows` directory in the VM, run: + * If you **built the binaries from inside the VM**, then from your `ebpf-for-windows` directory in the VM, run: ```ps .\x64\debug\deploy-ebpf -l ``` - - Otherwise, if you **built the binaries on the host machine**, then from your `ebpf-for-windows` + + * Otherwise, if you **built the binaries on the host machine**, then from your `ebpf-for-windows` directory on the host machine, start an admin Powershell on the host machine and run: ```ps .\x64\debug\deploy-ebpf --vm="" ``` + or, to also copy files needed to run various tests, run: + ```ps .\x64\debug\deploy-ebpf --vm="" -t ``` + or, to copy files to a specific directory, including file shares, run: + ```ps .\x64\debug\deploy-ebpf -l="c:\some\path" ``` @@ -112,24 +109,31 @@ has already built the binaries for `x64/Debug` or `x64/Release`. powershell -ExecutionPolicy Bypass .\scripts\setup-ebpf.ps1 ``` + ### Method 3 (Install files you built yourself, with a VM checkpoint) + This method uses a machine that has already built the binaries for `x64/Debug` or `x64/Release`. Copy the build output in `\x64\[Debug|Release]` to the host of the test VM and run the following in a Powershell command prompt: + 1. Create a snapshot of the test VM named **baseline**, by running: ```ps Checkpoint-VM -Name -CheckpointName baseline ``` + 1. Store the VM administrator credential, by running the following commands: + ```ps Install-Module CredentialManager -force ``` + ```ps New-StoredCredential -Target TEST_VM -Username -Password -Persist LocalMachine ``` + > Note that "`TEST_VM`" is literal and is used in step 5 below; it need not be the name of any actual test VM. 1. Enter the `\x64\[Debug|Release]` directory (`cd`) where the build artifacts are stored. 1. Modify `.\vm_list.json` to specify the name of the test VM under `VMList`, eg: @@ -146,11 +150,14 @@ command prompt: ] } ``` + 1. Run the following commands to setup to use the credentials saved with `TEST_VM` in step 2, for logging into each of the VMs named in `vm_list.json`: + ```ps Set-ExecutionPolicy unrestricted -Force ``` + ```ps .\setup_ebpf_cicd_tests.ps1 ``` @@ -162,7 +169,6 @@ to install eBPF on all Windows nodes in a Kubernetes cluster. 1. Download the `.msi` file from the [latest release on GitHub](https://github.com/microsoft/ebpf-for-windows/releases) and copy it over to [images](../images) directory. - 2. Build ebpf-for-windows image. * To **build the image on the Windows Host**, make sure docker is installed. [Install docker on Windows Server](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/set-up-environment?tabs=Windows-Server/). @@ -174,7 +180,8 @@ Start an admin Powershell on the Windows Host and run the following command and * To **build the image on a Linux machine** (e.g. Ubuntu), make sure docker is installed (see [install docker on Ubuntu](https://docs.docker.com/engine/install/ubuntu/)), and do the following: - - Run the following command and provide parameters for `repository`, `tag` and `OSVersion`: + * Run the following command and provide parameters for `repository`, `tag` and `OSVersion`: + ```bash $HOME/ebpf-for-windows-image/build-images.sh ```` @@ -182,6 +189,7 @@ Start an admin Powershell on the Windows Host and run the following command and 3. Push the `ebpf-for-windows` image to your repository. 4. Update `manifests/Kubernetes/ebpf-for-windows-daemonset.yaml` with the container image pointing to your image path. Run the following command: + ```cmd kubectl apply -f manifests/Kubernetes/ebpf-for-windows-daemonset.yaml ``` diff --git a/installer/Product.wxs b/installer/Product.wxs index 2fdd7ac7ad..b457c78dae 100644 --- a/installer/Product.wxs +++ b/installer/Product.wxs @@ -47,30 +47,6 @@ SPDX-License-Identifier: MIT - - - - - - - - - - - - - - - - - - - - - - - - @@ -83,9 +59,6 @@ SPDX-License-Identifier: MIT NOT Installed NOT Installed - NOT Installed - NOT Installed - NOT Installed NOT Installed @@ -98,9 +71,6 @@ SPDX-License-Identifier: MIT NOT Installed NOT Installed - (NOT Installed AND ) OR (Installed AND ) - (NOT Installed AND ) OR (Installed AND ) - REMOVE="ALL" @@ -110,9 +80,6 @@ SPDX-License-Identifier: MIT REMOVE="ALL" REMOVE="ALL" - REMOVE="ALL" OR REMOVE="eBPF_Testing" OR REMOVE="eBPF_Development,eBPF_Testing" - REMOVE="ALL" OR REMOVE="eBPF_Testing" OR REMOVE="eBPF_Development,eBPF_Testing" - REMOVE="ALL" REMOVE="ALL" @@ -135,35 +102,11 @@ SPDX-License-Identifier: MIT + + - - - - - - - - - - - - - - - - - - - - - - - - - - @@ -316,788 +259,4 @@ SPDX-License-Identifier: MIT - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/installer/ebpf-for-windows.wixproj b/installer/ebpf-for-windows.wixproj index 1a7c516426..ce7f968160 100644 --- a/installer/ebpf-for-windows.wixproj +++ b/installer/ebpf-for-windows.wixproj @@ -78,14 +78,6 @@ SPDX-License-Identifier: MIT Binaries;Content;Satellites INSTALLFOLDER - - EbpfCore_Usersim - {1fdad2fd-ebd8-462a-b285-ed5174e55079} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - ebpfsvc {ba065b6a-38f8-4197-8f66-87c84afad513} @@ -94,14 +86,6 @@ SPDX-License-Identifier: MIT Binaries;Content;Satellites INSTALLFOLDER - - usersim - {030a7ac6-14dc-45cf-af34-891057ab1402} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - api {c8bf60c3-40a9-43ad-891a-8aa34f1c3a68} @@ -118,78 +102,6 @@ SPDX-License-Identifier: MIT Binaries;Content;Satellites INSTALLFOLDER - - api_test - {12161211-5463-4aa9-9b6e-552552b475dc} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - - - bpftool_tests - {8b5b061b-3170-4d1b-8c5b-e86b890c14b8} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - - - sample_ext_app - {7358d8bd-4123-4b2d-86c0-87f6ba6ced76} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - - - sample_ebpf_ext - {c8d46543-5ae5-4e66-b9ce-8b84588b1c9e} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - - - sample - {b4ad72e3-754e-40ca-9cea-d3f2c9170e51} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - - - ebpf_stress_tests_km - {4f082524-9496-44fa-8cba-4bc0bdc62568} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - - - ebpf_stress_tests_um - {e51a27f5-a1d5-40da-a09b-850203d6b13d} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - - - unit_tests - {a4037e5d-b00c-4af1-9398-76b3ece95f62} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - - - xdp_tests - {07dc6181-84a2-4a14-a806-5e9af6c929c2} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - bpf2c {69b97e52-18dc-434e-a6e4-4c0f3e88c44a} @@ -198,14 +110,6 @@ SPDX-License-Identifier: MIT Binaries;Content;Satellites INSTALLFOLDER - - bpf2c_tests - {61df9973-81b9-4006-9148-52f58259bbcf} - True - True - Binaries;Content;Satellites - INSTALLFOLDER - bpftool {ca179b38-ebf5-4371-b830-73486e35250b} diff --git a/scripts/check_msi_installation.ps1 b/scripts/check_msi_installation.ps1 index d8ce3a5a40..09edbe3091 100644 --- a/scripts/check_msi_installation.ps1 +++ b/scripts/check_msi_installation.ps1 @@ -14,7 +14,7 @@ $InstallPath = "$env:ProgramFiles\ebpf-for-windows"; # - The expected file lists for each build artifact $buildArtifactParams = @{ "Build-x64_Debug" = @{ - "InstallComponents" = "ADDLOCAL=eBPF_Runtime_Components,eBPF_Runtime_Components_JIT,eBPF_Development,eBPF_Testing" + "InstallComponents" = "ADDLOCAL=eBPF_Runtime_Components,eBPF_Runtime_Components_JIT" "ExpectedFileList" = "..\..\scripts\check_msi_installation_files_regular_debug.txt" } "Build-x64-native-only_NativeOnlyRelease" = @{ diff --git a/scripts/check_msi_installation_files_regular_debug.txt b/scripts/check_msi_installation_files_regular_debug.txt index 5ce90d0496..aa7855544e 100644 --- a/scripts/check_msi_installation_files_regular_debug.txt +++ b/scripts/check_msi_installation_files_regular_debug.txt @@ -18,246 +18,5 @@ C:\Program Files\ebpf-for-windows\drivers\eBPFCore.sys C:\Program Files\ebpf-for-windows\drivers\NetEbpfExt.inf C:\Program Files\ebpf-for-windows\drivers\NetEbpfExt.pdb C:\Program Files\ebpf-for-windows\drivers\NetEbpfExt.sys -C:\Program Files\ebpf-for-windows\include\bpf2c.h -C:\Program Files\ebpf-for-windows\include\bpf_endian.h -C:\Program Files\ebpf-for-windows\include\bpf_helpers.h -C:\Program Files\ebpf-for-windows\include\bpf_helpers_platform.h -C:\Program Files\ebpf-for-windows\include\bpf_helper_defs.h -C:\Program Files\ebpf-for-windows\include\ebpf_api.h -C:\Program Files\ebpf-for-windows\include\ebpf_base.h -C:\Program Files\ebpf-for-windows\include\ebpf_core_structs.h -C:\Program Files\ebpf-for-windows\include\ebpf_execution_type.h -C:\Program Files\ebpf-for-windows\include\ebpf_extension.h -C:\Program Files\ebpf-for-windows\include\ebpf_extension_uuids.h -C:\Program Files\ebpf-for-windows\include\ebpf_nethooks.h -C:\Program Files\ebpf-for-windows\include\ebpf_program_attach_type_guids.h -C:\Program Files\ebpf-for-windows\include\ebpf_program_types.h -C:\Program Files\ebpf-for-windows\include\ebpf_result.h -C:\Program Files\ebpf-for-windows\include\ebpf_store_helper.h -C:\Program Files\ebpf-for-windows\include\ebpf_structs.h -C:\Program Files\ebpf-for-windows\include\ebpf_utilities.h -C:\Program Files\ebpf-for-windows\include\ebpf_windows.h -C:\Program Files\ebpf-for-windows\include\elfio_wrapper.hpp -C:\Program Files\ebpf-for-windows\include\git_commit_id.h -C:\Program Files\ebpf-for-windows\include\asm\errno.h -C:\Program Files\ebpf-for-windows\include\asm\posix_types.h -C:\Program Files\ebpf-for-windows\include\asm\types.h -C:\Program Files\ebpf-for-windows\include\bpf\bpf.h -C:\Program Files\ebpf-for-windows\include\bpf\bpf_legacy.h -C:\Program Files\ebpf-for-windows\include\bpf\hashmap.h -C:\Program Files\ebpf-for-windows\include\bpf\libbpf.h -C:\Program Files\ebpf-for-windows\include\bpf\libbpf_legacy.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\asm\barrier.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\linux\compiler.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\linux\err.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\linux\filter.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\linux\kernel.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\linux\list.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\linux\overflow.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\linux\ring_buffer.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\linux\types.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\uapi\linux\bpf.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\uapi\linux\bpf_common.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\uapi\linux\btf.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\uapi\linux\if_link.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\uapi\linux\if_xdp.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\uapi\linux\netlink.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\uapi\linux\perf_event.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\uapi\linux\pkt_cls.h -C:\Program Files\ebpf-for-windows\include\libbpf\include\uapi\linux\pkt_sched.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\bpf.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\bpf_core_read.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\bpf_endian.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\bpf_gen_internal.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\bpf_helpers.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\bpf_helper_defs.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\bpf_tracing.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\btf.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\hashmap.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\libbpf.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\libbpf_common.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\libbpf_internal.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\libbpf_legacy.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\libbpf_version.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\nlattr.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\relo_core.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\skel_internal.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\strset.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\str_error.h -C:\Program Files\ebpf-for-windows\include\libbpf\src\usdt.bpf.h -C:\Program Files\ebpf-for-windows\include\linux\bpf.h -C:\Program Files\ebpf-for-windows\include\linux\stddef.h -C:\Program Files\ebpf-for-windows\include\net\if_ether.h -C:\Program Files\ebpf-for-windows\include\net\ip.h -C:\Program Files\ebpf-for-windows\include\net\tcp.h -C:\Program Files\ebpf-for-windows\include\net\udp.h -C:\Program Files\ebpf-for-windows\include\uapi\linux\bpf.h -C:\Program Files\ebpf-for-windows\include\uapi\linux\if_ether.h -C:\Program Files\ebpf-for-windows\include\uapi\linux\in.h -C:\Program Files\ebpf-for-windows\include\uapi\linux\ip.h -C:\Program Files\ebpf-for-windows\include\uapi\linux\tcp.h C:\Program Files\ebpf-for-windows\JIT\ebpfsvc.exe -C:\Program Files\ebpf-for-windows\JIT\EbpfSvc.pdb -C:\Program Files\ebpf-for-windows\lib\EbpfApi.lib -C:\Program Files\ebpf-for-windows\scripts\Convert-BpfToNative.ps1 -C:\Program Files\ebpf-for-windows\testing\api_test.exe -C:\Program Files\ebpf-for-windows\testing\api_test.pdb -C:\Program Files\ebpf-for-windows\testing\bad_map_name.o -C:\Program Files\ebpf-for-windows\testing\bad_map_name_um.dll -C:\Program Files\ebpf-for-windows\testing\bad_map_name_um.pdb -C:\Program Files\ebpf-for-windows\testing\bindmonitor.o -C:\Program Files\ebpf-for-windows\testing\bindmonitor.sys -C:\Program Files\ebpf-for-windows\testing\bindmonitor_mt_tailcall.o -C:\Program Files\ebpf-for-windows\testing\bindmonitor_mt_tailcall.sys -C:\Program Files\ebpf-for-windows\testing\bindmonitor_mt_tailcall_um.dll -C:\Program Files\ebpf-for-windows\testing\bindmonitor_mt_tailcall_um.pdb -C:\Program Files\ebpf-for-windows\testing\bindmonitor_ringbuf.o -C:\Program Files\ebpf-for-windows\testing\bindmonitor_ringbuf.sys -C:\Program Files\ebpf-for-windows\testing\bindmonitor_ringbuf_um.dll -C:\Program Files\ebpf-for-windows\testing\bindmonitor_ringbuf_um.pdb -C:\Program Files\ebpf-for-windows\testing\bindmonitor_tailcall.o -C:\Program Files\ebpf-for-windows\testing\bindmonitor_tailcall.sys -C:\Program Files\ebpf-for-windows\testing\bindmonitor_tailcall_um.dll -C:\Program Files\ebpf-for-windows\testing\bindmonitor_tailcall_um.pdb -C:\Program Files\ebpf-for-windows\testing\bindmonitor_um.dll -C:\Program Files\ebpf-for-windows\testing\bindmonitor_um.pdb -C:\Program Files\ebpf-for-windows\testing\bpf.o -C:\Program Files\ebpf-for-windows\testing\bpf.sys -C:\Program Files\ebpf-for-windows\testing\bpftool_test.exe -C:\Program Files\ebpf-for-windows\testing\bpftool_test.pdb -C:\Program Files\ebpf-for-windows\testing\bpf_call.o -C:\Program Files\ebpf-for-windows\testing\bpf_call.sys -C:\Program Files\ebpf-for-windows\testing\cgroup_count_connect4.o -C:\Program Files\ebpf-for-windows\testing\cgroup_count_connect4.pdb -C:\Program Files\ebpf-for-windows\testing\cgroup_count_connect4.sys -C:\Program Files\ebpf-for-windows\testing\cgroup_count_connect4_um.dll -C:\Program Files\ebpf-for-windows\testing\cgroup_count_connect6.o -C:\Program Files\ebpf-for-windows\testing\cgroup_count_connect6.pdb -C:\Program Files\ebpf-for-windows\testing\cgroup_count_connect6.sys -C:\Program Files\ebpf-for-windows\testing\cgroup_count_connect6_um.dll -C:\Program Files\ebpf-for-windows\testing\cgroup_mt_connect4.o -C:\Program Files\ebpf-for-windows\testing\cgroup_mt_connect4.pdb -C:\Program Files\ebpf-for-windows\testing\cgroup_mt_connect4.sys -C:\Program Files\ebpf-for-windows\testing\cgroup_mt_connect4_um.dll -C:\Program Files\ebpf-for-windows\testing\cgroup_mt_connect6.o -C:\Program Files\ebpf-for-windows\testing\cgroup_mt_connect6.pdb -C:\Program Files\ebpf-for-windows\testing\cgroup_mt_connect6.sys -C:\Program Files\ebpf-for-windows\testing\cgroup_mt_connect6_um.dll -C:\Program Files\ebpf-for-windows\testing\cgroup_sock_addr.o -C:\Program Files\ebpf-for-windows\testing\cgroup_sock_addr.sys -C:\Program Files\ebpf-for-windows\testing\cgroup_sock_addr2.o -C:\Program Files\ebpf-for-windows\testing\cgroup_sock_addr2.sys -C:\Program Files\ebpf-for-windows\testing\cgroup_sock_addr2_um.dll -C:\Program Files\ebpf-for-windows\testing\cgroup_sock_addr2_um.pdb -C:\Program Files\ebpf-for-windows\testing\cgroup_sock_addr_um.dll -C:\Program Files\ebpf-for-windows\testing\cgroup_sock_addr_um.pdb -C:\Program Files\ebpf-for-windows\testing\decap_permit_packet.o -C:\Program Files\ebpf-for-windows\testing\decap_permit_packet.sys -C:\Program Files\ebpf-for-windows\testing\divide_by_zero.o -C:\Program Files\ebpf-for-windows\testing\divide_by_zero.sys -C:\Program Files\ebpf-for-windows\testing\divide_by_zero_um.dll -C:\Program Files\ebpf-for-windows\testing\divide_by_zero_um.pdb -C:\Program Files\ebpf-for-windows\testing\droppacket.o -C:\Program Files\ebpf-for-windows\testing\droppacket.sys -C:\Program Files\ebpf-for-windows\testing\droppacket_um.dll -C:\Program Files\ebpf-for-windows\testing\droppacket_um.pdb -C:\Program Files\ebpf-for-windows\testing\droppacket_unsafe.o -C:\Program Files\ebpf-for-windows\testing\ebpfcore_usersim.dll -C:\Program Files\ebpf-for-windows\testing\ebpfcore_usersim.pdb -C:\Program Files\ebpf-for-windows\testing\ebpf_stress_tests_km.exe -C:\Program Files\ebpf-for-windows\testing\ebpf_stress_tests_km.pdb -C:\Program Files\ebpf-for-windows\testing\ebpf_stress_tests_um.exe -C:\Program Files\ebpf-for-windows\testing\ebpf_stress_tests_um.pdb -C:\Program Files\ebpf-for-windows\testing\empty_um.dll -C:\Program Files\ebpf-for-windows\testing\empty_um.pdb -C:\Program Files\ebpf-for-windows\testing\encap_reflect_packet.o -C:\Program Files\ebpf-for-windows\testing\encap_reflect_packet.sys -C:\Program Files\ebpf-for-windows\testing\hash_of_map_um.dll -C:\Program Files\ebpf-for-windows\testing\hash_of_map_um.pdb -C:\Program Files\ebpf-for-windows\testing\hash_of_map.o -C:\Program Files\ebpf-for-windows\testing\hash_of_map.sys -C:\Program Files\ebpf-for-windows\testing\invalid_helpers_um.dll -C:\Program Files\ebpf-for-windows\testing\invalid_helpers_um.pdb -C:\Program Files\ebpf-for-windows\testing\invalid_maps1_um.dll -C:\Program Files\ebpf-for-windows\testing\invalid_maps1_um.pdb -C:\Program Files\ebpf-for-windows\testing\invalid_maps2_um.dll -C:\Program Files\ebpf-for-windows\testing\invalid_maps2_um.pdb -C:\Program Files\ebpf-for-windows\testing\invalid_maps3_um.dll -C:\Program Files\ebpf-for-windows\testing\invalid_maps3_um.pdb -C:\Program Files\ebpf-for-windows\testing\map.o -C:\Program Files\ebpf-for-windows\testing\map.sys -C:\Program Files\ebpf-for-windows\testing\map_in_map_btf.o -C:\Program Files\ebpf-for-windows\testing\map_in_map_btf.sys -C:\Program Files\ebpf-for-windows\testing\map_in_map_btf_um.dll -C:\Program Files\ebpf-for-windows\testing\map_in_map_btf_um.pdb -C:\Program Files\ebpf-for-windows\testing\map_in_map_legacy_id.o -C:\Program Files\ebpf-for-windows\testing\map_in_map_legacy_id.sys -C:\Program Files\ebpf-for-windows\testing\map_in_map_legacy_idx.o -C:\Program Files\ebpf-for-windows\testing\map_in_map_legacy_idx.sys -C:\Program Files\ebpf-for-windows\testing\map_in_map_legacy_idx_um.dll -C:\Program Files\ebpf-for-windows\testing\map_in_map_legacy_idx_um.pdb -C:\Program Files\ebpf-for-windows\testing\map_in_map_legacy_id_um.dll -C:\Program Files\ebpf-for-windows\testing\map_in_map_legacy_id_um.pdb -C:\Program Files\ebpf-for-windows\testing\map_reuse.o -C:\Program Files\ebpf-for-windows\testing\map_reuse.sys -C:\Program Files\ebpf-for-windows\testing\map_reuse_2.o -C:\Program Files\ebpf-for-windows\testing\map_reuse_2.sys -C:\Program Files\ebpf-for-windows\testing\map_reuse_2_um.dll -C:\Program Files\ebpf-for-windows\testing\map_reuse_2_um.pdb -C:\Program Files\ebpf-for-windows\testing\map_reuse_um.dll -C:\Program Files\ebpf-for-windows\testing\map_reuse_um.pdb -C:\Program Files\ebpf-for-windows\testing\map_um.dll -C:\Program Files\ebpf-for-windows\testing\map_um.pdb -C:\Program Files\ebpf-for-windows\testing\pidtgid.o -C:\Program Files\ebpf-for-windows\testing\pidtgid.sys -C:\Program Files\ebpf-for-windows\testing\printk.o -C:\Program Files\ebpf-for-windows\testing\printk.sys -C:\Program Files\ebpf-for-windows\testing\printk_legacy.o -C:\Program Files\ebpf-for-windows\testing\printk_legacy.sys -C:\Program Files\ebpf-for-windows\testing\printk_unsafe.o -C:\Program Files\ebpf-for-windows\testing\reflect_packet.o -C:\Program Files\ebpf-for-windows\testing\reflect_packet.sys -C:\Program Files\ebpf-for-windows\testing\run_tests.bat -C:\Program Files\ebpf-for-windows\testing\sample_ebpf_ext.pdb -C:\Program Files\ebpf-for-windows\testing\sample_ebpf_ext.sys -C:\Program Files\ebpf-for-windows\testing\sample_ext_app.exe -C:\Program Files\ebpf-for-windows\testing\sample_ext_app.pdb -C:\Program Files\ebpf-for-windows\testing\sockops.o -C:\Program Files\ebpf-for-windows\testing\sockops.sys -C:\Program Files\ebpf-for-windows\testing\tail_call.o -C:\Program Files\ebpf-for-windows\testing\tail_call.sys -C:\Program Files\ebpf-for-windows\testing\tail_call_bad.o -C:\Program Files\ebpf-for-windows\testing\tail_call_bad.sys -C:\Program Files\ebpf-for-windows\testing\tail_call_bad_um.dll -C:\Program Files\ebpf-for-windows\testing\tail_call_bad_um.pdb -C:\Program Files\ebpf-for-windows\testing\tail_call_map.o -C:\Program Files\ebpf-for-windows\testing\tail_call_map.sys -C:\Program Files\ebpf-for-windows\testing\tail_call_max_exceed.o -C:\Program Files\ebpf-for-windows\testing\tail_call_max_exceed.sys -C:\Program Files\ebpf-for-windows\testing\tail_call_max_exceed_um.dll -C:\Program Files\ebpf-for-windows\testing\tail_call_max_exceed_um.pdb -C:\Program Files\ebpf-for-windows\testing\tail_call_multiple.o -C:\Program Files\ebpf-for-windows\testing\tail_call_multiple.sys -C:\Program Files\ebpf-for-windows\testing\tail_call_multiple_um.dll -C:\Program Files\ebpf-for-windows\testing\tail_call_multiple_um.pdb -C:\Program Files\ebpf-for-windows\testing\tail_call_recursive.o -C:\Program Files\ebpf-for-windows\testing\tail_call_recursive.sys -C:\Program Files\ebpf-for-windows\testing\tail_call_recursive_um.dll -C:\Program Files\ebpf-for-windows\testing\tail_call_recursive_um.pdb -C:\Program Files\ebpf-for-windows\testing\tail_call_sequential.o -C:\Program Files\ebpf-for-windows\testing\tail_call_sequential.sys -C:\Program Files\ebpf-for-windows\testing\tail_call_sequential_um.dll -C:\Program Files\ebpf-for-windows\testing\tail_call_sequential_um.pdb -C:\Program Files\ebpf-for-windows\testing\tail_call_um.dll -C:\Program Files\ebpf-for-windows\testing\tail_call_um.pdb -C:\Program Files\ebpf-for-windows\testing\test_sample_ebpf.o -C:\Program Files\ebpf-for-windows\testing\test_sample_ebpf.sys -C:\Program Files\ebpf-for-windows\testing\test_utility_helpers.o -C:\Program Files\ebpf-for-windows\testing\test_utility_helpers_um.dll -C:\Program Files\ebpf-for-windows\testing\test_utility_helpers_um.pdb -C:\Program Files\ebpf-for-windows\testing\unit_tests.exe -C:\Program Files\ebpf-for-windows\testing\unit_tests.pdb -C:\Program Files\ebpf-for-windows\testing\usersim.dll -C:\Program Files\ebpf-for-windows\testing\usersim.pdb -C:\Program Files\ebpf-for-windows\testing\xdp_tests.exe -C:\Program Files\ebpf-for-windows\testing\xdp_tests.pdb \ No newline at end of file +C:\Program Files\ebpf-for-windows\JIT\EbpfSvc.pdb \ No newline at end of file From 3c6203e1a7c78d8915520ac609c3f8770f766888 Mon Sep 17 00:00:00 2001 From: Dhiren Vispute <86131363+dv-msft@users.noreply.github.com> Date: Wed, 13 Mar 2024 15:16:27 -0700 Subject: [PATCH 02/10] Use github-specified primary repository for CI/CD workflow. (#3345) Co-authored-by: Dhiren Vispute --- .github/workflows/cicd.yml | 5 +++++ .github/workflows/reusable-build.yml | 6 +++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 0ec34f5dba..29f3643f76 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -49,6 +49,7 @@ jobs: uses: ./.github/workflows/reusable-build.yml with: ref: ${{ github.ref }} + repository: ${{ github.repository }} build_artifact: Build-x64 generate_release_package: true build_msi: true @@ -63,6 +64,7 @@ jobs: uses: ./.github/workflows/reusable-build.yml with: ref: ${{ github.ref }} + repository: ${{ github.repository }} build_artifact: Build-x64-native-only build_msi: true build_nuget: true @@ -266,6 +268,7 @@ jobs: uses: ./.github/workflows/reusable-build.yml with: ref: ${{ github.ref }} + repository: ${{ github.repository }} build_artifact: Build-x64-Analyze # Analysis on external projects is conditional, as on small CI/CD VMs the compiler can run OOM build_options: /p:Analysis='True' /p:AnalysisOnExternal='False' @@ -277,6 +280,7 @@ jobs: uses: ./.github/workflows/reusable-build.yml with: ref: ${{ github.ref }} + repository: ${{ github.repository }} build_artifact: Build-x64-Sanitize build_options: /p:AddressSanitizer='True' @@ -476,6 +480,7 @@ jobs: uses: ./.github/workflows/reusable-build.yml with: ref: ${{ github.ref }} + repository: ${{ github.repository }} build_artifact: Build-x64-CodeQl build_codeql: true diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index db19b78786..96d7cf8a63 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -11,6 +11,10 @@ on: ref: required: true type: string + # repository to be used (needed for self-hosted runner setups) + repository: + required: true + type: string # Name associated with the output of this build. build_artifact: required: true @@ -87,7 +91,7 @@ jobs: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 if: steps.skip_check.outputs.should_skip != 'true' with: - repository: microsoft/ebpf-for-windows + repository: ${{inputs.repository}} submodules: 'recursive' ref: ${{inputs.ref}} From 37613367a007b6340d5e40d0fbb900f6851a2eb2 Mon Sep 17 00:00:00 2001 From: Anurag Saxena <43585259+saxena-anurag@users.noreply.github.com> Date: Fri, 15 Mar 2024 17:16:14 -0700 Subject: [PATCH 03/10] Fix ring_buf issues (#3354) * fix * add tests * code cleanup * fix tests * cr comments * cr comments --- libs/api/ebpf_api.cpp | 2 +- libs/runtime/ebpf_ring_buffer.c | 29 ++++++-- libs/runtime/ebpf_ring_buffer.h | 2 +- libs/shared/ebpf_ring_buffer_record.h | 1 + tests/api_test/api_test.cpp | 73 ++++++++++++++++++- .../expected/bindmonitor_ringbuf_dll.c | 6 +- .../expected/bindmonitor_ringbuf_raw.c | 6 +- .../expected/bindmonitor_ringbuf_sys.c | 6 +- tests/sample/bindmonitor_ringbuf.c | 2 +- 9 files changed, 108 insertions(+), 19 deletions(-) diff --git a/libs/api/ebpf_api.cpp b/libs/api/ebpf_api.cpp index 00ab929e22..ec9be180a1 100644 --- a/libs/api/ebpf_api.cpp +++ b/libs/api/ebpf_api.cpp @@ -4177,7 +4177,7 @@ _ebpf_ring_buffer_map_async_query_completion(_Inout_ void* completion_context) N // Async IOCTL operation returned with success status. Read the ring buffer records and indicate it to the // subscriber. - size_t ring_buffer_size; + size_t ring_buffer_size = 0; uint32_t dummy; result = _get_map_descriptor_properties( diff --git a/libs/runtime/ebpf_ring_buffer.c b/libs/runtime/ebpf_ring_buffer.c index 7fedec637e..abc1db4bc1 100644 --- a/libs/runtime/ebpf_ring_buffer.c +++ b/libs/runtime/ebpf_ring_buffer.c @@ -34,6 +34,13 @@ _ring_get_consumer_offset(_In_ const ebpf_ring_buffer_t* ring) return ring->consumer_offset % ring->length; } +inline static size_t +_ring_get_used_capacity(_In_ const ebpf_ring_buffer_t* ring) +{ + ebpf_assert(ring->producer_offset >= ring->consumer_offset); + return ring->producer_offset - ring->consumer_offset; +} + inline static void _ring_advance_producer_offset(_Inout_ ebpf_ring_buffer_t* ring, size_t length) { @@ -146,23 +153,30 @@ ebpf_ring_buffer_output(_Inout_ ebpf_ring_buffer_t* ring, _In_reads_bytes_(lengt } void -ebpf_ring_buffer_query(_In_ const ebpf_ring_buffer_t* ring, _Out_ size_t* consumer, _Out_ size_t* producer) +ebpf_ring_buffer_query(_In_ ebpf_ring_buffer_t* ring, _Out_ size_t* consumer, _Out_ size_t* producer) { + ebpf_lock_state_t state = ebpf_lock_lock(&ring->lock); *consumer = ring->consumer_offset; *producer = ring->producer_offset; + ebpf_lock_unlock(&ring->lock, state); } _Must_inspect_result_ ebpf_result_t ebpf_ring_buffer_return(_Inout_ ebpf_ring_buffer_t* ring, size_t length) { + EBPF_LOG_ENTRY(); ebpf_result_t result; ebpf_lock_state_t state = ebpf_lock_lock(&ring->lock); size_t local_length = length; size_t offset = _ring_get_consumer_offset(ring); - // Check if length is valid. - if ((length > _ring_get_length(ring)) || - (length + _ring_get_consumer_offset(ring) > _ring_get_producer_offset(ring))) { + if ((length > _ring_get_length(ring)) || length > _ring_get_used_capacity(ring)) { + EBPF_LOG_MESSAGE_UINT64_UINT64( + EBPF_TRACELOG_LEVEL_ERROR, + EBPF_TRACELOG_KEYWORD_MAP, + "ebpf_ring_buffer_return: Buffer too large", + ring->producer_offset, + ring->consumer_offset); result = EBPF_INVALID_ARGUMENT; goto Done; } @@ -178,6 +192,11 @@ ebpf_ring_buffer_return(_Inout_ ebpf_ring_buffer_t* ring, size_t length) } // Did it end on a record boundary? if (local_length != 0) { + EBPF_LOG_MESSAGE_UINT64( + EBPF_TRACELOG_LEVEL_ERROR, + EBPF_TRACELOG_KEYWORD_MAP, + "ebpf_ring_buffer_return: Invalid buffer length", + local_length); result = EBPF_INVALID_ARGUMENT; goto Done; } @@ -187,7 +206,7 @@ ebpf_ring_buffer_return(_Inout_ ebpf_ring_buffer_t* ring, size_t length) Done: ebpf_lock_unlock(&ring->lock, state); - return result; + EBPF_RETURN_RESULT(result); } _Must_inspect_result_ ebpf_result_t diff --git a/libs/runtime/ebpf_ring_buffer.h b/libs/runtime/ebpf_ring_buffer.h index 9a8343dc21..4a72d81ce1 100644 --- a/libs/runtime/ebpf_ring_buffer.h +++ b/libs/runtime/ebpf_ring_buffer.h @@ -49,7 +49,7 @@ ebpf_ring_buffer_output(_Inout_ ebpf_ring_buffer_t* ring_buffer, _In_reads_bytes * @param[out] producer Offset of the next buffer to be produced. */ void -ebpf_ring_buffer_query(_In_ const ebpf_ring_buffer_t* ring_buffer, _Out_ size_t* consumer, _Out_ size_t* producer); +ebpf_ring_buffer_query(_In_ ebpf_ring_buffer_t* ring_buffer, _Out_ size_t* consumer, _Out_ size_t* producer); /** * @brief Mark one or more records in the ring buffer as returned to the ring. diff --git a/libs/shared/ebpf_ring_buffer_record.h b/libs/shared/ebpf_ring_buffer_record.h index 019eb24ef1..8d5dc11c92 100644 --- a/libs/shared/ebpf_ring_buffer_record.h +++ b/libs/shared/ebpf_ring_buffer_record.h @@ -29,6 +29,7 @@ typedef struct _ebpf_ring_buffer_record inline const ebpf_ring_buffer_record_t* ebpf_ring_buffer_next_record(_In_ const uint8_t* buffer, size_t buffer_length, size_t consumer, size_t producer) { + ebpf_assert(producer >= consumer); if (producer == consumer) { return NULL; } diff --git a/tests/api_test/api_test.cpp b/tests/api_test/api_test.cpp index 5fcfa9b1b9..57b7f9d409 100644 --- a/tests/api_test/api_test.cpp +++ b/tests/api_test/api_test.cpp @@ -1121,6 +1121,7 @@ TEST_CASE("load_native_program_invalid5", "[native][negative]") { _load_invalid_program("invalid_maps3.sys", EBPF_EXECUTION_NATIVE, -EINVAL); } +#endif // _DEBUG TEST_CASE("ioctl_stress", "[stress]") { @@ -1211,7 +1212,7 @@ TEST_CASE("ioctl_stress", "[stress]") } } - // Wait for 10 seconds + // Wait for 60 seconds std::this_thread::sleep_for(std::chrono::seconds(60)); stop_requested = true; @@ -1228,4 +1229,72 @@ TEST_CASE("ioctl_stress", "[stress]") bpf_object__close(object); } -#endif \ No newline at end of file +TEST_CASE("test_ringbuffer_wraparound", "[stress]") +{ + // Load bindmonitor_ringbuf.sys. + struct bpf_object* object = nullptr; + fd_t program_fd; + uint32_t event_count = 0; + std::string app_id = "api_test.exe"; + uint32_t thread_count = 2; + native_module_helper_t native_helper; + native_helper.initialize("bindmonitor_ringbuf", EBPF_EXECUTION_NATIVE); + + REQUIRE( + _program_load_helper( + native_helper.get_file_name().c_str(), BPF_PROG_TYPE_BIND, EBPF_EXECUTION_NATIVE, &object, &program_fd) == + 0); + + // Get fd of process_map map. + fd_t process_map_fd = bpf_object__find_map_fd_by_name(object, "process_map"); + + uint32_t max_entries = bpf_map__max_entries(bpf_object__find_map_by_name(object, "process_map")); + uint32_t max_iterations = static_cast(10 * (max_entries / app_id.size())); + printf("max_iterations: %d\n", max_iterations); + REQUIRE(max_iterations % thread_count == 0); + uint32_t iterations_per_thread = max_iterations / thread_count; + + // Subscribe to the ring buffer. + auto ring = ring_buffer__new( + process_map_fd, + [](void* ctx, void*, size_t) { + (*((uint32_t*)ctx))++; + return 0; + }, + &event_count, + nullptr); + + // Create 2 threads that invoke the program to trigger ring buffer events. + std::vector threads; + for (uint32_t i = 0; i < thread_count; i++) { + threads.emplace_back([&]() { + bind_md_t ctx = {}; + bpf_test_run_opts opts = {}; + opts.ctx_in = &ctx; + opts.ctx_size_in = sizeof(ctx); + opts.ctx_out = &ctx; + opts.ctx_size_out = sizeof(ctx); + opts.data_in = app_id.data(); + opts.data_size_in = static_cast(app_id.size()); + opts.data_out = app_id.data(); + opts.data_size_out = static_cast(app_id.size()); + + for (uint32_t i = 0; i < iterations_per_thread; i++) { + int result = bpf_prog_test_run_opts(program_fd, &opts); + REQUIRE(result == 0); + } + }); + } + + // Wait for threads to complete. + for (auto& t : threads) { + t.join(); + } + REQUIRE(event_count == max_iterations); + + // Unsubscribe from the ring buffer. + ring_buffer__free(ring); + + // Clean up. + bpf_object__close(object); +} \ No newline at end of file diff --git a/tests/bpf2c_tests/expected/bindmonitor_ringbuf_dll.c b/tests/bpf2c_tests/expected/bindmonitor_ringbuf_dll.c index 66ec50fd54..1580177d48 100644 --- a/tests/bpf2c_tests/expected/bindmonitor_ringbuf_dll.c +++ b/tests/bpf2c_tests/expected/bindmonitor_ringbuf_dll.c @@ -45,7 +45,7 @@ static map_entry_t _maps[] = { BPF_MAP_TYPE_RINGBUF, // Type of map. 0, // Size in bytes of a map key. 0, // Size in bytes of a map value. - 262144, // Maximum number of entries allowed in the map. + 65536, // Maximum number of entries allowed in the map. 0, // Inner map index. LIBBPF_PIN_NONE, // Pinning type for the map. 7, // Identifier for a map template. @@ -111,7 +111,7 @@ bind_monitor(void* context) if (r2 != IMMEDIATE(0)) #line 26 "sample/bindmonitor_ringbuf.c" goto label_1; - // EBPF_OP_LDXDW pc=2 dst=r2 src=r1 offset=0 imm=0 + // EBPF_OP_LDXDW pc=2 dst=r2 src=r1 offset=0 imm=0 #line 28 "sample/bindmonitor_ringbuf.c" r2 = *(uint64_t*)(uintptr_t)(r1 + OFFSET(0)); // EBPF_OP_LDXDW pc=3 dst=r3 src=r1 offset=8 imm=0 @@ -122,7 +122,7 @@ bind_monitor(void* context) if (r2 >= r3) #line 28 "sample/bindmonitor_ringbuf.c" goto label_1; - // EBPF_OP_SUB64_REG pc=5 dst=r3 src=r2 offset=0 imm=0 + // EBPF_OP_SUB64_REG pc=5 dst=r3 src=r2 offset=0 imm=0 #line 29 "sample/bindmonitor_ringbuf.c" r3 -= r2; // EBPF_OP_LDDW pc=6 dst=r1 src=r0 offset=0 imm=0 diff --git a/tests/bpf2c_tests/expected/bindmonitor_ringbuf_raw.c b/tests/bpf2c_tests/expected/bindmonitor_ringbuf_raw.c index dbcceb11a2..0fd79e883a 100644 --- a/tests/bpf2c_tests/expected/bindmonitor_ringbuf_raw.c +++ b/tests/bpf2c_tests/expected/bindmonitor_ringbuf_raw.c @@ -19,7 +19,7 @@ static map_entry_t _maps[] = { BPF_MAP_TYPE_RINGBUF, // Type of map. 0, // Size in bytes of a map key. 0, // Size in bytes of a map value. - 262144, // Maximum number of entries allowed in the map. + 65536, // Maximum number of entries allowed in the map. 0, // Inner map index. LIBBPF_PIN_NONE, // Pinning type for the map. 7, // Identifier for a map template. @@ -85,7 +85,7 @@ bind_monitor(void* context) if (r2 != IMMEDIATE(0)) #line 26 "sample/bindmonitor_ringbuf.c" goto label_1; - // EBPF_OP_LDXDW pc=2 dst=r2 src=r1 offset=0 imm=0 + // EBPF_OP_LDXDW pc=2 dst=r2 src=r1 offset=0 imm=0 #line 28 "sample/bindmonitor_ringbuf.c" r2 = *(uint64_t*)(uintptr_t)(r1 + OFFSET(0)); // EBPF_OP_LDXDW pc=3 dst=r3 src=r1 offset=8 imm=0 @@ -96,7 +96,7 @@ bind_monitor(void* context) if (r2 >= r3) #line 28 "sample/bindmonitor_ringbuf.c" goto label_1; - // EBPF_OP_SUB64_REG pc=5 dst=r3 src=r2 offset=0 imm=0 + // EBPF_OP_SUB64_REG pc=5 dst=r3 src=r2 offset=0 imm=0 #line 29 "sample/bindmonitor_ringbuf.c" r3 -= r2; // EBPF_OP_LDDW pc=6 dst=r1 src=r0 offset=0 imm=0 diff --git a/tests/bpf2c_tests/expected/bindmonitor_ringbuf_sys.c b/tests/bpf2c_tests/expected/bindmonitor_ringbuf_sys.c index 69ff793b2f..db62b78c5f 100644 --- a/tests/bpf2c_tests/expected/bindmonitor_ringbuf_sys.c +++ b/tests/bpf2c_tests/expected/bindmonitor_ringbuf_sys.c @@ -180,7 +180,7 @@ static map_entry_t _maps[] = { BPF_MAP_TYPE_RINGBUF, // Type of map. 0, // Size in bytes of a map key. 0, // Size in bytes of a map value. - 262144, // Maximum number of entries allowed in the map. + 65536, // Maximum number of entries allowed in the map. 0, // Inner map index. LIBBPF_PIN_NONE, // Pinning type for the map. 7, // Identifier for a map template. @@ -246,7 +246,7 @@ bind_monitor(void* context) if (r2 != IMMEDIATE(0)) #line 26 "sample/bindmonitor_ringbuf.c" goto label_1; - // EBPF_OP_LDXDW pc=2 dst=r2 src=r1 offset=0 imm=0 + // EBPF_OP_LDXDW pc=2 dst=r2 src=r1 offset=0 imm=0 #line 28 "sample/bindmonitor_ringbuf.c" r2 = *(uint64_t*)(uintptr_t)(r1 + OFFSET(0)); // EBPF_OP_LDXDW pc=3 dst=r3 src=r1 offset=8 imm=0 @@ -257,7 +257,7 @@ bind_monitor(void* context) if (r2 >= r3) #line 28 "sample/bindmonitor_ringbuf.c" goto label_1; - // EBPF_OP_SUB64_REG pc=5 dst=r3 src=r2 offset=0 imm=0 + // EBPF_OP_SUB64_REG pc=5 dst=r3 src=r2 offset=0 imm=0 #line 29 "sample/bindmonitor_ringbuf.c" r3 -= r2; // EBPF_OP_LDDW pc=6 dst=r1 src=r0 offset=0 imm=0 diff --git a/tests/sample/bindmonitor_ringbuf.c b/tests/sample/bindmonitor_ringbuf.c index 166e4c86d0..3a65fa18d2 100644 --- a/tests/sample/bindmonitor_ringbuf.c +++ b/tests/sample/bindmonitor_ringbuf.c @@ -16,7 +16,7 @@ struct { __uint(type, BPF_MAP_TYPE_RINGBUF); - __uint(max_entries, 256 * 1024); + __uint(max_entries, 64 * 1024); } process_map SEC(".maps"); SEC("bind") From 5813961ab9a7579cc625eb725fc5ae2d26ec2f18 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 08:40:50 -0700 Subject: [PATCH 04/10] Bump external/Catch2 from `8ac8190` to `bfe3ff8` (#3366) Bumps [external/Catch2](https://github.com/catchorg/Catch2) from `8ac8190` to `bfe3ff8`. - [Release notes](https://github.com/catchorg/Catch2/releases) - [Commits](https://github.com/catchorg/Catch2/compare/8ac8190e494a381072c89f5e161b92a08d98b37b...bfe3ff8f19d3436381b4e3a18c13734d21a8b089) --- updated-dependencies: - dependency-name: external/Catch2 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- external/Catch2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/Catch2 b/external/Catch2 index 8ac8190e49..bfe3ff8f19 160000 --- a/external/Catch2 +++ b/external/Catch2 @@ -1 +1 @@ -Subproject commit 8ac8190e494a381072c89f5e161b92a08d98b37b +Subproject commit bfe3ff8f19d3436381b4e3a18c13734d21a8b089 From 06c38b54992ae262d811461b381da89cc9e18ee0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 09:22:22 -0700 Subject: [PATCH 05/10] Bump peter-evans/create-pull-request from 6.0.1 to 6.0.2 (#3363) Bumps [peter-evans/create-pull-request](https://github.com/peter-evans/create-pull-request) from 6.0.1 to 6.0.2. - [Release notes](https://github.com/peter-evans/create-pull-request/releases) - [Commits](https://github.com/peter-evans/create-pull-request/compare/a4f52f8033a6168103c2538976c07b467e8163bc...70a41aba780001da0a30141984ae2a0c95d8704e) --- updated-dependencies: - dependency-name: peter-evans/create-pull-request dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/nuget_update.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nuget_update.yaml b/.github/workflows/nuget_update.yaml index f5f62d9b63..5913354ab8 100644 --- a/.github/workflows/nuget_update.yaml +++ b/.github/workflows/nuget_update.yaml @@ -61,7 +61,7 @@ jobs: nuget update -noninteractive -verbosity detailed ${{env.SOLUTION_FILE_PATH}} - name: Create Pull Request - uses: peter-evans/create-pull-request@a4f52f8033a6168103c2538976c07b467e8163bc + uses: peter-evans/create-pull-request@70a41aba780001da0a30141984ae2a0c95d8704e with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: Update NuGet packages From b319f56bb7b49f034afc02689c8770cb30ef15d1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Mar 2024 13:50:02 -0700 Subject: [PATCH 06/10] Bump actions/checkout from 4.1.1 to 4.1.2 (#3362) Bumps [actions/checkout](https://github.com/actions/checkout) from 4.1.1 to 4.1.2. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/b4ffde65f46336ab88eb53be808477a3936bae11...9bb56186c3b09b4f86b1c65136769dd318469633) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/dependency-review.yml | 2 +- .github/workflows/nuget_update.yaml | 2 +- .github/workflows/ossar-scan.yml | 2 +- .github/workflows/reusable-build.yml | 2 +- .github/workflows/reusable-test.yml | 4 ++-- .github/workflows/scorecards-analysis.yml | 2 +- .github/workflows/update-docs.yml | 2 +- 7 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index 49d1efcbe0..1cadba96e1 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -22,6 +22,6 @@ jobs: egress-policy: audit - name: 'Checkout Repository' - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: 'Dependency Review' uses: actions/dependency-review-action@9129d7d40b8c12c1ed0f60400d00c92d437adcce # v4.1.3 diff --git a/.github/workflows/nuget_update.yaml b/.github/workflows/nuget_update.yaml index 5913354ab8..03908e4954 100644 --- a/.github/workflows/nuget_update.yaml +++ b/.github/workflows/nuget_update.yaml @@ -37,7 +37,7 @@ jobs: with: egress-policy: audit - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 with: # Only check out main repo, not submodules. ref: ${{ github.event.workflow_run.head_branch }} diff --git a/.github/workflows/ossar-scan.yml b/.github/workflows/ossar-scan.yml index 01eebdb68a..d3e0e57ff8 100644 --- a/.github/workflows/ossar-scan.yml +++ b/.github/workflows/ossar-scan.yml @@ -47,7 +47,7 @@ jobs: paths_ignore: '["**.md", "**/docs/**"]' # Checking out the branch is needed to correctly log security alerts. - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 if: steps.skip_check.outputs.should_skip != 'true' with: # Only check out main repo, not submodules. diff --git a/.github/workflows/reusable-build.yml b/.github/workflows/reusable-build.yml index 96d7cf8a63..46ac285555 100644 --- a/.github/workflows/reusable-build.yml +++ b/.github/workflows/reusable-build.yml @@ -88,7 +88,7 @@ jobs: powershell.exe "echo 'msvc_tools_version=%VCToolsVersion%' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append" powershell.exe "echo 'ASAN_WIN_CONTINUE_ON_INTERCEPTION_FAILURE=true' | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 if: steps.skip_check.outputs.should_skip != 'true' with: repository: ${{inputs.repository}} diff --git a/.github/workflows/reusable-test.yml b/.github/workflows/reusable-test.yml index 8a268542ff..c792428192 100644 --- a/.github/workflows/reusable-test.yml +++ b/.github/workflows/reusable-test.yml @@ -96,7 +96,7 @@ jobs: paths_ignore: '["**.md", "**/docs/**"]' # Checking out the branch is needed to gather correct code coverage data. - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # Only check out source code if code coverage is being gathered. if: (inputs.code_coverage == true) && (steps.skip_check.outputs.should_skip != 'true') with: @@ -104,7 +104,7 @@ jobs: ref: ${{ github.event.workflow_run.head_branch }} # Perform shallow checkout for self-hosted runner. - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 if: (inputs.environment == 'ebpf_cicd_tests_ws2019' || inputs.environment == 'ebpf_cicd_tests_ws2022' || inputs.environment == 'ebpf_cicd_perf_ws2022') && (steps.skip_check.outputs.should_skip != 'true') with: ref: ${{ github.event.workflow_run.head_branch }} diff --git a/.github/workflows/scorecards-analysis.yml b/.github/workflows/scorecards-analysis.yml index 86e5b4dbe7..2bbe2ced6a 100644 --- a/.github/workflows/scorecards-analysis.yml +++ b/.github/workflows/scorecards-analysis.yml @@ -45,7 +45,7 @@ jobs: - name: "Checkout code" if: github.ref_name == 'main' - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 with: persist-credentials: false diff --git a/.github/workflows/update-docs.yml b/.github/workflows/update-docs.yml index a99fe0cf4f..4ec903559d 100644 --- a/.github/workflows/update-docs.yml +++ b/.github/workflows/update-docs.yml @@ -41,7 +41,7 @@ jobs: run: | sudo apt install doxygen - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 - name: Clone docs run: | From f8be910986c9a10a75aa639fb15b4c8ca47717f8 Mon Sep 17 00:00:00 2001 From: Kevin Ji <1146876+kevinji@users.noreply.github.com> Date: Mon, 18 Mar 2024 16:49:12 -0700 Subject: [PATCH 07/10] docs: Mark C/JSON/XML code, remove stray backticks (#3353) --- docs/DevelopmentGuide.md | 2 +- docs/InstallEbpf.md | 4 ++-- docs/NativeCodeGeneration.md | 8 ++++---- docs/TroubleshootingGuide.md | 19 +++++++++---------- docs/debugging.md | 2 +- docs/eBpfExtensions.md | 16 ++++++++-------- docs/tutorial.md | 24 ++++++++++++------------ 7 files changed, 37 insertions(+), 38 deletions(-) diff --git a/docs/DevelopmentGuide.md b/docs/DevelopmentGuide.md index 403aadfdfe..98d2eb4be8 100644 --- a/docs/DevelopmentGuide.md +++ b/docs/DevelopmentGuide.md @@ -91,7 +91,7 @@ If you see unexpected formatting changes in the code, verify that you are runnin The following license header **must** be included at the top of every code file: -``` +```c // Copyright (c) Microsoft Corporation // SPDX-License-Identifier: MIT ``` diff --git a/docs/InstallEbpf.md b/docs/InstallEbpf.md index 48c53dc770..56b3e78add 100644 --- a/docs/InstallEbpf.md +++ b/docs/InstallEbpf.md @@ -176,7 +176,7 @@ Start an admin Powershell on the Windows Host and run the following command and ```ps .\images\build-images.ps1 - ```` + ``` * To **build the image on a Linux machine** (e.g. Ubuntu), make sure docker is installed (see [install docker on Ubuntu](https://docs.docker.com/engine/install/ubuntu/)), and do the following: @@ -184,7 +184,7 @@ Start an admin Powershell on the Windows Host and run the following command and ```bash $HOME/ebpf-for-windows-image/build-images.sh - ```` + ``` 3. Push the `ebpf-for-windows` image to your repository. diff --git a/docs/NativeCodeGeneration.md b/docs/NativeCodeGeneration.md index 2d5945547c..4abd391073 100644 --- a/docs/NativeCodeGeneration.md +++ b/docs/NativeCodeGeneration.md @@ -94,7 +94,7 @@ required, and to import the helper functions. Every generated C file contains a single global entry point of type metadata_table_t: -``` +```c typedef struct _metadata_table { void (*programs)(program_entry_t** programs, size_t* count); @@ -111,7 +111,7 @@ is not valid in a C variable name. This variable is the only globally visible va ## Exported programs Each program in the generated C file is exported via a program_entry_t: -``` +```c typedef struct _program_entry { uint64_t (*function)(void*); @@ -138,7 +138,7 @@ The skeleton framework then uses NMR to publish this information to the eBPF exe The generated C code exposes a table containing the address of each helper function, name, ID, and additional meta-data of the helper function. The C code generator emits a table for the helper functions referenced by the program: -``` +```c typedef struct _helper_function_entry { uint64_t (*address)(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); @@ -161,7 +161,7 @@ in the generated code are called indirectly via the address field. Each map referenced by any of the eBPF programs is added as a map_entry_t: -``` +```c typedef struct _map_entry { void* address; diff --git a/docs/TroubleshootingGuide.md b/docs/TroubleshootingGuide.md index aaad56f740..ca12fd829a 100644 --- a/docs/TroubleshootingGuide.md +++ b/docs/TroubleshootingGuide.md @@ -95,7 +95,7 @@ invoked, see [troubleshooting eBPF program issues](#Troubleshooting-general-eBPF ### Expected WFP objects for the program type BPF_PROG_TYPE_CGROUP_SOCK_ADDR The following are the expected `sublayer` objects for this program type: -``` +```xml {7c7b3fb9-3331-436a-98e1-b901df457fff} @@ -141,7 +141,7 @@ In this scenario, the `FWPM_LAYER_ALE_CONNECT_REDIRECT_V6` layer objects are nec handles dual-stack sockets. This is the expected `callout` and `filter` at the `FWPM_LAYER_ALE_CONNECT_REDIRECT_V4` layer: -``` +```xml {98849e0f-b07d-11ec-9a30-18602489beee} @@ -186,7 +186,7 @@ This is the expected `callout` and `filter` at the `FWPM_LAYER_ALE_CONNECT_REDIR ``` This is the expected `callout` and `filter` at the `FWPM_LAYER_ALE_CONNECT_REDIRECT_V6` layer: -``` +```xml {98849e10-b07d-11ec-9a30-18602489beee} @@ -232,7 +232,7 @@ This is the expected `callout` and `filter` at the `FWPM_LAYER_ALE_CONNECT_REDIR ``` This is the expected `callout` and `filter` at the `FWPM_LAYER_ALE_AUTH_CONNECT_V4` layer: -``` +```xml {98849e0b-b07d-11ec-9a30-18602489beee} @@ -298,7 +298,7 @@ In this output, check that you see the expected eBPF program, looking at the `na and `map_ids` for the next set of checks. Example Output: -``` +```json [{ "id": 196867, "type": "sock_addr", @@ -319,8 +319,7 @@ In this output, check for an entry with the `prog_id` which matches the `id` fro the `attach_type` is as expected. Example output: -``` -bpftool.exe -p link +```json [{ "id": 262403, "type": 2, @@ -341,7 +340,7 @@ bpftool.exe -p map show id In this output, use the `map_ids` from the above output. Map usage is up to the eBPF program developer, so you should confirm that the `type` and `name` is as expected for the scenario. This example output is from invoking the bpftool for each map: -``` +```json { "id": 66054, "type": "hash", @@ -519,7 +518,7 @@ callout here. If you see more than 1, then another WFP callout driver may be att that your eBPF program is, which may affect the final connection. Sample output: -``` +```xml {98849e0f-b07d-11ec-9a30-18602489beee} @@ -557,4 +556,4 @@ Sample output: uninstall or disable the other WFP callouts. Note that the `name` field in the `wfpstate` output may differ from the actual driver or product name. --------------------- \ No newline at end of file +-------------------- diff --git a/docs/debugging.md b/docs/debugging.md index 6ab7377bf7..3557d7afd7 100644 --- a/docs/debugging.md +++ b/docs/debugging.md @@ -262,7 +262,7 @@ Disassembly of section .text: We see that the code the ntohs() function in [tests\sample\ebpf.h](../tests/sample/ebpf.h): -``` +```c uint16_t ntohs(uint16_t us) { diff --git a/docs/eBpfExtensions.md b/docs/eBpfExtensions.md index 567232c239..54d1b7d9bd 100644 --- a/docs/eBpfExtensions.md +++ b/docs/eBpfExtensions.md @@ -100,7 +100,7 @@ invoking an eBPF program. The various fields of this struct are as follows. * `meta`: Offset (in bytes) to the field in the context structure that is pointing to the beginning of context metadata. For example, for the XDP_TEST program types, the context data structure is as follows: -``` +```c // XDP_TEST hook. We use "struct xdp_md" for cross-platform compatibility. typedef struct xdp_md { @@ -113,7 +113,7 @@ typedef struct xdp_md } xdp_md_t; ``` The corresponding context descriptor looks like: -``` +```c const ebpf_context_descriptor_t g_xdp_context_descriptor = {sizeof(xdp_md_t), EBPF_OFFSET_OF(xdp_md_t, data), EBPF_OFFSET_OF(xdp_md_t, data_end), @@ -134,7 +134,7 @@ helper function. #### `ebpf_argument_type_t` Enum This enum describes the various argument types that can be passed to an eBPF helper function. This is defined in the [PREVAIL Verifier](https://github.com/vbpf/ebpf-verifier) project. -``` +```c typedef enum _ebpf_argument_type { EBPF_ARGUMENT_TYPE_DONTCARE = 0, EBPF_ARGUMENT_TYPE_ANYTHING, @@ -154,7 +154,7 @@ typedef enum _ebpf_argument_type { #### `ebpf_return_type` Enum This enum describes the various return types from an eBPF helper function. This is defined in the [PREVAIL Verifier](https://github.com/vbpf/ebpf-verifier) project. -``` +```c typedef enum _ebpf_return_type { EBPF_RETURN_TYPE_INTEGER = 0, EBPF_RETURN_TYPE_PTR_TO_MAP_VALUE_OR_NULL, @@ -235,7 +235,7 @@ specific data that may be used by an extension for attaching an eBPF program. Fo being attached to an XDP_TEST hook, the network interface index can be passed via this parameter. This tells the extension to invoke the eBPF program whenever there are any inbound packets on that network interface. The attach parameter can be obtained as follows: -``` +```c ebpf_extension_data_t* extension_data = (ebpf_extension_data_t*)ClientRegistrationInstance->NpiSpecificCharacteristics; attach_parameter = extension_data->data; ``` @@ -250,7 +250,7 @@ the provider must free the per-client context passed in via `ProviderBindingCont To invoke an eBPF program, the extension uses the dispatch table supplied by the Hook NPI client during attaching. The client dispatch table contains the functions, with the following type prototypes: -``` +```c /** * @brief Invoke the eBPF program. * @@ -309,7 +309,7 @@ typedef ebpf_result_t (*ebpf_program_batch_end_invoke_function_t)( ``` The function pointer can be obtained from the client dispatch table as follows: -``` +```c invoke_program = (ebpf_program_invoke_function_t)client_dispatch_table->function[0]; ``` When an extension invokes this function pointer, then the call flows through the eBPF Execution Context and eventually @@ -344,7 +344,7 @@ itself. However, if a program type so chooses, it may provide implementations fo the extension would have to provide another Program Information NPI provider, which *does not* provide any program context descriptor. Instead, it only supplies the prototypes and addresses of the general helper functions. The NPI ID of this module defined as: -``` +```c GUID ebpf_general_helper_function_module_id = {/* 8d2a1d3f-9ce6-473d-b48e-17aa5c5581fe */ 0x8d2a1d3f, 0x9ce6, diff --git a/docs/tutorial.md b/docs/tutorial.md index f228194436..ca6be90115 100644 --- a/docs/tutorial.md +++ b/docs/tutorial.md @@ -25,7 +25,7 @@ However, we'll do this walkthrough assuming one is only using Windows. **Step 1)** Author a new file by putting some content into a file, say `bpf.c`: -``` +```c int func() { return 0; @@ -166,7 +166,7 @@ reached. In this way, there can even be multiple sections per source file. Author a new file, say in `bpf2.c` this time, with another function and a pragma above each one: -``` +```c #pragma clang section text="myprog" int func() @@ -493,7 +493,7 @@ hook-specific context structure passed as one argument.) The "xdp_test" hook point has the following prototype in `ebpf_nethooks.h`: -``` +```c typedef struct xdp_md { void* data; // Pointer to start of packet data. @@ -514,7 +514,7 @@ typedef xdp_action_t xdp_hook_t(xdp_md_t* context); A sample eBPF program might look like this: -``` +```c #include "bpf_helpers.h" #include "ebpf_nethooks.h" @@ -596,7 +596,7 @@ What would have happened had the prototype not matched? Let's say the verifier is the same as above but XDP_TEST instead had a different struct definition: -``` +```c typedef struct _xdp_md_t { uint64_t more; @@ -641,7 +641,7 @@ can vary by platform. For comparison, helpers for Linux are documented in the Let's say the following helper function prototype is exposed by Windows: -``` +```c #define EBPF_HELPER(return_type, name, args) typedef return_type(*name##_t) args EBPF_HELPER(int64_t, bpf_map_update_elem, (struct bpf_map * map, void* key, void* value, uint64_t flags)); ``` @@ -649,7 +649,7 @@ EBPF_HELPER(int64_t, bpf_map_update_elem, (struct bpf_map * map, void* key, void We'll cover in section 6.3 what this function does, but for now we only care about the prototype. We can create a sample (but, as we will see, invalid) program like so: -``` +```c #include "bpf_helpers.h" int func() @@ -758,7 +758,7 @@ of eBPF programs, or to expose information (e.g., statistics) to applications. To see how maps are exposed to eBPF programs, let's first start from a plain eBPF program: -``` +```c SEC("myprog") int func() { @@ -773,7 +773,7 @@ if multiple instances of our program are simultaneously running on different CPUs. -``` +```c #include "bpf_helpers.h" struct { @@ -810,7 +810,7 @@ Contents of section maps: ``` Now to make use of the map, we have to use helper functions to access it: -``` +```c void *bpf_map_lookup_elem(struct bpf_map* map, const void* key); int bpf_map_update_elem(struct bpf_map* map, const void* key, const void* value, uint64_t flags); int bpf_map_delete_elem(struct bpf_map* map, const void* key); @@ -818,7 +818,7 @@ int bpf_map_delete_elem(struct bpf_map* map, const void* key); Let's update the program to write the value "42" to the map section for the current CPU, by changing the "myprog" section to the following: -``` +```c SEC("myprog") int func1() { @@ -882,7 +882,7 @@ Above shows "call 2", but `netsh` shows more details ; C:\your\path\here/map.c:13 ; return result; 12: exit -```` +``` Notice from instruction 11 that `netsh` understands that `bpf_map_update_elem()` expects a map file descriptor (FD) in R1, a map key in R2, and a map value in R3. From 7c1ea2c23bd45d537cc930596d8761c439272162 Mon Sep 17 00:00:00 2001 From: Shankar Seal <74580197+shankarseal@users.noreply.github.com> Date: Mon, 18 Mar 2024 19:35:38 -0700 Subject: [PATCH 08/10] wait for all events. (#3368) --- tests/api_test/api_test.cpp | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/api_test/api_test.cpp b/tests/api_test/api_test.cpp index 57b7f9d409..cbba0eb309 100644 --- a/tests/api_test/api_test.cpp +++ b/tests/api_test/api_test.cpp @@ -27,6 +27,7 @@ #include #include #include +using namespace std::chrono_literals; CATCH_REGISTER_LISTENER(_watchdog) @@ -1229,12 +1230,19 @@ TEST_CASE("ioctl_stress", "[stress]") bpf_object__close(object); } +typedef struct _ring_buffer_test_context +{ + uint32_t event_count = 0; + uint32_t expected_event_count = 0; + std::promise promise; +} ring_buffer_test_context_t; + TEST_CASE("test_ringbuffer_wraparound", "[stress]") { // Load bindmonitor_ringbuf.sys. struct bpf_object* object = nullptr; fd_t program_fd; - uint32_t event_count = 0; + ring_buffer_test_context_t context; std::string app_id = "api_test.exe"; uint32_t thread_count = 2; native_module_helper_t native_helper; @@ -1254,14 +1262,21 @@ TEST_CASE("test_ringbuffer_wraparound", "[stress]") REQUIRE(max_iterations % thread_count == 0); uint32_t iterations_per_thread = max_iterations / thread_count; + // Initialize context. + context.event_count = 0; + context.expected_event_count = max_iterations; + auto ring_buffer_event_callback = context.promise.get_future(); // Subscribe to the ring buffer. auto ring = ring_buffer__new( process_map_fd, [](void* ctx, void*, size_t) { - (*((uint32_t*)ctx))++; + ring_buffer_test_context_t* context = reinterpret_cast(ctx); + if (++context->event_count == context->expected_event_count) { + context->promise.set_value(); + } return 0; }, - &event_count, + &context, nullptr); // Create 2 threads that invoke the program to trigger ring buffer events. @@ -1290,7 +1305,8 @@ TEST_CASE("test_ringbuffer_wraparound", "[stress]") for (auto& t : threads) { t.join(); } - REQUIRE(event_count == max_iterations); + // Wait for 1 second for the ring buffer to receive all events. + REQUIRE(ring_buffer_event_callback.wait_for(1s) == std::future_status::ready); // Unsubscribe from the ring buffer. ring_buffer__free(ring); From 706675ac1289e3b65d09f1fa1440b53c08820de6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Mar 2024 23:00:17 +0000 Subject: [PATCH 09/10] Bump external/usersim from `e01d44e` to `a7b1c28` (#3364) Bumps [external/usersim](https://github.com/microsoft/usersim) from `e01d44e` to `a7b1c28`. - [Commits](https://github.com/microsoft/usersim/compare/e01d44ed72222734852a3b9e22d974c17ce71b50...a7b1c281da61712a80f914182e46f8b2e5217665) --- updated-dependencies: - dependency-name: external/usersim dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- external/usersim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/external/usersim b/external/usersim index e01d44ed72..a7b1c281da 160000 --- a/external/usersim +++ b/external/usersim @@ -1 +1 @@ -Subproject commit e01d44ed72222734852a3b9e22d974c17ce71b50 +Subproject commit a7b1c281da61712a80f914182e46f8b2e5217665 From b494058832aa135c3fe0bc42891ec4f6f503757e Mon Sep 17 00:00:00 2001 From: Dhiren Vispute <86131363+dv-msft@users.noreply.github.com> Date: Thu, 21 Mar 2024 10:40:56 -0700 Subject: [PATCH 10/10] Rework dump handling, add trace messages, fix process id for user mode dump (#3355) Co-authored-by: Dhiren Vispute --- scripts/Run-Self-Hosted-Runner-Test.ps1 | 10 +- scripts/config_test_vm.psm1 | 149 ++++++++++++++++++++---- 2 files changed, 138 insertions(+), 21 deletions(-) diff --git a/scripts/Run-Self-Hosted-Runner-Test.ps1 b/scripts/Run-Self-Hosted-Runner-Test.ps1 index 080d97cf0a..e890cb86d4 100644 --- a/scripts/Run-Self-Hosted-Runner-Test.ps1 +++ b/scripts/Run-Self-Hosted-Runner-Test.ps1 @@ -179,8 +179,16 @@ if (-not $WaitResult) { $DriveFreeSpaceGB = GetDriveFreeSpaceGB -DriveSpecification $Env:SystemDrive Write-Log "Current available disk space: $DriveFreeSpaceGB GB`n" + # $TestProcess refers to 'cmd.exe' which ends up running the real test application. + # (This is done so that we can capture the stdout and stderr outputs from the test application + # itself. I have not been able to get Powershell's 'built-in' redirection mechanisms for the + # 'Process' object to play well in our scenario). + # We therefore need pass the test application's id to procdump64.exe + $TestCommandId = (Get-Process -Name $TestCommand).Id + Write-Log "Test Command:$TestCommand, Id:$TestCommandId" + Write-Log "Creating User mode dump @ $UserModeDumpFilePath" - $ProcDumpArguments = "-r -ma $($TestProcess.Id) $UserModeDumpFilePath" + $ProcDumpArguments = "-r -ma $($TestCommandId) $UserModeDumpFilePath" Write-Log "Dump Command: $ProcDumpBinaryPath $ProcDumpArguments" $ProcDumpProcess = Start-Process -NoNewWindow ` -FilePath $ProcDumpBinaryPath ` diff --git a/scripts/config_test_vm.psm1 b/scripts/config_test_vm.psm1 index 70a9c6607d..9d098a4d7e 100644 --- a/scripts/config_test_vm.psm1 +++ b/scripts/config_test_vm.psm1 @@ -210,10 +210,69 @@ function Export-BuildArtifactsToVMs Remove-Item -Force $tempFileName } +function ArchiveKernelModeDumpOnVM +{ + param ( + [Parameter(Mandatory = $True)] [System.Management.Automation.Runspaces.PSSession] $Session + ) + + Invoke-Command -Session $Session -ScriptBlock { + + $KernelModeDumpFileSourcePath = "$Env:WinDir" + $KernelModeDumpFileDestinationPath = "$Env:SystemDrive\KernelDumps" + + # Create the compressed dump folder if doesn't exist. + if (!(Test-Path $KernelModeDumpFileDestinationPath)) { + Write-Output "Creating $KernelModeDumpFileDestinationPath directory." + New-Item -ItemType Directory -Path $KernelModeDumpFileDestinationPath | Out-Null + + # Make sure it was created + if (!(Test-Path $KernelModeDumpFileDestinationPath)) { + $ErrorMessage = ` + "*** ERROR *** Create compressed dump file directory failed: $KernelModeDumpFileDestinationPath`n" + Write-Output $ErrorMessage + Start-Sleep -seconds 3 + Throw $ErrorMessage + } + } + + if (Test-Path $KernelModeDumpFileSourcePath\*.dmp -PathType Leaf) { + Write-Output "Found kernel mode dump(s) in $($KernelModeDumpFileSourcePath):" + $DumpFiles = get-childitem -Path $KernelModeDumpFileSourcePath\*.dmp + foreach ($DumpFile in $DumpFiles) { + Write-Output "`tName:$($DumpFile.Name), Size:$((($DumpFile.Length) / 1MB).ToString("F2")) MB" + } + Write-Output "`n" + + Write-Output ` + "Compressing kernel dump files: $KernelModeDumpFileSourcePath -> $KernelModeDumpFileDestinationPath" + Compress-Archive ` + -Path $KernelModeDumpFileSourcePath\*.dmp ` + -DestinationPath $KernelModeDumpFileDestinationPath\km_dumps.zip ` + -CompressionLevel Fastest ` + -Force + + if (Test-Path $KernelModeDumpFileDestinationPath\km_dumps.zip -PathType Leaf) { + $CompressedDumpFile = get-childitem -Path $KernelModeDumpFileDestinationPath\km_dumps.zip + Write-Output "Found compressed kernel mode dump file in $($KernelModeDumpFileDestinationPath):" + Write-Output ` + "`tName:$($CompressedDumpFile.Name), Size:$((($CompressedDumpFile.Length) / 1MB).ToString("F2")) MB" + } else { + $ErrorMessage = "*** ERROR *** kernel mode dump compressed file not found.`n`n" + Write-Output $ErrorMessage + Start-Sleep -seconds 3 + throw $ErrorMessage + } + } else { + Write-Output "`n" + Write-Output "No kernel mode dump(s) in $($KernelModeDumpFileSourcePath)." + } + } +} + # -# Import test logs from VM. +# Import test logs and dumps from VM. # - function Import-ResultsFromVM { param([Parameter(Mandatory=$True)] $VMList, @@ -236,21 +295,37 @@ function Import-ResultsFromVM } $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive} - # Copy kernel crash dumps if any. - Invoke-Command -Session $VMSession -ScriptBlock { - if (!(Test-Path "$Env:SystemDrive\KernelDumps")) { - New-Item -ItemType Directory -Path "$Env:SystemDrive\KernelDumps" - } - - if (Test-Path $Env:WinDir\*.dmp -PathType Leaf) { - tar czf $Env:SystemDrive\KernelDumps\km_dumps.tgz -C $Env:WinDir *.dmp - Remove-Item -Path $Env:WinDir\*.dmp - } + # Archive and copy kernel crash dumps, if any. + Write-Log "Processing kernel mode dump (if any) on VM $VMName" + ArchiveKernelModeDumpOnVM -Session $VMSession + + $LocalKernelArchiveLocation = ".\TestLogs\$VMName\KernelDumps" + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\KernelDumps" ` + -Destination $LocalKernelArchiveLocation ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log + + if (Test-Path $LocalKernelArchiveLocation\km_dumps.zip -PathType Leaf) { + $LocalFile = get-childitem -Path $LocalKernelArchiveLocation\km_dumps.zip + Write-Log "`n" + Write-Log "Local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $($VMName):" + Write-Log "`tName:$($LocalFile.Name), Size:$((($LocalFile.Length) / 1MB).ToString("F2")) MB" + } else { + Write-Log "`n" + Write-Log "No local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $VMName." } - Copy-Item -FromSession $VMSession "$VMSystemDrive\KernelDumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log # Copy user mode crash dumps if any. - Copy-Item -FromSession $VMSession "$VMSystemDrive\dumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\dumps" ` + -Destination ".\TestLogs\$VMName" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log # Copy logs from Test VM. if (!(Test-Path ".\TestLogs\$VMName\Logs")) { @@ -259,10 +334,22 @@ function Import-ResultsFromVM $VMTemp = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:TEMP} Write-Log ("Copy $LogFileName from $VMTemp on $VMName to $pwd\TestLogs") - Copy-Item -FromSession $VMSession "$VMTemp\$LogFileName" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMTemp\$LogFileName" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log Write-Log ("Copy CodeCoverage from eBPF on $VMName to $pwd\..\..") - Copy-Item -FromSession $VMSession "$VMSystemDrive\eBPF\ebpf_for_windows.xml" -Destination "$pwd\..\.." -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\ebpf_for_windows.xml" ` + -Destination "$pwd\..\.." ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log # Copy kernel mode traces, if enabled. if ($KmTracing) { @@ -273,7 +360,11 @@ function Import-ResultsFromVM [Parameter(Mandatory=$True)] [string] $LogFileName, [Parameter(Mandatory=$True)] [string] $EtlFile) $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory" - Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue + Import-Module ` + $WorkingDirectory\common.psm1 ` + -ArgumentList ($LogFileName) ` + -Force ` + -WarningAction SilentlyContinue Write-Log "Query KM ETL tracing status before trace stop" $ProcInfo = Start-Process -FilePath "wpr.exe" ` @@ -304,12 +395,24 @@ function Import-ResultsFromVM # Copy ETL from Test VM. Write-Log ("Copy $WorkingDirectory\$EtlFile on $VMName to $pwd\TestLogs\$VMName\Logs") - Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\$EtlFile" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\$EtlFile" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log } # Copy performance results from Test VM. Write-Log ("Copy performance results from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs") - Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\*.csv" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\*.csv" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log # Compress and copy the performance profile if present. Invoke-Command -Session $VMSession -ScriptBlock { @@ -320,7 +423,13 @@ function Import-ResultsFromVM } } Write-Log ("Copy performance profile from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs") - Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log } # Move runner test logs to TestLogs folder. Write-Host ("Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs")