diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 0ec34f5dba..52cd82328f 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -547,12 +547,14 @@ jobs: # against the kernel mode eBPF sub-system. km_mt_stress_tests_restart_extension: needs: regular - if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + # PR TESTING ONLY!!! + # if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + if: github.event_name == 'schedule' || github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch' uses: ./.github/workflows/reusable-test.yml with: name: km_mt_stress_tests_restart_extension pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "memory" - test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Stress" -Options @("RestartExtension") + test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Stress" -Options @("RestartExtension") -TestHangTimeout 30 post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true build_artifact: Build-x64 environment: ebpf_cicd_tests_ws2019 diff --git a/scripts/config_test_vm.psm1 b/scripts/config_test_vm.psm1 index 70a9c6607d..4137a92d01 100644 --- a/scripts/config_test_vm.psm1 +++ b/scripts/config_test_vm.psm1 @@ -210,10 +210,66 @@ function Export-BuildArtifactsToVMs Remove-Item -Force $tempFileName } +function ArchiveKernelModeDump +{ + param ( + [Parameter(Mandatory = $True)] [System.Management.Automation.Runspaces.PSSession] $Session + ) + + Invoke-Command -Session $Session -ScriptBlock { + + $KernelModeDumpFileSourcePath = "$Env:WinDir" + $KernelModeDumpFileDestinationPath = "$Env:SystemDrive\KernelDumps" + + # Create the compressed dump folder if doesn't exist. + if (!(Test-Path $KernelModeDumpFileDestinationPath)) { + Write-Log "Creating $KernelModeDumpFileDestinationPath directory." + New-Item -ItemType Directory -Path $KernelModeDumpFileDestinationPath + + # Make sure it was created + if (!(Test-Path $KernelModeDumpFileDestinationPath)) { + $ErrorMessage = ` + "*** ERROR *** Create compressed dump file directory failed: $KernelModeDumpFileDestinationPath`n" + Write-Log $ErrorMessage + Start-Sleep -seconds 3 + Throw $ErrorMessage + } + } + + if (Test-Path $KernelModeDumpFileSourcePath\*.dmp -PathType Leaf) { + Write-Log "Found kernel mode dump(s) in $($KernelModeDumpFileSourcePath):" + $DumpFiles = get-childitem -Path $KernelModeDumpFileSourcePath\*.dmp + foreach ($DumpFile in $DumpFiles) { + Write-Log "`tName:$($DumpFile.Name), Size:$((($DumpFile.Length) / 1MB).ToString("F2")) MB" + } + Write-Log "`n" + + Write-Log ` + "Compressing kernel dump files: $KernelModeDumpFileSourcePath -> $KernelModeDumpFileDestinationPath" + Compress-Archive ` + -Path $KernelModeDumpFileSourcePath\*.dmp ` + -DestinationPath $KernelModeDumpFileDestinationPath\km_dumps.zip ` + -CompressionLevel Fastest ` + -Force + + if (Test-Path $KernelModeDumpFileDestinationPath\km_dumps.zip -PathType Leaf) { + $CompressedDumpFile = get-childitem -Path $KernelModeDumpFileDestinationPath\km_dumps.zip + Write-Log "Found compressed kernel mode dump file in $($KernelModeDumpFileDestinationPath):" + Write-Log ` + "`tName:$($CompressedDumpFile.Name), Size:$((($CompressedDumpFile.Length) / 1MB).ToString("F2")) MB" + } else { + $ErrorMessage = "*** ERROR *** kernel mode dump compressed file not found.`n`n" + Write-Log $ErrorMessage + Start-Sleep -seconds 3 + throw $ErrorMessage + } + } + } +} + # -# Import test logs from VM. +# Import test logs and dumps from VM. # - function Import-ResultsFromVM { param([Parameter(Mandatory=$True)] $VMList, @@ -236,21 +292,32 @@ function Import-ResultsFromVM } $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive} - # Copy kernel crash dumps if any. - Invoke-Command -Session $VMSession -ScriptBlock { - if (!(Test-Path "$Env:SystemDrive\KernelDumps")) { - New-Item -ItemType Directory -Path "$Env:SystemDrive\KernelDumps" - } - - if (Test-Path $Env:WinDir\*.dmp -PathType Leaf) { - tar czf $Env:SystemDrive\KernelDumps\km_dumps.tgz -C $Env:WinDir *.dmp - Remove-Item -Path $Env:WinDir\*.dmp - } + # Archive and copy kernel crash dumps, if any. + $LocalKernelDumpFileDestination = ".\TestLogs\$VMName" + ArchiveKernelModeDump -Session $VMSession + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\KernelDumps" ` + -Destination $LocalKernelDumpFileDestination ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log + + if (Test-Path $LocalKernelDumpFileDestination\km_dumps.zip -PathType Leaf) { + $LocalFile = get-childitem -Path $LocalKernelDumpFileDestination\km_dumps.zip + Write-Log "`n" + Write-Log "Found local copy of compressed kernel mode dump file in $($LocalKernelDumpFileDestination):" + Write-Log "`tName:$($LocalFile.Name), Size:$((($LocalFile.Length) / 1MB).ToString("F2")) MB" } - Copy-Item -FromSession $VMSession "$VMSystemDrive\KernelDumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log # Copy user mode crash dumps if any. - Copy-Item -FromSession $VMSession "$VMSystemDrive\dumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\dumps" ` + -Destination ".\TestLogs\$VMName" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log # Copy logs from Test VM. if (!(Test-Path ".\TestLogs\$VMName\Logs")) { @@ -259,10 +326,22 @@ function Import-ResultsFromVM $VMTemp = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:TEMP} Write-Log ("Copy $LogFileName from $VMTemp on $VMName to $pwd\TestLogs") - Copy-Item -FromSession $VMSession "$VMTemp\$LogFileName" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMTemp\$LogFileName" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log Write-Log ("Copy CodeCoverage from eBPF on $VMName to $pwd\..\..") - Copy-Item -FromSession $VMSession "$VMSystemDrive\eBPF\ebpf_for_windows.xml" -Destination "$pwd\..\.." -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\ebpf_for_windows.xml" ` + -Destination "$pwd\..\.." ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log # Copy kernel mode traces, if enabled. if ($KmTracing) { @@ -273,7 +352,11 @@ function Import-ResultsFromVM [Parameter(Mandatory=$True)] [string] $LogFileName, [Parameter(Mandatory=$True)] [string] $EtlFile) $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory" - Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue + Import-Module ` + $WorkingDirectory\common.psm1 ` + -ArgumentList ($LogFileName) ` + -Force ` + -WarningAction SilentlyContinue Write-Log "Query KM ETL tracing status before trace stop" $ProcInfo = Start-Process -FilePath "wpr.exe" ` @@ -304,12 +387,24 @@ function Import-ResultsFromVM # Copy ETL from Test VM. Write-Log ("Copy $WorkingDirectory\$EtlFile on $VMName to $pwd\TestLogs\$VMName\Logs") - Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\$EtlFile" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\$EtlFile" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log } # Copy performance results from Test VM. Write-Log ("Copy performance results from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs") - Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\*.csv" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\*.csv" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log # Compress and copy the performance profile if present. Invoke-Command -Session $VMSession -ScriptBlock { @@ -320,7 +415,13 @@ function Import-ResultsFromVM } } Write-Log ("Copy performance profile from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs") - Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log + Copy-Item ` + -FromSession $VMSession ` + -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" ` + -Destination ".\TestLogs\$VMName\Logs" ` + -Recurse ` + -Force ` + -ErrorAction Ignore 2>&1 | Write-Log } # Move runner test logs to TestLogs folder. Write-Host ("Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs") diff --git a/tests/stress/km/stress_tests_km.cpp b/tests/stress/km/stress_tests_km.cpp index caa6794836..402d59595a 100644 --- a/tests/stress/km/stress_tests_km.cpp +++ b/tests/stress/km/stress_tests_km.cpp @@ -1416,6 +1416,10 @@ TEST_CASE("jit_load_attach_detach_unload_random_v4_test", "[jit_mt_stress_test]" LOG_INFO("\nStarting test *** jit_load_attach_detach_unload_random_v4_test ***"); test_control_info local_test_control_info = _global_test_control_info; + // FIXME!!! PR VALIDATION ONLY!!! + LOG_INFO("*** PR TESTING!!! SLEEPING FOREVER!!! ***"); + std::this_thread::sleep_for(std::chrono::seconds(0xFFFFFFFF)); + _print_test_control_info(local_test_control_info); _mt_prog_load_stress_test(EBPF_EXECUTION_JIT, local_test_control_info); }