WiP - rework dump handling, add trace messages

microsoft · Mar 14, 2024 · a83ee2e · a83ee2e
1 parent 5e84f5c
commit a83ee2e
Show file tree

Hide file tree

Showing 3 changed files with 129 additions and 22 deletions.
diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml
@@ -547,12 +547,14 @@ jobs:
   # against the kernel mode eBPF sub-system.
   km_mt_stress_tests_restart_extension:
     needs: regular
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    # PR TESTING ONLY!!!
+    # if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    if: github.event_name == 'schedule' || github.event_name == 'pull_request' || github.event_name == 'merge_group' || github.event_name == 'workflow_dispatch'
     uses: ./.github/workflows/reusable-test.yml
     with:
       name: km_mt_stress_tests_restart_extension
       pre_test: .\setup_ebpf_cicd_tests.ps1 -KmTracing $true -KmTraceType "memory"
-      test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Stress" -Options @("RestartExtension")
+      test_command: .\execute_ebpf_cicd_tests.ps1 -TestMode "Stress" -Options @("RestartExtension") -TestHangTimeout 30
       post_test: .\cleanup_ebpf_cicd_tests.ps1 -KmTracing $true
       build_artifact: Build-x64
       environment: ebpf_cicd_tests_ws2019

diff --git a/scripts/config_test_vm.psm1 b/scripts/config_test_vm.psm1
@@ -210,10 +210,66 @@ function Export-BuildArtifactsToVMs
     Remove-Item -Force $tempFileName
 }
 
+function ArchiveKernelModeDump
+{
+    param (
+        [Parameter(Mandatory = $True)] [System.Management.Automation.Runspaces.PSSession] $Session
+    )
+
+    Invoke-Command -Session $Session -ScriptBlock {
+
+        $KernelModeDumpFileSourcePath = "$Env:WinDir"
+        $KernelModeDumpFileDestinationPath = "$Env:SystemDrive\KernelDumps"
+
+        # Create the compressed dump folder if doesn't exist.
+        if (!(Test-Path $KernelModeDumpFileDestinationPath)) {
+            Write-Log "Creating $KernelModeDumpFileDestinationPath directory."
+            New-Item -ItemType Directory -Path $KernelModeDumpFileDestinationPath
+
+            # Make sure it was created
+            if (!(Test-Path $KernelModeDumpFileDestinationPath)) {
+                $ErrorMessage = `
+                    "*** ERROR *** Create compressed dump file directory failed: $KernelModeDumpFileDestinationPath`n"
+                Write-Log $ErrorMessage
+                Start-Sleep -seconds 3
+                Throw $ErrorMessage
+            }
+        }
+
+        if (Test-Path $KernelModeDumpFileSourcePath\*.dmp -PathType Leaf) {
+            Write-Log "Found kernel mode dump(s) in $($KernelModeDumpFileSourcePath):"
+            $DumpFiles = get-childitem -Path $KernelModeDumpFileSourcePath\*.dmp
+            foreach ($DumpFile in $DumpFiles) {
+                Write-Log "`tName:$($DumpFile.Name), Size:$((($DumpFile.Length) / 1MB).ToString("F2")) MB"
+            }
+            Write-Log "`n"
+
+            Write-Log `
+                "Compressing kernel dump files: $KernelModeDumpFileSourcePath -> $KernelModeDumpFileDestinationPath"
+            Compress-Archive `
+                -Path $KernelModeDumpFileSourcePath\*.dmp `
+                -DestinationPath $KernelModeDumpFileDestinationPath\km_dumps.zip `
+                -CompressionLevel Fastest `
+                -Force
+
+            if (Test-Path $KernelModeDumpFileDestinationPath\km_dumps.zip -PathType Leaf) {
+                $CompressedDumpFile = get-childitem -Path $KernelModeDumpFileDestinationPath\km_dumps.zip
+                Write-Log "Found compressed kernel mode dump file in $($KernelModeDumpFileDestinationPath):"
+                Write-Log `
+                    "`tName:$($CompressedDumpFile.Name), Size:$((($CompressedDumpFile.Length) / 1MB).ToString("F2")) MB"
+            } else {
+                $ErrorMessage = "*** ERROR *** kernel mode dump compressed file not found.`n`n"
+                Write-Log $ErrorMessage
+                Start-Sleep -seconds 3
+                throw $ErrorMessage
+            }
+        }
+    }
+}
+
 #
-# Import test logs from VM.
+# Import test logs and dumps from VM.
 #
-
 function Import-ResultsFromVM
 {
     param([Parameter(Mandatory=$True)] $VMList,
@@ -236,21 +292,32 @@ function Import-ResultsFromVM
         }
         $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive}
 
-        # Copy kernel crash dumps if any.
-        Invoke-Command -Session $VMSession -ScriptBlock {
-            if (!(Test-Path "$Env:SystemDrive\KernelDumps")) {
-                New-Item -ItemType Directory -Path "$Env:SystemDrive\KernelDumps"
-            }
-
-            if (Test-Path $Env:WinDir\*.dmp -PathType Leaf) {
-                tar czf $Env:SystemDrive\KernelDumps\km_dumps.tgz -C $Env:WinDir *.dmp
-                Remove-Item -Path $Env:WinDir\*.dmp
-            }
+        # Archive and copy kernel crash dumps, if any.
+        $LocalKernelDumpFileDestination = ".\TestLogs\$VMName"
+        ArchiveKernelModeDump -Session $VMSession
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\KernelDumps" `
+            -Destination $LocalKernelDumpFileDestination `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
+
+        if (Test-Path $LocalKernelDumpFileDestination\km_dumps.zip -PathType Leaf) {
+            $LocalFile = get-childitem -Path $LocalKernelDumpFileDestination\km_dumps.zip
+            Write-Log "`n"
+            Write-Log "Found local copy of compressed kernel mode dump file in $($LocalKernelDumpFileDestination):"
+            Write-Log "`tName:$($LocalFile.Name), Size:$((($LocalFile.Length) / 1MB).ToString("F2")) MB"
         }
-        Copy-Item -FromSession $VMSession "$VMSystemDrive\KernelDumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
 
         # Copy user mode crash dumps if any.
-        Copy-Item -FromSession $VMSession "$VMSystemDrive\dumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\dumps" `
+            -Destination ".\TestLogs\$VMName" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
 
         # Copy logs from Test VM.
         if (!(Test-Path ".\TestLogs\$VMName\Logs")) {
@@ -259,10 +326,22 @@ function Import-ResultsFromVM
 
         $VMTemp = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:TEMP}
         Write-Log ("Copy $LogFileName from $VMTemp on $VMName to $pwd\TestLogs")
-        Copy-Item -FromSession $VMSession "$VMTemp\$LogFileName" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMTemp\$LogFileName" `
+            -Destination ".\TestLogs\$VMName\Logs" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
 
         Write-Log ("Copy CodeCoverage from eBPF on $VMName to $pwd\..\..")
-        Copy-Item -FromSession $VMSession "$VMSystemDrive\eBPF\ebpf_for_windows.xml" -Destination "$pwd\..\.." -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\eBPF\ebpf_for_windows.xml" `
+            -Destination "$pwd\..\.." `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
 
         # Copy kernel mode traces, if enabled.
         if ($KmTracing) {
@@ -273,7 +352,11 @@ function Import-ResultsFromVM
                       [Parameter(Mandatory=$True)] [string] $LogFileName,
                       [Parameter(Mandatory=$True)] [string] $EtlFile)
                 $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory"
-                Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
+                Import-Module `
+                    $WorkingDirectory\common.psm1 `
+                    -ArgumentList ($LogFileName) `
+                    -Force `
+                    -WarningAction SilentlyContinue
 
                 Write-Log "Query KM ETL tracing status before trace stop"
                 $ProcInfo = Start-Process -FilePath "wpr.exe" `
@@ -304,12 +387,24 @@ function Import-ResultsFromVM
 
             # Copy ETL from Test VM.
             Write-Log ("Copy $WorkingDirectory\$EtlFile on $VMName to $pwd\TestLogs\$VMName\Logs")
-            Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\$EtlFile" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+            Copy-Item `
+                -FromSession $VMSession `
+                -Path "$VMSystemDrive\eBPF\$EtlFile" `
+                -Destination ".\TestLogs\$VMName\Logs" `
+                -Recurse `
+                -Force `
+                -ErrorAction Ignore 2>&1 | Write-Log
         }
 
         # Copy performance results from Test VM.
         Write-Log ("Copy performance results from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs")
-        Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\*.csv" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\eBPF\*.csv" `
+            -Destination ".\TestLogs\$VMName\Logs" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
 
         # Compress and copy the performance profile if present.
         Invoke-Command -Session $VMSession -ScriptBlock {
@@ -320,7 +415,13 @@ function Import-ResultsFromVM
             }
         }
         Write-Log ("Copy performance profile from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs")
-        Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" `
+            -Destination ".\TestLogs\$VMName\Logs" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
     }
     # Move runner test logs to TestLogs folder.
     Write-Host ("Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs")

diff --git a/tests/stress/km/stress_tests_km.cpp b/tests/stress/km/stress_tests_km.cpp
@@ -1416,6 +1416,10 @@ TEST_CASE("jit_load_attach_detach_unload_random_v4_test", "[jit_mt_stress_test]"
     LOG_INFO("\nStarting test *** jit_load_attach_detach_unload_random_v4_test ***");
     test_control_info local_test_control_info = _global_test_control_info;
 
+    // FIXME!!! PR VALIDATION ONLY!!!
+    LOG_INFO("*** PR TESTING!!! SLEEPING FOREVER!!! ***");
+    std::this_thread::sleep_for(std::chrono::seconds(0xFFFFFFFF));
+
     _print_test_control_info(local_test_control_info);
     _mt_prog_load_stress_test(EBPF_EXECUTION_JIT, local_test_control_info);
 }