Rework dump handling, add trace messages, fix process id for user mod…

…e dump (#3355) Co-authored-by: Dhiren Vispute <[email protected]>
microsoft · Mar 21, 2024 · b494058 · b494058
1 parent 706675a
commit b494058
Show file tree

Hide file tree

Showing 2 changed files with 138 additions and 21 deletions.
diff --git a/scripts/Run-Self-Hosted-Runner-Test.ps1 b/scripts/Run-Self-Hosted-Runner-Test.ps1
@@ -179,8 +179,16 @@ if (-not $WaitResult) {
     $DriveFreeSpaceGB = GetDriveFreeSpaceGB -DriveSpecification $Env:SystemDrive
     Write-Log "Current available disk space: $DriveFreeSpaceGB GB`n"
 
+    # $TestProcess refers to 'cmd.exe' which ends up running the real test application.
+    # (This is done so that we can capture the stdout and stderr outputs from the test application
+    # itself. I have not been able to get Powershell's 'built-in' redirection mechanisms for the
+    # 'Process' object to play well in our scenario).
+    # We therefore need pass the test application's id to procdump64.exe
+    $TestCommandId = (Get-Process -Name $TestCommand).Id
+    Write-Log "Test Command:$TestCommand, Id:$TestCommandId"
+
     Write-Log "Creating User mode dump @ $UserModeDumpFilePath"
-    $ProcDumpArguments = "-r -ma $($TestProcess.Id) $UserModeDumpFilePath"
+    $ProcDumpArguments = "-r -ma $($TestCommandId) $UserModeDumpFilePath"
     Write-Log "Dump Command: $ProcDumpBinaryPath $ProcDumpArguments"
     $ProcDumpProcess = Start-Process -NoNewWindow `
         -FilePath $ProcDumpBinaryPath `

diff --git a/scripts/config_test_vm.psm1 b/scripts/config_test_vm.psm1
@@ -210,10 +210,69 @@ function Export-BuildArtifactsToVMs
     Remove-Item -Force $tempFileName
 }
 
+function ArchiveKernelModeDumpOnVM
+{
+    param (
+        [Parameter(Mandatory = $True)] [System.Management.Automation.Runspaces.PSSession] $Session
+    )
+
+    Invoke-Command -Session $Session -ScriptBlock {
+
+        $KernelModeDumpFileSourcePath = "$Env:WinDir"
+        $KernelModeDumpFileDestinationPath = "$Env:SystemDrive\KernelDumps"
+
+        # Create the compressed dump folder if doesn't exist.
+        if (!(Test-Path $KernelModeDumpFileDestinationPath)) {
+            Write-Output "Creating $KernelModeDumpFileDestinationPath directory."
+            New-Item -ItemType Directory -Path $KernelModeDumpFileDestinationPath | Out-Null
+
+            # Make sure it was created
+            if (!(Test-Path $KernelModeDumpFileDestinationPath)) {
+                $ErrorMessage = `
+                    "*** ERROR *** Create compressed dump file directory failed: $KernelModeDumpFileDestinationPath`n"
+                Write-Output $ErrorMessage
+                Start-Sleep -seconds 3
+                Throw $ErrorMessage
+            }
+        }
+
+        if (Test-Path $KernelModeDumpFileSourcePath\*.dmp -PathType Leaf) {
+            Write-Output "Found kernel mode dump(s) in $($KernelModeDumpFileSourcePath):"
+            $DumpFiles = get-childitem -Path $KernelModeDumpFileSourcePath\*.dmp
+            foreach ($DumpFile in $DumpFiles) {
+                Write-Output "`tName:$($DumpFile.Name), Size:$((($DumpFile.Length) / 1MB).ToString("F2")) MB"
+            }
+            Write-Output "`n"
+
+            Write-Output `
+                "Compressing kernel dump files: $KernelModeDumpFileSourcePath -> $KernelModeDumpFileDestinationPath"
+            Compress-Archive `
+                -Path $KernelModeDumpFileSourcePath\*.dmp `
+                -DestinationPath $KernelModeDumpFileDestinationPath\km_dumps.zip `
+                -CompressionLevel Fastest `
+                -Force
+
+            if (Test-Path $KernelModeDumpFileDestinationPath\km_dumps.zip -PathType Leaf) {
+                $CompressedDumpFile = get-childitem -Path $KernelModeDumpFileDestinationPath\km_dumps.zip
+                Write-Output "Found compressed kernel mode dump file in $($KernelModeDumpFileDestinationPath):"
+                Write-Output `
+                    "`tName:$($CompressedDumpFile.Name), Size:$((($CompressedDumpFile.Length) / 1MB).ToString("F2")) MB"
+            } else {
+                $ErrorMessage = "*** ERROR *** kernel mode dump compressed file not found.`n`n"
+                Write-Output $ErrorMessage
+                Start-Sleep -seconds 3
+                throw $ErrorMessage
+            }
+        } else {
+            Write-Output "`n"
+            Write-Output "No kernel mode dump(s) in $($KernelModeDumpFileSourcePath)."
+        }
+    }
+}
+
 #
-# Import test logs from VM.
+# Import test logs and dumps from VM.
 #
-
 function Import-ResultsFromVM
 {
     param([Parameter(Mandatory=$True)] $VMList,
@@ -236,21 +295,37 @@ function Import-ResultsFromVM
         }
         $VMSystemDrive = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:SystemDrive}
 
-        # Copy kernel crash dumps if any.
-        Invoke-Command -Session $VMSession -ScriptBlock {
-            if (!(Test-Path "$Env:SystemDrive\KernelDumps")) {
-                New-Item -ItemType Directory -Path "$Env:SystemDrive\KernelDumps"
-            }
-
-            if (Test-Path $Env:WinDir\*.dmp -PathType Leaf) {
-                tar czf $Env:SystemDrive\KernelDumps\km_dumps.tgz -C $Env:WinDir *.dmp
-                Remove-Item -Path $Env:WinDir\*.dmp
-            }
+        # Archive and copy kernel crash dumps, if any.
+        Write-Log "Processing kernel mode dump (if any) on VM $VMName"
+        ArchiveKernelModeDumpOnVM -Session $VMSession
+
+        $LocalKernelArchiveLocation = ".\TestLogs\$VMName\KernelDumps"
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\KernelDumps" `
+            -Destination $LocalKernelArchiveLocation `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
+
+        if (Test-Path $LocalKernelArchiveLocation\km_dumps.zip -PathType Leaf) {
+            $LocalFile = get-childitem -Path $LocalKernelArchiveLocation\km_dumps.zip
+            Write-Log "`n"
+            Write-Log "Local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $($VMName):"
+            Write-Log "`tName:$($LocalFile.Name), Size:$((($LocalFile.Length) / 1MB).ToString("F2")) MB"
+        } else {
+            Write-Log "`n"
+            Write-Log "No local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $VMName."
         }
-        Copy-Item -FromSession $VMSession "$VMSystemDrive\KernelDumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
 
         # Copy user mode crash dumps if any.
-        Copy-Item -FromSession $VMSession "$VMSystemDrive\dumps" -Destination ".\TestLogs\$VMName" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\dumps" `
+            -Destination ".\TestLogs\$VMName" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
 
         # Copy logs from Test VM.
         if (!(Test-Path ".\TestLogs\$VMName\Logs")) {
@@ -259,10 +334,22 @@ function Import-ResultsFromVM
 
         $VMTemp = Invoke-Command -Session $VMSession -ScriptBlock {return $Env:TEMP}
         Write-Log ("Copy $LogFileName from $VMTemp on $VMName to $pwd\TestLogs")
-        Copy-Item -FromSession $VMSession "$VMTemp\$LogFileName" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMTemp\$LogFileName" `
+            -Destination ".\TestLogs\$VMName\Logs" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
 
         Write-Log ("Copy CodeCoverage from eBPF on $VMName to $pwd\..\..")
-        Copy-Item -FromSession $VMSession "$VMSystemDrive\eBPF\ebpf_for_windows.xml" -Destination "$pwd\..\.." -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\eBPF\ebpf_for_windows.xml" `
+            -Destination "$pwd\..\.." `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
 
         # Copy kernel mode traces, if enabled.
         if ($KmTracing) {
@@ -273,7 +360,11 @@ function Import-ResultsFromVM
                       [Parameter(Mandatory=$True)] [string] $LogFileName,
                       [Parameter(Mandatory=$True)] [string] $EtlFile)
                 $WorkingDirectory = "$env:SystemDrive\$WorkingDirectory"
-                Import-Module $WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue
+                Import-Module `
+                    $WorkingDirectory\common.psm1 `
+                    -ArgumentList ($LogFileName) `
+                    -Force `
+                    -WarningAction SilentlyContinue
 
                 Write-Log "Query KM ETL tracing status before trace stop"
                 $ProcInfo = Start-Process -FilePath "wpr.exe" `
@@ -304,12 +395,24 @@ function Import-ResultsFromVM
 
             # Copy ETL from Test VM.
             Write-Log ("Copy $WorkingDirectory\$EtlFile on $VMName to $pwd\TestLogs\$VMName\Logs")
-            Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\$EtlFile" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+            Copy-Item `
+                -FromSession $VMSession `
+                -Path "$VMSystemDrive\eBPF\$EtlFile" `
+                -Destination ".\TestLogs\$VMName\Logs" `
+                -Recurse `
+                -Force `
+                -ErrorAction Ignore 2>&1 | Write-Log
         }
 
         # Copy performance results from Test VM.
         Write-Log ("Copy performance results from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs")
-        Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\*.csv" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\eBPF\*.csv" `
+            -Destination ".\TestLogs\$VMName\Logs" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
 
         # Compress and copy the performance profile if present.
         Invoke-Command -Session $VMSession -ScriptBlock {
@@ -320,7 +423,13 @@ function Import-ResultsFromVM
             }
         }
         Write-Log ("Copy performance profile from eBPF on $VMName to $pwd\TestLogs\$VMName\Logs")
-        Copy-Item -FromSession $VMSession -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" -Destination ".\TestLogs\$VMName\Logs" -Recurse -Force -ErrorAction Ignore 2>&1 | Write-Log
+        Copy-Item `
+            -FromSession $VMSession `
+            -Path "$VMSystemDrive\eBPF\bpf_perf_etls.tgz" `
+            -Destination ".\TestLogs\$VMName\Logs" `
+            -Recurse `
+            -Force `
+            -ErrorAction Ignore 2>&1 | Write-Log
     }
     # Move runner test logs to TestLogs folder.
     Write-Host ("Copy $LogFileName from $env:TEMP on host runner to $pwd\TestLogs")