-
Notifications
You must be signed in to change notification settings - Fork 241
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
WiP - Use custom script wrapper for all km tests to capture user, km …
…dumps on hangs
- Loading branch information
Dhiren Vispute
authored and
Dhiren Vispute
committed
Feb 26, 2024
1 parent
12f73c3
commit f421d23
Showing
7 changed files
with
802 additions
and
327 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,226 @@ | ||
| # Copyright (c) Microsoft Corporation | ||
| # SPDX-License-Identifier: MIT | ||
| # | ||
| # This script executes the provided test command, waits for <TestHangTimeout> (in seconds) and then triggers user and | ||
| # kernel dumps if the test process is still ruunning (This is typically the case when the test process is hung for some | ||
| # reason). | ||
| # | ||
| # The user mode dump is created using 'procdump64.exe' and the kernel dump using 'notmyfault64.exe' utilities | ||
| # respectively, both from the SysInternals Suite. The script assumes the presence of these utilities under the current | ||
| # working directory. | ||
| # | ||
| # (While 'procdump64.exe' also has the ability to generate a kernel dump, that dump is restricted to the kernel | ||
| # portions of the user mode app's threads _only_ and does not provide any visibility into other kernel threads. This | ||
| # script therefore uses the 'notmyfault64.exe' tool to generate a 'true' kernel dump which captures the state of all | ||
| # kernel threads and related data structures) | ||
| # | ||
|
|
||
| [CmdletBinding()] | ||
| Param( | ||
| [Parameter(Mandatory = $true)] [string] $TestCommand, | ||
| [Parameter(Mandatory = $false)] [string] $TestArguments = "", | ||
| [Parameter(Mandatory = $false)] [int] $TestHangTimeout = 3600, | ||
| [Parameter(Mandatory = $false)] [string] $UserModeDumpFolder = "C:\Dumps", | ||
| [Parameter(Mandatory = $false)] [bool] $NeedKernelDump = $true | ||
| ) | ||
|
|
||
| # Finds and returns the specified tool's location under the current directory. If not found, throws an exception. | ||
| function GetToolLocationPath | ||
| { | ||
| param( | ||
| [Parameter(Mandatory = $True)] [string] $ToolName | ||
| ) | ||
|
|
||
| $ToolLocationPath = Get-ChildItem -Path "$Pwd" ` | ||
| -Recurse -Filter "$ToolName" -ErrorAction SilentlyContinue | Select-Object -First 1 | ||
| if ($ToolLocationPath -eq $null) { | ||
| $ErrorMessage = "*** ERROR *** $ToolName not found under $Pwd." | ||
| Write-output $ErrorMessage | ||
| Start-Sleep -seconds 5 | ||
| throw $ErrorMessage | ||
| } | ||
|
|
||
| return $ToolLocationPath.FullName | ||
| } | ||
|
|
||
| function ThrowWithErrorMessage | ||
| { | ||
| Param( | ||
| [Parameter(Mandatory = $True)] [string] $ErrorMessage | ||
| ) | ||
|
|
||
| Write-Log $ErrorMessage | ||
| # Wait a bit to let the above message to show up. | ||
| Start-Sleep -seconds 5 | ||
| throw $ErrorMessage | ||
| } | ||
|
|
||
| if ($VerbosePreference -eq 'Continue') { | ||
| Write-Output "Command : $TestCommand" | ||
| Write-Output "Arguments : $TestArguments" | ||
| Write-Output "Test Hang Timeout : $TestHangTimeout" | ||
| Write-Output "User mode dump Folder : $UserModeDumpFolder" | ||
| Write-Output "Kernel dump needed : $NeedKernelDump" | ||
| } | ||
|
|
||
| # Verify timeout is non-zero. | ||
| if ($TestHangTimeout -le 0) { | ||
| ThrowWithErrorMessage ` | ||
| -ErrorMessage "*** ERROR *** Invalid test hang timeout value: $TestHangTimeout" | ||
| } | ||
|
|
||
| # Create dump folder if not present. | ||
| if (-not (Test-Path -Path $UserModeDumpFolder)) { | ||
| New-Item -Path $UserModeDumpFolder -ItemType Directory -Force | ||
|
|
||
| # verify dump folder presence. | ||
| if (-not (Test-Path -Path $UserModeDumpFolder)) { | ||
| ThrowWithErrorMessage ` | ||
| -ErrorMessage "*** ERROR *** User mode dump folder not found: $UserModeDumpFolder" | ||
| } | ||
| } | ||
|
|
||
| # Check if procdump64.exe and notmyfault64.exe are present on the system. | ||
| $ProcDumpBinary = "ProcDump64.exe" | ||
| Write-Log "Verifying $ProcDumpBinary presence in $Pwd..." | ||
| $ProcDumpBinaryPath = GetToolLocationPath -ToolName $ProcDumpBinary | ||
| Write-Log "$ProcDumpBinary location: $ProcDumpBinaryPath" | ||
| Write-Log "`n" | ||
|
|
||
| $NotMyFaultBinary = "NotMyFault64.exe" | ||
| Write-Log "Verifying $NotMyFaultBinary presence in $Pwd..." | ||
| $NotMyFaultBinaryPath = GetToolLocationPath -ToolName $NotMyFaultBinary | ||
| Write-Log "$NotMyFaultBinary location: $NotMyFaultBinaryPath" | ||
| Write-Log "`n" | ||
|
|
||
| # While at it, enable EULA for all SysInternals tools. | ||
| REG ADD HKCU\Software\Sysinternals /v EulaAccepted /t REG_DWORD /d 1 /f | Out-Null | ||
|
|
||
| # The following command will enable full memory dump. | ||
| # Not enabled yet as it needs a VM with an explicitly created page file of at least (physical_memory + 1MB) in size. | ||
| # The default value of the 'CrashDumpEnabled' key is 7 ('automatic' sizing of dump file size (system determined)). | ||
| # | ||
| # Set-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\CrashControl' -Name 'CrashDumpEnabled' -Value 1 | ||
| # | ||
| if ($VerbosePreference -eq 'Continue') { | ||
| # Dump current kernel mode dump settings. | ||
| Write-Log "`n" | ||
| Write-Log "Current kernel dump configuration:`n" | ||
| $KernelDumpInfo = Get-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\CrashControl' | ||
| $Lines = $KernelDumpInfo -split '; ' | ||
| foreach($Line in $Lines) { | ||
| Write-Log "`t$Line" | ||
| } | ||
| Write-Log "`n" | ||
| } | ||
|
|
||
| $BeforeTestFreeSpaceGB = (((Get-Volume C).SizeRemaining) / 1GB).ToString("F2") | ||
| Write-Log "Available disk space (Before test start): $BeforeTestFreeSpaceGB GB" | ||
|
|
||
| $FullTestCommandSpec = Join-Path $Pwd $TestCommand | ||
|
|
||
| # Start the test process using the provided command and arguments. | ||
| Write-Log "`n`n" | ||
| Write-Log "Staring Test command: $FullTestCommandSpec $TestArguments" | ||
| Write-Log "Test hang timeout: $TestHangTimeout (seconds)" | ||
| Write-Log "`n" | ||
|
|
||
| # Create a temporary file for standard output and error. | ||
| $TestTempOutFile = [System.IO.Path]::GetTempFileName() | ||
|
|
||
| # Form the complete command line with output redirection for cmd.exe | ||
| $TestCommandLine = "$FullTestCommandSpec $($TestArguments) > $TestTempOutFile 2>&1" | ||
|
|
||
| # Start the test process and wait for it to exit or timeout | ||
| $TestProcess = Start-Process -FilePath "cmd.exe" -ArgumentList "/c $TestCommandLine" -PassThru -NoNewWindow -Wait | ||
| $WaitResult = $TestProcess.WaitForExit($TestHangTimeout * 1000) | ||
|
|
||
| if (-not $WaitResult) { | ||
| Write-Log "`n" | ||
| Write-Log "*** ERROR *** Test execution hang timeout ($TestHangTimeout seconds) expired.`n" | ||
|
|
||
| # First generate a user mode dump. | ||
| $UserModeDumpFileName = "$($TestCommand)_$(Get-Date -Format 'yyyy-MM-dd_HH-mm-ss').dmp" | ||
| $UserModeDumpFilePath = Join-Path $UserModeDumpFolder $UserModeDumpFileName | ||
|
|
||
| if ($VerbosePreference -eq 'Continue') { | ||
| Write-Log "User mode Dumpfile name: $UserModeDumpFileName" | ||
| Write-Log "User mode Dumpfile Path: $UserModeDumpFilePath" | ||
| } | ||
|
|
||
| $DriveFreeSpaceGB = (((Get-Volume C).SizeRemaining) /1GB).ToString("F2") | ||
| Write-Log "Current available disk space: $DriveFreeSpaceGB GB`n" | ||
|
|
||
| Write-Log "Creating User mode dump @ $UserModeDumpFilePath" | ||
| $ProcDumpArguments = "-r -ma $($TestProcess.Id) $UserModeDumpFilePath" | ||
| Write-Log "Dump Command: $ProcDumpBinaryPath $ProcDumpArguments" | ||
| $ProcDumpProcess = Start-Process -NoNewWindow ` | ||
| -FilePath $ProcDumpBinaryPath ` | ||
| -ArgumentList $ProcDumpArguments ` | ||
| -Wait -PassThru | ||
| Write-Log "Waiting for user mode dump to complete..." | ||
| $ProcDumpProcess.WaitForExit() | ||
|
|
||
| # Get procdump64.exe's exit code. | ||
| $ExitCode = $($ProcDumpProcess.ExitCode) | ||
| Write-Log "$ProcDumpBinaryPath completed with exit code: $ExitCode`n" | ||
| Write-Log "User mode dump completed. Flushing disk buffers..." | ||
| Write-VolumeCache -DriveLetter C | ||
|
|
||
| # Wait for a bit for things to settle down to ensure the user mode dump is completely flushed. | ||
| Start-Sleep -seconds 10 | ||
|
|
||
| $UserModeDumpSizeMB = | ||
| (((Get-ItemProperty -Path $UserModeDumpFilePath).Length) /1MB).ToString("F2") | ||
| if ($UserModeDumpSizeMB -eq 0) { | ||
| Write-Log "* WARNING * User mode dump $UserModeDumpFilePath NOT CREATED" | ||
| } else { | ||
| Write-Log "`n`n" | ||
| Write-Log "Created $UserModeDumpFilePath, size: $UserModeDumpSizeMB MB" | ||
| Write-Log "`n`n" | ||
| } | ||
|
|
||
| if ($NeedKernelDump) { | ||
| Write-Log "Creating kernel dump...`n" | ||
| # Wait a bit for the above message to show up in the log. | ||
| Start-Sleep -seconds 5 | ||
|
|
||
| # This will/should not return (system will/should bluescreen and reboot). | ||
| Start-Process -NoNewWindow -Wait -FilePath $NotMyFaultBinaryPath -ArgumentList "/crash" | ||
|
|
||
| # If we get here, notmyfault64.exe failed for some reason. Kill the hung process, throw error. | ||
| $TestProcess.Kill() | ||
| ThrowWithErrorMessage ` | ||
| -ErrorMessage "*** ERROR *** $($PSCommandPath): kernel mode dump creation FAILED" | ||
| } else { | ||
| Write-Log "`n`n" | ||
| Write-Log "Kernel dump not needed, killing test process $($TestProcess.ProcessName)..." | ||
| $TestProcess.Kill() | ||
| Write-Log "`n`n" | ||
|
|
||
| $ErrorMessage = "*** ERROR *** $($PSCommandPath): $FullTestCommandSpec.exe exceeded test hang timout of " + | ||
| "$TestHangTimeout seconds" | ||
| ThrowWithErrorMessage -ErrorMessage $ErrorMessage | ||
| } | ||
| } else { | ||
|
|
||
| # Ensure the process has completely exited. | ||
| $TestProcess.WaitForExit() | ||
|
|
||
| # Read and display the output (if any) from the temporary output file. | ||
| $TestStandardOutput = Get-Content -Path $TestTempOutFile | ||
| if ($TestStandardOutput) { | ||
| Write-Log "`n`n" | ||
| Write-Log "Test Program output:`n" | ||
| foreach($Line in $TestStandardOutput) { | ||
| Write-Log "$Line" | ||
| } | ||
| } | ||
| Write-Log "`n" | ||
|
|
||
| if ($($TestProcess.ExitCode) -ne 0) { | ||
| $ErrorMessage = "*** ERROR *** $($PSCommandPath): $FullTestCommandSpec " + | ||
| "exited with exit code: $($TestProcess.ExitCode)" | ||
| ThrowWithErrorMessage -ErrorMessage $ErrorMessage | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.