From a3edec538e85e16ea816881c009df79afaaa10f5 Mon Sep 17 00:00:00 2001 From: Shankar Seal <74580197+shankarseal@users.noreply.github.com> Date: Mon, 4 Nov 2024 20:37:30 -0800 Subject: [PATCH] Run all scripts as job with timeout. (#3971) * Run all scripts as job with timeout. * compress KM trace file. * refactored common code. --- scripts/cleanup_ebpf_cicd_tests.ps1 | 111 ++++++++++++++++++---------- scripts/common.psm1 | 70 ++++++++++++++++++ scripts/config_test_vm.psm1 | 64 ++++++---------- scripts/execute_ebpf_cicd_tests.ps1 | 80 ++++++++------------ scripts/install_ebpf.psm1 | 1 + scripts/setup_ebpf_cicd_tests.ps1 | 77 +++++++++++++++---- 6 files changed, 257 insertions(+), 146 deletions(-) diff --git a/scripts/cleanup_ebpf_cicd_tests.ps1 b/scripts/cleanup_ebpf_cicd_tests.ps1 index 8c64d76b68..e718e14715 100644 --- a/scripts/cleanup_ebpf_cicd_tests.ps1 +++ b/scripts/cleanup_ebpf_cicd_tests.ps1 @@ -6,54 +6,87 @@ param ([parameter(Mandatory=$false)][string] $Target = "TEST_VM", [parameter(Mandatory=$false)][string] $LogFileName = "TestLog.log", [parameter(Mandatory=$false)][string] $WorkingDirectory = $pwd.ToString(), [parameter(Mandatory=$false)][string] $TestExecutionJsonFileName = "test_execution.json", - [parameter(Mandatory=$false)][string] $SelfHostedRunnerName) + [parameter(Mandatory=$false)][string] $SelfHostedRunnerName = [System.Net.Dns]::GetHostName(), + [Parameter(Mandatory = $false)][int] $TestJobTimeout = (30*60)) Push-Location $WorkingDirectory -$TestVMCredential = Get-StoredCredential -Target $Target -ErrorAction Stop - -# Load other utility modules. Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue -Import-Module .\config_test_vm.psm1 -Force -ArgumentList ($TestVMCredential.UserName, $TestVMCredential.Password, $WorkingDirectory, $LogFileName) -WarningAction SilentlyContinue -Import-Module .\install_ebpf.psm1 -ArgumentList ($WorkingDirectory, $LogFileName) -Force -WarningAction SilentlyContinue + +$TestVMCredential = Get-StoredCredential -Target $Target -ErrorAction Stop # Read the test execution json. -$TestExecutionConfig = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json -$VMList = $TestExecutionConfig.VMMap.$SelfHostedRunnerName - -# Wait for all VMs to be in ready state, in case the test run caused any VM to crash. -Wait-AllVMsToInitialize ` - -VMList $VMList ` - -UserName $TestVMCredential.UserName ` - -AdminPassword $TestVMCredential.Password - -# Check if we're here after a crash (we are if c:\windows\memory.dmp exists on the VM). If so, -# we need to skip the stopping of the drivers as they may be in a wedged state as a result of the -# crash. We will be restoring the VM's 'baseline' snapshot next, so the step is redundant anyway. -foreach ($VM in $VMList) { - $VMName = $VM.Name - $DumpFound = Invoke-Command ` - -VMName $VMName ` - -Credential $TestVMCredential ` - -ScriptBlock { - Test-Path -Path "c:\windows\memory.dmp" -PathType leaf - } +$Config = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json - if ($DumpFound -eq $True) { - Write-Log "Post-crash reboot detected on VM $VMName" - } else { - # Stop eBPF Components on the test VM. (Un-install is not necessary.) - # We *MUST* be able to stop all drivers cleanly after a test. Failure to do so indicates a fatal bug in - # one/some of the ebpf driver-set. - Stop-eBPFComponentsOnVM -VMName $VMname -ErrorAction Stop +$Job = Start-Job -ScriptBlock { + param ([Parameter(Mandatory = $True)] [PSCredential] $TestVMCredential, + [Parameter(Mandatory = $true)] [PSCustomObject] $Config, + [Parameter(Mandatory = $true)] [string] $SelfHostedRunnerName, + [parameter(Mandatory = $true)] [string] $LogFileName, + [parameter(Mandatory = $true)] [string] $WorkingDirectory = $pwd.ToString(), + [parameter(Mandatory = $true)] [bool] $KmTracing + ) + Push-Location $WorkingDirectory + + # Load other utility modules. + Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue + Import-Module .\config_test_vm.psm1 -Force -ArgumentList ($TestVMCredential.UserName, $TestVMCredential.Password, $WorkingDirectory, $LogFileName) -WarningAction SilentlyContinue + + $VMList = $Config.VMMap.$SelfHostedRunnerName + # Wait for all VMs to be in ready state, in case the test run caused any VM to crash. + Wait-AllVMsToInitialize ` + -VMList $VMList ` + -UserName $TestVMCredential.UserName ` + -AdminPassword $TestVMCredential.Password + + # Check if we're here after a crash (we are if c:\windows\memory.dmp exists on the VM). If so, + # we need to skip the stopping of the drivers as they may be in a wedged state as a result of the + # crash. We will be restoring the VM's 'baseline' snapshot next, so the step is redundant anyway. + foreach ($VM in $VMList) { + $VMName = $VM.Name + $DumpFound = Invoke-Command ` + -VMName $VMName ` + -Credential $TestVMCredential ` + -ScriptBlock { + Test-Path -Path "c:\windows\memory.dmp" -PathType leaf + } + + if ($DumpFound -eq $True) { + Write-Log "Post-crash reboot detected on VM $VMName" + } } -} -# Import logs from VMs. -Import-ResultsFromVM -VMList $VMList -KmTracing $KmTracing + # Import logs from VMs. + Import-ResultsFromVM -VMList $VMList -KmTracing $KmTracing + + # Stop the VMs. + Stop-AllVMs -VMList $VMList + + Pop-Location + +} -ArgumentList ( + $TestVMCredential, + $Config, + $SelfHostedRunnerName, + $LogFileName, + $WorkingDirectory, + $KmTracing) + -# Stop the VMs. -Stop-AllVMs -VMList $VMList -Restore-AllVMs -VMList $VMList +# Wait for the job to complete +$JobTimedOut = ` + Wait-TestJobToComplete -Job $Job ` + -Config $Config ` + -SelfHostedRunnerName $SelfHostedRunnerName ` + -TestJobTimeout $TestJobTimeout ` + -CheckpointPrefix "Cleanup" + +# Clean up +Remove-Job -Job $Job -Force Pop-Location + +if ($JobTimedOut) { + exit 1 +} + diff --git a/scripts/common.psm1 b/scripts/common.psm1 index 1e07c877d7..a4d4f6d536 100644 --- a/scripts/common.psm1 +++ b/scripts/common.psm1 @@ -42,3 +42,73 @@ function New-Credential $Credential = New-Object -TypeName System.Management.Automation.PSCredential -ArgumentList @($UserName, $AdminPassword) return $Credential } + + +function Compress-File +{ + param ([Parameter(Mandatory = $True)] [string] $SourcePath, + [Parameter(Mandatory = $True)] [string] $DestinationPath + ) + + Write-Log "Compressing $SourcePath -> $DestinationPath" + + # Retry 3 times to ensure compression operation succeeds. + # To mitigate error message: "The process cannot access the file because it is being used by another process." + $retryCount = 1 + while ($retryCount -lt 4) { + $error.clear() + Compress-Archive ` + -Path $SourcePath ` + -DestinationPath $DestinationPath ` + -CompressionLevel Fastest ` + -Force + if ($error[0] -ne $null) { + $ErrorMessage = "*** ERROR *** Failed to compress kernel mode dump files: $error. Retrying $retryCount" + Write-Output $ErrorMessage + Start-Sleep -seconds (5 * $retryCount) + $retryCount++ + } else { + # Compression succeeded. + break; + } + } +} + +function Wait-TestJobToComplete +{ + param([Parameter(Mandatory = $true)] [System.Management.Automation.Job] $Job, + [Parameter(Mandatory = $true)] [PSCustomObject] $Config, + [Parameter(Mandatory = $true)] [string] $SelfHostedRunnerName, + [Parameter(Mandatory = $true)] [int] $TestJobTimeout, + [Parameter(Mandatory = $true)] [string] $CheckpointPrefix) + $TimeElapsed = 0 + # Loop to fetch and print job output in near real-time + while ($Job.State -eq 'Running') { + $JobOutput = Receive-Job -Job $job + $JobOutput | ForEach-Object { Write-Host $_ } + + Start-Sleep -Seconds 2 + $TimeElapsed += 2 + + if ($TimeElapsed -gt $TestJobTimeout) { + if ($Job.State -eq "Running") { + $VMList = $Config.VMMap.$SelfHostedRunnerName + # currently one VM runs per runner. + $TestVMName = $VMList[0].Name + Write-Host "Running kernel tests on $TestVMName has timed out after one hour" -ForegroundColor Yellow + $Timestamp = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss' + $CheckpointName = "$CheckpointPrefix-$TestVMName-Checkpoint-$Timestamp" + Write-Log "Taking snapshot $CheckpointName of $TestVMName" + Checkpoint-VM -Name $TestVMName -SnapshotName $CheckpointName + $JobTimedOut = $true + break + } + } + } + + # Print any remaining output after the job completes + $JobOutput = Receive-Job -Job $job + $JobOutput | ForEach-Object { Write-Host $_ } + + return $JobTimedOut +} \ No newline at end of file diff --git a/scripts/config_test_vm.psm1 b/scripts/config_test_vm.psm1 index fd94f3a3fa..25b8efa8bc 100644 --- a/scripts/config_test_vm.psm1 +++ b/scripts/config_test_vm.psm1 @@ -302,80 +302,59 @@ function Stop-eBPFComponentsOnVM Write-Log "eBPF components stopped on $VMName" -ForegroundColor Green } -function ArchiveKernelModeDumpOnVM +function Compress-KernelModeDumpOnVM { param ( [Parameter(Mandatory = $True)] [System.Management.Automation.Runspaces.PSSession] $Session ) Invoke-Command -Session $Session -ScriptBlock { + param([Parameter(Mandatory=$True)] [string] $WorkingDirectory) + + Import-Module $env:SystemDrive\$WorkingDirectory\common.psm1 -ArgumentList ($LogFileName) -Force -WarningAction SilentlyContinue $KernelModeDumpFileSourcePath = "$Env:WinDir" $KernelModeDumpFileDestinationPath = "$Env:SystemDrive\KernelDumps" - + # Create the compressed dump folder if doesn't exist. if (!(Test-Path $KernelModeDumpFileDestinationPath)) { - Write-Output "Creating $KernelModeDumpFileDestinationPath directory." + Write-Log "Creating $KernelModeDumpFileDestinationPath directory." New-Item -ItemType Directory -Path $KernelModeDumpFileDestinationPath | Out-Null # Make sure it was created if (!(Test-Path $KernelModeDumpFileDestinationPath)) { $ErrorMessage = ` "*** ERROR *** Create compressed dump file directory failed: $KernelModeDumpFileDestinationPath`n" - Write-Output $ErrorMessage - Start-Sleep -seconds 3 + Write-Log $ErrorMessage Throw $ErrorMessage } } if (Test-Path $KernelModeDumpFileSourcePath\*.dmp -PathType Leaf) { - Write-Output "Found kernel mode dump(s) in $($KernelModeDumpFileSourcePath):" + Write-Log "Found kernel mode dump(s) in $($KernelModeDumpFileSourcePath):" $DumpFiles = get-childitem -Path $KernelModeDumpFileSourcePath\*.dmp foreach ($DumpFile in $DumpFiles) { - Write-Output "`tName:$($DumpFile.Name), Size:$((($DumpFile.Length) / 1MB).ToString("F2")) MB" + Write-Log "`tName:$($DumpFile.Name), Size:$((($DumpFile.Length) / 1MB).ToString("F2")) MB" } - Write-Output "`n" - Write-Output ` + Write-Log ` "Compressing kernel dump files: $KernelModeDumpFileSourcePath -> $KernelModeDumpFileDestinationPath" - # Retry 3 times to ensure compression operation succeeds. - # To mitigate error message: "The process cannot access the file 'C:\Windows\MEMORY.DMP' because it is being used by another process." - $retryCount = 1 - while ($retryCount -lt 4) { - $error.clear() - Compress-Archive ` - -Path "$KernelModeDumpFileSourcePath\*.dmp" ` - -DestinationPath "$KernelModeDumpFileDestinationPath\km_dumps.zip" ` - -CompressionLevel Fastest ` - -Force - if ($error[0] -ne $null) { - $ErrorMessage = "*** ERROR *** Failed to compress kernel mode dump files: $error. Retrying $retryCount" - Write-Output $ErrorMessage - Start-Sleep -seconds (5 * $retryCount) - $retryCount++ - } else { - # Compression succeeded. - break; - } - } - + Compress-File -SourcePath $KernelModeDumpFileSourcePath\*.dmp -DestinationPath $KernelModeDumpFileDestinationPath\km_dumps.zip if (Test-Path $KernelModeDumpFileDestinationPath\km_dumps.zip -PathType Leaf) { $CompressedDumpFile = get-childitem -Path $KernelModeDumpFileDestinationPath\km_dumps.zip - Write-Output "Found compressed kernel mode dump file in $($KernelModeDumpFileDestinationPath):" - Write-Output ` + Write-Log "Found compressed kernel mode dump file in $($KernelModeDumpFileDestinationPath):" + Write-Log ` "`tName:$($CompressedDumpFile.Name), Size:$((($CompressedDumpFile.Length) / 1MB).ToString("F2")) MB" } else { $ErrorMessage = "*** ERROR *** kernel mode dump compressed file not found.`n`n" - Write-Output $ErrorMessage - Start-Sleep -seconds 3 + Write-Log $ErrorMessage throw $ErrorMessage } } else { - Write-Output "`n" - Write-Output "No kernel mode dump(s) in $($KernelModeDumpFileSourcePath)." + Write-Log "No kernel mode dump(s) in $($KernelModeDumpFileSourcePath)." } - } + } -ArgumentList ("eBPF") -ErrorAction Ignore } # @@ -405,7 +384,7 @@ function Import-ResultsFromVM # Archive and copy kernel crash dumps, if any. Write-Log "Processing kernel mode dump (if any) on VM $VMName" - ArchiveKernelModeDumpOnVM -Session $VMSession + Compress-KernelModeDumpOnVM -Session $VMSession $LocalKernelArchiveLocation = ".\TestLogs\$VMName\KernelDumps" Copy-Item ` @@ -418,11 +397,9 @@ function Import-ResultsFromVM if (Test-Path $LocalKernelArchiveLocation\km_dumps.zip -PathType Leaf) { $LocalFile = get-childitem -Path $LocalKernelArchiveLocation\km_dumps.zip - Write-Log "`n" Write-Log "Local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $($VMName):" Write-Log "`tName:$($LocalFile.Name), Size:$((($LocalFile.Length) / 1MB).ToString("F2")) MB" } else { - Write-Log "`n" Write-Log "No local copy of kernel mode dump archive in $($LocalKernelArchiveLocation) for VM $VMName." } @@ -489,13 +466,16 @@ function Import-ResultsFromVM $EtlFileSize = (Get-ChildItem $WorkingDirectory\$EtlFile).Length/1MB Write-Log "ETL file Size: $EtlFileSize MB" + + Write-Log "Compressing $WorkingDirectory\$EtlFile ..." + Compress-File -SourcePath "$WorkingDirectory\$EtlFile" -DestinationPath "$WorkingDirectory\$EtlFile.zip" } -ArgumentList ("eBPF", $LogFileName, $EtlFile) -ErrorAction Ignore # Copy ETL from Test VM. - Write-Log ("Copy $WorkingDirectory\$EtlFile on $VMName to $pwd\TestLogs\$VMName\Logs") + Write-Log ("Copy $VMSystemDrive\eBPF\$EtlFile.zip on $VMName to $pwd\TestLogs\$VMName\Logs") Copy-Item ` -FromSession $VMSession ` - -Path "$VMSystemDrive\eBPF\$EtlFile" ` + -Path "$VMSystemDrive\eBPF\$EtlFile.zip" ` -Destination ".\TestLogs\$VMName\Logs" ` -Recurse ` -Force ` diff --git a/scripts/execute_ebpf_cicd_tests.ps1 b/scripts/execute_ebpf_cicd_tests.ps1 index 83e67b25cd..c5b39e220f 100644 --- a/scripts/execute_ebpf_cicd_tests.ps1 +++ b/scripts/execute_ebpf_cicd_tests.ps1 @@ -23,24 +23,18 @@ $StandardUserTestVMCredential = Get-StoredCredential -Target $StandardUserTarget # Read the test execution json. $Config = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json -$VMList = $Config.VMMap.$SelfHostedRunnerName -# currently one VM runs per runner. -$TestVMName = $VMList[0].Name $Job = Start-Job -ScriptBlock { - param ( - [Parameter(Mandatory = $True)][string] $TestVMName, - [Parameter(Mandatory = $true)] [PSCustomObject] $Config, - [Parameter(Mandatory = $True)] [string] $Admin, - [Parameter(Mandatory = $True)] [SecureString] $AdminPassword, - [Parameter(Mandatory = $True)] [string] $StandardUser, - [Parameter(Mandatory = $True)] [SecureString] $StandardUserPassword, + param ([Parameter(Mandatory = $True)] [PSCredential] $AdminTestVMCredential, + [Parameter(Mandatory = $True)] [PSCredential] $StandardUserTestVMCredential, + [Parameter(Mandatory = $true)] [PSCustomObject] $Config, + [Parameter(Mandatory = $true)] [string] $SelfHostedRunnerName, [Parameter(Mandatory = $True)] [string] $WorkingDirectory, [Parameter(Mandatory = $True)] [string] $LogFileName, - [Parameter(Mandatory = $True)][string] $TestMode, - [Parameter(Mandatory = $True)][string[]] $Options, - [Parameter(Mandatory = $True)][int] $TestHangTimeout, - [Parameter(Mandatory = $True)][string] $UserModeDumpFolder) + [Parameter(Mandatory = $True)] [string] $TestMode, + [Parameter(Mandatory = $True)] [string[]] $Options, + [Parameter(Mandatory = $True)] [int] $TestHangTimeout, + [Parameter(Mandatory = $True)] [string] $UserModeDumpFolder) Push-Location $WorkingDirectory @@ -49,10 +43,10 @@ $Job = Start-Job -ScriptBlock { Import-Module $WorkingDirectory\vm_run_tests.psm1 ` -Force ` -ArgumentList ( - $Admin, - $AdminPassword, - $StandardUser, - $StandardUserPassword, + $AdminTestVMCredential.UserName, + $AdminTestVMCredential.Password, + $StandardUserTestVMCredential.UserName, + $StandardUserTestVMCredential.Password, $WorkingDirectory, $LogFileName, $TestMode, @@ -61,19 +55,23 @@ $Job = Start-Job -ScriptBlock { $UserModeDumpFolder) ` -WarningAction SilentlyContinue + $VMList = $Config.VMMap.$SelfHostedRunnerName + # currently one VM runs per runner. + $TestVMName = $VMList[0].Name + # Run Kernel tests on test VM. Write-Log "Running kernel tests on $TestVMName" Run-KernelTestsOnVM -VMName $TestVMName -Config $Config # Stop eBPF components on test VMs. Stop-eBPFComponentsOnVM -VMName $TestVMName + + Pop-Location } -ArgumentList ( - $TestVMName, + $AdminTestVMCredential, + $StandardUserTestVMCredential, $Config, - $AdminTestVMCredential.UserName, - $AdminTestVMCredential.Password, - $StandardUserTestVMCredential.UserName, - $StandardUserTestVMCredential.Password, + $SelfHostedRunnerName, $WorkingDirectory, $LogFileName, $TestMode, @@ -82,34 +80,18 @@ $Job = Start-Job -ScriptBlock { $UserModeDumpFolder) # Keep track of the last received output count -$LastCount = 0 -$TimeElapsed = 0 - -# Loop to fetch and print job output in near real-time -while ($Job.State -eq 'Running') { - $JobOutput = Receive-Job -Job $job - $JobOutput | ForEach-Object { Write-Host $_ } - - Start-Sleep -Seconds 2 - $TimeElapsed += 2 - - if ($TimeElapsed -gt $TestJobTimeout) { - if ($Job.State -eq "Running") { - Write-Host "Running kernel tests on $TestVMName has timed out after one hour" -ForegroundColor Yellow - $Timestamp = Get-Date -Format 'yyyy-MM-dd_HH-mm-ss' - $CheckpointName = "$TestVMName-Checkpoint-$Timestamp" - Write-Log "Taking snapshot $CheckpointName of $TestVMName" - Checkpoint-VM -Name $TestVMName -SnapshotName $CheckpointName - break - } - } -} - -# Print any remaining output after the job completes -$JobOutput = Receive-Job -Job $job -$JobOutput | ForEach-Object { Write-Host $_ } +$JobTimedOut = ` + Wait-TestJobToComplete -Job $Job ` + -Config $Config ` + -SelfHostedRunnerName $SelfHostedRunnerName ` + -TestJobTimeout $TestJobTimeout ` + -CheckpointPrefix "Execute" # Clean up Remove-Job -Job $Job -Force Pop-Location + +if ($JobTimedOut) { + exit 1 +} diff --git a/scripts/install_ebpf.psm1 b/scripts/install_ebpf.psm1 index 2c0e58efd3..28ea727165 100644 --- a/scripts/install_ebpf.psm1 +++ b/scripts/install_ebpf.psm1 @@ -139,6 +139,7 @@ function Stop-eBPFComponents { $EbpfDrivers.GetEnumerator() | ForEach-Object { try { if ($_.Value.IsDriver) { + Write-Log "Stopping $($_.Key) driver..." Stop-Service $_.Name -ErrorAction Stop 2>&1 | Write-Log Write-Log "$($_.Key) driver stopped." -ForegroundColor Green } diff --git a/scripts/setup_ebpf_cicd_tests.ps1 b/scripts/setup_ebpf_cicd_tests.ps1 index 17f639534c..f2d715fc5b 100644 --- a/scripts/setup_ebpf_cicd_tests.ps1 +++ b/scripts/setup_ebpf_cicd_tests.ps1 @@ -10,7 +10,8 @@ param ([parameter(Mandatory=$false)][string] $Target = "TEST_VM", [parameter(Mandatory=$false)][string] $RegressionArtifactsVersion = "", [parameter(Mandatory=$false)][string] $RegressionArtifactsConfiguration = "", [parameter(Mandatory=$false)][string] $TestExecutionJsonFileName = "test_execution.json", - [parameter(Mandatory=$false)][string] $SelfHostedRunnerName = [System.Net.Dns]::GetHostName()) + [parameter(Mandatory=$false)][string] $SelfHostedRunnerName = [System.Net.Dns]::GetHostName(), + [Parameter(Mandatory = $false)][int] $TestJobTimeout = (30*60)) Push-Location $WorkingDirectory @@ -21,8 +22,8 @@ Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction S Import-Module .\config_test_vm.psm1 -Force -ArgumentList ($TestVMCredential.UserName, $TestVMCredential.Password, $WorkingDirectory, $LogFileName) -WarningAction SilentlyContinue # Read the test execution json. -$TestExecutionConfig = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json -$VMList = $TestExecutionConfig.VMMap.$SelfHostedRunnerName +$Config = Get-Content ("{0}\{1}" -f $PSScriptRoot, $TestExecutionJsonFileName) | ConvertFrom-Json +$VMList = $Config.VMMap.$SelfHostedRunnerName # Delete old log files if any. Remove-Item "$env:TEMP\$LogFileName" -ErrorAction SilentlyContinue @@ -32,9 +33,6 @@ foreach($VM in $VMList) { } Remove-Item ".\TestLogs" -Recurse -Confirm:$false -ErrorAction SilentlyContinue -# Get all VMs to ready state. -Initialize-AllVMs -VMList $VMList -ErrorAction Stop - if ($TestMode -eq "Regression") { # Download the release artifacts for regression tests. @@ -50,16 +48,63 @@ if ($TestMode -eq "CI/CD" -or $TestMode -eq "Regression") { Get-CoreNetTools Get-PSExec -# Export build artifacts to the test VMs. -Export-BuildArtifactsToVMs -VMList $VMList -ErrorAction Stop +$Job = Start-Job -ScriptBlock { + param ([Parameter(Mandatory = $True)] [PSCredential] $TestVMCredential, + [Parameter(Mandatory = $true)] [PSCustomObject] $Config, + [Parameter(Mandatory = $true)] [string] $SelfHostedRunnerName, + [parameter(Mandatory = $true)] [string] $TestMode, + [parameter(Mandatory = $true)] [string] $LogFileName, + [parameter(Mandatory = $true)] [string] $WorkingDirectory = $pwd.ToString(), + [parameter(Mandatory = $true)] [bool] $KmTracing, + [parameter(Mandatory = $true)] [string] $KmTraceType + ) + Push-Location $WorkingDirectory + + # Load other utility modules. + Import-Module .\common.psm1 -Force -ArgumentList ($LogFileName) -WarningAction SilentlyContinue + Import-Module .\config_test_vm.psm1 -Force -ArgumentList ($TestVMCredential.UserName, $TestVMCredential.Password, $WorkingDirectory, $LogFileName) -WarningAction SilentlyContinue + + $VMList = $Config.VMMap.$SelfHostedRunnerName + + # Get all VMs to ready state. + Initialize-AllVMs -VMList $VMList -ErrorAction Stop + + # Export build artifacts to the test VMs. + Export-BuildArtifactsToVMs -VMList $VMList -ErrorAction Stop + + # Configure network adapters on VMs. + Initialize-NetworkInterfacesOnVMs $VMList -ErrorAction Stop + + # Install eBPF Components on the test VM. + foreach($VM in $VMList) { + $VMName = $VM.Name + Install-eBPFComponentsOnVM -VMName $VMname -TestMode $TestMode -KmTracing $KmTracing -KmTraceType $KmTraceType -ErrorAction Stop + } + + Pop-Location +} -ArgumentList ( + $TestVMCredential, + $Config, + $SelfHostedRunnerName, + $TestMode, + $LogFileName, + $WorkingDirectory, + $KmTracing, + $KmTraceType) + +# wait for the job to complete +$JobTimedOut = ` + Wait-TestJobToComplete -Job $Job ` + -Config $Config ` + -SelfHostedRunnerName $SelfHostedRunnerName ` + -TestJobTimeout $TestJobTimeout ` + -CheckpointPrefix "Setup" + +# Clean up +Remove-Job -Job $Job -Force -# Configure network adapters on VMs. -Initialize-NetworkInterfacesOnVMs $VMList -ErrorAction Stop +Pop-Location -# Install eBPF Components on the test VM. -foreach($VM in $VMList) { - $VMName = $VM.Name - Install-eBPFComponentsOnVM -VMName $VMname -TestMode $TestMode -KmTracing $KmTracing -KmTraceType $KmTraceType -ErrorAction Stop +if ($JobTimedOut) { + exit 1 } - -Pop-Location