From dd7d7a608d20b8f6bd9d1110bee1c8e73023376f Mon Sep 17 00:00:00 2001 From: Freddy Kristiansen Date: Fri, 21 Jun 2024 10:42:35 +0200 Subject: [PATCH] BCPT Test Report (#619) This PR will add a BCPT Test Report to the build summary, looking like this: ![image](https://github.com/microsoft/AL-Go/assets/10775043/7a9d2e42-e017-41fc-9bc2-62cc2534bea1) You can also add a bcptBaseLine.json to the project in order to establish a baseline for the performance tests. It looks like this: ![image](https://github.com/microsoft/AL-Go/assets/10775043/fd5612e5-11c9-4483-bd27-c49dc6ed05c0) TODOs: - [x] Add tests - [x] Add thresholds to project settings - [x] Determine how thresholds should work? threshold on very small items like (enter account no.) doesn't make much sense. It absolutely makes sense to have threshold on scenarios. - [x] Determine sorting of test results? (codeunitID, codeunitName or ???) - [x] Issue GitHub warnings and errors when thresholds are exceeded - [x] Is durationMin in milliseconds, seconds or what? how many decimal digits should be displayed? - [x] Get BCPT Backend Team signoff that BCPT Test Results are correctly understood and compared - [x] Add scenario documentation Example of bcpt tests with failures and warnings: ![image](https://github.com/microsoft/AL-Go/assets/10775043/85b16114-687f-435d-bc93-0d54757b7196) --------- Co-authored-by: freddydk Co-authored-by: Alexander Holstrup <117829001+aholstrup1@users.noreply.github.com> Co-authored-by: Maria Zhelezova <43066499+mazhelez@users.noreply.github.com> --- Actions/AL-Go-Helper.ps1 | 6 + Actions/AnalyzeTests/AnalyzeTests.ps1 | 45 ++- Actions/AnalyzeTests/TestResultAnalyzer.ps1 | 377 ++++++++++++++---- Actions/AnalyzeTests/action.yaml | 8 +- .../CheckForUpdates.HelperFunctions.ps1 | 2 +- RELEASENOTES.md | 15 + Scenarios/AddAPerformanceTestApp.md | 59 ++- Scenarios/settings.md | 1 + .../CreateOnlineDevelopmentEnvironment.yaml | 4 +- .../workflows/PublishToEnvironment.yaml | 4 +- .../CreateOnlineDevelopmentEnvironment.yaml | 4 +- .../workflows/PublishToEnvironment.yaml | 4 +- Tests/AnalyzeTests.Test.ps1 | 145 +++++++ 13 files changed, 554 insertions(+), 120 deletions(-) create mode 100644 Tests/AnalyzeTests.Test.ps1 diff --git a/Actions/AL-Go-Helper.ps1 b/Actions/AL-Go-Helper.ps1 index b4f62578e..972c1393d 100644 --- a/Actions/AL-Go-Helper.ps1 +++ b/Actions/AL-Go-Helper.ps1 @@ -638,6 +638,12 @@ function ReadSettings { "buildModes" = @() "useCompilerFolder" = $false "pullRequestTrigger" = "pull_request_target" + "bcptThresholds" = [ordered]@{ + "DurationWarning" = 10 + "DurationError" = 25 + "NumberOfSqlStmtsWarning" = 5 + "NumberOfSqlStmtsError" = 10 + } "fullBuildPatterns" = @() "excludeEnvironments" = @() "alDoc" = [ordered]@{ diff --git a/Actions/AnalyzeTests/AnalyzeTests.ps1 b/Actions/AnalyzeTests/AnalyzeTests.ps1 index ffddd95c6..1b6589cca 100644 --- a/Actions/AnalyzeTests/AnalyzeTests.ps1 +++ b/Actions/AnalyzeTests/AnalyzeTests.ps1 @@ -2,7 +2,7 @@ [Parameter(HelpMessage = "Specifies the parent telemetry scope for the telemetry signal", Mandatory = $false)] [string] $parentTelemetryScopeJson = '7b7d', [Parameter(HelpMessage = "Project to analyze", Mandatory = $false)] - [string] $project + [string] $project = '.' ) $telemetryScope = $null @@ -17,25 +17,40 @@ try { . (Join-Path -Path $PSScriptRoot 'TestResultAnalyzer.ps1') $testResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\TestResults.xml" - if (Test-Path $testResultsFile) { - $testResults = [xml](Get-Content "$project\TestResults.xml") - $testResultSummary = GetTestResultSummary -testResults $testResults -includeFailures 50 + $testResultsSummaryMD, $testResultsfailuresMD, $testResultsFailuresSummaryMD = GetTestResultSummaryMD -testResultsFile $testResultsFile - Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "TestResultMD=$testResultSummary" - Write-Host "TestResultMD=$testResultSummary" + $settings = $env:Settings | ConvertFrom-Json + $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptTestResults.json" + $bcptBaseLineFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptBaseLine.json" + $bcptThresholdsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\bcptThresholds.json" + $bcptSummaryMD = GetBcptSummaryMD ` + -bcptTestResultsFile $bcptTestResultsFile ` + -baseLinePath $bcptBaseLineFile ` + -thresholdsPath $bcptThresholdsFile ` + -bcptThresholds ($settings.bcptThresholds | ConvertTo-HashTable) - Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultSummary.Replace("\n","`n"))`n" + # If summary fits, we will display it in the GitHub summary + if ($testResultsSummaryMD.Length -gt 65000) { + # If Test results summary is too long, we will not display it in the GitHub summary, instead we will display a message to download the test results + $testResultsSummaryMD = "Test results summary size exceeds GitHub summary capacity. Download **TestResults** artifact to see details." } - else { - Write-Host "Test results not found" + # If summary AND BCPT summary fits, we will display both in the GitHub summary + if ($testResultsSummaryMD.Length + $bcptSummaryMD.Length -gt 65000) { + # If Combined Test Results and BCPT summary exceeds GitHub summary capacity, we will not display the BCPT summary + $bcptSummaryMD = "Performance test results summary size exceeds GitHub summary capacity. Download **BcptTestResults** artifact to see details." } - - $bcptTestResultsFile = Join-Path $ENV:GITHUB_WORKSPACE "$project\BCPTTestResults.json" - if (Test-Path $bcptTestResultsFile) { - # TODO Display BCPT Test Results + # If summary AND BCPT summary AND failures summary fits, we will display all in the GitHub summary + if ($testResultsSummaryMD.Length + $testResultsfailuresMD.Length + $bcptSummaryMD.Length -gt 65000) { + # If Combined Test Results, failures and BCPT summary exceeds GitHub summary capacity, we will not display the failures details, only the failures summary + $testResultsfailuresMD = $testResultsFailuresSummaryMD } - else { - #Add-Content -path $ENV:GITHUB_STEP_SUMMARY -value "*BCPT test results not found*`n`n" + + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "## Test results`n`n" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsSummaryMD.Replace("\n","`n"))`n`n" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($testResultsfailuresMD.Replace("\n","`n"))`n`n" + if ($bcptSummaryMD) { + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "## Performance test results`n`n" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "$($bcptSummaryMD.Replace("\n","`n"))`n`n" } TrackTrace -telemetryScope $telemetryScope diff --git a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 index b089cd3f4..f3efad58f 100644 --- a/Actions/AnalyzeTests/TestResultAnalyzer.ps1 +++ b/Actions/AnalyzeTests/TestResultAnalyzer.ps1 @@ -1,107 +1,324 @@ -function GetTestResultSummary { +$statusOK = " :heavy_check_mark:" +$statusWarning = " :warning:" +$statusError = " :x:" +$statusSkipped = " :question:" + +# Build MarkDown of TestResults file +# This function will not fail if the file does not exist or if any test errors are found +# TestResults is in JUnit format +# Returns both a summary part and a failures part +function GetTestResultSummaryMD { Param( - [xml] $testResults, - [int] $includeFailures + [string] $testResultsFile ) - $totalTests = 0 - $totalTime = 0.0 - $totalFailed = 0 - $totalSkipped = 0 - $failuresIncluded = 0 $summarySb = [System.Text.StringBuilder]::new() $failuresSb = [System.Text.StringBuilder]::new() - if ($testResults.testsuites) { - $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "appName" } | ForEach-Object { $_.Value } } | Select-Object -Unique) - if (-not $appNames) { - $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "extensionId" } | ForEach-Object { $_.Value } } | Select-Object -Unique) - } - foreach($testsuite in $testResults.testsuites.testsuite) { - $totalTests += $testsuite.Tests - $totalTime += [decimal]::Parse($testsuite.time, [System.Globalization.CultureInfo]::InvariantCulture) - $totalFailed += $testsuite.failures - $totalSkipped += $testsuite.skipped - } - Write-Host "$($appNames.Count) TestApps, $totalTests tests, $totalFailed failed, $totalSkipped skipped, $totalTime seconds" - $summarySb.Append('|Test app|Tests|Passed|Failed|Skipped|Time|\n|:---|---:|---:|---:|---:|---:|\n') | Out-Null - foreach($appName in $appNames) { - $appTests = 0 - $appTime = 0.0 - $appFailed = 0 - $appSkipped = 0 - $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { $_.Value -eq $appName } } - foreach($suite in $suites) { - $appTests += [int]$suite.tests - $appFailed += [int]$suite.failures - $appSkipped += [int]$suite.skipped - $appTime += [decimal]::Parse($suite.time, [System.Globalization.CultureInfo]::InvariantCulture) - } - $appPassed = $appTests-$appFailed-$appSkipped - Write-Host "- $appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds" - $summarySb.Append("|$appName|$appTests|") | Out-Null - if ($appPassed -gt 0) { - $summarySb.Append("$($appPassed) :white_check_mark:") | Out-Null - } - $summarySb.Append("|") | Out-Null - if ($appFailed -gt 0) { - $summarySb.Append("$($appFailed) :x:") | Out-Null + if (Test-Path -Path $testResultsFile -PathType Leaf) { + $testResults = [xml](Get-Content -path $testResultsFile -Encoding UTF8) + $totalTests = 0 + $totalTime = 0.0 + $totalFailed = 0 + $totalSkipped = 0 + if ($testResults.testsuites) { + $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "appName" } | ForEach-Object { $_.Value } } | Select-Object -Unique) + if (-not $appNames) { + $appNames = @($testResults.testsuites.testsuite | ForEach-Object { $_.Properties.property | Where-Object { $_.Name -eq "extensionId" } | ForEach-Object { $_.Value } } | Select-Object -Unique) } - $summarySb.Append("|") | Out-Null - if ($appSkipped -gt 0) { - $summarySb.Append("$($appSkipped) :white_circle:") | Out-Null + foreach($testsuite in $testResults.testsuites.testsuite) { + $totalTests += $testsuite.Tests + $totalTime += [decimal]::Parse($testsuite.time, [System.Globalization.CultureInfo]::InvariantCulture) + $totalFailed += $testsuite.failures + $totalSkipped += $testsuite.skipped } - $summarySb.Append("|$($appTime)s|\n") | Out-Null - if ($appFailed -gt 0) { - $failuresSb.Append("
$appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds\n") | Out-Null + Write-Host "$($appNames.Count) TestApps, $totalTests tests, $totalFailed failed, $totalSkipped skipped, $totalTime seconds" + $summarySb.Append('|Test app|Tests|Passed|Failed|Skipped|Time|\n|:---|---:|---:|---:|---:|---:|\n') | Out-Null + foreach($appName in $appNames) { + $appTests = 0 + $appTime = 0.0 + $appFailed = 0 + $appSkipped = 0 + $suites = $testResults.testsuites.testsuite | where-Object { $_.Properties.property | Where-Object { ($_.Name -eq 'appName' -or $_.Name -eq 'extensionId') -and $_.Value -eq $appName } } foreach($suite in $suites) { - Write-Host " - $($suite.name), $($suite.tests) tests, $($suite.failures) failed, $($suite.skipped) skipped, $($suite.time) seconds" - if ($suite.failures -gt 0 -and $failuresSb.Length -lt 32000 -and $includeFailures -gt $failuresIncluded) { - $failuresSb.Append("
$($suite.name), $($suite.tests) tests, $($suite.failures) failed, $($suite.skipped) skipped, $($suite.time) seconds") | Out-Null - foreach($testcase in $suite.testcase) { - if ($testcase.ChildNodes.Count -gt 0) { - Write-Host " - $($testcase.name), Failure, $($testcase.time) seconds" - $failuresSb.Append("
$($testcase.name), Failure") | Out-Null - foreach($failure in $testcase.ChildNodes) { - Write-Host " - Error: $($failure.message)" - Write-Host " Stacktrace:" - Write-Host " $($failure."#text".Trim().Replace("`n","`n "))" - $failuresSb.Append("      Error: $($failure.message)
") | Out-Null - $failuresSb.Append("      Stack trace
") | Out-Null - $failuresSb.Append("      $($failure."#text".Trim().Replace("`n","
      "))
") | Out-Null + $appTests += [int]$suite.tests + $appFailed += [int]$suite.failures + $appSkipped += [int]$suite.skipped + $appTime += [decimal]::Parse($suite.time, [System.Globalization.CultureInfo]::InvariantCulture) + } + $appPassed = $appTests-$appFailed-$appSkipped + Write-Host "- $appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds" + $summarySb.Append("|$appName|$appTests|") | Out-Null + if ($appPassed -gt 0) { + $summarySb.Append("$($appPassed)$statusOK") | Out-Null + } + $summarySb.Append("|") | Out-Null + if ($appFailed -gt 0) { + $summarySb.Append("$($appFailed)$statusError") | Out-Null + } + $summarySb.Append("|") | Out-Null + if ($appSkipped -gt 0) { + $summarySb.Append("$($appSkipped)$statusSkipped") | Out-Null + } + $summarySb.Append("|$($appTime)s|\n") | Out-Null + if ($appFailed -gt 0) { + $failuresSb.Append("
$appName, $appTests tests, $appPassed passed, $appFailed failed, $appSkipped skipped, $appTime seconds\n") | Out-Null + foreach($suite in $suites) { + Write-Host " - $($suite.name), $($suite.tests) tests, $($suite.failures) failed, $($suite.skipped) skipped, $($suite.time) seconds" + if ($suite.failures -gt 0 -and $failuresSb.Length -lt 32000) { + $failuresSb.Append("
$($suite.name), $($suite.tests) tests, $($suite.failures) failed, $($suite.skipped) skipped, $($suite.time) seconds") | Out-Null + foreach($testcase in $suite.testcase) { + if ($testcase.ChildNodes.Count -gt 0) { + Write-Host " - $($testcase.name), Failure, $($testcase.time) seconds" + $failuresSb.Append("
$($testcase.name), Failure") | Out-Null + foreach($failure in $testcase.ChildNodes) { + Write-Host " - Error: $($failure.message)" + Write-Host " Stacktrace:" + Write-Host " $($failure."#text".Trim().Replace("`n","`n "))" + $failuresSb.Append("      Error: $($failure.message)
") | Out-Null + $failuresSb.Append("      Stack trace
") | Out-Null + $failuresSb.Append("      $($failure."#text".Trim().Replace("`n","
      "))
") | Out-Null + } + $failuresSb.Append("
") | Out-Null } - $failuresSb.Append("
") | Out-Null } + $failuresSb.Append("
") | Out-Null } - $failuresSb.Append("
") | Out-Null - $failuresIncluded++ } + $failuresSb.Append("
") | Out-Null } - $failuresSb.Append("
") | Out-Null } } - } - if ($totalFailed -gt 0) { - if ($totalFailed -gt $failuresIncluded) { - $failuresSb.Insert(0,"
$totalFailed failing tests (showing the first $failuresIncluded here, download test results to see all)") | Out-Null + if ($totalFailed -gt 0) { + $failuresSummaryMD = "$totalFailed failing tests, download test results to see details" + $failuresSb.Insert(0,"
$failuresSummaryMD") | Out-Null + $failuresSb.Append("
") | Out-Null } else { - $failuresSb.Insert(0,"
$totalFailed failing tests") | Out-Null + $failuresSummaryMD = "No test failures" + $failuresSb.Append($failuresSummaryMD) | Out-Null } - $failuresSb.Append("
") | Out-Null - if (($summarySb.Length + $failuresSb.Length) -lt 65000) { - $summarySb.Append("\n\n$($failuresSb.ToString())") | Out-Null + } + else { + $summarySb.Append("No test results found") | Out-Null + $failuresSummaryMD = '' + } + $summarySb.ToString() + $failuresSb.ToString() + $failuresSummaryMD +} + +function ReadBcptFile { + Param( + [string] $bcptTestResultsFile + ) + + if ((-not $bcptTestResultsFile) -or (-not (Test-Path -Path $bcptTestResultsFile -PathType Leaf))) { + return $null + } + + # Read BCPT file + $bcptResult = Get-Content -Path $bcptTestResultsFile -Encoding UTF8 | ConvertFrom-Json + $suites = [ordered]@{} + # Sort by bcptCode, codeunitID, operation + foreach($measure in $bcptResult) { + $bcptCode = $measure.bcptCode + $codeunitID = $measure.codeunitID + $codeunitName = $measure.codeunitName + $operation = $measure.operation + + # Create Suite if it doesn't exist + if(-not $suites.Contains($bcptCode)) { + $suites."$bcptCode" = [ordered]@{} } - else { - $summarySb.Append("\n\n$totalFailed failing tests. Download test results to see all") | Out-Null + # Create Codeunit under Suite if it doesn't exist + if (-not $suites."$bcptCode".Contains("$codeunitID")) { + $suites."$bcptCode"."$codeunitID" = @{ + "codeunitName" = $codeunitName + "operations" = [ordered]@{} + } } + # Create Operation under Codeunit if it doesn't exist + if (-not $suites."$bcptCode"."$codeunitID"."operations".Contains($operation)) { + $suites."$bcptCode"."$codeunitID"."operations"."$operation" = @{ + "measurements" = @() + } + } + # Add measurement to measurements under operation + $suites."$bcptCode"."$codeunitID"."operations"."$operation".measurements += @(@{ + "durationMin" = $measure.durationMin + "numberOfSQLStmts" = $measure.numberOfSQLStmts + }) + } + $suites +} + +function GetBcptSummaryMD { + Param( + [string] $bcptTestResultsFile, + [string] $baseLinePath = '', + [string] $thresholdsPath = '', + [int] $skipMeasurements = 0, + [hashtable] $bcptThresholds = $null + ) + + $bcpt = ReadBcptFile -bcptTestResultsFile $bcptTestResultsFile + if (-not $bcpt) { + return '' + } + $baseLine = ReadBcptFile -bcptTestResultsFile $baseLinePath + if ($baseLine) { + if ($null -eq $bcptThresholds) { + throw "Thresholds must be provided when comparing to a baseline" + } + # Override thresholds if thresholds file exists + if ($thresholdsPath -and (Test-Path -path $thresholdsPath)) { + Write-Host "Reading thresholds from $thresholdsPath" + $thresholds = Get-Content -Path $thresholdsPath -Encoding UTF8 | ConvertFrom-Json + foreach($threshold in 'durationWarning', 'durationError', 'numberOfSqlStmtsWarning', 'numberOfSqlStmtsError') { + if ($thresholds.PSObject.Properties.Name -eq $threshold) { + $bcptThresholds."$threshold" = $thresholds."$threshold" + } + } + } + Write-Host "Using thresholds:" + Write-Host "- DurationWarning: $($bcptThresholds.durationWarning)" + Write-Host "- DurationError: $($bcptThresholds.durationError)" + Write-Host "- NumberOfSqlStmtsWarning: $($bcptThresholds.numberOfSqlStmtsWarning)" + Write-Host "- NumberOfSqlStmtsError: $($bcptThresholds.numberOfSqlStmtsError)" + } + + $summarySb = [System.Text.StringBuilder]::new() + if ($baseLine) { + $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Status|Duration (ms)|Duration base (ms)|Duration diff (ms)|Duration diff|SQL Stmts|SQL Stmts base|SQL Stmts diff|SQL Stmts diff|\n") | Out-Null + $summarySb.Append("|:---------|:----------|:------------|:--------|:----:|------------:|-----------------:|-----------------:|------------:|--------:|-------------:|-------------:|-------------:|\n") | Out-Null } else { - $summarySb.Append("\n\nNo test failures") | Out-Null + $summarySb.Append("|BCPT Suite|Codeunit ID|Codeunit Name|Operation|Duration (ms)|SQL Stmts|\n") | Out-Null + $summarySb.Append("|:---------|:----------|:------------|:--------|------------:|--------:|\n") | Out-Null + } + + $lastSuiteName = '' + $lastCodeunitID = '' + $lastCodeunitName = '' + $lastOperationName = '' + + # calculate statistics on measurements, skipping the $skipMeasurements longest measurements + foreach($suiteName in $bcpt.Keys) { + $suite = $bcpt."$suiteName" + foreach($codeUnitID in $suite.Keys) { + $codeunit = $suite."$codeunitID" + $codeUnitName = $codeunit.codeunitName + foreach($operationName in $codeunit."operations".Keys) { + $operation = $codeunit."operations"."$operationName" + # Get measurements to use for statistics + $measurements = @($operation."measurements" | Sort-Object -Descending { $_.durationMin } | Select-Object -Skip $skipMeasurements) + # Calculate statistics and store them in the operation + $durationMin = ($measurements | ForEach-Object { $_.durationMin } | Measure-Object -Minimum).Minimum + $numberOfSQLStmts = ($measurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum + + $baseLineFound = $true + try { + $baseLineMeasurements = @($baseLine."$suiteName"."$codeUnitID"."operations"."$operationName"."measurements" | Sort-Object -Descending { $_.durationMin } | Select-Object -Skip $skipMeasurements) + if ($baseLineMeasurements.Count -eq 0) { + throw "No base line measurements" + } + $baseDurationMin = ($baseLineMeasurements | ForEach-Object { $_.durationMin } | Measure-Object -Minimum).Minimum + $diffDurationMin = $durationMin-$baseDurationMin + $baseNumberOfSQLStmts = ($baseLineMeasurements | ForEach-Object { $_.numberOfSQLStmts } | Measure-Object -Minimum).Minimum + $diffNumberOfSQLStmts = $numberOfSQLStmts-$baseNumberOfSQLStmts + } + catch { + $baseLineFound = $false + $baseDurationMin = $durationMin + $diffDurationMin = 0 + $baseNumberOfSQLStmts = $numberOfSQLStmts + $diffNumberOfSQLStmts = 0 + } + + $pctDurationMin = ($durationMin-$baseDurationMin)*100/$baseDurationMin + $durationMinStr = "$($durationMin.ToString("#"))|" + $baseDurationMinStr = "$($baseDurationMin.ToString("#"))|" + $diffDurationMinStr = "$($diffDurationMin.ToString("+#;-#;0"))|$($pctDurationMin.ToString('+#;-#;0'))%|" + + $pctNumberOfSQLStmts = ($numberOfSQLStmts-$baseNumberOfSQLStmts)*100/$baseNumberOfSQLStmts + $numberOfSQLStmtsStr = "$($numberOfSQLStmts.ToString("#"))|" + $baseNumberOfSQLStmtsStr = "$($baseNumberOfSQLStmts.ToString("#"))|" + $diffNumberOfSQLStmtsStr = "$($diffNumberOfSQLStmts.ToString("+#;-#;0"))|$($pctNumberOfSQLStmts.ToString('+#;-#;0'))%|" + + $thisOperationName = ''; if ($operationName -ne $lastOperationName) { $thisOperationName = $operationName } + $thisCodeunitName = ''; if ($codeunitName -ne $lastCodeunitName) { $thisCodeunitName = $codeunitName; $thisOperationName = $operationName } + $thisCodeunitID = ''; if ($codeunitID -ne $lastCodeunitID) { $thisCodeunitID = $codeunitID; $thisOperationName = $operationName } + $thisSuiteName = ''; if ($suiteName -ne $lastSuiteName) { $thisSuiteName = $suiteName; $thisOperationName = $operationName } + + if (!$baseLine) { + # No baseline provided + $statusStr = '' + $baseDurationMinStr = '' + $diffDurationMinStr = '' + $baseNumberOfSQLStmtsStr = '' + $diffNumberOfSQLStmtsStr = '' + } + else { + if (!$baseLineFound) { + # Baseline provided, but not found for this operation + $statusStr = $statusSkipped + $baseDurationMinStr = 'N/A|' + $diffDurationMinStr = '||' + $baseNumberOfSQLStmtsStr = 'N/A|' + $diffNumberOfSQLStmtsStr = '||' + } + else { + $statusStr = $statusOK + if ($pctDurationMin -ge $bcptThresholds.durationError) { + $statusStr = $statusError + if ($thisCodeunitName) { + # Only give errors and warnings on top level operation + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the error threshold of $($bcptThresholds.durationError)% for duration" + } + } + if ($pctNumberOfSQLStmts -ge $bcptThresholds.numberOfSqlStmtsError) { + $statusStr = $statusError + if ($thisCodeunitName) { + # Only give errors and warnings on top level operation + OutputError -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the error threshold of $($bcptThresholds.numberOfSqlStmtsError)% for number of SQL statements" + } + } + if ($statusStr -eq $statusOK) { + if ($pctDurationMin -ge $bcptThresholds.durationWarning) { + $statusStr = $statusWarning + if ($thisCodeunitName) { + # Only give errors and warnings on top level operation + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctDurationMin.ToString('N0'))%, which exceeds the warning threshold of $($bcptThresholds.durationWarning)% for duration" + } + } + if ($pctNumberOfSQLStmts -ge $bcptThresholds.numberOfSqlStmtsWarning) { + $statusStr = $statusWarning + if ($thisCodeunitName) { + # Only give errors and warnings on top level operation + OutputWarning -message "$operationName in $($suiteName):$codeUnitID degrades $($pctNumberOfSQLStmts.ToString('N0'))%, which exceeds the warning threshold of $($bcptThresholds.numberOfSqlStmtsWarning)% for number of SQL statements" + } + } + } + } + $statusStr += '|' + } + + $summarySb.Append("|$thisSuiteName|$thisCodeunitID|$thisCodeunitName|$thisOperationName|$statusStr$durationMinStr$baseDurationMinStr$diffDurationMinStr$numberOfSQLStmtsStr$baseNumberOfSQLStmtsStr$diffNumberOfSQLStmtsStr\n") | Out-Null + + $lastSuiteName = $suiteName + $lastCodeunitID = $codeUnitID + $lastCodeunitName = $codeUnitName + $lastOperationName = $operationName + } + } } - if ($summarySb.Length -lt 65500) { - $summarySb.ToString() + + if ($baseLine) { + $summarySb.Append("\nUsed baseline provided in $([System.IO.Path]::GetFileName($baseLinePath)).") | Out-Null } else { - "$totalFailed failing tests. Download test results to see all" + $summarySb.Append("\nNo baseline provided. Copy a set of BCPT results to $([System.IO.Path]::GetFileName($baseLinePath)) in the project folder in order to establish a baseline.") | Out-Null } + + $summarySb.ToString() } diff --git a/Actions/AnalyzeTests/action.yaml b/Actions/AnalyzeTests/action.yaml index 7e4f21123..17eb64b5f 100644 --- a/Actions/AnalyzeTests/action.yaml +++ b/Actions/AnalyzeTests/action.yaml @@ -11,17 +11,13 @@ inputs: default: '7b7d' project: description: Project to analyze - required: true -outputs: - TestResultMD: - description: MarkDown of test result - value: ${{ steps.AnalyzeTests.outputs.TestResultMD }} + required: false + default: '.' runs: using: composite steps: - name: run shell: ${{ inputs.shell }} - id: AnalyzeTests env: _parentTelemetryScopeJson: ${{ inputs.parentTelemetryScopeJson }} _project: ${{ inputs.project }} diff --git a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 index 9a312a3d7..4c1956b5c 100644 --- a/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 +++ b/Actions/CheckForUpdates/CheckForUpdates.HelperFunctions.ps1 @@ -28,7 +28,7 @@ function DownloadTemplateRepository { if ($downloadLatest) { # Get Branches from template repository - $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches" -retry + $response = InvokeWebRequest -Headers $headers -Uri "$apiUrl/branches?per_page=100" -retry $branchInfo = ($response.content | ConvertFrom-Json) | Where-Object { $_.Name -eq $branch } if (!$branchInfo) { throw "$templateUrl doesn't exist" diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 30dcf2c79..6ff757800 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -1,4 +1,19 @@ +### Business Central Performance Toolkit Test Result Viewer + +In the summary after a Test Run, you now also have the result of performance tests. + +### New Settings + +- `bcptThresholds` is a JSON object with properties for the default thresholds for the Business Central Performance Toolkit + - **DurationWarning** - a warning is issued if the duration of a bcpt test degrades more than this percentage (default 10) + - **DurationError** - an error is issued if the duration of a bcpt test degrades more than this percentage (default 25) + - **NumberOfSqlStmtsWarning** - a warning is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 5) + - **NumberOfSqlStmtsError** - an error is issued if the number of SQL statements from a bcpt test increases more than this percentage (default 10) + +> [!NOTE] +> Duration thresholds are subject to varying results depending on the performance of the agent running the tests. Number of SQL statements executed by a test is often the most reliable indicator of performance degredation. + ## v5.2 ### Issues diff --git a/Scenarios/AddAPerformanceTestApp.md b/Scenarios/AddAPerformanceTestApp.md index d57f04ae8..8c27fc9a7 100644 --- a/Scenarios/AddAPerformanceTestApp.md +++ b/Scenarios/AddAPerformanceTestApp.md @@ -2,16 +2,55 @@ *Prerequisites: A completed [scenario 1](GetStarted.md)* 1. On **github.com**, open **Actions** on your solution, select **Create a new performance test app** and then choose **Run workflow**. Enter values for **name**, **publisher**, and **ID range** and choose **Run workflow** -![Run Workflow](https://github.com/microsoft/AL-Go/assets/10775043/40f9bda7-578b-4844-9c2b-f59200d04584) -1. When the workflow is done, navigate to **Pull Requests**, **inspect the PR** and **Merge the pull request** -![Pull Request](https://github.com/microsoft/AL-Go/assets/10775043/e97ef897-93f4-4c9f-9ce2-e747d7021003) -1. Under **Actions**, you will see that a Merge pull request **CI workflow** has been kicked off -![Workflows](https://github.com/microsoft/AL-Go/assets/10775043/90ee2dee-4e2a-4d16-80bc-63d3ce1f53b5) -1. If you wait for the workflow to complete, you will see that it completes and one of the build artifacts are the **BCPT Test Results** -![Fail](https://github.com/microsoft/AL-Go/assets/10775043/ad154e32-34d4-49f1-a8de-e74ed5a79217) -1. Opening the **BCPT Test Results** and inspecting the results looks like this -![Test failure](https://github.com/microsoft/AL-Go/assets/10775043/0869601d-55e6-4e1d-9d1e-fb1a2c0c6b05) -1. Currently there isn't a visual viewer of these results. The goal is to have a PowerBI dashboard, which can gather BCPT test results from multiple builds and compare. + + ![Run workflow](https://github.com/microsoft/AL-Go/assets/10775043/d499294e-8c88-4f2d-9bb4-b34bad276a6b) + + Note that if workflows are not allowed to create pull requests due to GitHub Settings, you can create the PR manually by following the link in the annotation + + ![Annotation](https://github.com/microsoft/AL-Go/assets/10775043/d346f0fc-5db4-4ff1-9c76-e93cb03ae504) + +2. When the workflow is done, navigate to **Pull Requests**, **inspect the PR** and **Merge the pull request** + + ![Pull Request](https://github.com/microsoft/AL-Go/assets/10775043/d2831620-3bc9-4808-aa7a-997944aaaa33) + +3. Under **Actions**, you will see that a Merge pull request **CI/CD workflow** has been kicked off + + ![Workflows](https://github.com/microsoft/AL-Go/assets/10775043/37f6c5b9-aaac-4cdc-b1d0-ef661cd2bfbe) + +4. If you wait for the workflow to complete, you will see that it completes and one of the build artifacts are the **BCPT Test Results** + + ![BCPT Test Results](https://github.com/microsoft/AL-Go/assets/10775043/cb206f91-3b83-4000-987c-39faa9765695) + +5. Opening the **BCPT Test Results** and inspecting the results looks like this + + ![BCPT Test Results.json](https://github.com/microsoft/AL-Go/assets/10775043/27acb70c-1ead-4832-b22a-b022c578250d) + +6. Scrolling down further reveals the Performance Test Results in a table, which also indicates that if we want to set a baseline for comparing future BCPT Test Results, we need to add a `bcptBaseLine.json` file in the project folder. + + ![BCPT Test Results viewer](https://github.com/microsoft/AL-Go/assets/10775043/4b263e9e-7ec9-4101-92a7-046e7807e797) + +7. After uploading a `bcptBaseLine.json`, to the project root (which is the repo root in single project repositories), another CI/CD workflow will be kicked off, which now compares the results with the baseline: + + ![With BaseLine](https://github.com/microsoft/AL-Go/assets/10775043/c00840d5-4c67-4a72-a4d9-cdebe62e54c0) + + Where negative numbers in the diff fields indicates faster execution or lower number of SQL statements than the baseline. + +> [!NOTE] +> +> You can specify thresholds for performance testing in project settings (see [https://aka.ms/algosettings#bcptThresholds](https://aka.ms/algosettings#bcptThresholds)) or in a file called `bcptThresholds.json`, which should be located next to the `bcptBaseLine.json` file. + +8. After uploading a `bcptThresholds.json` file with this content: + + ``` + { + "durationWarning": 0, + "durationError": 1 + } + ``` + + The CI/CD workflow now uses these thresholds for the CI/CD run: + + ![Warnings and Error](https://github.com/microsoft/AL-Go/assets/10775043/be85d4c1-c710-410d-aba3-b55de8750396) --- [back](../README.md) diff --git a/Scenarios/settings.md b/Scenarios/settings.md index 80b320157..7db7706d1 100644 --- a/Scenarios/settings.md +++ b/Scenarios/settings.md @@ -33,6 +33,7 @@ When running a workflow or a local script, the settings are applied by reading s | bcptTestFolders | bcptTestFolders should be an array of folders (relative to project root), which contains performance test apps for this project. Apps in these folders are sorted based on dependencies and built, published and bcpt tests are run in that order.
If bcptTestFolders are not specified, AL-Go for GitHub will try to locate bcptTestFolders in the root of the project. | [ ] | | appDependencyProbingPaths | Array of dependency specifications, from which apps will be downloaded when the CI/CD workflow is starting. Every dependency specification consists of the following properties:
**repo** = repository
**version** = version (default latest)
**release_status** = latestBuild/release/prerelease/draft (default release)
**projects** = projects (default * = all)
**branch** = branch (default main)
**AuthTokenSecret** = Name of secret containing auth token (default none)
| [ ] | | cleanModePreprocessorSymbols | List of clean tags to be used in _Clean_ build mode | [ ] | +| bcptThresholds | Structure with properties for the thresholds when running performance tests using the Business Central Performance Toolkit.
**DurationWarning** = a warning is shown if the duration of a bcpt test degrades more than this percentage (default 10)
**DurationError** - an error is shown if the duration of a bcpt test degrades more than this percentage (default 25)
**NumberOfSqlStmtsWarning** - a warning is shown if the number of SQL statements from a bcpt test increases more than this percentage (default 5)
**NumberOfSqlStmtsError** - an error is shown if the number of SQL statements from a bcpt test increases more than this percentage (default 10)
*Note that errors and warnings on the build in GitHub are only issued when a threshold is exceeded on the codeunit level, when an individual operation threshold is exceeded, it is only shown in the test results viewer.* | ## AppSource specific basic project settings | Name | Description | Default value | diff --git a/Templates/AppSource App/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml b/Templates/AppSource App/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml index 5984a0b2e..45cd0f1c4 100644 --- a/Templates/AppSource App/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml +++ b/Templates/AppSource App/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml @@ -83,7 +83,7 @@ jobs: $settings = $env:Settings | ConvertFrom-Json if ('${{ fromJson(steps.ReadSecrets.outputs.Secrets).adminCenterApiCredentials }}') { Write-Host "AdminCenterApiCredentials provided in secret $($settings.adminCenterApiCredentialsSecretName)!" - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "Admin Center Api Credentials was provided in a secret called $($settings.adminCenterApiCredentialsSecretName). Using this information for authentication." + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "Admin Center Api Credentials was provided in a secret called $($settings.adminCenterApiCredentialsSecretName). Using this information for authentication." } else { Write-Host "AdminCenterApiCredentials not provided, initiating Device Code flow" @@ -93,7 +93,7 @@ jobs: . $ALGoHelperPath DownloadAndImportBcContainerHelper $authContext = New-BcAuthContext -includeDeviceLogin -deviceLoginTimeout ([TimeSpan]::FromSeconds(0)) - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Admin Center Api and could not locate a secret called $($settings.adminCenterApiCredentialsSecretName) (https://aka.ms/ALGoSettings#AdminCenterApiCredentialsSecretName)`n`n$($authContext.message)" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Admin Center Api and could not locate a secret called $($settings.adminCenterApiCredentialsSecretName) (https://aka.ms/ALGoSettings#AdminCenterApiCredentialsSecretName)`n`n$($authContext.message)" Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "deviceCode=$($authContext.deviceCode)" } diff --git a/Templates/AppSource App/.github/workflows/PublishToEnvironment.yaml b/Templates/AppSource App/.github/workflows/PublishToEnvironment.yaml index cbc851345..06f4e3e15 100644 --- a/Templates/AppSource App/.github/workflows/PublishToEnvironment.yaml +++ b/Templates/AppSource App/.github/workflows/PublishToEnvironment.yaml @@ -101,7 +101,7 @@ jobs: } if ($authContext) { Write-Host "AuthContext provided in secret $secretName!" - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AuthContext was provided in a secret called $secretName. Using this information for authentication." + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AuthContext was provided in a secret called $secretName. Using this information for authentication." } else { Write-Host "No AuthContext provided for $envName, initiating Device Code flow" @@ -111,7 +111,7 @@ jobs: . $ALGoHelperPath DownloadAndImportBcContainerHelper $authContext = New-BcAuthContext -includeDeviceLogin -deviceLoginTimeout ([TimeSpan]::FromSeconds(0)) - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Environment $('${{ steps.envName.outputs.envName }}'.Split(' ')[0]) and could not locate a secret called ${{ steps.envName.outputs.envName }}_AuthContext`n`n$($authContext.message)" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Environment $('${{ steps.envName.outputs.envName }}'.Split(' ')[0]) and could not locate a secret called ${{ steps.envName.outputs.envName }}_AuthContext`n`n$($authContext.message)" Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "deviceCode=$($authContext.deviceCode)" } diff --git a/Templates/Per Tenant Extension/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml b/Templates/Per Tenant Extension/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml index 5984a0b2e..45cd0f1c4 100644 --- a/Templates/Per Tenant Extension/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/CreateOnlineDevelopmentEnvironment.yaml @@ -83,7 +83,7 @@ jobs: $settings = $env:Settings | ConvertFrom-Json if ('${{ fromJson(steps.ReadSecrets.outputs.Secrets).adminCenterApiCredentials }}') { Write-Host "AdminCenterApiCredentials provided in secret $($settings.adminCenterApiCredentialsSecretName)!" - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "Admin Center Api Credentials was provided in a secret called $($settings.adminCenterApiCredentialsSecretName). Using this information for authentication." + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "Admin Center Api Credentials was provided in a secret called $($settings.adminCenterApiCredentialsSecretName). Using this information for authentication." } else { Write-Host "AdminCenterApiCredentials not provided, initiating Device Code flow" @@ -93,7 +93,7 @@ jobs: . $ALGoHelperPath DownloadAndImportBcContainerHelper $authContext = New-BcAuthContext -includeDeviceLogin -deviceLoginTimeout ([TimeSpan]::FromSeconds(0)) - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Admin Center Api and could not locate a secret called $($settings.adminCenterApiCredentialsSecretName) (https://aka.ms/ALGoSettings#AdminCenterApiCredentialsSecretName)`n`n$($authContext.message)" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Admin Center Api and could not locate a secret called $($settings.adminCenterApiCredentialsSecretName) (https://aka.ms/ALGoSettings#AdminCenterApiCredentialsSecretName)`n`n$($authContext.message)" Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "deviceCode=$($authContext.deviceCode)" } diff --git a/Templates/Per Tenant Extension/.github/workflows/PublishToEnvironment.yaml b/Templates/Per Tenant Extension/.github/workflows/PublishToEnvironment.yaml index cbc851345..06f4e3e15 100644 --- a/Templates/Per Tenant Extension/.github/workflows/PublishToEnvironment.yaml +++ b/Templates/Per Tenant Extension/.github/workflows/PublishToEnvironment.yaml @@ -101,7 +101,7 @@ jobs: } if ($authContext) { Write-Host "AuthContext provided in secret $secretName!" - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AuthContext was provided in a secret called $secretName. Using this information for authentication." + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AuthContext was provided in a secret called $secretName. Using this information for authentication." } else { Write-Host "No AuthContext provided for $envName, initiating Device Code flow" @@ -111,7 +111,7 @@ jobs: . $ALGoHelperPath DownloadAndImportBcContainerHelper $authContext = New-BcAuthContext -includeDeviceLogin -deviceLoginTimeout ([TimeSpan]::FromSeconds(0)) - Set-Content -Path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Environment $('${{ steps.envName.outputs.envName }}'.Split(' ')[0]) and could not locate a secret called ${{ steps.envName.outputs.envName }}_AuthContext`n`n$($authContext.message)" + Add-Content -Encoding UTF8 -path $ENV:GITHUB_STEP_SUMMARY -value "AL-Go needs access to the Business Central Environment $('${{ steps.envName.outputs.envName }}'.Split(' ')[0]) and could not locate a secret called ${{ steps.envName.outputs.envName }}_AuthContext`n`n$($authContext.message)" Add-Content -Encoding UTF8 -Path $env:GITHUB_OUTPUT -Value "deviceCode=$($authContext.deviceCode)" } diff --git a/Tests/AnalyzeTests.Test.ps1 b/Tests/AnalyzeTests.Test.ps1 new file mode 100644 index 000000000..7f0b542b8 --- /dev/null +++ b/Tests/AnalyzeTests.Test.ps1 @@ -0,0 +1,145 @@ +Get-Module TestActionsHelper | Remove-Module -Force +Import-Module (Join-Path $PSScriptRoot 'TestActionsHelper.psm1') + +Describe "AnalyzeTests Action Tests" { + BeforeAll { + function GetBcptTestResultFile { + Param( + [int] $noOfSuites = 1, + [int] $noOfCodeunits = 1, + [int] $noOfOperations = 1, + [int] $noOfMeasurements = 1, + [int] $durationOffset = 0, + [int] $numberOfSQLStmtsOffset = 0 + ) + + $bcpt = @() + for($suiteNo = 1; $suiteNo -le $noOfSuites; $suiteNo++) { + $suiteName = "SUITE$suiteNo" + for($codeUnitID = 1; $codeunitID -le $noOfCodeunits; $codeunitID++) { + $codeunitName = "Codeunit$codeunitID" + for($operationNo = 1; $operationNo -le $noOfOperations; $operationNo++) { + $operationName = "Operation$operationNo" + for($no = 1; $no -le $noOfMeasurements; $no++) { + $bcpt += @(@{ + "id" = [GUID]::NewGuid().ToString() + "bcptCode" = $suiteName + "codeunitID" = $codeunitID + "codeunitName" = $codeunitName + "operation" = $operationName + "durationMin" = $operationNo*10+$no+$durationOffset + "numberOfSQLStmts" = $operationNo+$numberOfSQLStmtsOffset + }) + } + } + } + } + $filename = Join-Path ([System.IO.Path]::GetTempPath()) "$([GUID]::NewGuid().ToString()).json" + $bcpt | ConvertTo-Json -Depth 100 | Set-Content -Path $filename -Encoding UTF8 + return $filename + } + + $actionName = "AnalyzeTests" + $scriptRoot = Join-Path $PSScriptRoot "..\Actions\$actionName" -Resolve + $scriptName = "$actionName.ps1" + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'actionScript', Justification = 'False positive.')] + $actionScript = GetActionScript -scriptRoot $scriptRoot -scriptName $scriptName + + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'bcptFilename', Justification = 'False positive.')] + $bcptFilename = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 5 -noOfMeasurements 4 + # BaseLine1 has overall highter duration and more SQL statements than bcptFilename (+ one more opearion) + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'bcptBaseLine1', Justification = 'False positive.')] + $bcptBaseLine1 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 4 -noOfOperations 6 -noOfMeasurements 4 -durationOffset 5 -numberOfSQLStmtsOffset 1 + # BaseLine2 has overall lower duration and less SQL statements than bcptFilename (+ one less opearion) + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'bcptBaseLine2', Justification = 'False positive.')] + $bcptBaseLine2 = GetBcptTestResultFile -noOfSuites 1 -noOfCodeunits 2 -noOfOperations 4 -noOfMeasurements 4 -durationOffset -2 -numberOfSQLStmtsOffset 0 + [Diagnostics.CodeAnalysis.SuppressMessageAttribute('PSUseDeclaredVarsMoreThanAssignments', 'thresholdsFile', Justification = 'False positive.')] + $thresholdsFile = Join-Path ([System.IO.Path]::GetTempPath()) "$([GUID]::NewGuid().ToString()).json" + @{ "NumberOfSqlStmtsThresholdWarning" = 1; "NumberOfSqlStmtsThresholdError" = 2 } | ConvertTo-Json | Set-Content -Path $thresholdsFile -Encoding UTF8 + } + + It 'Compile Action' { + Invoke-Expression $actionScript + } + + It 'Test action.yaml matches script' { + $permissions = [ordered]@{ + } + $outputs = [ordered]@{ + } + YamlTest -scriptRoot $scriptRoot -actionName $actionName -actionScript $actionScript -permissions $permissions -outputs $outputs + } + + It 'Test ReadBcptFile' { + . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') + . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') + $bcpt = ReadBcptFile -bcptTestResultsFile $bcptFilename + $bcpt.Count | should -Be 1 + $bcpt."SUITE1".Count | should -Be 2 + $bcpt."SUITE1"."1".operations.Count | should -Be 5 + $bcpt."SUITE1"."1".operations."operation2".measurements.Count | should -Be 4 + } + + It 'Test GetBcptSummaryMD (no baseline)' { + . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') + . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') + $md = GetBcptSummaryMD -bcptTestResultsFile $bcptFilename + Write-Host $md.Replace('\n',"`n") + $md | should -Match 'No baseline provided' + $columns = 6 + $rows = 12 + [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 + [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 + [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 + [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) + } + + It 'Test GetBcptSummaryMD (with worse baseline)' { + . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') + . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') + $md = GetBcptSummaryMD -bcptTestResultsFile $bcptFilename -baselinePath $bcptBaseLine1 -bcptThresholds @{"durationWarning"=10;"durationError"=25;"numberOfSqlStmtsWarning"=5;"numberOfSqlStmtsError"=10} + Write-Host $md.Replace('\n',"`n") + $md | should -Not -Match 'No baseline provided' + $columns = 13 + $rows = 12 + [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 + [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 + [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 + [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 10 + [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 0 + [regex]::Matches($md, "\|$statusError\|").Count | should -Be 0 + [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) + } + + It 'Test GetBcptSummaryMD (with better baseline)' { + . (Join-Path $scriptRoot '../AL-Go-Helper.ps1') + . (Join-Path $scriptRoot 'TestResultAnalyzer.ps1') + + $script:errorCount = 0 + Mock OutputError { Param([string] $message) Write-Host "ERROR: $message"; $script:errorCount++ } + $script:warningCount = 0 + Mock OutputWarning { Param([string] $message) Write-Host "WARNING: $message"; $script:warningCount++ } + + $md = GetBcptSummaryMD -bcptTestResultsFile $bcptFilename -baselinePath $bcptBaseLine2 -thresholdsPath $thresholdsFile -bcptThresholds @{"durationWarning"=5;"durationError"=10;"numberOfSqlStmtsWarning"=5;"numberOfSqlStmtsError"=10} + Write-Host $md.Replace('\n',"`n") + $md | should -Not -Match 'No baseline provided' + $columns = 13 + $rows = 12 + [regex]::Matches($md, '\|SUITE1\|').Count | should -Be 1 + [regex]::Matches($md, '\|Codeunit.\|').Count | should -Be 2 + [regex]::Matches($md, '\|Operation.\|').Count | should -Be 10 + [regex]::Matches($md, '\|N\/A\|').Count | should -Be 4 + [regex]::Matches($md, "\|$statusOK\|").Count | should -Be 0 + [regex]::Matches($md, "\|$statusWarning\|").Count | should -Be 4 + [regex]::Matches($md, "\|$statusError\|").Count | should -Be 4 + [regex]::Matches($md, '\|').Count | should -Be (($columns+1)*$rows) + $script:errorCount | Should -be 2 + $script:warningCount | Should -be 0 + } + + AfterAll { + Remove-Item -Path $bcptFilename -Force -ErrorAction SilentlyContinue + Remove-Item -Path $bcptBaseLine1 -Force -ErrorAction SilentlyContinue + Remove-Item -Path $bcptBaseLine2 -Force -ErrorAction SilentlyContinue + } +}