Merge branch 'release/1.1'

DaedalicEntertainment · Jul 9, 2020 · c60cde4 · c60cde4
2 parents 8546898 + df416e5
commit c60cde4
Show file tree

Hide file tree

Showing 67 changed files with 1,395 additions and 279 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -37,7 +37,7 @@ Internally, we're using the following checklist when preparing for a new release
 * Create release branch
 * Add examples for new features where appropriate
 * Run all tests
-* Update documentation (README, images, spelling, table of contents)
+* Update documentation (README, images, code samples, spelling, table of contents)
 * Increase version number (and engine version, if necessary)
 * Create plugin package
 * Check plugin package in another project

diff --git a/DaedalicTestAutomationPlugin.Automation/DaeTestConfig.cs b/DaedalicTestAutomationPlugin.Automation/DaeTestConfig.cs
@@ -11,6 +11,12 @@ public class DaeTestConfig : EpicGame.EpicGameTestConfig
 		[AutoParam]
         public string JUnitReportPath;
 
+        /// <summary>
+        /// Where to write test reports to.
+        /// </summary>
+        [AutoParam]
+        public string ReportPath;
+
         /// <summary>
         /// Which single test to run, instead of all available tests.
         /// </summary>
@@ -26,6 +32,11 @@ public override void ApplyToConfig(UnrealAppConfig AppConfig, UnrealSessionRole
                 AppConfig.CommandLine += string.Format(" JUnitReportPath=\"{0}\"", JUnitReportPath);
             }
 
+            if (!string.IsNullOrEmpty(ReportPath))
+            {
+                AppConfig.CommandLine += string.Format(" ReportPath=\"{0}\"", ReportPath);
+            }
+
             if (!string.IsNullOrEmpty(TestName))
             {
                 AppConfig.CommandLine += string.Format(" TestName=\"{0}\"", TestName);

diff --git a/DaedalicTestAutomationPlugin/Content/ReportTemplates/PerformanceReport.template.html b/DaedalicTestAutomationPlugin/Content/ReportTemplates/PerformanceReport.template.html
@@ -0,0 +1,24 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+    <link rel="stylesheet" href="bootstrap.min.css">
+    <title>Performance Report</title>
+  </head>
+  <body>
+    <div class="container">
+      <h1>Performance Report</h1>
+      <div class="row">
+        <div class="col-3"><strong>Start Time:</strong></div>
+        <div class="col-3">{START_TIME}</div>
+      </div>
+      <div class="row">
+        <div class="col-3"><strong>Total Duration (Seconds):</strong></div>
+        <div class="col-3">{TOTAL_DURATION}</div>
+      </div>
+
+{MAP_RESULTS}
+    </div>
+  </body>
+</html>
diff --git a/...stAutomationPlugin/Content/ReportTemplates/PerformanceReportBudgetViolation.template.html b/...stAutomationPlugin/Content/ReportTemplates/PerformanceReportBudgetViolation.template.html
@@ -0,0 +1,10 @@
+          <tr>
+            <td>{LOCATION}</td>
+            <td>{PREVIOUS}</td>
+            <td>{NEXT}</td>
+            <td>{FPS}</td>
+            <td>{GAME_TIME}&nbsp;ms</td>
+            <td>{RENDER_TIME}&nbsp;ms</td>
+            <td>{GPU_TIME}&nbsp;ms</td>
+            <td><a href="{SCREENSHOT_PATH}" target="_blank"><img src="{SCREENSHOT_PATH}" class="img-fluid" alt="{SCREENSHOT_PATH}"/></a></td>
+          </tr>
diff --git a/DaedalicTestAutomationPlugin/Content/ReportTemplates/PerformanceReportMap.template.html b/DaedalicTestAutomationPlugin/Content/ReportTemplates/PerformanceReportMap.template.html
@@ -0,0 +1,24 @@
+      <p></p>
+      <h2>{MAP_NAME}</h2>
+      <div class="row">
+        <div class="col-3"><strong>Duration (Seconds):</strong></div>
+        <div class="col-3">{MAP_DURATION}</div>
+      </div>
+      <p></p>
+      <table class="table table-striped">
+        <thead>
+          <tr>
+            <th scope="col">Location</th>
+            <th scope="col">Between</th>
+            <th scope="col">And</th>
+            <th scope="col">FPS</th>
+            <th scope="col">Game</th>
+            <th scope="col">Render</th>
+            <th scope="col">GPU</th>
+            <th scope="col">Screenshot</th>
+          </tr>
+        </thead>
+        <tbody>
+{BUDGET_VIOLATIONS}
+        </tbody>
+      </table>
diff --git a/DaedalicTestAutomationPlugin/Content/ReportTemplates/bootstrap.min.css b/DaedalicTestAutomationPlugin/Content/ReportTemplates/bootstrap.min.css
diff --git a/DaedalicTestAutomationPlugin/Content/TestPerformance/BP_TestPerformancePawn.uasset b/DaedalicTestAutomationPlugin/Content/TestPerformance/BP_TestPerformancePawn.uasset
diff --git a/DaedalicTestAutomationPlugin/Content/TestPerformance/TestPerformance.umap b/DaedalicTestAutomationPlugin/Content/TestPerformance/TestPerformance.umap
diff --git a/DaedalicTestAutomationPlugin/DaedalicTestAutomationPlugin.uplugin b/DaedalicTestAutomationPlugin/DaedalicTestAutomationPlugin.uplugin
@@ -1,14 +1,14 @@
 {
   "FileVersion": 3,
   "Version": 1,
-  "VersionName": "1.0.0",
+  "VersionName": "1.1.0",
   "FriendlyName": "Daedalic Test Automation Plugin",
-  "Description": "Facilitates setting up integration test suits with Gauntlet.",
+  "Description": "Facilitates setting up integration test suites with Gauntlet.",
   "Category": "Daedalic Entertainment",
   "CreatedBy": "Daedalic Entertainment GmbH",
   "CreatedByURL": "https://www.daedalic.com/",
   "DocsURL": "https://github.com/DaedalicEntertainment/ue4-test-automation",
-  "EngineVersion": "4.23",
+  "EngineVersion": "4.23.0",
   "MarketplaceURL": "",
   "SupportURL": "https://github.com/DaedalicEntertainment/ue4-test-automation/issues",
   "EnabledByDefault": false,

diff --git a/...utomationPlugin/Source/DaedalicTestAutomationPlugin/DaedalicTestAutomationPlugin.Build.cs b/...utomationPlugin/Source/DaedalicTestAutomationPlugin/DaedalicTestAutomationPlugin.Build.cs
@@ -29,7 +29,9 @@ public DaedalicTestAutomationPlugin(ReadOnlyTargetRules Target) : base(Target)
                     "InputCore",
                     "Gauntlet",
                     "UMG",
-                    "SlateCore"
+                    "SlateCore",
+                    "RenderCore",
+                    "Projects"
                 }
 				);
 

diff --git a/...utomationPlugin/Source/DaedalicTestAutomationPlugin/Private/DaeGauntletTestController.cpp b/...utomationPlugin/Source/DaedalicTestAutomationPlugin/Private/DaeGauntletTestController.cpp
@@ -1,9 +1,10 @@
 #include "DaeGauntletTestController.h"
 #include "Kismet/GameplayStatics.h"
 #include "DaeGauntletStates.h"
-#include "DaeJUnitReportWriter.h"
 #include "DaeTestAutomationPluginSettings.h"
 #include "DaeTestLogCategory.h"
+#include "DaeTestReportWriter.h"
+#include "DaeTestReportWriterSet.h"
 #include "DaeTestSuiteActor.h"
 #include <AssetRegistryModule.h>
 #include <EngineUtils.h>
@@ -18,8 +19,10 @@ void UDaeGauntletTestController::OnInit()
     const UDaeTestAutomationPluginSettings* TestAutomationPluginSettings =
         GetDefault<UDaeTestAutomationPluginSettings>();
 
-    UE_LOG(LogDaeTest, Display, TEXT("Discovering tests from: %s"),
-           *TestAutomationPluginSettings->TestMapPath);
+    for (const FString& TestMapFolder : TestAutomationPluginSettings->TestMapFolders)
+    {
+        UE_LOG(LogDaeTest, Display, TEXT("Discovering tests from: %s"), *TestMapFolder);
+    }
 
     // Build list of tests (based on FAutomationEditorCommonUtils::CollectTestsByClass).
     FAssetRegistryModule& AssetRegistryModule =
@@ -29,24 +32,37 @@ void UDaeGauntletTestController::OnInit()
     AssetRegistryModule.Get().SearchAllAssets(true);
     AssetRegistryModule.Get().GetAssetsByClass(UWorld::StaticClass()->GetFName(), AssetDataArray);
 
-    const FString PatternToCheck =
-        FString::Printf(TEXT("/%s/"), *TestAutomationPluginSettings->TestMapPath);
-
     for (auto ObjIter = AssetDataArray.CreateConstIterator(); ObjIter; ++ObjIter)
     {
         const FAssetData& AssetData = *ObjIter;
 
-        FString Filename = FPackageName::LongPackageNameToFilename(AssetData.ObjectPath.ToString());
+        FString FileName = FPackageName::LongPackageNameToFilename(AssetData.ObjectPath.ToString());
+        FName MapName = AssetData.AssetName;
 
-        if (Filename.Contains(*PatternToCheck))
+        bool bIsTestMap = TestAutomationPluginSettings->IsTestMap(FileName, MapName);
+
+        if (bIsTestMap)
         {
-            FName MapName = AssetData.AssetName;
             MapNames.Add(MapName);
 
             UE_LOG(LogDaeTest, Display, TEXT("Discovered test: %s"), *MapName.ToString());
         }
     }
 
+    // Set console variables.
+    for (auto& ConsoleVariable : TestAutomationPluginSettings->ConsoleVariables)
+    {
+        IConsoleVariable* CVar = IConsoleManager::Get().FindConsoleVariable(*ConsoleVariable.Key);
+
+        if (CVar)
+        {
+            CVar->Set(*ConsoleVariable.Value);
+
+            UE_LOG(LogDaeTest, Log, TEXT("Setting console variable %s = %s"), *ConsoleVariable.Key,
+                   *ConsoleVariable.Value);
+        }
+    }
+
     GetGauntlet()->BroadcastStateChange(FDaeGauntletStates::Initialized);
 }
 
@@ -150,20 +166,6 @@ void UDaeGauntletTestController::LoadNextTestMap()
         UE_LOG(LogDaeTest, Display,
                TEXT("UDaeGauntletTestController::LoadNextTestMap - All tests finished."));
 
-        // Write final test report.
-        FDaeJUnitReportWriter JUnitReportWriter;
-        FString TestReport =
-            JUnitReportWriter.CreateReport(TEXT("DaeGauntletTestController"), Results);
-        UE_LOG(LogDaeTest, Log, TEXT("Test report:\r\n%s"), *TestReport);
-
-        FString JUnitReportPath = ParseCommandLineOption(TEXT("JUnitReportPath"));
-
-        if (!JUnitReportPath.IsEmpty())
-        {
-            UE_LOG(LogDaeTest, Display, TEXT("Writing test report to: %s"), *JUnitReportPath);
-            FFileHelper::SaveStringToFile(TestReport, *JUnitReportPath);
-        }
-
         // Finish Gauntlet.
         GetGauntlet()->BroadcastStateChange(FDaeGauntletStates::Finished);
 
@@ -182,7 +184,20 @@ void UDaeGauntletTestController::LoadNextTestMap()
 
 void UDaeGauntletTestController::OnTestSuiteFinished(ADaeTestSuiteActor* TestSuite)
 {
+    // Store result.
     Results.Add(TestSuite->GetResult());
+
+    // Update test reports on disk.
+    FString ReportPath = ParseCommandLineOption(TEXT("ReportPath"));
+
+    FDaeTestReportWriterSet ReportWriters = TestSuite->GetReportWriters();
+
+    for (TSharedPtr<FDaeTestReportWriter> ReportWriter : ReportWriters.GetReportWriters())
+    {
+        ReportWriter->WriteReport(Results, ReportPath);
+    }
+
+    // Proceed with next test.
     LoadNextTestMap();
 }
 

diff --git a/DaedalicTestAutomationPlugin/Source/DaedalicTestAutomationPlugin/Private/DaeTestActor.cpp b/DaedalicTestAutomationPlugin/Source/DaedalicTestAutomationPlugin/Private/DaeTestActor.cpp
@@ -1,6 +1,8 @@
 #include "DaeTestActor.h"
 #include "DaeTestLogCategory.h"
 #include "DaeTestParameterProviderActor.h"
+#include "DaeTestReportWriterJUnit.h"
+#include "DaeTestResult.h"
 
 ADaeTestActor::ADaeTestActor(
     const FObjectInitializer& ObjectInitializer /*= FObjectInitializer::Get()*/)
@@ -92,6 +94,18 @@ UObject* ADaeTestActor::GetCurrentParameter() const
     return CurrentParameter;
 }
 
+TSharedPtr<FDaeTestResultData> ADaeTestActor::CollectResults() const
+{
+    return MakeShareable(new FDaeTestResultData());
+}
+
+FDaeTestReportWriterSet ADaeTestActor::GetReportWriters() const
+{
+    FDaeTestReportWriterSet ReportWriters;
+    ReportWriters.Add(MakeShareable(new FDaeTestReportWriterJUnit()));
+    return ReportWriters;
+}
+
 void ADaeTestActor::NotifyOnTestSuccessful()
 {
     if (bHasResult)

diff --git a/...gin/Source/DaedalicTestAutomationPlugin/Private/DaeTestAssertBlueprintFunctionLibrary.cpp b/...gin/Source/DaedalicTestAutomationPlugin/Private/DaeTestAssertBlueprintFunctionLibrary.cpp
@@ -192,20 +192,21 @@ void UDaeTestAssertBlueprintFunctionLibrary::AssertCompareInt64(int64 First,
 
 void UDaeTestAssertBlueprintFunctionLibrary::AssertEqualFloat(float Actual, float Expected,
                                                               const FString& What,
-                                                              UObject* Context /*= nullptr*/)
+                                                              UObject* Context /*= nullptr*/,
+                                                              const float Tolerance /*= 0.0001f*/)
 {
-    if (!FMath::IsNearlyEqual(Actual, Expected))
+    if (!FMath::IsNearlyEqual(Actual, Expected, Tolerance))
     {
         FString Message = FString::Format(*ErrorMessageFormatEqual, {What, Expected, Actual});
         OnTestFailed(Context, Message);
     }
 }
 
-void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualFloat(float Actual, float Unexpected,
-                                                                 const FString& What,
-                                                                 UObject* Context /*= nullptr*/)
+void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualFloat(
+    float Actual, float Unexpected, const FString& What, UObject* Context /*= nullptr*/,
+    const float Tolerance /*= 0.0001f*/)
 {
-    if (FMath::IsNearlyEqual(Actual, Unexpected))
+    if (FMath::IsNearlyEqual(Actual, Unexpected, Tolerance))
     {
         FString Message = FString::Format(*ErrorMessageFormatNotEqual, {What, Unexpected});
         OnTestFailed(Context, Message);
@@ -319,20 +320,20 @@ void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualText(const FText& Act
 void UDaeTestAssertBlueprintFunctionLibrary::AssertEqualVector(const FVector& Actual,
                                                                const FVector& Expected,
                                                                const FString& What,
-                                                               UObject* Context /*= nullptr*/)
+                                                               UObject* Context /*= nullptr*/,
+                                                               const float Tolerance /*= 0.0001f*/)
 {
-    if (!Actual.Equals(Expected))
+    if (!Actual.Equals(Expected, Tolerance))
     {
         FString Message = FString::Format(*ErrorMessageFormatEqual,
                                           {What, Expected.ToString(), Actual.ToString()});
         OnTestFailed(Context, Message);
     }
 }
 
-void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualVector(const FVector& Actual,
-                                                                  const FVector& Unexpected,
-                                                                  const FString& What,
-                                                                  UObject* Context /*= nullptr*/)
+void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualVector(
+    const FVector& Actual, const FVector& Unexpected, const FString& What,
+    UObject* Context /*= nullptr*/, const float Tolerance /*= 0.0001f*/)
 {
     if (Actual.Equals(Unexpected))
     {
@@ -345,48 +346,46 @@ void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualVector(const FVector&
 void UDaeTestAssertBlueprintFunctionLibrary::AssertEqualRotator(const FRotator& Actual,
                                                                 const FRotator& Expected,
                                                                 const FString& What,
-                                                                UObject* Context /*= nullptr*/)
+                                                                UObject* Context /*= nullptr*/,
+                                                                const float Tolerance /*= 0.0001f*/)
 {
-    if (!Actual.Equals(Expected))
+    if (!Actual.Equals(Expected, Tolerance))
     {
         FString Message = FString::Format(*ErrorMessageFormatEqual,
                                           {What, Expected.ToString(), Actual.ToString()});
         OnTestFailed(Context, Message);
     }
 }
 
-void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualRotator(const FRotator& Actual,
-                                                                   const FRotator& Unexpected,
-                                                                   const FString& What,
-                                                                   UObject* Context /*= nullptr*/)
+void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualRotator(
+    const FRotator& Actual, const FRotator& Unexpected, const FString& What,
+    UObject* Context /*= nullptr*/, const float Tolerance /*= 0.0001f*/)
 {
-    if (Actual.Equals(Unexpected))
+    if (Actual.Equals(Unexpected, Tolerance))
     {
         FString Message =
             FString::Format(*ErrorMessageFormatNotEqual, {What, Unexpected.ToString()});
         OnTestFailed(Context, Message);
     }
 }
 
-void UDaeTestAssertBlueprintFunctionLibrary::AssertEqualTransform(const FTransform& Actual,
-                                                                  const FTransform& Expected,
-                                                                  const FString& What,
-                                                                  UObject* Context /*= nullptr*/)
+void UDaeTestAssertBlueprintFunctionLibrary::AssertEqualTransform(
+    const FTransform& Actual, const FTransform& Expected, const FString& What,
+    UObject* Context /*= nullptr*/, const float Tolerance /*= 0.0001f*/)
 {
-    if (!Actual.Equals(Expected))
+    if (!Actual.Equals(Expected, Tolerance))
     {
         FString Message = FString::Format(*ErrorMessageFormatEqual,
                                           {What, Expected.ToString(), Actual.ToString()});
         OnTestFailed(Context, Message);
     }
 }
 
-void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualTransform(const FTransform& Actual,
-                                                                     const FTransform& Unexpected,
-                                                                     const FString& What,
-                                                                     UObject* Context /*= nullptr*/)
+void UDaeTestAssertBlueprintFunctionLibrary::AssertNotEqualTransform(
+    const FTransform& Actual, const FTransform& Unexpected, const FString& What,
+    UObject* Context /*= nullptr*/, const float Tolerance /*= 0.0001f*/)
 {
-    if (Actual.Equals(Unexpected))
+    if (Actual.Equals(Unexpected, Tolerance))
     {
         FString Message =
             FString::Format(*ErrorMessageFormatNotEqual, {What, Unexpected.ToString()});

diff --git a/...gin/Source/DaedalicTestAutomationPlugin/Private/DaeTestAssumeBlueprintFunctionLibrary.cpp b/...gin/Source/DaedalicTestAutomationPlugin/Private/DaeTestAssumeBlueprintFunctionLibrary.cpp
@@ -1,4 +1,6 @@
 #include "DaeTestAssumeBlueprintFunctionLibrary.h"
+#include "DaeTestActor.h"
+#include "DaeTestLogCategory.h"
 
 void UDaeTestAssumeBlueprintFunctionLibrary::AssumeTrue(bool bValue, const FString& What,
                                                         UObject* Context /*= nullptr*/)