Merge pull request #1693 from AzureAD/fadi/ci-repair

Work to get 2 MSAL Automation pipelines to green
AzureAD · Sep 14, 2022 · a4d18f4 · a4d18f4
2 parents 11b3049 + c4decdd
commit a4d18f4
Show file tree

Hide file tree

Showing 16 changed files with 48 additions and 19 deletions.
diff --git a/azure-pipelines/ui-automation/msal-broker-BrokerHost-nightly.yml b/azure-pipelines/ui-automation/msal-broker-BrokerHost-nightly.yml
@@ -45,7 +45,7 @@ parameters:
   - name: testTargetPackages
     displayName: Packages as Test Targets
     type: string
-    default: package com.microsoft.identity.client.msal.automationapp.testpass.broker
+    default: package com.microsoft.identity.client.msal.automationapp.testpass.broker, notAnnotation org.junit.Ignore
   - name: oldBrokerHostVersion
     displayName: Old Broker host Version
     type: string

diff --git a/azure-pipelines/ui-automation/msal-broker-ProdBroker-nightly.yml b/azure-pipelines/ui-automation/msal-broker-ProdBroker-nightly.yml
@@ -45,7 +45,7 @@ parameters:
   - name: testTargetPackages
     displayName: Packages as Test Targets
     type: string
-    default: package com.microsoft.identity.client.msal.automationapp.testpass.broker
+    default: package com.microsoft.identity.client.msal.automationapp.testpass.broker, notAnnotation org.junit.Ignore
   - name: oldBrokerHostVersion
     displayName: Old Broker host Version
     type: string

diff --git a/azure-pipelines/ui-automation/msal-local-nightly.yml b/azure-pipelines/ui-automation/msal-local-nightly.yml
@@ -45,7 +45,7 @@ parameters:
   - name: testTargetPackages
     displayName: Packages as Test Targets
     type: string
-    default: package com.microsoft.identity.client.msal.automationapp.testpass.msalonly, package com.microsoft.identity.client.msal.automationapp.testpass.b2c
+    default: package com.microsoft.identity.client.msal.automationapp.testpass.msalonly, package com.microsoft.identity.client.msal.automationapp.testpass.b2c, notAnnotation org.junit.Ignore
 
 variables:
   engineeringProjectId: 'fac9d424-53d2-45c0-91b5-ef6ba7a6bf26'

diff --git a/azure-pipelines/ui-automation/templates/flank/flank.yml b/azure-pipelines/ui-automation/templates/flank/flank.yml
@@ -1,19 +1,27 @@
 flank:
-  ## Max Test Shards
-  # test shards - the amount of groups to split the test suite into
-  # set to -1 to use one shard per test. default: 1
+   ## Max Test Shards
+   # test shards - the amount of groups to split the test suite into
+   # set to -1 to use one shard per test. default: 1
    max-test-shards: 5
 
-  ## Number of Test Runs
-  # test runs - the amount of times to run the tests.
-  # 1 runs the tests once. 10 runs all the tests 10x
+   ## Number of Test Runs
+   # test runs - the amount of times to run the tests.
+   # 1 runs the tests once. 10 runs all the tests 10x
    num-test-runs: 1
 
-  ## Use Average Test Time for New Tests flag
-  # Enable using average time from previous tests duration when using SmartShard and tests did not run before.
-  # Default: false
+   ## Use Average Test Time for New Tests flag
+   # Enable using average time from previous tests duration when using SmartShard and tests did not run before.
+   # Default: false
    use-average-test-time-for-new-tests: true
 
-  ## Local Results Directory
-  # Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
-   local-result-dir: flank
+   ## Local Results Directory
+   # Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved.
+   local-result-dir: flank
+
+   ## Output Style flag
+   # Output style of execution status. May be one of [verbose, multi, single, compact].
+   # For runs with only one test execution the default value is 'verbose', in other cases
+   # 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles
+   # which don't support ansi codes, to avoid corrupted output use single or verbose.
+   # The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports.
+   output-style: single
diff --git a/common b/common
diff --git a/...crosoft/identity/client/msal/automationapp/testpass/broker/brokerapi/TestCase1561137.java b/...crosoft/identity/client/msal/automationapp/testpass/broker/brokerapi/TestCase1561137.java
@@ -29,6 +29,7 @@
 import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk;
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers;
 import com.microsoft.identity.client.ui.automation.broker.BrokerHost;
 import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired;
@@ -44,6 +45,7 @@
 // Remove Broker Account
 // https://identitydivision.visualstudio.com/Engineering/_workitems/edit/1561137
 @SupportedBrokers(brokers = BrokerHost.class)
+@RetryOnFailure(retryCount = 2)
 public class TestCase1561137 extends AbstractMsalBrokerTest {
     @Test
     public void test_1561137() throws Throwable {

diff --git a/...rosoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1420494.java b/...rosoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1420494.java
@@ -33,6 +33,7 @@
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractGuestAccountMsalBrokerUiTest;
 import com.microsoft.identity.client.ui.automation.TestContext;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.constants.GlobalConstants;
 import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired;
 import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters;
@@ -55,6 +56,7 @@
 
 // Acquire token for cross cloud guest account (with broker)
 // https://identitydivision.visualstudio.com/DefaultCollection/IDDP/_workitems/edit/1420494
+@RetryOnFailure(retryCount = 2)
 @RunWith(Parameterized.class)
 public class TestCase1420494 extends AbstractGuestAccountMsalBrokerUiTest {
 

diff --git a/...rosoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1592468.java b/...rosoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1592468.java
@@ -32,6 +32,7 @@
 import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk;
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractGuestAccountMsalBrokerUiTest;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.constants.GlobalConstants;
 import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired;
 import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters;
@@ -57,6 +58,7 @@
 
 // Acquire token from cross cloud when Device CA is required (via PKeyAuth)
 // https://identitydivision.visualstudio.com/DefaultCollection/IDDP/_workitems/edit/1592468
+@RetryOnFailure(retryCount = 2)
 @RunWith(Parameterized.class)
 public class TestCase1592468 extends AbstractGuestAccountMsalBrokerUiTest {
 

diff --git a/.../com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833513.java b/.../com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833513.java
@@ -31,6 +31,7 @@
 import com.microsoft.identity.client.exception.MsalException;
 import com.microsoft.identity.client.msal.automationapp.R;
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers;
 import com.microsoft.identity.client.ui.automation.broker.BrokerHost;
 import com.microsoft.identity.client.ui.automation.broker.BrokerMicrosoftAuthenticator;
@@ -50,10 +51,12 @@
 import org.junit.Test;
 
 import java.util.Arrays;
+import java.util.concurrent.TimeUnit;
 
 // End My Shift - In Shared device mode, only account from the same tenant should be able to acquire token.
 // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/833513
 @SupportedBrokers(brokers = {BrokerMicrosoftAuthenticator.class, BrokerHost.class})
+@RetryOnFailure(retryCount = 2)
 public class TestCase833513 extends AbstractMsalBrokerTest {
 
     @Test
@@ -119,7 +122,7 @@ public void test_833513() throws MsalException, InterruptedException, LabApiExce
 
         // expecting error in WebView now
         final UiObject errMsg = UiAutomatorUtils.obtainUiObjectWithText("AADSTS50020");
-        Assert.assertTrue(errMsg.exists());
+        Assert.assertTrue(errMsg.waitForExists(TimeUnit.MINUTES.toMillis(1)));
     }
 
     @Override

diff --git a/.../com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833514.java b/.../com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833514.java
@@ -36,6 +36,7 @@
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
 import com.microsoft.identity.client.ui.automation.TokenRequestLatch;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers;
 import com.microsoft.identity.client.ui.automation.app.AzureSampleApp;
 import com.microsoft.identity.client.ui.automation.broker.BrokerHost;
@@ -58,6 +59,7 @@
 // End My Shift - In Shared device mode, an account signed in through App A can be used by App B.
 // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/833514
 @SupportedBrokers(brokers = {BrokerMicrosoftAuthenticator.class, BrokerHost.class})
+@RetryOnFailure(retryCount = 2)
 public class TestCase833514 extends AbstractMsalBrokerTest {
 
     @Test

diff --git a/.../com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833515.java b/.../com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833515.java
@@ -32,6 +32,7 @@
 import com.microsoft.identity.client.ui.automation.TokenRequestLatch;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers;
 import com.microsoft.identity.client.ui.automation.app.AzureSampleApp;
 import com.microsoft.identity.client.ui.automation.broker.BrokerHost;
@@ -42,7 +43,6 @@
 import com.microsoft.identity.client.ui.automation.interaction.PromptParameter;
 import com.microsoft.identity.client.ui.automation.interaction.microsoftsts.AadLoginComponentHandler;
 import com.microsoft.identity.client.ui.automation.interaction.microsoftsts.AadPromptHandler;
-import com.microsoft.identity.client.ui.automation.utils.UiAutomatorUtils;
 import com.microsoft.identity.labapi.utilities.client.ILabAccount;
 import com.microsoft.identity.labapi.utilities.client.LabQuery;
 import com.microsoft.identity.labapi.utilities.constants.TempUserType;
@@ -58,6 +58,7 @@
 // End My Shift - In Shared device mode, global sign out should work.
 // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/833515
 @SupportedBrokers(brokers = {BrokerMicrosoftAuthenticator.class, BrokerHost.class})
+@RetryOnFailure(retryCount = 2)
 public class TestCase833515 extends AbstractMsalBrokerTest {
 
     final static String MY_APPS_URL = "myapps.microsoft.com";

diff --git a/.../microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase1561151.java b/.../microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase1561151.java
@@ -31,6 +31,7 @@
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
 import com.microsoft.identity.client.ui.automation.TestContext;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters;
 import com.microsoft.identity.client.ui.automation.interaction.PromptParameter;
 import com.microsoft.identity.client.ui.automation.interaction.microsoftsts.AadPromptHandler;
@@ -46,6 +47,7 @@
 
 // [Joined][MSAL] Password Change
 // https://identitydivision.visualstudio.com/Engineering/_workitems/edit/1561151
+@RetryOnFailure(retryCount = 2)
 public class TestCase1561151 extends AbstractMsalBrokerTest {
     @Test
     public void test_1561151() throws Throwable {

diff --git a/...m/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase796050.java b/...m/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase796050.java
@@ -31,6 +31,7 @@
 import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk;
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
 import com.microsoft.identity.client.ui.automation.TestContext;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
 import com.microsoft.identity.client.ui.automation.broker.BrokerMicrosoftAuthenticator;
@@ -51,6 +52,7 @@
 // The goal of the test case is to ensure that we can add accounts in broker via the
 // "Add another account" option in Account Chooser Activity
 // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/796050
+@RetryOnFailure(retryCount = 2)
 public class TestCase796050 extends AbstractMsalBrokerTest {
 
     @Test

diff --git a/.../com/microsoft/identity/client/msal/automationapp/testpass/broker/mam/TestCase850457.java b/.../com/microsoft/identity/client/msal/automationapp/testpass/broker/mam/TestCase850457.java
@@ -30,6 +30,7 @@
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
 import com.microsoft.identity.client.ui.automation.TestContext;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters;
 import com.microsoft.identity.client.ui.automation.interaction.PromptParameter;
 import com.microsoft.identity.client.ui.automation.interaction.microsoftsts.AadPromptHandler;
@@ -44,6 +45,7 @@
 
 // [MSAL] password reset for MAM_CA account
 // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/850457
+@RetryOnFailure(retryCount = 2)
 public class TestCase850457 extends AbstractMsalBrokerTest {
 
     @Test

diff --git a/...crosoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1561152.java b/...crosoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1561152.java
@@ -32,6 +32,7 @@
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
 import com.microsoft.identity.client.ui.automation.TestContext;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired;
 import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters;
 import com.microsoft.identity.client.ui.automation.interaction.PromptParameter;
@@ -46,6 +47,7 @@
 
 // [Non-Joined][MSAL] Password Change (bad_token)
 // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/1561152
+@RetryOnFailure(retryCount = 2)
 public class TestCase1561152 extends AbstractMsalBrokerTest {
 
     @Test

diff --git a/...crosoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1600592.java b/...crosoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1600592.java
@@ -24,13 +24,13 @@
 
 import com.microsoft.identity.client.Prompt;
 import com.microsoft.identity.client.exception.MsalServiceException;
-import com.microsoft.identity.client.msal.automationapp.AbstractMsalUiTest;
 import com.microsoft.identity.client.msal.automationapp.R;
 import com.microsoft.identity.client.msal.automationapp.sdk.MsalAuthResult;
 import com.microsoft.identity.client.msal.automationapp.sdk.MsalAuthTestParams;
 import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk;
 import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest;
 import com.microsoft.identity.client.ui.automation.TokenRequestTimeout;
+import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure;
 import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired;
 import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters;
 import com.microsoft.identity.client.ui.automation.interaction.PromptParameter;
@@ -45,6 +45,7 @@
 
 // [Non-joined] A single-tenant app makes a silent request with common authority. It should fail.
 // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/1600592
+@RetryOnFailure(retryCount = 2)
 public class TestCase1600592 extends AbstractMsalBrokerTest {
 
     @Test