diff --git a/azure-pipelines/ui-automation/msal-broker-BrokerHost-nightly.yml b/azure-pipelines/ui-automation/msal-broker-BrokerHost-nightly.yml index 98c50f1be..21252168a 100644 --- a/azure-pipelines/ui-automation/msal-broker-BrokerHost-nightly.yml +++ b/azure-pipelines/ui-automation/msal-broker-BrokerHost-nightly.yml @@ -45,7 +45,7 @@ parameters: - name: testTargetPackages displayName: Packages as Test Targets type: string - default: package com.microsoft.identity.client.msal.automationapp.testpass.broker + default: package com.microsoft.identity.client.msal.automationapp.testpass.broker, notAnnotation org.junit.Ignore - name: oldBrokerHostVersion displayName: Old Broker host Version type: string diff --git a/azure-pipelines/ui-automation/msal-broker-ProdBroker-nightly.yml b/azure-pipelines/ui-automation/msal-broker-ProdBroker-nightly.yml index a47669c6e..4df4bbc29 100644 --- a/azure-pipelines/ui-automation/msal-broker-ProdBroker-nightly.yml +++ b/azure-pipelines/ui-automation/msal-broker-ProdBroker-nightly.yml @@ -45,7 +45,7 @@ parameters: - name: testTargetPackages displayName: Packages as Test Targets type: string - default: package com.microsoft.identity.client.msal.automationapp.testpass.broker + default: package com.microsoft.identity.client.msal.automationapp.testpass.broker, notAnnotation org.junit.Ignore - name: oldBrokerHostVersion displayName: Old Broker host Version type: string diff --git a/azure-pipelines/ui-automation/msal-local-nightly.yml b/azure-pipelines/ui-automation/msal-local-nightly.yml index 434ae879f..e9a45cc22 100644 --- a/azure-pipelines/ui-automation/msal-local-nightly.yml +++ b/azure-pipelines/ui-automation/msal-local-nightly.yml @@ -45,7 +45,7 @@ parameters: - name: testTargetPackages displayName: Packages as Test Targets type: string - default: package com.microsoft.identity.client.msal.automationapp.testpass.msalonly, package com.microsoft.identity.client.msal.automationapp.testpass.b2c + default: package com.microsoft.identity.client.msal.automationapp.testpass.msalonly, package com.microsoft.identity.client.msal.automationapp.testpass.b2c, notAnnotation org.junit.Ignore variables: engineeringProjectId: 'fac9d424-53d2-45c0-91b5-ef6ba7a6bf26' diff --git a/azure-pipelines/ui-automation/templates/flank/flank.yml b/azure-pipelines/ui-automation/templates/flank/flank.yml index 4380c852d..304ffd928 100644 --- a/azure-pipelines/ui-automation/templates/flank/flank.yml +++ b/azure-pipelines/ui-automation/templates/flank/flank.yml @@ -1,19 +1,27 @@ flank: - ## Max Test Shards - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. default: 1 + ## Max Test Shards + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. default: 1 max-test-shards: 5 - ## Number of Test Runs - # test runs - the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x + ## Number of Test Runs + # test runs - the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x num-test-runs: 1 - ## Use Average Test Time for New Tests flag - # Enable using average time from previous tests duration when using SmartShard and tests did not run before. - # Default: false + ## Use Average Test Time for New Tests flag + # Enable using average time from previous tests duration when using SmartShard and tests did not run before. + # Default: false use-average-test-time-for-new-tests: true - ## Local Results Directory - # Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. - local-result-dir: flank \ No newline at end of file + ## Local Results Directory + # Local folder to store the test result. Folder is DELETED before each run to ensure only artifacts from the new run are saved. + local-result-dir: flank + + ## Output Style flag + # Output style of execution status. May be one of [verbose, multi, single, compact]. + # For runs with only one test execution the default value is 'verbose', in other cases + # 'multi' is used as the default. The output style 'multi' is not displayed correctly on consoles + # which don't support ansi codes, to avoid corrupted output use single or verbose. + # The output style `compact` is used to produce less detailed output, it prints just Args, test and matrix count, weblinks, cost, and result reports. + output-style: single diff --git a/common b/common index db036836b..21adc22e7 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit db036836baa6929bb856c4ef134174b3976b985e +Subproject commit 21adc22e7dabdb47949b01e565172dd9308572d2 diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/brokerapi/TestCase1561137.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/brokerapi/TestCase1561137.java index adf26f950..5a18a0251 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/brokerapi/TestCase1561137.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/brokerapi/TestCase1561137.java @@ -29,6 +29,7 @@ import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk; import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers; import com.microsoft.identity.client.ui.automation.broker.BrokerHost; import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired; @@ -44,6 +45,7 @@ // Remove Broker Account // https://identitydivision.visualstudio.com/Engineering/_workitems/edit/1561137 @SupportedBrokers(brokers = BrokerHost.class) +@RetryOnFailure(retryCount = 2) public class TestCase1561137 extends AbstractMsalBrokerTest { @Test public void test_1561137() throws Throwable { diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1420494.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1420494.java index ef04abcd1..5e7badf58 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1420494.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1420494.java @@ -33,6 +33,7 @@ import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractGuestAccountMsalBrokerUiTest; import com.microsoft.identity.client.ui.automation.TestContext; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.constants.GlobalConstants; import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired; import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters; @@ -55,6 +56,7 @@ // Acquire token for cross cloud guest account (with broker) // https://identitydivision.visualstudio.com/DefaultCollection/IDDP/_workitems/edit/1420494 +@RetryOnFailure(retryCount = 2) @RunWith(Parameterized.class) public class TestCase1420494 extends AbstractGuestAccountMsalBrokerUiTest { diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1592468.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1592468.java index b3ecaa2cd..764e46841 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1592468.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/crosscloud/TestCase1592468.java @@ -32,6 +32,7 @@ import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk; import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractGuestAccountMsalBrokerUiTest; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.constants.GlobalConstants; import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired; import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters; @@ -57,6 +58,7 @@ // Acquire token from cross cloud when Device CA is required (via PKeyAuth) // https://identitydivision.visualstudio.com/DefaultCollection/IDDP/_workitems/edit/1592468 +@RetryOnFailure(retryCount = 2) @RunWith(Parameterized.class) public class TestCase1592468 extends AbstractGuestAccountMsalBrokerUiTest { diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833513.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833513.java index 7badff1c7..d8cbc0a5c 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833513.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833513.java @@ -31,6 +31,7 @@ import com.microsoft.identity.client.exception.MsalException; import com.microsoft.identity.client.msal.automationapp.R; import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers; import com.microsoft.identity.client.ui.automation.broker.BrokerHost; import com.microsoft.identity.client.ui.automation.broker.BrokerMicrosoftAuthenticator; @@ -50,10 +51,12 @@ import org.junit.Test; import java.util.Arrays; +import java.util.concurrent.TimeUnit; // End My Shift - In Shared device mode, only account from the same tenant should be able to acquire token. // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/833513 @SupportedBrokers(brokers = {BrokerMicrosoftAuthenticator.class, BrokerHost.class}) +@RetryOnFailure(retryCount = 2) public class TestCase833513 extends AbstractMsalBrokerTest { @Test @@ -119,7 +122,7 @@ public void test_833513() throws MsalException, InterruptedException, LabApiExce // expecting error in WebView now final UiObject errMsg = UiAutomatorUtils.obtainUiObjectWithText("AADSTS50020"); - Assert.assertTrue(errMsg.exists()); + Assert.assertTrue(errMsg.waitForExists(TimeUnit.MINUTES.toMillis(1))); } @Override diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833514.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833514.java index 7c0fedc7f..b125437c5 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833514.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833514.java @@ -36,6 +36,7 @@ import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest; import com.microsoft.identity.client.ui.automation.TokenRequestLatch; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers; import com.microsoft.identity.client.ui.automation.app.AzureSampleApp; import com.microsoft.identity.client.ui.automation.broker.BrokerHost; @@ -58,6 +59,7 @@ // End My Shift - In Shared device mode, an account signed in through App A can be used by App B. // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/833514 @SupportedBrokers(brokers = {BrokerMicrosoftAuthenticator.class, BrokerHost.class}) +@RetryOnFailure(retryCount = 2) public class TestCase833514 extends AbstractMsalBrokerTest { @Test diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833515.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833515.java index f6c1b0bc5..ee2f0f288 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833515.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/flw/TestCase833515.java @@ -32,6 +32,7 @@ import com.microsoft.identity.client.ui.automation.TokenRequestLatch; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.annotations.SupportedBrokers; import com.microsoft.identity.client.ui.automation.app.AzureSampleApp; import com.microsoft.identity.client.ui.automation.broker.BrokerHost; @@ -42,7 +43,6 @@ import com.microsoft.identity.client.ui.automation.interaction.PromptParameter; import com.microsoft.identity.client.ui.automation.interaction.microsoftsts.AadLoginComponentHandler; import com.microsoft.identity.client.ui.automation.interaction.microsoftsts.AadPromptHandler; -import com.microsoft.identity.client.ui.automation.utils.UiAutomatorUtils; import com.microsoft.identity.labapi.utilities.client.ILabAccount; import com.microsoft.identity.labapi.utilities.client.LabQuery; import com.microsoft.identity.labapi.utilities.constants.TempUserType; @@ -58,6 +58,7 @@ // End My Shift - In Shared device mode, global sign out should work. // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/833515 @SupportedBrokers(brokers = {BrokerMicrosoftAuthenticator.class, BrokerHost.class}) +@RetryOnFailure(retryCount = 2) public class TestCase833515 extends AbstractMsalBrokerTest { final static String MY_APPS_URL = "myapps.microsoft.com"; diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase1561151.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase1561151.java index f0c40b824..b7e4e6fde 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase1561151.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase1561151.java @@ -31,6 +31,7 @@ import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest; import com.microsoft.identity.client.ui.automation.TestContext; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters; import com.microsoft.identity.client.ui.automation.interaction.PromptParameter; import com.microsoft.identity.client.ui.automation.interaction.microsoftsts.AadPromptHandler; @@ -46,6 +47,7 @@ // [Joined][MSAL] Password Change // https://identitydivision.visualstudio.com/Engineering/_workitems/edit/1561151 +@RetryOnFailure(retryCount = 2) public class TestCase1561151 extends AbstractMsalBrokerTest { @Test public void test_1561151() throws Throwable { diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase796050.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase796050.java index e57c8f752..77266cda9 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase796050.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/joined/TestCase796050.java @@ -31,6 +31,7 @@ import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk; import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest; import com.microsoft.identity.client.ui.automation.TestContext; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; import com.microsoft.identity.client.ui.automation.broker.BrokerMicrosoftAuthenticator; @@ -51,6 +52,7 @@ // The goal of the test case is to ensure that we can add accounts in broker via the // "Add another account" option in Account Chooser Activity // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/796050 +@RetryOnFailure(retryCount = 2) public class TestCase796050 extends AbstractMsalBrokerTest { @Test diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/mam/TestCase850457.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/mam/TestCase850457.java index daf0aab98..f6a3d8da2 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/mam/TestCase850457.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/mam/TestCase850457.java @@ -30,6 +30,7 @@ import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest; import com.microsoft.identity.client.ui.automation.TestContext; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters; import com.microsoft.identity.client.ui.automation.interaction.PromptParameter; import com.microsoft.identity.client.ui.automation.interaction.microsoftsts.AadPromptHandler; @@ -44,6 +45,7 @@ // [MSAL] password reset for MAM_CA account // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/850457 +@RetryOnFailure(retryCount = 2) public class TestCase850457 extends AbstractMsalBrokerTest { @Test diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1561152.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1561152.java index 97b72ba71..8c3208c97 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1561152.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1561152.java @@ -32,6 +32,7 @@ import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest; import com.microsoft.identity.client.ui.automation.TestContext; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired; import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters; import com.microsoft.identity.client.ui.automation.interaction.PromptParameter; @@ -46,6 +47,7 @@ // [Non-Joined][MSAL] Password Change (bad_token) // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/1561152 +@RetryOnFailure(retryCount = 2) public class TestCase1561152 extends AbstractMsalBrokerTest { @Test diff --git a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1600592.java b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1600592.java index dfac3b90f..222f70fde 100644 --- a/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1600592.java +++ b/msalautomationapp/src/androidTest/java/com/microsoft/identity/client/msal/automationapp/testpass/broker/nonjoined/TestCase1600592.java @@ -24,13 +24,13 @@ import com.microsoft.identity.client.Prompt; import com.microsoft.identity.client.exception.MsalServiceException; -import com.microsoft.identity.client.msal.automationapp.AbstractMsalUiTest; import com.microsoft.identity.client.msal.automationapp.R; import com.microsoft.identity.client.msal.automationapp.sdk.MsalAuthResult; import com.microsoft.identity.client.msal.automationapp.sdk.MsalAuthTestParams; import com.microsoft.identity.client.msal.automationapp.sdk.MsalSdk; import com.microsoft.identity.client.msal.automationapp.testpass.broker.AbstractMsalBrokerTest; import com.microsoft.identity.client.ui.automation.TokenRequestTimeout; +import com.microsoft.identity.client.ui.automation.annotations.RetryOnFailure; import com.microsoft.identity.client.ui.automation.interaction.OnInteractionRequired; import com.microsoft.identity.client.ui.automation.interaction.PromptHandlerParameters; import com.microsoft.identity.client.ui.automation.interaction.PromptParameter; @@ -45,6 +45,7 @@ // [Non-joined] A single-tenant app makes a silent request with common authority. It should fail. // https://identitydivision.visualstudio.com/DevEx/_workitems/edit/1600592 +@RetryOnFailure(retryCount = 2) public class TestCase1600592 extends AbstractMsalBrokerTest { @Test