From ca144772928f33d9c17e5c2566bd6a5ad5c0f747 Mon Sep 17 00:00:00 2001
From: Norm Johanson <normj@amazon.com>
Date: Thu, 5 Sep 2024 09:04:44 -0700
Subject: [PATCH] Add a wait after integ test CloudFormation deployment to wait
 for any eventually consistency sync up.

---
 .../CustomResponse.cs                         | 37 +++++++++++++++++--
 .../IntegrationTestContextFixture.cs          |  3 ++
 2 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/Libraries/test/TestServerlessApp.IntegrationTests/CustomResponse.cs b/Libraries/test/TestServerlessApp.IntegrationTests/CustomResponse.cs
index 3d987c391..433adac4d 100644
--- a/Libraries/test/TestServerlessApp.IntegrationTests/CustomResponse.cs
+++ b/Libraries/test/TestServerlessApp.IntegrationTests/CustomResponse.cs
@@ -2,6 +2,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http;
 using System.Text;
 using System.Threading.Tasks;
 using Xunit;
@@ -21,7 +22,7 @@ public CustomResponse(IntegrationTestContextFixture fixture)
         [Fact]
         public async Task OkResponseWithHeader_Returns200Status()
         {
-            var response = await _fixture.HttpClient.GetAsync($"{_fixture.RestApiUrlPrefix}/okresponsewithheader/1");
+            var response = await GetWithRetryAsync($"{_fixture.RestApiUrlPrefix}/okresponsewithheader/1");
             response.EnsureSuccessStatusCode();
             Assert.Equal("All Good", await response.Content.ReadAsStringAsync());
         }
@@ -29,7 +30,7 @@ public async Task OkResponseWithHeader_Returns200Status()
         [Fact]
         public async Task OkResponseWithHeader_ReturnsValidationErrors()
         {
-            var response = await _fixture.HttpClient.GetAsync($"{_fixture.RestApiUrlPrefix}/okresponsewithheader/hello");
+            var response = await GetWithRetryAsync($"{_fixture.RestApiUrlPrefix}/okresponsewithheader/hello");
             Assert.Equal(400, (int)response.StatusCode);
             var content = await response.Content.ReadAsStringAsync();
             var errorJson = JObject.Parse(content);
@@ -41,7 +42,7 @@ public async Task OkResponseWithHeader_ReturnsValidationErrors()
         [Fact]
         public async Task OkResponseWithCustomSerializer_Returns200Status()
         {
-            var response = await _fixture.HttpClient.GetAsync($"{_fixture.HttpApiUrlPrefix}/okresponsewithcustomserializerasync/John/Doe");
+            var response = await GetWithRetryAsync($"{_fixture.HttpApiUrlPrefix}/okresponsewithcustomserializerasync/John/Doe");
             response.EnsureSuccessStatusCode();
 
             var content = await response.Content.ReadAsStringAsync();
@@ -49,5 +50,35 @@ public async Task OkResponseWithCustomSerializer_Returns200Status()
             Assert.Equal("John", person["FIRST_NAME"]);
             Assert.Equal("Doe", person["LAST_NAME"]);
         }
+
+        private async Task<HttpResponseMessage> GetWithRetryAsync(string path)
+        {
+            int MAX_ATTEMPTS = 10;
+            HttpResponseMessage response = null;
+            for (var retryAttempt = 0; retryAttempt < MAX_ATTEMPTS; retryAttempt++)
+            {
+                await Task.Delay(retryAttempt * 1000);
+                try
+                {
+                    response = await _fixture.HttpClient.GetAsync(path);
+
+                    // No tests are coded to return 403 Forbidden. If this is returned it is likely
+                    // an eventual consistency issue.
+                    if (response.StatusCode == System.Net.HttpStatusCode.Forbidden)
+                        continue;
+
+                    break;
+                }
+                catch
+                {
+                    if (retryAttempt + 1 == MAX_ATTEMPTS)
+                    {
+                        throw;
+                    }
+                }
+            }
+
+            return response;
+        }
     }
 }
diff --git a/Libraries/test/TestServerlessApp.IntegrationTests/IntegrationTestContextFixture.cs b/Libraries/test/TestServerlessApp.IntegrationTests/IntegrationTestContextFixture.cs
index b5e27f60a..cbae5fd6a 100644
--- a/Libraries/test/TestServerlessApp.IntegrationTests/IntegrationTestContextFixture.cs
+++ b/Libraries/test/TestServerlessApp.IntegrationTests/IntegrationTestContextFixture.cs
@@ -62,6 +62,9 @@ public async Task InitializeAsync()
             Assert.False(string.IsNullOrEmpty(RestApiUrlPrefix));
 
             await LambdaHelper.WaitTillNotPending(LambdaFunctions.Select(x => x.Name).ToList());
+
+            // Wait an additional 10 seconds for any other eventually consistency state to finish up.
+            await Task.Delay(10000);
         }
 
         public async Task DisposeAsync()