Add max elapsed time check for rest fallback host retries for TO3l6

Add missing `httpMaxRetryDuration` value to Defaults.TIMEOUTS

Resolves #1717

src/common/lib/util/defaults.ts

@@ -23,6 +23,7 @@ type CompleteDefaults = IDefaults & {
     disconnectedRetryTimeout: number;
     suspendedRetryTimeout: number;
     httpRequestTimeout: number;
+    httpMaxRetryDuration: number;
     channelRetryTimeout: number;
     fallbackRetryTimeout: number;
     connectionStateTtl: number;
@@ -74,6 +75,7 @@ const Defaults = {
     suspendedRetryTimeout: 30000,
     /* Undocumented, but part of the api and can be used by customers: */
     httpRequestTimeout: 10000,
+    httpMaxRetryDuration: 15000,
     channelRetryTimeout: 15000,
     fallbackRetryTimeout: 600000,
     /* For internal / test use only: */

src/common/types/http.ts

@@ -192,10 +192,24 @@ export class Http {
         return this.doUri(method, uriFromHost(hosts[0]), headers, body, params);
       }
 
+      let tryAHostStartedAt: number;
       const tryAHost = async (candidateHosts: Array<string>, persistOnSuccess?: boolean): Promise<RequestResult> => {
         const host = candidateHosts.shift();
+        tryAHostStartedAt = tryAHostStartedAt ?? Date.now();
         const result = await this.doUri(method, uriFromHost(host as string), headers, body, params);
         if (result.error && this.platformHttp.shouldFallback(result.error as ErrnoException) && candidateHosts.length) {
+          // TO3l6
+          const elapsedTime = Date.now() - tryAHostStartedAt;
+          if (elapsedTime > client.options.timeouts.httpMaxRetryDuration) {
+            return {
+              error: new ErrorInfo(
+                `Timeout for trying fallback hosts retries. Total elapsed time exceeded the ${client.options.timeouts.httpMaxRetryDuration}ms limit`,
+                50003,
+                500,
+              ),
+            };
+          }
+
           return tryAHost(candidateHosts, true);
         }
         if (persistOnSuccess) {