From f03de9b145e26dc1996af68a66cac3aa4873d1fa Mon Sep 17 00:00:00 2001
From: Kevin Porras <kevin.porras@pantheon.io>
Date: Wed, 10 Jan 2024 10:54:42 -0600
Subject: [PATCH] [LOPS-1951] Avoid hitting APIs so hard. (#2525)

* Add retry backoff.

* Restrict values for maxRetries and retryBackoff.

* Workflow processing should not hit api that hard.

* Workflow polling should be 5s default.

* Remove unneeded changes.
---
 src/Commands/WorkflowProcessingTrait.php |  6 +++++-
 src/ProgressBars/WorkflowProgressBar.php | 13 +++++++++++++
 src/Request/Request.php                  | 15 +++++++++++++--
 3 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/src/Commands/WorkflowProcessingTrait.php b/src/Commands/WorkflowProcessingTrait.php
index fe0cbaa58..1cf5d5658 100644
--- a/src/Commands/WorkflowProcessingTrait.php
+++ b/src/Commands/WorkflowProcessingTrait.php
@@ -24,7 +24,11 @@ public function processWorkflow(Workflow $workflow): ?Workflow
             $progressBar = $this->getContainer()->get($nickname);
             return $progressBar->cycle();
         }
-        $retry_interval = $this->getConfig()->get('http_retry_delay_ms', 100);
+        $retry_interval = $this->getConfig()->get('workflow_polling_delay_ms', 5000);
+        if ($retry_interval < 1000) {
+            // The API will not allow polling faster than once per second.
+            $retry_interval = 1000;
+        }
         do {
             $workflow->fetch();
             usleep($retry_interval * 1000);
diff --git a/src/ProgressBars/WorkflowProgressBar.php b/src/ProgressBars/WorkflowProgressBar.php
index 9ff0d4f1d..72b10dddb 100644
--- a/src/ProgressBars/WorkflowProgressBar.php
+++ b/src/ProgressBars/WorkflowProgressBar.php
@@ -47,6 +47,19 @@ public function cycle()
         }
     }
 
+    /**
+     * Sleeps to prevent spamming the API.
+     */
+    protected function sleep()
+    {
+        $retry_interval = $this->getConfig()->get('workflow_polling_delay_ms', 5000);
+        if ($retry_interval < 1000) {
+            // The API will not allow polling faster than once per second.
+            $retry_interval = 1000;
+        }
+        usleep($retry_interval * 1000);
+    }
+
     /**
      * Runs a single iteration of the progress bar.
      * @return bool
diff --git a/src/Request/Request.php b/src/Request/Request.php
index 60fabfd7a..40b6f3520 100755
--- a/src/Request/Request.php
+++ b/src/Request/Request.php
@@ -176,7 +176,12 @@ private function getClient($base_uri = null): ClientInterface
     private function createRetryDecider(): callable
     {
         $config = $this->getConfig();
-        $maxRetries = $config->get('http_max_retries', 5);
+        $maxRetries = $config->get('http_max_retries', $defaultMaxRetries);
+        // Cap max retries at 10.
+        $maxRetries = $maxRetries > 10 ? 10 : $maxRetries;
+        $retryBackoff = $config->get('http_retry_backoff', 5);
+        // Retry backoff should be at least 3.
+        $retryBackoff = $retryBackoff < 3 ? 3 : $retryBackoff;
         $logger = $this->logger;
         $logWarning = function (string $message) use ($logger) {
             if ($this->output()->isVerbose()) {
@@ -191,7 +196,8 @@ private function createRetryDecider(): callable
             ?Exception $exception = null
         ) use (
             $maxRetries,
-            $logWarning
+            $logWarning,
+            $retryBackoff
         ) {
             $logWarningOnRetry = fn (string $reason) => 0 === $retry
                 ? $logWarning(
@@ -217,6 +223,8 @@ private function createRetryDecider(): callable
                 // Retry on connection-related exceptions such as "Connection refused" and "Operation timed out".
                 if ($retry !== $maxRetries) {
                     $logWarningOnRetry($exception->getMessage());
+                    $logWarning(sprintf("Retrying in %s seconds.", $retryBackoff * ($retry + 1)));
+                    sleep($retryBackoff * ($retry + 1));
 
                     return true;
                 }
@@ -240,6 +248,9 @@ private function createRetryDecider(): callable
                         ['body' => $response->getBody()->getContents()]
                     );
 
+                    $logWarning(sprintf("Retrying in %s seconds.", $retryBackoff * ($retry + 1)));
+                    sleep($retryBackoff * ($retry + 1));
+
                     return true;
                 }
             }