[LOPS-1951] Avoid hitting APIs so hard. (#2525)

* Add retry backoff. * Restrict values for maxRetries and retryBackoff. * Workflow processing should not hit api that hard. * Workflow polling should be 5s default. * Remove unneeded changes.
pantheon-systems · Jan 10, 2024 · f03de9b · f03de9b
1 parent ee00b7a
commit f03de9b
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 3 deletions.
diff --git a/src/Commands/WorkflowProcessingTrait.php b/src/Commands/WorkflowProcessingTrait.php
@@ -24,7 +24,11 @@ public function processWorkflow(Workflow $workflow): ?Workflow
             $progressBar = $this->getContainer()->get($nickname);
             return $progressBar->cycle();
         }
-        $retry_interval = $this->getConfig()->get('http_retry_delay_ms', 100);
+        $retry_interval = $this->getConfig()->get('workflow_polling_delay_ms', 5000);
+        if ($retry_interval < 1000) {
+            // The API will not allow polling faster than once per second.
+            $retry_interval = 1000;
+        }
         do {
             $workflow->fetch();
             usleep($retry_interval * 1000);

diff --git a/src/ProgressBars/WorkflowProgressBar.php b/src/ProgressBars/WorkflowProgressBar.php
@@ -47,6 +47,19 @@ public function cycle()
         }
     }
 
+    /**
+     * Sleeps to prevent spamming the API.
+     */
+    protected function sleep()
+    {
+        $retry_interval = $this->getConfig()->get('workflow_polling_delay_ms', 5000);
+        if ($retry_interval < 1000) {
+            // The API will not allow polling faster than once per second.
+            $retry_interval = 1000;
+        }
+        usleep($retry_interval * 1000);
+    }
+
     /**
      * Runs a single iteration of the progress bar.
      * @return bool

diff --git a/src/Request/Request.php b/src/Request/Request.php
@@ -176,7 +176,12 @@ private function getClient($base_uri = null): ClientInterface
     private function createRetryDecider(): callable
     {
         $config = $this->getConfig();
-        $maxRetries = $config->get('http_max_retries', 5);
+        $maxRetries = $config->get('http_max_retries', $defaultMaxRetries);
+        // Cap max retries at 10.
+        $maxRetries = $maxRetries > 10 ? 10 : $maxRetries;
+        $retryBackoff = $config->get('http_retry_backoff', 5);
+        // Retry backoff should be at least 3.
+        $retryBackoff = $retryBackoff < 3 ? 3 : $retryBackoff;
         $logger = $this->logger;
         $logWarning = function (string $message) use ($logger) {
             if ($this->output()->isVerbose()) {
@@ -191,7 +196,8 @@ private function createRetryDecider(): callable
             ?Exception $exception = null
         ) use (
             $maxRetries,
-            $logWarning
+            $logWarning,
+            $retryBackoff
         ) {
             $logWarningOnRetry = fn (string $reason) => 0 === $retry
                 ? $logWarning(
@@ -217,6 +223,8 @@ private function createRetryDecider(): callable
                 // Retry on connection-related exceptions such as "Connection refused" and "Operation timed out".
                 if ($retry !== $maxRetries) {
                     $logWarningOnRetry($exception->getMessage());
+                    $logWarning(sprintf("Retrying in %s seconds.", $retryBackoff * ($retry + 1)));
+                    sleep($retryBackoff * ($retry + 1));
 
                     return true;
                 }
@@ -240,6 +248,9 @@ private function createRetryDecider(): callable
                         ['body' => $response->getBody()->getContents()]
                     );
 
+                    $logWarning(sprintf("Retrying in %s seconds.", $retryBackoff * ($retry + 1)));
+                    sleep($retryBackoff * ($retry + 1));
+
                     return true;
                 }
             }