feat: adds a rate limiting utility method and uses it in ape-node p…

…rovider (#2398)

src/ape/utils/__init__.py

@@ -84,7 +84,13 @@ def __getattr__(name: str):
 
         return getattr(process_module, name)
 
-    elif name in ("USER_AGENT", "RPCHeaders", "allow_disconnected", "stream_response"):
+    elif name in (
+        "USER_AGENT",
+        "RPCHeaders",
+        "allow_disconnected",
+        "request_with_retry",
+        "stream_response",
+    ):
         import ape.utils.rpc as rpc_module
 
         return getattr(rpc_module, name)
@@ -166,6 +172,7 @@ def __getattr__(name: str):
     "path_match",
     "raises_not_implemented",
     "returns_array",
+    "request_with_retry",
     "RPCHeaders",
     "run_in_tempdir",
     "run_until_complete",

src/ape/utils/rpc.py

@@ -1,10 +1,13 @@
+import time
 from collections.abc import Callable
+from random import randint
+from typing import Optional
 
 import requests
 from requests.models import CaseInsensitiveDict
 from tqdm import tqdm  # type: ignore
 
-from ape.exceptions import ProviderNotConnectedError
+from ape.exceptions import ProviderError, ProviderNotConnectedError
 from ape.logging import logger
 from ape.utils.misc import __version__, _python_version
 
@@ -89,3 +92,60 @@ def __setitem__(self, key, value):
 
         if new_user_agent := " ".join(new_parts):
             super().__setitem__(key, f"{existing_user_agent} {new_user_agent}")
+
+
+def request_with_retry(
+    func: Callable,
+    min_retry_delay: int = 1_000,
+    retry_backoff_factor: int = 2,
+    max_retry_delay: int = 30_000,
+    max_retries: int = 10,
+    retry_jitter: int = 250,
+    is_rate_limit: Optional[Callable[[Exception], bool]] = None,
+):
+    """
+    Make a request with 429/rate-limit retry logic.
+
+    Args:
+        func (Callable): The function to run with rate-limit handling logic.
+        min_retry_delay (int): The amount of milliseconds to wait before
+          retrying the request. Defaults to ``1_000`` (one second).
+        retry_backoff_factor (int): The multiplier applied to the retry delay
+          after each failed attempt. Defaults to ``2``.
+        max_retry_delay (int): The maximum length of the retry delay.
+          Defaults to ``30_000`` (30 seconds).
+        max_retries (int): The maximum number of retries.
+          Defaults to ``10``.
+        retry_jitter (int): A random number of milliseconds up to this limit
+          is added to each retry delay. Defaults to ``250`` milliseconds.
+        is_rate_limit (Callable[[Exception], bool] | None): A custom handler
+          for detecting rate-limits. Defaults to checking for a 429 status
+          code on an HTTPError.
+    """
+    if not is_rate_limit:
+        # Use default checker.
+        def checker(err: Exception) -> bool:
+            return isinstance(err, requests.HTTPError) and err.response.status_code == 429
+
+        is_rate_limit = checker
+
+    for attempt in range(max_retries):
+        try:
+            return func()
+        except Exception as err:
+            if not is_rate_limit(err):
+                # It was not a rate limit error. Raise whatever exception it is.
+                raise
+
+            else:
+                # We were rate-limited. Invoke retry/backoff logic.
+                logger.warning("Request was rate-limited. Backing-off and then retrying...")
+                retry_interval = min(
+                    max_retry_delay, min_retry_delay * retry_backoff_factor**attempt
+                )
+                delay = retry_interval + randint(0, retry_jitter)
+                time.sleep(delay / 1000)
+                continue
+
+    # If we get here, we over-waited. Raise custom exception.
+    raise ProviderError(f"Rate limit retry-mechanism exceeded after '{max_retries}' attempts.")

src/ape_ethereum/provider.py

@@ -30,7 +30,7 @@
 )
 
 try:
-    from web3.exceptions import Web3RPCError
+    from web3.exceptions import Web3RPCError  # type: ignore
 except ImportError:
     Web3RPCError = ValueError  # type: ignore
 
@@ -66,6 +66,7 @@
 from ape.utils._web3_compat import ExtraDataToPOAMiddleware, WebsocketProvider
 from ape.utils.basemodel import ManagerAccessMixin
 from ape.utils.misc import DEFAULT_MAX_RETRIES_TX, gas_estimation_error_message, to_int
+from ape.utils.rpc import request_with_retry
 from ape_ethereum._print import CONSOLE_ADDRESS, console_contract
 from ape_ethereum.trace import CallTrace, TraceApproach, TransactionTrace
 from ape_ethereum.transactions import AccessList, AccessListTransaction, TransactionStatusEnum
@@ -1134,8 +1135,10 @@ def _post_connect(self):
         self.chain_manager.contracts._cache_contract_type(CONSOLE_ADDRESS, console_contract)
 
     def make_request(self, rpc: str, parameters: Optional[Iterable] = None) -> Any:
-        parameters = parameters or []
+        return request_with_retry(lambda: self._make_request(rpc, parameters=parameters))
 
+    def _make_request(self, rpc: str, parameters: Optional[Iterable] = None) -> Any:
+        parameters = parameters or []
         try:
             result = self.web3.provider.make_request(RPCEndpoint(rpc), parameters)
         except HTTPError as err:
@@ -1144,6 +1147,9 @@ def make_request(self, rpc: str, parameters: Optional[Iterable] = None) -> Any:
                     f"RPC method '{rpc}' is not implemented by this node instance."
                 )
 
+            elif err.response.status_code == 429:
+                raise  # Raise as-is so rate-limit handling picks it up.
+
             raise ProviderError(str(err)) from err
 
         if "error" in result:

tests/functional/test_provider.py

@@ -22,7 +22,12 @@
 )
 from ape.types.events import LogFilter
 from ape.utils.testing import DEFAULT_TEST_ACCOUNT_BALANCE, DEFAULT_TEST_CHAIN_ID
-from ape_ethereum.provider import WEB3_PROVIDER_URI_ENV_VAR_NAME, Web3Provider, _sanitize_web3_url
+from ape_ethereum.provider import (
+    WEB3_PROVIDER_URI_ENV_VAR_NAME,
+    EthereumNodeProvider,
+    Web3Provider,
+    _sanitize_web3_url,
+)
 from ape_ethereum.transactions import TransactionStatusEnum, TransactionType
 from ape_test import LocalProvider
 
@@ -468,6 +473,33 @@ def custom_make_request(rpc, params):
         eth_tester_provider._web3 = real_web3
 
 
+def test_make_request_rate_limiting(mocker, ethereum, mock_web3):
+    provider = EthereumNodeProvider(network=ethereum.local)
+    provider._web3 = mock_web3
+
+    class RateLimitTester:
+        tries = 3
+        _try = 0
+        tries_made = 0
+
+        def rate_limit_hook(self, rpc, params):
+            self.tries_made += 1
+            if self._try >= self.tries:
+                self._try = 0
+                return {"success": True}
+            else:
+                self._try += 1
+                response = mocker.MagicMock()
+                response.status_code = 429
+                raise HTTPError(response=response)
+
+    rate_limit_tester = RateLimitTester()
+    mock_web3.provider.make_request.side_effect = rate_limit_tester.rate_limit_hook
+    result = provider.make_request("ape_testRateLimiting", parameters=[])
+    assert rate_limit_tester.tries_made == rate_limit_tester.tries + 1
+    assert result == {"success": True}
+
+
 def test_base_fee(eth_tester_provider):
     actual = eth_tester_provider.base_fee
     assert actual > 0