From c8713221b4cada28f56cddbd1726c6b3b8b82833 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Ricks?= Date: Fri, 3 Nov 2023 11:01:20 +0100 Subject: [PATCH] Change: Improve rate limit algorithm for requests against the NVT API The original algorithm was very simple by just applying a pause after a specific number of allowed requests. When doing additional costly tasks after each requests this algorithm results in unnecessary delays and the sleep might me completely obsolete. Therefore calculate the time since the last check for a sleep. If this time delta is within the rate limit we wait for the required time before doing the next request. --- pontos/nvd/api.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pontos/nvd/api.py b/pontos/nvd/api.py index 598d2d359..34949b329 100644 --- a/pontos/nvd/api.py +++ b/pontos/nvd/api.py @@ -16,6 +16,7 @@ # along with this program. If not, see . import asyncio +import time from abc import ABC from datetime import datetime, timezone from types import TracebackType @@ -76,10 +77,6 @@ def convert_camel_case(dct: Dict[str, Any]) -> Dict[str, Any]: return converted -async def sleep() -> None: - await asyncio.sleep(SLEEP_TIMEOUT) - - class NVDApi(ABC): """ Abstract base class for querying the NIST NVD API. @@ -120,6 +117,7 @@ def __init__( self._rate_limit = None self._request_count = 0 + self._last_sleep = time.monotonic() def _request_headers(self) -> Headers: """ @@ -141,7 +139,13 @@ async def _consider_rate_limit(self) -> None: self._request_count += 1 if self._request_count > self._rate_limit: - await sleep() + time_since_last_sleep = time.monotonic() - self._last_sleep + + if time_since_last_sleep < SLEEP_TIMEOUT: + time_to_sleep = SLEEP_TIMEOUT - time_since_last_sleep + await asyncio.sleep(time_to_sleep) + + self._last_sleep = time.monotonic() self._request_count = 0 async def _get(