diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index a27249f..525784d 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,7 +9,7 @@ jobs: steps: - uses: actions/checkout@master - name: Setting up tox environment - uses: docker://kiwicom/tox:3.21.2 + uses: docker://kiwicom/tox:3.25.0 env: TOXENV: ${{ matrix.toxenv }} XDG_CACHE_HOME: /tmp/cache @@ -27,7 +27,7 @@ jobs: key: ${{ runner.os }}-${{ hashFiles('requirements.txt') }} strategy: matrix: - toxenv: [py37, py38, py39] + toxenv: [py39, py310, py311] lint: name: pylint @@ -66,17 +66,29 @@ jobs: pre-commit: name: Static checks runs-on: ubuntu-latest - container: kiwicom/pre-commit:2.9.3 + container: kiwicom/pre-commit:3.6.0 + steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: 3.11 - name: Cache - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: .pre-commit-cache key: static-checks-${{ hashFiles('.pre-commit-config.yaml') }} + # git checkout is not creating a working git folder, it has dubious ownership if not configured + # The pre-commit hook will not work on this original git folder + # https://github.com/pre-commit/pre-commit/issues/2125 + - name: fix git + run: git config --system --add safe.directory '*' + - run: mkdir -p .pre-commit-cache - run: pre-commit install --install-hooks - run: pre-commit run -a diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4180fe3..89810b2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - pyver: ["3.6", "3.7", "3.8", "3.9"] + pyver: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v2 - name: Set up Python diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 98745b2..e6a2fb3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ --- default_language_version: - python: python3 + python: python3.11 exclude: "^.github.*" repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.6.0 hooks: - id: trailing-whitespace exclude: ^.*\.md$ @@ -24,7 +24,7 @@ repos: - id: gitlint - repo: https://github.com/adrienverge/yamllint - rev: v1.26.0 + rev: v1.30.0 hooks: - id: yamllint @@ -35,7 +35,7 @@ repos: language_version: system - repo: https://github.com/pycqa/isort - rev: 5.7.0 + rev: 5.13.2 hooks: - id: isort additional_dependencies: [".[pyproject]"] @@ -46,7 +46,7 @@ repos: - id: black - repo: https://github.com/PyCQA/pylint - rev: pylint-2.6.0 + rev: v3.2.6 hooks: - id: pylint exclude: ^(docs/).*$ diff --git a/.pylintrc b/.pylintrc index 8bde44c..9fb7c69 100644 --- a/.pylintrc +++ b/.pylintrc @@ -11,16 +11,13 @@ disable = no-member, unused-argument, broad-except, - relative-import, wrong-import-position, bare-except, locally-disabled, protected-access, abstract-method, - no-self-use, fixme, too-few-public-methods, - bad-continuation, useless-object-inheritance, too-many-arguments, too-many-locals, diff --git a/docs-requirements.txt b/docs-requirements.txt index 1b7ab40..903b6dc 100644 --- a/docs-requirements.txt +++ b/docs-requirements.txt @@ -1,62 +1,57 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # # pip-compile --output-file=./docs-requirements.txt ./docs-requirements.in # -alabaster==0.7.12 +alabaster==0.7.16 # via sphinx -babel==2.10.3 +babel==2.16.0 # via sphinx -certifi==2022.6.15 +certifi==2024.7.4 # via requests -charset-normalizer==2.0.12 +charset-normalizer==3.3.2 # via requests -docutils==0.17.1 +docutils==0.20.1 # via # sphinx # sphinx-rtd-theme -idna==3.3 +idna==3.7 # via requests -imagesize==1.3.0 +imagesize==1.4.1 # via sphinx -importlib-metadata==4.12.0 +jinja2==3.1.4 # via sphinx -jinja2==3.1.2 - # via sphinx -markupsafe==2.1.1 +markupsafe==2.1.5 # via jinja2 -packaging==21.3 +packaging==24.1 # via sphinx -pygments==2.12.0 +pygments==2.18.0 # via sphinx -pyparsing==3.0.9 - # via packaging -pytz==2022.1 - # via babel -requests==2.28.0 +requests==2.32.3 # via sphinx snowballstemmer==2.2.0 # via sphinx -sphinx==5.0.2 +sphinx==7.4.7 # via # -r ./docs-requirements.in # sphinx-rtd-theme -sphinx-rtd-theme==1.0.0 + # sphinxcontrib-jquery +sphinx-rtd-theme==2.0.0 # via -r ./docs-requirements.in -sphinxcontrib-applehelp==1.0.2 +sphinxcontrib-applehelp==2.0.0 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.0 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx +sphinxcontrib-jquery==4.1 + # via sphinx-rtd-theme sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.5 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx -urllib3==1.26.9 +urllib3==2.2.2 # via requests -zipp==3.8.0 - # via importlib-metadata diff --git a/mypy.ini b/mypy.ini index 4373a3b..3f00513 100644 --- a/mypy.ini +++ b/mypy.ini @@ -1,5 +1,5 @@ [mypy] -python_version = 3.7 +python_version = 3.11 ignore_missing_imports = True disallow_untyped_defs = True disallow_incomplete_defs = True diff --git a/request_session/_compat.py b/request_session/_compat.py index 0f58073..3fa0b08 100644 --- a/request_session/_compat.py +++ b/request_session/_compat.py @@ -1,4 +1,5 @@ """Module ensuring lib compatibilty.""" + from __future__ import absolute_import try: diff --git a/request_session/exceptions.py b/request_session/exceptions.py index a50ee05..8fed82c 100644 --- a/request_session/exceptions.py +++ b/request_session/exceptions.py @@ -1,4 +1,5 @@ """Module contains exceptions from requests and request_session.""" + from typing import Any # pylint: disable=unused-import from requests.exceptions import ConnectionError # pylint: disable=redefined-builtin diff --git a/request_session/protocols.py b/request_session/protocols.py index e378772..9f1b2ef 100644 --- a/request_session/protocols.py +++ b/request_session/protocols.py @@ -1,4 +1,5 @@ """Simple protocols to duck type dependency injections.""" + from typing import Any, List, Optional diff --git a/request_session/request_session.py b/request_session/request_session.py index d870404..691d9a3 100644 --- a/request_session/request_session.py +++ b/request_session/request_session.py @@ -1,4 +1,5 @@ """Main RequestSession module.""" + import re import time from collections import namedtuple @@ -113,7 +114,9 @@ def __init__( self.logger = logger self.log_prefix = log_prefix self.allowed_log_levels = allowed_log_levels - self.retriable_client_errors = retriable_client_errors if retriable_client_errors else [408] + self.retriable_client_errors = ( + retriable_client_errors if retriable_client_errors else [408] + ) self.prepare_new_session() @@ -149,17 +152,7 @@ def set_user_agent(self): # type: () -> None """Set proper user-agent string to header according to RFC22.""" pattern = r"^(?P\S.+?)\/(?P\S.+?) \((?P\S.+?) (?P\S.+?)\)(?: ?(?P.*))$" - string = ( - "{service_name}/{version} ({organization} {environment}) {sys_info}".format( - service_name=self.user_agent_components.service_name, # type: ignore - version=self.user_agent_components.version, # type: ignore - organization=self.user_agent_components.organization, # type: ignore - environment=self.user_agent_components.environment, # type: ignore - sys_info=self.user_agent_components.sys_info # type: ignore - if self.user_agent_components.sys_info # type: ignore - else "", - ).strip() - ) + string = f"{self.user_agent_components.service_name}/{self.user_agent_components.version} ({self.user_agent_components.organization} {self.user_agent_components.environment}) {self.user_agent_components.sys_info if self.user_agent_components.sys_info else ''}".strip() if not re.match(pattern, string): raise InvalidUserAgentString("Provided User-Agent string is not valid.") self.user_agent = string @@ -273,7 +266,7 @@ def _process( sleep_before_repeat=None, # type: Optional[float] tags=None, # type: Optional[list] raise_for_status=None, # type: Optional[bool] - **request_kwargs # type: Any + **request_kwargs, # type: Any ): # pylint: disable=too-many-statements # type: (...) -> Optional[requests.Response] r"""Run a request against a service depending on a request type. @@ -366,9 +359,11 @@ def _process( attempt=run, ) - if self.is_server_error(error, status_code) or self.retry_on_client_errors(status_code): + if self.is_server_error( + error, status_code + ) or self.retry_on_client_errors(status_code): if is_econnreset_error: - self.log("info", "{}.session_replace".format(request_category)) + self.log("info", f"{request_category}.session_replace") self.remove_session() self.prepare_new_session() @@ -444,9 +439,7 @@ def _send_request(self, request_type, request_params, tags, run, request_categor :param str request_category: Category for log and metric reporting. :return requests.Response: HTTP Response Object. """ - metric_name = "{request_category}.response_time".format( - request_category=request_category - ) + metric_name = f"{request_category}.response_time" if not self.statsd: return self.session.request(method=request_type, **request_params) @@ -491,7 +484,7 @@ def _log_with_params( status_code=response.status_code, attempt=attempt, url=url, - **extra_params + **extra_params, ) def sleep(self, seconds, request_category, tags): @@ -518,12 +511,10 @@ def metric_increment(self, metric, request_category, tags, attempt=None): """ new_tags = list(tags) if tags else [] if attempt: - new_tags.append("attempt:{attempt}".format(attempt=attempt)) + new_tags.append(f"attempt:{attempt}") if self.statsd is not None: - metric_name = "{metric_base}.{metric_type}".format( - metric_base=request_category, metric_type=metric - ) + metric_name = f"{request_category}.{metric}" self.statsd.increment(metric_name, tags=new_tags) def log(self, level, event, **kwargs): @@ -537,7 +528,7 @@ def log(self, level, event, **kwargs): """ if not level in self.allowed_log_levels: raise AttributeError("Provided log level is not allowed.") - event_name = "{prefix}.{event}".format(prefix=self.log_prefix, event=event) + event_name = f"{self.log_prefix}.{event}" if self.logger is not None: getattr(self.logger, level)(event_name, **kwargs) @@ -586,17 +577,17 @@ def _exception_log_and_metrics( self.log( "exception", - "{}.failed".format(request_category), + f"{request_category}.failed", error_type=error_type, status_code=status_code, attempt=attempt, - **extra_params + **extra_params, ) self.metric_increment( metric="request", request_category=request_category, - tags=tags + ["attempt:{}".format(attempt)], + tags=tags + [f"attempt:{attempt}"], ) @staticmethod diff --git a/request_session/utils.py b/request_session/utils.py index f8e90e9..a65ca9d 100644 --- a/request_session/utils.py +++ b/request_session/utils.py @@ -1,8 +1,8 @@ """Utilites used in RequestSession.""" -import logging + import sys import time -from typing import Any, Dict, Iterator, List, Optional, Text +from typing import Any, Dict, List, Optional from .protocols import Ddtrace @@ -36,11 +36,9 @@ def split_tags_and_update(dictionary, tags): def dict_to_string(dictionary): - # type: (Dict[str, Any]) -> Text + # type: (Dict[str, Any]) -> str """Convert dictionary to key=value pairs separated by a space.""" - return " ".join( - ["{}={}".format(key, value) for key, value in sorted(dictionary.items())] - ) + return " ".join([f"{key}={value}" for key, value in sorted(dictionary.items())]) def traced_sleep(trace_name, seconds, ddtrace, tags=None): diff --git a/requirements.txt b/requirements.txt index 914f6fe..92a736a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,20 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # # pip-compile requirements.in # -certifi==2022.6.15 +certifi==2024.7.4 # via requests -charset-normalizer==2.0.12 +charset-normalizer==3.3.2 # via requests -idna==3.3 +idna==3.7 # via requests -requests==2.28.0 +requests==2.32.3 # via -r requirements.in -simplejson==3.17.6 +simplejson==3.19.2 # via -r requirements.in -structlog==21.5.0 +structlog==24.4.0 # via -r requirements.in -urllib3==1.26.9 +urllib3==2.2.2 # via requests diff --git a/setup.py b/setup.py index f49e6fe..0806d1f 100644 --- a/setup.py +++ b/setup.py @@ -5,15 +5,15 @@ with open("README.md", encoding="utf-8") as f: readme = f.read() -with open("requirements.in") as f: +with open("requirements.in", encoding="utf-8") as f: install_requires = [line for line in f if line and line[0] not in "#-"] -with open("test-requirements.in") as f: +with open("test-requirements.in", encoding="utf-8") as f: tests_require = [line for line in f if line and line[0] not in "#-"] setup( name="request_session", - version="0.15.0", + version="0.16.0", url="https://github.com/kiwicom/request-session", description="Python HTTP requests on steroids", long_description=readme, @@ -28,9 +28,9 @@ "Development Status :: 4 - Beta", "Operating System :: OS Independent", "Topic :: Internet :: WWW/HTTP", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", "Intended Audience :: Developers", ], ) diff --git a/test-requirements.txt b/test-requirements.txt index 3372c9f..79356ce 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,82 +1,86 @@ # -# This file is autogenerated by pip-compile with python 3.9 -# To update, run: +# This file is autogenerated by pip-compile with Python 3.11 +# by the following command: # # pip-compile --output-file=./test-requirements.txt ./test-requirements.in # -attrs==21.4.0 - # via pytest -blinker==1.4 - # via raven -brotlipy==0.7.0 +attrs==24.2.0 + # via + # jsonschema + # referencing +brotlicffi==1.1.0.0 # via httpbin -cffi==1.15.0 - # via brotlipy -click==8.1.3 +cffi==1.17.0 + # via brotlicffi +click==8.1.7 # via flask -coverage[toml]==6.4.1 +coverage[toml]==7.6.1 # via pytest-cov decorator==5.1.1 # via httpbin -flask==2.1.2 +flasgger==0.9.7.1 + # via httpbin +flask==2.1.3 # via + # flasgger # httpbin - # raven -httpbin==0.7.0 +greenlet==2.0.2 + # via httpbin +httpbin==0.10.1 # via pytest-httpbin -importlib-metadata==4.12.0 - # via flask -iniconfig==1.1.1 +iniconfig==2.0.0 # via pytest -itsdangerous==2.1.2 - # via - # flask - # httpbin -jinja2==3.1.2 +itsdangerous==2.2.0 # via flask -markupsafe==2.1.1 - # via - # httpbin - # jinja2 -more-itertools==8.13.0 +jinja2==3.1.4 + # via flask +jsonschema==4.23.0 + # via flasgger +jsonschema-specifications==2023.12.1 + # via jsonschema +markupsafe==2.1.5 + # via jinja2 +mistune==3.0.2 + # via flasgger +more-itertools==10.4.0 # via -r ./test-requirements.in -packaging==21.3 - # via pytest -pluggy==1.0.0 - # via pytest -py==1.11.0 +packaging==24.1 + # via + # flasgger + # pytest +pluggy==1.5.0 # via pytest -pycparser==2.21 +pycparser==2.22 # via cffi -pyparsing==3.0.9 - # via packaging -pytest==7.1.2 +pytest==8.3.2 # via # -r ./test-requirements.in # pytest-cov # pytest-mock -pytest-cov==3.0.0 +pytest-cov==5.0.0 # via -r ./test-requirements.in -pytest-httpbin==1.0.2 +pytest-httpbin==2.0.0 # via -r ./test-requirements.in -pytest-mock==3.8.1 +pytest-mock==3.14.0 # via -r ./test-requirements.in -raven[flask]==6.10.0 - # via httpbin +pyyaml==6.0.2 + # via flasgger +referencing==0.35.1 + # via + # jsonschema + # jsonschema-specifications +rpds-py==0.20.0 + # via + # jsonschema + # referencing six==1.16.0 # via + # flasgger # httpbin - # pytest-httpbin -structlog==21.5.0 +structlog==24.4.0 # via -r ./test-requirements.in -tomli==2.0.1 - # via - # coverage - # pytest werkzeug==2.0.3 # via # -r ./test-requirements.in # flask # httpbin -zipp==3.8.0 - # via importlib-metadata diff --git a/test/conftest.py b/test/conftest.py index 6466e69..57a49df 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,4 +1,5 @@ """Just a conftest.""" + from typing import Any, Callable import httpbin as Httpbin diff --git a/test/test_request_session.py b/test/test_request_session.py index 4557560..742d4b3 100644 --- a/test/test_request_session.py +++ b/test/test_request_session.py @@ -1,4 +1,5 @@ """Test the main module.""" + import itertools import sys from typing import Any, Callable, Dict, Iterator, List, Union @@ -29,7 +30,7 @@ def test_init(mocker, httpbin): # type: (Mock, Httpbin) -> None """Test initialization of RequestSession.""" mock_ddtrace = mocker.Mock(spec_set=Ddtrace) - mock_tracing_config = dict() # type: Dict[Any, Any] + mock_tracing_config: dict = {} mock_ddtrace.config.get_from.return_value = mock_tracing_config session = RequestSession( @@ -179,13 +180,13 @@ def test_raise_for_status(mocker, httpbin, status_code, raises): mock_sys.exc_info.return_value = (HTTPError, HTTPError(), "fake_traceback") if raises: with pytest.raises(raises): - session.get(path="/status/{status_code}".format(status_code=status_code)) + session.get(path=f"/status/{status_code}") if isinstance(raises, HTTPError): assert mock_sys.exc_info()[1].__sentry_source == "third_party" assert mock_sys.exc_info()[1].__sentry_pd_alert == "disabled" else: - session.get(path="/status/{status_code}".format(status_code=status_code)) + session.get(path=f"/status/{status_code}") @pytest.mark.parametrize( @@ -239,9 +240,7 @@ def _prepare_new_session(self): # type: ignore client.get("/status/500") actual_call_count = sum(session.request.call_count for session in used_sessions) - mock_log.assert_called_with( - "info", "{category}.session_replace".format(category=client.request_category) - ) + mock_log.assert_called_with("info", f"{client.request_category}.session_replace") assert mock_exception_log_and_metrics.call_count == 1 assert actual_call_count == expected_call_count @@ -305,7 +304,7 @@ def test_logging(mocker, request_session, inputs, expected): ) client = request_session(max_retries=inputs["max_retries"]) client._exception_log_and_metrics = mock_exception_log_and_metrics - expected["request_params"]["url"] = "{}{}".format(client.host, inputs["path"]) + expected["request_params"]["url"] = f"{client.host}{inputs['path']}" calls = [] for attempt in range(1, expected["call_count"] + 1): @@ -360,11 +359,11 @@ def test_metric_increment( calls = [] for attempt in range(1, call_count + 1): - metric = "{}.{}".format(client._get_request_category(), "request") - tags = ["status:{}".format(status)] + metric = f"{client._get_request_category()}.request" + tags = [f"status:{status}"] if error: - tags.append("error:{}".format(error)) - calls.append(mocker.call(metric, tags=tags + ["attempt:{}".format(attempt)])) + tags.append(f"error:{error}") + calls.append(mocker.call(metric, tags=tags + [f"attempt:{attempt}"])) assert mock_statsd.increment.call_count == call_count mock_statsd.increment.assert_has_calls(calls) @@ -458,7 +457,7 @@ def test_send_request(request_session, mocker, inputs, expected): assert isinstance(response, requests.Response) mock_statsd.timed.assert_called_once_with( - "{}.response_time".format(client.request_category), + f"{client.request_category}.response_time", use_ms=True, tags=inputs["tags"], ) @@ -608,7 +607,7 @@ def test_exception_and_log_metrics(request_session, mocker, inputs, expected): mock_log.assert_called_once_with( "exception", - "{}.failed".format(client.request_category), + f"{client.request_category}.failed", error_type=expected["error_type"], status_code=inputs["status_code"], **expected["extra_params"], diff --git a/test/test_utils.py b/test/test_utils.py index c1f819d..3895e7e 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -1,4 +1,5 @@ """Test the utilities.""" + import sys from typing import Dict, List diff --git a/tox.ini b/tox.ini index e938c3f..84f9116 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = pylint,tests-{py37,py38,py39} +envlist = pylint,tests-{py39,py310,py311} [testenv] deps =