feat: support redis as cache backend for http requests and installati…

…on tokens

.pre-commit-config.yaml

@@ -31,7 +31,7 @@ repos:
     rev: v1.8.0
     hooks:
       - id: mypy
-        additional_dependencies: [types-requests, types-aiofiles]
+        additional_dependencies: [types-requests, types-aiofiles, types-redis]
   - repo: https://github.com/asottile/pyupgrade
     rev: v3.15.0
     hooks:

DEPENDENCIES

@@ -1,8 +1,9 @@
 pypi/pypi/-/aiofiles/23.2.1
 pypi/pypi/-/aiohttp/3.9.3
 pypi/pypi/-/aiohttp-client-cache/0.11.0
+pypi/pypi/-/aiohttp-client-cache/0.11.0
 pypi/pypi/-/aiosignal/1.3.1
-pypi/pypi/-/aiosqlite/0.19.0
+pypi/pypi/-/aiosqlite/0.20.0
 pypi/pypi/-/annotated-types/0.6.0
 pypi/pypi/-/async-timeout/4.0.3
 pypi/pypi/-/attrs/23.2.0
@@ -15,8 +16,8 @@ pypi/pypi/-/charset-normalizer/3.3.2
 pypi/pypi/-/chevron/0.14.0
 pypi/pypi/-/click/8.1.7
 pypi/pypi/-/colorama/0.4.6
-pypi/pypi/-/cryptography/42.0.4
-pypi/pypi/-/dnspython/2.5.0
+pypi/pypi/-/cryptography/42.0.5
+pypi/pypi/-/dnspython/2.6.1
 pypi/pypi/-/exceptiongroup/1.2.0
 pypi/pypi/-/flask/3.0.2
 pypi/pypi/-/flask-minify/0.42
@@ -47,7 +48,7 @@ pypi/pypi/-/mergedeep/1.3.4
 pypi/pypi/-/mintotp/0.3.0
 pypi/pypi/-/mkdocs/1.5.3
 pypi/pypi/-/mkdocs-exclude/1.0.2
-pypi/pypi/-/mkdocs-material/9.5.9
+pypi/pypi/-/mkdocs-material/9.5.11
 pypi/pypi/-/mkdocs-material-extensions/1.3.1
 pypi/pypi/-/motor/3.3.2
 pypi/pypi/-/multidict/6.0.5
@@ -60,12 +61,12 @@ pypi/pypi/-/playwright/1.41.2
 pypi/pypi/-/ply/3.11
 pypi/pypi/-/priority/2.0.0
 pypi/pypi/-/pycparser/2.21
-pypi/pypi/-/pydantic/2.6.1
-pypi/pypi/-/pydantic-core/2.16.2
+pypi/pypi/-/pydantic/2.6.2
+pypi/pypi/-/pydantic-core/2.16.3
 pypi/pypi/-/pyee/11.0.1
 pypi/pypi/-/pygments/2.17.2
 pypi/pypi/-/pymdown-extensions/10.7
-pypi/pypi/-/pymongo/4.6.1
+pypi/pypi/-/pymongo/4.6.2
 pypi/pypi/-/pynacl/1.5.0
 pypi/pypi/-/python-dateutil/2.8.2
 pypi/pypi/-/python-decouple/3.8
@@ -75,20 +76,22 @@ pypi/pypi/-/pyyaml-env-tag/0.1
 pypi/pypi/-/quart/0.19.4
 pypi/pypi/-/quart-auth/0.9.1
 pypi/pypi/-/quart-flask-patch/0.3.0
+pypi/pypi/-/quart-redis/2.0.0
 pypi/pypi/-/quart-uploads/0.0.2
 pypi/pypi/-/quart-wtforms/1.0.1
 pypi/pypi/-/rcssmin/1.1.2
+pypi/pypi/-/redis/4.6.0
 pypi/pypi/-/referencing/0.33.0
 pypi/pypi/-/regex/2023.12.25
 pypi/pypi/-/requests/2.31.0
-pypi/pypi/-/requests-cache/1.1.1
+pypi/pypi/-/requests-cache/1.2.0
 pypi/pypi/-/rpds-py/0.18.0
 pypi/pypi/-/six/1.16.0
 pypi/pypi/-/taskgroup/0.0.0a4
 pypi/pypi/-/tomli/2.0.1
-pypi/pypi/-/typing-extensions/4.9.0
+pypi/pypi/-/typing-extensions/4.10.0
 pypi/pypi/-/url-normalize/1.4.3
-pypi/pypi/-/urllib3/2.2.0
+pypi/pypi/-/urllib3/2.2.1
 pypi/pypi/-/watchdog/4.0.0
 pypi/pypi/-/werkzeug/3.0.1
 pypi/pypi/-/wsproto/1.2.0

docker/docker-compose.base.yml

@@ -5,6 +5,7 @@ services:
     restart: always
     depends_on:
       - mongodb
+      - redis
     env_file: ../.env
     build:
       context: ..
@@ -29,6 +30,15 @@ services:
     image: mongo:7.0.5
     command: mongod --quiet --logpath /dev/null
     ports:
-      - 27017:27017
+      - '27017:27017'
     volumes:
       - ../approot/db:/data/db
+
+  redis:
+    image: redis:7.2.4-alpine
+    restart: always
+    ports:
+      - '6379:6379'
+    command: redis-server --save 20 1 --loglevel warning
+    volumes:
+      - ../approot/cache:/data/

otterdog/providers/github/cache/__init__.py

@@ -0,0 +1,52 @@
+#  *******************************************************************************
+#  Copyright (c) 2024 Eclipse Foundation and others.
+#  This program and the accompanying materials are made available
+#  under the terms of the Eclipse Public License 2.0
+#  which is available at http://www.eclipse.org/legal/epl-v20.html
+#  SPDX-License-Identifier: EPL-2.0
+#  *******************************************************************************
+
+from abc import ABC, abstractmethod
+
+from aiohttp_client_cache import CacheBackend
+from redis.asyncio.client import Redis
+
+
+class CacheStrategy(ABC):
+    @abstractmethod
+    def get_cache_backend(self) -> CacheBackend: ...
+
+
+_AIOHTTP_CACHE_DIR = ".cache/async_http"
+
+
+class _FileCache(CacheStrategy):
+    def __init__(self, cache_dir: str):
+        self._cache_dir = cache_dir
+
+    def get_cache_backend(self) -> CacheBackend:
+        from aiohttp_client_cache.backends import FileBackend
+
+        return FileBackend(
+            cache_name=self._cache_dir,
+            use_temp=False,
+        )
+
+
+class _RedisCache(CacheStrategy):
+    def __init__(self, redis_uri: str, connection: Redis | None):
+        self._redis_uri = redis_uri
+        self._connection = connection
+
+    def get_cache_backend(self) -> CacheBackend:
+        from aiohttp_client_cache.backends import RedisBackend
+
+        return RedisBackend(address=self._redis_uri, connection=self._connection)
+
+
+def file_cache(cache_dir: str = _AIOHTTP_CACHE_DIR) -> CacheStrategy:
+    return _FileCache(cache_dir)
+
+
+def redis_cache(uri: str, connection: Redis | None = None) -> CacheStrategy:
+    return _RedisCache(uri, connection)

otterdog/providers/github/rest/__init__.py

@@ -13,6 +13,7 @@
 from functools import cached_property
 
 from otterdog.providers.github.auth import AuthStrategy
+from otterdog.providers.github.cache import CacheStrategy, file_cache
 
 from .requester import Requester
 
@@ -22,9 +23,10 @@ class RestApi:
     _GH_API_VERSION = "2022-11-28"
     _GH_API_URL_ROOT = "https://api.github.com"
 
-    def __init__(self, auth_strategy: AuthStrategy | None = None):
+    def __init__(self, auth_strategy: AuthStrategy | None = None, cache_strategy: CacheStrategy = file_cache()):
         self._auth_strategy = auth_strategy
-        self._requester = Requester(auth_strategy, self._GH_API_URL_ROOT, self._GH_API_VERSION)
+        self._cache_strategy = cache_strategy
+        self._requester = Requester(auth_strategy, cache_strategy, self._GH_API_URL_ROOT, self._GH_API_VERSION)
 
     async def __aenter__(self):
         return self
@@ -130,8 +132,5 @@ def encrypt_value(public_key: str, secret_value: str) -> str:
     return b64encode(encrypted).decode("utf-8")
 
 
-_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
-
-
-def parse_date_string(date: str) -> datetime:
-    return datetime.strptime(date, _FORMAT)
+def parse_iso_date_string(date: str) -> datetime:
+    return datetime.fromisoformat(date)

otterdog/providers/github/rest/app_client.py

@@ -12,7 +12,7 @@
 from otterdog.providers.github.exception import GitHubException
 from otterdog.utils import print_debug
 
-from . import RestApi, RestClient, parse_date_string
+from . import RestApi, RestClient, parse_iso_date_string
 
 
 class AppClient(RestClient):
@@ -42,7 +42,7 @@ async def create_installation_access_token(self, installation_id: str) -> tuple[
 
         try:
             response = await self.requester.request_json("POST", f"/app/installations/{installation_id}/access_tokens")
-            return response["token"], parse_date_string(response["expires_at"])
+            return response["token"], parse_iso_date_string(response["expires_at"])
         except GitHubException as ex:
             tb = ex.__traceback__
             raise RuntimeError(f"failed creating installation access token:\n{ex}").with_traceback(tb)

otterdog/providers/github/rest/requester.py

@@ -10,18 +10,24 @@
 from collections.abc import AsyncIterable
 from typing import Any
 
-from aiohttp_client_cache.backends import FileBackend
 from aiohttp_client_cache.session import CachedSession as AsyncCachedSession
 
 from otterdog.providers.github.auth import AuthStrategy
+from otterdog.providers.github.cache import CacheStrategy
 from otterdog.providers.github.exception import BadCredentialsException, GitHubException
 from otterdog.utils import is_debug_enabled, is_trace_enabled, print_debug, print_trace
 
 _AIOHTTP_CACHE_DIR = ".cache/async_http"
 
 
 class Requester:
-    def __init__(self, auth_strategy: AuthStrategy | None, base_url: str, api_version: str):
+    def __init__(
+        self,
+        auth_strategy: AuthStrategy | None,
+        cache_strategy: CacheStrategy,
+        base_url: str,
+        api_version: str,
+    ):
         self._base_url = base_url
         self._auth = auth_strategy.get_auth() if auth_strategy is not None else None
 
@@ -31,12 +37,7 @@ def __init__(self, auth_strategy: AuthStrategy | None, base_url: str, api_versio
             "X-Github-Next-Global-ID": "1",
         }
 
-        self._session = AsyncCachedSession(
-            cache=FileBackend(
-                cache_name=_AIOHTTP_CACHE_DIR,
-                use_temp=False,
-            ),
-        )
+        self._session = AsyncCachedSession(cache=cache_strategy.get_cache_backend())
 
     async def close(self) -> None:
         await self._session.close()

otterdog/webapp/__init__.py

@@ -16,19 +16,24 @@
 from quart import Quart
 from quart.json.provider import DefaultJSONProvider
 from quart_auth import QuartAuth
+from quart_redis import RedisHandler  # type: ignore
 
 from .config import AppConfig
 from .db import Mongo, init_mongo_database
+from .filters import register_filters
+from .utils import close_rest_apis
 
 _BLUEPRINT_MODULES: list[str] = ["home", "api"]
 
 mongo = Mongo()
 auth_manager = QuartAuth()
+redis_handler = RedisHandler()
 
 
 def register_extensions(app):
     mongo.init_app(app)
     auth_manager.init_app(app)
+    redis_handler.init_app(app, decode_responses=True)
 
 
 def register_github_webhook(app) -> None:
@@ -55,49 +60,6 @@ async def configure():
             await init_mongo_database(mongo)
 
 
-def register_filters(app):
-    @app.template_filter("status")
-    def status_color(status):
-        from otterdog.webapp.db.models import InstallationStatus
-
-        match status:
-            case InstallationStatus.INSTALLED:
-                return "success"
-            case InstallationStatus.NOT_INSTALLED:
-                return "danger"
-            case InstallationStatus.SUSPENDED:
-                return "warning"
-            case _:
-                return "info"
-
-    @app.template_filter("is_dict")
-    def is_dict(value):
-        return isinstance(value, dict)
-
-    @app.template_filter("length_to_color")
-    def length_to_color(value):
-        if len(value) == 0:
-            return "primary"
-        else:
-            return "success"
-
-    @app.template_filter("has_dummy_secret")
-    def has_dummy_secret(value):
-        return value.has_dummy_secret()
-
-    @app.template_filter("has_dummy_secrets")
-    def any_has_dummy_secrets(value):
-        return any(map(lambda x: x.has_dummy_secret(), value))
-
-    @app.template_filter("pretty_format_model")
-    def pretty_format_model(value):
-        from otterdog.models import ModelObject
-        from otterdog.utils import PrettyFormatter
-
-        assert isinstance(value, ModelObject)
-        return PrettyFormatter().format(value.to_model_dict(False, False))
-
-
 def create_app(app_config: AppConfig):
     app = Quart(app_config.QUART_APP)
     app.config.from_object(app_config)
@@ -117,4 +79,8 @@ def default(self, o):
 
     app.json = CustomJSONProvider(app)
 
+    @app.after_serving
+    async def close_resources() -> None:
+        await close_rest_apis()
+
     return app

otterdog/webapp/config.py

@@ -23,6 +23,7 @@ class AppConfig:
     DB_ROOT = os.path.join(APP_ROOT, "db")
 
     MONGO_URI = config("MONGO_URI", default="mongodb://mongodb:27017/otterdog")
+    REDIS_URI = config("REDIS_URI", default="redis://redis:6379")
 
     OTTERDOG_CONFIG_OWNER = config("OTTERDOG_CONFIG_OWNER", default=None)
     OTTERDOG_CONFIG_REPO = config("OTTERDOG_CONFIG_REPO", default=None)

otterdog/webapp/filters.py

@@ -0,0 +1,50 @@
+#  *******************************************************************************
+#  Copyright (c) 2024 Eclipse Foundation and others.
+#  This program and the accompanying materials are made available
+#  under the terms of the Eclipse Public License 2.0
+#  which is available at http://www.eclipse.org/legal/epl-v20.html
+#  SPDX-License-Identifier: EPL-2.0
+#  *******************************************************************************
+
+
+def register_filters(app):
+    @app.template_filter("status")
+    def status_color(status):
+        from otterdog.webapp.db.models import InstallationStatus
+
+        match status:
+            case InstallationStatus.INSTALLED:
+                return "success"
+            case InstallationStatus.NOT_INSTALLED:
+                return "danger"
+            case InstallationStatus.SUSPENDED:
+                return "warning"
+            case _:
+                return "info"
+
+    @app.template_filter("is_dict")
+    def is_dict(value):
+        return isinstance(value, dict)
+
+    @app.template_filter("length_to_color")
+    def length_to_color(value):
+        if len(value) == 0:
+            return "primary"
+        else:
+            return "success"
+
+    @app.template_filter("has_dummy_secret")
+    def has_dummy_secret(value):
+        return value.has_dummy_secret()
+
+    @app.template_filter("has_dummy_secrets")
+    def any_has_dummy_secrets(value):
+        return any(map(lambda x: x.has_dummy_secret(), value))
+
+    @app.template_filter("pretty_format_model")
+    def pretty_format_model(value):
+        from otterdog.models import ModelObject
+        from otterdog.utils import PrettyFormatter
+
+        assert isinstance(value, ModelObject)
+        return PrettyFormatter().format(value.to_model_dict(False, False))