[#20] * Added configurable postprocessing, that allows to modify valu…

…e retrieved from the cache

  * Added built-in implementation, that applies deep-copy
* Fix MANIFEST.in

CHANGELOG.rst

@@ -1,3 +1,10 @@
+2.1.0
+-----
+
+* Added configurable postprocessing, that allows to modify value retrieved from the cache
+  * Added built-in implementation, that applies deep-copy
+* Fix MANIFEST.in
+
 2.0.0
 -----
 

MANIFEST.in

@@ -1,3 +1,3 @@
 include requirements.txt
 include README.rst
-include CHANGELOG.md
+include CHANGELOG.rst

README.rst

@@ -192,6 +192,10 @@ With *memoize* you have under control:
   least-recently-updated strategy is already provided;
 * entry builder (see :class:`memoize.entrybuilder.CacheEntryBuilder`)
   which has control over ``update_after``  & ``expires_after`` described in `Tunable eviction & async refreshing`_
+* value post-processing (see :class:`memoize.postprocessing.Postprocessing`);
+  noop is the default one;
+  deep-copy post-processing is also provided (be wary of deep-copy cost & limitations,
+  but deep-copying allows callers to safely modify values retrieved from an in-memory cache).
 
 All of these elements are open for extension (you can implement and plug-in your own).
 Please contribute!
@@ -209,19 +213,21 @@ Example how to customize default config (everything gets overridden):
     from memoize.entrybuilder import ProvidedLifeSpanCacheEntryBuilder
     from memoize.eviction import LeastRecentlyUpdatedEvictionStrategy
     from memoize.key import EncodedMethodNameAndArgsKeyExtractor
+    from memoize.postprocessing import DeepcopyPostprocessing
     from memoize.storage import LocalInMemoryCacheStorage
     from memoize.wrapper import memoize
 
-
-    @memoize(configuration=MutableCacheConfiguration
-             .initialized_with(DefaultInMemoryCacheConfiguration())
-             .set_method_timeout(value=timedelta(minutes=2))
-             .set_entry_builder(ProvidedLifeSpanCacheEntryBuilder(update_after=timedelta(minutes=2),
-                                                                  expire_after=timedelta(minutes=5)))
-             .set_eviction_strategy(LeastRecentlyUpdatedEvictionStrategy(capacity=2048))
-             .set_key_extractor(EncodedMethodNameAndArgsKeyExtractor(skip_first_arg_as_self=False))
-             .set_storage(LocalInMemoryCacheStorage())
-             )
+    @memoize(
+        configuration=MutableCacheConfiguration
+        .initialized_with(DefaultInMemoryCacheConfiguration())
+        .set_method_timeout(value=timedelta(minutes=2))
+        .set_entry_builder(ProvidedLifeSpanCacheEntryBuilder(update_after=timedelta(minutes=2),
+                                                             expire_after=timedelta(minutes=5)))
+        .set_eviction_strategy(LeastRecentlyUpdatedEvictionStrategy(capacity=2048))
+        .set_key_extractor(EncodedMethodNameAndArgsKeyExtractor(skip_first_arg_as_self=False))
+        .set_storage(LocalInMemoryCacheStorage())
+        .set_postprocessing(DeepcopyPostprocessing())
+    )
     async def cached():
         return 'dummy'
 
@@ -232,7 +238,8 @@ Still, you can use default configuration which:
 * uses in-memory storage;
 * uses method instance & arguments to infer cache key;
 * stores up to 4096 elements in cache and evicts entries according to least recently updated policy;
-* refreshes elements after 10 minutes & ignores unrefreshed elements after 30 minutes.
+* refreshes elements after 10 minutes & ignores unrefreshed elements after 30 minutes;
+* does not post-process cached values.
 
 If that satisfies you, just use default config:
 

examples/configuration/custom_configuration.py

@@ -4,18 +4,21 @@
 from memoize.entrybuilder import ProvidedLifeSpanCacheEntryBuilder
 from memoize.eviction import LeastRecentlyUpdatedEvictionStrategy
 from memoize.key import EncodedMethodNameAndArgsKeyExtractor
+from memoize.postprocessing import DeepcopyPostprocessing
 from memoize.storage import LocalInMemoryCacheStorage
 from memoize.wrapper import memoize
 
 
-@memoize(configuration=MutableCacheConfiguration
-         .initialized_with(DefaultInMemoryCacheConfiguration())
-         .set_method_timeout(value=timedelta(minutes=2))
-         .set_entry_builder(ProvidedLifeSpanCacheEntryBuilder(update_after=timedelta(minutes=2),
-                                                              expire_after=timedelta(minutes=5)))
-         .set_eviction_strategy(LeastRecentlyUpdatedEvictionStrategy(capacity=2048))
-         .set_key_extractor(EncodedMethodNameAndArgsKeyExtractor(skip_first_arg_as_self=False))
-         .set_storage(LocalInMemoryCacheStorage())
-         )
+@memoize(
+    configuration=MutableCacheConfiguration
+    .initialized_with(DefaultInMemoryCacheConfiguration())
+    .set_method_timeout(value=timedelta(minutes=2))
+    .set_entry_builder(ProvidedLifeSpanCacheEntryBuilder(update_after=timedelta(minutes=2),
+                                                         expire_after=timedelta(minutes=5)))
+    .set_eviction_strategy(LeastRecentlyUpdatedEvictionStrategy(capacity=2048))
+    .set_key_extractor(EncodedMethodNameAndArgsKeyExtractor(skip_first_arg_as_self=False))
+    .set_storage(LocalInMemoryCacheStorage())
+    .set_postprocessing(DeepcopyPostprocessing())
+)
 async def cached():
     return 'dummy'

memoize/configuration.py

@@ -9,6 +9,7 @@
 from memoize.entrybuilder import CacheEntryBuilder, ProvidedLifeSpanCacheEntryBuilder
 from memoize.eviction import EvictionStrategy, LeastRecentlyUpdatedEvictionStrategy
 from memoize.key import KeyExtractor, EncodedMethodReferenceAndArgsKeyExtractor
+from memoize.postprocessing import Postprocessing, NoPostprocessing
 from memoize.storage import CacheStorage
 from memoize.storage import LocalInMemoryCacheStorage
 
@@ -51,30 +52,40 @@ def eviction_strategy(self) -> EvictionStrategy:
         """ Determines which EvictionStrategy is to be used by cache. """
         raise NotImplementedError()
 
+    @abstractmethod
+    def postprocessing(self) -> Postprocessing:
+        """ Determines which/if Postprocessing is to be used by cache. """
+        raise NotImplementedError()
+
     def __str__(self) -> str:
         return self.__repr__()
 
     def __repr__(self) -> str:
-        return "{name}[configured={configured}, method_timeout={method_timeout}, entry_builder={entry_builder}," \
-               " key_extractor={key_extractor}, storage={storage}, eviction_strategy={eviction_strategy}]" \
-            .format(name=self.__class__, configured=self.configured(), method_timeout=self.method_timeout(),
-                    entry_builder=self.entry_builder(), key_extractor=self.key_extractor(), storage=self.storage(),
-                    eviction_strategy=self.eviction_strategy())
+        return (f"{self.__class__}["
+                f"configured={self.configured()}, "
+                f"method_timeout={self.method_timeout()}, "
+                f"entry_builder={self.entry_builder()}, "
+                f"key_extractor={self.key_extractor()}, "
+                f"storage={self.storage()}, "
+                f"eviction_strategy={self.eviction_strategy()}, "
+                f"postprocessing={self.postprocessing()}"
+                f"]")
 
 
 class MutableCacheConfiguration(CacheConfiguration):
     """ Mutable configuration which can be change at runtime.
     May be also used to customize existing configuration (for example a default one, which is immutable)."""
 
     def __init__(self, configured: bool, storage: CacheStorage, key_extractor: KeyExtractor,
-                 eviction_strategy: EvictionStrategy, entry_builder: CacheEntryBuilder,
+                 eviction_strategy: EvictionStrategy, entry_builder: CacheEntryBuilder, postprocessing: Postprocessing,
                  method_timeout: timedelta) -> None:
         self.__storage = storage
         self.__configured = configured
         self.__key_extractor = key_extractor
         self.__entry_builder = entry_builder
         self.__method_timeout = method_timeout
         self.__eviction_strategy = eviction_strategy
+        self.__postprocessing = postprocessing
 
     @staticmethod
     def initialized_with(configuration: CacheConfiguration) -> 'MutableCacheConfiguration':
@@ -85,6 +96,7 @@ def initialized_with(configuration: CacheConfiguration) -> 'MutableCacheConfigur
             entry_builder=configuration.entry_builder(),
             method_timeout=configuration.method_timeout(),
             eviction_strategy=configuration.eviction_strategy(),
+            postprocessing=configuration.postprocessing(),
         )
 
     def method_timeout(self) -> timedelta:
@@ -105,6 +117,9 @@ def entry_builder(self) -> CacheEntryBuilder:
     def eviction_strategy(self) -> EvictionStrategy:
         return self.__eviction_strategy
 
+    def postprocessing(self) -> Postprocessing:
+        return self.__postprocessing
+
     def set_method_timeout(self, value: timedelta) -> 'MutableCacheConfiguration':
         self.__method_timeout = value
         return self
@@ -129,6 +144,10 @@ def set_eviction_strategy(self, value: EvictionStrategy) -> 'MutableCacheConfigu
         self.__eviction_strategy = value
         return self
 
+    def set_postprocessing(self, value: Postprocessing) -> 'MutableCacheConfiguration':
+        self.__postprocessing = value
+        return self
+
 
 class DefaultInMemoryCacheConfiguration(CacheConfiguration):
     """ Default parameters that describe in-memory cache. Be ware that parameters used do not suit every case. """
@@ -142,6 +161,7 @@ def __init__(self, capacity: int = 4096, method_timeout: timedelta = timedelta(m
         self.__key_extractor = EncodedMethodReferenceAndArgsKeyExtractor()
         self.__eviction_strategy = LeastRecentlyUpdatedEvictionStrategy(capacity=capacity)
         self.__entry_builder = ProvidedLifeSpanCacheEntryBuilder(update_after=update_after, expire_after=expire_after)
+        self.__postprocessing = NoPostprocessing()
 
     def configured(self) -> bool:
         return self.__configured
@@ -160,3 +180,6 @@ def eviction_strategy(self) -> LeastRecentlyUpdatedEvictionStrategy:
 
     def key_extractor(self) -> EncodedMethodReferenceAndArgsKeyExtractor:
         return self.__key_extractor
+
+    def postprocessing(self) -> Postprocessing:
+        return self.__postprocessing

memoize/postprocessing.py

@@ -0,0 +1,30 @@
+import copy
+from abc import ABCMeta, abstractmethod
+from typing import Any
+
+ValueType = Any
+
+
+class Postprocessing(metaclass=ABCMeta):
+    @abstractmethod
+    def apply(self, original: ValueType) -> ValueType:
+        """Transforms value just before returning from the cache."""
+        raise NotImplementedError()
+
+
+class NoPostprocessing(Postprocessing):
+    def apply(self, original: ValueType) -> ValueType:
+        """Applies no postprocessing (returns original value)."""
+        return original
+
+
+class DeepcopyPostprocessing(Postprocessing):
+    def apply(self, original: ValueType) -> ValueType:
+        """
+        Performs deep copy of the value. Useful when you want to prevent modifying the value cached in memory
+        (so callers could modify their copies safely).
+
+        Have in mind that this operation may be expensive,
+        and may not be suitable for all types of values (see docs on copy.deepcopy).
+        """
+        return copy.deepcopy(original)

memoize/wrapper.py

@@ -145,6 +145,6 @@ def value_future_provider():
         else:
             result = current_entry
 
-        return result.value
+        return configuration_snapshot.postprocessing().apply(result.value)
 
     return wrapper

setup.py

@@ -11,7 +11,7 @@ def prepare_description():
 
 setup(
     name='py-memoize',
-    version='2.0.0',
+    version='2.1.0',
     author='Michal Zmuda',
     author_email='[email protected]',
     url='https://github.com/DreamLab/memoize',

tests/unit/test_postprocessing.py

@@ -0,0 +1,60 @@
+from memoize.postprocessing import DeepcopyPostprocessing
+from tests.py310workaround import fix_python_3_10_compatibility
+
+fix_python_3_10_compatibility()
+
+from unittest.mock import Mock
+
+from tornado import gen
+from tornado.testing import AsyncTestCase, gen_test
+
+from memoize.configuration import MutableCacheConfiguration, DefaultInMemoryCacheConfiguration
+from memoize.wrapper import memoize
+
+
+class KeyExtractorInteractionsTests(AsyncTestCase):
+
+    @gen_test
+    def test_postprocessing_is_applied(self):
+        # given
+        postprocessing = Mock()
+        postprocessing.apply = Mock(return_value='overridden-by-postprocessing')
+
+        @memoize(
+            configuration=MutableCacheConfiguration
+            .initialized_with(DefaultInMemoryCacheConfiguration())
+            .set_postprocessing(postprocessing)
+        )
+        @gen.coroutine
+        def sample_method(arg):
+            return f"value-for-{arg}"
+
+        # when
+        result = yield sample_method('test')
+
+        # then
+        postprocessing.apply.assert_called_once()
+        postprocessing.apply.assert_called_once_with('value-for-test')
+        self.assertEqual(result, 'overridden-by-postprocessing')
+
+    @gen_test
+    def test_postprocessing_based_on_deepcopy_prevents_modifying_value_cached_in_memory(self):
+        # given
+
+        @memoize(
+            configuration=MutableCacheConfiguration
+            .initialized_with(DefaultInMemoryCacheConfiguration())
+            .set_postprocessing(DeepcopyPostprocessing())
+        )
+        @gen.coroutine
+        def sample_method(arg):
+            return {'arg': arg, 'list': [4, 5, 1, 2, 3]}  # unsorted
+
+        # when
+        result1 = yield sample_method('test')
+        result2 = yield sample_method('test')
+        result1['list'].sort()
+
+        # then
+        self.assertEqual(result1, {'arg': 'test', 'list': [1, 2, 3, 4, 5]})  # sorted in-place
+        self.assertEqual(result2, {'arg': 'test', 'list': [4, 5, 1, 2, 3]})  # still unsorted