From 6d85c94bf11e580a687aca009aa3a2f9f29776c6 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sun, 30 Jul 2023 14:49:43 +0300 Subject: [PATCH 01/25] add serializer/deserilizer --- .../utilities/idempotency/base.py | 18 +++++++++++++++--- .../utilities/idempotency/idempotency.py | 18 +++++++++++++++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 46aa5ef896..b578360eb1 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -51,6 +51,8 @@ def __init__( function_payload: Any, config: IdempotencyConfig, persistence_store: BasePersistenceLayer, + serializer: Optional[Callable[[Any], Dict]] = None, + deserializer: Optional[Callable[[Dict], Any]] = None, function_args: Optional[Tuple] = None, function_kwargs: Optional[Dict] = None, ): @@ -65,13 +67,20 @@ def __init__( Idempotency Configuration persistence_store : BasePersistenceLayer Instance of persistence layer to store idempotency records + serializer: Optional[Callable[[Any], Dict]] + Custom function to serialize the given object into a dictionary + deserializer: Optional[Callable[[Dict], Any]] + Custom function to deserialize dictionary representation into an object function_args: Optional[Tuple] Function arguments function_kwargs: Optional[Dict] Function keyword arguments + """ self.function = function - self.data = deepcopy(_prepare_data(function_payload)) + self.serializer = serializer or _prepare_data + self.deserializer = deserializer + self.data = deepcopy(self.serializer(function_payload)) self.fn_args = function_args self.fn_kwargs = function_kwargs self.config = config @@ -206,8 +215,11 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] f"Execution already in progress with idempotency key: " f"{self.persistence_store.event_key_jmespath}={data_record.idempotency_key}", ) - - return data_record.response_json_as_dict() + + response_dict: Optional[dict] = data_record.response_json_as_dict() + if response_dict and self.deserializer: + return self.deserializer(response_dict) + return response_dict def _get_function_response(self): try: diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 76d353d205..dcab86d64b 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -26,6 +26,8 @@ def idempotent( context: LambdaContext, persistence_store: BasePersistenceLayer, config: Optional[IdempotencyConfig] = None, + serializer: Optional[Callable[[Any], Dict]] = None, + deserializer: Optional[Callable[[Dict], Any]] = None, **kwargs, ) -> Any: """ @@ -43,7 +45,10 @@ def idempotent( Instance of BasePersistenceLayer to store data config: IdempotencyConfig Configuration - + serializer: Optional[Callable[[Any], Dict]] + Custom function to serialize the given object into a dictionary + deserializer: Optional[Callable[[Dict], Any]] + Custom function to deserialize dictionary representation into an object Examples -------- **Processes Lambda's event in an idempotent manner** @@ -72,6 +77,8 @@ def idempotent( function_payload=event, config=config, persistence_store=persistence_store, + serializer=serializer, + deserializer=deserializer, function_args=args, function_kwargs=kwargs, ) @@ -85,6 +92,9 @@ def idempotent_function( data_keyword_argument: str, persistence_store: BasePersistenceLayer, config: Optional[IdempotencyConfig] = None, + serializer: Optional[Callable[[Any], Dict]] = None, + deserializer: Optional[Callable[[Dict], Any]] = None, + ) -> Any: """ Decorator to handle idempotency of any function @@ -99,6 +109,10 @@ def idempotent_function( Instance of BasePersistenceLayer to store data config: IdempotencyConfig Configuration + serializer: Optional[Callable[[Any], Dict]] + Custom function to serialize the given object into a dictionary + deserializer: Optional[Callable[[Dict], Any]] + Custom function to deserialize dictionary representation into an object Examples -------- @@ -149,6 +163,8 @@ def decorate(*args, **kwargs): persistence_store=persistence_store, function_args=args, function_kwargs=kwargs, + serializer=serializer, + deserializer=deserializer, ) return idempotency_handler.handle() From e142c4bea57abfe8a5067d42c4219fc18c86bf89 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sun, 30 Jul 2023 14:54:22 +0300 Subject: [PATCH 02/25] improve docs --- aws_lambda_powertools/utilities/idempotency/base.py | 2 ++ .../utilities/idempotency/idempotency.py | 8 ++++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index b578360eb1..b5e2857236 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -69,8 +69,10 @@ def __init__( Instance of persistence layer to store idempotency records serializer: Optional[Callable[[Any], Dict]] Custom function to serialize the given object into a dictionary + If not supplied, best effort will be done to serialize known object representations deserializer: Optional[Callable[[Dict], Any]] Custom function to deserialize dictionary representation into an object + If not supplied, a dictionary is returned function_args: Optional[Tuple] Function arguments function_kwargs: Optional[Dict] diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index dcab86d64b..570c1642e9 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -46,9 +46,11 @@ def idempotent( config: IdempotencyConfig Configuration serializer: Optional[Callable[[Any], Dict]] - Custom function to serialize the given object into a dictionary + Custom function to serialize the given object into a dictionary. + If not supplied, best effort will be done to serialize known object representations deserializer: Optional[Callable[[Dict], Any]] - Custom function to deserialize dictionary representation into an object + Custom function to deserialize dictionary representation into an object. + If not supplied, a dictionary is returned Examples -------- **Processes Lambda's event in an idempotent manner** @@ -111,8 +113,10 @@ def idempotent_function( Configuration serializer: Optional[Callable[[Any], Dict]] Custom function to serialize the given object into a dictionary + If not supplied, best effort will be done to serialize known object representations deserializer: Optional[Callable[[Dict], Any]] Custom function to deserialize dictionary representation into an object + If not supplied, a dictionary is returned Examples -------- From 151080b118db363c265e29f73551a742c6ffc093 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sun, 30 Jul 2023 15:51:17 +0300 Subject: [PATCH 03/25] seperate serializers of input and output --- .../utilities/idempotency/base.py | 36 ++++++++------- .../utilities/idempotency/idempotency.py | 44 +++++++++++-------- .../idempotency/test_idempotency.py | 39 ++++++++++++++++ 3 files changed, 86 insertions(+), 33 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index b5e2857236..82b38d8ffd 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -51,8 +51,9 @@ def __init__( function_payload: Any, config: IdempotencyConfig, persistence_store: BasePersistenceLayer, - serializer: Optional[Callable[[Any], Dict]] = None, - deserializer: Optional[Callable[[Dict], Any]] = None, + input_serializer: Optional[Callable[[Any], Dict]] = None, + output_serializer: Optional[Callable[[Any], Dict]] = None, + output_deserializer: Optional[Callable[[Dict], Any]] = None, function_args: Optional[Tuple] = None, function_kwargs: Optional[Dict] = None, ): @@ -67,22 +68,26 @@ def __init__( Idempotency Configuration persistence_store : BasePersistenceLayer Instance of persistence layer to store idempotency records - serializer: Optional[Callable[[Any], Dict]] - Custom function to serialize the given object into a dictionary + input_serializer: Optional[Callable[[Any], Dict]] + Custom function to serialize the given object into a dictionary. If not supplied, best effort will be done to serialize known object representations - deserializer: Optional[Callable[[Dict], Any]] - Custom function to deserialize dictionary representation into an object - If not supplied, a dictionary is returned + output_serializer: Optional[Callable[[Any], Dict]] + Custom function to serialize the returned object into a dictionary. + If not supplied, no serialization is done + output_deserializer: Optional[Callable[[Dict], Any]] + Custom function to deserialize dictionary representation into an object. + If not supplied, no deserialization is done function_args: Optional[Tuple] Function arguments function_kwargs: Optional[Dict] Function keyword arguments - + """ self.function = function - self.serializer = serializer or _prepare_data - self.deserializer = deserializer - self.data = deepcopy(self.serializer(function_payload)) + self.input_serializer = input_serializer or _prepare_data + self.output_serializer = output_serializer + self.output_deserializer = output_deserializer + self.data = deepcopy(self.input_serializer(function_payload)) self.fn_args = function_args self.fn_kwargs = function_kwargs self.config = config @@ -217,10 +222,10 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] f"Execution already in progress with idempotency key: " f"{self.persistence_store.event_key_jmespath}={data_record.idempotency_key}", ) - + response_dict: Optional[dict] = data_record.response_json_as_dict() - if response_dict and self.deserializer: - return self.deserializer(response_dict) + if response_dict and self.output_deserializer: + return self.output_deserializer(response_dict) return response_dict def _get_function_response(self): @@ -240,7 +245,8 @@ def _get_function_response(self): else: try: - self.persistence_store.save_success(data=self.data, result=response) + saved_response: dict = self.output_serializer(response) if self.output_deserializer else response + self.persistence_store.save_success(data=self.data, result=saved_response) except Exception as save_exception: raise IdempotencyPersistenceLayerError( "Failed to update record state to success in idempotency store", diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 570c1642e9..6ba508f73a 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -26,8 +26,9 @@ def idempotent( context: LambdaContext, persistence_store: BasePersistenceLayer, config: Optional[IdempotencyConfig] = None, - serializer: Optional[Callable[[Any], Dict]] = None, - deserializer: Optional[Callable[[Dict], Any]] = None, + input_serializer: Optional[Callable[[Any], Dict]] = None, + output_serializer: Optional[Callable[[Any], Dict]] = None, + output_deserializer: Optional[Callable[[Dict], Any]] = None, **kwargs, ) -> Any: """ @@ -45,12 +46,15 @@ def idempotent( Instance of BasePersistenceLayer to store data config: IdempotencyConfig Configuration - serializer: Optional[Callable[[Any], Dict]] + input_serializer: Optional[Callable[[Any], Dict]] Custom function to serialize the given object into a dictionary. If not supplied, best effort will be done to serialize known object representations - deserializer: Optional[Callable[[Dict], Any]] + output_serializer: Optional[Callable[[Any], Dict]] + Custom function to serialize the returned object into a dictionary. + If not supplied, no serialization is done + output_deserializer: Optional[Callable[[Dict], Any]] Custom function to deserialize dictionary representation into an object. - If not supplied, a dictionary is returned + If not supplied, no deserialization is done Examples -------- **Processes Lambda's event in an idempotent manner** @@ -79,8 +83,9 @@ def idempotent( function_payload=event, config=config, persistence_store=persistence_store, - serializer=serializer, - deserializer=deserializer, + input_serializer=input_serializer, + output_serializer=output_serializer, + output_deserializer=output_deserializer, function_args=args, function_kwargs=kwargs, ) @@ -94,9 +99,9 @@ def idempotent_function( data_keyword_argument: str, persistence_store: BasePersistenceLayer, config: Optional[IdempotencyConfig] = None, - serializer: Optional[Callable[[Any], Dict]] = None, - deserializer: Optional[Callable[[Dict], Any]] = None, - + input_serializer: Optional[Callable[[Any], Dict]] = None, + output_serializer: Optional[Callable[[Any], Dict]] = None, + output_deserializer: Optional[Callable[[Dict], Any]] = None, ) -> Any: """ Decorator to handle idempotency of any function @@ -111,13 +116,15 @@ def idempotent_function( Instance of BasePersistenceLayer to store data config: IdempotencyConfig Configuration - serializer: Optional[Callable[[Any], Dict]] - Custom function to serialize the given object into a dictionary + input_serializer: Optional[Callable[[Any], Dict]] + Custom function to serialize the given object into a dictionary. If not supplied, best effort will be done to serialize known object representations - deserializer: Optional[Callable[[Dict], Any]] - Custom function to deserialize dictionary representation into an object - If not supplied, a dictionary is returned - + output_serializer: Optional[Callable[[Any], Dict]] + Custom function to serialize the returned object into a dictionary. + If not supplied, no serialization is done + output_deserializer: Optional[Callable[[Dict], Any]] + Custom function to deserialize dictionary representation into an object. + If not supplied, no deserialization is done Examples -------- **Processes an order in an idempotent manner** @@ -165,10 +172,11 @@ def decorate(*args, **kwargs): function_payload=payload, config=config, persistence_store=persistence_store, + input_serializer=input_serializer, + output_serializer=output_serializer, + output_deserializer=output_deserializer, function_args=args, function_kwargs=kwargs, - serializer=serializer, - deserializer=deserializer, ) return idempotency_handler.handle() diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 47b2474466..11a5d8cf76 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -1196,6 +1196,45 @@ def record_handler(record): assert result == expected_result +@pytest.mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or higher for dataclasses") +def test_idempotent_function_serialization_pydantic(): + # GIVEN + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class PaymentInput(BaseModel): + customer_id: str + transaction_id: str + + class PaymentOutput(BaseModel): + customer_id: str + transaction_id: str + + @idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + input_serializer=lambda x: x.dict(), + output_serializer=lambda x: x.dict(), + output_deserializer=PaymentOutput.parse_obj, + ) + def collect_payment(payment: PaymentInput) -> PaymentOutput: + return PaymentOutput.parse_obj(payment) + + # WHEN + payment = PaymentInput(**mock_event) + first_call: PaymentOutput = collect_payment(payment=payment) + assert first_call.customer_id == payment.customer_id + assert first_call.transaction_id == payment.transaction_id + assert isinstance(first_call, PaymentOutput) + second_call: PaymentOutput = collect_payment(payment=payment) + assert isinstance(second_call, PaymentOutput) + assert second_call.customer_id == payment.customer_id + assert second_call.transaction_id == payment.transaction_id + + def test_idempotent_function_arbitrary_args_kwargs(): # Scenario to validate we can use idempotent_function with a function # with an arbitrary number of args and kwargs From 97742489bbe370fbb1b362e082dccdce950e6b22 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Fri, 4 Aug 2023 10:48:46 +0300 Subject: [PATCH 04/25] feat(idempotency): custom serialization - add a helper serailizer class, in order to allow refactoring later on in regards to string seriliztion --- .../utilities/idempotency/__init__.py | 12 ++++- .../utilities/idempotency/base.py | 28 ++++------ .../utilities/idempotency/idempotency.py | 34 ++---------- .../idempotency/serialization/__init__.py | 0 .../idempotency/serialization/base.py | 22 ++++++++ .../idempotency/serialization/custom_dict.py | 15 ++++++ .../idempotency/serialization/pydantic.py | 24 +++++++++ .../idempotency/test_idempotency.py | 54 ++++++++++++++++--- 8 files changed, 133 insertions(+), 56 deletions(-) create mode 100644 aws_lambda_powertools/utilities/idempotency/serialization/__init__.py create mode 100644 aws_lambda_powertools/utilities/idempotency/serialization/base.py create mode 100644 aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py create mode 100644 aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py diff --git a/aws_lambda_powertools/utilities/idempotency/__init__.py b/aws_lambda_powertools/utilities/idempotency/__init__.py index 148b291ea6..20a23c5eb5 100644 --- a/aws_lambda_powertools/utilities/idempotency/__init__.py +++ b/aws_lambda_powertools/utilities/idempotency/__init__.py @@ -8,7 +8,17 @@ from aws_lambda_powertools.utilities.idempotency.persistence.dynamodb import ( DynamoDBPersistenceLayer, ) +from aws_lambda_powertools.utilities.idempotency.serialization.custom_dict import CustomDictSerializer +from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import PydanticSerializer from .idempotency import IdempotencyConfig, idempotent, idempotent_function -__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent", "idempotent_function", "IdempotencyConfig") +__all__ = ( + "DynamoDBPersistenceLayer", + "BasePersistenceLayer", + "idempotent", + "idempotent_function", + "IdempotencyConfig", + "PydanticSerializer", + "CustomDictSerializer", +) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 82b38d8ffd..be0f6a01b2 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -18,6 +18,9 @@ BasePersistenceLayer, DataRecord, ) +from aws_lambda_powertools.utilities.idempotency.serialization.base import ( + BaseDictSerializer, +) MAX_RETRIES = 2 logger = logging.getLogger(__name__) @@ -51,9 +54,7 @@ def __init__( function_payload: Any, config: IdempotencyConfig, persistence_store: BasePersistenceLayer, - input_serializer: Optional[Callable[[Any], Dict]] = None, - output_serializer: Optional[Callable[[Any], Dict]] = None, - output_deserializer: Optional[Callable[[Dict], Any]] = None, + output_serializer: Optional[BaseDictSerializer] = None, function_args: Optional[Tuple] = None, function_kwargs: Optional[Dict] = None, ): @@ -67,16 +68,9 @@ def __init__( config: IdempotencyConfig Idempotency Configuration persistence_store : BasePersistenceLayer - Instance of persistence layer to store idempotency records - input_serializer: Optional[Callable[[Any], Dict]] - Custom function to serialize the given object into a dictionary. - If not supplied, best effort will be done to serialize known object representations - output_serializer: Optional[Callable[[Any], Dict]] - Custom function to serialize the returned object into a dictionary. + output_serializer: Optional[BaseDictSerializer] + Serializer to transform the data to and from a dictionary. If not supplied, no serialization is done - output_deserializer: Optional[Callable[[Dict], Any]] - Custom function to deserialize dictionary representation into an object. - If not supplied, no deserialization is done function_args: Optional[Tuple] Function arguments function_kwargs: Optional[Dict] @@ -84,10 +78,8 @@ def __init__( """ self.function = function - self.input_serializer = input_serializer or _prepare_data self.output_serializer = output_serializer - self.output_deserializer = output_deserializer - self.data = deepcopy(self.input_serializer(function_payload)) + self.data = deepcopy(_prepare_data(function_payload)) self.fn_args = function_args self.fn_kwargs = function_kwargs self.config = config @@ -224,8 +216,8 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] ) response_dict: Optional[dict] = data_record.response_json_as_dict() - if response_dict and self.output_deserializer: - return self.output_deserializer(response_dict) + if response_dict and self.output_serializer: + return self.output_serializer.from_dict(response_dict) return response_dict def _get_function_response(self): @@ -245,7 +237,7 @@ def _get_function_response(self): else: try: - saved_response: dict = self.output_serializer(response) if self.output_deserializer else response + saved_response: dict = self.output_serializer.to_dict(response) if self.output_serializer else response self.persistence_store.save_success(data=self.data, result=saved_response) except Exception as save_exception: raise IdempotencyPersistenceLayerError( diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 6ba508f73a..6f6584a10f 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -14,6 +14,7 @@ from aws_lambda_powertools.utilities.idempotency.persistence.base import ( BasePersistenceLayer, ) +from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseDictSerializer from aws_lambda_powertools.utilities.typing import LambdaContext logger = logging.getLogger(__name__) @@ -26,9 +27,6 @@ def idempotent( context: LambdaContext, persistence_store: BasePersistenceLayer, config: Optional[IdempotencyConfig] = None, - input_serializer: Optional[Callable[[Any], Dict]] = None, - output_serializer: Optional[Callable[[Any], Dict]] = None, - output_deserializer: Optional[Callable[[Dict], Any]] = None, **kwargs, ) -> Any: """ @@ -46,15 +44,6 @@ def idempotent( Instance of BasePersistenceLayer to store data config: IdempotencyConfig Configuration - input_serializer: Optional[Callable[[Any], Dict]] - Custom function to serialize the given object into a dictionary. - If not supplied, best effort will be done to serialize known object representations - output_serializer: Optional[Callable[[Any], Dict]] - Custom function to serialize the returned object into a dictionary. - If not supplied, no serialization is done - output_deserializer: Optional[Callable[[Dict], Any]] - Custom function to deserialize dictionary representation into an object. - If not supplied, no deserialization is done Examples -------- **Processes Lambda's event in an idempotent manner** @@ -83,9 +72,6 @@ def idempotent( function_payload=event, config=config, persistence_store=persistence_store, - input_serializer=input_serializer, - output_serializer=output_serializer, - output_deserializer=output_deserializer, function_args=args, function_kwargs=kwargs, ) @@ -99,9 +85,7 @@ def idempotent_function( data_keyword_argument: str, persistence_store: BasePersistenceLayer, config: Optional[IdempotencyConfig] = None, - input_serializer: Optional[Callable[[Any], Dict]] = None, - output_serializer: Optional[Callable[[Any], Dict]] = None, - output_deserializer: Optional[Callable[[Dict], Any]] = None, + output_serializer: Optional[BaseDictSerializer] = None, ) -> Any: """ Decorator to handle idempotency of any function @@ -116,15 +100,9 @@ def idempotent_function( Instance of BasePersistenceLayer to store data config: IdempotencyConfig Configuration - input_serializer: Optional[Callable[[Any], Dict]] - Custom function to serialize the given object into a dictionary. - If not supplied, best effort will be done to serialize known object representations - output_serializer: Optional[Callable[[Any], Dict]] - Custom function to serialize the returned object into a dictionary. - If not supplied, no serialization is done - output_deserializer: Optional[Callable[[Dict], Any]] - Custom function to deserialize dictionary representation into an object. - If not supplied, no deserialization is done + output_serializer: Optional[BaseDictSerializer] + Serializer to transform the data to and from a dictionary. + If not supplied, no serialization is done Examples -------- **Processes an order in an idempotent manner** @@ -172,9 +150,7 @@ def decorate(*args, **kwargs): function_payload=payload, config=config, persistence_store=persistence_store, - input_serializer=input_serializer, output_serializer=output_serializer, - output_deserializer=output_deserializer, function_args=args, function_kwargs=kwargs, ) diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/__init__.py b/aws_lambda_powertools/utilities/idempotency/serialization/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/base.py b/aws_lambda_powertools/utilities/idempotency/serialization/base.py new file mode 100644 index 0000000000..4b3de88893 --- /dev/null +++ b/aws_lambda_powertools/utilities/idempotency/serialization/base.py @@ -0,0 +1,22 @@ +""" +Serialization for supporting idempotency +""" +import logging +from abc import ABC, abstractmethod +from typing import Any, Dict + +logger = logging.getLogger(__name__) + + +class BaseDictSerializer(ABC): + """ + Abstract Base Class for Idempotency serialization layer, supporting dict operations. + """ + + @abstractmethod + def to_dict(self, data: Any) -> Dict: + pass + + @abstractmethod + def from_dict(self, data: Dict) -> Any: + pass diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py b/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py new file mode 100644 index 0000000000..4e8d6edd86 --- /dev/null +++ b/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py @@ -0,0 +1,15 @@ +from typing import Any, Callable, Dict + +from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseDictSerializer + + +class CustomDictSerializer(BaseDictSerializer): + def __init__(self, to_dict: Callable[[Any], Dict], from_dict: Callable[[Dict], Any]): + self.__to_dict: Callable[[Any], Dict] = to_dict + self.__from_dict: Callable[[Dict], Any] = from_dict + + def to_dict(self, data: Any) -> Dict: + return self.__to_dict(data) + + def from_dict(self, data: Dict) -> Any: + return self.__from_dict(data) diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py new file mode 100644 index 0000000000..df752d6f5e --- /dev/null +++ b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py @@ -0,0 +1,24 @@ +from typing import Dict, TypeVar + +from pydantic import BaseModel + +from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseDictSerializer + +Model = TypeVar("Model", bound=BaseModel) + + +class PydanticSerializer(BaseDictSerializer): + def __init__(self, model: Model): + self.__model: Model = model + + def to_dict(self, data: Model) -> Dict: + if callable(getattr(data, "model_dump", None)): + # Support for pydantic V2 + return data.model_dump() + return data.dict() + + def from_dict(self, data: Dict) -> Model: + if callable(getattr(self.__model, "model_validate", None)): + # Support for pydantic V2 + return self.__model.model_validate(data) + return self.__model.parse_obj(data) diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 11a5d8cf76..53601bf6de 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -15,8 +15,12 @@ event_source, ) from aws_lambda_powertools.utilities.idempotency import ( + CustomDictSerializer, DynamoDBPersistenceLayer, IdempotencyConfig, + PydanticSerializer, + idempotent, + idempotent_function, ) from aws_lambda_powertools.utilities.idempotency.base import ( MAX_RETRIES, @@ -31,10 +35,6 @@ IdempotencyPersistenceLayerError, IdempotencyValidationError, ) -from aws_lambda_powertools.utilities.idempotency.idempotency import ( - idempotent, - idempotent_function, -) from aws_lambda_powertools.utilities.idempotency.persistence.base import ( BasePersistenceLayer, DataRecord, @@ -1196,7 +1196,45 @@ def record_handler(record): assert result == expected_result -@pytest.mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or higher for dataclasses") +def test_idempotent_function_serialization_custom_dict(): + # GIVEN + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class PaymentInput(BaseModel): + customer_id: str + transaction_id: str + + class PaymentOutput(BaseModel): + customer_id: str + transaction_id: str + + @idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=CustomDictSerializer( + to_dict=lambda x: x.dict(), + from_dict=PaymentOutput.parse_obj, + ), + ) + def collect_payment(payment: PaymentInput) -> PaymentOutput: + return PaymentOutput.parse_obj(payment) + + # WHEN + payment = PaymentInput(**mock_event) + first_call: PaymentOutput = collect_payment(payment=payment) + assert first_call.customer_id == payment.customer_id + assert first_call.transaction_id == payment.transaction_id + assert isinstance(first_call, PaymentOutput) + second_call: PaymentOutput = collect_payment(payment=payment) + assert isinstance(second_call, PaymentOutput) + assert second_call.customer_id == payment.customer_id + assert second_call.transaction_id == payment.transaction_id + + def test_idempotent_function_serialization_pydantic(): # GIVEN config = IdempotencyConfig(use_local_cache=True) @@ -1216,9 +1254,9 @@ class PaymentOutput(BaseModel): data_keyword_argument="payment", persistence_store=persistence_layer, config=config, - input_serializer=lambda x: x.dict(), - output_serializer=lambda x: x.dict(), - output_deserializer=PaymentOutput.parse_obj, + output_serializer=PydanticSerializer( + model=PaymentOutput, + ), ) def collect_payment(payment: PaymentInput) -> PaymentOutput: return PaymentOutput.parse_obj(payment) From 641905c9d2e0ebb43b2be585bcfcfcc19419e368 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sat, 12 Aug 2023 18:28:01 +0300 Subject: [PATCH 05/25] fix tests --- .../utilities/idempotency/base.py | 1 - .../utilities/idempotency/idempotency.py | 2 +- .../idempotency/test_idempotency.py | 50 ++++++++++--------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index be0f6a01b2..5d12fa42c6 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -214,7 +214,6 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] f"Execution already in progress with idempotency key: " f"{self.persistence_store.event_key_jmespath}={data_record.idempotency_key}", ) - response_dict: Optional[dict] = data_record.response_json_as_dict() if response_dict and self.output_serializer: return self.output_serializer.from_dict(response_dict) diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 6f6584a10f..40f5de223e 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -127,6 +127,7 @@ def process_order(customer_id: str, order: dict, **kwargs): data_keyword_argument=data_keyword_argument, persistence_store=persistence_store, config=config, + output_serializer=output_serializer, ), ) @@ -144,7 +145,6 @@ def decorate(*args, **kwargs): ) payload = kwargs.get(data_keyword_argument) - idempotency_handler = IdempotencyHandler( function=function, function_payload=payload, diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 53601bf6de..fe4a242e02 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -1200,39 +1200,41 @@ def test_idempotent_function_serialization_custom_dict(): # GIVEN config = IdempotencyConfig(use_local_cache=True) mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} - idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_custom_dict..record_handler#{hash_idempotency_key(mock_event)}" # noqa E501 persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) - class PaymentInput(BaseModel): - customer_id: str - transaction_id: str + to_dict_called = False + from_dict_called = False - class PaymentOutput(BaseModel): - customer_id: str - transaction_id: str + def to_dict(data): + nonlocal to_dict_called + to_dict_called = True + return data + + def from_dict(data): + nonlocal from_dict_called + from_dict_called = True + return data + + expected_result = {"message": "Foo"} + output_serializer = CustomDictSerializer( + to_dict=to_dict, + from_dict=from_dict, + ) @idempotent_function( - data_keyword_argument="payment", persistence_store=persistence_layer, + data_keyword_argument="record", config=config, - output_serializer=CustomDictSerializer( - to_dict=lambda x: x.dict(), - from_dict=PaymentOutput.parse_obj, - ), + output_serializer=output_serializer, ) - def collect_payment(payment: PaymentInput) -> PaymentOutput: - return PaymentOutput.parse_obj(payment) + def record_handler(record): + return expected_result - # WHEN - payment = PaymentInput(**mock_event) - first_call: PaymentOutput = collect_payment(payment=payment) - assert first_call.customer_id == payment.customer_id - assert first_call.transaction_id == payment.transaction_id - assert isinstance(first_call, PaymentOutput) - second_call: PaymentOutput = collect_payment(payment=payment) - assert isinstance(second_call, PaymentOutput) - assert second_call.customer_id == payment.customer_id - assert second_call.transaction_id == payment.transaction_id + record_handler(record=mock_event) + assert to_dict_called + record_handler(record=mock_event) + assert from_dict_called def test_idempotent_function_serialization_pydantic(): From f1daf0723caf98ab2059f2df5f151946fbe0503b Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sun, 13 Aug 2023 10:01:37 +0300 Subject: [PATCH 06/25] idemoponcy changes --- .../idempotency/serialization/base.py | 1 + .../idempotency/serialization/custom_dict.py | 8 +++ .../idempotency/serialization/pydantic.py | 18 ++++--- docs/utilities/idempotency.md | 16 ++++++ ...nt_function_dataclass_output_serializer.py | 54 +++++++++++++++++++ ...ent_function_pydantic_output_serializer.py | 44 +++++++++++++++ .../idempotency/test_idempotency.py | 2 +- 7 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py create mode 100644 examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/base.py b/aws_lambda_powertools/utilities/idempotency/serialization/base.py index 4b3de88893..cb37eb1f34 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/base.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/base.py @@ -11,6 +11,7 @@ class BaseDictSerializer(ABC): """ Abstract Base Class for Idempotency serialization layer, supporting dict operations. + This interface not be inherited by end user implementation. """ @abstractmethod diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py b/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py index 4e8d6edd86..875e20dcfb 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py @@ -5,6 +5,14 @@ class CustomDictSerializer(BaseDictSerializer): def __init__(self, to_dict: Callable[[Any], Dict], from_dict: Callable[[Dict], Any]): + """ + Parameters + ---------- + to_dict: Callable[[Any], Dict] + A function capable of transforming the saved data object representation into a dictionary + from_dict: str + A function capable of transforming the saved dictionary into the original data object representation + """ self.__to_dict: Callable[[Any], Dict] = to_dict self.__from_dict: Callable[[Dict], Any] = from_dict diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py index df752d6f5e..21aeaa888b 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py @@ -1,24 +1,30 @@ -from typing import Dict, TypeVar +from typing import Dict, Type from pydantic import BaseModel from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseDictSerializer -Model = TypeVar("Model", bound=BaseModel) +Model = BaseModel class PydanticSerializer(BaseDictSerializer): - def __init__(self, model: Model): - self.__model: Model = model + def __init__(self, model: Type[Model]): + """ + Parameters + ---------- + model: Model + A model of the type to transform + """ + self.__model: Type[Model] = model def to_dict(self, data: Model) -> Dict: if callable(getattr(data, "model_dump", None)): # Support for pydantic V2 - return data.model_dump() + return data.model_dump() # type: ignore[unused-ignore,attr-defined] return data.dict() def from_dict(self, data: Dict) -> Model: if callable(getattr(self.__model, "model_validate", None)): # Support for pydantic V2 - return self.__model.model_validate(data) + return self.__model.model_validate(data) # type: ignore[unused-ignore,attr-defined] return self.__model.parse_obj(data) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 1b2128cd7f..035d9e09cf 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -152,6 +152,22 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo --8<-- "examples/idempotency/src/working_with_idempotent_function_pydantic.py" ``` +#### Output Serialization + +By supplying an output serializer, you can control the return type of the function, allowing cleaner integration with the rest of your code base. + +=== "Using A Custom Type (Dataclasses)" + + ```python hl_lines="3-8 26-28 32-35 42 48" + --8<-- "examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py" + ``` + +=== "Using Pydantic" + + ```python hl_lines="3-8 23-24 32 35" + --8<-- "examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py" + ``` + #### Batch integration You can can easily integrate with [Batch utility](batch.md){target="_blank"} via context manager. This ensures that you process each record in an idempotent manner, and guard against a [Lambda timeout](#lambda-timeouts) idempotent situation. diff --git a/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py b/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py new file mode 100644 index 0000000000..13fdc793cd --- /dev/null +++ b/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py @@ -0,0 +1,54 @@ +from dataclasses import dataclass + +from aws_lambda_powertools.utilities.idempotency import ( + CustomDictSerializer, + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig(event_key_jmespath="order_id") # see Choosing a payload subset section + + +@dataclass +class OrderItem: + sku: str + description: str + + +@dataclass +class Order: + item: OrderItem + order_id: int + + +@dataclass +class OrderOutput: + order_id: int + + +order_output_serializer: CustomDictSerializer = CustomDictSerializer( + to_dict=lambda x: x.asdict(), + from_dict=lambda x: OrderOutput(**x), +) + + +@idempotent_function( + data_keyword_argument="order", + config=config, + persistence_store=dynamodb, + output_serializer=order_output_serializer, +) +def process_order(order: Order) -> OrderOutput: + return OrderOutput(order_id=order.order_id) + + +def lambda_handler(event: dict, context: LambdaContext): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id=1) + + # `order` parameter must be called as a keyword argument to work + process_order(order=order) diff --git a/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py b/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py new file mode 100644 index 0000000000..2b56c0e22e --- /dev/null +++ b/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py @@ -0,0 +1,44 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + PydanticSerializer, + idempotent_function, +) +from aws_lambda_powertools.utilities.parser import BaseModel +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig(event_key_jmespath="order_id") # see Choosing a payload subset section + + +class OrderItem(BaseModel): + sku: str + description: str + + +class Order(BaseModel): + item: OrderItem + order_id: int + + +class OrderOutput(BaseModel): + order_id: int + + +@idempotent_function( + data_keyword_argument="order", + config=config, + persistence_store=dynamodb, + output_serializer=PydanticSerializer(model=OrderOutput), +) +def process_order(order: Order) -> OrderOutput: + return OrderOutput(order_id=order.order_id) + + +def lambda_handler(event: dict, context: LambdaContext): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id=1) + + # `order` parameter must be called as a keyword argument to work + process_order(order=order) diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index fe4a242e02..0491f2e693 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -1261,7 +1261,7 @@ class PaymentOutput(BaseModel): ), ) def collect_payment(payment: PaymentInput) -> PaymentOutput: - return PaymentOutput.parse_obj(payment) + return PaymentOutput(**payment.dict()) # WHEN payment = PaymentInput(**mock_event) From 008aecfa2fb94a4d4fa25239c81d58638ebcb82f Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Fri, 18 Aug 2023 16:02:34 +0300 Subject: [PATCH 07/25] * Add NoOpSerializer instead of checking for None * remove import of pydantic in order to not enforce pydantic existence * fix examples * fix bad docs --- .../utilities/idempotency/__init__.py | 4 ---- .../utilities/idempotency/base.py | 14 ++++++++------ .../utilities/idempotency/idempotency.py | 2 ++ .../idempotency/serialization/no_op.py | 18 ++++++++++++++++++ ...ent_function_dataclass_output_serializer.py | 2 +- ...tent_function_pydantic_output_serializer.py | 2 +- .../functional/idempotency/test_idempotency.py | 4 ++-- 7 files changed, 32 insertions(+), 14 deletions(-) create mode 100644 aws_lambda_powertools/utilities/idempotency/serialization/no_op.py diff --git a/aws_lambda_powertools/utilities/idempotency/__init__.py b/aws_lambda_powertools/utilities/idempotency/__init__.py index 20a23c5eb5..ae27330cc1 100644 --- a/aws_lambda_powertools/utilities/idempotency/__init__.py +++ b/aws_lambda_powertools/utilities/idempotency/__init__.py @@ -8,8 +8,6 @@ from aws_lambda_powertools.utilities.idempotency.persistence.dynamodb import ( DynamoDBPersistenceLayer, ) -from aws_lambda_powertools.utilities.idempotency.serialization.custom_dict import CustomDictSerializer -from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import PydanticSerializer from .idempotency import IdempotencyConfig, idempotent, idempotent_function @@ -19,6 +17,4 @@ "idempotent", "idempotent_function", "IdempotencyConfig", - "PydanticSerializer", - "CustomDictSerializer", ) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 5d12fa42c6..1cb586666d 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -21,6 +21,9 @@ from aws_lambda_powertools.utilities.idempotency.serialization.base import ( BaseDictSerializer, ) +from aws_lambda_powertools.utilities.idempotency.serialization.no_op import ( + NoOpSerializer, +) MAX_RETRIES = 2 logger = logging.getLogger(__name__) @@ -68,6 +71,7 @@ def __init__( config: IdempotencyConfig Idempotency Configuration persistence_store : BasePersistenceLayer + Instance of persistence layer to store idempotency records output_serializer: Optional[BaseDictSerializer] Serializer to transform the data to and from a dictionary. If not supplied, no serialization is done @@ -78,7 +82,7 @@ def __init__( """ self.function = function - self.output_serializer = output_serializer + self.output_serializer = output_serializer or NoOpSerializer() self.data = deepcopy(_prepare_data(function_payload)) self.fn_args = function_args self.fn_kwargs = function_kwargs @@ -215,9 +219,7 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] f"{self.persistence_store.event_key_jmespath}={data_record.idempotency_key}", ) response_dict: Optional[dict] = data_record.response_json_as_dict() - if response_dict and self.output_serializer: - return self.output_serializer.from_dict(response_dict) - return response_dict + return self.output_serializer.from_dict(response_dict) def _get_function_response(self): try: @@ -236,8 +238,8 @@ def _get_function_response(self): else: try: - saved_response: dict = self.output_serializer.to_dict(response) if self.output_serializer else response - self.persistence_store.save_success(data=self.data, result=saved_response) + serialized_response: dict = self.output_serializer.to_dict(response) + self.persistence_store.save_success(data=self.data, result=serialized_response) except Exception as save_exception: raise IdempotencyPersistenceLayerError( "Failed to update record state to success in idempotency store", diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 40f5de223e..b390d543aa 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -44,6 +44,7 @@ def idempotent( Instance of BasePersistenceLayer to store data config: IdempotencyConfig Configuration + Examples -------- **Processes Lambda's event in an idempotent manner** @@ -145,6 +146,7 @@ def decorate(*args, **kwargs): ) payload = kwargs.get(data_keyword_argument) + idempotency_handler = IdempotencyHandler( function=function, function_payload=payload, diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/no_op.py b/aws_lambda_powertools/utilities/idempotency/serialization/no_op.py new file mode 100644 index 0000000000..d48369116b --- /dev/null +++ b/aws_lambda_powertools/utilities/idempotency/serialization/no_op.py @@ -0,0 +1,18 @@ +from typing import Dict + +from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseDictSerializer + + +class NoOpSerializer(BaseDictSerializer): + def __init__(self): + """ + Parameters + ---------- + default serializer, does not transform data + """ + + def to_dict(self, data: Dict) -> Dict: + return data + + def from_dict(self, data: Dict) -> Dict: + return data diff --git a/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py b/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py index 13fdc793cd..193de89cb9 100644 --- a/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py +++ b/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py @@ -1,11 +1,11 @@ from dataclasses import dataclass from aws_lambda_powertools.utilities.idempotency import ( - CustomDictSerializer, DynamoDBPersistenceLayer, IdempotencyConfig, idempotent_function, ) +from aws_lambda_powertools.utilities.idempotency.serialization.custom_dict import CustomDictSerializer from aws_lambda_powertools.utilities.typing import LambdaContext dynamodb = DynamoDBPersistenceLayer(table_name="IdempotencyTable") diff --git a/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py b/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py index 2b56c0e22e..46adbe283d 100644 --- a/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py +++ b/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py @@ -1,9 +1,9 @@ from aws_lambda_powertools.utilities.idempotency import ( DynamoDBPersistenceLayer, IdempotencyConfig, - PydanticSerializer, idempotent_function, ) +from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import PydanticSerializer from aws_lambda_powertools.utilities.parser import BaseModel from aws_lambda_powertools.utilities.typing import LambdaContext diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 0491f2e693..13d0700e4b 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -15,10 +15,8 @@ event_source, ) from aws_lambda_powertools.utilities.idempotency import ( - CustomDictSerializer, DynamoDBPersistenceLayer, IdempotencyConfig, - PydanticSerializer, idempotent, idempotent_function, ) @@ -39,6 +37,8 @@ BasePersistenceLayer, DataRecord, ) +from aws_lambda_powertools.utilities.idempotency.serialization.custom_dict import CustomDictSerializer +from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import PydanticSerializer from aws_lambda_powertools.utilities.validation import envelopes, validator from tests.functional.idempotency.utils import ( build_idempotency_put_item_stub, From 1b39f319de3d94fda362aca53bad15c8a3c51ada Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sun, 20 Aug 2023 11:40:46 +0300 Subject: [PATCH 08/25] make pr --- .../utilities/idempotency/base.py | 9 ++-- .../idempotency/test_idempotency.py | 41 +++++++++++++++++++ 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 1cb586666d..e6555e4d96 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -182,7 +182,7 @@ def _get_idempotency_record(self) -> Optional[DataRecord]: return data_record - def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any]]: + def _handle_for_status(self, data_record: DataRecord) -> Optional[Any]: """ Take appropriate action based on data_record's status @@ -192,8 +192,9 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] Returns ------- - Optional[Dict[Any, Any] + Optional[Any] Function's response previously used for this idempotency key, if it has successfully executed already. + In case an output serializer is configured, the response is deserialized. Raises ------ @@ -219,7 +220,7 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Dict[Any, Any] f"{self.persistence_store.event_key_jmespath}={data_record.idempotency_key}", ) response_dict: Optional[dict] = data_record.response_json_as_dict() - return self.output_serializer.from_dict(response_dict) + return self.output_serializer.from_dict(response_dict) if response_dict else None def _get_function_response(self): try: @@ -238,7 +239,7 @@ def _get_function_response(self): else: try: - serialized_response: dict = self.output_serializer.to_dict(response) + serialized_response: dict = self.output_serializer.to_dict(response) if response else None self.persistence_store.save_success(data=self.data, result=serialized_response) except Exception as save_exception: raise IdempotencyPersistenceLayerError( diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 13d0700e4b..808b2ac55f 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -1237,6 +1237,47 @@ def record_handler(record): assert from_dict_called +def test_idempotent_function_serialization_no_response(): + # GIVEN + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_no_response..record_handler#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + to_dict_called = False + from_dict_called = False + + def to_dict(data): + nonlocal to_dict_called + to_dict_called = True + return data + + def from_dict(data): + nonlocal from_dict_called + from_dict_called = True + return data + + output_serializer = CustomDictSerializer( + to_dict=to_dict, + from_dict=from_dict, + ) + + @idempotent_function( + persistence_store=persistence_layer, + data_keyword_argument="record", + config=config, + output_serializer=output_serializer, + ) + def record_handler(record): + return None + + record_handler(record=mock_event) + assert to_dict_called is False, "in case response is None, to_dict should not be called" + response = record_handler(record=mock_event) + assert response is None + assert from_dict_called is False, "in case response is None, from_dict should not be called" + + def test_idempotent_function_serialization_pydantic(): # GIVEN config = IdempotencyConfig(use_local_cache=True) From bf25130b249c30707978772f3b0f2ec667a52b76 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Mon, 21 Aug 2023 17:16:36 +0300 Subject: [PATCH 09/25] update function description --- aws_lambda_powertools/utilities/idempotency/base.py | 2 +- aws_lambda_powertools/utilities/idempotency/idempotency.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index e6555e4d96..6a8a9e7180 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -74,7 +74,7 @@ def __init__( Instance of persistence layer to store idempotency records output_serializer: Optional[BaseDictSerializer] Serializer to transform the data to and from a dictionary. - If not supplied, no serialization is done + If not supplied, no serialization is done via the NoOpSerializer function_args: Optional[Tuple] Function arguments function_kwargs: Optional[Dict] diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index b390d543aa..8cb376fab1 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -103,7 +103,7 @@ def idempotent_function( Configuration output_serializer: Optional[BaseDictSerializer] Serializer to transform the data to and from a dictionary. - If not supplied, no serialization is done + If not supplied, no serialization is done via the NoOpSerializer Examples -------- **Processes an order in an idempotent manner** From baf49bc6bad2d5a7ca1f15769f4b5d4a3dc4b74e Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Fri, 25 Aug 2023 15:30:16 +0300 Subject: [PATCH 10/25] Cr fixes --- aws_lambda_powertools/utilities/idempotency/base.py | 6 +++--- aws_lambda_powertools/utilities/idempotency/idempotency.py | 6 +++--- .../utilities/idempotency/serialization/base.py | 6 +----- .../utilities/idempotency/serialization/custom_dict.py | 6 +++--- .../utilities/idempotency/serialization/no_op.py | 6 +++--- .../utilities/idempotency/serialization/pydantic.py | 6 +++--- 6 files changed, 16 insertions(+), 20 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 6a8a9e7180..29aeb91383 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -19,7 +19,7 @@ DataRecord, ) from aws_lambda_powertools.utilities.idempotency.serialization.base import ( - BaseDictSerializer, + BaseIdempotencySerializer, ) from aws_lambda_powertools.utilities.idempotency.serialization.no_op import ( NoOpSerializer, @@ -57,7 +57,7 @@ def __init__( function_payload: Any, config: IdempotencyConfig, persistence_store: BasePersistenceLayer, - output_serializer: Optional[BaseDictSerializer] = None, + output_serializer: Optional[BaseIdempotencySerializer] = None, function_args: Optional[Tuple] = None, function_kwargs: Optional[Dict] = None, ): @@ -72,7 +72,7 @@ def __init__( Idempotency Configuration persistence_store : BasePersistenceLayer Instance of persistence layer to store idempotency records - output_serializer: Optional[BaseDictSerializer] + output_serializer: Optional[BaseIdempotencySerializer] Serializer to transform the data to and from a dictionary. If not supplied, no serialization is done via the NoOpSerializer function_args: Optional[Tuple] diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 8cb376fab1..2c61248433 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -14,7 +14,7 @@ from aws_lambda_powertools.utilities.idempotency.persistence.base import ( BasePersistenceLayer, ) -from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseDictSerializer +from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseIdempotencySerializer from aws_lambda_powertools.utilities.typing import LambdaContext logger = logging.getLogger(__name__) @@ -86,7 +86,7 @@ def idempotent_function( data_keyword_argument: str, persistence_store: BasePersistenceLayer, config: Optional[IdempotencyConfig] = None, - output_serializer: Optional[BaseDictSerializer] = None, + output_serializer: Optional[BaseIdempotencySerializer] = None, ) -> Any: """ Decorator to handle idempotency of any function @@ -101,7 +101,7 @@ def idempotent_function( Instance of BasePersistenceLayer to store data config: IdempotencyConfig Configuration - output_serializer: Optional[BaseDictSerializer] + output_serializer: Optional[BaseIdempotencySerializer] Serializer to transform the data to and from a dictionary. If not supplied, no serialization is done via the NoOpSerializer Examples diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/base.py b/aws_lambda_powertools/utilities/idempotency/serialization/base.py index cb37eb1f34..de6e7c1fbb 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/base.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/base.py @@ -1,17 +1,13 @@ """ Serialization for supporting idempotency """ -import logging from abc import ABC, abstractmethod from typing import Any, Dict -logger = logging.getLogger(__name__) - -class BaseDictSerializer(ABC): +class BaseIdempotencySerializer(ABC): """ Abstract Base Class for Idempotency serialization layer, supporting dict operations. - This interface not be inherited by end user implementation. """ @abstractmethod diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py b/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py index 875e20dcfb..2af8bed08b 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/custom_dict.py @@ -1,16 +1,16 @@ from typing import Any, Callable, Dict -from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseDictSerializer +from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseIdempotencySerializer -class CustomDictSerializer(BaseDictSerializer): +class CustomDictSerializer(BaseIdempotencySerializer): def __init__(self, to_dict: Callable[[Any], Dict], from_dict: Callable[[Dict], Any]): """ Parameters ---------- to_dict: Callable[[Any], Dict] A function capable of transforming the saved data object representation into a dictionary - from_dict: str + from_dict: Callable[[Dict], Any] A function capable of transforming the saved dictionary into the original data object representation """ self.__to_dict: Callable[[Any], Dict] = to_dict diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/no_op.py b/aws_lambda_powertools/utilities/idempotency/serialization/no_op.py index d48369116b..59185f704e 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/no_op.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/no_op.py @@ -1,14 +1,14 @@ from typing import Dict -from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseDictSerializer +from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseIdempotencySerializer -class NoOpSerializer(BaseDictSerializer): +class NoOpSerializer(BaseIdempotencySerializer): def __init__(self): """ Parameters ---------- - default serializer, does not transform data + Default serializer, does not transform data """ def to_dict(self, data: Dict) -> Dict: diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py index 21aeaa888b..8c0eb01d93 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py @@ -2,18 +2,18 @@ from pydantic import BaseModel -from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseDictSerializer +from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseIdempotencySerializer Model = BaseModel -class PydanticSerializer(BaseDictSerializer): +class PydanticSerializer(BaseIdempotencySerializer): def __init__(self, model: Type[Model]): """ Parameters ---------- model: Model - A model of the type to transform + A Pydantic model of the type to transform """ self.__model: Type[Model] = model From fc38f63c7d56703994a84b04dcb1d1d30dd4f6cc Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Fri, 25 Aug 2023 23:47:54 +0300 Subject: [PATCH 11/25] add support to dynamic model deducation --- .../utilities/idempotency/exceptions.py | 12 ++++++++ .../utilities/idempotency/idempotency.py | 19 +++++++++---- .../idempotency/serialization/base.py | 28 +++++++++++++++++++ .../idempotency/serialization/pydantic.py | 23 +++++++++++++-- .../idempotency/test_idempotency.py | 14 +++++++--- 5 files changed, 84 insertions(+), 12 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/exceptions.py b/aws_lambda_powertools/utilities/idempotency/exceptions.py index 67a8d6721b..af92b5ae65 100644 --- a/aws_lambda_powertools/utilities/idempotency/exceptions.py +++ b/aws_lambda_powertools/utilities/idempotency/exceptions.py @@ -71,3 +71,15 @@ class IdempotencyKeyError(BaseError): """ Payload does not contain an idempotent key """ + + +class IdempotencyModelTypeError(BaseError): + """ + Model type does not match expected type + """ + + +class IdempotencyNoSerializationModelError(BaseError): + """ + No model was supplied to the serializer + """ diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index 2c61248433..c13a637818 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -4,7 +4,8 @@ import functools import logging import os -from typing import Any, Callable, Dict, Optional, cast +from inspect import isclass +from typing import Any, Callable, Dict, Optional, Type, Union, cast from aws_lambda_powertools.middleware_factory import lambda_handler_decorator from aws_lambda_powertools.shared import constants @@ -14,7 +15,10 @@ from aws_lambda_powertools.utilities.idempotency.persistence.base import ( BasePersistenceLayer, ) -from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseIdempotencySerializer +from aws_lambda_powertools.utilities.idempotency.serialization.base import ( + BaseIdempotencyModelSerializer, + BaseIdempotencySerializer, +) from aws_lambda_powertools.utilities.typing import LambdaContext logger = logging.getLogger(__name__) @@ -86,7 +90,7 @@ def idempotent_function( data_keyword_argument: str, persistence_store: BasePersistenceLayer, config: Optional[IdempotencyConfig] = None, - output_serializer: Optional[BaseIdempotencySerializer] = None, + output_serializer: Optional[Union[BaseIdempotencySerializer, Type[BaseIdempotencyModelSerializer]]] = None, ) -> Any: """ Decorator to handle idempotency of any function @@ -101,9 +105,11 @@ def idempotent_function( Instance of BasePersistenceLayer to store data config: IdempotencyConfig Configuration - output_serializer: Optional[BaseIdempotencySerializer] + output_serializer: Optional[Union[BaseIdempotencySerializer, Type[BaseIdempotencyModelSerializer]]] Serializer to transform the data to and from a dictionary. - If not supplied, no serialization is done via the NoOpSerializer + If not supplied, no serialization is done via the NoOpSerializer. + In case a serializer of type inheriting BaseIdempotencyModelSerializer] is given, + the serializer is deduced from the function return type. Examples -------- **Processes an order in an idempotent manner** @@ -131,6 +137,9 @@ def process_order(customer_id: str, order: dict, **kwargs): output_serializer=output_serializer, ), ) + if isclass(output_serializer) and issubclass(output_serializer, BaseIdempotencyModelSerializer): + # instantiate an instance of the serializer class + output_serializer = output_serializer.instantiate(function.__annotations__.get("return", None)) config = config or IdempotencyConfig() diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/base.py b/aws_lambda_powertools/utilities/idempotency/serialization/base.py index de6e7c1fbb..2dde3a931e 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/base.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/base.py @@ -17,3 +17,31 @@ def to_dict(self, data: Any) -> Dict: @abstractmethod def from_dict(self, data: Dict) -> Any: pass + + +class BaseIdempotencyModelSerializer(BaseIdempotencySerializer): + """ + Abstract Base Class for Idempotency serialization layer, for using a model as data object representation. + """ + + @classmethod + @abstractmethod + def instantiate(cls, model_type: Any) -> BaseIdempotencySerializer: + """ + Instantiate the serializer from the given model type. + In case there is the model_type is unknown, None will be sent to the method. + It's on the implementer to verify that: + - None is handled + - A model type not matching the expected types is handled + + Parameters + ---------- + model_type: Any + The model type to instantiate the class for + + Returns + ------- + BaseIdempotencySerializer + Instance of the serializer class + """ + pass diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py index 8c0eb01d93..df89233e15 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py @@ -1,13 +1,20 @@ -from typing import Dict, Type +from typing import Any, Dict, Type from pydantic import BaseModel -from aws_lambda_powertools.utilities.idempotency.serialization.base import BaseIdempotencySerializer +from aws_lambda_powertools.utilities.idempotency.exceptions import ( + IdempotencyModelTypeError, + IdempotencyNoSerializationModelError, +) +from aws_lambda_powertools.utilities.idempotency.serialization.base import ( + BaseIdempotencyModelSerializer, + BaseIdempotencySerializer, +) Model = BaseModel -class PydanticSerializer(BaseIdempotencySerializer): +class PydanticSerializer(BaseIdempotencyModelSerializer): def __init__(self, model: Type[Model]): """ Parameters @@ -28,3 +35,13 @@ def from_dict(self, data: Dict) -> Model: # Support for pydantic V2 return self.__model.model_validate(data) # type: ignore[unused-ignore,attr-defined] return self.__model.parse_obj(data) + + @classmethod + def instantiate(cls, model_type: Any) -> BaseIdempotencySerializer: + if model_type is None: + raise IdempotencyNoSerializationModelError("No serialization model was supplied") + + if not issubclass(model_type, BaseModel): + raise IdempotencyModelTypeError("Model type is not inherited from pydantic BaseModel") + + return cls(model=model_type) diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 808b2ac55f..a12a4bd504 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -1278,7 +1278,8 @@ def record_handler(record): assert from_dict_called is False, "in case response is None, from_dict should not be called" -def test_idempotent_function_serialization_pydantic(): +@pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"]) +def test_idempotent_function_serialization_pydantic(output_serializer_type: str): # GIVEN config = IdempotencyConfig(use_local_cache=True) mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} @@ -1293,13 +1294,18 @@ class PaymentOutput(BaseModel): customer_id: str transaction_id: str + if output_serializer_type == "explicit": + output_serializer = PydanticSerializer( + model=PaymentOutput, + ) + else: + output_serializer = PydanticSerializer + @idempotent_function( data_keyword_argument="payment", persistence_store=persistence_layer, config=config, - output_serializer=PydanticSerializer( - model=PaymentOutput, - ), + output_serializer=output_serializer, ) def collect_payment(payment: PaymentInput) -> PaymentOutput: return PaymentOutput(**payment.dict()) From 9f08bf0f48dc9d25467e34d577486fa96339e6ec Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Fri, 25 Aug 2023 23:56:34 +0300 Subject: [PATCH 12/25] add negative tests --- .../idempotency/test_idempotency.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index a12a4bd504..626755acba 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -30,6 +30,8 @@ IdempotencyInconsistentStateError, IdempotencyInvalidStatusError, IdempotencyKeyError, + IdempotencyModelTypeError, + IdempotencyNoSerializationModelError, IdempotencyPersistenceLayerError, IdempotencyValidationError, ) @@ -1322,6 +1324,62 @@ def collect_payment(payment: PaymentInput) -> PaymentOutput: assert second_call.transaction_id == payment.transaction_id +def test_idempotent_function_serialization_pydantic_failure_no_return_type(): + # GIVEN + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class PaymentInput(BaseModel): + customer_id: str + transaction_id: str + + class PaymentOutput(BaseModel): + customer_id: str + transaction_id: str + + idempotent_function_decorator = idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=PydanticSerializer, + ) + with pytest.raises(IdempotencyNoSerializationModelError, match="No serialization model was supplied"): + + @idempotent_function_decorator + def collect_payment(payment: PaymentInput): + return PaymentOutput(**payment.dict()) + + +def test_idempotent_function_serialization_pydantic_failure_bad_type(): + # GIVEN + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + class PaymentInput(BaseModel): + customer_id: str + transaction_id: str + + class PaymentOutput(BaseModel): + customer_id: str + transaction_id: str + + idempotent_function_decorator = idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=PydanticSerializer, + ) + with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"): + + @idempotent_function_decorator + def collect_payment(payment: PaymentInput) -> dict: + return PaymentOutput(**payment.dict()) + + def test_idempotent_function_arbitrary_args_kwargs(): # Scenario to validate we can use idempotent_function with a function # with an arbitrary number of args and kwargs From fdde7b9636ee27e774856949315958bb86819aa0 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sat, 26 Aug 2023 00:14:14 +0300 Subject: [PATCH 13/25] add implelementation for dataclass --- .../idempotency/serialization/dataclass.py | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 aws_lambda_powertools/utilities/idempotency/serialization/dataclass.py diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/dataclass.py b/aws_lambda_powertools/utilities/idempotency/serialization/dataclass.py new file mode 100644 index 0000000000..e698c22d6f --- /dev/null +++ b/aws_lambda_powertools/utilities/idempotency/serialization/dataclass.py @@ -0,0 +1,39 @@ +from dataclasses import asdict, is_dataclass +from typing import Any, Dict, Type + +from aws_lambda_powertools.utilities.idempotency.exceptions import ( + IdempotencyModelTypeError, + IdempotencyNoSerializationModelError, +) +from aws_lambda_powertools.utilities.idempotency.serialization.base import ( + BaseIdempotencyModelSerializer, + BaseIdempotencySerializer, +) + +DataClass = Any + + +class DataclassSerializer(BaseIdempotencyModelSerializer): + def __init__(self, model: Type[DataClass]): + """ + Parameters + ---------- + model: Model + A Pydantic model of the type to transform + """ + self.__model: Type[DataClass] = model + + def to_dict(self, data: DataClass) -> Dict: + return asdict(data) + + def from_dict(self, data: Dict) -> DataClass: + return self.__model(**data) + + @classmethod + def instantiate(cls, model_type: Any) -> BaseIdempotencySerializer: + if model_type is None: + raise IdempotencyNoSerializationModelError("No serialization model was supplied") + + if not is_dataclass(model_type): + raise IdempotencyModelTypeError("Model type is not inherited of dataclass type") + return cls(model=model_type) From fe4ff3d6ca92b96455e58bb1b6cdbfcbc60fafc0 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sat, 26 Aug 2023 00:26:19 +0300 Subject: [PATCH 14/25] add dataclass serializer --- .../idempotency/test_idempotency.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/tests/functional/idempotency/test_idempotency.py b/tests/functional/idempotency/test_idempotency.py index 626755acba..24fcd76b4d 100644 --- a/tests/functional/idempotency/test_idempotency.py +++ b/tests/functional/idempotency/test_idempotency.py @@ -40,6 +40,7 @@ DataRecord, ) from aws_lambda_powertools.utilities.idempotency.serialization.custom_dict import CustomDictSerializer +from aws_lambda_powertools.utilities.idempotency.serialization.dataclass import DataclassSerializer from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import PydanticSerializer from aws_lambda_powertools.utilities.validation import envelopes, validator from tests.functional.idempotency.utils import ( @@ -1380,6 +1381,115 @@ def collect_payment(payment: PaymentInput) -> dict: return PaymentOutput(**payment.dict()) +@pytest.mark.parametrize("output_serializer_type", ["explicit", "deduced"]) +def test_idempotent_function_serialization_dataclass(output_serializer_type: str): + # GIVEN + dataclasses = get_dataclasses_lib() + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_dataclass..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + @dataclasses.dataclass + class PaymentInput: + customer_id: str + transaction_id: str + + @dataclasses.dataclass + class PaymentOutput: + customer_id: str + transaction_id: str + + if output_serializer_type == "explicit": + output_serializer = DataclassSerializer( + model=PaymentOutput, + ) + else: + output_serializer = DataclassSerializer + + @idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=output_serializer, + ) + def collect_payment(payment: PaymentInput) -> PaymentOutput: + return PaymentOutput(**dataclasses.asdict(payment)) + + # WHEN + payment = PaymentInput(**mock_event) + first_call: PaymentOutput = collect_payment(payment=payment) + assert first_call.customer_id == payment.customer_id + assert first_call.transaction_id == payment.transaction_id + assert isinstance(first_call, PaymentOutput) + second_call: PaymentOutput = collect_payment(payment=payment) + assert isinstance(second_call, PaymentOutput) + assert second_call.customer_id == payment.customer_id + assert second_call.transaction_id == payment.transaction_id + + +def test_idempotent_function_serialization_dataclass_failure_no_return_type(): + # GIVEN + dataclasses = get_dataclasses_lib() + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + @dataclasses.dataclass + class PaymentInput: + customer_id: str + transaction_id: str + + @dataclasses.dataclass + class PaymentOutput: + customer_id: str + transaction_id: str + + idempotent_function_decorator = idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=DataclassSerializer, + ) + with pytest.raises(IdempotencyNoSerializationModelError, match="No serialization model was supplied"): + + @idempotent_function_decorator + def collect_payment(payment: PaymentInput): + return PaymentOutput(**payment.dict()) + + +def test_idempotent_function_serialization_dataclass_failure_bad_type(): + # GIVEN + dataclasses = get_dataclasses_lib() + config = IdempotencyConfig(use_local_cache=True) + mock_event = {"customer_id": "fake", "transaction_id": "fake-id"} + idempotency_key = f"{TESTS_MODULE_PREFIX}.test_idempotent_function_serialization_pydantic_failure_no_return_type..collect_payment#{hash_idempotency_key(mock_event)}" # noqa E501 + persistence_layer = MockPersistenceLayer(expected_idempotency_key=idempotency_key) + + @dataclasses.dataclass + class PaymentInput: + customer_id: str + transaction_id: str + + @dataclasses.dataclass + class PaymentOutput: + customer_id: str + transaction_id: str + + idempotent_function_decorator = idempotent_function( + data_keyword_argument="payment", + persistence_store=persistence_layer, + config=config, + output_serializer=PydanticSerializer, + ) + with pytest.raises(IdempotencyModelTypeError, match="Model type is not inherited from pydantic BaseModel"): + + @idempotent_function_decorator + def collect_payment(payment: PaymentInput) -> dict: + return PaymentOutput(**payment.dict()) + + def test_idempotent_function_arbitrary_args_kwargs(): # Scenario to validate we can use idempotent_function with a function # with an arbitrary number of args and kwargs From 18ae349d21fb64fce6d67fe54cb5f7852dfb245c Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Thu, 31 Aug 2023 17:37:55 +0300 Subject: [PATCH 15/25] cr change requests --- .../utilities/idempotency/serialization/base.py | 4 ++-- ...potent_function_dataclass_output_serializer.py | 15 ++++++++++++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/base.py b/aws_lambda_powertools/utilities/idempotency/serialization/base.py index 2dde3a931e..b82f070617 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/base.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/base.py @@ -12,11 +12,11 @@ class BaseIdempotencySerializer(ABC): @abstractmethod def to_dict(self, data: Any) -> Dict: - pass + raise NotImplementedError("Implementation of to_dict is required") @abstractmethod def from_dict(self, data: Dict) -> Any: - pass + raise NotImplementedError("Implementation of from_dict is required") class BaseIdempotencyModelSerializer(BaseIdempotencySerializer): diff --git a/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py b/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py index 193de89cb9..ca366a49aa 100644 --- a/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py +++ b/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py @@ -1,4 +1,5 @@ -from dataclasses import dataclass +from dataclasses import asdict, dataclass +from typing import Any, Dict from aws_lambda_powertools.utilities.idempotency import ( DynamoDBPersistenceLayer, @@ -29,9 +30,17 @@ class OrderOutput: order_id: int +def custom_to_dict(x: Any) -> Dict: + return asdict(x) + + +def custom_from_dict(x: Dict) -> Any: + return OrderOutput(**x) + + order_output_serializer: CustomDictSerializer = CustomDictSerializer( - to_dict=lambda x: x.asdict(), - from_dict=lambda x: OrderOutput(**x), + to_dict=custom_to_dict, + from_dict=custom_from_dict, ) From f6dc6959352bb2cf5d11bc964fcf3a2b60453f9a Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Thu, 31 Aug 2023 17:40:11 +0300 Subject: [PATCH 16/25] rename example file --- docs/utilities/idempotency.md | 2 +- ...orking_with_idempotent_function_custom_output_serializer.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename examples/idempotency/src/{working_with_idempotent_function_dataclass_output_serializer.py => working_with_idempotent_function_custom_output_serializer.py} (100%) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 78d2954d92..d343501d1c 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -159,7 +159,7 @@ By supplying an output serializer, you can control the return type of the functi === "Using A Custom Type (Dataclasses)" ```python hl_lines="3-8 26-28 32-35 42 48" - --8<-- "examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py" + --8<-- "examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py" ``` === "Using Pydantic" diff --git a/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py b/examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py similarity index 100% rename from examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py rename to examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py From cc5a1bd718db84cdf019859cfc04d27e61d9d64d Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Thu, 31 Aug 2023 17:41:38 +0300 Subject: [PATCH 17/25] fix lines --- docs/utilities/idempotency.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index d343501d1c..7f6bf66ed4 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -158,7 +158,7 @@ By supplying an output serializer, you can control the return type of the functi === "Using A Custom Type (Dataclasses)" - ```python hl_lines="3-8 26-28 32-35 42 48" + ```python hl_lines="3-8 26-28 32-44 51" --8<-- "examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py" ``` From ba2ebc6d6c3187858047e6109972762d57e81186 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Fri, 1 Sep 2023 12:00:49 +0300 Subject: [PATCH 18/25] add more docs and examples --- docs/utilities/idempotency.md | 26 ++++++-- ...nt_function_dataclass_output_serializer.py | 60 +++++++++++++++++++ ...ent_function_pydantic_output_serializer.py | 16 ++++- 3 files changed, 94 insertions(+), 8 deletions(-) create mode 100644 examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index 7f6bf66ed4..cbba4cc398 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -156,16 +156,30 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo By supplying an output serializer, you can control the return type of the function, allowing cleaner integration with the rest of your code base. -=== "Using A Custom Type (Dataclasses)" +=== "Using Pydantic" +Explicitly passing the model type + ```python hl_lines="3-8 24-25 32-35 55" + --8<-- "examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py" + ``` +Deducing the model type from the return type annotation + ```python hl_lines="3-8 24-25 42-46 55" + --8<-- "examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py" + ``` - ```python hl_lines="3-8 26-28 32-44 51" - --8<-- "examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py" +=== "Using Dataclasses" +Explicitly passing the model type + ```python hl_lines="1 5-8 27-29 36-39 59" + --8<-- "examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py" + ``` +Deducing the model type from the return type annotation + ```python hl_lines="1 5-8 27-29 46-50 60" + --8<-- "examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py" ``` -=== "Using Pydantic" +=== "Using A Custom Type (Dataclasses)" - ```python hl_lines="3-8 23-24 32 35" - --8<-- "examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py" + ```python hl_lines="3-8 26-28 32-44 51" + --8<-- "examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py" ``` #### Batch integration diff --git a/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py b/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py new file mode 100644 index 0000000000..2459fbf587 --- /dev/null +++ b/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py @@ -0,0 +1,60 @@ +from dataclasses import dataclass + +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.idempotency.serialization.dataclass import DataclassSerializer +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig(event_key_jmespath="order_id") # see Choosing a payload subset section + + +@dataclass +class OrderItem: + sku: str + description: str + + +@dataclass +class Order: + item: OrderItem + order_id: int + + +@dataclass +class OrderOutput: + order_id: int + + +@idempotent_function( + data_keyword_argument="order", + config=config, + persistence_store=dynamodb, + output_serializer=DataclassSerializer(model=OrderOutput), +) +def explicit_order_output_serializer(order: Order): + return OrderOutput(order_id=order.order_id) + + +@idempotent_function( + data_keyword_argument="order", + config=config, + persistence_store=dynamodb, + output_serializer=DataclassSerializer, +) +# order output is deduced from return type +def deduced_order_output_serializer(order: Order) -> OrderOutput: + return OrderOutput(order_id=order.order_id) + + +def lambda_handler(event: dict, context: LambdaContext): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id=1) + + # `order` parameter must be called as a keyword argument to work + explicit_order_output_serializer(order=order) + deduced_order_output_serializer(order=order) diff --git a/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py b/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py index 46adbe283d..d5d449cecc 100644 --- a/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py +++ b/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py @@ -31,7 +31,18 @@ class OrderOutput(BaseModel): persistence_store=dynamodb, output_serializer=PydanticSerializer(model=OrderOutput), ) -def process_order(order: Order) -> OrderOutput: +def explicit_order_output_serializer(order: Order): + return OrderOutput(order_id=order.order_id) + + +@idempotent_function( + data_keyword_argument="order", + config=config, + persistence_store=dynamodb, + output_serializer=PydanticSerializer, +) +# order output is deduced from return type +def deduced_order_output_serializer(order: Order) -> OrderOutput: return OrderOutput(order_id=order.order_id) @@ -41,4 +52,5 @@ def lambda_handler(event: dict, context: LambdaContext): order = Order(item=order_item, order_id=1) # `order` parameter must be called as a keyword argument to work - process_order(order=order) + explicit_order_output_serializer(order=order) + deduced_order_output_serializer(order=order) From 605278300af7b995d56b3ce0bece9ba68e8ee905 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Fri, 1 Sep 2023 12:03:41 +0300 Subject: [PATCH 19/25] remove redundant type annotation --- ...working_with_idempotent_function_custom_output_serializer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py b/examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py index ca366a49aa..f8ef30c7ab 100644 --- a/examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py +++ b/examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py @@ -38,7 +38,7 @@ def custom_from_dict(x: Dict) -> Any: return OrderOutput(**x) -order_output_serializer: CustomDictSerializer = CustomDictSerializer( +order_output_serializer = CustomDictSerializer( to_dict=custom_to_dict, from_dict=custom_from_dict, ) From 39b27a0e0203ce5e86e7bee0e64429a5c9ec1139 Mon Sep 17 00:00:00 2001 From: Ruben Fonseca Date: Fri, 1 Sep 2023 13:53:14 +0200 Subject: [PATCH 20/25] fix: reading improvements --- .../utilities/idempotency/exceptions.py | 2 +- .../utilities/idempotency/idempotency.py | 8 ++++---- .../utilities/idempotency/serialization/base.py | 6 +++--- .../idempotency/serialization/dataclass.py | 8 ++++++-- .../idempotency/serialization/pydantic.py | 14 +++++++------- 5 files changed, 21 insertions(+), 17 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/exceptions.py b/aws_lambda_powertools/utilities/idempotency/exceptions.py index af92b5ae65..6e5930549c 100644 --- a/aws_lambda_powertools/utilities/idempotency/exceptions.py +++ b/aws_lambda_powertools/utilities/idempotency/exceptions.py @@ -75,7 +75,7 @@ class IdempotencyKeyError(BaseError): class IdempotencyModelTypeError(BaseError): """ - Model type does not match expected type + Model type does not match expected payload output """ diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index c13a637818..fd68c81739 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -4,7 +4,6 @@ import functools import logging import os -from inspect import isclass from typing import Any, Callable, Dict, Optional, Type, Union, cast from aws_lambda_powertools.middleware_factory import lambda_handler_decorator @@ -108,8 +107,8 @@ def idempotent_function( output_serializer: Optional[Union[BaseIdempotencySerializer, Type[BaseIdempotencyModelSerializer]]] Serializer to transform the data to and from a dictionary. If not supplied, no serialization is done via the NoOpSerializer. - In case a serializer of type inheriting BaseIdempotencyModelSerializer] is given, - the serializer is deduced from the function return type. + In case a serializer of type inheriting BaseIdempotencyModelSerializer is given, + the serializer is derived from the function return type. Examples -------- **Processes an order in an idempotent manner** @@ -137,7 +136,8 @@ def process_order(customer_id: str, order: dict, **kwargs): output_serializer=output_serializer, ), ) - if isclass(output_serializer) and issubclass(output_serializer, BaseIdempotencyModelSerializer): + + if issubclass(output_serializer, BaseIdempotencyModelSerializer): # instantiate an instance of the serializer class output_serializer = output_serializer.instantiate(function.__annotations__.get("return", None)) diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/base.py b/aws_lambda_powertools/utilities/idempotency/serialization/base.py index b82f070617..45317bd031 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/base.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/base.py @@ -28,10 +28,10 @@ class BaseIdempotencyModelSerializer(BaseIdempotencySerializer): @abstractmethod def instantiate(cls, model_type: Any) -> BaseIdempotencySerializer: """ - Instantiate the serializer from the given model type. - In case there is the model_type is unknown, None will be sent to the method. + Creates an instance of a serializer based on a provided model type. + In case the model_type is unknown, None will be sent as `model_type`. It's on the implementer to verify that: - - None is handled + - None is handled correctly - A model type not matching the expected types is handled Parameters diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/dataclass.py b/aws_lambda_powertools/utilities/idempotency/serialization/dataclass.py index e698c22d6f..dac77ed734 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/dataclass.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/dataclass.py @@ -14,12 +14,16 @@ class DataclassSerializer(BaseIdempotencyModelSerializer): + """ + A serializer class for transforming data between dataclass objects and dictionaries. + """ + def __init__(self, model: Type[DataClass]): """ Parameters ---------- - model: Model - A Pydantic model of the type to transform + model: Type[DataClass] + A dataclass type to be used for serialization and deserialization """ self.__model: Type[DataClass] = model diff --git a/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py index df89233e15..0c168233bf 100644 --- a/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py +++ b/aws_lambda_powertools/utilities/idempotency/serialization/pydantic.py @@ -11,26 +11,26 @@ BaseIdempotencySerializer, ) -Model = BaseModel - class PydanticSerializer(BaseIdempotencyModelSerializer): - def __init__(self, model: Type[Model]): + """Pydantic serializer for idempotency models""" + + def __init__(self, model: Type[BaseModel]): """ Parameters ---------- model: Model - A Pydantic model of the type to transform + Pydantic model to be used for serialization """ - self.__model: Type[Model] = model + self.__model: Type[BaseModel] = model - def to_dict(self, data: Model) -> Dict: + def to_dict(self, data: BaseModel) -> Dict: if callable(getattr(data, "model_dump", None)): # Support for pydantic V2 return data.model_dump() # type: ignore[unused-ignore,attr-defined] return data.dict() - def from_dict(self, data: Dict) -> Model: + def from_dict(self, data: Dict) -> BaseModel: if callable(getattr(self.__model, "model_validate", None)): # Support for pydantic V2 return self.__model.model_validate(data) # type: ignore[unused-ignore,attr-defined] From e71dd19ec21eb055c206faa0c1eff22dea1a4c3a Mon Sep 17 00:00:00 2001 From: Ruben Fonseca Date: Fri, 1 Sep 2023 14:05:08 +0200 Subject: [PATCH 21/25] fix: mypy errors --- aws_lambda_powertools/utilities/idempotency/idempotency.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index fd68c81739..d32f67ffde 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -4,6 +4,7 @@ import functools import logging import os +from inspect import isclass from typing import Any, Callable, Dict, Optional, Type, Union, cast from aws_lambda_powertools.middleware_factory import lambda_handler_decorator @@ -137,7 +138,7 @@ def process_order(customer_id: str, order: dict, **kwargs): ), ) - if issubclass(output_serializer, BaseIdempotencyModelSerializer): + if isclass(output_serializer) and issubclass(output_serializer, BaseIdempotencyModelSerializer): # instantiate an instance of the serializer class output_serializer = output_serializer.instantiate(function.__annotations__.get("return", None)) From a08ba6d2dc00e5aa565ae89e40ef88cdbf0d2add Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Fri, 1 Sep 2023 23:03:02 +0300 Subject: [PATCH 22/25] add newline --- aws_lambda_powertools/utilities/idempotency/idempotency.py | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_lambda_powertools/utilities/idempotency/idempotency.py b/aws_lambda_powertools/utilities/idempotency/idempotency.py index d32f67ffde..f38a860a6c 100644 --- a/aws_lambda_powertools/utilities/idempotency/idempotency.py +++ b/aws_lambda_powertools/utilities/idempotency/idempotency.py @@ -110,6 +110,7 @@ def idempotent_function( If not supplied, no serialization is done via the NoOpSerializer. In case a serializer of type inheriting BaseIdempotencyModelSerializer is given, the serializer is derived from the function return type. + Examples -------- **Processes an order in an idempotent manner** From af2ecb4d6c8b9b5bc08e91b8d75ecc855e77a18a Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sat, 2 Sep 2023 13:36:44 +0300 Subject: [PATCH 23/25] revert file format change --- aws_lambda_powertools/utilities/idempotency/__init__.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/__init__.py b/aws_lambda_powertools/utilities/idempotency/__init__.py index ae27330cc1..148b291ea6 100644 --- a/aws_lambda_powertools/utilities/idempotency/__init__.py +++ b/aws_lambda_powertools/utilities/idempotency/__init__.py @@ -11,10 +11,4 @@ from .idempotency import IdempotencyConfig, idempotent, idempotent_function -__all__ = ( - "DynamoDBPersistenceLayer", - "BasePersistenceLayer", - "idempotent", - "idempotent_function", - "IdempotencyConfig", -) +__all__ = ("DynamoDBPersistenceLayer", "BasePersistenceLayer", "idempotent", "idempotent_function", "IdempotencyConfig") From c0469b697de9adf27e153433a167250dae7b9e13 Mon Sep 17 00:00:00 2001 From: "arad.yaron" Date: Sat, 2 Sep 2023 13:41:23 +0300 Subject: [PATCH 24/25] add newline fix edge case where dict is {} and not None --- aws_lambda_powertools/utilities/idempotency/base.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/aws_lambda_powertools/utilities/idempotency/base.py b/aws_lambda_powertools/utilities/idempotency/base.py index 29aeb91383..a8d509b86e 100644 --- a/aws_lambda_powertools/utilities/idempotency/base.py +++ b/aws_lambda_powertools/utilities/idempotency/base.py @@ -79,7 +79,6 @@ def __init__( Function arguments function_kwargs: Optional[Dict] Function keyword arguments - """ self.function = function self.output_serializer = output_serializer or NoOpSerializer() @@ -220,7 +219,9 @@ def _handle_for_status(self, data_record: DataRecord) -> Optional[Any]: f"{self.persistence_store.event_key_jmespath}={data_record.idempotency_key}", ) response_dict: Optional[dict] = data_record.response_json_as_dict() - return self.output_serializer.from_dict(response_dict) if response_dict else None + if response_dict is not None: + return self.output_serializer.from_dict(response_dict) + return None def _get_function_response(self): try: From bfbc63632cdbea2da73a021c7afaf7724d4202e5 Mon Sep 17 00:00:00 2001 From: Leandro Damascena Date: Tue, 5 Sep 2023 12:25:27 +0100 Subject: [PATCH 25/25] Adjusts in the documentation --- docs/utilities/idempotency.md | 43 ++++++++++------- ...th_dataclass_deduced_output_serializer.py} | 11 ----- ..._dataclass_explicitly_output_serializer.py | 48 +++++++++++++++++++ ...ith_pydantic_deduced_output_serializer.py} | 11 ----- ...h_pydantic_explicitly_output_serializer.py | 44 +++++++++++++++++ 5 files changed, 118 insertions(+), 39 deletions(-) rename examples/idempotency/src/{working_with_idempotent_function_dataclass_output_serializer.py => working_with_dataclass_deduced_output_serializer.py} (80%) create mode 100644 examples/idempotency/src/working_with_dataclass_explicitly_output_serializer.py rename examples/idempotency/src/{working_with_idempotent_function_pydantic_output_serializer.py => working_with_pydantic_deduced_output_serializer.py} (81%) create mode 100644 examples/idempotency/src/working_with_pydantic_explicitly_output_serializer.py diff --git a/docs/utilities/idempotency.md b/docs/utilities/idempotency.md index cbba4cc398..3f55b34c25 100644 --- a/docs/utilities/idempotency.md +++ b/docs/utilities/idempotency.md @@ -152,33 +152,42 @@ When using `idempotent_function`, you must tell us which keyword parameter in yo --8<-- "examples/idempotency/src/working_with_idempotent_function_pydantic.py" ``` -#### Output Serialization +#### Output serialization -By supplying an output serializer, you can control the return type of the function, allowing cleaner integration with the rest of your code base. +The default return of the `idempotent_function` decorator is a JSON object, but you can customize the function's return type by utilizing the `output_serializer` parameter. The output serializer supports any JSON serializable data, **Python Dataclasses** and **Pydantic Models**. -=== "Using Pydantic" -Explicitly passing the model type - ```python hl_lines="3-8 24-25 32-35 55" - --8<-- "examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py" +!!! info "When using the `output_serializer` parameter, the data will continue to be stored in DynamoDB as a JSON object." + +Working with Pydantic Models: + +=== "Explicitly passing the Pydantic model type" + + ```python hl_lines="6 24 25 32 35 44" + --8<-- "examples/idempotency/src/working_with_pydantic_explicitly_output_serializer.py" ``` -Deducing the model type from the return type annotation - ```python hl_lines="3-8 24-25 42-46 55" - --8<-- "examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py" +=== "Deducing the Pydantic model type from the return type annotation" + + ```python hl_lines="6 24 25 32 36 45" + --8<-- "examples/idempotency/src/working_with_pydantic_deduced_output_serializer.py" ``` -=== "Using Dataclasses" -Explicitly passing the model type - ```python hl_lines="1 5-8 27-29 36-39 59" - --8<-- "examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py" +Working with Python Dataclasses: + +=== "Explicitly passing the model type" + + ```python hl_lines="8 27-29 36 39 48" + --8<-- "examples/idempotency/src/working_with_dataclass_explicitly_output_serializer.py" ``` -Deducing the model type from the return type annotation - ```python hl_lines="1 5-8 27-29 46-50 60" - --8<-- "examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py" + +=== "Deducing the model type from the return type annotation" + + ```python hl_lines="8 27-29 36 40 49" + --8<-- "examples/idempotency/src/working_with_dataclass_deduced_output_serializer.py" ``` === "Using A Custom Type (Dataclasses)" - ```python hl_lines="3-8 26-28 32-44 51" + ```python hl_lines="9 33 37 41-44 51 54" --8<-- "examples/idempotency/src/working_with_idempotent_function_custom_output_serializer.py" ``` diff --git a/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py b/examples/idempotency/src/working_with_dataclass_deduced_output_serializer.py similarity index 80% rename from examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py rename to examples/idempotency/src/working_with_dataclass_deduced_output_serializer.py index 2459fbf587..3feb5153e3 100644 --- a/examples/idempotency/src/working_with_idempotent_function_dataclass_output_serializer.py +++ b/examples/idempotency/src/working_with_dataclass_deduced_output_serializer.py @@ -29,16 +29,6 @@ class OrderOutput: order_id: int -@idempotent_function( - data_keyword_argument="order", - config=config, - persistence_store=dynamodb, - output_serializer=DataclassSerializer(model=OrderOutput), -) -def explicit_order_output_serializer(order: Order): - return OrderOutput(order_id=order.order_id) - - @idempotent_function( data_keyword_argument="order", config=config, @@ -56,5 +46,4 @@ def lambda_handler(event: dict, context: LambdaContext): order = Order(item=order_item, order_id=1) # `order` parameter must be called as a keyword argument to work - explicit_order_output_serializer(order=order) deduced_order_output_serializer(order=order) diff --git a/examples/idempotency/src/working_with_dataclass_explicitly_output_serializer.py b/examples/idempotency/src/working_with_dataclass_explicitly_output_serializer.py new file mode 100644 index 0000000000..95b65c570e --- /dev/null +++ b/examples/idempotency/src/working_with_dataclass_explicitly_output_serializer.py @@ -0,0 +1,48 @@ +from dataclasses import dataclass + +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.idempotency.serialization.dataclass import DataclassSerializer +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig(event_key_jmespath="order_id") # see Choosing a payload subset section + + +@dataclass +class OrderItem: + sku: str + description: str + + +@dataclass +class Order: + item: OrderItem + order_id: int + + +@dataclass +class OrderOutput: + order_id: int + + +@idempotent_function( + data_keyword_argument="order", + config=config, + persistence_store=dynamodb, + output_serializer=DataclassSerializer(model=OrderOutput), +) +def explicit_order_output_serializer(order: Order): + return OrderOutput(order_id=order.order_id) + + +def lambda_handler(event: dict, context: LambdaContext): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id=1) + + # `order` parameter must be called as a keyword argument to work + explicit_order_output_serializer(order=order) diff --git a/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py b/examples/idempotency/src/working_with_pydantic_deduced_output_serializer.py similarity index 81% rename from examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py rename to examples/idempotency/src/working_with_pydantic_deduced_output_serializer.py index d5d449cecc..98b7ed52bf 100644 --- a/examples/idempotency/src/working_with_idempotent_function_pydantic_output_serializer.py +++ b/examples/idempotency/src/working_with_pydantic_deduced_output_serializer.py @@ -25,16 +25,6 @@ class OrderOutput(BaseModel): order_id: int -@idempotent_function( - data_keyword_argument="order", - config=config, - persistence_store=dynamodb, - output_serializer=PydanticSerializer(model=OrderOutput), -) -def explicit_order_output_serializer(order: Order): - return OrderOutput(order_id=order.order_id) - - @idempotent_function( data_keyword_argument="order", config=config, @@ -52,5 +42,4 @@ def lambda_handler(event: dict, context: LambdaContext): order = Order(item=order_item, order_id=1) # `order` parameter must be called as a keyword argument to work - explicit_order_output_serializer(order=order) deduced_order_output_serializer(order=order) diff --git a/examples/idempotency/src/working_with_pydantic_explicitly_output_serializer.py b/examples/idempotency/src/working_with_pydantic_explicitly_output_serializer.py new file mode 100644 index 0000000000..6219e688e1 --- /dev/null +++ b/examples/idempotency/src/working_with_pydantic_explicitly_output_serializer.py @@ -0,0 +1,44 @@ +from aws_lambda_powertools.utilities.idempotency import ( + DynamoDBPersistenceLayer, + IdempotencyConfig, + idempotent_function, +) +from aws_lambda_powertools.utilities.idempotency.serialization.pydantic import PydanticSerializer +from aws_lambda_powertools.utilities.parser import BaseModel +from aws_lambda_powertools.utilities.typing import LambdaContext + +dynamodb = DynamoDBPersistenceLayer(table_name="IdempotencyTable") +config = IdempotencyConfig(event_key_jmespath="order_id") # see Choosing a payload subset section + + +class OrderItem(BaseModel): + sku: str + description: str + + +class Order(BaseModel): + item: OrderItem + order_id: int + + +class OrderOutput(BaseModel): + order_id: int + + +@idempotent_function( + data_keyword_argument="order", + config=config, + persistence_store=dynamodb, + output_serializer=PydanticSerializer(model=OrderOutput), +) +def explicit_order_output_serializer(order: Order): + return OrderOutput(order_id=order.order_id) + + +def lambda_handler(event: dict, context: LambdaContext): + config.register_lambda_context(context) # see Lambda timeouts section + order_item = OrderItem(sku="fake", description="sample") + order = Order(item=order_item, order_id=1) + + # `order` parameter must be called as a keyword argument to work + explicit_order_output_serializer(order=order)