From 2cfdc0dc7921c4c874107c97c41badcbe3f296c8 Mon Sep 17 00:00:00 2001 From: "roger.foskett" Date: Thu, 16 May 2024 13:57:06 +0100 Subject: [PATCH] simplify database provider common code --- abnosql/plugins/table/cosmos.py | 63 ++++++++------------------- abnosql/plugins/table/dynamodb.py | 59 ++++++++----------------- abnosql/plugins/table/firestore.py | 56 +++++++----------------- abnosql/plugins/table/memory.py | 46 +++++++++----------- abnosql/table.py | 70 +++++++++++++++++++++++++++++- 5 files changed, 139 insertions(+), 155 deletions(-) diff --git a/abnosql/plugins/table/cosmos.py b/abnosql/plugins/table/cosmos.py index 22e6b8c..bfb6c37 100644 --- a/abnosql/plugins/table/cosmos.py +++ b/abnosql/plugins/table/cosmos.py @@ -9,20 +9,19 @@ import abnosql.exceptions as ex from abnosql.plugin import PM -from abnosql.table import add_audit from abnosql.table import add_change_meta -from abnosql.table import audit_callback -from abnosql.table import check_exists from abnosql.table import check_exists_enabled +from abnosql.table import delete_item_post +from abnosql.table import delete_item_pre +from abnosql.table import get_item_post +from abnosql.table import get_item_pre from abnosql.table import get_key_attrs from abnosql.table import get_sql_params -from abnosql.table import kms_decrypt_item -from abnosql.table import kms_encrypt_item from abnosql.table import kms_process_query_items from abnosql.table import parse_connstr +from abnosql.table import put_item_post +from abnosql.table import put_item_pre from abnosql.table import TableBase -from abnosql.table import validate_item -from abnosql.table import validate_key_attrs from abnosql.table import validate_query_attrs hookimpl = pluggy.HookimplMarker('abnosql.table') @@ -201,9 +200,8 @@ def _container(self, name): @cosmos_ex_handler() def get_item(self, **kwargs) -> t.Optional[t.Dict]: - key = validate_key_attrs(self.key_attrs, dict(**kwargs), False) - self.pm.hook.get_item_pre(table=self.name, key=key) - _check_exists = dict(**kwargs).pop('abnosql_check_exists', None) + audit_key, _check_exists = get_item_pre(self, dict(**kwargs)) + item = None try: item = strip_cosmos_attrs( @@ -217,12 +215,7 @@ def get_item(self, **kwargs) -> t.Optional[t.Dict]: else: raise ex.NotFoundException('item not found') - _item = self.pm.hook.get_item_post(table=self.name, item=item) - if _item: - item = _item - item = kms_decrypt_item(self.config, item) - audit_callback(self, 'get', key) - return item + return get_item_post(self, dict(**kwargs), item, audit_key) @cosmos_ex_handler() def put_item( @@ -231,27 +224,11 @@ def put_item( update: t.Optional[bool] = False, audit_user: t.Optional[str] = None ) -> t.Dict: - operation = 'update' if update else 'create' - key = validate_key_attrs(self.key_attrs, item) + # cosmos has to do create/update on delete but don't audit this abnosql_audit_callback = item.pop('abnosql_audit_callback', None) - validate_item(self.config, operation, item) - item = check_exists(self, operation, item) - - audit_user = audit_user or self.config.get('audit_user') - if audit_user: - item = add_audit(item, update or False, audit_user) + item, key = put_item_pre(self, item, update, audit_user) - # add change metadata if enabled - if self.change_meta is True: - item = add_change_meta( - item, self.name, 'MODIFY' if update is True else 'INSERT' - ) - - _item = self.pm.hook.put_item_pre(table=self.name, item=item) - if _item: - item = _item[0] - item = kms_encrypt_item(self.config, item) # do update if update is True: kwargs = { @@ -267,12 +244,10 @@ def put_item( else: item = self._container(self.name).upsert_item(item) item = strip_cosmos_attrs(item) - self.pm.hook.put_item_post(table=self.name, item=item) - if abnosql_audit_callback is not False: - audit_callback( - self, 'update' if update else 'create', key, audit_user - ) - return item + + return put_item_post( + self, item, update, audit_user, abnosql_audit_callback + ) @cosmos_ex_handler() def put_items( @@ -288,8 +263,7 @@ def put_items( @cosmos_ex_handler() def delete_item(self, **kwargs): - check_exists(self, 'delete', dict(kwargs)) - key = validate_key_attrs(self.key_attrs, dict(**kwargs), False) + key = delete_item_pre(self, dict(kwargs)) # if change metadata enabled do update first then delete if self.change_meta is True: @@ -315,9 +289,8 @@ def delete_item(self, **kwargs): self._container(self.name).delete_item( **get_key_kwargs(**kwargs) ) - self.pm.hook.delete_item_post(table=self.name, key=dict(kwargs)) - if item is not None: - audit_callback(self, 'delete', key) + + delete_item_post(self, key) @cosmos_ex_handler() def query( diff --git a/abnosql/plugins/table/dynamodb.py b/abnosql/plugins/table/dynamodb.py index 149831d..60c4719 100644 --- a/abnosql/plugins/table/dynamodb.py +++ b/abnosql/plugins/table/dynamodb.py @@ -11,18 +11,17 @@ import abnosql.exceptions as ex from abnosql.plugin import PM -from abnosql.table import add_audit -from abnosql.table import audit_callback -from abnosql.table import check_exists from abnosql.table import check_exists_enabled +from abnosql.table import delete_item_post +from abnosql.table import delete_item_pre +from abnosql.table import get_item_post +from abnosql.table import get_item_pre from abnosql.table import get_key_attrs from abnosql.table import get_sql_params -from abnosql.table import kms_decrypt_item -from abnosql.table import kms_encrypt_item from abnosql.table import kms_process_query_items +from abnosql.table import put_item_post +from abnosql.table import put_item_pre from abnosql.table import TableBase -from abnosql.table import validate_item -from abnosql.table import validate_key_attrs from abnosql.table import validate_query_attrs hookimpl = pluggy.HookimplMarker('abnosql.table') @@ -180,23 +179,15 @@ def set_config(self, config: t.Optional[dict]): @dynamodb_ex_handler() def get_item(self, **kwargs) -> t.Optional[t.Dict]: - key = validate_key_attrs(self.key_attrs, dict(**kwargs), False) - self.pm.hook.get_item_pre(table=self.name, key=key) + audit_key, _ = get_item_pre(self, dict(**kwargs)) + response = deserialize(self.table.get_item( TableName=self.name, Key=get_key(**kwargs) ), self.config.get('deserializer')) - _check_exists = dict(**kwargs).pop('abnosql_check_exists', None) item = response.get('Item') - _item = self.pm.hook.get_item_post(table=self.name, item=item) - if _item: - item = _item - item = kms_decrypt_item(self.config, item) - if _check_exists is not False: - check_exists(self, 'get', item) - if item is not None: - audit_callback(self, 'get', key) - return item + + return get_item_post(self, dict(**kwargs), item, audit_key) @dynamodb_ex_handler() def put_item( @@ -205,18 +196,7 @@ def put_item( update: t.Optional[bool] = False, audit_user: t.Optional[str] = None ) -> t.Dict: - operation = 'update' if update else 'create' - key = validate_key_attrs(self.key_attrs, item) - validate_item(self.config, operation, item) - item = check_exists(self, operation, item) - - audit_user = audit_user or self.config.get('audit_user') - if audit_user: - item = add_audit(item, update or False, audit_user) - _item = self.pm.hook.put_item_pre(table=self.name, item=item) - if _item: - item = _item[0] - item = kms_encrypt_item(self.config, item) + item, _ = put_item_pre(self, item, update, audit_user) # do update if update is True: @@ -243,11 +223,7 @@ def put_item( else: self.table.put_item(Item=item) - self.pm.hook.put_item_post(table=self.name, item=item) - audit_callback( - self, 'update' if update else 'create', key, audit_user - ) - return item + return put_item_post(self, item, update, audit_user) @dynamodb_ex_handler() def put_items( @@ -263,12 +239,11 @@ def put_items( @dynamodb_ex_handler() def delete_item(self, **kwargs): - key = validate_key_attrs(self.key_attrs, dict(**kwargs), False) - check_exists(self, 'delete', dict(kwargs)) - _key = get_key(**kwargs) - self.table.delete_item(Key=_key) - self.pm.hook.delete_item_post(table=self.name, key=key) - audit_callback(self, 'delete', key) + key = delete_item_pre(self, dict(kwargs)) + + self.table.delete_item(Key=get_key(**kwargs)) + + delete_item_post(self, key) @dynamodb_ex_handler() def query( diff --git a/abnosql/plugins/table/firestore.py b/abnosql/plugins/table/firestore.py index 3c70211..5beabe9 100644 --- a/abnosql/plugins/table/firestore.py +++ b/abnosql/plugins/table/firestore.py @@ -13,18 +13,17 @@ import abnosql.exceptions as ex from abnosql.plugin import PM -from abnosql.table import add_audit -from abnosql.table import audit_callback -from abnosql.table import check_exists from abnosql.table import check_exists_enabled +from abnosql.table import delete_item_post +from abnosql.table import delete_item_pre +from abnosql.table import get_item_post +from abnosql.table import get_item_pre from abnosql.table import get_key_attrs -from abnosql.table import kms_decrypt_item -from abnosql.table import kms_encrypt_item from abnosql.table import kms_process_query_items from abnosql.table import parse_connstr +from abnosql.table import put_item_post +from abnosql.table import put_item_pre from abnosql.table import TableBase -from abnosql.table import validate_item -from abnosql.table import validate_key_attrs from abnosql.table import validate_query_attrs import sqlglot @@ -139,20 +138,12 @@ def set_config(self, config: t.Optional[dict]): @firestore_ex_handler() def get_item(self, **kwargs) -> t.Optional[t.Dict]: - key = validate_key_attrs(self.key_attrs, dict(**kwargs), False) - self.pm.hook.get_item_pre(table=self.name, key=key) - _check_exists = dict(**kwargs).pop('abnosql_check_exists', None) + audit_key, _ = get_item_pre(self, dict(**kwargs)) + doc = self.table.document(self._docid(**kwargs)).get() item = doc.to_dict() if doc.exists else None - _item = self.pm.hook.get_item_post(table=self.name, item=item) - if _item: - item = _item - item = kms_decrypt_item(self.config, item) - if _check_exists is not False: - check_exists(self, 'get', item) - if item is not None: - audit_callback(self, 'get', key) - return item + + return get_item_post(self, dict(**kwargs), item, audit_key) @firestore_ex_handler() def put_item( @@ -161,18 +152,7 @@ def put_item( update: t.Optional[bool] = False, audit_user: t.Optional[str] = None ) -> t.Dict: - operation = 'update' if update else 'create' - key = validate_key_attrs(self.key_attrs, item) - validate_item(self.config, operation, item) - item = check_exists(self, operation, item) - - audit_user = audit_user or self.config.get('audit_user') - if audit_user: - item = add_audit(item, update or False, audit_user) - _item = self.pm.hook.put_item_pre(table=self.name, item=item) - if _item: - item = _item[0] - item = kms_encrypt_item(self.config, item) + item, _ = put_item_pre(self, item, update, audit_user) # do update docid = self._docid(**item) @@ -190,17 +170,12 @@ def put_item( else: ref.set(item) - self.pm.hook.put_item_post(table=self.name, item=item) - # firestore doesnt return updated item, so make this optional if needed # note encrypted attrs won't be decrypted if self.config.get('put_get') is True: item = self.table.document(docid).get().to_dict() - audit_callback( - self, 'update' if update else 'create', key, audit_user - ) - return item + return put_item_post(self, item, update, audit_user) @firestore_ex_handler() def put_items( @@ -221,12 +196,11 @@ def put_items( @firestore_ex_handler() def delete_item(self, **kwargs): - key = validate_key_attrs(self.key_attrs, dict(**kwargs), False) - check_exists(self, 'delete', dict(kwargs)) + key = delete_item_pre(self, dict(kwargs)) + docid = self._docid(**kwargs) self.table.document(docid).delete() - self.pm.hook.delete_item_post(table=self.name, key=dict(kwargs)) - audit_callback(self, 'delete', key) + delete_item_post(self, key) @firestore_ex_handler() def query( diff --git a/abnosql/plugins/table/memory.py b/abnosql/plugins/table/memory.py index 4b734b0..1c2a575 100644 --- a/abnosql/plugins/table/memory.py +++ b/abnosql/plugins/table/memory.py @@ -10,16 +10,18 @@ import abnosql.exceptions as ex from abnosql.plugin import PM -from abnosql.table import add_audit -from abnosql.table import audit_callback +from abnosql.table import check_exists_enabled +from abnosql.table import delete_item_post +from abnosql.table import delete_item_pre +from abnosql.table import get_item_post +from abnosql.table import get_item_pre from abnosql.table import get_key_attrs from abnosql.table import get_sql_params -from abnosql.table import kms_decrypt_item -from abnosql.table import kms_encrypt_item from abnosql.table import kms_process_query_items +from abnosql.table import put_item_post +from abnosql.table import put_item_pre from abnosql.table import quote_str from abnosql.table import TableBase -from abnosql.table import validate_key_attrs from abnosql.table import validate_query_attrs @@ -170,6 +172,7 @@ def __init__( self.database = 'memory' self.set_config(config) self.key_attrs = get_key_attrs(self.config) + self.check_exists = check_exists_enabled(self.config) self.items = self.config.get('items', {}) @memory_ex_handler() @@ -183,8 +186,8 @@ def set_config(self, config: t.Optional[dict]): @memory_ex_handler() def get_item(self, **kwargs) -> t.Dict: - key = validate_key_attrs(self.key_attrs, dict(**kwargs), False) - self.pm.hook.get_item_pre(table=self.name, key=key) + audit_key, _ = get_item_pre(self, dict(**kwargs)) + key = get_key(**kwargs) item = None if self.items: @@ -192,13 +195,8 @@ def get_item(self, **kwargs) -> t.Dict: else: global TABLES item = TABLES.get(self.name, {}).get(key) - _item = self.pm.hook.get_item_post(table=self.name, item=item) - if _item: - item = _item - item = kms_decrypt_item(self.config, item) - if item is not None: - audit_callback(self, 'get', key) - return item + + return get_item_post(self, dict(**kwargs), item, audit_key) @memory_ex_handler() def put_item( @@ -207,11 +205,9 @@ def put_item( update: t.Optional[bool] = False, audit_user: t.Optional[str] = None ) -> t.Dict: - key = validate_key_attrs(self.key_attrs, item) - if audit_user: - item = add_audit(item, update or False, audit_user) + item, _ = put_item_pre(self, item, update, audit_user) + _key = ':'.join([item[_] for _ in self.key_attrs]) - item = kms_encrypt_item(self.config, item) if self.items: if update is True: self.items[_key].update(item) @@ -226,11 +222,8 @@ def put_item( else: TABLES[self.name][_key] = item item = TABLES[self.name][_key].copy() - self.pm.hook.put_item_post(table=self.name, item=item) - audit_callback( - self, 'update' if update else 'create', key, audit_user - ) - return item + + return put_item_post(self, item, update, audit_user) @memory_ex_handler() def put_items( @@ -245,15 +238,16 @@ def put_items( @memory_ex_handler() def delete_item(self, **kwargs): - key = validate_key_attrs(self.key_attrs, dict(**kwargs), False) + key = delete_item_pre(self, dict(kwargs)) + _key = get_key(**kwargs) if self.items: self.items.pop(_key, None) else: global TABLES TABLES.get(self.name, {}).pop(_key, None) - self.pm.hook.delete_item_post(table=self.name, key=_key) - audit_callback(self, 'delete', key) + + delete_item_post(self, key) @memory_ex_handler() def query( diff --git a/abnosql/table.py b/abnosql/table.py index aa4f76b..3741ee8 100644 --- a/abnosql/table.py +++ b/abnosql/table.py @@ -296,6 +296,70 @@ def quote_str(string): ) + "'" +def get_item_pre(tb, kwargs): + key = validate_key_attrs(tb.key_attrs, dict(**kwargs), False) + tb.pm.hook.get_item_pre(table=tb.name, key=key) + _check_exists = kwargs.pop('abnosql_check_exists', None) + return key, _check_exists + + +def get_item_post(tb, kwargs, item, audit_key): + _check_exists = kwargs.pop('abnosql_check_exists', None) + _item = tb.pm.hook.get_item_post(table=tb.name, item=item) + if _item: + item = _item + item = kms_decrypt_item(tb.config, item) + if _check_exists is not False: + check_exists(tb, 'get', item) + if item is not None: + audit_callback(tb, 'get', audit_key) + return item + + +def put_item_pre(tb, item, update, audit_user): + operation = 'update' if update else 'create' + key = validate_key_attrs(tb.key_attrs, item) + validate_item(tb.config, operation, item) + item = check_exists(tb, operation, item) + + audit_user = audit_user or tb.config.get('audit_user') + if audit_user: + item = add_audit(item, update or False, audit_user) + + # add change metadata if enabled + if hasattr(tb, 'change_meta') and tb.change_meta is True: + item = add_change_meta( + item, tb.name, 'MODIFY' if update is True else 'INSERT' + ) + + _item = tb.pm.hook.put_item_pre(table=tb.name, item=item) + if _item: + item = _item[0] + item = kms_encrypt_item(tb.config, item) + return item, key + + +def put_item_post(tb, item, update, audit_user, abnosql_audit_callback=True): + key = validate_key_attrs(tb.key_attrs, item) + tb.pm.hook.put_item_post(table=tb.name, item=item) + if abnosql_audit_callback is not False: + audit_callback( + tb, 'update' if update else 'create', key, audit_user + ) + return item + + +def delete_item_pre(tb, kwargs): + key = validate_key_attrs(tb.key_attrs, dict(**kwargs), False) + check_exists(tb, 'delete', dict(kwargs)) + return key + + +def delete_item_post(tb, key): + tb.pm.hook.delete_item_post(table=tb.name, key=key) + audit_callback(tb, 'delete', key) + + def validate_query_attrs(key: t.Dict, filters: t.Dict): """Validate that the query and filter attributes are named correctly @@ -605,7 +669,11 @@ def check_exists_enabled(config): def check_exists(obj: TableBase, operation: str, item: dict): - if len(obj.key_attrs) == 0 or obj.check_exists is False: # type: ignore + if ( + len(obj.key_attrs) == 0 # type: ignore + or not hasattr(obj, 'check_exists') + or obj.check_exists is False + ): return item key = { k: item.get(k) for k in obj.key_attrs # type: ignore