diff --git a/acstore/containers/manager.py b/acstore/containers/manager.py index fce892e..347737e 100644 --- a/acstore/containers/manager.py +++ b/acstore/containers/manager.py @@ -75,7 +75,7 @@ def GetSchema(cls, container_type): container_class = cls._attribute_container_classes.get( container_type, None) if not container_class: - raise ValueError(f'Unsupported container type: {container_type:s}') + raise ValueError(f'Unsupported container type: {container_type!s}') return getattr(container_class, 'SCHEMA', {}) diff --git a/acstore/helpers/json_serializer.py b/acstore/helpers/json_serializer.py new file mode 100644 index 0000000..e0ab0d7 --- /dev/null +++ b/acstore/helpers/json_serializer.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +"""Attribute container JSON serializer.""" + +from acstore.containers import manager as containers_manager +from acstore.helpers import schema as schema_helper + + +class AttributeContainerJSONSerializer(object): + """Attribute container JSON serializer.""" + + _CONTAINERS_MANAGER = containers_manager.AttributeContainersManager + + @classmethod + def ConvertAttributeContainerToJSON(cls, attribute_container): + """Converts an attribute container object into a JSON dictioary. + + The resulting dictionary of the JSON serialized objects consists of: + { + '__type__': 'AttributeContainer' + '__container_type__': ... + ... + } + + Here '__type__' indicates the object base type. In this case + 'AttributeContainer'. + + '__container_type__' indicates the container type and rest of the elements + of the dictionary that make up the attributes of the container. + + Args: + attribute_container (AttributeContainer): attribute container. + + Returns: + dict[str, object]: JSON serialized objects. + """ + try: + schema = cls._CONTAINERS_MANAGER.GetSchema( + attribute_container.CONTAINER_TYPE) + except ValueError: + schema = {} + + json_dict = { + '__type__': 'AttributeContainer', + '__container_type__': attribute_container.CONTAINER_TYPE} + + for attribute_name, attribute_value in attribute_container.GetAttributes(): + data_type = schema.get(attribute_name, None) + if data_type: + serializer = schema_helper.SchemaHelper.GetAttributeSerializer( + data_type, 'json') + + attribute_value = serializer.SerializeValue(attribute_value) + + # JSON will not serialize certain runtime types like set, therefore + # these are cast to list first. + if isinstance(attribute_value, set): + attribute_value = list(attribute_value) + + json_dict[attribute_name] = attribute_value + + return json_dict + + @classmethod + def ConvertJSONToAttributeContainer(cls, json_dict): + """Converts a JSON dictionary into an attribute container object. + + The dictionary of the JSON serialized objects consists of: + { + '__type__': 'AttributeContainer' + '__container_type__': ... + ... + } + + Here '__type__' indicates the object base type. In this case + 'AttributeContainer'. + + '__container_type__' indicates the container type and rest of the elements + of the dictionary that make up the attributes of the container. + + Args: + json_dict (dict[str, object]): JSON serialized objects. + + Returns: + AttributeContainer: attribute container. + """ + # Use __container_type__ to indicate the attribute container type. + container_type = json_dict.get('__container_type__', None) + + attribute_container = cls._CONTAINERS_MANAGER.CreateAttributeContainer( + container_type) + + supported_attribute_names = attribute_container.GetAttributeNames() + for attribute_name, attribute_value in json_dict.items(): + if attribute_name in ('__container_type__', '__type__'): + continue + + # Be strict about which attributes to set. + if attribute_name not in supported_attribute_names: + continue + + setattr(attribute_container, attribute_name, attribute_value) + + return attribute_container diff --git a/acstore/interface.py b/acstore/interface.py index 3e382e9..f3cd713 100644 --- a/acstore/interface.py +++ b/acstore/interface.py @@ -238,3 +238,55 @@ def UpdateAttributeContainer(self, container): """ self._RaiseIfNotWritable() self._WriteExistingAttributeContainer(container) + + +class AttributeContainerStoreWithReadCache(AttributeContainerStore): + """Interface of an attribute container store with read cache. + + Attributes: + format_version (int): storage format version. + """ + + # pylint: disable=abstract-method + + # The maximum number of cached attribute containers + _MAXIMUM_CACHED_CONTAINERS = 32 * 1024 + + def __init__(self): + """Initializes an attribute container store with read cache.""" + super(AttributeContainerStoreWithReadCache, self).__init__() + self._attribute_container_cache = collections.OrderedDict() + + def _CacheAttributeContainerByIndex(self, attribute_container, index): + """Caches a specific attribute container. + + Args: + attribute_container (AttributeContainer): attribute container. + index (int): attribute container index. + """ + if len(self._attribute_container_cache) >= self._MAXIMUM_CACHED_CONTAINERS: + self._attribute_container_cache.popitem(last=True) + + lookup_key = f'{attribute_container.CONTAINER_TYPE:s}.{index:d}' + self._attribute_container_cache[lookup_key] = attribute_container + self._attribute_container_cache.move_to_end(lookup_key, last=False) + + def _GetCachedAttributeContainer(self, container_type, index): + """Retrieves a specific cached attribute container. + + Args: + container_type (str): attribute container type. + index (int): attribute container index. + + Returns: + AttributeContainer: attribute container or None if not available. + + Raises: + IOError: when there is an error querying the attribute container store. + OSError: when there is an error querying the attribute container store. + """ + lookup_key = f'{container_type:s}.{index:d}' + attribute_container = self._attribute_container_cache.get(lookup_key, None) + if attribute_container: + self._attribute_container_cache.move_to_end(lookup_key, last=False) + return attribute_container diff --git a/acstore/sqlite_store.py b/acstore/sqlite_store.py index bf01adf..26af910 100644 --- a/acstore/sqlite_store.py +++ b/acstore/sqlite_store.py @@ -2,8 +2,8 @@ """SQLite-based attribute container store.""" import ast -import collections import itertools +import json import os import pathlib import sqlite3 @@ -122,7 +122,8 @@ def DeserializeValue(self, data_type, value): elif data_type not in self._MAPPINGS: serializer = schema_helper.SchemaHelper.GetAttributeSerializer( data_type, 'json') - value = serializer.DeserializeValue(value) + json_dict = json.loads(value) + value = serializer.DeserializeValue(json_dict) return value @@ -160,12 +161,14 @@ def SerializeValue(self, data_type, value): if isinstance(value, set): value = list(value) - return serializer.SerializeValue(value) + json_dict = serializer.SerializeValue(value) + return json.dumps(json_dict) return value -class SQLiteAttributeContainerStore(interface.AttributeContainerStore): +class SQLiteAttributeContainerStore( + interface.AttributeContainerStoreWithReadCache): """SQLite-based attribute container store. Attributes: @@ -205,15 +208,11 @@ class SQLiteAttributeContainerStore(interface.AttributeContainerStore): _INSERT_METADATA_VALUE_QUERY = ( 'INSERT INTO metadata (key, value) VALUES (?, ?)') - # The maximum number of cached attribute containers - _MAXIMUM_CACHED_CONTAINERS = 32 * 1024 - _MAXIMUM_WRITE_CACHE_SIZE = 50 def __init__(self): """Initializes a SQLite attribute container store.""" super(SQLiteAttributeContainerStore, self).__init__() - self._attribute_container_cache = collections.OrderedDict() self._connection = None self._cursor = None self._is_open = False @@ -224,20 +223,6 @@ def __init__(self): self.format_version = self._FORMAT_VERSION self.serialization_format = 'json' - def _CacheAttributeContainerByIndex(self, attribute_container, index): - """Caches a specific attribute container. - - Args: - attribute_container (AttributeContainer): attribute container. - index (int): attribute container index. - """ - if len(self._attribute_container_cache) >= self._MAXIMUM_CACHED_CONTAINERS: - self._attribute_container_cache.popitem(last=True) - - lookup_key = f'{attribute_container.CONTAINER_TYPE:s}.{index:d}' - self._attribute_container_cache[lookup_key] = attribute_container - self._attribute_container_cache.move_to_end(lookup_key, last=False) - def _CacheAttributeContainerForWrite( self, container_type, column_names, values): """Caches an attribute container for writing. @@ -515,26 +500,6 @@ def _GetAttributeContainersWithFilter( if self._storage_profiler: self._storage_profiler.StopTiming('get_containers') - def _GetCachedAttributeContainer(self, container_type, index): - """Retrieves a specific cached attribute container. - - Args: - container_type (str): attribute container type. - index (int): attribute container index. - - Returns: - AttributeContainer: attribute container or None if not available. - - Raises: - IOError: when there is an error querying the attribute container store. - OSError: when there is an error querying the attribute container store. - """ - lookup_key = f'{container_type:s}.{index:d}' - attribute_container = self._attribute_container_cache.get(lookup_key, None) - if attribute_container: - self._attribute_container_cache.move_to_end(lookup_key, last=False) - return attribute_container - def _HasTable(self, table_name): """Determines if a specific table exists. @@ -835,7 +800,7 @@ def Close(self): OSError: if the attribute container store is already closed. """ if not self._is_open: - raise IOError('Storage file already closed.') + raise IOError('Attribute container store already closed.') if self._connection: self._Flush() @@ -1034,7 +999,7 @@ def Open(self, path=None, read_only=True, **unused_kwargs): # pylint: disable=a ValueError: if path is missing. """ if self._is_open: - raise IOError('Storage file already opened.') + raise IOError('Attribute container store already opened.') if not path: raise ValueError('Missing path.') diff --git a/config/dpkg/changelog b/config/dpkg/changelog index 96b4a42..4c9d140 100644 --- a/config/dpkg/changelog +++ b/config/dpkg/changelog @@ -2,4 +2,4 @@ acstore (20240121-1) unstable; urgency=low * Auto-generated - -- Log2Timeline maintainers Sun, 21 Jan 2024 06:24:34 +0100 + -- Log2Timeline maintainers Sun, 21 Jan 2024 08:55:35 +0100 diff --git a/docs/sources/api/acstore.helpers.rst b/docs/sources/api/acstore.helpers.rst index 89c81f0..45823d3 100644 --- a/docs/sources/api/acstore.helpers.rst +++ b/docs/sources/api/acstore.helpers.rst @@ -4,6 +4,14 @@ acstore.helpers package Submodules ---------- +acstore.helpers.json\_serializer module +--------------------------------------- + +.. automodule:: acstore.helpers.json_serializer + :members: + :undoc-members: + :show-inheritance: + acstore.helpers.schema module ----------------------------- diff --git a/tests/containers/interface.py b/tests/containers/interface.py index 1553dd5..9f8be08 100644 --- a/tests/containers/interface.py +++ b/tests/containers/interface.py @@ -32,12 +32,12 @@ class AttributeContainerTest(test_lib.BaseTestCase): def testCopyToDict(self): """Tests the CopyToDict function.""" attribute_container = interface.AttributeContainer() - attribute_container.attribute_name = 'attribute_name' - attribute_container.attribute_value = 'attribute_value' + attribute_container.attribute_name = 'MyName' + attribute_container.attribute_value = 'MyValue' expected_dict = { - 'attribute_name': 'attribute_name', - 'attribute_value': 'attribute_value'} + 'attribute_name': 'MyName', + 'attribute_value': 'MyValue'} test_dict = attribute_container.CopyToDict() @@ -47,8 +47,8 @@ def testGetAttributeNames(self): """Tests the GetAttributeNames function.""" attribute_container = interface.AttributeContainer() attribute_container._protected_attribute = 'protected' - attribute_container.attribute_name = 'attribute_name' - attribute_container.attribute_value = 'attribute_value' + attribute_container.attribute_name = 'MyName' + attribute_container.attribute_value = 'MyValue' expected_attribute_names = ['attribute_name', 'attribute_value'] @@ -70,12 +70,12 @@ def testGetAttributes(self): """Tests the GetAttributes function.""" attribute_container = interface.AttributeContainer() attribute_container._protected_attribute = 'protected' - attribute_container.attribute_name = 'attribute_name' - attribute_container.attribute_value = 'attribute_value' + attribute_container.attribute_name = 'MyName' + attribute_container.attribute_value = 'MyValue' expected_attributes = [ - ('attribute_name', 'attribute_name'), - ('attribute_value', 'attribute_value')] + ('attribute_name', 'MyName'), + ('attribute_value', 'MyValue')] attributes = sorted(attribute_container.GetAttributes()) @@ -86,8 +86,8 @@ def testGetAttributes(self): expected_attributes = [ ('_protected_attribute', 'protected'), - ('attribute_name', 'attribute_name'), - ('attribute_value', 'attribute_value')] + ('attribute_name', 'MyName'), + ('attribute_value', 'MyValue')] attributes = sorted(attribute_container.GetAttributes()) @@ -97,8 +97,8 @@ def testGetAttributeValueHash(self): """Tests the GetAttributeValuesHash function.""" attribute_container = interface.AttributeContainer() attribute_container._protected_attribute = 'protected' - attribute_container.attribute_name = 'attribute_name' - attribute_container.attribute_value = 'attribute_value' + attribute_container.attribute_name = 'MyName' + attribute_container.attribute_value = 'MyValue' attribute_values_hash1 = attribute_container.GetAttributeValuesHash() @@ -108,7 +108,7 @@ def testGetAttributeValueHash(self): self.assertNotEqual(attribute_values_hash1, attribute_values_hash2) - attribute_container.attribute_value = 'attribute_value' + attribute_container.attribute_value = 'MyValue' setattr(attribute_container, '_SERIALIZABLE_PROTECTED_ATTRIBUTES', [ '_protected_attribute']) @@ -121,8 +121,8 @@ def testGetAttributeValuesString(self): """Tests the GetAttributeValuesString function.""" attribute_container = interface.AttributeContainer() attribute_container._protected_attribute = 'protected' - attribute_container.attribute_name = 'attribute_name' - attribute_container.attribute_value = 'attribute_value' + attribute_container.attribute_name = 'MyName' + attribute_container.attribute_value = 'MyValue' attribute_values_string1 = attribute_container.GetAttributeValuesString() @@ -132,7 +132,7 @@ def testGetAttributeValuesString(self): self.assertNotEqual(attribute_values_string1, attribute_values_string2) - attribute_container.attribute_value = 'attribute_value' + attribute_container.attribute_value = 'MyValue' setattr(attribute_container, '_SERIALIZABLE_PROTECTED_ATTRIBUTES', [ '_protected_attribute']) diff --git a/tests/containers/manager.py b/tests/containers/manager.py index 7f302e4..6f98bd8 100644 --- a/tests/containers/manager.py +++ b/tests/containers/manager.py @@ -14,77 +14,77 @@ class AttributeContainersManagerTest(shared_test_lib.BaseTestCase): # pylint: disable=protected-access + _TEST_MANAGER = manager.AttributeContainersManager + def testCreateAttributeContainer(self): """Tests the CreateAttributeContainer function.""" - manager.AttributeContainersManager.RegisterAttributeContainer( + self._TEST_MANAGER.RegisterAttributeContainer( shared_test_lib.TestAttributeContainer) try: - attribute_container = ( - manager.AttributeContainersManager.CreateAttributeContainer( - 'test_container')) + attribute_container = self._TEST_MANAGER.CreateAttributeContainer( + 'test_container') self.assertIsNotNone(attribute_container) with self.assertRaises(ValueError): - manager.AttributeContainersManager.CreateAttributeContainer('bogus') + self._TEST_MANAGER.CreateAttributeContainer('bogus') finally: - manager.AttributeContainersManager.DeregisterAttributeContainer( + self._TEST_MANAGER.DeregisterAttributeContainer( shared_test_lib.TestAttributeContainer) def testGetContainerTypes(self): """Tests the GetContainerTypes function.""" - manager.AttributeContainersManager.RegisterAttributeContainer( + self._TEST_MANAGER.RegisterAttributeContainer( shared_test_lib.TestAttributeContainer) try: - container_types = manager.AttributeContainersManager.GetContainerTypes() + container_types = self._TEST_MANAGER.GetContainerTypes() self.assertIn('test_container', container_types) finally: - manager.AttributeContainersManager.DeregisterAttributeContainer( + self._TEST_MANAGER.DeregisterAttributeContainer( shared_test_lib.TestAttributeContainer) def testGetSchema(self): """Tests the GetSchema function.""" - manager.AttributeContainersManager.RegisterAttributeContainer( + self._TEST_MANAGER.RegisterAttributeContainer( shared_test_lib.TestAttributeContainer) try: - schema = manager.AttributeContainersManager.GetSchema('test_container') + schema = self._TEST_MANAGER.GetSchema('test_container') self.assertIsNotNone(schema) self.assertEqual(schema, shared_test_lib.TestAttributeContainer.SCHEMA) with self.assertRaises(ValueError): - manager.AttributeContainersManager.GetSchema('bogus') + self._TEST_MANAGER.GetSchema('bogus') finally: - manager.AttributeContainersManager.DeregisterAttributeContainer( + self._TEST_MANAGER.DeregisterAttributeContainer( shared_test_lib.TestAttributeContainer) def testAttributeContainerRegistration(self): """Tests the Register and DeregisterAttributeContainer functions.""" - number_of_classes = len( - manager.AttributeContainersManager._attribute_container_classes) + number_of_classes = len(self._TEST_MANAGER._attribute_container_classes) - manager.AttributeContainersManager.RegisterAttributeContainer( + self._TEST_MANAGER.RegisterAttributeContainer( shared_test_lib.TestAttributeContainer) try: self.assertEqual( - len(manager.AttributeContainersManager._attribute_container_classes), + len(self._TEST_MANAGER._attribute_container_classes), number_of_classes + 1) with self.assertRaises(KeyError): - manager.AttributeContainersManager.RegisterAttributeContainer( + self._TEST_MANAGER.RegisterAttributeContainer( shared_test_lib.TestAttributeContainer) finally: - manager.AttributeContainersManager.DeregisterAttributeContainer( + self._TEST_MANAGER.DeregisterAttributeContainer( shared_test_lib.TestAttributeContainer) self.assertEqual( - len(manager.AttributeContainersManager._attribute_container_classes), + len(self._TEST_MANAGER._attribute_container_classes), number_of_classes) diff --git a/tests/helpers/json_serializer.py b/tests/helpers/json_serializer.py new file mode 100644 index 0000000..afb5e99 --- /dev/null +++ b/tests/helpers/json_serializer.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +"""Tests for the attribute container JSON serializer.""" + +import unittest + +from acstore.containers import manager +from acstore.helpers import json_serializer + +from tests import test_lib as shared_test_lib + + +class AttributeContainerJSONSerializerTest(shared_test_lib.BaseTestCase): + """Tests for the attribute container JSON serializer.""" + + _TEST_MANAGER = manager.AttributeContainersManager + _TEST_SERIALIZER = json_serializer.AttributeContainerJSONSerializer + + def testConvertAttributeContainerToJSON(self): + """Tests the ConvertAttributeContainerToJSON function.""" + attribute_container = shared_test_lib.TestAttributeContainer() + attribute_container.attribute = 'MyAttribute' + + expected_json_dict = { + '__container_type__': 'test_container', + '__type__': 'AttributeContainer', + 'attribute': 'MyAttribute'} + + json_dict = self._TEST_SERIALIZER.ConvertAttributeContainerToJSON( + attribute_container) + self.assertEqual(json_dict, expected_json_dict) + + def testConvertJSONToAttributeContainer(self): + """Tests the ConvertJSONToAttributeContainer function.""" + json_dict = { + '__container_type__': 'test_container', + '__type__': 'AttributeContainer', + 'attribute': 'MyAttribute'} + + self._TEST_MANAGER.RegisterAttributeContainer( + shared_test_lib.TestAttributeContainer) + + try: + attribute_container = ( + self._TEST_SERIALIZER.ConvertJSONToAttributeContainer(json_dict)) + + finally: + self._TEST_MANAGER.DeregisterAttributeContainer( + shared_test_lib.TestAttributeContainer) + + self.assertIsNotNone(attribute_container) + self.assertEqual(attribute_container.CONTAINER_TYPE, 'test_container') + self.assertEqual(attribute_container.attribute, 'MyAttribute') + + +if __name__ == '__main__': + unittest.main()