From 0fe1e1600982c681addc8b77eb216edcdfcc9073 Mon Sep 17 00:00:00 2001 From: Iryna Shevchenko Date: Tue, 8 Oct 2024 11:41:32 +0300 Subject: [PATCH] fix mapper and tests --- nightingale/mapper.py | 6 ++-- tests/test_loader.py | 2 +- tests/test_publisher.py | 5 +++- tests/test_unflatten.py | 65 ++++++++++++++++++++++++++++++++++++++++- 4 files changed, 73 insertions(+), 5 deletions(-) diff --git a/nightingale/mapper.py b/nightingale/mapper.py index 1eca1bd..c595a43 100644 --- a/nightingale/mapper.py +++ b/nightingale/mapper.py @@ -205,6 +205,8 @@ def set_nested_value(nested_dict, keys, value, schema, add_new=False, append_onc # case for /parties/roles set_nested_value(result, keys, value, flattened_schema, add_new=True, append_once=True) continue + elif array_counters is None: + array_counters = {} elif array_path in array_counters: if add_new := is_new_array(array_counters, child_path, last_key_name, array_value, array_path): array_counters[array_path] = array_value @@ -212,7 +214,7 @@ def set_nested_value(nested_dict, keys, value, schema, add_new=False, append_onc else: if last_key_name == "id": array_counters[array_path] = array_value - set_nested_value(result, keys[:-1], [{}], flattened_schema) + set_nested_value(result, keys[:-1], {}, flattened_schema, True) current = result for i, key in enumerate(keys[:-1]): @@ -236,7 +238,7 @@ def set_nested_value(nested_dict, keys, value, schema, add_new=False, append_onc def shift_current_array(self, current, array_path, array_counters): if not current: current.append({}) - return find_array_element_by_id(current, array_counters.get(array_path)) + return find_array_element_by_id(current, array_counters.get(array_path) if array_counters else None) def make_release_id(self, curr_row: dict) -> None: """ diff --git a/tests/test_loader.py b/tests/test_loader.py index 3684bd4..7d735d6 100644 --- a/tests/test_loader.py +++ b/tests/test_loader.py @@ -33,7 +33,7 @@ def test_get_connection(self): def test_load(self): data = self.loader.load("SELECT * FROM test_table") - self.assertEqual(data, [{"column1": "value1"}]) + self.assertEqual(list(data), [{"column1": "value1"}]) if __name__ == "__main__": diff --git a/tests/test_publisher.py b/tests/test_publisher.py index 9d9ce18..6c6c19d 100644 --- a/tests/test_publisher.py +++ b/tests/test_publisher.py @@ -1,4 +1,5 @@ import unittest +from unittest import mock from nightingale.config import Publishing from nightingale.publisher import DataPublisher @@ -7,7 +8,9 @@ class TestDataPublisher(unittest.TestCase): def setUp(self): self.config = Publishing(**{"version": "1.0", "publisher": "test_publisher", "base_uri": "http://example.com"}) - self.publisher = DataPublisher(self.config) + self.mapping = mock.Mock() + self.mapping.extensions = [{"url": "http://example.com/extension1"}, {"url": None}] + self.publisher = DataPublisher(self.config, self.mapping) self.data = [{"ocid": "ocid_prefix-1234"}] def test_publish(self): diff --git a/tests/test_unflatten.py b/tests/test_unflatten.py index d36fc87..4d5886e 100644 --- a/tests/test_unflatten.py +++ b/tests/test_unflatten.py @@ -25,6 +25,10 @@ def get_ocid_mapping(self): def get_containing_array_path(self, path): return get_longest_array_path(self.array_paths, path) +@pytest.fixture(autouse=True) +def mock_load_workbook(): + with mock.patch("openpyxl.load_workbook", return_value=mock.MagicMock()): + yield @pytest.fixture def mock_config(): @@ -54,12 +58,12 @@ def mock_config(): MockMappingConfig( { "flat_col1": "/object/field1", - "flat_col_s": "/object/field2/string_field", "flat_col2": "/object/field2/array_field/id", "flat_col3": "/object/field2/array_field/array_field2/id", "flat_col4": "/object/field2/array_field/array_field2/id", "flat_col5": "/object/field2/array_field/id", "flat_col6": "/object/field2/array_field/array_field2/id", + "flat_col_s": "/object/field2/string_field", }, array_paths=["/object/field2/array_field", "/object/field2/array_field/array_field2"], ), @@ -289,6 +293,65 @@ def mock_config(): None, {"root": {"field1": "1", "array1": [{"id": "2", "field2": "3"}, {"id": "4"}]}}, ), + ( + { + "col1": "val1", + "col2": "val2", + "col3": "val3", + "col4": "val4", + "col5": "val5", + "col6": "val6", + "col7": "val7" + }, + MockMappingConfig( + { + "col1": "/object/lvl1/field", + "col2": "/object/lvl1/array1/field1", + "col3": "/object/lvl1/array1/field2", + "col4": "/object/lvl1/array1/id", + "col5": "/object/lvl1/lvl2/field", + "col6": "/object/lvl1/lvl2/array2/field1", + "col7": "/object/lvl1/lvl2/array2/id", + }, + array_paths=["/lvl1/array"], + ), + { + "/object/lvl1/field": {"type": "string"}, + "/object/lvl1/array1": {"type": "array"}, + "/object/lvl1/array1/field1": {"type": "string"}, + "/object/lvl1/array1/field2": {"type": "string"}, + "/object/lvl1/array1/id": {"type": "string"}, + "/object/lvl1/lvl2/field": {"type": "string"}, + "/object/lvl1/lvl2/array2": {"type": "array"}, + "/object/lvl1/lvl2/array2/field1": {"type": "string"}, + "/object/lvl1/lvl2/array2/id": {"type": "string"}, + }, + None, + { + "object": { + "lvl1": { + "field": "val1", + "array1": [ + { + "id": "val4", + "field1": "val2", + "field2": "val3" + }, + ], + "lvl2": { + "field": "val5", + "array2": [ + { + "id": "val7", + "field1": "val6" + } + ] + } + + } + } + } + ), ], ) def test_transform_data(input_data, mapping_config, flattened_schema, result, expected_output, mock_config):