From 0526e21946bcee1340f24e362845225e32b88610 Mon Sep 17 00:00:00 2001 From: Markus Dregi Date: Fri, 19 Jun 2020 21:26:24 +0200 Subject: [PATCH] Support AllowEmpty for variable length containers --- configsuite/meta_keys.py | 1 + configsuite/schema.py | 29 ++- configsuite/validator.py | 10 + docs/source/release_notes.rst | 3 + docs/source/user_guide/getting_started.rst | 33 +++ tests/{test_types.py => test_basic_types.py} | 138 ++---------- tests/test_containers.py | 211 +++++++++++++++++++ 7 files changed, 294 insertions(+), 131 deletions(-) rename tests/{test_types.py => test_basic_types.py} (61%) create mode 100644 tests/test_containers.py diff --git a/configsuite/meta_keys.py b/configsuite/meta_keys.py index 626364d3..e2168eb6 100644 --- a/configsuite/meta_keys.py +++ b/configsuite/meta_keys.py @@ -35,3 +35,4 @@ class MetaKeys(enum.Enum): LayerTransformation = "layer_transformation" AllowNone = "allow_none" Default = "default" + AllowEmpty = "allow_empty" diff --git a/configsuite/schema.py b/configsuite/schema.py index 6fb48eed..dffe2aa9 100644 --- a/configsuite/schema.py +++ b/configsuite/schema.py @@ -76,9 +76,17 @@ def _check_required_not_default(schema_level): return True +@configsuite.validator_msg("Only variable length containers can specify AllowEmpty") +def _check_allowempty_only_variable_containers(schema_level): + level_type = schema_level.get(MK.Type) + const_len = isinstance(level_type, types.BasicType) or level_type == types.NamedDict + return not (const_len and MK.AllowEmpty in schema_level) + + _SCHEMA_LEVEL_DEFAULTS = { MK.Required: True, MK.AllowNone: False, + MK.AllowEmpty: True, MK.Description: "", MK.ElementValidators: (), MK.ContextValidators: (), @@ -93,6 +101,7 @@ def _check_required_not_default(schema_level): _check_default_type, _check_allownone_required, _check_required_not_default, + _check_allowempty_only_variable_containers, ), MK.Content: { MK.Type: {MK.Type: types.Type}, @@ -136,14 +145,19 @@ def _check_required_not_default(schema_level): MK.AllowNone: True, }, MK.Content: {MK.Type: _Anytype}, + MK.AllowEmpty: { + MK.Type: types.Bool, + MK.Required: False, + MK.Default: _SCHEMA_LEVEL_DEFAULTS[MK.AllowEmpty], + }, }, } -def _build_meta_schema(deduce_required, basic_type): +def _build_meta_schema(deduce_required, schema_type): meta_schema = copy.deepcopy(META_SCHEMA) - if basic_type: + if isinstance(schema_type, types.BasicType): meta_schema[MK.Content].pop(MK.Content) if deduce_required: @@ -151,6 +165,7 @@ def _build_meta_schema(deduce_required, basic_type): _check_allownone_type, _check_required_type, _check_default_type, + _check_allowempty_only_variable_containers, ) return meta_schema @@ -210,17 +225,22 @@ def _assert_valid_schema(schema, allow_default, validate_named_keys, deduce_requ def _build_level_schema(schema): schema = copy.deepcopy(schema) level_schema = copy.deepcopy(_SCHEMA_LEVEL_DEFAULTS) + is_basic_type = isinstance(schema[MK.Type], types.BasicType) # Discard ignore from default if not in level schema if MK.Required not in schema: level_schema.pop(MK.Required) # Discard basic type defaults for non-basic types - if not isinstance(schema[MK.Type], types.BasicType): + if not is_basic_type: for basic_key in (MK.Required, MK.Default, MK.AllowNone): if basic_key in level_schema: level_schema.pop(basic_key) + # Discard allow_empty default for basic types and named dicts + if is_basic_type or schema[MK.Type] == types.NamedDict: + level_schema.pop(MK.AllowEmpty) + level_schema.update(schema) return level_schema @@ -235,8 +255,7 @@ def _assert_valid_schema_level(schema, allow_default, deduce_required): fmt = "Default value is only allowed for contents in NamedDict" raise ValueError(fmt) - is_basic_type = isinstance(schema.get(MK.Type), types.BasicType) - meta_schema = _build_meta_schema(deduce_required, is_basic_type) + meta_schema = _build_meta_schema(deduce_required, schema.get(MK.Type)) level_validator = configsuite.Validator(meta_schema) result = level_validator.validate(schema) diff --git a/configsuite/validator.py b/configsuite/validator.py index e81ef26c..be16bdd7 100644 --- a/configsuite/validator.py +++ b/configsuite/validator.py @@ -75,6 +75,9 @@ def _validate(self, config, schema): if self._apply_validators and valid: valid &= self._context_validation(config, schema) + if self._apply_validators and valid: + valid &= self._length_validation(config, schema) + return bool(valid) def _element_validation(self, config, schema): @@ -89,6 +92,13 @@ def _element_validation(self, config, schema): return valid + def _length_validation(self, config, schema): + if not schema.get(MK.AllowEmpty, True) and len(config) == 0: + self._add_invalid_value_error("Expected non-empty container") + return False + + return True + def _context_validation(self, config, schema): context_validators = schema.get(MK.ContextValidators, ()) diff --git a/docs/source/release_notes.rst b/docs/source/release_notes.rst index cb08aee7..fb3c13eb 100644 --- a/docs/source/release_notes.rst +++ b/docs/source/release_notes.rst @@ -13,6 +13,9 @@ Release Notes dev --- +**New features** + - Add possibility in schema to specify whether a variable length container (list or dict) is allowed to be empty + 0.6.4 (2020-09-25) ------------------ diff --git a/docs/source/user_guide/getting_started.rst b/docs/source/user_guide/getting_started.rst index 0f7fb26a..9ee60617 100644 --- a/docs/source/user_guide/getting_started.rst +++ b/docs/source/user_guide/getting_started.rst @@ -568,6 +568,37 @@ in order for a configuration with ``None`` to pass. Scrooge has a credit of None +Allow Empty +----------- + +Occasionally one would like to require variable length containers (lists and +dicts) to have at least one element. This can be implemented with a validator +on the container (see :ref:`Validators-section`). However, as this feature have +been requested multiple times, we've decided to implement +explicit support for it without having to implement your own validator. Note +that by default all containers are allowed to be empty. + +.. testcode:: [allowempty] + + import configsuite + from configsuite import types + from configsuite import MetaKeys as MK + + schema = { + MK.Type: types.List, + MK.AllowEmpty: False, + MK.Content: { + MK.Item: {MK.Type: types.Integer}, + }, + } + + non_empty_suite = configsuite.ConfigSuite([0, 1, 2, 3, 4], schema) + assert non_empty_suite.valid + + empty_suite = configsuite.ConfigSuite([], schema) + assert not empty_suite.valid + + Default values -------------- @@ -881,6 +912,8 @@ will be replaced by the title. You can then include a customized ``.css`` file that acts on each item. +.. _validators-section: + Validators ---------- diff --git a/tests/test_types.py b/tests/test_basic_types.py similarity index 61% rename from tests/test_types.py rename to tests/test_basic_types.py index 91d518fe..345f5199 100644 --- a/tests/test_types.py +++ b/tests/test_basic_types.py @@ -1,4 +1,4 @@ -"""Copyright 2018 Equinor ASA and The Netherlands Organisation for +"""Copyright 2020 Equinor ASA and The Netherlands Organisation for Applied Scientific Research TNO. Licensed under the MIT license. @@ -26,7 +26,7 @@ from . import data -class TestTypes(unittest.TestCase): +class TestBasicTypes(unittest.TestCase): def test_name_pet_accepted(self): raw_config = data.pets.build_config() config_suite = configsuite.ConfigSuite(raw_config, data.pets.build_schema()) @@ -58,28 +58,6 @@ def test_invalid_string(self): raw_config["pet"]["favourite_food"], config.pet.favourite_food ) - def test_invalid_base_dict(self): - for raw_config in [14, None, [{"name": "Markus"}], ()]: - config_suite = configsuite.ConfigSuite(raw_config, data.pets.build_schema()) - self.assertFalse(config_suite.valid) - - self.assertEqual(1, len(config_suite.errors)) - err = config_suite.errors[0] - self.assertIsInstance(err, configsuite.InvalidTypeError) - self.assertEqual((), err.key_path) - - def test_invalid_dict(self): - for pet in [14, None, [{"name": "Markus"}], ()]: - raw_config = data.pets.build_config() - raw_config["pet"] = pet - config_suite = configsuite.ConfigSuite(raw_config, data.pets.build_schema()) - self.assertFalse(config_suite.valid) - - self.assertEqual(1, len(config_suite.errors)) - err = config_suite.errors[0] - self.assertIsInstance(err, configsuite.InvalidTypeError) - self.assertEqual(("pet",), err.key_path) - def test_string(self): for strval in ["fdnsjk", "", "str4thewin", u"ustr", True, [], 1, 1.2, {}, None]: raw_config = data.pets.build_config() @@ -148,108 +126,6 @@ def test_bool(self): self.assertIsInstance(err, configsuite.InvalidTypeError) self.assertEqual(("pet", "likeable"), err.key_path) - def test_list(self): - playgrounds = [{"name": "superfun", "score": 1}, {"name": "Hell", "score": 99}] - - for end_idx, _ in enumerate(playgrounds): - raw_config = data.pets.build_config() - raw_config["playgrounds"] = playgrounds[:end_idx] - config_suite = configsuite.ConfigSuite(raw_config, data.pets.build_schema()) - - self.assertTrue(config_suite.valid, "Errors: %r" % (config_suite.errors,)) - config = config_suite.snapshot - self.assertEqual(end_idx, len(config.playgrounds)) - - for idx, plgr in enumerate(playgrounds[:end_idx]): - self.assertEqual(plgr["name"], config.playgrounds[idx].name) - self.assertEqual(plgr["score"], config.playgrounds[idx].score) - - def test_containers_in_named_dicts(self): - schema = { - MK.Type: configsuite.types.NamedDict, - MK.Content: { - "list": { - MK.Type: configsuite.types.List, - MK.Content: {MK.Item: {MK.Type: configsuite.types.Number}}, - }, - "dict": { - MK.Type: configsuite.types.Dict, - MK.Content: { - MK.Key: {MK.Type: configsuite.types.String}, - MK.Value: {MK.Type: configsuite.types.Number}, - }, - }, - "named_dict": { - MK.Type: configsuite.types.NamedDict, - MK.Content: { - "a": {MK.Type: configsuite.types.Number, MK.Default: 0} - }, - }, - }, - } - - suite = configsuite.ConfigSuite({}, schema, deduce_required=True) - self.assertTrue(suite.valid, suite.errors) - self.assertEqual((), suite.snapshot.list) - self.assertEqual((), suite.snapshot.dict) - self.assertEqual(0, suite.snapshot.named_dict.a) - - def test_list_errors(self): - raw_config = data.pets.build_config() - raw_config["playgrounds"] = [{"name": "faulty grounds"}] - config_suite = configsuite.ConfigSuite(raw_config, data.pets.build_schema()) - - self.assertFalse(config_suite.valid) - self.assertEqual(1, len(config_suite.errors)) - err = config_suite.errors[0] - self.assertIsInstance(err, configsuite.MissingKeyError) - self.assertEqual(("playgrounds", 0), err.key_path) - - def test_list_cannot_spec_required(self): - schema = { - MK.Type: configsuite.types.List, - MK.Content: {MK.Item: {MK.Type: configsuite.types.Integer}}, - } - config_suite = configsuite.ConfigSuite([], schema) - self.assertTrue(config_suite.valid) - - for req in (True, False): - schema[MK.Required] = req - with self.assertRaises(ValueError) as err: - configsuite.ConfigSuite([], schema) - self.assertIn("Required can only be used for BasicType", str(err.exception)) - - def test_named_dict_cannot_spec_required(self): - schema = { - MK.Type: configsuite.types.NamedDict, - MK.Content: {"key": {MK.Type: configsuite.types.Integer}}, - } - config_suite = configsuite.ConfigSuite({"key": 42}, schema) - self.assertTrue(config_suite.valid) - - for req in (True, False): - schema[MK.Required] = req - with self.assertRaises(ValueError) as err: - configsuite.ConfigSuite({"key": 42}, schema) - self.assertIn("Required can only be used for BasicType", str(err.exception)) - - def test_dict_cannot_spec_required(self): - schema = { - MK.Type: configsuite.types.Dict, - MK.Content: { - MK.Key: {MK.Type: configsuite.types.String}, - MK.Value: {MK.Type: configsuite.types.String}, - }, - } - config_suite = configsuite.ConfigSuite({}, schema) - self.assertTrue(config_suite.valid) - - for req in (True, False): - schema[MK.Required] = req - with self.assertRaises(ValueError) as err: - configsuite.ConfigSuite({}, schema) - self.assertIn("Required can only be used for BasicType", str(err.exception)) - def test_date(self): timestamp = datetime.datetime(2015, 1, 2, 3, 4, 5) date = datetime.date(2010, 1, 1) @@ -309,3 +185,13 @@ def test_invalid_boolean_result_type(self): for elem in elements: with self.assertRaises(TypeError): configsuite.types.BooleanResult(elem, "incorrect", "type") + + def test_allow_empty_basic_type_invalid(self): + for val in [True, False]: + schema = {MK.Type: configsuite.types.String, MK.AllowEmpty: val} + with self.assertRaises(ValueError) as err: + configsuite.ConfigSuite("fdsfds", schema) + self.assertIn( + "Only variable length containers can specify AllowEmpty is false", + str(err.exception), + ) diff --git a/tests/test_containers.py b/tests/test_containers.py new file mode 100644 index 00000000..17d90a7b --- /dev/null +++ b/tests/test_containers.py @@ -0,0 +1,211 @@ +"""Copyright 2020 Equinor ASA and The Netherlands Organisation for +Applied Scientific Research TNO. + +Licensed under the MIT license. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the conditions stated in the LICENSE file in the project root for +details. + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. +""" + +import unittest + +import configsuite +from configsuite import MetaKeys as MK + +from . import data + + +class TestContainers(unittest.TestCase): + def test_invalid_base_dict(self): + for raw_config in [14, None, [{"name": "Markus"}], ()]: + config_suite = configsuite.ConfigSuite(raw_config, data.pets.build_schema()) + self.assertFalse(config_suite.valid) + + self.assertEqual(1, len(config_suite.errors)) + err = config_suite.errors[0] + self.assertIsInstance(err, configsuite.InvalidTypeError) + self.assertEqual((), err.key_path) + + def test_invalid_dict(self): + for pet in [14, None, [{"name": "Markus"}], ()]: + raw_config = data.pets.build_config() + raw_config["pet"] = pet + config_suite = configsuite.ConfigSuite(raw_config, data.pets.build_schema()) + self.assertFalse(config_suite.valid) + + self.assertEqual(1, len(config_suite.errors)) + err = config_suite.errors[0] + self.assertIsInstance(err, configsuite.InvalidTypeError) + self.assertEqual(("pet",), err.key_path) + + def test_list(self): + playgrounds = [{"name": "superfun", "score": 1}, {"name": "Hell", "score": 99}] + + for end_idx, _ in enumerate(playgrounds): + raw_config = data.pets.build_config() + raw_config["playgrounds"] = playgrounds[:end_idx] + config_suite = configsuite.ConfigSuite(raw_config, data.pets.build_schema()) + + self.assertTrue(config_suite.valid, "Errors: %r" % (config_suite.errors,)) + config = config_suite.snapshot + self.assertEqual(end_idx, len(config.playgrounds)) + + for idx, plgr in enumerate(playgrounds[:end_idx]): + self.assertEqual(plgr["name"], config.playgrounds[idx].name) + self.assertEqual(plgr["score"], config.playgrounds[idx].score) + + def test_containers_in_named_dicts(self): + schema = { + MK.Type: configsuite.types.NamedDict, + MK.Content: { + "list": { + MK.Type: configsuite.types.List, + MK.Content: {MK.Item: {MK.Type: configsuite.types.Number}}, + }, + "dict": { + MK.Type: configsuite.types.Dict, + MK.Content: { + MK.Key: {MK.Type: configsuite.types.String}, + MK.Value: {MK.Type: configsuite.types.Number}, + }, + }, + "named_dict": { + MK.Type: configsuite.types.NamedDict, + MK.Content: { + "a": {MK.Type: configsuite.types.Number, MK.Default: 0} + }, + }, + }, + } + + suite = configsuite.ConfigSuite({}, schema, deduce_required=True) + self.assertTrue(suite.valid, suite.errors) + self.assertEqual((), suite.snapshot.list) + self.assertEqual((), suite.snapshot.dict) + self.assertEqual(0, suite.snapshot.named_dict.a) + + def test_list_errors(self): + raw_config = data.pets.build_config() + raw_config["playgrounds"] = [{"name": "faulty grounds"}] + config_suite = configsuite.ConfigSuite(raw_config, data.pets.build_schema()) + + self.assertFalse(config_suite.valid) + self.assertEqual(1, len(config_suite.errors)) + err = config_suite.errors[0] + self.assertIsInstance(err, configsuite.MissingKeyError) + self.assertEqual(("playgrounds", 0), err.key_path) + + def test_list_cannot_spec_required(self): + schema = { + MK.Type: configsuite.types.List, + MK.Content: {MK.Item: {MK.Type: configsuite.types.Integer}}, + } + config_suite = configsuite.ConfigSuite([], schema) + self.assertTrue(config_suite.valid) + + for req in (True, False): + schema[MK.Required] = req + with self.assertRaises(ValueError) as err: + configsuite.ConfigSuite([], schema) + self.assertIn("Required can only be used for BasicType", str(err.exception)) + + def test_list_disallow_empty(self): + schema = { + MK.Type: configsuite.types.List, + MK.AllowEmpty: False, + MK.Content: {MK.Item: {MK.Type: configsuite.types.Integer}}, + } + + non_empty_suite = configsuite.ConfigSuite([1, 2, 3], schema) + self.assertTrue(non_empty_suite.valid) + + empty_suite = configsuite.ConfigSuite([], schema) + self.assertFalse(empty_suite.valid) + self.assertEqual(1, len(empty_suite.errors)) + self.assertIn("Expected non-empty container", empty_suite.errors[0].msg) + + def test_list_disallow_empty_layers(self): + schema = { + MK.Type: configsuite.types.List, + MK.AllowEmpty: False, + MK.Content: {MK.Item: {MK.Type: configsuite.types.Integer}}, + } + + suite = configsuite.ConfigSuite([], schema, layers=([1, 2, 3],)) + self.assertTrue(suite.valid) + + suite = configsuite.ConfigSuite([1, 2, 3], schema, layers=([],)) + self.assertTrue(suite.valid) + + def test_named_dict_cannot_spec_required(self): + schema = { + MK.Type: configsuite.types.NamedDict, + MK.Content: {"key": {MK.Type: configsuite.types.Integer}}, + } + config_suite = configsuite.ConfigSuite({"key": 42}, schema) + self.assertTrue(config_suite.valid) + + for req in (True, False): + schema[MK.Required] = req + with self.assertRaises(ValueError) as err: + configsuite.ConfigSuite({"key": 42}, schema) + self.assertIn("Required can only be used for BasicType", str(err.exception)) + + def test_dict_cannot_spec_required(self): + schema = { + MK.Type: configsuite.types.Dict, + MK.Content: { + MK.Key: {MK.Type: configsuite.types.String}, + MK.Value: {MK.Type: configsuite.types.String}, + }, + } + config_suite = configsuite.ConfigSuite({}, schema) + self.assertTrue(config_suite.valid) + + for req in (True, False): + schema[MK.Required] = req + with self.assertRaises(ValueError) as err: + configsuite.ConfigSuite({}, schema) + self.assertIn("Required can only be used for BasicType", str(err.exception)) + + def test_dict_disallow_empty(self): + schema = { + MK.Type: configsuite.types.Dict, + MK.AllowEmpty: False, + MK.Content: { + MK.Key: {MK.Type: configsuite.types.String}, + MK.Value: {MK.Type: configsuite.types.Integer}, + }, + } + + non_empty_suite = configsuite.ConfigSuite({"a": 1, "b": 2}, schema) + self.assertTrue(non_empty_suite.valid) + + empty_suite = configsuite.ConfigSuite({}, schema) + self.assertFalse(empty_suite.valid) + self.assertEqual(1, len(empty_suite.errors)) + self.assertIn("Expected non-empty container", empty_suite.errors[0].msg) + + def test_allow_empty_named_dict_invalid(self): + for val in [True, False]: + schema = { + MK.Type: configsuite.types.NamedDict, + MK.AllowEmpty: val, + MK.Content: {"a": {MK.Type: configsuite.types.Integer}}, + } + + with self.assertRaises(ValueError) as err: + configsuite.ConfigSuite("fdsfds", schema) + self.assertIn( + "Only variable length containers can specify AllowEmpty is false", + str(err.exception), + )