From f0d561e2b4f1533b472b15801e42c6df0cbbb889 Mon Sep 17 00:00:00 2001 From: Elijah Date: Tue, 25 Jul 2023 22:06:31 +0000 Subject: [PATCH 01/13] WIP: Open Knowledge Validator --- tools/ok_validator/__init__.py | 1 + tools/ok_validator/okh.py | 15 +++++++++++ tools/ok_validator/okw.py | 0 tools/ok_validator/validate.py | 47 ++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 tools/ok_validator/__init__.py create mode 100644 tools/ok_validator/okh.py create mode 100644 tools/ok_validator/okw.py create mode 100644 tools/ok_validator/validate.py diff --git a/tools/ok_validator/__init__.py b/tools/ok_validator/__init__.py new file mode 100644 index 0000000..966a9e0 --- /dev/null +++ b/tools/ok_validator/__init__.py @@ -0,0 +1 @@ +from .okh import OKHValidator \ No newline at end of file diff --git a/tools/ok_validator/okh.py b/tools/ok_validator/okh.py new file mode 100644 index 0000000..edbe95d --- /dev/null +++ b/tools/ok_validator/okh.py @@ -0,0 +1,15 @@ +from .validate import OKValidator + +__REQUIRED_FIELDS__ = [ + "bom", + "tools", + "title", + "description", + "intended-output", + "intended-use" +] + +class OKHValidator(OKValidator): + + def __int__(self): + super().__int__(__REQUIRED_FIELDS__) diff --git a/tools/ok_validator/okw.py b/tools/ok_validator/okw.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/ok_validator/validate.py b/tools/ok_validator/validate.py new file mode 100644 index 0000000..4f2c626 --- /dev/null +++ b/tools/ok_validator/validate.py @@ -0,0 +1,47 @@ +"""Open knowledge validator""" +import yaml +from pathlib import Path +from typing import List, Union + + +class OKValidator: + def __init__(self, required_fields: List[str]): + self.required_fields = required_fields + + def _validate_yaml(self, yaml_content: dict) -> bool: + """ + Validate the YAML content to check if it contains the required fields. + """ + for field in self.required_fields: + if field not in yaml_content: + return False + return True + + def validate(self, src: Union[str, Path, dict]) -> bool: + """ + Validate the YAML source, which can be a file path, YAML content string, Path object, or YAML dictionary. + """ + if not isinstance(src, Union[str, dict]): + raise TypeError("src should be one of the following: a string path, a yaml string content, " + "a Path object, or Yaml dict") + + if isinstance(src, dict): + return self._validate_yaml(src) + + if isinstance(src, Path): + src = str(src) + + try: + with open(src, 'r') as yaml_file: + yaml_content = yaml.safe_load(yaml_file) + if yaml_content is None: + print("Error: The YAML file is empty or contains invalid syntax.") + return False + else: + return self._validate_yaml(yaml_content) + except FileNotFoundError: + print("Error: File not found. Please provide a valid YAML file path.") + return False + except yaml.YAMLError as e: + print(f"Error: Invalid YAML syntax. {e}") + return False From 1577afc87194d58115860136a279c8c1a71944e3 Mon Sep 17 00:00:00 2001 From: Elijah Date: Tue, 25 Jul 2023 22:11:39 +0000 Subject: [PATCH 02/13] fix typo --- tools/ok_validator/okh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ok_validator/okh.py b/tools/ok_validator/okh.py index edbe95d..01852c9 100644 --- a/tools/ok_validator/okh.py +++ b/tools/ok_validator/okh.py @@ -12,4 +12,4 @@ class OKHValidator(OKValidator): def __int__(self): - super().__int__(__REQUIRED_FIELDS__) + super().__init__(__REQUIRED_FIELDS__) From acf5cd1216e05430e50af71c51a0edc8de580ff8 Mon Sep 17 00:00:00 2001 From: Elijah Date: Thu, 3 Aug 2023 00:05:49 +0000 Subject: [PATCH 03/13] add more validation checks --- tools/ok_validator/__init__.py | 1 - tools/ok_validator/okh.py | 8 ++--- tools/ok_validator/validate.py | 63 ++++++++++++++++++++++++++++------ 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/tools/ok_validator/__init__.py b/tools/ok_validator/__init__.py index 966a9e0..e69de29 100644 --- a/tools/ok_validator/__init__.py +++ b/tools/ok_validator/__init__.py @@ -1 +0,0 @@ -from .okh import OKHValidator \ No newline at end of file diff --git a/tools/ok_validator/okh.py b/tools/ok_validator/okh.py index 01852c9..d154d07 100644 --- a/tools/ok_validator/okh.py +++ b/tools/ok_validator/okh.py @@ -2,14 +2,10 @@ __REQUIRED_FIELDS__ = [ "bom", - "tools", "title", - "description", - "intended-output", - "intended-use" ] -class OKHValidator(OKValidator): - def __int__(self): +class OKHValidator(OKValidator): + def __init__(self): super().__init__(__REQUIRED_FIELDS__) diff --git a/tools/ok_validator/validate.py b/tools/ok_validator/validate.py index 4f2c626..a40399c 100644 --- a/tools/ok_validator/validate.py +++ b/tools/ok_validator/validate.py @@ -2,6 +2,9 @@ import yaml from pathlib import Path from typing import List, Union +from collections import namedtuple + +Error = namedtuple("Error", ["type", "msg"]) class OKValidator: @@ -17,13 +20,21 @@ def _validate_yaml(self, yaml_content: dict) -> bool: return False return True - def validate(self, src: Union[str, Path, dict]) -> bool: + def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: """ - Validate the YAML source, which can be a file path, YAML content string, Path object, or YAML dictionary. + Validate the YAML source, which can be a file path, YAML content string, + Path object, or YAML dictionary. """ if not isinstance(src, Union[str, dict]): - raise TypeError("src should be one of the following: a string path, a yaml string content, " - "a Path object, or Yaml dict") + return self.return_value_or_error( + Error( + ValueError, + "src should be one of the following: a string path, " + "a yaml string content," + "a Path object, or Yaml dict", + ), + raise_exception, + ) if isinstance(src, dict): return self._validate_yaml(src) @@ -32,16 +43,46 @@ def validate(self, src: Union[str, Path, dict]) -> bool: src = str(src) try: - with open(src, 'r') as yaml_file: + with open(src, "r") as yaml_file: yaml_content = yaml.safe_load(yaml_file) if yaml_content is None: - print("Error: The YAML file is empty or contains invalid syntax.") - return False + return self.return_value_or_error( + Error( + ValueError, + "The YAML file is empty or contains invalid " "syntax.", + ), + raise_exception, + ) else: return self._validate_yaml(yaml_content) except FileNotFoundError: - print("Error: File not found. Please provide a valid YAML file path.") - return False - except yaml.YAMLError as e: - print(f"Error: Invalid YAML syntax. {e}") + return self.return_value_or_error( + Error( + FileNotFoundError, + "File not found. Please provide a valid YAML " "file path.", + ), + raise_exception, + ) + except yaml.YAMLError: + return self.return_value_or_error( + Error(yaml.YAMLError, ""), raise_exception + ) + + @staticmethod + def return_value_or_error(result: Error, raise_exception: bool = False): + """Return a bool or raise an exception. + + Args: + result: exception to raise. + raise_exception: If set to true, the provided exception will be raised. + """ + if not raise_exception: return False + + if not isinstance(result, Error): + raise TypeError( + f"result arg needs to be of type,{type(Error)}. " + f"Got {type(result)} instead." + ) + + raise result.type(result.msg) From 502cd3c6402f85c8f487bd1959152da153e17fad Mon Sep 17 00:00:00 2001 From: Elijah Date: Thu, 3 Aug 2023 00:10:56 +0000 Subject: [PATCH 04/13] test files initial commit --- tools/ok_validator/tests/__init__.py | 0 tools/ok_validator/tests/test_okh.py | 3 +++ tools/ok_validator/tests/test_validate.py | 3 +++ 3 files changed, 6 insertions(+) create mode 100644 tools/ok_validator/tests/__init__.py create mode 100644 tools/ok_validator/tests/test_okh.py create mode 100644 tools/ok_validator/tests/test_validate.py diff --git a/tools/ok_validator/tests/__init__.py b/tools/ok_validator/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/ok_validator/tests/test_okh.py b/tools/ok_validator/tests/test_okh.py new file mode 100644 index 0000000..2eea3f3 --- /dev/null +++ b/tools/ok_validator/tests/test_okh.py @@ -0,0 +1,3 @@ +"""Unit tests for okh validator""" + +# TODO: add tests diff --git a/tools/ok_validator/tests/test_validate.py b/tools/ok_validator/tests/test_validate.py new file mode 100644 index 0000000..556b80c --- /dev/null +++ b/tools/ok_validator/tests/test_validate.py @@ -0,0 +1,3 @@ +"""Unit tests for ok validator""" + +# TODO: add tests From 671228cd8535c07a48c5e5a8c93cac18e475e768 Mon Sep 17 00:00:00 2001 From: Elijah Date: Mon, 7 Aug 2023 14:36:11 +0000 Subject: [PATCH 05/13] refactor and added tests --- tools/ok_validator/{ => src}/okh.py | 0 tools/ok_validator/{ => src}/okw.py | 0 tools/ok_validator/{ => src}/validate.py | 4 +- tools/ok_validator/tests/test_validate.py | 106 +++++++++++++++++++++- 4 files changed, 107 insertions(+), 3 deletions(-) rename tools/ok_validator/{ => src}/okh.py (100%) rename tools/ok_validator/{ => src}/okw.py (100%) rename tools/ok_validator/{ => src}/validate.py (96%) diff --git a/tools/ok_validator/okh.py b/tools/ok_validator/src/okh.py similarity index 100% rename from tools/ok_validator/okh.py rename to tools/ok_validator/src/okh.py diff --git a/tools/ok_validator/okw.py b/tools/ok_validator/src/okw.py similarity index 100% rename from tools/ok_validator/okw.py rename to tools/ok_validator/src/okw.py diff --git a/tools/ok_validator/validate.py b/tools/ok_validator/src/validate.py similarity index 96% rename from tools/ok_validator/validate.py rename to tools/ok_validator/src/validate.py index a40399c..311dce0 100644 --- a/tools/ok_validator/validate.py +++ b/tools/ok_validator/src/validate.py @@ -25,12 +25,12 @@ def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: Validate the YAML source, which can be a file path, YAML content string, Path object, or YAML dictionary. """ - if not isinstance(src, Union[str, dict]): + # breakpoint() + if not isinstance(src, Union[str, dict, Path]): return self.return_value_or_error( Error( ValueError, "src should be one of the following: a string path, " - "a yaml string content," "a Path object, or Yaml dict", ), raise_exception, diff --git a/tools/ok_validator/tests/test_validate.py b/tools/ok_validator/tests/test_validate.py index 556b80c..8a923ef 100644 --- a/tools/ok_validator/tests/test_validate.py +++ b/tools/ok_validator/tests/test_validate.py @@ -1,3 +1,107 @@ """Unit tests for ok validator""" -# TODO: add tests +import pytest +from tools.ok_validator.src.validate import OKValidator, Error + + +@pytest.fixture +def okh_string(): + return """ +title: mock okh title. +description: mock description for okh. +bom: mock bom field. + """ + + +@pytest.fixture +def okh_yaml_file(tmp_path, okh_string): + (okh_file := tmp_path / "okh.yaml").touch() + okh_file.write_text(okh_string) + + return okh_file + + +@pytest.fixture +def okh_dict(): + return {"title": "mock okh title.", "description": "mock description for okh.", "bom": "mock bom field."} + + +@pytest.fixture +def ok_validator(): + return OKValidator(["bom", "title"]) + + +@pytest.mark.parametrize( + "fixture", ["okh_yaml_file", "okh_dict"] +) +def test_validate_yaml_file(fixture, request, ok_validator): + """Test that the validate method returns the correct boolean. + + Args: + fixture: fixtures of accepted src formats. + request: pytest request. + ok_validator: Validator instance. + """ + okh = request.getfixturevalue(fixture) + assert ok_validator.validate(okh) + + +def test_validate_with_non_existent_file(tmp_path, ok_validator): + """Test that the validate method returns False when file does not + exist. + + Args: + tmp_path: Test location. + ok_validator: Validator instance. + """ + file = tmp_path / "okh.yaml" + assert not ok_validator.validate(file) + + +def test_validate_with_invalid_file(tmp_path, ok_validator): + """Test that the validate method returns False when an invalid Yaml file + is passed. + + Args: + tmp_path: Test location. + ok_validator: Validator Instance. + """ + (file := tmp_path / "okh.yaml").touch() + assert not ok_validator.validate(file) + + file.write_text("""title: invalid_field: mock title +description: mock description. + """) + + assert not ok_validator.validate(file) + + +def test_validate_raise_exception(tmp_path, ok_validator): + """Test that an exception is raised when the raise_exception parameter + is set to True on the validator method. + + Args: + tmp_path: The test location. + ok_validator: Validator instance. + """ + file = tmp_path / "okh.yaml" + + with pytest.raises(FileNotFoundError): + ok_validator.validate(file, raise_exception=True) + + +def test_return_value_or_error(): + """Test that the right exception is raised.""" + + error = Error(type=ValueError, msg="raised a value error") + with pytest.raises(error.type) as err: + OKValidator.return_value_or_error(error, raise_exception=True) + assert err.value.args[0] == error.msg + + +def test_return_value_or_error_with_wrong_error_type(): + """Test that an exception is raised when the wrong type is provided + with the `raise_exception` flag set to True. + """ + with pytest.raises(TypeError): + OKValidator.return_value_or_error(str, raise_exception=True) From 4e1fb834527672bfaccb57979d4859d76c980288 Mon Sep 17 00:00:00 2001 From: Elijah Date: Mon, 7 Aug 2023 15:26:27 +0000 Subject: [PATCH 06/13] python 3.7 support for tests --- tools/ok_validator/tests/test_validate.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/ok_validator/tests/test_validate.py b/tools/ok_validator/tests/test_validate.py index 8a923ef..3b86c5e 100644 --- a/tools/ok_validator/tests/test_validate.py +++ b/tools/ok_validator/tests/test_validate.py @@ -15,7 +15,8 @@ def okh_string(): @pytest.fixture def okh_yaml_file(tmp_path, okh_string): - (okh_file := tmp_path / "okh.yaml").touch() + okh_file = tmp_path / "okh.yaml" + okh_file.touch() okh_file.write_text(okh_string) return okh_file @@ -66,7 +67,8 @@ def test_validate_with_invalid_file(tmp_path, ok_validator): tmp_path: Test location. ok_validator: Validator Instance. """ - (file := tmp_path / "okh.yaml").touch() + file = tmp_path / "okh.yaml" + file.touch() assert not ok_validator.validate(file) file.write_text("""title: invalid_field: mock title From ecb22839092bc3e27872494ce6ac468688d54a5b Mon Sep 17 00:00:00 2001 From: Elijah Date: Mon, 7 Aug 2023 15:38:19 +0000 Subject: [PATCH 07/13] add conftest --- tools/ok_validator/tests/conftest.py | 31 +++++++++++++++++++++++ tools/ok_validator/tests/test_validate.py | 28 -------------------- 2 files changed, 31 insertions(+), 28 deletions(-) create mode 100644 tools/ok_validator/tests/conftest.py diff --git a/tools/ok_validator/tests/conftest.py b/tools/ok_validator/tests/conftest.py new file mode 100644 index 0000000..2c5911c --- /dev/null +++ b/tools/ok_validator/tests/conftest.py @@ -0,0 +1,31 @@ +import pytest +from tools.ok_validator.src.validate import OKValidator + + +@pytest.fixture +def okh_string(): + return """ +title: mock okh title. +description: mock description for okh. +bom: mock bom field. + """ + + +@pytest.fixture +def okh_yaml_file(tmp_path, okh_string): + okh_file = tmp_path / "okh.yaml" + okh_file.touch() + okh_file.write_text(okh_string) + + return okh_file + + +@pytest.fixture +def okh_dict(): + return {"title": "mock okh title.", "description": "mock description for okh.", "bom": "mock bom field."} + + +@pytest.fixture +def ok_validator(): + return OKValidator(["bom", "title"]) + diff --git a/tools/ok_validator/tests/test_validate.py b/tools/ok_validator/tests/test_validate.py index 3b86c5e..0f67d8e 100644 --- a/tools/ok_validator/tests/test_validate.py +++ b/tools/ok_validator/tests/test_validate.py @@ -4,34 +4,6 @@ from tools.ok_validator.src.validate import OKValidator, Error -@pytest.fixture -def okh_string(): - return """ -title: mock okh title. -description: mock description for okh. -bom: mock bom field. - """ - - -@pytest.fixture -def okh_yaml_file(tmp_path, okh_string): - okh_file = tmp_path / "okh.yaml" - okh_file.touch() - okh_file.write_text(okh_string) - - return okh_file - - -@pytest.fixture -def okh_dict(): - return {"title": "mock okh title.", "description": "mock description for okh.", "bom": "mock bom field."} - - -@pytest.fixture -def ok_validator(): - return OKValidator(["bom", "title"]) - - @pytest.mark.parametrize( "fixture", ["okh_yaml_file", "okh_dict"] ) From 12711ae7faf9143fd4fffead2761517b7cc6f3c0 Mon Sep 17 00:00:00 2001 From: Elijah Date: Mon, 7 Aug 2023 15:52:37 +0000 Subject: [PATCH 08/13] more tests --- tools/ok_validator/tests/conftest.py | 17 ++++++++++- tools/ok_validator/tests/test_okh.py | 23 ++++++++++++++- tools/ok_validator/tests/test_validate.py | 36 +++++++++++------------ 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/tools/ok_validator/tests/conftest.py b/tools/ok_validator/tests/conftest.py index 2c5911c..f887049 100644 --- a/tools/ok_validator/tests/conftest.py +++ b/tools/ok_validator/tests/conftest.py @@ -1,5 +1,7 @@ import pytest + from tools.ok_validator.src.validate import OKValidator +from tools.ok_validator.src.okh import OKHValidator @pytest.fixture @@ -22,10 +24,23 @@ def okh_yaml_file(tmp_path, okh_string): @pytest.fixture def okh_dict(): - return {"title": "mock okh title.", "description": "mock description for okh.", "bom": "mock bom field."} + return { + "title": "mock okh title.", + "description": "mock description for okh.", + "bom": "mock bom field.", + } + + +@pytest.fixture +def okh_dict_partial(): + return {"title": "mock okh title."} @pytest.fixture def ok_validator(): return OKValidator(["bom", "title"]) + +@pytest.fixture +def okh_validator(): + return OKHValidator() diff --git a/tools/ok_validator/tests/test_okh.py b/tools/ok_validator/tests/test_okh.py index 2eea3f3..a0fdef6 100644 --- a/tools/ok_validator/tests/test_okh.py +++ b/tools/ok_validator/tests/test_okh.py @@ -1,3 +1,24 @@ """Unit tests for okh validator""" -# TODO: add tests + +def test_okh_validate(okh_validator, okh_dict): + """Test that the OKH validator return the correct boolean + on validate. + + Args: + okh_validator: OKHValidator instance. + okh_dict: Yaml dict containing all required OKH fields. + """ + assert okh_validator.required_fields == ["bom", "title"] + assert okh_validator.validate(okh_dict) + + +def test_okh_validate_partial_fields(okh_validator, okh_dict_partial): + """Test that the OKH validator return the correct boolean + on validate. + + Args: + okh_validator: OKHValidator instance. + okh_dict_partial: Yaml dict with missing required OKH fields. + """ + assert not okh_validator.validate(okh_dict_partial) diff --git a/tools/ok_validator/tests/test_validate.py b/tools/ok_validator/tests/test_validate.py index 0f67d8e..3859e7c 100644 --- a/tools/ok_validator/tests/test_validate.py +++ b/tools/ok_validator/tests/test_validate.py @@ -4,9 +4,7 @@ from tools.ok_validator.src.validate import OKValidator, Error -@pytest.mark.parametrize( - "fixture", ["okh_yaml_file", "okh_dict"] -) +@pytest.mark.parametrize("fixture", ["okh_yaml_file", "okh_dict"]) def test_validate_yaml_file(fixture, request, ok_validator): """Test that the validate method returns the correct boolean. @@ -21,11 +19,11 @@ def test_validate_yaml_file(fixture, request, ok_validator): def test_validate_with_non_existent_file(tmp_path, ok_validator): """Test that the validate method returns False when file does not - exist. + exist. - Args: - tmp_path: Test location. - ok_validator: Validator instance. + Args: + tmp_path: Test location. + ok_validator: Validator instance. """ file = tmp_path / "okh.yaml" assert not ok_validator.validate(file) @@ -33,30 +31,32 @@ def test_validate_with_non_existent_file(tmp_path, ok_validator): def test_validate_with_invalid_file(tmp_path, ok_validator): """Test that the validate method returns False when an invalid Yaml file - is passed. + is passed. - Args: - tmp_path: Test location. - ok_validator: Validator Instance. + Args: + tmp_path: Test location. + ok_validator: Validator Instance. """ file = tmp_path / "okh.yaml" file.touch() assert not ok_validator.validate(file) - file.write_text("""title: invalid_field: mock title + file.write_text( + """title: invalid_field: mock title description: mock description. - """) + """ + ) assert not ok_validator.validate(file) def test_validate_raise_exception(tmp_path, ok_validator): """Test that an exception is raised when the raise_exception parameter - is set to True on the validator method. + is set to True on the validator method. - Args: - tmp_path: The test location. - ok_validator: Validator instance. + Args: + tmp_path: The test location. + ok_validator: Validator instance. """ file = tmp_path / "okh.yaml" @@ -75,7 +75,7 @@ def test_return_value_or_error(): def test_return_value_or_error_with_wrong_error_type(): """Test that an exception is raised when the wrong type is provided - with the `raise_exception` flag set to True. + with the `raise_exception` flag set to True. """ with pytest.raises(TypeError): OKValidator.return_value_or_error(str, raise_exception=True) From 35496a294030abe3b843ef4a44c94e630bd9d0dc Mon Sep 17 00:00:00 2001 From: Elijah Date: Tue, 8 Aug 2023 08:01:59 +0000 Subject: [PATCH 09/13] remove dead code --- tools/ok_validator/src/validate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/ok_validator/src/validate.py b/tools/ok_validator/src/validate.py index 311dce0..5b25c7b 100644 --- a/tools/ok_validator/src/validate.py +++ b/tools/ok_validator/src/validate.py @@ -25,7 +25,6 @@ def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: Validate the YAML source, which can be a file path, YAML content string, Path object, or YAML dictionary. """ - # breakpoint() if not isinstance(src, Union[str, dict, Path]): return self.return_value_or_error( Error( From 2a66b7e352856f56b8ad7c281adab20d86e23a4f Mon Sep 17 00:00:00 2001 From: Elijah Date: Tue, 8 Aug 2023 22:19:44 +0000 Subject: [PATCH 10/13] update docstrings --- tools/ok_validator/src/validate.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/tools/ok_validator/src/validate.py b/tools/ok_validator/src/validate.py index 5b25c7b..f7b61f4 100644 --- a/tools/ok_validator/src/validate.py +++ b/tools/ok_validator/src/validate.py @@ -12,8 +12,10 @@ def __init__(self, required_fields: List[str]): self.required_fields = required_fields def _validate_yaml(self, yaml_content: dict) -> bool: - """ - Validate the YAML content to check if it contains the required fields. + """Validate the YAML content to check if it contains the required fields. + + Args: + yaml_content: The parsed yaml dictionary. """ for field in self.required_fields: if field not in yaml_content: @@ -24,12 +26,16 @@ def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: """ Validate the YAML source, which can be a file path, YAML content string, Path object, or YAML dictionary. + + Args: + src: The string path or parsed yaml contents represented as dictionary. + raise_exception: Raises an exception if true otherwise returns a boolean result. """ if not isinstance(src, Union[str, dict, Path]): return self.return_value_or_error( Error( ValueError, - "src should be one of the following: a string path, " + "`src` should be one of the following: a string path, " "a Path object, or Yaml dict", ), raise_exception, @@ -48,7 +54,7 @@ def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: return self.return_value_or_error( Error( ValueError, - "The YAML file is empty or contains invalid " "syntax.", + "The YAML file is empty or contains invalid syntax.", ), raise_exception, ) @@ -64,7 +70,7 @@ def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: ) except yaml.YAMLError: return self.return_value_or_error( - Error(yaml.YAMLError, ""), raise_exception + Error(yaml.YAMLError, "The YAML file is empty or contains invalid syntax."), raise_exception ) @staticmethod @@ -72,7 +78,7 @@ def return_value_or_error(result: Error, raise_exception: bool = False): """Return a bool or raise an exception. Args: - result: exception to raise. + result: Exception to raise. raise_exception: If set to true, the provided exception will be raised. """ if not raise_exception: From b0240a7759660d7f534cd5f0da38bec4af465997 Mon Sep 17 00:00:00 2001 From: Elijah Date: Sun, 22 Oct 2023 10:48:48 +0000 Subject: [PATCH 11/13] use pydantic-yaml --- tools/ok_validator/src/okh.py | 12 ++-- tools/ok_validator/src/validate.py | 78 +++++----------------- tools/ok_validator/tests/conftest.py | 11 +-- tools/ok_validator/tests/test_okh.py | 9 +-- tools/ok_validator/tests/test_validate.py | 81 ----------------------- 5 files changed, 33 insertions(+), 158 deletions(-) delete mode 100644 tools/ok_validator/tests/test_validate.py diff --git a/tools/ok_validator/src/okh.py b/tools/ok_validator/src/okh.py index d154d07..4b94c75 100644 --- a/tools/ok_validator/src/okh.py +++ b/tools/ok_validator/src/okh.py @@ -1,11 +1,13 @@ +from pydantic import BaseModel + from .validate import OKValidator -__REQUIRED_FIELDS__ = [ - "bom", - "title", -] + +class OKH(BaseModel, extra="allow"): + title: str + bom: list[str] class OKHValidator(OKValidator): def __init__(self): - super().__init__(__REQUIRED_FIELDS__) + super().__init__(OKH) diff --git a/tools/ok_validator/src/validate.py b/tools/ok_validator/src/validate.py index f7b61f4..ddec2d5 100644 --- a/tools/ok_validator/src/validate.py +++ b/tools/ok_validator/src/validate.py @@ -1,26 +1,16 @@ """Open knowledge validator""" -import yaml +import json from pathlib import Path -from typing import List, Union -from collections import namedtuple +from typing import Type, TypeVar, Union +from pydantic import BaseModel +from pydantic_yaml import parse_yaml_file_as, parse_yaml_raw_as -Error = namedtuple("Error", ["type", "msg"]) +T = TypeVar("T", bound=BaseModel) class OKValidator: - def __init__(self, required_fields: List[str]): - self.required_fields = required_fields - - def _validate_yaml(self, yaml_content: dict) -> bool: - """Validate the YAML content to check if it contains the required fields. - - Args: - yaml_content: The parsed yaml dictionary. - """ - for field in self.required_fields: - if field not in yaml_content: - return False - return True + def __init__(self, model_type: Type[T]): + self.model_type = model_type def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: """ @@ -33,61 +23,29 @@ def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: """ if not isinstance(src, Union[str, dict, Path]): return self.return_value_or_error( - Error( - ValueError, + ValueError( "`src` should be one of the following: a string path, " "a Path object, or Yaml dict", ), raise_exception, ) - - if isinstance(src, dict): - return self._validate_yaml(src) - - if isinstance(src, Path): - src = str(src) - try: - with open(src, "r") as yaml_file: - yaml_content = yaml.safe_load(yaml_file) - if yaml_content is None: - return self.return_value_or_error( - Error( - ValueError, - "The YAML file is empty or contains invalid syntax.", - ), - raise_exception, - ) - else: - return self._validate_yaml(yaml_content) - except FileNotFoundError: - return self.return_value_or_error( - Error( - FileNotFoundError, - "File not found. Please provide a valid YAML " "file path.", - ), - raise_exception, - ) - except yaml.YAMLError: - return self.return_value_or_error( - Error(yaml.YAMLError, "The YAML file is empty or contains invalid syntax."), raise_exception - ) + if isinstance(src, dict): + parse_yaml_raw_as(self.model_type, json.dumps(src)) + else: + parse_yaml_file_as(self.model_type, src) + return True + except Exception as err: + return self.return_value_or_error(err, raise_exception) @staticmethod - def return_value_or_error(result: Error, raise_exception: bool = False): + def return_value_or_error(error: Exception, raise_exception: bool = False): """Return a bool or raise an exception. Args: - result: Exception to raise. + error: Exception to raise. raise_exception: If set to true, the provided exception will be raised. """ if not raise_exception: return False - - if not isinstance(result, Error): - raise TypeError( - f"result arg needs to be of type,{type(Error)}. " - f"Got {type(result)} instead." - ) - - raise result.type(result.msg) + raise error diff --git a/tools/ok_validator/tests/conftest.py b/tools/ok_validator/tests/conftest.py index f887049..b0c9e20 100644 --- a/tools/ok_validator/tests/conftest.py +++ b/tools/ok_validator/tests/conftest.py @@ -1,6 +1,5 @@ import pytest -from tools.ok_validator.src.validate import OKValidator from tools.ok_validator.src.okh import OKHValidator @@ -9,7 +8,8 @@ def okh_string(): return """ title: mock okh title. description: mock description for okh. -bom: mock bom field. +bom: + - mock bom field. """ @@ -27,7 +27,7 @@ def okh_dict(): return { "title": "mock okh title.", "description": "mock description for okh.", - "bom": "mock bom field.", + "bom": ["mock bom field."], } @@ -36,11 +36,6 @@ def okh_dict_partial(): return {"title": "mock okh title."} -@pytest.fixture -def ok_validator(): - return OKValidator(["bom", "title"]) - - @pytest.fixture def okh_validator(): return OKHValidator() diff --git a/tools/ok_validator/tests/test_okh.py b/tools/ok_validator/tests/test_okh.py index a0fdef6..25e6fb9 100644 --- a/tools/ok_validator/tests/test_okh.py +++ b/tools/ok_validator/tests/test_okh.py @@ -1,16 +1,17 @@ """Unit tests for okh validator""" +import pytest -def test_okh_validate(okh_validator, okh_dict): +@pytest.mark.parametrize("src_fixture", ["okh_yaml_file", "okh_dict"]) +def test_okh_validate(okh_validator, src_fixture, request): """Test that the OKH validator return the correct boolean on validate. Args: okh_validator: OKHValidator instance. - okh_dict: Yaml dict containing all required OKH fields. + src_fixture: src dict or file containing all required OKH fields. """ - assert okh_validator.required_fields == ["bom", "title"] - assert okh_validator.validate(okh_dict) + assert okh_validator.validate(request.getfixturevalue(src_fixture)) def test_okh_validate_partial_fields(okh_validator, okh_dict_partial): diff --git a/tools/ok_validator/tests/test_validate.py b/tools/ok_validator/tests/test_validate.py deleted file mode 100644 index 3859e7c..0000000 --- a/tools/ok_validator/tests/test_validate.py +++ /dev/null @@ -1,81 +0,0 @@ -"""Unit tests for ok validator""" - -import pytest -from tools.ok_validator.src.validate import OKValidator, Error - - -@pytest.mark.parametrize("fixture", ["okh_yaml_file", "okh_dict"]) -def test_validate_yaml_file(fixture, request, ok_validator): - """Test that the validate method returns the correct boolean. - - Args: - fixture: fixtures of accepted src formats. - request: pytest request. - ok_validator: Validator instance. - """ - okh = request.getfixturevalue(fixture) - assert ok_validator.validate(okh) - - -def test_validate_with_non_existent_file(tmp_path, ok_validator): - """Test that the validate method returns False when file does not - exist. - - Args: - tmp_path: Test location. - ok_validator: Validator instance. - """ - file = tmp_path / "okh.yaml" - assert not ok_validator.validate(file) - - -def test_validate_with_invalid_file(tmp_path, ok_validator): - """Test that the validate method returns False when an invalid Yaml file - is passed. - - Args: - tmp_path: Test location. - ok_validator: Validator Instance. - """ - file = tmp_path / "okh.yaml" - file.touch() - assert not ok_validator.validate(file) - - file.write_text( - """title: invalid_field: mock title -description: mock description. - """ - ) - - assert not ok_validator.validate(file) - - -def test_validate_raise_exception(tmp_path, ok_validator): - """Test that an exception is raised when the raise_exception parameter - is set to True on the validator method. - - Args: - tmp_path: The test location. - ok_validator: Validator instance. - """ - file = tmp_path / "okh.yaml" - - with pytest.raises(FileNotFoundError): - ok_validator.validate(file, raise_exception=True) - - -def test_return_value_or_error(): - """Test that the right exception is raised.""" - - error = Error(type=ValueError, msg="raised a value error") - with pytest.raises(error.type) as err: - OKValidator.return_value_or_error(error, raise_exception=True) - assert err.value.args[0] == error.msg - - -def test_return_value_or_error_with_wrong_error_type(): - """Test that an exception is raised when the wrong type is provided - with the `raise_exception` flag set to True. - """ - with pytest.raises(TypeError): - OKValidator.return_value_or_error(str, raise_exception=True) From dd7161f870bb5ec730c7b078943e62a12d823792 Mon Sep 17 00:00:00 2001 From: Elijah Date: Sun, 29 Oct 2023 16:26:29 +0000 Subject: [PATCH 12/13] add pyproject file and readme --- tools/ok_validator/README.md | 2 ++ tools/ok_validator/pyproject.toml | 0 2 files changed, 2 insertions(+) create mode 100644 tools/ok_validator/README.md create mode 100644 tools/ok_validator/pyproject.toml diff --git a/tools/ok_validator/README.md b/tools/ok_validator/README.md new file mode 100644 index 0000000..667eeca --- /dev/null +++ b/tools/ok_validator/README.md @@ -0,0 +1,2 @@ +## Open Knowledge Validator + diff --git a/tools/ok_validator/pyproject.toml b/tools/ok_validator/pyproject.toml new file mode 100644 index 0000000..e69de29 From bc695d34080f3728a7d1c3fe881d44f868c88044 Mon Sep 17 00:00:00 2001 From: Elijah Date: Sun, 29 Oct 2023 16:29:53 +0000 Subject: [PATCH 13/13] update pyproject file --- tools/ok_validator/pyproject.toml | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tools/ok_validator/pyproject.toml b/tools/ok_validator/pyproject.toml index e69de29..e0b2699 100644 --- a/tools/ok_validator/pyproject.toml +++ b/tools/ok_validator/pyproject.toml @@ -0,0 +1,31 @@ +[tool.poetry] +name="okvalidator" +version="0.1.0" +description="An Open Knowledge validator" +authors = [ + "Helpful Engineering", +] +readme="README.md" + +repository = "https://github.com/helpfulengineering/project-data-platform/tools/ok_validator" + +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Software Development :: Open Knowledge :: Supply Chain", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", +] + + +[tool.poetry.dependencies] +python = "^3.7" +pyyaml = "6.0" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.2.1" \ No newline at end of file