From f0d561e2b4f1533b472b15801e42c6df0cbbb889 Mon Sep 17 00:00:00 2001 From: Elijah Date: Tue, 25 Jul 2023 22:06:31 +0000 Subject: [PATCH 1/7] WIP: Open Knowledge Validator --- tools/ok_validator/__init__.py | 1 + tools/ok_validator/okh.py | 15 +++++++++++ tools/ok_validator/okw.py | 0 tools/ok_validator/validate.py | 47 ++++++++++++++++++++++++++++++++++ 4 files changed, 63 insertions(+) create mode 100644 tools/ok_validator/__init__.py create mode 100644 tools/ok_validator/okh.py create mode 100644 tools/ok_validator/okw.py create mode 100644 tools/ok_validator/validate.py diff --git a/tools/ok_validator/__init__.py b/tools/ok_validator/__init__.py new file mode 100644 index 0000000..966a9e0 --- /dev/null +++ b/tools/ok_validator/__init__.py @@ -0,0 +1 @@ +from .okh import OKHValidator \ No newline at end of file diff --git a/tools/ok_validator/okh.py b/tools/ok_validator/okh.py new file mode 100644 index 0000000..edbe95d --- /dev/null +++ b/tools/ok_validator/okh.py @@ -0,0 +1,15 @@ +from .validate import OKValidator + +__REQUIRED_FIELDS__ = [ + "bom", + "tools", + "title", + "description", + "intended-output", + "intended-use" +] + +class OKHValidator(OKValidator): + + def __int__(self): + super().__int__(__REQUIRED_FIELDS__) diff --git a/tools/ok_validator/okw.py b/tools/ok_validator/okw.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/ok_validator/validate.py b/tools/ok_validator/validate.py new file mode 100644 index 0000000..4f2c626 --- /dev/null +++ b/tools/ok_validator/validate.py @@ -0,0 +1,47 @@ +"""Open knowledge validator""" +import yaml +from pathlib import Path +from typing import List, Union + + +class OKValidator: + def __init__(self, required_fields: List[str]): + self.required_fields = required_fields + + def _validate_yaml(self, yaml_content: dict) -> bool: + """ + Validate the YAML content to check if it contains the required fields. + """ + for field in self.required_fields: + if field not in yaml_content: + return False + return True + + def validate(self, src: Union[str, Path, dict]) -> bool: + """ + Validate the YAML source, which can be a file path, YAML content string, Path object, or YAML dictionary. + """ + if not isinstance(src, Union[str, dict]): + raise TypeError("src should be one of the following: a string path, a yaml string content, " + "a Path object, or Yaml dict") + + if isinstance(src, dict): + return self._validate_yaml(src) + + if isinstance(src, Path): + src = str(src) + + try: + with open(src, 'r') as yaml_file: + yaml_content = yaml.safe_load(yaml_file) + if yaml_content is None: + print("Error: The YAML file is empty or contains invalid syntax.") + return False + else: + return self._validate_yaml(yaml_content) + except FileNotFoundError: + print("Error: File not found. Please provide a valid YAML file path.") + return False + except yaml.YAMLError as e: + print(f"Error: Invalid YAML syntax. {e}") + return False From 1577afc87194d58115860136a279c8c1a71944e3 Mon Sep 17 00:00:00 2001 From: Elijah Date: Tue, 25 Jul 2023 22:11:39 +0000 Subject: [PATCH 2/7] fix typo --- tools/ok_validator/okh.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ok_validator/okh.py b/tools/ok_validator/okh.py index edbe95d..01852c9 100644 --- a/tools/ok_validator/okh.py +++ b/tools/ok_validator/okh.py @@ -12,4 +12,4 @@ class OKHValidator(OKValidator): def __int__(self): - super().__int__(__REQUIRED_FIELDS__) + super().__init__(__REQUIRED_FIELDS__) From acf5cd1216e05430e50af71c51a0edc8de580ff8 Mon Sep 17 00:00:00 2001 From: Elijah Date: Thu, 3 Aug 2023 00:05:49 +0000 Subject: [PATCH 3/7] add more validation checks --- tools/ok_validator/__init__.py | 1 - tools/ok_validator/okh.py | 8 ++--- tools/ok_validator/validate.py | 63 ++++++++++++++++++++++++++++------ 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/tools/ok_validator/__init__.py b/tools/ok_validator/__init__.py index 966a9e0..e69de29 100644 --- a/tools/ok_validator/__init__.py +++ b/tools/ok_validator/__init__.py @@ -1 +0,0 @@ -from .okh import OKHValidator \ No newline at end of file diff --git a/tools/ok_validator/okh.py b/tools/ok_validator/okh.py index 01852c9..d154d07 100644 --- a/tools/ok_validator/okh.py +++ b/tools/ok_validator/okh.py @@ -2,14 +2,10 @@ __REQUIRED_FIELDS__ = [ "bom", - "tools", "title", - "description", - "intended-output", - "intended-use" ] -class OKHValidator(OKValidator): - def __int__(self): +class OKHValidator(OKValidator): + def __init__(self): super().__init__(__REQUIRED_FIELDS__) diff --git a/tools/ok_validator/validate.py b/tools/ok_validator/validate.py index 4f2c626..a40399c 100644 --- a/tools/ok_validator/validate.py +++ b/tools/ok_validator/validate.py @@ -2,6 +2,9 @@ import yaml from pathlib import Path from typing import List, Union +from collections import namedtuple + +Error = namedtuple("Error", ["type", "msg"]) class OKValidator: @@ -17,13 +20,21 @@ def _validate_yaml(self, yaml_content: dict) -> bool: return False return True - def validate(self, src: Union[str, Path, dict]) -> bool: + def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: """ - Validate the YAML source, which can be a file path, YAML content string, Path object, or YAML dictionary. + Validate the YAML source, which can be a file path, YAML content string, + Path object, or YAML dictionary. """ if not isinstance(src, Union[str, dict]): - raise TypeError("src should be one of the following: a string path, a yaml string content, " - "a Path object, or Yaml dict") + return self.return_value_or_error( + Error( + ValueError, + "src should be one of the following: a string path, " + "a yaml string content," + "a Path object, or Yaml dict", + ), + raise_exception, + ) if isinstance(src, dict): return self._validate_yaml(src) @@ -32,16 +43,46 @@ def validate(self, src: Union[str, Path, dict]) -> bool: src = str(src) try: - with open(src, 'r') as yaml_file: + with open(src, "r") as yaml_file: yaml_content = yaml.safe_load(yaml_file) if yaml_content is None: - print("Error: The YAML file is empty or contains invalid syntax.") - return False + return self.return_value_or_error( + Error( + ValueError, + "The YAML file is empty or contains invalid " "syntax.", + ), + raise_exception, + ) else: return self._validate_yaml(yaml_content) except FileNotFoundError: - print("Error: File not found. Please provide a valid YAML file path.") - return False - except yaml.YAMLError as e: - print(f"Error: Invalid YAML syntax. {e}") + return self.return_value_or_error( + Error( + FileNotFoundError, + "File not found. Please provide a valid YAML " "file path.", + ), + raise_exception, + ) + except yaml.YAMLError: + return self.return_value_or_error( + Error(yaml.YAMLError, ""), raise_exception + ) + + @staticmethod + def return_value_or_error(result: Error, raise_exception: bool = False): + """Return a bool or raise an exception. + + Args: + result: exception to raise. + raise_exception: If set to true, the provided exception will be raised. + """ + if not raise_exception: return False + + if not isinstance(result, Error): + raise TypeError( + f"result arg needs to be of type,{type(Error)}. " + f"Got {type(result)} instead." + ) + + raise result.type(result.msg) From 502cd3c6402f85c8f487bd1959152da153e17fad Mon Sep 17 00:00:00 2001 From: Elijah Date: Thu, 3 Aug 2023 00:10:56 +0000 Subject: [PATCH 4/7] test files initial commit --- tools/ok_validator/tests/__init__.py | 0 tools/ok_validator/tests/test_okh.py | 3 +++ tools/ok_validator/tests/test_validate.py | 3 +++ 3 files changed, 6 insertions(+) create mode 100644 tools/ok_validator/tests/__init__.py create mode 100644 tools/ok_validator/tests/test_okh.py create mode 100644 tools/ok_validator/tests/test_validate.py diff --git a/tools/ok_validator/tests/__init__.py b/tools/ok_validator/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/ok_validator/tests/test_okh.py b/tools/ok_validator/tests/test_okh.py new file mode 100644 index 0000000..2eea3f3 --- /dev/null +++ b/tools/ok_validator/tests/test_okh.py @@ -0,0 +1,3 @@ +"""Unit tests for okh validator""" + +# TODO: add tests diff --git a/tools/ok_validator/tests/test_validate.py b/tools/ok_validator/tests/test_validate.py new file mode 100644 index 0000000..556b80c --- /dev/null +++ b/tools/ok_validator/tests/test_validate.py @@ -0,0 +1,3 @@ +"""Unit tests for ok validator""" + +# TODO: add tests From 671228cd8535c07a48c5e5a8c93cac18e475e768 Mon Sep 17 00:00:00 2001 From: Elijah Date: Mon, 7 Aug 2023 14:36:11 +0000 Subject: [PATCH 5/7] refactor and added tests --- tools/ok_validator/{ => src}/okh.py | 0 tools/ok_validator/{ => src}/okw.py | 0 tools/ok_validator/{ => src}/validate.py | 4 +- tools/ok_validator/tests/test_validate.py | 106 +++++++++++++++++++++- 4 files changed, 107 insertions(+), 3 deletions(-) rename tools/ok_validator/{ => src}/okh.py (100%) rename tools/ok_validator/{ => src}/okw.py (100%) rename tools/ok_validator/{ => src}/validate.py (96%) diff --git a/tools/ok_validator/okh.py b/tools/ok_validator/src/okh.py similarity index 100% rename from tools/ok_validator/okh.py rename to tools/ok_validator/src/okh.py diff --git a/tools/ok_validator/okw.py b/tools/ok_validator/src/okw.py similarity index 100% rename from tools/ok_validator/okw.py rename to tools/ok_validator/src/okw.py diff --git a/tools/ok_validator/validate.py b/tools/ok_validator/src/validate.py similarity index 96% rename from tools/ok_validator/validate.py rename to tools/ok_validator/src/validate.py index a40399c..311dce0 100644 --- a/tools/ok_validator/validate.py +++ b/tools/ok_validator/src/validate.py @@ -25,12 +25,12 @@ def validate(self, src: Union[str, Path, dict], raise_exception=False) -> bool: Validate the YAML source, which can be a file path, YAML content string, Path object, or YAML dictionary. """ - if not isinstance(src, Union[str, dict]): + # breakpoint() + if not isinstance(src, Union[str, dict, Path]): return self.return_value_or_error( Error( ValueError, "src should be one of the following: a string path, " - "a yaml string content," "a Path object, or Yaml dict", ), raise_exception, diff --git a/tools/ok_validator/tests/test_validate.py b/tools/ok_validator/tests/test_validate.py index 556b80c..8a923ef 100644 --- a/tools/ok_validator/tests/test_validate.py +++ b/tools/ok_validator/tests/test_validate.py @@ -1,3 +1,107 @@ """Unit tests for ok validator""" -# TODO: add tests +import pytest +from tools.ok_validator.src.validate import OKValidator, Error + + +@pytest.fixture +def okh_string(): + return """ +title: mock okh title. +description: mock description for okh. +bom: mock bom field. + """ + + +@pytest.fixture +def okh_yaml_file(tmp_path, okh_string): + (okh_file := tmp_path / "okh.yaml").touch() + okh_file.write_text(okh_string) + + return okh_file + + +@pytest.fixture +def okh_dict(): + return {"title": "mock okh title.", "description": "mock description for okh.", "bom": "mock bom field."} + + +@pytest.fixture +def ok_validator(): + return OKValidator(["bom", "title"]) + + +@pytest.mark.parametrize( + "fixture", ["okh_yaml_file", "okh_dict"] +) +def test_validate_yaml_file(fixture, request, ok_validator): + """Test that the validate method returns the correct boolean. + + Args: + fixture: fixtures of accepted src formats. + request: pytest request. + ok_validator: Validator instance. + """ + okh = request.getfixturevalue(fixture) + assert ok_validator.validate(okh) + + +def test_validate_with_non_existent_file(tmp_path, ok_validator): + """Test that the validate method returns False when file does not + exist. + + Args: + tmp_path: Test location. + ok_validator: Validator instance. + """ + file = tmp_path / "okh.yaml" + assert not ok_validator.validate(file) + + +def test_validate_with_invalid_file(tmp_path, ok_validator): + """Test that the validate method returns False when an invalid Yaml file + is passed. + + Args: + tmp_path: Test location. + ok_validator: Validator Instance. + """ + (file := tmp_path / "okh.yaml").touch() + assert not ok_validator.validate(file) + + file.write_text("""title: invalid_field: mock title +description: mock description. + """) + + assert not ok_validator.validate(file) + + +def test_validate_raise_exception(tmp_path, ok_validator): + """Test that an exception is raised when the raise_exception parameter + is set to True on the validator method. + + Args: + tmp_path: The test location. + ok_validator: Validator instance. + """ + file = tmp_path / "okh.yaml" + + with pytest.raises(FileNotFoundError): + ok_validator.validate(file, raise_exception=True) + + +def test_return_value_or_error(): + """Test that the right exception is raised.""" + + error = Error(type=ValueError, msg="raised a value error") + with pytest.raises(error.type) as err: + OKValidator.return_value_or_error(error, raise_exception=True) + assert err.value.args[0] == error.msg + + +def test_return_value_or_error_with_wrong_error_type(): + """Test that an exception is raised when the wrong type is provided + with the `raise_exception` flag set to True. + """ + with pytest.raises(TypeError): + OKValidator.return_value_or_error(str, raise_exception=True) From 4e1fb834527672bfaccb57979d4859d76c980288 Mon Sep 17 00:00:00 2001 From: Elijah Date: Mon, 7 Aug 2023 15:26:27 +0000 Subject: [PATCH 6/7] python 3.7 support for tests --- tools/ok_validator/tests/test_validate.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/ok_validator/tests/test_validate.py b/tools/ok_validator/tests/test_validate.py index 8a923ef..3b86c5e 100644 --- a/tools/ok_validator/tests/test_validate.py +++ b/tools/ok_validator/tests/test_validate.py @@ -15,7 +15,8 @@ def okh_string(): @pytest.fixture def okh_yaml_file(tmp_path, okh_string): - (okh_file := tmp_path / "okh.yaml").touch() + okh_file = tmp_path / "okh.yaml" + okh_file.touch() okh_file.write_text(okh_string) return okh_file @@ -66,7 +67,8 @@ def test_validate_with_invalid_file(tmp_path, ok_validator): tmp_path: Test location. ok_validator: Validator Instance. """ - (file := tmp_path / "okh.yaml").touch() + file = tmp_path / "okh.yaml" + file.touch() assert not ok_validator.validate(file) file.write_text("""title: invalid_field: mock title From 1c1a1c644e820b8bb69b068768e9e5811c26fb35 Mon Sep 17 00:00:00 2001 From: Elijah Date: Mon, 7 Aug 2023 15:32:09 +0000 Subject: [PATCH 7/7] OK validator dependency management with pyproject.toml --- tools/ok_validator/README.md | 1 + tools/ok_validator/pyproject.toml | 33 +++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 tools/ok_validator/README.md create mode 100644 tools/ok_validator/pyproject.toml diff --git a/tools/ok_validator/README.md b/tools/ok_validator/README.md new file mode 100644 index 0000000..1c31d70 --- /dev/null +++ b/tools/ok_validator/README.md @@ -0,0 +1 @@ +# OPen Knowledge Validator \ No newline at end of file diff --git a/tools/ok_validator/pyproject.toml b/tools/ok_validator/pyproject.toml new file mode 100644 index 0000000..96f3c83 --- /dev/null +++ b/tools/ok_validator/pyproject.toml @@ -0,0 +1,33 @@ +[tool.poetry] +name="okvalidator" +version="0.1.0" +description="An Open Knowledge validator" +authors = [ + "Elijah Ahianyo elijahahianyo@gmail.com", +] +readme="README.md" + +repository = "https://github.com/helpfulengineering/project-data-platform/tools/ok_validator" + +classifiers = [ + "Development Status :: 3 - Alpha", + + "Intended Audience :: Developers", + "Topic :: Software Development :: Open Knowledge", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3 :: Only", +] + + +[tool.poetry.dependencies] +python = "^3.7" +pyyaml = "6.0" + +[tool.poetry.group.dev.dependencies] +pytest = "^7.2.1" \ No newline at end of file