tests: validate schemas in regular testing (#450)

Followup to #447
based on testing suggestion from @abravalheri.

Signed-off-by: Henry Schreiner <[email protected]>

.github/workflows/ci.yml

@@ -89,7 +89,7 @@ jobs:
           allow-prereleases: true
 
       - name: Install package
-        run: pip install -e.[test,test-meta,test-numpy,wheels,cov]
+        run: pip install -e.[test,test-meta,test-numpy,test-schema,wheels,cov]
 
       - name: Test package
         run: >-

noxfile.py

@@ -69,23 +69,12 @@ def generate_schema(session: nox.Session) -> None:
     DIR.joinpath("src/scikit_build_core/resources/scikit-build.schema.json").write_text(schema)  # type: ignore[arg-type]
 
 
-@nox.session
-def validate_schemas(session: nox.Session) -> None:
-    session.install("-e.", "validate-pyproject")
-
-    docs_examples = Path("docs/examples").glob("**/pyproject.toml")
-    tests_packages = Path("tests/packages").glob("**/pyproject.toml")
-    paths = [*docs_examples, *tests_packages]
-    # map currently required to make mypy happy, future nox version should fix
-    session.run("validate-pyproject", *map(str, paths))
-
-
 @nox.session
 def tests(session: nox.Session) -> None:
     """
     Run the unit and regular tests. Includes coverage if --cov passed.
     """
-    _run_tests(session, extras=["test-meta,test-numpy"])
+    _run_tests(session, extras=["test-meta,test-numpy,test-schema"])
 
 
 @nox.session

pyproject.toml

@@ -70,6 +70,10 @@ test-numpy = [
     "numpy~=1.24.0; python_version=='3.8' and platform_python_implementation=='PyPy'",
     "numpy~=1.25.0; python_version=='3.9' and platform_python_implementation=='PyPy'",
 ]
+test-schema = [
+    "fastjsonschema",
+    "validate-pyproject",
+]
 cov = [
     "pytest-cov[toml]",
 ]

src/scikit_build_core/_compat/tomllib.py

@@ -3,11 +3,11 @@
 import sys
 
 if sys.version_info < (3, 11):
-    from tomli import load
+    from tomli import load, loads
 else:
-    from tomllib import load
+    from tomllib import load, loads
 
-__all__ = ["load"]
+__all__ = ["load", "loads"]
 
 
 def __dir__() -> list[str]:

tests/test_schema.py

@@ -1,12 +1,63 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from scikit_build_core._compat import tomllib
 from scikit_build_core.settings.skbuild_schema import (
     generate_skbuild_schema,
     get_skbuild_schema,
 )
 
+DIR = Path(__file__).parent.resolve()
+
 
 def test_compare_schemas():
     """
     Should be the same. If not, run nox -s generate_schema
     """
 
     assert generate_skbuild_schema() == get_skbuild_schema()
+
+
+@pytest.mark.parametrize(
+    "filepath",
+    [
+        *DIR.parent.joinpath("docs/examples").glob("**/pyproject.toml"),
+        *DIR.joinpath("packages").glob("**/pyproject.toml"),
+    ],
+)
+def test_valid_schemas_files(filepath: Path) -> None:
+    api = pytest.importorskip("validate_pyproject.api")
+
+    with filepath.open("rb") as f:
+        example = tomllib.load(f)
+
+    validator = api.Validator()
+    assert validator(example) is not None
+
+
+@pytest.mark.parametrize(
+    "addition", [{"minimum-version": 0.3}, {"random": "not valid"}]
+)
+def test_invalid_schemas(addition: dict[str, Any]) -> None:
+    fastjsonschema = pytest.importorskip("fastjsonschema")
+    api = pytest.importorskip("validate_pyproject.api")
+
+    example_toml = """\
+    [project]
+    name = "myproj"
+    version = "0"
+
+    [tool.scikit-build]
+    minimum-version = "0.3"
+    """
+
+    example = tomllib.loads(example_toml)
+    example["tool"]["scikit-build"].update(**addition)
+
+    validator = api.Validator()
+    with pytest.raises(fastjsonschema.JsonSchemaValueException):
+        validator(example)