From 0d7643a4549497eac252bbd3d3cf56539254b0d1 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 3 Dec 2024 12:47:50 -0500 Subject: [PATCH] Config file: use pydantic for all config models (#11798) We aren't using the whole power of pydantic, but it's a good start. --- readthedocs/config/config.py | 22 ++----- readthedocs/config/models.py | 114 ++++++++++++++++------------------- readthedocs/config/utils.py | 11 ---- 3 files changed, 56 insertions(+), 91 deletions(-) diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index 53690534716..e1cbbd60b33 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -7,8 +7,9 @@ from functools import lru_cache from django.conf import settings +from pydantic import BaseModel -from readthedocs.config.utils import list_to_dict, to_dict +from readthedocs.config.utils import list_to_dict from readthedocs.core.utils.filesystem import safe_open from readthedocs.projects.constants import GENERIC @@ -23,7 +24,6 @@ Mkdocs, Python, PythonInstall, - PythonInstallRequirements, Search, Sphinx, Submodules, @@ -207,7 +207,7 @@ def as_dict(self): config = {} for name in self.PUBLIC_ATTRIBUTES: attr = getattr(self, name) - config[name] = to_dict(attr) + config[name] = attr.model_dump() if isinstance(attr, BaseModel) else attr return config def __getattr__(self, name): @@ -793,21 +793,7 @@ def build(self): @property def python(self): - python_install = [] - python = self._config["python"] - for install in python["install"]: - if "requirements" in install: - python_install.append( - PythonInstallRequirements(**install), - ) - elif "path" in install: - python_install.append( - PythonInstall(**install), - ) - - return Python( - install=python_install, - ) + return Python(**self._config["python"]) @property def sphinx(self): diff --git a/readthedocs/config/models.py b/readthedocs/config/models.py index c96e2462586..5d63e11ad90 100644 --- a/readthedocs/config/models.py +++ b/readthedocs/config/models.py @@ -1,45 +1,24 @@ -"""Models for the response of the configuration object.""" -from pydantic import BaseModel - -from readthedocs.config.utils import to_dict - - -class Base: - - """ - Base class for every configuration. - - Each inherited class should define - its attributes in the `__slots__` attribute. - - We are using `__slots__` so we can't add more attributes by mistake, - this is similar to a namedtuple. - """ - - def __init__(self, **kwargs): - for name in self.__slots__: - setattr(self, name, kwargs[name]) +""" +Models for the response of the configuration object. - def as_dict(self): - return {name: to_dict(getattr(self, name)) for name in self.__slots__} +We make use of pydantic to define the models/dataclasses for all the +options that the user can define in the configuration file. +Pydantic does runtime type checking and validation, +but we aren't using it yet, and instead we are doing the validation +in a separate step. +""" +from typing import Literal -# TODO: rename this class to `Build` -class BuildWithOs(Base): - __slots__ = ("os", "tools", "jobs", "apt_packages", "commands") - - def __init__(self, **kwargs): - kwargs.setdefault("apt_packages", []) - kwargs.setdefault("commands", []) - super().__init__(**kwargs) +from pydantic import BaseModel -class BuildTool(Base): - __slots__ = ("version", "full_version") +class BuildTool(BaseModel): + version: str + full_version: str class BuildJobsBuildTypes(BaseModel): - """Object used for `build.jobs.build` key.""" html: list[str] | None = None @@ -47,13 +26,8 @@ class BuildJobsBuildTypes(BaseModel): epub: list[str] | None = None htmlzip: list[str] | None = None - def as_dict(self): - # Just to keep compatibility with the old implementation. - return self.model_dump() - class BuildJobs(BaseModel): - """Object used for `build.jobs` key.""" pre_checkout: list[str] = [] @@ -70,42 +44,58 @@ class BuildJobs(BaseModel): build: BuildJobsBuildTypes = BuildJobsBuildTypes() post_build: list[str] = [] - def as_dict(self): - # Just to keep compatibility with the old implementation. - return self.model_dump() + +# TODO: rename this class to `Build` +class BuildWithOs(BaseModel): + os: str + tools: dict[str, BuildTool] + jobs: BuildJobs = BuildJobs() + apt_packages: list[str] = [] + commands: list[str] = [] -class Python(Base): - __slots__ = ("install",) +class PythonInstallRequirements(BaseModel): + requirements: str -class PythonInstallRequirements(Base): - __slots__ = ("requirements",) +class PythonInstall(BaseModel): + path: str + method: Literal["pip", "setuptools"] = "pip" + extra_requirements: list[str] = [] -class PythonInstall(Base): - __slots__ = ( - "path", - "method", - "extra_requirements", - ) +class Python(BaseModel): + install: list[PythonInstall | PythonInstallRequirements] = [] -class Conda(Base): - __slots__ = ("environment",) +class Conda(BaseModel): + environment: str -class Sphinx(Base): - __slots__ = ("builder", "configuration", "fail_on_warning") +class Sphinx(BaseModel): + configuration: str | None + # NOTE: This is how we save the object in the DB, + # the actual options for users are "html", "htmldir", "singlehtml". + builder: Literal["sphinx", "sphinx_htmldir", "sphinx_singlehtml"] = "sphinx" + fail_on_warning: bool = False -class Mkdocs(Base): - __slots__ = ("configuration", "fail_on_warning") +class Mkdocs(BaseModel): + configuration: str | None + fail_on_warning: bool = False -class Submodules(Base): - __slots__ = ("include", "exclude", "recursive") +class Submodules(BaseModel): + include: list[str] | Literal["all"] = [] + exclude: list[str] | Literal["all"] = [] + recursive: bool = False -class Search(Base): - __slots__ = ("ranking", "ignore") +class Search(BaseModel): + ranking: dict[str, int] = {} + ignore: list[str] = [ + "search.html", + "search/index.html", + "404.html", + "404/index.html", + ] diff --git a/readthedocs/config/utils.py b/readthedocs/config/utils.py index da9dd666c77..52014ef0475 100644 --- a/readthedocs/config/utils.py +++ b/readthedocs/config/utils.py @@ -1,17 +1,6 @@ """Shared functions for the config module.""" -def to_dict(value): - """Recursively transform a class from `config.models` to a dict.""" - if hasattr(value, "as_dict"): - return value.as_dict() - if isinstance(value, list): - return [to_dict(v) for v in value] - if isinstance(value, dict): - return {k: to_dict(v) for k, v in value.items()} - return value - - def list_to_dict(list_): """Transform a list to a dictionary with its indices as keys.""" dict_ = {str(i): element for i, element in enumerate(list_)}