From b2d40a07f5b8f65fcab4f03e64d0923a3fa3b658 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 00:10:34 +0200 Subject: [PATCH 01/10] Add sphinx extension --- .pre-commit-config.yaml | 1 + procrastinate/contrib/sphinx/__init__.py | 35 ++++++++++++++++++++++++ pyproject.toml | 16 +++++++++-- 3 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 procrastinate/contrib/sphinx/__init__.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07b837005..6b3966db5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -47,6 +47,7 @@ repos: - psycopg[pool]==3.2.1 - python-dateutil==2.9.0.post0 - sqlalchemy==2.0.32 + - sphinx==7.1.2 - typing-extensions==4.12.2 - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.5.7 diff --git a/procrastinate/contrib/sphinx/__init__.py b/procrastinate/contrib/sphinx/__init__.py new file mode 100644 index 000000000..e3b84398e --- /dev/null +++ b/procrastinate/contrib/sphinx/__init__.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +from typing import Any + +from sphinx.application import Sphinx +from sphinx.ext.autodoc import FunctionDocumenter + +from procrastinate import tasks + + +class ProcrastinateTaskDocumenter(FunctionDocumenter): + objtype = "procrastinate_task" + directivetype = "function" + member_order = 40 + + @classmethod + def can_document_member( + cls, + member: Any, + membername: str, + isattr: bool, + parent: Any, + ) -> bool: + return isinstance(member, tasks.Task) + + +def setup(app: Sphinx): + app.setup_extension("sphinx.ext.autodoc") # Require autodoc extension + + app.add_autodocumenter(ProcrastinateTaskDocumenter) + + return { + "version": "1", + "parallel_read_safe": True, + } diff --git a/pyproject.toml b/pyproject.toml index 1ff26d963..1a90c8a6c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,12 +38,14 @@ psycopg2-binary = { version = "*", optional = true } python-dateutil = "*" sqlalchemy = { version = "^2.0", optional = true } typing-extensions = { version = "*", python = "<3.8" } +sphinx = { version = "*", optional = true } [tool.poetry.extras] django = ["django"] sqlalchemy = ["sqlalchemy"] aiopg = ["aiopg", "psycopg2-binary"] psycopg2 = ["psycopg2-binary"] +sphinx = ["sphinx"] [tool.poetry.group.types] optional = true @@ -62,9 +64,17 @@ aiopg = "*" sqlalchemy = { extras = ["mypy"], version = "*" } psycopg2-binary = "*" psycopg = [ - { version = "^3.1.13", extras = ["binary", "pool"], markers = "sys_platform != 'darwin' or platform_machine != 'arm64'"}, - { version = "^3.1.13", extras = ["binary", "pool"], markers = "sys_platform == 'darwin' and platform_machine == 'arm64'", python = ">=3.10"}, - { version = "^3.1.13", extras = ["pool"], markers = "sys_platform == 'darwin' and platform_machine == 'arm64'", python = "<3.10" } + { version = "^3.1.13", extras = [ + "binary", + "pool", + ], markers = "sys_platform != 'darwin' or platform_machine != 'arm64'" }, + { version = "^3.1.13", extras = [ + "binary", + "pool", + ], markers = "sys_platform == 'darwin' and platform_machine == 'arm64'", python = ">=3.10" }, + { version = "^3.1.13", extras = [ + "pool", + ], markers = "sys_platform == 'darwin' and platform_machine == 'arm64'", python = "<3.10" }, ] [tool.poetry.group.django.dependencies] From b8e4869044e3c62f0e5f54d9d6f0ccc1411bf3c7 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 00:08:06 +0200 Subject: [PATCH 02/10] Document the new extension --- docs/howto/advanced/sphinx.md | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 docs/howto/advanced/sphinx.md diff --git a/docs/howto/advanced/sphinx.md b/docs/howto/advanced/sphinx.md new file mode 100644 index 000000000..452bf8d37 --- /dev/null +++ b/docs/howto/advanced/sphinx.md @@ -0,0 +1,30 @@ +# Document my tasks with Sphinx & Autodoc + +If you use Sphinx's `autodoc` extension to document your project, you might +have noticed that your tasks are absent from the documentation. This is because +when you apply the `@app.task` decorator, you're actually replacing the +function with a Procrastinate Task object, which `autodoc` doesn't know how to +process. + +Procrastinate provides a small Sphinx extension to fix that. You may want to +ensure procrastinate is installed with the `sphinx` extra in the environment +where you build your doc. This is not mandatory, as it only adds sphinx itself +as a dependency, but if the extension ever needs other dependencies in the +future, they will be installed through the `sphinx` extra as well. + +```bash +$ pip install procrastinate[sphinx] +``` + +Then, add the following to your `conf.py`: + +```python +extensions = [ + # ... + "procrastinate.contrib.sphinx", + # ... +] +``` + +That's it. Your tasks will now be picked up by `autodoc` and included in your +documentation. From 98e02c26aaf9ffe83f6e9070abb045f6a008dafd Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 00:09:07 +0200 Subject: [PATCH 03/10] Drink our own champagne: use our new extension This is the reason why the builtin ask didn't appear in the doc --- docs/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/conf.py b/docs/conf.py index 749c801f8..07f38d9dd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -42,6 +42,7 @@ "sphinxcontrib.programoutput", "sphinx_github_changelog", "sphinx_copybutton", + "procrastinate.contrib.sphinx", ] myst_enable_extensions = [ From cbb58fb04fc302846544ba2e897ff4f831da6eba Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 00:10:58 +0200 Subject: [PATCH 04/10] Add docstrings to tasks in the demo so that autodoc considers them --- procrastinate_demos/demo_async/tasks.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/procrastinate_demos/demo_async/tasks.py b/procrastinate_demos/demo_async/tasks.py index 2b7e54cf7..437ab3f3d 100644 --- a/procrastinate_demos/demo_async/tasks.py +++ b/procrastinate_demos/demo_async/tasks.py @@ -5,9 +5,11 @@ @app.task(queue="sums") async def sum(a, b): + """Sum two numbers.""" return a + b @app.task(queue="defer") async def defer(): + """Defer a nother task.""" await sum.defer_async(a=1, b=2) From 31eb95b0dd4a2004404745d73cd0c1c5e3ad9216 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 00:11:12 +0200 Subject: [PATCH 05/10] Add integration test --- tests/integration/contrib/sphinx/__init__.py | 0 tests/integration/contrib/sphinx/conftest.py | 20 +++++++++++++++++++ .../contrib/sphinx/test-root/conf.py | 8 ++++++++ .../contrib/sphinx/test-root/index.rst | 5 +++++ .../contrib/sphinx/test_autodoc.py | 11 ++++++++++ 5 files changed, 44 insertions(+) create mode 100644 tests/integration/contrib/sphinx/__init__.py create mode 100644 tests/integration/contrib/sphinx/conftest.py create mode 100644 tests/integration/contrib/sphinx/test-root/conf.py create mode 100644 tests/integration/contrib/sphinx/test-root/index.rst create mode 100644 tests/integration/contrib/sphinx/test_autodoc.py diff --git a/tests/integration/contrib/sphinx/__init__.py b/tests/integration/contrib/sphinx/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/contrib/sphinx/conftest.py b/tests/integration/contrib/sphinx/conftest.py new file mode 100644 index 000000000..7398f5bc9 --- /dev/null +++ b/tests/integration/contrib/sphinx/conftest.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +import pytest +from sphinx.testing.path import path + +pytest_plugins = ["sphinx.testing.fixtures"] + + +@pytest.fixture(scope="session") +def rootdir(): + return path(__file__).parent + + +@pytest.fixture +def sphinx_app(app_params, make_app): + """ + Provides the 'sphinx.application.Sphinx' object + """ + args, kwargs = app_params + return make_app(*args, **kwargs) diff --git a/tests/integration/contrib/sphinx/test-root/conf.py b/tests/integration/contrib/sphinx/test-root/conf.py new file mode 100644 index 000000000..abbececb4 --- /dev/null +++ b/tests/integration/contrib/sphinx/test-root/conf.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +extensions = [ + "sphinx.ext.autodoc", + "procrastinate.contrib.sphinx", +] + +buildername = "html" diff --git a/tests/integration/contrib/sphinx/test-root/index.rst b/tests/integration/contrib/sphinx/test-root/index.rst new file mode 100644 index 000000000..0f287a494 --- /dev/null +++ b/tests/integration/contrib/sphinx/test-root/index.rst @@ -0,0 +1,5 @@ +Tasks +===== + +.. automodule:: procrastinate_demos.demo_async.tasks + :members: diff --git a/tests/integration/contrib/sphinx/test_autodoc.py b/tests/integration/contrib/sphinx/test_autodoc.py new file mode 100644 index 000000000..20c84a3d0 --- /dev/null +++ b/tests/integration/contrib/sphinx/test_autodoc.py @@ -0,0 +1,11 @@ +from __future__ import annotations + +import pytest + + +@pytest.mark.sphinx(buildername="html", testroot="root") +def test_autodoc_task(sphinx_app): + sphinx_app.build() + content = (sphinx_app.outdir / "index.html").read_text() + # Check that the docstring of one of the task appears in the generated documentation + assert "Sum two numbers." in content From af31ab352882bf0abae5cd01aa753004964ecaf9 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 00:18:58 +0200 Subject: [PATCH 06/10] Poetry lock --- .pre-commit-config.yaml | 2 +- poetry.lock | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6b3966db5..146c7dd95 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -46,8 +46,8 @@ repos: - psycopg2-binary==2.9.9 - psycopg[pool]==3.2.1 - python-dateutil==2.9.0.post0 - - sqlalchemy==2.0.32 - sphinx==7.1.2 + - sqlalchemy==2.0.32 - typing-extensions==4.12.2 - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.5.7 diff --git a/poetry.lock b/poetry.lock index e148c3715..4371a31cf 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. [[package]] name = "aiopg" @@ -1833,9 +1833,10 @@ test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", aiopg = ["aiopg", "psycopg2-binary"] django = ["django"] psycopg2 = ["psycopg2-binary"] +sphinx = ["sphinx"] sqlalchemy = ["sqlalchemy"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "4c0b8b8ffde08064ffd59f6cf47bffff22f08479bfeba08fa9592672cf9f7659" +content-hash = "6c0aeb48705cf04301eaf320ef5df2399eae7fae612cdfad816b785c1623a715" From 2d23861eb9de42a4ff1d03fcd6b9c1d4858f1dc6 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 00:29:12 +0200 Subject: [PATCH 07/10] Add doc to toctree --- docs/howto/advanced.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/howto/advanced.md b/docs/howto/advanced.md index aee8516a6..dd810e5c3 100644 --- a/docs/howto/advanced.md +++ b/docs/howto/advanced.md @@ -17,4 +17,5 @@ advanced/events advanced/sync_defer advanced/custom_json_encoder_decoder advanced/blueprints +advanced/sphinx ::: From 1b8b33ae107595ad17d1e2928faa94bcab358ad4 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 00:34:25 +0200 Subject: [PATCH 08/10] define pytest_plugins in the top-level conftest --- tests/conftest.py | 4 ++++ tests/integration/contrib/sphinx/conftest.py | 20 ------------------- .../contrib/sphinx/test_autodoc.py | 15 ++++++++++++++ 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6e4ab323d..965b70aee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,6 +25,10 @@ if key.startswith("PROCRASTINATE_"): os.environ.pop(key) +# Infortunately, we need the sphinx fixtures even though they generate an "app" fixture +# that conflicts with our own "app" fixture +pytest_plugins = ["sphinx.testing.fixtures"] + def cursor_execute(cursor, query, *identifiers): if identifiers: diff --git a/tests/integration/contrib/sphinx/conftest.py b/tests/integration/contrib/sphinx/conftest.py index 7398f5bc9..e69de29bb 100644 --- a/tests/integration/contrib/sphinx/conftest.py +++ b/tests/integration/contrib/sphinx/conftest.py @@ -1,20 +0,0 @@ -from __future__ import annotations - -import pytest -from sphinx.testing.path import path - -pytest_plugins = ["sphinx.testing.fixtures"] - - -@pytest.fixture(scope="session") -def rootdir(): - return path(__file__).parent - - -@pytest.fixture -def sphinx_app(app_params, make_app): - """ - Provides the 'sphinx.application.Sphinx' object - """ - args, kwargs = app_params - return make_app(*args, **kwargs) diff --git a/tests/integration/contrib/sphinx/test_autodoc.py b/tests/integration/contrib/sphinx/test_autodoc.py index 20c84a3d0..30b191355 100644 --- a/tests/integration/contrib/sphinx/test_autodoc.py +++ b/tests/integration/contrib/sphinx/test_autodoc.py @@ -1,6 +1,21 @@ from __future__ import annotations import pytest +from sphinx.testing.path import path + + +@pytest.fixture +def rootdir(): + return path(__file__).parent + + +@pytest.fixture +def sphinx_app(app_params, make_app): + """ + Provides the 'sphinx.application.Sphinx' object + """ + args, kwargs = app_params + return make_app(*args, **kwargs) @pytest.mark.sphinx(buildername="html", testroot="root") From 4633dcef3588da5310b2c825fb8946d0c104ce40 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 22:17:55 +0200 Subject: [PATCH 09/10] Update procrastinate_demos/demo_async/tasks.py --- procrastinate_demos/demo_async/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/procrastinate_demos/demo_async/tasks.py b/procrastinate_demos/demo_async/tasks.py index 437ab3f3d..49f6be684 100644 --- a/procrastinate_demos/demo_async/tasks.py +++ b/procrastinate_demos/demo_async/tasks.py @@ -11,5 +11,5 @@ async def sum(a, b): @app.task(queue="defer") async def defer(): - """Defer a nother task.""" + """Defer a another task.""" await sum.defer_async(a=1, b=2) From 86ea2a2411b55c3495c0031c9b28368811674b19 Mon Sep 17 00:00:00 2001 From: Joachim Jablon Date: Sun, 11 Aug 2024 22:18:10 +0200 Subject: [PATCH 10/10] Update tests/conftest.py --- tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 965b70aee..836c610a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -25,7 +25,7 @@ if key.startswith("PROCRASTINATE_"): os.environ.pop(key) -# Infortunately, we need the sphinx fixtures even though they generate an "app" fixture +# Unfortunately, we need the sphinx fixtures even though they generate an "app" fixture # that conflicts with our own "app" fixture pytest_plugins = ["sphinx.testing.fixtures"]