Sphinx documentation pipeline (#8)

* Refactored retrievers into a directory with separate files

* Adds HybridCypherRetriever, refactor query construction, add tests

* Put docs/_build in gitignore

* Adding content to docs

* Edited Makefile to use poetry

* Use separate token for creating git tags in actions

* Bump version to 0.2.0a1

* Rename GitHub Action Workflow

* Missing docstring for filters argument in VectorRetriever

---------

Co-authored-by: Oskar Hane <[email protected]>
Co-authored-by: Neo4j-GenAI GitHub Action <[email protected]>

.pre-commit-config.yaml

@@ -4,3 +4,11 @@ repos:
     hooks:
     - id: trailing-whitespace
     - id: end-of-file-fixer
+  - repo: local
+    hooks:
+      - id: ruff-lint
+        name: Ruff Lint Check
+        entry: poetry run ruff format --check
+        language: system
+        types: [ python ]
+        stages: [ commit, push ]

CONTRIBUTING.md

@@ -54,7 +54,7 @@ Remember that many community members have become regular contributors and some a
 ## Specifically for this project
 Setting up the development environment:
 
-1. Install Python 3.9.1+
+1. Install Python 3.8.1+
 2. Install poetry (see https://python-poetry.org/docs/#installation)
 3. Install dependencies:
 

README.md

@@ -140,7 +140,7 @@ and/or [Discord](https://discord.gg/neo4j).
 ### Make changes
 
 1. Fork the repository.
-2. Install Python and Poetry. For more information, see [the development guide](./docs/contributing/DEVELOPING.md).
+2. Install Python and Poetry.
 3. Create a working branch from `main` and start with your changes!
 
 ### Pull request

docs/Makefile

@@ -0,0 +1,194 @@
+# Makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+VERSION       = 0.1.2
+PRODUCT       = neo4j-genai-python
+SPHINXOPTS    =
+SPHINXBUILD   = poetry run sphinx-build
+PAPER         =
+BUILDDIR      = build
+
+# User-friendly check for sphinx-build via Poetry
+ifeq ($(shell poetry run which sphinx-build >/dev/null 2>&1; echo $$?), 1)
+$(error The 'sphinx-build' command was not found within the Poetry environment. Make sure Sphinx is installed in your project dependencies.)
+endif
+
+# Internal variables.
+PAPEROPT_a4     = -D latex_paper_size=a4
+PAPEROPT_letter = -D latex_paper_size=letter
+ALLSPHINXOPTS   = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+# the i18n builder cannot share the environment and doctrees with the others
+I18NSPHINXOPTS  = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source
+
+.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext
+
+help:
+	@echo "Please use \`make <target>' where <target> is one of"
+	@echo "  html       to make standalone HTML files"
+	@echo "  dirhtml    to make HTML files named index.html in directories"
+	@echo "  singlehtml to make a single large HTML file"
+	@echo "  pickle     to make pickle files"
+	@echo "  json       to make JSON files"
+	@echo "  htmlhelp   to make HTML files and a HTML help project"
+	@echo "  qthelp     to make HTML files and a qthelp project"
+	@echo "  applehelp  to make an Apple Help Book"
+	@echo "  devhelp    to make HTML files and a Devhelp project"
+	@echo "  epub       to make an epub"
+	@echo "  latex      to make LaTeX files, you can set PAPER=a4 or PAPER=letter"
+	@echo "  latexpdf   to make LaTeX files and run them through pdflatex"
+	@echo "  latexpdfja to make LaTeX files and run them through platex/dvipdfmx"
+	@echo "  text       to make text files"
+	@echo "  man        to make manual pages"
+	@echo "  texinfo    to make Texinfo files"
+	@echo "  info       to make Texinfo files and run them through makeinfo"
+	@echo "  gettext    to make PO message catalogs"
+	@echo "  changes    to make an overview of all changed/added/deprecated items"
+	@echo "  xml        to make Docutils-native XML files"
+	@echo "  pseudoxml  to make pseudoxml-XML files for display purposes"
+	@echo "  linkcheck  to check all external links for integrity"
+	@echo "  doctest    to run all doctests embedded in the documentation (if enabled)"
+	@echo "  coverage   to run coverage check of the documentation (if enabled)"
+
+clean:
+	rm -rf $(BUILDDIR)/*
+
+html:
+	$(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/$(PRODUCT)/$(VERSION)
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/$(PRODUCT)/$(VERSION)."
+
+dirhtml:
+	$(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml
+	@echo
+	@echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml."
+
+singlehtml:
+	$(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml
+	@echo
+	@echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml."
+
+pickle:
+	$(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle
+	@echo
+	@echo "Build finished; now you can process the pickle files."
+
+json:
+	$(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json
+	@echo
+	@echo "Build finished; now you can process the JSON files."
+
+htmlhelp:
+	$(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp
+	@echo
+	@echo "Build finished; now you can run HTML Help Workshop with the" \
+	      ".hhp project file in $(BUILDDIR)/htmlhelp."
+
+qthelp:
+	$(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp
+	@echo
+	@echo "Build finished; now you can run "qcollectiongenerator" with the" \
+	      ".qhcp project file in $(BUILDDIR)/qthelp, like this:"
+	@echo "# qcollectiongenerator $(BUILDDIR)/qthelp/Neo4jBoltDriverforPython.qhcp"
+	@echo "To view the help file:"
+	@echo "# assistant -collectionFile $(BUILDDIR)/qthelp/Neo4jBoltDriverforPython.qhc"
+
+applehelp:
+	$(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp
+	@echo
+	@echo "Build finished. The help book is in $(BUILDDIR)/applehelp."
+	@echo "N.B. You won't be able to view it unless you put it in" \
+	      "~/Library/Documentation/Help or install it in your application" \
+	      "bundle."
+
+devhelp:
+	$(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp
+	@echo
+	@echo "Build finished."
+	@echo "To view the help file:"
+	@echo "# mkdir -p $$HOME/.local/share/devhelp/Neo4jBoltDriverforPython"
+	@echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/Neo4jBoltDriverforPython"
+	@echo "# devhelp"
+
+epub:
+	$(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub
+	@echo
+	@echo "Build finished. The epub file is in $(BUILDDIR)/epub."
+
+latex:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo
+	@echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex."
+	@echo "Run \`make' in that directory to run these through (pdf)latex" \
+	      "(use \`make latexpdf' here to do that automatically)."
+
+latexpdf:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through pdflatex..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+latexpdfja:
+	$(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex
+	@echo "Running LaTeX files through platex and dvipdfmx..."
+	$(MAKE) -C $(BUILDDIR)/latex all-pdf-ja
+	@echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex."
+
+text:
+	$(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text
+	@echo
+	@echo "Build finished. The text files are in $(BUILDDIR)/text."
+
+man:
+	$(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man
+	@echo
+	@echo "Build finished. The manual pages are in $(BUILDDIR)/man."
+
+texinfo:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo
+	@echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo."
+	@echo "Run \`make' in that directory to run these through makeinfo" \
+	      "(use \`make info' here to do that automatically)."
+
+info:
+	$(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo
+	@echo "Running Texinfo files through makeinfo..."
+	make -C $(BUILDDIR)/texinfo info
+	@echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo."
+
+gettext:
+	$(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale
+	@echo
+	@echo "Build finished. The message catalogs are in $(BUILDDIR)/locale."
+
+changes:
+	$(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes
+	@echo
+	@echo "The overview file is in $(BUILDDIR)/changes."
+
+linkcheck:
+	$(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck
+	@echo
+	@echo "Link check complete; look for any errors in the above output " \
+	      "or in $(BUILDDIR)/linkcheck/output.txt."
+
+doctest:
+	$(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest
+	@echo "Testing of doctests in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/doctest/output.txt."
+
+coverage:
+	$(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage
+	@echo "Testing of coverage in the sources finished, look at the " \
+	      "results in $(BUILDDIR)/coverage/python.txt."
+
+xml:
+	$(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml
+	@echo
+	@echo "Build finished. The XML files are in $(BUILDDIR)/xml."
+
+pseudoxml:
+	$(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml
+	@echo
+	@echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml."

docs/README.md

@@ -0,0 +1,16 @@
+# Sphinx Documentation
+
+Building the docs requires Python 3.8.1+
+
+Ensure the dev dependencies in `pyproject.toml` are installed.
+
+From the root directory, run the Makefile:
+
+```
+make -C docs html
+```
+
+You can now view your build locally.
+
+When you open a PR, a TeamCity build will trigger when it's ready and someone from Neo4j
+can then preview this.

docs/source/api.rst

@@ -0,0 +1,42 @@
+.. _api-documentation:
+
+API Documentation
+#################
+
+************************************
+Retrieval-Augmented Generation (RAG)
+************************************
+RAG is a technique that enhances Large Language Model (LLM) responses by retrieving
+source information from external data stores to augment generated responses.
+
+This package enables Python developers to perform RAG using Neo4j.
+
+**********
+Retrievers
+**********
+
+VectorRetriever
+===============
+
+.. autoclass:: neo4j_genai.retrievers.vector.VectorRetriever
+    :members:
+
+VectorCypherRetriever
+=====================
+
+.. autoclass:: neo4j_genai.retrievers.vector.VectorCypherRetriever
+   :members:
+
+
+HybridRetriever
+===============
+
+.. autoclass:: neo4j_genai.retrievers.hybrid.HybridRetriever
+   :members:
+
+
+HybridCypherRetriever
+=====================
+
+.. autoclass:: neo4j_genai.retrievers.hybrid.HybridCypherRetriever
+   :members:

docs/source/conf.py

@@ -0,0 +1,103 @@
+#  Copyright (c) "Neo4j"
+#  Neo4j Sweden AB [https://neo4j.com]
+#  #
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  #
+#      https://www.apache.org/licenses/LICENSE-2.0
+#  #
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+import sys
+import typing
+
+# Add the root of the project to the path so that Sphinx can find the Python sources
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "src"))
+
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = "neo4j-genai-python"
+copyright = "2024, Neo4j, Inc."
+author = "Neo4j, Inc."
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    "sphinx.ext.autodoc",
+    "sphinx.ext.doctest",
+    "sphinx.ext.todo",
+    "sphinx.ext.coverage",
+    "sphinx.ext.ifconfig",
+    "sphinx.ext.intersphinx",
+    "sphinx.ext.napoleon",
+    "sphinx.ext.viewcode",
+]
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+# source_suffix = ['.rst', '.md']
+source_suffix = ".rst"
+
+# The master toctree document.
+master_doc = "index"
+
+intersphinx_mapping = {
+    "python": ("https://docs.python.org/3", None),
+}
+
+templates_path = ["_templates"]
+exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"]
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = "neo4j"
+html_theme_path = ["themes"]
+
+# The name of the Pygments (syntax highlighting) style to use.
+# pygments_style = "sphinx"
+pygments_style = "friendly"
+
+# Don't include type hints in function signatures
+autodoc_typehints = "description"
+
+autodoc_type_aliases = {
+    # The code-base uses `import typing_extensions as te`.
+    # Re-write these to use `typing` instead, as Sphinx always resolves against
+    # the latest version of the `typing` module.
+    # This is a work-around to make Sphinx resolve type hints correctly, even
+    # though we're using `from __future__ import annotations`.
+    "te": typing,
+    # Type alias that's only defined and imported if `typing.TYPE_CHECKING`
+    # is `True`.
+    "_TAuth": "typing.Tuple[typing.Any, typing.Any] | Auth | None",
+}
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+html_theme_options = {
+    "sidebar_includehidden": True,
+    "sidebar_collapse": True,
+}
+
+autodoc_default_options = {
+    "member-order": "bysource",
+    # 'special-members': '__init__',
+    "undoc-members": True,
+    "exclude-members": "__weakref__",
+}