diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8ae20323cd..abfb78973a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -19,6 +19,16 @@ build: os: ubuntu-22.04 tools: python: '3' + # Override the Read the Docs build with our own. This uses a beta + # feature of Read the Docs that loses some features + # + # but attempts at using post_build to do post-processing have so far + # failed + # . + commands: + - make -C manual html mmref + - mkdir -p _readthedocs + - if test -n "$MMREF"; then mv manual/mmref _readthedocs/html; else mv manual/html _readthedocs/html; fi python: install: @@ -37,6 +47,7 @@ sphinx: # B. DOCUMENT HISTORY # # 2023-02-02 RB Created as part of MPS GitHub migration. +# 2023-03-03 RB Updated to also build the Memory Management Reference. # # # C. COPYRIGHT AND LICENSE diff --git a/manual/.gitignore b/manual/.gitignore index f2a531e43c..5a17543431 100644 --- a/manual/.gitignore +++ b/manual/.gitignore @@ -2,6 +2,7 @@ doctrees converted epub html +mmref.in mmref source/design/*.rst source/design/*.svg diff --git a/manual/Makefile b/manual/Makefile index 25615b8571..f739874811 100644 --- a/manual/Makefile +++ b/manual/Makefile @@ -18,7 +18,7 @@ ALLSPHINXOPTS = -T -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext mmref help: @echo "Please use \`make ' where is one of" @@ -41,10 +41,11 @@ help: @echo " changes to make an overview of all changed/added/deprecated items" @echo " linkcheck to check all external links for integrity" @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " mmref to make the Memory Management Reference" @echo " tools to install a local copy of the Python tools using virtualenv" clean: - -for dir in changes devhelp dirhtml doctest doctrees epub html htmlhelp json latex linkcheck locale man pickle qthelp singlehtml texinfo text; do rm -rf $(BUILDDIR)/$$dir; done + -for dir in changes devhelp dirhtml doctest doctrees epub html htmlhelp json latex linkcheck locale man pickle qthelp singlehtml texinfo text mmref.in mmref; do rm -rf $(BUILDDIR)/$$dir; done -find $(BUILDDIR)/source/design -name '*.rst' ! -name 'index.rst' ! -name 'old.rst' -exec rm -f '{}' ';' html: $(SPHINXBUILD) @@ -158,6 +159,13 @@ doctest: $(SPHINXBUILD) @echo "Testing of doctests in the sources finished, look at the " \ "results in $(BUILDDIR)/doctest/output.txt." +.PHONY: mmref +mmref: $(SPHINXBUILD) + MMREF=1 $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/mmref.in + tool/bin/$(PYTHON) ./make-mmref.py $(BUILDDIR)/mmref.in $(BUILDDIR)/mmref + @echo + @echo "Build finished. The Memory Management Reference is $(BUILDDIR)/mmref." + .PHONY: tools tools: tool/bin/pip requirements.pip tool/bin/pip install -r requirements.pip diff --git a/manual/make-mmref.py b/manual/make-mmref.py new file mode 100755 index 0000000000..2046be0cd5 --- /dev/null +++ b/manual/make-mmref.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +# +# MAKE THE MEMORY MANAGEMENT REFERENCE +# Gareth Rees, Ravenbrook Limited, 2014-05-23 +# +# $Id$ +# +# +# 1. INTRODUCTION +# +# This script builds the Memory Management Reference website from the +# Memory Pool System manual. +# +# The whole build procedure is as follows: +# +# 1. Sync //info.ravenbrook.com/project/mps/master/manual/... +# 2. make html MMREF=1 +# 3. Run this script +# +# +# 2. DESIGN +# +# We build the Memory Management Reference out of the Memory Pool +# System manual because: +# +# 1. having a single set of sources makes it easier to work on; +# 2. the glossary is a vital tool in organizing the MPS manual; +# 3. cross-references from the MMRef to the MPS are an opportunity +# for advertising the latter to readers of the former. +# +# +# 3. DEPENDENCIES +# +# html5lib +# six + +import os +import re +from shutil import copyfile +import sys +from urllib.parse import urljoin +from xml.etree.cElementTree import Element + +import html5lib + + +# 4. CONFIGURATION + +# Subdirectories of the MPS manual that belong in the MMRef. +MMREF_DIRS = 'glossary mmref _images _static'.split() + +# Top-level files that belong in the MMRef. +MMREF_FILES = dict(index='mmref-index', bib='bib', copyright='mmref-copyright') + +# Regular expression matching files to be included in the MMRef. +URL_FILTER_RE = re.compile(r'/html/(?:(?:{})\.html)?(?:#.*)?$|/(?:{})/'.format( + '|'.join(MMREF_FILES), '|'.join(MMREF_DIRS))) + +# Root URL for the MPS manual. +REWRITE_URL = 'https://www.ravenbrook.com/project/mps/master/manual/html/' + + +def rewrite_links(src, src_base, url_filter, rewrite_base, + url_attributes=(('a', 'href'),)): + """Rewrite URLs and anchors in src and return the result. + + First, src is parsed as HTML. Second, all URLs found in the + document are resolved relative to src_base and the result passed + to the functions url_filter. If this returns False, the URL is + resolved again, this time relative to rewrite_base, and the result + stored back to the document. Third, all DT elements with an id + starting "term-X" are rewritten to contain another anchor with id + "X" (for backwards compatiblity with links to the old manual). + Finally, the updated document is serialized as HTML and returned. + + The keyword argument url_attributes is a sequence of (tag, + attribute) pairs that contain URLs to be rewritten. + + """ + tree = html5lib.parse(src, treebuilder='etree') + + for tag, attr in url_attributes: + for e in tree.iter('{http://www.w3.org/1999/xhtml}' + tag): + u = e.get(attr) + if u and not url_filter(urljoin(src_base, u)): + rewritten = urljoin(rewrite_base, u) + if u != rewritten: + e.set(attr, rewritten) + + id_re = re.compile(r'-([a-z])') + for dt in tree.iter('{http://www.w3.org/1999/xhtml}dt'): + id = dt.get('id') + if id and id.startswith('term-'): + span = Element('span', id=id_re.sub(r'.\1', id[5:])) + span.text = dt.text + dt.text = '' + for child in list(dt): + dt.remove(child) + span.append(child) + dt.insert(0, span) + + tree_walker = html5lib.getTreeWalker('etree') + html_serializer = html5lib.serializer.HTMLSerializer() + return u''.join(html_serializer.serialize(tree_walker(tree))) + + +def newer(src, target): + """Return True if src is newer (that is, modified more recently) than + target, False otherwise. + + """ + return (not os.path.isfile(target) + or os.path.getmtime(target) < os.path.getmtime(src) + or os.path.getmtime(target) < os.path.getmtime(__file__)) + + +def rewrite_file(src_dir, rel_dir, src_filename, target_path, rewrite_url): + src_path = os.path.join(src_dir, src_filename) + if not newer(src_path, target_path): + return + print("Rewriting links in {} -> {}".format(src_path, target_path)) + with open(src_path) as file: + src = file.read() + src_base = '/{}/'.format(src_dir) + url_filter = URL_FILTER_RE.search + rewrite_base = urljoin(rewrite_url, rel_dir + '/') + result = rewrite_links(src, src_base, url_filter, rewrite_base) + with open(target_path, 'w') as file: + file.write(result) + + +def main(src_root='html', target_root='mmref'): + for directory in MMREF_DIRS: + src_dir = os.path.join(src_root, directory) + target_dir = os.path.join(target_root, directory) + os.makedirs(target_dir, exist_ok=True) + for filename in os.listdir(src_dir): + src_path = os.path.join(src_dir, filename) + target_path = os.path.join(target_dir, filename) + if filename.endswith('.html'): + rewrite_file(src_dir, directory, filename, target_path, + REWRITE_URL) + elif os.path.isfile(src_path): + copyfile(src_path, target_path) + for target, src in MMREF_FILES.items(): + rewrite_file(src_root, '', '{}.html'.format(src), + os.path.join(target_root, '{}.html'.format(target)), + REWRITE_URL) + + +if __name__ == '__main__': + main(*sys.argv[1:]) + + +# B. DOCUMENT HISTORY +# +# 2014-05-23 GDR Created. +# 2018-10-29 GDR Port to Python 3 and html5lib 0.9999999. +# +# +# C. COPYRIGHT AND LICENCE +# +# Copyright (c) 2014-2018 Ravenbrook Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# +# $Id$ diff --git a/manual/requirements.pip b/manual/requirements.pip index f5e50bda3e..d22bb939d9 100644 --- a/manual/requirements.pip +++ b/manual/requirements.pip @@ -2,3 +2,7 @@ # Layout breaks in Sphinx >= 5. See . sphinx >= 4, < 5 + +# HTML parser based on the WHATWG HTML specification +# Required by make-mmref.py +html5lib