Skip to content

Commit

Permalink
move footnote ordering to transforms
Browse files Browse the repository at this point in the history
  • Loading branch information
chrisjsewell committed Aug 5, 2024
1 parent f0c2dcd commit 7a17af9
Show file tree
Hide file tree
Showing 21 changed files with 591 additions and 359 deletions.
4 changes: 2 additions & 2 deletions docs/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -195,13 +195,13 @@ tasklist
(myst-warnings)=
## Build Warnings

Below lists the MyST specific warnings that may be emitted during the build process. These will be prepended to the end of the warning message, e.g.
Below lists the MyST specific warnings that may be emitted during the build process. These will be prepended to the end of the warning message (see also <inv:sphinx#show_warning_types>), e.g.

```
WARNING: Non-consecutive header level increase; H1 to H3 [myst.header]
```

**In general, if your build logs any warnings, you should either fix them or [raise an Issue](https://github.com/executablebooks/MyST-Parser/issues/new/choose) if you think the warning is erroneous.**
In general, if your build logs any warnings, you should either fix them or [raise an Issue](https://github.com/executablebooks/MyST-Parser/issues/new/choose) if you think the warning is erroneous.

However, in some circumstances if you wish to suppress the warning you can use the <inv:sphinx#suppress_warnings> configuration option, e.g.

Expand Down
10 changes: 9 additions & 1 deletion myst_parser/config/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -319,11 +319,19 @@ def __repr__(self) -> str:
},
)

footnote_sort: bool = dc.field(
default=True,
metadata={
"validator": instance_of(bool),
"help": "Move all footnotes to the end of the document, and sort by reference order",
},
)

footnote_transition: bool = dc.field(
default=True,
metadata={
"validator": instance_of(bool),
"help": "Place a transition before any footnotes",
"help": "Place a transition before sorted footnotes",
},
)

Expand Down
94 changes: 24 additions & 70 deletions myst_parser/mdit_to_docutils/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,9 @@ def sphinx_env(self) -> BuildEnvironment | None:
def create_warning(
self,
message: str,
subtype: MystWarnings,
subtype: MystWarnings | str,
*,
wtype: str | None = None,
line: int | None = None,
append_to: nodes.Element | None = None,
) -> nodes.system_message | None:
Expand All @@ -172,6 +173,7 @@ def create_warning(
self.document,
message,
subtype,
wtype=wtype,
line=line,
append_to=append_to,
)
Expand All @@ -189,22 +191,6 @@ def _render_tokens(self, tokens: list[Token]) -> None:

# nest tokens
node_tree = SyntaxTreeNode(tokens)

# move footnote definitions to env
self.md_env.setdefault("footnote_definitions", {})
for node in node_tree.walk(include_self=True):
new_children = []
for child in node.children:
if child.type == "footnote_reference":
label = child.meta["label"]
self.md_env["footnote_definitions"].setdefault(label, []).append(
child
)
else:
new_children.append(child)

node.children = new_children

# render
for child in node_tree.children:
# skip hidden?
Expand Down Expand Up @@ -255,6 +241,12 @@ def _render_finalise(self) -> None:
self._heading_slugs
)

# ensure these settings are set for later footnote transforms
self.document.settings.myst_footnote_transition = (
self.md_config.footnote_transition
)
self.document.settings.myst_footnote_sort = self.md_config.footnote_sort

# log warnings for duplicate reference definitions
# "duplicate_refs": [{"href": "ijk", "label": "B", "map": [4, 5], "title": ""}],
for dup_ref in self.md_env.get("duplicate_refs", []):
Expand All @@ -265,59 +257,6 @@ def _render_finalise(self) -> None:
append_to=self.document,
)

# we don't use the foot_references stored in the env
# since references within directives/roles will have been added after
# those from the initial markdown parse
# instead we gather them from a walk of the created document
foot_refs: dict[str, list[nodes.footnote_reference]] = {}
for refnode in findall(self.document)(nodes.footnote_reference):
foot_refs.setdefault(refnode["refname"], []).append(refnode)
if foot_refs and self.md_config.footnote_transition:
self.current_node.append(nodes.transition(classes=["footnotes"]))
for foot_label, foot_ref_nodes in foot_refs.items():
foot_def_tokens = self.md_env["footnote_definitions"].get(foot_label, [])
if len(foot_def_tokens) < 1:
if (
self.document.current_source
and self.document.current_source.endswith("<translated>")
):
# TODO this is a bit of a hack for now, to detect if we are parsing a translation snippet
# in which case we won't have the definition loaded and should not warn/remove
# I think in the future we should look to move this footnote logic to a transform
continue
for node in foot_ref_nodes:
self.create_warning(
f"No footnote definition found for label: '{foot_label}'",
MystWarnings.MD_FOOTNOTE_MISSING,
line=node.line,
append_to=self.current_node,
)
# lets remove the footnote references, so that docutils does not produce any more warnings
# we need to replace them with an element though, so that ids can be moved over (otherwise docutils excepts)
if node.get("auto"):
self.document.autofootnote_refs.remove(node)
node.replace_self(nodes.inline(text=f"[^{node['refname']}]"))
else:
# render the first one, create a warning for any duplicates
self.render_footnote_reference(foot_def_tokens[0])
for foot_def_token in foot_def_tokens[1:]:
self.create_warning(
f"Duplicate footnote definition found for label: '{foot_label}'",
MystWarnings.MD_FOOTNOTE_DUPE,
line=token_line(foot_def_token),
append_to=self.current_node,
)
# finally lets warn about any unused footnotes definitions
for foot_label, foot_def_tokens in self.md_env["footnote_definitions"].items():
if foot_label not in foot_refs:
for foot_def_token in foot_def_tokens:
self.create_warning(
f"Footnote definition not referenced: '{foot_label}'",
MystWarnings.MD_FOOTNOTE_UNUSED,
line=token_line(foot_def_token),
append_to=self.current_node,
)

# Add the wordcount, generated by the ``mdit_py_plugins.wordcount_plugin``.
wordcount_metadata = self.md_env.get("wordcount", {})
if wordcount_metadata:
Expand Down Expand Up @@ -1511,6 +1450,21 @@ def render_footnote_reference(self, token: SyntaxTreeNode) -> None:
"""Despite the name, this is actually a footnote definition, e.g. `[^a]: ...`"""
target = token.meta["label"]

if target in self.document.nameids:
# note we chose to directly omit these footnotes in the parser,
# rather than let docutils/sphinx handle them, since otherwise you end up with a confusing warning:
# WARNING: Duplicate explicit target name: "x". [docutils]
# we use [ref.footnote] as the type/subtype, rather than a myst specific warning,
# to make it more aligned with sphinx warnings for unreferenced footnotes
self.create_warning(
f"Duplicate footnote definition found for label: '{target}'",
"footnote",
wtype="ref",
line=token_line(token),
append_to=self.current_node,
)
return

footnote = nodes.footnote()
self.add_line_and_source_path(footnote, token)
footnote["names"].append(target)
Expand Down
127 changes: 126 additions & 1 deletion myst_parser/mdit_to_docutils/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,140 @@

from docutils import nodes
from docutils.transforms import Transform
from docutils.transforms.references import Footnotes
from markdown_it.common.normalize_url import normalizeLink

from myst_parser._compat import findall
from myst_parser.mdit_to_docutils.base import clean_astext
from myst_parser.warnings_ import MystWarnings, create_warning


class UnreferencedFootnotesDetector(Transform):
"""Detect unreferenced footnotes and emit warnings.
Replicates https://github.com/sphinx-doc/sphinx/pull/12730,
but also allows for use in docutils (without sphinx).
"""

default_priority = Footnotes.default_priority + 2

# document: nodes.document

def apply(self, **kwargs: t.Any) -> None:
"""Apply the transform."""

for node in self.document.footnotes:
# note we do not warn on duplicate footnotes here
# (i.e. where the name has been moved to dupnames)
# since this is already reported by docutils
if not node["backrefs"] and node["names"]:
create_warning(
self.document,
"Footnote [{}] is not referenced.".format(node["names"][0])
if node["names"]
else node["dupnames"][0],
wtype="ref",
subtype="footnote",
node=node,
)
for node in self.document.symbol_footnotes:
if not node["backrefs"]:
create_warning(

Check warning on line 47 in myst_parser/mdit_to_docutils/transforms.py

View check run for this annotation

Codecov / codecov/patch

myst_parser/mdit_to_docutils/transforms.py#L46-L47

Added lines #L46 - L47 were not covered by tests
self.document,
"Footnote [*] is not referenced.",
wtype="ref",
subtype="footnote",
node=node,
)
for node in self.document.autofootnotes:
# note we do not warn on duplicate footnotes here
# (i.e. where the name has been moved to dupnames)
# since this is already reported by docutils
if not node["backrefs"] and node["names"]:
create_warning(
self.document,
"Footnote [#] is not referenced.",
wtype="ref",
subtype="footnote",
node=node,
)


class SortFootnotes(Transform):
"""Sort auto-numbered, labelled footnotes by the order they are referenced.
This is run before the docutils ``Footnote`` transform, where numbered labels are assigned.
"""

default_priority = Footnotes.default_priority - 2

# document: nodes.document

def apply(self, **kwargs: t.Any) -> None:
"""Apply the transform."""
if not self.document.settings.myst_footnote_sort:
return

Check warning on line 81 in myst_parser/mdit_to_docutils/transforms.py

View check run for this annotation

Codecov / codecov/patch

myst_parser/mdit_to_docutils/transforms.py#L81

Added line #L81 was not covered by tests

ref_order: list[str] = [
node["refname"]
for node in self.document.autofootnote_refs
if "refname" in node
]

def _sort_key(node: nodes.footnote) -> int:
if node["names"] and node["names"][0] in ref_order:
return ref_order.index(node["names"][0])
return 999

self.document.autofootnotes.sort(key=_sort_key)


class CollectFootnotes(Transform):
"""Transform to move footnotes to the end of the document, and sort by label."""

default_priority = Footnotes.default_priority + 3

# document: nodes.document

def apply(self, **kwargs: t.Any) -> None:
"""Apply the transform."""
if not self.document.settings.myst_footnote_sort:
return

Check warning on line 107 in myst_parser/mdit_to_docutils/transforms.py

View check run for this annotation

Codecov / codecov/patch

myst_parser/mdit_to_docutils/transforms.py#L107

Added line #L107 was not covered by tests

footnotes: list[tuple[str, nodes.footnote]] = []
for footnote in (
self.document.symbol_footnotes
+ self.document.footnotes
+ self.document.autofootnotes
):
label = footnote.children[0]
footnotes.append((label.astext(), footnote))

if (
footnotes
and self.document.settings.myst_footnote_transition
# avoid warning: Document or section may not begin with a transition
and not all(isinstance(c, nodes.footnote) for c in self.document.children)
):
transition = nodes.transition(classes=["footnotes"])
transition.source = self.document.source
self.document += transition

def _sort_key(footnote: tuple[str, nodes.footnote]) -> int | str:
label, _ = footnote
try:
# ensure e.g 10 comes after 2
return int(label)
except ValueError:
return label

Check warning on line 134 in myst_parser/mdit_to_docutils/transforms.py

View check run for this annotation

Codecov / codecov/patch

myst_parser/mdit_to_docutils/transforms.py#L133-L134

Added lines #L133 - L134 were not covered by tests

for _, footnote in sorted(footnotes, key=_sort_key):
footnote.parent.remove(footnote)
self.document += footnote


class ResolveAnchorIds(Transform):
"""Directive for resolving `[name](#id)` type links."""
"""Transform for resolving `[name](#id)` type links."""

default_priority = 879 # this is the same as Sphinx's StandardDomain.process_doc

Expand Down
14 changes: 12 additions & 2 deletions myst_parser/parsers/docutils_.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,12 @@
read_topmatter,
)
from myst_parser.mdit_to_docutils.base import DocutilsRenderer
from myst_parser.mdit_to_docutils.transforms import ResolveAnchorIds
from myst_parser.mdit_to_docutils.transforms import (
CollectFootnotes,
ResolveAnchorIds,
SortFootnotes,
UnreferencedFootnotesDetector,
)
from myst_parser.parsers.mdit import create_md_parser
from myst_parser.warnings_ import MystWarnings, create_warning

Expand Down Expand Up @@ -246,7 +251,12 @@ class Parser(RstParser):
translate_section_name = None

def get_transforms(self):
return super().get_transforms() + [ResolveAnchorIds]
return super().get_transforms() + [
UnreferencedFootnotesDetector,
SortFootnotes,
CollectFootnotes,
ResolveAnchorIds,
]

def parse(self, inputstring: str, document: nodes.document) -> None:
"""Parse source text.
Expand Down
12 changes: 10 additions & 2 deletions myst_parser/parsers/sphinx_.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@
read_topmatter,
)
from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer
from myst_parser.mdit_to_docutils.transforms import ResolveAnchorIds
from myst_parser.mdit_to_docutils.transforms import (
CollectFootnotes,
ResolveAnchorIds,
SortFootnotes,
)
from myst_parser.parsers.mdit import create_md_parser
from myst_parser.warnings_ import create_warning

Expand Down Expand Up @@ -46,7 +50,11 @@ class MystParser(SphinxParser):
translate_section_name = None

def get_transforms(self):
return super().get_transforms() + [ResolveAnchorIds]
return super().get_transforms() + [
SortFootnotes,
CollectFootnotes,
ResolveAnchorIds,
]

def parse(self, inputstring: str, document: nodes.document) -> None:
"""Parse source text.
Expand Down
10 changes: 10 additions & 0 deletions myst_parser/sphinx_ext/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,11 @@
import sphinx
from docutils import nodes
from sphinx.application import Sphinx
from sphinx.transforms import (
UnreferencedFootnotesDetector as SphinxUnreferencedFootnotesDetector,
)

from myst_parser.mdit_to_docutils.transforms import UnreferencedFootnotesDetector
from myst_parser.parsers.docutils_ import (
depart_container_html,
depart_rubric_html,
Expand Down Expand Up @@ -39,6 +43,12 @@ def setup_sphinx(app: Sphinx, load_parser: bool = False) -> None:
app.add_role("sub-ref", SubstitutionReferenceRole())
app.add_directive("figure-md", FigureMarkdown)

# TODO currently we globally replace sphinx's transform,
# to overcome issues it has (https://github.com/sphinx-doc/sphinx/pull/12730),
# but once this PR is merged/released, we should remove this
app.registry.transforms.remove(SphinxUnreferencedFootnotesDetector)
app.add_transform(UnreferencedFootnotesDetector)

app.add_post_transform(MystReferenceResolver)

# override only the html writer visit methods for rubric, to use the "level" attribute
Expand Down
Loading

0 comments on commit 7a17af9

Please sign in to comment.