From 36b2c34b35c22633f333e0ee1e462dac8a580ea6 Mon Sep 17 00:00:00 2001 From: Hood Chatham Date: Sat, 23 Sep 2023 15:22:10 -0700 Subject: [PATCH] Add Type XRefs to IR (#51) This updates the IR representation of types to a possible list of strings or xref objects. The xrefs can be internal xrefs to types in the project or external ones to types defined in dependencies. I don't currently add xrefs to intrinsic types but that would be a natural thing to do. The renderer currently throws this information away. Rendering it is left to a followup. --- sphinx_js/ir.py | 22 ++- sphinx_js/renderers.py | 50 ++++- sphinx_js/typedoc.py | 173 ++++++++++++------ tests/test_build_js/test_build_js.py | 2 +- .../test_typedoc_analysis.py | 126 +++++++++---- 5 files changed, 270 insertions(+), 103 deletions(-) diff --git a/sphinx_js/ir.py b/sphinx_js/ir.py index 11f0a1b8..7e9c42d3 100644 --- a/sphinx_js/ir.py +++ b/sphinx_js/ir.py @@ -28,8 +28,25 @@ from .analyzer_utils import dotted_path + +@dataclass +class TypeXRef: + name: str + + +@dataclass +class TypeXRefInternal(TypeXRef): + path: list[str] + + +@dataclass +class TypeXRefExternal(TypeXRef): + sourcefilename: str + qualifiedName: str + + #: Human-readable type of a value. None if we don't know the type. -Type = str | None +Type = str | list[str | TypeXRef] | None # In the far future, we may take full control of our RST templates rather than # using the js-domain directives provided by Sphinx. This would give us the # freedom to link type names in formal param lists and param description lists @@ -38,6 +55,7 @@ # (simple for JS, fancy for TS) and can, on request, render it out as either # text or link-having RST. + #: Pathname, full or not, to an object: ReStructuredText = str @@ -98,7 +116,7 @@ class or interface""" @dataclass class TypeParam: name: str - extends: str | None + extends: Type description: ReStructuredText = ReStructuredText("") diff --git a/sphinx_js/renderers.py b/sphinx_js/renderers.py index 6ba5c168..6b592968 100644 --- a/sphinx_js/renderers.py +++ b/sphinx_js/renderers.py @@ -23,7 +23,9 @@ Pathname, Return, TopLevel, + Type, TypeParam, + TypeXRef, ) from .jsdoc import Analyzer as JsAnalyzer from .parsers import PathVisitor @@ -205,16 +207,51 @@ def _formal_params(self, obj: Function | Class) -> str: return "({})".format(", ".join(formals)) + def format_type(self, type: Type, escape: bool = False) -> str: + if not type: + return "" + if isinstance(type, str): + return self.format_type([type]) + it = iter(type) + + def strs() -> Iterator[str]: + for elem in it: + if isinstance(elem, str): + yield elem + else: + xref.append(elem) + return + + res = [] + while True: + xref: list[TypeXRef] = [] + s = "".join(strs()) + if escape: + s = rst.escape(s) + res.append(s) + if not xref: + break + res.append(self.render_xref(xref[0], escape)) + + return "".join(res) + + def render_xref(self, s: TypeXRef, escape: bool = False) -> str: + if escape: + return rst.escape(s.name) + return s.name + def _return_formatter(self, return_: Return) -> tuple[list[str], str]: """Derive heads and tail from ``@returns`` blocks.""" - tail = ("**%s** -- " % rst.escape(return_.type)) if return_.type else "" + tail = "" + if return_.type: + tail += "**%s** -- " % self.format_type(return_.type, escape=True) tail += return_.description return ["returns"], tail def _type_param_formatter(self, tparam: TypeParam) -> tuple[list[str], str] | None: v = tparam.name if tparam.extends: - v += f" extends {tparam.extends}" + v += " extends " + self.format_type(tparam.extends) heads = ["typeparam", v] return heads, tparam.description @@ -225,8 +262,9 @@ def _param_formatter(self, param: Param) -> tuple[list[str], str] | None: return None heads = ["param"] if param.type: - heads.append(param.type) + heads.append(self.format_type(param.type)) heads.append(param.name) + tail = param.description return heads, tail @@ -235,14 +273,14 @@ def _param_type_formatter(self, param: Param) -> tuple[list[str], str] | None: if not param.type: return None heads = ["type", param.name] - tail = rst.escape(param.type) + tail = self.format_type(param.type) return heads, tail def _exception_formatter(self, exception: Exc) -> tuple[list[str], str]: """Derive heads and tail from ``@throws`` blocks.""" heads = ["throws"] if exception.type: - heads.append(exception.type) + heads.append(self.format_type(exception.type)) tail = exception.description return heads, tail @@ -453,7 +491,7 @@ def _template_vars(self, name: str, obj: Attribute) -> dict[str, Any]: # type: is_optional=obj.is_optional, see_also=obj.see_alsos, examples=obj.examples, - type=obj.type, + type=self.format_type(obj.type), content="\n".join(self._content), ) diff --git a/sphinx_js/typedoc.py b/sphinx_js/typedoc.py index c00825d4..1dcfa235 100644 --- a/sphinx_js/typedoc.py +++ b/sphinx_js/typedoc.py @@ -5,7 +5,7 @@ import re import subprocess import textwrap -from collections.abc import Sequence +from collections.abc import Iterable, Iterator, Sequence from errno import ENOENT from functools import cache from inspect import isclass @@ -491,7 +491,7 @@ def _related_types( for t in orig_types: if t.type != "reference": continue - if not t.target: + if not isinstance(t.target, int): continue rtype = converter.index[t.target] pathname = ir.Pathname(rtype.path) @@ -620,26 +620,33 @@ class TypeLiteral(NodeBase): indexSignature: "Signature | None" = None children: Sequence["Member"] = [] - def render(self, converter: Converter) -> str: + def render(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: if self.signatures: - return self.signatures[0].type.render_name(converter) - children = [] + yield from self.signatures[0].type._render_name(converter) + return + yield "{ " index_sig = self.indexSignature if index_sig: assert len(index_sig.parameters) == 1 key = index_sig.parameters[0] - keyname = key.name - keytype = key.type.render_name(converter) - valuetype = index_sig.type.render_name(converter) - children.append(f"[{keyname}: {keytype}]: {valuetype}") + yield "[" + yield key.name + yield ": " + yield from key.type._render_name(converter) + yield "]" + yield ": " + yield from index_sig.type._render_name(converter) + yield "; " for child in self.children: - maybe_optional = "" + yield child.name if child.flags.isOptional: - maybe_optional = "?" - child_type_name = child.type.render_name(converter) - children.append(child.name + maybe_optional + ": " + child_type_name) - return "{" + ", ".join(children) + "}" + yield "?: " + else: + yield ": " + yield from child.type._render_name(converter) + yield "; " + yield "}" def to_ir( self, converter: Converter @@ -884,21 +891,36 @@ def to_ir( return result, self.children +def riffle( + t: Iterable[Iterable[str | ir.TypeXRef]], other: str +) -> Iterator[str | ir.TypeXRef]: + it = iter(t) + try: + yield from next(it) + except StopIteration: + return + for i in it: + yield other + yield from i + + class TypeBase(Base): typeArguments: list["TypeD"] = [] - def render_name(self, converter: Converter) -> str: - name = self._render_name_root(converter) + def render_name(self, converter: Converter) -> list[str | ir.TypeXRef]: + return list(self._render_name(converter)) - if self.typeArguments: - arg_names = ", ".join( - arg.render_name(converter) for arg in self.typeArguments - ) - name += f"<{arg_names}>" + def _render_name(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: + yield from self._render_name_root(converter) - return name + if not self.typeArguments: + return + yield "<" + gen = (arg._render_name(converter) for arg in self.typeArguments) + yield from riffle(gen, ", ") + yield ">" - def _render_name_root(self, converter: Converter) -> str: + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: raise NotImplementedError @@ -906,19 +928,22 @@ class AndOrType(TypeBase): type: Literal["union", "intersection"] types: list["TypeD"] - def _render_name_root(self, converter: Converter) -> str: + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: if self.type == "union": - return "|".join(t.render_name(converter) for t in self.types) + symbol = "|" elif self.type == "intersection": - return " & ".join(t.render_name(converter) for t in self.types) + symbol = " & " + gen = (t._render_name(converter) for t in self.types) + yield from riffle(gen, symbol) class ArrayType(TypeBase): type: Literal["array"] elementType: "TypeD" - def _render_name_root(self, converter: Converter) -> str: - return self.elementType.render_name(converter) + "[]" + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: + yield from self.elementType._render_name(converter) + yield "[]" class OperatorType(TypeBase): @@ -926,78 +951,112 @@ class OperatorType(TypeBase): operator: str target: "TypeD" - def _render_name_root(self, converter: Converter) -> str: - return self.operator + " " + self.target.render_name(converter) + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: + yield self.operator + " " + yield from self.target._render_name(converter) + + +class Target(BaseModel): + sourceFileName: str + qualifiedName: str class IntrinsicType(TypeBase): type: Literal["intrinsic"] name: str - def _render_name_root(self, converter: Converter) -> str: - return self.name + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: + yield self.name class ReferenceType(TypeBase): type: Literal["reference"] name: str - target: Any - - def _render_name_root(self, converter: Converter) -> str: - return self.name + target: int | Target | None + refersToTypeParameter: bool = False + + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: + if self.refersToTypeParameter: + yield self.name + return + if isinstance(self.target, int) and self.target > 0: + node = converter.index[self.target] + yield ir.TypeXRefInternal(self.name, node.path) + return + assert isinstance(self.target, Target) + yield ir.TypeXRefExternal( + self.name, self.target.sourceFileName, self.target.qualifiedName + ) class ReflectionType(TypeBase): type: Literal["reflection"] declaration: Node - def _render_name_root(self, converter: Converter) -> str: + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: if isinstance(self.declaration, TypeLiteral): - return self.declaration.render(converter) + yield from self.declaration.render(converter) + return if isinstance(self.declaration, Callable): + if self.declaration.kindString == "Constructor": + yield "{new (" + else: + yield "(" sig = self.declaration.signatures[0] - params = [] - for param in sig.parameters: - name = param.name - type_name = param.type.render_name(converter) - params.append(f"{name}: {type_name}") - params_str = ", ".join(params) + + def inner(param: Param) -> Iterator[str | ir.TypeXRef]: + yield param.name + ": " + yield from param.type._render_name(converter) + + yield from riffle((inner(param) for param in sig.parameters), ", ") + + yield "): " ret = sig.return_type(converter)[0].type - sig_str = f"({params_str}): {ret}" + assert ret + if isinstance(ret, str): + yield ret + else: + yield from ret if self.declaration.kindString == "Constructor": - sig_str = f"{{new {sig_str}}}" - return sig_str - return "" + yield "}" + return + yield "" + return class LiteralType(TypeBase): type: Literal["literal"] value: Any - def _render_name_root(self, converter: Converter) -> str: + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: if self.value is None: - return "null" + yield "null" + return # TODO: it could be a bigint or a string? if isinstance(self.value, int): - return "number" - return "" + yield "number" + return + yield "" + return class TupleType(TypeBase): type: Literal["tuple"] elements: list["TypeD"] - def _render_name_root(self, converter: Converter) -> str: - types = [t.render_name(converter) for t in self.elements] - return "[" + ", ".join(types) + "]" + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: + types = (t._render_name(converter) for t in self.elements) + yield "[" + yield from riffle(types, ", ") + yield "]" class OtherType(TypeBase): type: Literal["indexedAccess"] - def _render_name_root(self, converter: Converter) -> str: - return "" + def _render_name_root(self, converter: Converter) -> Iterator[str | ir.TypeXRef]: + yield "" AnyNode = Node | Project | Signature diff --git a/tests/test_build_js/test_build_js.py b/tests/test_build_js/test_build_js.py index ca6c245a..0b9e85ba 100644 --- a/tests/test_build_js/test_build_js.py +++ b/tests/test_build_js/test_build_js.py @@ -361,7 +361,7 @@ def test_restructuredtext_injection(self): "injection(a_, b)\n\n" " Arguments:\n" " * **a_** -- Snorf\n\n" - " * **b** (*type_*) -- >>Borf_<<\n\n" + " * **b** (>>type_<<) -- >>Borf_<<\n\n" " Returns:\n" " **rtype_** -- >>Dorf_<<\n", ) diff --git a/tests/test_typedoc_analysis/test_typedoc_analysis.py b/tests/test_typedoc_analysis/test_typedoc_analysis.py index 74af4b7d..3f507fa4 100644 --- a/tests/test_typedoc_analysis/test_typedoc_analysis.py +++ b/tests/test_typedoc_analysis/test_typedoc_analysis.py @@ -1,14 +1,35 @@ +from copy import copy, deepcopy from json import loads from unittest import TestCase import pytest -from sphinx_js.ir import Attribute, Class, Function, Param, Pathname, Return, TypeParam +from sphinx_js.ir import ( + Attribute, + Class, + Function, + Param, + Pathname, + Return, + Type, + TypeParam, + TypeXRef, + TypeXRefExternal, + TypeXRefInternal, +) from sphinx_js.renderers import AutoClassRenderer, AutoFunctionRenderer from sphinx_js.typedoc import Comment, Converter, Summary, parse from tests.testing import NO_MATCH, TypeDocAnalyzerTestCase, TypeDocTestCase, dict_where +def join_type(t: Type) -> str: + if not t: + return "" + if isinstance(t, str): + return t + return "".join(e.name if isinstance(e, TypeXRef) else e for e in t) + + class PopulateIndexTests(TestCase): def test_top_level_function(self): """Make sure nodes get indexed.""" @@ -287,7 +308,7 @@ def test_interface_function_member(self): def test_variable(self): """Make sure top-level consts and vars are found.""" const = self.analyzer.get_object(["topLevelConst"]) - assert const.type == "number" + assert const.type == ["number"] def test_function(self): """Make sure Functions, Params, and Returns are built properly for @@ -304,7 +325,7 @@ def test_function(self): description="Some number", has_default=True, is_variadic=False, - type="number", + type=["number"], default="1", ), Param( @@ -312,11 +333,11 @@ def test_function(self): description="Some strings", has_default=False, is_variadic=True, - type="string[]", + type=["string", "[]"], ), ] assert func.exceptions == [] - assert func.returns == [Return(type="number", description="The best number")] + assert func.returns == [Return(type=["number"], description="The best number")] def test_constructor(self): """Make sure constructors get attached to classes and analyzed into @@ -371,14 +392,14 @@ def test_getter(self): types.""" getter = self.analyzer.get_object(["gettable"]) assert isinstance(getter, Attribute) - assert getter.type == "number" + assert getter.type == ["number"] def test_setter(self): """Test that we represent setters as Attributes and find the type of their 1 param.""" setter = self.analyzer.get_object(["settable"]) assert isinstance(setter, Attribute) - assert setter.type == "string" + assert setter.type == ["string"] class TypeNameTests(TypeDocAnalyzerTestCase): @@ -406,12 +427,14 @@ def test_basic(self): ("sym", "symbol"), ]: obj = self.analyzer.get_object([obj_name]) - assert obj.type == type_name + assert join_type(obj.type) == type_name def test_named_interface(self): """Make sure interfaces can be referenced by name.""" obj = self.analyzer.get_object(["interfacer"]) - assert obj.params[0].type == "Interface" + assert obj.params[0].type == [ + TypeXRefInternal(name="Interface", path=["./", "types.", "Interface"]) + ] def test_interface_readonly_member(self): """Make sure the readonly modifier doesn't keep us from computing the @@ -419,7 +442,7 @@ def test_interface_readonly_member(self): obj = self.analyzer.get_object(["Interface"]) read_only_num = obj.members[0] assert read_only_num.name == "readOnlyNum" - assert read_only_num.type == "number" + assert read_only_num.type == ["number"] def test_array(self): """Make sure array types are rendered correctly. @@ -429,59 +452,82 @@ def test_array(self): """ obj = self.analyzer.get_object(["overload"]) - assert obj.params[0].type == "string[]" + assert obj.params[0].type == ["string", "[]"] def test_literal_types(self): """Make sure a thing of a named literal type has that type name attached.""" obj = self.analyzer.get_object(["certainNumbers"]) - assert obj.type == "CertainNumbers" + assert obj.type == [ + TypeXRefInternal( + name="CertainNumbers", path=["./", "types.", "CertainNumbers"] + ) + ] def test_unions(self): """Make sure unions get rendered properly.""" obj = self.analyzer.get_object(["union"]) - assert obj.type == "number|string|Color" + assert obj.type == [ + "number", + "|", + "string", + "|", + TypeXRefInternal(name="Color", path=["./", "types.", "Color"]), + ] def test_intersection(self): obj = self.analyzer.get_object(["intersection"]) - assert obj.type == "FooHaver & BarHaver" + assert obj.type == [ + TypeXRefInternal(name="FooHaver", path=["./", "types.", "FooHaver"]), + " & ", + TypeXRefInternal(name="BarHaver", path=["./", "types.", "BarHaver"]), + ] def test_generic_function(self): """Make sure type params appear in args and return types.""" obj = self.analyzer.get_object(["aryIdentity"]) - assert obj.params[0].type == "T[]" - assert obj.returns[0].type == "T[]" + T = ["T", "[]"] + assert obj.params[0].type == T + assert obj.returns[0].type == T def test_generic_member(self): """Make sure members of a class have their type params taken into account.""" obj = self.analyzer.get_object(["add"]) assert len(obj.params) == 2 - assert obj.params[0].type == "T" - assert obj.params[1].type == "T" - assert obj.returns[0].type == "T" + T = ["T"] + assert obj.params[0].type == T + assert obj.params[1].type == T + assert obj.returns[0].type == T def test_constrained_by_interface(self): """Make sure ``extends SomeInterface`` constraints are rendered.""" obj = self.analyzer.get_object(["constrainedIdentity"]) - assert obj.params[0].type == "T" - assert obj.returns[0].type == "T" + T = ["T"] + assert obj.params[0].type == T + assert obj.returns[0].type == T assert obj.type_params[0] == TypeParam( - name="T", extends="Lengthwise", description="the identity type" + name="T", + extends=[ + TypeXRefInternal(name="Lengthwise", path=["./", "types.", "Lengthwise"]) + ], + description="the identity type", ) def test_constrained_by_key(self): """Make sure ``extends keyof SomeObject`` constraints are rendered.""" obj: Function = self.analyzer.get_object(["getProperty"]) assert obj.params[0].name == "obj" - assert obj.params[0].type == "T" - assert obj.params[1].type == "K" + assert join_type(obj.params[0].type) == "T" + assert join_type(obj.params[1].type) == "K" # TODO? # assert obj.returns[0].type == "" assert obj.type_params[0] == TypeParam( name="T", extends=None, description="The type of the object" ) - assert obj.type_params[1] == TypeParam( + tp = copy(obj.type_params[1]) + tp.extends = join_type(tp.extends) + assert tp == TypeParam( name="K", extends="string|number|symbol", description="The type of the key" ) @@ -498,7 +544,9 @@ def test_constrained_by_key(self): def test_class_constrained(self): # TODO: this may belong somewhere else obj: Class = self.analyzer.get_object(["ParamClass"]) - assert obj.type_params[0] == TypeParam( + tp = copy(obj.type_params[0]) + tp.extends = join_type(tp.extends) + assert tp == TypeParam( name="S", extends="number[]", description="The type we contain" ) a = AutoClassRenderer.__new__(AutoClassRenderer) @@ -512,25 +560,29 @@ def test_constrained_by_constructor(self): """Make sure ``new ()`` expressions and, more generally, per-property constraints are rendered properly.""" obj = self.analyzer.get_object(["create1"]) - assert obj.params[0].type == "{new (x: number): A}" + assert join_type(obj.params[0].type) == "{new (x: number): A}" obj = self.analyzer.get_object(["create2"]) - assert obj.params[0].type == "{new (): T}" + assert join_type(obj.params[0].type) == "{new (): T}" def test_utility_types(self): """Test that a representative one of TS's utility types renders.""" obj = self.analyzer.get_object(["partial"]) - assert obj.type == "Partial" + t = deepcopy(obj.type) + t[0].sourcefilename = "xxx" + assert t == [TypeXRefExternal("Partial", "xxx", "Partial"), "<", "string", ">"] def test_constrained_by_property(self): obj = self.analyzer.get_object(["objProps"]) - assert obj.params[0].type == "{label: string}" - assert obj.params[1].type == "{[key: number]: string, label: string}" + assert obj.params[0].type == ["{ ", "label", ": ", "string", "; ", "}"] + assert ( + join_type(obj.params[1].type) == "{ [key: number]: string; label: string; }" + ) def test_optional_property(self): """Make sure optional properties render properly.""" obj = self.analyzer.get_object(["option"]) - assert obj.type == "{a: number, b?: string}" + assert join_type(obj.type) == "{ a: number; b?: string; }" def test_code_in_description(self): obj = self.analyzer.get_object(["codeInDescription"]) @@ -552,14 +604,14 @@ def test_code_in_description(self): def test_destructured(self): obj = self.analyzer.get_object(["destructureTest"]) assert obj.params[0].name == "options.a" - assert obj.params[0].type == "string" + assert join_type(obj.params[0].type) == "string" assert obj.params[1].name == "options.b" - assert obj.params[1].type == "{c: string}" + assert join_type(obj.params[1].type) == "{ c: string; }" obj = self.analyzer.get_object(["destructureTest2"]) assert obj.params[0].name == "options.a" - assert obj.params[0].type == "string" + assert join_type(obj.params[0].type) == "string" assert obj.params[1].name == "options.b" - assert obj.params[1].type == "{c: string}" + assert join_type(obj.params[1].type) == "{ c: string; }" obj = self.analyzer.get_object(["destructureTest3"]) assert obj.params[0].name == "options" - assert obj.params[0].type == "{a: string, b: {c: string}}" + assert join_type(obj.params[0].type) == "{ a: string; b: { c: string; }; }"