Refactor content parsing and ref resolution

Reference resolution logic has been moved from the renderer to the
parser (invoked by the prerenderer), where refs are now converted to
markdown links using an intermediate `luadox:<refid>` link format. It's
up to the renderer to resolve these links to whatever is appropriate.

This required introducing the notion of an id to references. Ids are
globally unique opaque strings that are tracked by the parser, which the
renderer can consult in order to convert an id to a Reference object.

This refactoring continues to pave the way for #5 and will allow for
different kinds of renderers (not just HTML), where the common logic
that applies to all renderers has been moved to the parser and run
during the prerender stage.

Additionally, tag parsing within content blocks (e.g. handling @tparam,
@note, etc.) has been rewritten and hopefully simplified.
(Parser.parse_raw_content())

Finally, this commit includes some optimizations:
  * Compiled regexp objects are now cached and reused, reducing
    compilation overhead
  * First sentence detection has been rewritten using a more naive,
    lower level approach that is significantly faster.  During
    profiling, get_first_sentence() was the most disproportionately
    expensive functions called.

src/main.py

@@ -31,11 +31,9 @@
 from typing import Generator, Union, Dict, Tuple, Set
 
 from .log import log
-from .assets import assets
 from .parse import *
 from .render import *
 from .prerender import Prerenderer
-from .reference import ManualRef
 
 try:
     # version.py is generated at build time, so we are running from the proper
@@ -51,21 +49,6 @@
 # which case the module name will be inferred.
 BasePathsType = Dict[Union[Tuple[str, ...], None], Set[str]]
 
-# Files from the assets directory to be copied
-ASSETS = [
-    'luadox.css',
-    'prism.css',
-    'prism.js',
-    'js-search.min.js',
-    'search.js',
-    'img/i-left.svg',
-    'img/i-right.svg',
-    'img/i-download.svg',
-    'img/i-github.svg',
-    'img/i-gitlab.svg',
-    'img/i-bitbucket.svg',
-]
-
 class FullHelpParser(argparse.ArgumentParser):
     def error(self, message: str) -> None:
         sys.stderr.write('error: %s\n' % message)
@@ -265,43 +248,9 @@ def main():
     try:
         log.info('prerendering %d pages', len(parser.topsyms))
         toprefs = Prerenderer(parser).process()
-
-        for ref in toprefs:
-            if ref.userdata.get('empty') and ref.implicit:
-                # Reference has no content and it was also implicitly generated, so we don't render it.
-                log.info('not rendering empty %s %s', ref.type, ref.name)
-                continue
-            if isinstance(ref, ManualRef) and ref.name == 'index':
-                typedir = outdir
-            else:
-                typedir = os.path.join(outdir, ref.type)
-            os.makedirs(typedir, exist_ok=True)
-            outfile = os.path.join(typedir, ref.name + '.html')
-            log.info('rendering %s %s -> %s', ref.type, ref.name, outfile)
-            html = renderer.render(ref)
-            with open(outfile, 'w', encoding='utf8') as f:
-                f.write(html)
-
-        js = renderer.render_search_index()
-        with open(os.path.join(outdir, 'index.js'), 'w', encoding='utf8') as f:
-            f.write(js)
-
-        html = renderer.render_search_page()
-        with open(os.path.join(outdir, 'search.html'), 'w', encoding='utf8') as f:
-            f.write(html)
-
-        if not parser.get_reference(ManualRef, 'index'):
-            # The user hasn't specified an index manual page, so we generate a blank
-            # landing page that at least presents the sidebar with available links.
-            html = renderer.render_landing_page()
-            with open(os.path.join(outdir, 'index.html'), 'w', encoding='utf8') as f:
-                f.write(html)
-
-        for name in ASSETS:
-            outfile = os.path.join(outdir, name)
-            if os.path.dirname(name):
-                os.makedirs(os.path.dirname(outfile), exist_ok=True)
-            with open(outfile, 'wb') as f:
-                f.write(assets.get(name))
+        renderer.render(toprefs, outdir)
     except Exception as e:
         log.exception('unhandled error rendering around %s:%s: %s', parser.ctx.file, parser.ctx.line, e)
+        sys.exit(1)
+
+    log.info('done')