Break Up LaTeX Source on Section Macros

.github/workflows/on-pull-request.yml

@@ -2,9 +2,9 @@ name: Node.js CI
 
 on:
     push:
-        branches: ["main"]
+        branches: ["main", "pretext"]
     pull_request:
-        branches: ["main"]
+        branches: ["main", "pretext"]
 
 jobs:
     build:

packages/unified-latex-to-pretext/libs/pre-conversion-subs/break-on-boundaries.ts

@@ -0,0 +1,134 @@
+import { env, arg } from "@unified-latex/unified-latex-builder";
+import * as Ast from "@unified-latex/unified-latex-types";
+import { getNamedArgsContent } from "@unified-latex/unified-latex-util-arguments";
+import {
+    anyEnvironment,
+    anyMacro,
+    match,
+} from "@unified-latex/unified-latex-util-match";
+import { printRaw } from "@unified-latex/unified-latex-util-print-raw";
+import { replaceNode } from "@unified-latex/unified-latex-util-replace";
+import {
+    splitOnMacro,
+    unsplitOnMacro,
+} from "@unified-latex/unified-latex-util-split";
+import { visit } from "@unified-latex/unified-latex-util-visit";
+
+// all divisions [division macro, environment]
+const divisions: [string, string][] = [
+    ["part", "_part"],
+    ["chapter", "_chapter"],
+    ["section", "_section"],
+    ["subsection", "_subsection"],
+    ["subsubsection", "_subsubsection"],
+    ["paragraph", "_paragraph"],
+    ["subparagraph", "_subparagraph"],
+];
+
+/**
+ * Breaks up division macros into environments. Returns a list of warning messages
+ * for any groups that were removed.
+ */
+export function breakOnBoundaries(ast: Ast.Ast): string[] {
+    // messages for any groups removed
+    const messages: string[] = [];
+
+    visit(ast, (node, info) => {
+        // needs to be an environment, root, or group node
+        if (
+            !(
+                anyEnvironment(node) ||
+                node.type === "root" ||
+                match.group(node)
+            ) ||
+            // skip math mode
+            info.context.hasMathModeAncestor
+        ) {
+            return;
+        }
+        // if it's an environment, make sure it isn't a newly created one
+        else if (
+            anyEnvironment(node) &&
+            divisions.map((x) => x[1]).includes(node.env)
+        ) {
+            return;
+        }
+
+        // if it's a group, push a warning message
+        if (match.group(node)) {
+            messages.push(
+                `Warning: hoisted out of a group, which might break the LaTeX code. { group: ${printRaw(
+                    node
+                )} }`
+            );
+        }
+
+        // now break up the divisions, starting at part
+        node.content = breakUp(node.content, divisions, 0);
+    });
+
+    replaceNode(ast, (node) => {
+        // remove all old division nodes
+        if (
+            anyMacro(node) &&
+            divisions.map((x) => x[0]).includes(node.content)
+        ) {
+            return null;
+        }
+        // remove groups
+        else if (match.group(node)) {
+            return node.content;
+        }
+    });
+
+    return messages;
+}
+
+/**
+ * Recursively breaks up the ast at the division macros.
+ */
+function breakUp(
+    content: Ast.Node[],
+    divisions: [string, string][],
+    depth: number
+): Ast.Node[] {
+    // broke up all divisions
+    if (depth > 6) {
+        return content;
+    }
+
+    const splits = splitOnMacro(content, divisions[depth][0]);
+
+    // go through each segment to recursively break
+    for (let i = 0; i < splits.segments.length; i++) {
+        splits.segments[i] = breakUp(splits.segments[i], divisions, depth + 1);
+    }
+
+    createEnvironments(splits, divisions[depth][1]);
+
+    // rebuild this part of the ast
+    return unsplitOnMacro(splits);
+}
+
+/**
+ * Create the new environments that replace the division macros
+ */
+function createEnvironments(
+    splits: { segments: Ast.Node[][]; macros: Ast.Macro[] },
+    newEnviron: string
+): void {
+    // loop through segments (skipping first segment)
+    for (let i = 1; i < splits.segments.length; i++) {
+        // get the title
+        const title = getNamedArgsContent(splits.macros[i - 1])["title"];
+        const titleArg: Ast.Argument[] = [];
+
+        // create title argument
+        if (title) {
+            titleArg.push(arg(title, { braces: "[]" }));
+        }
+
+        // wrap segment with a new environment
+        splits.segments[i] = [env(newEnviron, splits.segments[i], titleArg)];
+    }
+}

packages/unified-latex-to-pretext/libs/pre-conversion-subs/expand-user-defined-macros.ts

@@ -0,0 +1,49 @@
+import * as Ast from "@unified-latex/unified-latex-types";
+import {
+    expandMacrosExcludingDefinitions,
+    listNewcommands,
+} from "@unified-latex/unified-latex-util-macros";
+import { attachMacroArgs } from "@unified-latex/unified-latex-util-arguments";
+import { anyMacro } from "@unified-latex/unified-latex-util-match";
+import { EXIT, visit } from "@unified-latex/unified-latex-util-visit";
+
+type NewCommandSpec = ReturnType<typeof listNewcommands>[number];
+
+/**
+ * Expands user-defined macros
+ */
+export function expandUserDefinedMacros(ast: Ast.Ast): void {
+    const newcommands = listNewcommands(ast);
+
+    // get a set of all macros to be expanded
+    const macrosToExpand = new Set(newcommands.map((command) => command.name));
+
+    const macroInfo = Object.fromEntries(
+        newcommands.map((m) => [m.name, { signature: m.signature }])
+    );
+
+    // recursively expand at most 100 times
+    for (let i = 0; i < 100; i++) {
+        // check if any macros still need expanding
+        if (!needToExpand(ast, macrosToExpand)) {
+            break;
+        }
+
+        // attach the arguments to each macro before processing it
+        attachMacroArgs(ast, macroInfo);
+        expandMacrosExcludingDefinitions(ast, newcommands);
+    }
+}
+
+function needToExpand(ast: Ast.Ast, macros: Set<string>): boolean {
+    let needExpand = false;
+
+    visit(ast, (node) => {
+        if (anyMacro(node) && macros.has(node.content)) {
+            needExpand = true;
+            EXIT;
+        }
+    });
+
+    return needExpand;
+}

packages/unified-latex-to-pretext/libs/pre-conversion-subs/report-unsupported-macro-katex.ts

@@ -0,0 +1,27 @@
+import * as Ast from "@unified-latex/unified-latex-types";
+import { anyMacro, match } from "@unified-latex/unified-latex-util-match";
+import { visit } from "@unified-latex/unified-latex-util-visit";
+import { KATEX_SUPPORT } from "./katex-subs";
+
+/**
+ * Return list of macros unsupported by Katex
+ */
+export function reportMacrosUnsupportedByKatex(ast: Ast.Ast): string[] {
+    const unsupported: string[] = [];
+
+    // match a macro supported by Katex
+    const isSupported = match.createMacroMatcher(KATEX_SUPPORT.macros);
+
+    // visit all nodes
+    visit(ast, (node, info) => {
+        // macro in math mode
+        if (anyMacro(node) && info.context.hasMathModeAncestor) {
+            // check if not supported by katex
+            if (!isSupported(node)) {
+                unsupported.push((node as Ast.Macro).content);
+            }
+        }
+    });
+
+    return unsupported;
+}

packages/unified-latex-to-pretext/tests/break-on-boundaries.test.ts

@@ -0,0 +1,149 @@
+import { describe, it, expect } from "vitest";
+import util from "util";
+import { getParser } from "@unified-latex/unified-latex-util-parse";
+import { printRaw } from "@unified-latex/unified-latex-util-print-raw";
+import { breakOnBoundaries } from "../libs/pre-conversion-subs/break-on-boundaries";
+
+// Make console.log pretty-print by default
+const origLog = console.log;
+console.log = (...args) => {
+    origLog(...args.map((x) => util.inspect(x, false, 10, true)));
+};
+
+describe("unified-latex-to-pretext:break-on-boundaries", () => {
+    let value: string;
+
+    it("can break on parts", () => {
+        value = String.raw`\part{Foo}Hi, this is a part\part{Bar}This is another part`;
+
+        const parser = getParser();
+        const ast = parser.parse(value);
+
+        expect(breakOnBoundaries(ast)).toEqual([]);
+
+        expect(printRaw(ast)).toEqual(
+            String.raw`\begin{_part}[Foo]Hi, this is a part\end{_part}\begin{_part}[Bar]This is another part\end{_part}`
+        );
+    });
+
+    it("can break on a combination of divisions", () => {
+        value = String.raw`\part{part1}\section{Section1}Hi, this is a section\chapter{chap1}This is a chapter\section{Subsection2}`;
+
+        const parser = getParser();
+        const ast = parser.parse(value);
+
+        expect(breakOnBoundaries(ast)).toEqual([]);
+
+        expect(printRaw(ast)).toEqual(
+            String.raw`\begin{_part}[part1]` +
+                String.raw`\begin{_section}[Section1]Hi, this is a section\end{_section}` +
+                String.raw`\begin{_chapter}[chap1]This is a chapter` +
+                String.raw`\begin{_section}[Subsection2]\end{_section}\end{_chapter}\end{_part}`
+        );
+    });
+
+    it("can break on divisions wrapped around by a document environment", () => {
+        value = String.raw`\begin{document}\section{Baz}Hi, this is a subsection\subsubsection{Foo}description.\end{document}`;
+
+        const parser = getParser();
+        const ast = parser.parse(value);
+
+        expect(breakOnBoundaries(ast)).toEqual([]);
+
+        expect(printRaw(ast)).toEqual(
+            String.raw`\begin{document}\begin{_section}[Baz]Hi, this is a subsection` +
+                String.raw`\begin{_subsubsection}[Foo]description.\end{_subsubsection}` +
+                String.raw`\end{_section}\end{document}`
+        );
+    });
+
+    it("can break on divisions wrapped around by different environments", () => {
+        value =
+            String.raw`\begin{center}\part{name}Hi, this is a part\begin{environ}` +
+            String.raw`\subparagraph{title}description.\end{environ}\end{center}`;
+
+        const parser = getParser();
+        const ast = parser.parse(value);
+
+        expect(breakOnBoundaries(ast)).toEqual([]);
+
+        expect(printRaw(ast)).toEqual(
+            String.raw`\begin{center}\begin{_part}[name]Hi, this is a part` +
+                String.raw`\begin{environ}\begin{_subparagraph}[title]description.` +
+                String.raw`\end{_subparagraph}\end{environ}\end{_part}\end{center}`
+        );
+    });
+
+    it("can break on divisions in a group", () => {
+        value =
+            String.raw`\begin{document}\chapter{Chap}` +
+            String.raw`{\paragraph{Intro}Introduction.\begin{center}\subparagraph{Conclusion}Conclusion.\end{center}}` +
+            String.raw`Chapter finished.\end{document}`;
+
+        const parser = getParser();
+        const ast = parser.parse(value);
+
+        expect(breakOnBoundaries(ast)).toEqual([
+            String.raw`Warning: hoisted out of a group, which might break the LaTeX code. ` +
+                String.raw`{ group: {\paragraph{Intro}Introduction.\begin{center}\subparagraph{Conclusion}Conclusion.\end{center}} }`,
+        ]);
+
+        expect(printRaw(ast)).toEqual(
+            String.raw`\begin{document}\begin{_chapter}[Chap]\begin{_paragraph}[Intro]Introduction.` +
+                String.raw`\begin{center}\begin{_subparagraph}[Conclusion]Conclusion.\end{_subparagraph}` +
+                String.raw`\end{center}\end{_paragraph}Chapter finished.\end{_chapter}\end{document}`
+        );
+    });
+
+    it("can break on divisions in nested groups", () => {
+        value =
+            String.raw`\part{part1}{\subsection{Intro}description.` +
+            String.raw`\subsubsection{body}more text.{\subparagraph{Conclusion}Conclusion.}}`;
+
+        const parser = getParser();
+        const ast = parser.parse(value);
+
+        expect(breakOnBoundaries(ast)).toEqual([
+            String.raw`Warning: hoisted out of a group, which might break the LaTeX code. ` +
+                String.raw`{ group: {\subsection{Intro}description.\subsubsection{body}more text.{\subparagraph{Conclusion}Conclusion.}} }`,
+            String.raw`Warning: hoisted out of a group, which might break the LaTeX code. ` +
+                String.raw`{ group: {\subparagraph{Conclusion}Conclusion.} }`,
+        ]);
+
+        expect(printRaw(ast)).toEqual(
+            String.raw`\begin{_part}[part1]\begin{_subsection}[Intro]description.` +
+                String.raw`\begin{_subsubsection}[body]more text.\begin{_subparagraph}[Conclusion]Conclusion.` +
+                String.raw`\end{_subparagraph}\end{_subsubsection}\end{_subsection}\end{_part}`
+        );
+    });
+
+    it("can break on divisions with latex in their titles", () => {
+        value = String.raw`\chapter{$x = \frac{1}{2}$}Chapter 1\subsection{\"name\_1\" \(\mathbb{R}\)}This is subsection`;
+
+        const parser = getParser();
+        const ast = parser.parse(value);
+
+        expect(breakOnBoundaries(ast)).toEqual([]);
+
+        expect(printRaw(ast)).toEqual(
+            String.raw`\begin{_chapter}[$x = \frac{1}{2}$]Chapter 1` +
+                String.raw`\begin{_subsection}[\"name\_1\" \(\mathbb{R}\)]This is subsection` +
+                String.raw`\end{_subsection}\end{_chapter}`
+        );
+    });
+
+    it("can break on divisions and trim whitespace around division beginnings and endings", () => {
+        value = String.raw` \subsubsection{first}subsection 1  \paragraph{body}This is paragraph    `;
+
+        const parser = getParser();
+        const ast = parser.parse(value);
+
+        expect(breakOnBoundaries(ast)).toEqual([]);
+
+        expect(printRaw(ast)).toEqual(
+            String.raw`\begin{_subsubsection}[first]subsection 1` +
+                String.raw`\begin{_paragraph}[body]This is paragraph` +
+                String.raw`\end{_paragraph}\end{_subsubsection}`
+        );
+    });
+});