diff --git a/packages/unified-latex-util-argspec/libs/argspec-parser.ts b/packages/unified-latex-util-argspec/libs/argspec-parser.ts index d32b3d52..8c3015c1 100644 --- a/packages/unified-latex-util-argspec/libs/argspec-parser.ts +++ b/packages/unified-latex-util-argspec/libs/argspec-parser.ts @@ -17,7 +17,10 @@ function getDecorators(node: ArgSpec.Node): string { * Print an `xparse` argument specification AST * to a string. */ -export function printRaw(node: ArgSpec.Ast, root = false): string { +export function printRaw( + node: ArgSpec.Node | string | (ArgSpec.Node | string)[], + root = false +): string { if (typeof node === "string") { return node; } diff --git a/packages/unified-latex-util-argspec/libs/argspec-types.ts b/packages/unified-latex-util-argspec/libs/argspec-types.ts index 998aa470..a6dfca7c 100644 --- a/packages/unified-latex-util-argspec/libs/argspec-types.ts +++ b/packages/unified-latex-util-argspec/libs/argspec-types.ts @@ -1,12 +1,5 @@ export type Ast = Node[] | Node; -export type Node = - | Optional - | Mandatory - | Verbatim - | Body - | Group - | Until - | string; +export type Node = Optional | Mandatory | Verbatim | Body | Group | Until; type Optional = OptionalArg | OptionalStar | OptionalToken | Embellishment; interface AstNode { type: string; @@ -36,7 +29,7 @@ interface OptionalToken extends LeadingWhitespace, AstNode { } export interface Embellishment extends DefaultArgument, AstNode { type: "embellishment"; - embellishmentTokens: (Group | string)[]; + embellishmentTokens: string[]; } interface Mandatory extends LeadingWhitespace, DefaultArgument, Arg { type: "mandatory"; diff --git a/packages/unified-latex-util-arguments/libs/gobble-arguments.ts b/packages/unified-latex-util-arguments/libs/gobble-arguments.ts index d4cd86c1..004ac706 100644 --- a/packages/unified-latex-util-arguments/libs/gobble-arguments.ts +++ b/packages/unified-latex-util-arguments/libs/gobble-arguments.ts @@ -29,63 +29,65 @@ export function gobbleArguments( argSpec = parseArgspec(argSpec); } - // argSpec may be mutated below. - argSpec = structuredClone(argSpec); - const args: Ast.Argument[] = []; - let totalNodesRemoved = 0; + let nodesRemoved = 0; for (const spec of argSpec) { - const innerArgs: Ast.Argument[] = []; - let argument: Ast.Argument | null; - let nodesRemoved: number, matchNum: number | undefined; - do { - ({ argument, nodesRemoved, matchNum } = gobbleSingleArgument( + if (spec.type === "embellishment") { + // We need special behavior for embellishment argspecs. + // Because an embellishment argspec specifies more than one argument, + // we need to keep gobbling arguments until we've got them all. + const remainingTokens = new Set(spec.embellishmentTokens); + const argForToken = Object.fromEntries( + spec.embellishmentTokens.map((t) => [t, emptyArg()]) + ); + + let { argument, nodesRemoved: removed } = gobbleSingleArgument( nodes, - spec, + embellishmentSpec(remainingTokens), startPos - )); - if (argument) { - innerArgs[nthHoleIndex(innerArgs, matchNum || 1)] = argument; - totalNodesRemoved += nodesRemoved; + ); + while (argument) { + const token = argument.openMark; + remainingTokens.delete(token); + argForToken[token] = argument; + nodesRemoved += removed; + const newSpec = embellishmentSpec(remainingTokens); + ({ argument, nodesRemoved: removed } = gobbleSingleArgument( + nodes, + newSpec, + startPos + )); } - // Usual ArgSpec ends this loop by returning `matchNum === undefined`. - // Embellishment argspecs always return matchNum. They end this loop - // by returning falsy `argument` value. - } while (argument && matchNum !== undefined); - // Fill out missing arguments. - if (matchNum === undefined) { - matchNum = argument ? 0 : 1; - } - let i = -1; - while (matchNum--) { - i = nextHoleIndex(innerArgs, i); - innerArgs[i] = arg([], { openMark: "", closeMark: "" }); + args.push(...spec.embellishmentTokens.map((t) => argForToken[t])); + } else { + const { argument, nodesRemoved: removed } = gobbleSingleArgument( + nodes, + spec, + startPos + ); + args.push(argument || emptyArg()); + nodesRemoved += removed; } - args.push(...innerArgs); } - return { args, nodesRemoved: totalNodesRemoved }; + return { args, nodesRemoved }; } -function nextHoleIndex( - arr: (NonNullable | undefined)[], - startPos: number -) { - do { - startPos++; - } while (typeof arr[startPos] !== "undefined"); - return startPos; +/** + * Create an embellishment argspec from a set of tokens. + */ +function embellishmentSpec(tokens: Set): ArgSpec.Embellishment { + return { + type: "embellishment", + embellishmentTokens: [...tokens], + }; } + /** - * Get n-th left-most hole in `arr`. `n` is a 1-based integer, - * so putting ([], 1) would return 0. + * Create an empty argument. */ -function nthHoleIndex(arr: (NonNullable | undefined)[], n: number) { - let i = -1; - while (n--) { - i = nextHoleIndex(arr, i); - } - return i; +function emptyArg(): Ast.Argument { + return arg([], { openMark: "", closeMark: "" }); } diff --git a/packages/unified-latex-util-arguments/libs/gobble-single-argument.ts b/packages/unified-latex-util-arguments/libs/gobble-single-argument.ts index 2167e709..57315eed 100644 --- a/packages/unified-latex-util-arguments/libs/gobble-single-argument.ts +++ b/packages/unified-latex-util-arguments/libs/gobble-single-argument.ts @@ -12,11 +12,6 @@ import { scan } from "@unified-latex/unified-latex-util-scan"; * Gobbles an argument of whose type is specified * by `argSpec` starting at the position `startPos`. * If an argument couldn't be found, `argument` will be `null`. - * `matchNum` is undefined in most cases. It is optionally provided - * if the provided `argSpec` may match multiple arguments. In such cases, - * if there is a matched `argument`, `matchNum` represents a 1-based index of - * that argument, and if there's no `argument`, it represents the count of - * missing arguments. */ export function gobbleSingleArgument( nodes: Ast.Node[], @@ -25,7 +20,6 @@ export function gobbleSingleArgument( ): { argument: Ast.Argument | null; nodesRemoved: number; - matchNum?: number; } { if (typeof argSpec === "string" || !argSpec.type) { throw new Error( @@ -39,8 +33,6 @@ export function gobbleSingleArgument( let currPos = startPos; - let matchNum: number | undefined = undefined; - // Gobble whitespace from `currPos` onward, updating `currPos`. // If `argSpec` specifies leading whitespace is not allowed, // this function does nothing. @@ -76,13 +68,10 @@ export function gobbleSingleArgument( match.comment(currNode) || match.parbreak(currNode) ) { - const ret: { argument: null; nodesRemoved: number; matchNum?: number } = - { argument, nodesRemoved: 0 }; - if (argSpec.type === "embellishment") { - ret.matchNum = normalizeEmbellishmentTokens( - argSpec.embellishmentTokens - ).length; - } + const ret: { argument: null; nodesRemoved: number } = { + argument, + nodesRemoved: 0, + }; return ret; } @@ -195,13 +184,7 @@ export function gobbleSingleArgument( break; } case "embellishment": { - // Split tokens into single characters - const tokens = normalizeEmbellishmentTokens( - argSpec.embellishmentTokens - ); - argSpec.embellishmentTokens = tokens; // ArgSpec is mutated here - for (let i = 0; i < tokens.length; i++) { - const token = tokens[i]; + for (const token of argSpec.embellishmentTokens) { const bracePos = findBracePositions(nodes, currPos, token); if (!bracePos) { continue; @@ -215,13 +198,8 @@ export function gobbleSingleArgument( } ); currPos = bracePos[1] + 1; - matchNum = i + 1; // 1-based indices - tokens.splice(i, 1); break; } - if (!argument) { - matchNum = tokens.length; - } break; } default: @@ -234,7 +212,7 @@ export function gobbleSingleArgument( // if we did not consume an argument, we don't want to consume the whitespace. const nodesRemoved = argument ? currPos - startPos : 0; nodes.splice(startPos, nodesRemoved); - return { argument, nodesRemoved, matchNum }; + return { argument, nodesRemoved }; } function cloneStringNode(node: Ast.String, content: string): Ast.String { @@ -333,27 +311,3 @@ function findBracePositions( } return [openMarkPos, closeMarkPos]; } - -function normalizeEmbellishmentTokens( - tokens: (ArgSpec.Group | string)[] -): string[] { - return tokens.flatMap((token) => { - if (typeof token === "string") { - return token.split(""); - } - // xparse (as of 2023-02-02) accepts single character enclosed in braces {}. - // It does not allow more nesting, e.g. e{{{_}}} produces an error. - if (token.content.length === 1) { - const bracedToken = token.content[0]; - if (typeof bracedToken === "string" && bracedToken.length === 1) { - return bracedToken; - } - } - console.warn( - `Embellishment token should be a single character, but got ${printRaw( - token - )}` - ); - return []; - }); -} diff --git a/packages/unified-latex-util-arguments/tests/attach-arguments-in-array.test.ts b/packages/unified-latex-util-arguments/tests/attach-arguments-in-array.test.ts index 421e19e5..5efb02cf 100644 --- a/packages/unified-latex-util-arguments/tests/attach-arguments-in-array.test.ts +++ b/packages/unified-latex-util-arguments/tests/attach-arguments-in-array.test.ts @@ -1,3 +1,4 @@ +import { describe, expect, it } from "vitest"; import util from "util"; import { strToNodes } from "../../test-common"; import { attachMacroArgsInArray } from "../libs/attach-arguments"; @@ -228,9 +229,10 @@ describe("unified-latex-util-arguments", () => { ], }, ]); - + }); + it("can attach embellishment arguments in array", () => { // embellishments - nodes = strToNodes("\\xxx^a\\xxx_b\\xxx_b^a"); + let nodes = strToNodes("\\xxx^a\\xxx_b\\xxx_b^a"); attachMacroArgsInArray(nodes, { xxx: { signature: "e{^_}" } }); expect(nodes).toEqual([ { diff --git a/packages/unified-latex-util-arguments/tests/attach-arguments.test.ts b/packages/unified-latex-util-arguments/tests/attach-arguments.test.ts index dfe85172..447e0ee5 100644 --- a/packages/unified-latex-util-arguments/tests/attach-arguments.test.ts +++ b/packages/unified-latex-util-arguments/tests/attach-arguments.test.ts @@ -1,3 +1,4 @@ +import { describe, expect, it } from "vitest"; import util from "util"; import * as Ast from "@unified-latex/unified-latex-types"; import { attachMacroArgs } from "../libs/attach-arguments"; diff --git a/packages/unified-latex-util-arguments/tests/gobble-arguments.test.ts b/packages/unified-latex-util-arguments/tests/gobble-arguments.test.ts index 6fe09513..d9e68e3d 100644 --- a/packages/unified-latex-util-arguments/tests/gobble-arguments.test.ts +++ b/packages/unified-latex-util-arguments/tests/gobble-arguments.test.ts @@ -1,8 +1,9 @@ +import { describe, expect, it } from "vitest"; import { VFile } from "unified-lint-rule/lib"; import util from "util"; import { trimRenderInfo } from "../../unified-latex-util-render-info"; import * as Ast from "@unified-latex/unified-latex-types"; -import { parse as parseArgspec } from "../../unified-latex-util-argspec"; +import { parse as parseArgspec } from "@unified-latex/unified-latex-util-argspec"; import { gobbleArguments } from "../libs/gobble-arguments"; import { processLatexToAstViaUnified } from "@unified-latex/unified-latex"; import { arg, s, SP } from "@unified-latex/unified-latex-builder"; @@ -75,7 +76,7 @@ describe("unified-latex-util-arguments", () => { ]); }); - it("can gobble arguments that represents mutiple embellishments", () => { + it("can gobble arguments that represents multiple embellishments", () => { let argspec = parseArgspec("e{_ad}"); value = "_{1234}abcde"; file = processLatexToAstViaUnified().processSync({ value }); @@ -135,7 +136,7 @@ describe("unified-latex-util-arguments", () => { }); expect(nodes).toEqual([]); - // Whitespaces between embellishment arguments should be ignored. + // Whitespace between embellishment arguments should be ignored. argspec = parseArgspec("e{^_}"); value = "^1 _2"; file = processLatexToAstViaUnified().processSync({ value }); diff --git a/packages/unified-latex-util-arguments/tests/gobble-single-argument.test.ts b/packages/unified-latex-util-arguments/tests/gobble-single-argument.test.ts index 7c2f7e22..0a0695f1 100644 --- a/packages/unified-latex-util-arguments/tests/gobble-single-argument.test.ts +++ b/packages/unified-latex-util-arguments/tests/gobble-single-argument.test.ts @@ -1,8 +1,9 @@ +import { describe, expect, it } from "vitest"; import { VFile } from "unified-lint-rule/lib"; import util from "util"; import { trimRenderInfo } from "../../unified-latex-util-render-info"; import * as Ast from "@unified-latex/unified-latex-types"; -import { parse as parseArgspec } from "../../unified-latex-util-argspec"; +import { parse as parseArgspec } from "@unified-latex/unified-latex-util-argspec"; import { gobbleSingleArgument } from "../libs/gobble-single-argument"; import { processLatexToAstViaUnified } from "@unified-latex/unified-latex"; @@ -697,7 +698,6 @@ describe("unified-latex-util-arguments", () => { { argument: null, nodesRemoved: 0, - matchNum: 0, } ); @@ -716,7 +716,6 @@ describe("unified-latex-util-arguments", () => { closeMark: "", }, nodesRemoved: 3, - matchNum: 1, }); ast = [ @@ -734,7 +733,6 @@ describe("unified-latex-util-arguments", () => { closeMark: "", }, nodesRemoved: 2, - matchNum: 2, }); }); it("can gobble embellishments whose token is in a group one level deep", () => { @@ -749,7 +747,6 @@ describe("unified-latex-util-arguments", () => { closeMark: "", }, nodesRemoved: 2, - matchNum: 1, }); }); });