feat: deprecate markdown-to-jsx and support Latex in markdown

apps/console/package.json

@@ -57,9 +57,7 @@
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-hook-form": "^7.51.0",
-    "react-markdown": "^8.0.3",
     "reactflow": "^11.8.3",
-    "remark-frontmatter": "^4.0.1",
     "sharp": "^0.32.6",
     "shiki": "^0.11.1",
     "tailwindcss-animate": "^1.0.6",

apps/console/src/components/index.ts

@@ -1,5 +1,4 @@
 export * from "./AuthPageBase";
 export * from "./ChangePasswordForm";
 export * from "./LoginForm";
-export * from "./ModelReadmeMarkdown";
 export * from "./OnboardingForm";

package.json

@@ -36,6 +36,7 @@
     "turbo": "latest",
     "typescript": "^5.5.4",
     "yaml": "^2.5.0"
+
   },
   "engines": {
     "node": ">=20.0.0"

packages/toolkit/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@instill-ai/toolkit",
-  "version": "0.112.0",
+  "version": "0.113.0-rc.2",
   "description": "Instill AI's frontend toolkit",
   "repository": "https://github.com/instill-ai/design-system.git",
   "bugs": "https://github.com/instill-ai/design-system/issues",
@@ -158,9 +158,13 @@
     "react-avatar-editor": "^13.0.2",
     "react-error-boundary": "^4.0.13",
     "react-hook-form": "^7.51.0",
+    "react-markdown": "^9.0.1",
     "react-syntax-highlighter": "^15.5.0",
     "reactflow": "^11.10.0",
     "recharts": "2.12.7",
+    "rehype-katex": "^7.0.1",
+    "remark-gfm": "^4.0.0",
+    "remark-math": "^6.0.0",
     "sanitize-html": "^2.13.0",
     "semver": "^7.5.4",
     "server-only": "^0.0.1",

packages/toolkit/src/lib/markdown/MarkdownViewer.tsx

@@ -0,0 +1,81 @@
+"use client";
+
+import * as React from "react";
+import ReactMarkdown from "react-markdown";
+import rehypeKatex from "rehype-katex";
+import remarkGfm from "remark-gfm";
+import remarkMath from "remark-math";
+import sanitizeHtml from "sanitize-html";
+
+import { cn } from "@instill-ai/design-system";
+
+import { preprocessLaTeX } from "./preprocessLatex";
+
+export const MarkdownViewer = ({
+  className,
+  markdown,
+}: {
+  className?: string;
+  markdown: string;
+}) => {
+  const sanitizedHtmlText = sanitizeHtml(markdown ?? "");
+
+  const processedText = preprocessLaTeX(sanitizedHtmlText);
+
+  const remarkPlugins = [
+    remarkGfm,
+    [remarkMath, { singleDollarTextMath: true }],
+  ];
+
+  const rehypePlugins = [[rehypeKatex, { output: "mathml" }]];
+
+  return (
+    <React.Fragment>
+      <style jsx={true}>{`
+        .markdown-body a {
+          word-break: break-all !important;
+        }
+
+        .markdown-body pre code {
+          white-space: pre-wrap !important;
+        }
+
+        .markdown-body p {
+          white-space: pre-wrap !important;
+        }
+
+        .markdown-body ul > li {
+          white-space: pre-wrap !important;
+        }
+
+        .markdown-body ol > li {
+          white-space: pre-wrap !important;
+        }
+
+        .markdown-body h1,
+        h2,
+        h3,
+        h4,
+        h5,
+        h6 {
+          white-space: pre-wrap !important;
+        }
+      `}</style>
+      <article
+        className={cn(
+          "markdown-body w-full overflow-x-scroll rounded-b-sm px-1.5 py-1",
+          className,
+        )}
+      >
+        <ReactMarkdown
+          /* @ts-expect-error remark and rehype has type conflicts */
+          remarkPlugins={remarkPlugins}
+          /* @ts-expect-error remark and rehype has type conflicts */
+          rehypePlugins={rehypePlugins}
+        >
+          {processedText}
+        </ReactMarkdown>
+      </article>
+    </React.Fragment>
+  );
+};

packages/toolkit/src/lib/markdown/index.ts

@@ -0,0 +1,2 @@
+export * from "./MarkdownViewer";
+export * from "./preprocessLatex";

packages/toolkit/src/lib/markdown/preprocessLatex.test.ts

@@ -0,0 +1,92 @@
+import { describe, expect, test } from "vitest";
+
+import { preprocessLaTeX } from "./preprocessLatex";
+
+describe("preprocessLaTeX", () => {
+  test("returns the same string if no LaTeX patterns are found", () => {
+    const content = "This is a test string without LaTeX";
+    expect(preprocessLaTeX(content)).toBe(content);
+  });
+
+  test("escapes dollar signs followed by digits", () => {
+    const content = "Price is $50 and $100";
+    const expected = "Price is \\$50 and \\$100";
+    expect(preprocessLaTeX(content)).toBe(expected);
+  });
+
+  test("does not escape dollar signs not followed by digits", () => {
+    const content = "This $variable is not escaped";
+    expect(preprocessLaTeX(content)).toBe(content);
+  });
+
+  test("preserves existing LaTeX expressions", () => {
+    const content = "Inline $x^2 + y^2 = z^2$ and block $$E = mc^2$$";
+    expect(preprocessLaTeX(content)).toBe(content);
+  });
+
+  test("handles mixed LaTeX and currency", () => {
+    const content = "LaTeX $x^2$ and price $50";
+    const expected = "LaTeX $x^2$ and price \\$50";
+    expect(preprocessLaTeX(content)).toBe(expected);
+  });
+
+  test("converts LaTeX delimiters", () => {
+    const content = "Brackets \\[x^2\\] and parentheses \\(y^2\\)";
+    const expected = "Brackets $$x^2$$ and parentheses $y^2$";
+    expect(preprocessLaTeX(content)).toBe(expected);
+  });
+
+  test("escapes mhchem commands", () => {
+    const content = "$\\ce{H2O}$ and $\\pu{123 J}$";
+    const expected = "$\\\\ce{H2O}$ and $\\\\pu{123 J}$";
+    expect(preprocessLaTeX(content)).toBe(expected);
+  });
+
+  test("handles complex mixed content", () => {
+    const content = `
+      LaTeX inline $x^2$ and block $$y^2$$
+      Currency $100 and $200
+      Chemical $\\ce{H2O}$
+      Brackets \\[z^2\\]
+    `;
+    const expected = `
+      LaTeX inline $x^2$ and block $$y^2$$
+      Currency \\$100 and \\$200
+      Chemical $\\\\ce{H2O}$
+      Brackets $$z^2$$
+    `;
+    expect(preprocessLaTeX(content)).toBe(expected);
+  });
+
+  test("handles empty string", () => {
+    expect(preprocessLaTeX("")).toBe("");
+  });
+
+  test("preserves code blocks", () => {
+    const content = "```\n$100\n```\nOutside $200";
+    const expected = "```\n$100\n```\nOutside \\$200";
+    expect(preprocessLaTeX(content)).toBe(expected);
+  });
+
+  test("handles multiple currency values in a sentence", () => {
+    const content = "I have $50 in my wallet and $100 in the bank.";
+    const expected = "I have \\$50 in my wallet and \\$100 in the bank.";
+    expect(preprocessLaTeX(content)).toBe(expected);
+  });
+
+  test("preserves LaTeX expressions with numbers", () => {
+    const content = "The equation is $f(x) = 2x + 3$ where x is a variable.";
+    expect(preprocessLaTeX(content)).toBe(content);
+  });
+
+  test("handles currency values with commas", () => {
+    const content = "The price is $1,000,000 for this item.";
+    const expected = "The price is \\$1,000,000 for this item.";
+    expect(preprocessLaTeX(content)).toBe(expected);
+  });
+
+  test("preserves LaTeX expressions with special characters", () => {
+    const content = "The set is defined as $\\{x | x > 0\\}$.";
+    expect(preprocessLaTeX(content)).toBe(content);
+  });
+});

packages/toolkit/src/lib/markdown/preprocessLatex.ts

@@ -0,0 +1,85 @@
+/*
+  Credit:
+  https://github.com/danny-avila/LibreChat/blob/8178ae2a20f95525c3cb41e49409ffd7281ca743/client/src/utils/latex.ts
+*/
+
+/**
+ * Preprocesses LaTeX content by replacing delimiters and escaping certain characters.
+ *
+ * @param content The input string containing LaTeX expressions.
+ * @returns The processed string with replaced delimiters and escaped characters.
+ */
+export function preprocessLaTeX(content: string): string {
+  // Step 1: Protect code blocks
+  const codeBlocks: string[] = [];
+  content = content.replace(/(```[\s\S]*?```|`[^`\n]+`)/g, (_, code) => {
+    codeBlocks.push(code);
+    return `<<CODE_BLOCK_${codeBlocks.length - 1}>>`;
+  });
+
+  // Step 2: Protect existing LaTeX expressions
+  const latexExpressions: string[] = [];
+  content = content.replace(
+    /(\$\$[\s\S]*?\$\$|\\\[[\s\S]*?\\\]|\\\(.*?\\\))/g,
+    (match) => {
+      latexExpressions.push(match);
+      return `<<LATEX_${latexExpressions.length - 1}>>`;
+    },
+  );
+
+  // Step 3: Escape dollar signs that are likely currency indicators
+  content = content.replace(/\$(?=\d)/g, "\\$");
+
+  // Step 4: Restore LaTeX expressions
+  content = content.replace(/<<LATEX_(\d+)>>/g, (match, index) => {
+    const idx = parseInt(index);
+    if (latexExpressions[idx] != null) {
+      return latexExpressions[idx];
+    } else {
+      return match;
+    }
+  });
+
+  // Step 5: Restore code blocks
+  content = content.replace(/<<CODE_BLOCK_(\d+)>>/g, (match, index) => {
+    const idx = parseInt(index);
+    if (codeBlocks[idx] != null) {
+      return codeBlocks[idx];
+    } else {
+      return match;
+    }
+  });
+
+  // Step 6: Apply additional escaping functions
+  content = escapeBrackets(content);
+  content = escapeMhchem(content);
+
+  return content;
+}
+
+export function escapeBrackets(text: string): string {
+  const pattern =
+    /(```[\S\s]*?```|`.*?`)|\\\[([\S\s]*?[^\\])\\]|\\\((.*?)\\\)/g;
+  return text.replace(
+    pattern,
+    (
+      match: string,
+      codeBlock: string | undefined,
+      squareBracket: string | undefined,
+      roundBracket: string | undefined,
+    ): string => {
+      if (codeBlock != null) {
+        return codeBlock;
+      } else if (squareBracket != null) {
+        return `$$${squareBracket}$$`;
+      } else if (roundBracket != null) {
+        return `$${roundBracket}$`;
+      }
+      return match;
+    },
+  );
+}
+
+export function escapeMhchem(text: string) {
+  return text.replaceAll("$\\ce{", "$\\\\ce{").replaceAll("$\\pu{", "$\\\\pu{");
+}