From 149934b534fb16456f6ef16066a329759c4bcf02 Mon Sep 17 00:00:00 2001 From: Shuyou Date: Mon, 19 Aug 2024 17:01:02 +0800 Subject: [PATCH 1/2] refactor: md2html --- package.json | 3 + pnpm-lock.yaml | 49 ++++++-- src/blogs | 2 +- src/docs | 2 +- src/utils/common.ts | 272 +------------------------------------------- 5 files changed, 48 insertions(+), 280 deletions(-) diff --git a/package.json b/package.json index 7d40413c5..20937c386 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,7 @@ "@mui/x-tree-view": "^6.17.0", "@octokit/core": "^3.5.1", "@radix-ui/react-slot": "^1.0.2", + "@zilliz/toolkit": "^0.0.9", "axios": "^0.28.0", "class-variance-authority": "^0.7.0", "clsx": "^1.2.1", @@ -38,6 +39,7 @@ "highlight.js": "^11.6.0", "i18next": "^21.9.1", "instantsearch.css": "^7.4.5", + "katex": "^0.16.11", "less": "^4.1.3", "less-loader": "^11.0.0", "lucide-react": "^0.379.0", @@ -57,6 +59,7 @@ "rehype-image-size": "^1.0.0", "rehype-img-size": "^1.0.1", "remarkable": "^2.0.1", + "remarkable-katex": "^1.2.1", "rooks": "^5.8.0", "tailwind-merge": "^2.3.0", "tailwindcss-animate": "^1.0.7" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48877d0a4..dbcd06b7e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -53,6 +53,9 @@ importers: '@radix-ui/react-slot': specifier: ^1.0.2 version: 1.0.2(@types/react@18.3.3)(react@18.2.0) + '@zilliz/toolkit': + specifier: ^0.0.9 + version: 0.0.9 axios: specifier: ^0.28.0 version: 0.28.0 @@ -89,6 +92,9 @@ importers: instantsearch.css: specifier: ^7.4.5 version: 7.4.5 + katex: + specifier: ^0.16.11 + version: 0.16.11 less: specifier: ^4.1.3 version: 4.2.0 @@ -146,6 +152,9 @@ importers: remarkable: specifier: ^2.0.1 version: 2.0.1 + remarkable-katex: + specifier: ^1.2.1 + version: 1.2.1 rooks: specifier: ^5.8.0 version: 5.14.1(react-dom@18.2.0(react@18.2.0))(react@18.2.0) @@ -1544,6 +1553,9 @@ packages: '@zag-js/visually-hidden@0.20.0': resolution: {integrity: sha512-TrAoiymPNoBgx1f9pOyaM6j0xlxBgGDX209WZvphJDg4616S8KICUSqmC2u0c6gtahkypxGdYDfyj6I83hVuzQ==} + '@zilliz/toolkit@0.0.9': + resolution: {integrity: sha512-6+NwTS2R94lOOlLx6WyC3vrvSCyYomBDCD+j3kJPyc23+oK223Efk9ASHTE0tAvZafWFExmTf9VPlWKd1hXePg==} + acorn-import-assertions@1.9.0: resolution: {integrity: sha512-cmMwop9x+8KFhxvKrKfPYmN6/pKTYYHBqLa0DfvVZcKMJWNyWLnaqND7dx/qn66R7ewM1UX5XMaDVP5wlVTaVA==} peerDependencies: @@ -1852,6 +1864,10 @@ packages: resolution: {integrity: sha512-NOKm8xhkzAjzFx8B2v5OAHT+u5pRQc2UCa2Vq9jYL/31o2wi9mxBA7LIFs3sV5VSC49z6pEhfbMULvShKj26WA==} engines: {node: '>= 6'} + commander@8.3.0: + resolution: {integrity: sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==} + engines: {node: '>= 12'} + concat-map@0.0.1: resolution: {integrity: sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==} @@ -2846,6 +2862,10 @@ packages: resolution: {integrity: sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==} engines: {node: '>=4.0'} + katex@0.16.11: + resolution: {integrity: sha512-RQrI8rlHY92OLf3rho/Ts8i/XvjgguEjOkO1BEXcU3N8BqPpSzBNwV/G0Ukr+P/l3ivvJUE/Fa/CwbS6HesGNQ==} + hasBin: true + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -3745,6 +3765,9 @@ packages: remark-rehype@10.1.0: resolution: {integrity: sha512-EFmR5zppdBp0WQeDVZ/b66CWJipB2q2VLNFMabzDSGR66Z2fQii83G5gTBbgGEnEEA0QRussvrFHxk1HWGJskw==} + remarkable-katex@1.2.1: + resolution: {integrity: sha512-Y1VquJBZnaVsfsVcKW2hmjT+pDL7mp8l5WAVlvuvViltrdok2m1AIKmJv8SsH+mBY84PoMw67t3kTWw1dIm8+g==} + remarkable@2.0.1: resolution: {integrity: sha512-YJyMcOH5lrR+kZdmB0aJJ4+93bEojRZ1HGDn9Eagu6ibg7aVZhc3OWbbShRid+Q5eAfsEqWxpe+g5W5nYNfNiA==} engines: {node: '>= 6.0.0'} @@ -6552,6 +6575,8 @@ snapshots: '@zag-js/visually-hidden@0.20.0': {} + '@zilliz/toolkit@0.0.9': {} + acorn-import-assertions@1.9.0(acorn@8.11.3): dependencies: acorn: 8.11.3 @@ -6905,6 +6930,8 @@ snapshots: commander@4.1.1: {} + commander@8.3.0: {} + concat-map@0.0.1: {} confbox@0.1.7: {} @@ -7225,8 +7252,8 @@ snapshots: '@typescript-eslint/parser': 5.62.0(eslint@8.19.0)(typescript@5.4.5) eslint: 8.19.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 2.7.1(eslint-plugin-import@2.29.1)(eslint@8.19.0) - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-typescript@2.7.1)(eslint@8.19.0) + eslint-import-resolver-typescript: 2.7.1(eslint-plugin-import@2.29.1(eslint@8.19.0))(eslint@8.19.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-typescript@2.7.1(eslint-plugin-import@2.29.1(eslint@8.19.0))(eslint@8.19.0))(eslint@8.19.0) eslint-plugin-jsx-a11y: 6.8.0(eslint@8.19.0) eslint-plugin-react: 7.34.2(eslint@8.19.0) eslint-plugin-react-hooks: 4.6.2(eslint@8.19.0) @@ -7244,11 +7271,11 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-import-resolver-typescript@2.7.1(eslint-plugin-import@2.29.1)(eslint@8.19.0): + eslint-import-resolver-typescript@2.7.1(eslint-plugin-import@2.29.1(eslint@8.19.0))(eslint@8.19.0): dependencies: debug: 4.3.4 eslint: 8.19.0 - eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-typescript@2.7.1)(eslint@8.19.0) + eslint-plugin-import: 2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-typescript@2.7.1(eslint-plugin-import@2.29.1(eslint@8.19.0))(eslint@8.19.0))(eslint@8.19.0) glob: 7.2.3 is-glob: 4.0.3 resolve: 1.22.8 @@ -7256,18 +7283,18 @@ snapshots: transitivePeerDependencies: - supports-color - eslint-module-utils@2.8.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@2.7.1(eslint-plugin-import@2.29.1)(eslint@8.19.0))(eslint@8.19.0): + eslint-module-utils@2.8.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@2.7.1(eslint-plugin-import@2.29.1(eslint@8.19.0))(eslint@8.19.0))(eslint@8.19.0): dependencies: debug: 3.2.7 optionalDependencies: '@typescript-eslint/parser': 5.62.0(eslint@8.19.0)(typescript@5.4.5) eslint: 8.19.0 eslint-import-resolver-node: 0.3.9 - eslint-import-resolver-typescript: 2.7.1(eslint-plugin-import@2.29.1)(eslint@8.19.0) + eslint-import-resolver-typescript: 2.7.1(eslint-plugin-import@2.29.1(eslint@8.19.0))(eslint@8.19.0) transitivePeerDependencies: - supports-color - eslint-plugin-import@2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-typescript@2.7.1)(eslint@8.19.0): + eslint-plugin-import@2.29.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-typescript@2.7.1(eslint-plugin-import@2.29.1(eslint@8.19.0))(eslint@8.19.0))(eslint@8.19.0): dependencies: array-includes: 3.1.8 array.prototype.findlastindex: 1.2.5 @@ -7277,7 +7304,7 @@ snapshots: doctrine: 2.1.0 eslint: 8.19.0 eslint-import-resolver-node: 0.3.9 - eslint-module-utils: 2.8.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@2.7.1(eslint-plugin-import@2.29.1)(eslint@8.19.0))(eslint@8.19.0) + eslint-module-utils: 2.8.1(@typescript-eslint/parser@5.62.0(eslint@8.19.0)(typescript@5.4.5))(eslint-import-resolver-node@0.3.9)(eslint-import-resolver-typescript@2.7.1(eslint-plugin-import@2.29.1(eslint@8.19.0))(eslint@8.19.0))(eslint@8.19.0) hasown: 2.0.2 is-core-module: 2.13.1 is-glob: 4.0.3 @@ -8088,6 +8115,10 @@ snapshots: object.assign: 4.1.5 object.values: 1.2.0 + katex@0.16.11: + dependencies: + commander: 8.3.0 + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -9191,6 +9222,8 @@ snapshots: mdast-util-to-hast: 12.3.0 unified: 10.1.2 + remarkable-katex@1.2.1: {} + remarkable@2.0.1: dependencies: argparse: 1.0.10 diff --git a/src/blogs b/src/blogs index 41303aa39..3424abe0d 160000 --- a/src/blogs +++ b/src/blogs @@ -1 +1 @@ -Subproject commit 41303aa391bdb3db8444d7a4631601efc33946f4 +Subproject commit 3424abe0d630af684fc7646e84fc55546e86fba3 diff --git a/src/docs b/src/docs index 175ec4701..31191945c 160000 --- a/src/docs +++ b/src/docs @@ -1 +1 @@ -Subproject commit 175ec47016df0eab7753aae4198ee782784f561f +Subproject commit 31191945c1fe2598c0723e953df6a86fa73cbc18 diff --git a/src/utils/common.ts b/src/utils/common.ts index 0ebe2064e..7e5fad27b 100644 --- a/src/utils/common.ts +++ b/src/utils/common.ts @@ -1,274 +1,6 @@ -import { Remarkable } from 'remarkable'; -import { CUSTOM_H1_TAG } from '@/consts'; -import hljs from 'highlight.js/lib/core'; -import bash from 'highlight.js/lib/languages/bash'; -import python from 'highlight.js/lib/languages/python'; -import javascript from 'highlight.js/lib/languages/javascript'; -import go from 'highlight.js/lib/languages/go'; -import csharp from 'highlight.js/lib/languages/csharp'; -import java from 'highlight.js/lib/languages/java'; -import json from 'highlight.js/lib/languages/json'; -import { DocAnchorItemType } from '@/types/docs'; +import { Milvus } from '@zilliz/toolkit'; -// no language named shell, use bash instead -hljs.registerLanguage('shell', bash); -hljs.registerLanguage('python', python); -hljs.registerLanguage('javascript', javascript); -hljs.registerLanguage('go', go); -hljs.registerLanguage('csharp', csharp); -hljs.registerLanguage('java', java); -hljs.registerLanguage('json', json); - -// hljs.configure({ -// languages: ['python', 'javascript', 'bash', 'go', 'java'], -// }); - -function addPrefixToHref(htmlString: string, prefix: string) { - const hrefRegex = /href="([^"]*)"/g; - const prefixedHtmlString = htmlString.replace(hrefRegex, (match, href) => { - const newHref = - href.charAt(0) === '#' || - href.charAt(0) === '/' || - href.includes('http') || - href.includes('mailto:') - ? href - : `${prefix}${href}`; - return `href="${newHref}"`; - }); - return prefixedHtmlString; -} - -const convertImgSrc = (version: string, src: string) => { - if (src.includes('http')) { - return src; - } - - const path = src - .split('/') - .filter(v => v !== '.' && v !== '..') - .join('/'); - if (version === 'blog') { - return `/blogs/${path}`; - } - - return `/docs/${version}/${path}`; -}; - -const getHeadingIdFromToken = token => { - // in order to ensure every heading href is unique even when content is same, we use heading content + lines as id - const pattern = - /[`~!@#_$%^&*()=|{}':;',\\\[\\\].<>/?~!@#￥……&*（）——|{}【】‘；：”“""'。，、？]/g; - const content = token.content; - const formatText = `${content}`.replaceAll(pattern, ''); - return formatText.replaceAll(/\s/g, '-'); -}; - -export async function markdownToHtml( - markdown: string, - options: { - showAnchor?: boolean; - version?: string; - needCaption?: boolean; - path?: string; - } = {} -) { - const { - showAnchor = false, - version = 'v2.1.x', - needCaption = false, - path = '', - } = options; - - const md = new Remarkable({ - html: true, - highlight: function (str: string, lang: string) { - try { - return ( - hljs.highlightAuto(str, [ - 'python', - 'javascript', - 'bash', - 'go', - 'java', - 'csharp', - 'json', - ]).value + `` - ); - } catch (err) { - console.log('error', err); - return ''; // use external default escaping - } - }, - }); - - const codeList = []; - let titleContent = ''; - const headingAnchorList: DocAnchorItemType[] = []; - const REMOVE_SPECIAL_CHAR = /[`*]/g; - - // get code content of code block, used for code copy button - const getCodesPlugin = md => { - const rule = md.renderer.rules.fence; - md.renderer.rules.fence = function (tokens, idx, options, env, instance) { - codeList.push(tokens[idx].content); - return rule(tokens, idx, options, env, instance); - }; - }; - - // add copy link icon after headings - const getAnchorsPlugin = md => { - md.renderer.rules.heading_open = function (tokens, idx) { - const id = getHeadingIdFromToken(tokens[idx + 1]); - - return ``; - }; - - md.renderer.rules.heading_close = function (tokens, idx) { - const headingLevel = tokens[idx].hLevel; - const href = getHeadingIdFromToken(tokens[idx - 1]); - - if (headingLevel > 2) { - return ``; - } - - const label = tokens[idx - 1].content.replaceAll(REMOVE_SPECIAL_CHAR, ''); - headingAnchorList.push({ - label, - href, - type: headingLevel, - isActive: false, - }); - - return ` - `; - }; - }; - - // generate a caption at bottom of image, use alt props as content - const generateImgCaptionPlugin = md => { - md.renderer.rules.image = function (tokens, idx) { - const src = tokens[idx].src; - const alt = tokens[idx].alt; - const id = alt.toLocaleLowerCase().replaceAll(/\s/g, '-'); - - const url = convertImgSrc(version, src); - - return ` - - ${alt} - ${alt} - - `; - }; - }; - - // replace img src in html block like
- const formatHtmlBlockImgPlugin = md => { - const rule = md.renderer.rules.htmlblock; - md.renderer.rules.htmlblock = function ( - tokens, - idx, - options, - env, - instance - ) { - const content = tokens[idx].content; - - if (content.includes('src=')) { - const tpl = content.replaceAll( - /\.\.(\S*)\.[svg|jpg|jpeg|png]/g, - string => { - const url = convertImgSrc(version, string); - return url; - } - ); - tokens[idx].content = tpl; - } - - return rule(tokens, idx, options, env, instance); - }; - }; - - // replace img src in html block like - const formatHtmlInlineImgPlugin = md => { - const rule = md.renderer.rules.htmltag; - md.renderer.rules.htmltag = function (tokens, idx, options, env, instance) { - const content = tokens[idx].content; - - if (content.includes('src=')) { - const tpl = content.replace(/\.\.(\S*)\.[svg|jpg|jpeg|png]/, string => { - const url = convertImgSrc(version, string); - return url; - }); - tokens[idx].content = tpl; - } - - return rule(tokens, idx, options, env, instance); - }; - }; - - const getPageTitlePlugin = md => { - const rule = md.renderer.rules.heading_open; - - md.renderer.rules.heading_open = function ( - tokens, - idx, - options, - env, - instance - ) { - if (tokens[idx].hLevel === 1) { - titleContent = tokens[idx + 1].content ?? null; - } - if (version === 'blog' && tokens[idx].hLevel === 1) { - // return a special tag and remove it on client - return `<${CUSTOM_H1_TAG}>`; - } - return rule(tokens, idx, options, env, instance); - }; - }; - - const removeH1CloseTagFromBlog = md => { - const rule = md.renderer.rules.heading_close; - md.renderer.rules.heading_close = (tokens, idx, options, env, self) => { - if (version === 'blog' && tokens[idx].hLevel === 1) { - // return a special tag and remove it on client - return ``; - } else { - return rule(tokens, idx, options, env, self); - } - }; - }; - - md.use(getCodesPlugin); - showAnchor && md.use(getAnchorsPlugin); - md.use(generateImgCaptionPlugin); - md.use(getPageTitlePlugin); - md.use(formatHtmlBlockImgPlugin); - md.use(formatHtmlInlineImgPlugin); - md.use(removeH1CloseTagFromBlog); - - const tree = addPrefixToHref(md.render(markdown), path); - return { - tree, - codeList, - headingContent: titleContent, - anchorList: headingAnchorList, - }; -} +export const markdownToHtml = Milvus.md2html; export const copyToCommand = async (value: string, callback = () => {}) => { await navigator.clipboard.writeText(value); From bec981185fa4f5040accd38e42efdbab603e15a8 Mon Sep 17 00:00:00 2001 From: ryjiang Date: Tue, 20 Aug 2024 10:33:37 +0800 Subject: [PATCH 2/2] add redirects Signed-off-by: ryjiang --- conf/conf.d/client.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/conf/conf.d/client.conf b/conf/conf.d/client.conf index 002d9307a..aa0eec4d4 100644 --- a/conf/conf.d/client.conf +++ b/conf/conf.d/client.conf @@ -170,6 +170,13 @@ server { rewrite ^/api-reference/pymilvus/v2.4.x/EmbeddingModels/CohereEmbeddingFunction/__call()__.md /api-reference/pymilvus/v2.4.x/EmbeddingModels/CohereEmbeddingFunction/__call__().md permanent; rewrite ^/api-reference/pymilvus/v2.4.x/EmbeddingModels/JinaEmbeddingFunction/__call()__.md /api-reference/pymilvus/v2.4.x/EmbeddingModels/JinaEmbeddingFunction/__call__().md permanent; + # Aug 20 + rewrite ^/docs/configure_kafka.md /docs/system_configuration.md permanent; + rewrite ^/docs/configure_knowhere.md /docs/system_configuration.md permanent; + rewrite ^/docs/configure_messagechannel.md /docs/configure_msgchannel.md permanent; + rewrite ^/docs/configure_nats.md /docs/configure_natsmq.md permanent; + rewrite ^/docs/configure_quota_limits.md /docs/configure_quotaandlimits.md permanent; + # redirect old replicated links rewrite ^/docs/text_search_engine.md /docs/build-rag-with-milvus.md permanent; rewrite ^/docs/question_answering_system.md /docs/build-rag-with-milvus.md permanent;