From 3833fa03968c6343670a22b29f06e6db7150db9b Mon Sep 17 00:00:00 2001 From: don Date: Thu, 19 Oct 2023 15:47:55 +0800 Subject: [PATCH] add cli.js && stage-deploy-s3 github action --- .github/workflows/stage-deploy-s3.yml | 58 +++++++++++++++++ _IGNORE/clean_up.sh | 26 ++++++++ cli.js | 92 +++++++++++++++++++++++++++ docusaurus.config.js | 2 +- package.json | 5 +- src/config.js | 26 ++++++++ src/locales.json | 8 +++ src/versions.json | 11 ++++ 8 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/stage-deploy-s3.yml create mode 100755 _IGNORE/clean_up.sh create mode 100644 cli.js create mode 100644 src/config.js create mode 100644 src/locales.json create mode 100644 src/versions.json diff --git a/.github/workflows/stage-deploy-s3.yml b/.github/workflows/stage-deploy-s3.yml new file mode 100644 index 0000000000..6f0aeb11cd --- /dev/null +++ b/.github/workflows/stage-deploy-s3.yml @@ -0,0 +1,58 @@ +name: __Stage__Deploy document site to s3 +on: workflow_dispatch +env: + AWS_KEY: ${{ secrets.AWS_KEY }} + AWS_SECRET: ${{ secrets.AWS_SECRET }} +jobs: + stage_deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: 18 + cache: 'yarn' + cache-dependency-path: 'yarn.lock' + + - uses: actions/cache@v3 + id: cache + with: + path: ./node_modules + key: ${{ runner.os }}-node-${{ hashFiles('yarn.lock') }} + restore-keys: | + ${{ runner.os }}-node- + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_KEY }} + aws-secret-access-key: ${{ secrets.AWS_SECRET }} + aws-region: us-west-2 + + - uses: actions/checkout@v3 + with: + repository: starrocks/starrocks + fetch-depth: 0 + path: ./temp/en-us + + - uses: actions/checkout@v3 + with: + repository: starrocks/docs.zh-cn + fetch-depth: 0 + path: ./temp/zh-cn + + - name: Install Dependencies + if: steps.cache.outputs.cache-hit != 'true' + run: | + yarn install --frozen-lockfile + + - run: npm run copy + - name: Build website + run: | + export NODE_OPTIONS="--max_old_space_size=8192" + yarn build + - run: aws s3 cp build/ s3://starrocks-io-docs-stage/ --recursive + - name: clean cdn cache + run: | + aws cloudfront create-invalidation --distribution-id E3A0PTQYDTB1UQ --paths "/*" \ No newline at end of file diff --git a/_IGNORE/clean_up.sh b/_IGNORE/clean_up.sh new file mode 100755 index 0000000000..ffdef38724 --- /dev/null +++ b/_IGNORE/clean_up.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +echo "cleanup before running yarn build" +find . -type f -name TOC.md | xargs rm +rm i18n/zh/docusaurus-plugin-content-docs/*/README.md +rm versioned_docs/version-*/README.md + +echo "replacing StarRocks intro page\n\n" +find versioned_docs -type f -name "StarRocks_intro.md" -print0 -exec cp _IGNORE/_StarRocks_intro_English.mdx "{}" \; +find i18n/zh/docusaurus-plugin-content-docs -type f -name "StarRocks_intro.md" -print0 -exec cp _IGNORE/_StarRocks_intro_Chinese.mdx "{}" \; + +echo "adding index pages" +find versioned_docs -type d -name quick_start -print0 -exec cp _IGNORE/index_pages/English/quick_start.mdx "{}" \; +find i18n/zh/docusaurus-plugin-content-docs -type d -name quick_start -print0 -exec cp _IGNORE/index_pages/Chinese/quick_start.mdx "{}" \; + +echo "\nadd placeholders for pages in Chinese docs but not in English docs" +_IGNORE/add_missing_english_files.sh + +echo "\nadding frontmatter for sidebar language" +_IGNORE/add_chinese_sidebar.sh +_IGNORE/add_english_sidebar.sh + +echo "verifying Markdown" +npx docusaurus-mdx-checker -c versioned_docs +npx docusaurus-mdx-checker -c docs +npx docusaurus-mdx-checker -c i18n diff --git a/cli.js b/cli.js new file mode 100644 index 0000000000..3e762d753a --- /dev/null +++ b/cli.js @@ -0,0 +1,92 @@ +const args = process.argv; +const path = require("path"); +const config = require("./src/config"); +const fse = require("fs-extra"); +const fs = require("fs"); +const locales = config.locales; +const versions = config.versions.filter((i) => i.branch !== "latest"); +const exec = require("child_process").exec; +const execSync = require("child_process").execSync; + +const tempDir = path.join(__dirname, "temp"); +const docsDir = path.join(__dirname, config.docDir); +const cloneDocs = () => { + if (fs.existsSync(tempDir)) { + fs.rmSync(tempDir, { recursive: true }); + } + if (fs.existsSync(docsDir)) { + fs.rmSync(docsDir, { recursive: true }); + } + fs.mkdirSync(docsDir); + fs.mkdirSync(tempDir); + Promise.all( + locales.map((l) => { + const tempLocaleDir = `${tempDir}/${l.id}`; + console.log("Cloning docs:" + l.repoUrl); + let repoUrl = l.repoUrl; + if (process.env.DOCS_GITHUB_USER) { + repoUrl = + repoUrl.slice(0, 8) + + `${process.env.DOCS_GITHUB_USER}:${process.env.DOCS_GITHUB_TOKEN}@` + + repoUrl.slice(8); + } + return new Promise((resolve, reject) => { + exec( + `git clone ${repoUrl} ${tempLocaleDir}`, + { env: process.env }, + () => { + resolve(); + } + ); + }); + }) + ).then(() => { + copyDocs(); + }); +}; +const copyDocs = () => { + const deleteExistDir = () => { + const enDir = path.join(__dirname, "versioned_docs"); + const zhDir = path.join( + __dirname, + "i18n/zh/docusaurus-plugin-content-docs/" + ); + if (fs.existsSync(enDir)) { + fs.rmSync(enDir, { recursive: true }); + } + const zhFolders = fs.readdirSync(zhDir); + zhFolders.forEach((folder) => { + if (folder.startsWith("version-")) { + fs.rmSync(zhDir + "/" + folder, { recursive: true }); + } + }); + }; + deleteExistDir(); + locales.map((l) => { + const tempLocaleDir = `${tempDir}/${l.id}`; + const from = path.join(tempLocaleDir, ("/" + l.docsPath).slice(0, -1)); + versions.map((v) => { + const targetBranch = + v.branch === "main" ? "main" : l.branchPrefix + v.branch; + let to = path.join(__dirname, `versioned_docs/version-${v.branch}`); + if (l.id === "zh-cn") { + to = path.join( + __dirname, + `i18n/zh/docusaurus-plugin-content-docs/version-${v.branch}` + ); + } + execSync(`cd ${from} && git checkout ${targetBranch}`); + fse.copySync(from, to); + }); + }); + execSync(`./_IGNORE/clean_up.sh`); + console.log("done"); +}; + +if (args[2] === "clone") { + cloneDocs(); +} + +if (args[2] === "copy") { + copyDocs(); +} diff --git a/docusaurus.config.js b/docusaurus.config.js index 0d63a2d651..6ae3d5b339 100644 --- a/docusaurus.config.js +++ b/docusaurus.config.js @@ -18,7 +18,7 @@ const config = { url: 'https://danroscigno.github.io/', // Set the // pathname under which your site is served // For GitHub pages deployment, it is often '//' - baseUrl: '/doc/', + baseUrl: '/', //https://danroscigno.github.io/docusaurusv3/docs/3.0/administration/Authentication.html // GitHub pages deployment config. diff --git a/package.json b/package.json index b0662604de..d17d998987 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,8 @@ "deploy": "docusaurus deploy", "clear": "docusaurus clear", "serve": "docusaurus serve", + "clone": "node cli.js clone", + "copy": "node cli.js copy", "write-translations": "docusaurus write-translations", "write-heading-ids": "docusaurus write-heading-ids" }, @@ -21,6 +23,7 @@ "@docusaurus/theme-search-algolia": "^2.4.3", "@mdx-js/react": "^2.3.0", "clsx": "^1.2.1", + "fs-extra": "^11.1.1", "prism-react-renderer": "^1.3.5", "react": "^18.0.0", "react-dom": "^18.0.0" @@ -43,4 +46,4 @@ "engines": { "node": ">=16.14" } -} +} \ No newline at end of file diff --git a/src/config.js b/src/config.js new file mode 100644 index 0000000000..4e0ae71251 --- /dev/null +++ b/src/config.js @@ -0,0 +1,26 @@ +let locales = require("./locales.json"); +let versions = require("./versions.json"); + +if (process.env.DOC_LOCALE && process.env.DOC_LOCALE !== "all") { + locales = locales.filter((l) => l.id === process.env.DOC_LOCALE); +} +if (process.env.DOC_VERSION && process.env.DOC_VERSION !== "all") { + versions = versions.filter((v) => v.branch === process.env.DOC_VERSION); +} + +const docDir = "docs"; + +module.exports = { + docDir, + locales: locales.map((l) => ({ + ...l, + path: `${docDir}/${l}`, + repoUrl: + l.id === "en-us" + ? `https://github.com/StarRocks/starrocks` + : `https://github.com/StarRocks/docs${"." + l.id}`, + docsPath: l.id === "en-us" ? "docs/" : "", + branchPrefix: l.id === "en-us" ? "branch-" : "", + })), + versions, +}; diff --git a/src/locales.json b/src/locales.json new file mode 100644 index 0000000000..6b8b6d0b1c --- /dev/null +++ b/src/locales.json @@ -0,0 +1,8 @@ +[ + { + "id": "en-us" + }, + { + "id": "zh-cn" + } +] \ No newline at end of file diff --git a/src/versions.json b/src/versions.json new file mode 100644 index 0000000000..83866a49d4 --- /dev/null +++ b/src/versions.json @@ -0,0 +1,11 @@ +[ + { + "branch": "3.1" + }, + { + "branch": "3.0" + }, + { + "branch": "2.5" + } +] \ No newline at end of file