add cli.js && stage-deploy-s3 github action

.github/workflows/stage-deploy-s3.yml

@@ -0,0 +1,58 @@
+name: __Stage__Deploy document site to s3
+on: workflow_dispatch
+env:
+  AWS_KEY: ${{ secrets.AWS_KEY }}
+  AWS_SECRET: ${{ secrets.AWS_SECRET }}
+jobs:
+  stage_deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: 'yarn'
+          cache-dependency-path: 'yarn.lock'
+
+      - uses: actions/cache@v3
+        id: cache
+        with:
+          path: ./node_modules
+          key: ${{ runner.os }}-node-${{ hashFiles('yarn.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_KEY }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET }}
+          aws-region: us-west-2
+
+      - uses: actions/checkout@v3
+        with:
+          repository: starrocks/starrocks
+          fetch-depth: 0
+          path: ./temp/en-us
+
+      - uses: actions/checkout@v3
+        with:
+          repository: starrocks/docs.zh-cn
+          fetch-depth: 0
+          path: ./temp/zh-cn
+
+      - name: Install Dependencies
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          yarn install --frozen-lockfile
+
+      - run: npm run copy
+      - name: Build website
+        run:  |
+          export NODE_OPTIONS="--max_old_space_size=8192"
+          yarn build
+      - run: aws s3 cp build/ s3://starrocks-io-docs-stage/ --recursive
+      - name: clean cdn cache
+        run: |
+          aws cloudfront create-invalidation --distribution-id E3A0PTQYDTB1UQ --paths "/*"

_IGNORE/clean_up.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+echo "cleanup before running yarn build"
+find . -type f -name TOC.md | xargs rm
+rm i18n/zh/docusaurus-plugin-content-docs/*/README.md
+rm versioned_docs/version-*/README.md
+
+echo "replacing StarRocks intro page\n\n"
+find versioned_docs -type f -name "StarRocks_intro.md"  -print0 -exec cp _IGNORE/_StarRocks_intro_English.mdx "{}" \;
+find i18n/zh/docusaurus-plugin-content-docs -type f -name "StarRocks_intro.md"  -print0 -exec cp _IGNORE/_StarRocks_intro_Chinese.mdx "{}" \;
+
+echo "adding index pages"
+find versioned_docs -type d -name quick_start -print0 -exec cp _IGNORE/index_pages/English/quick_start.mdx "{}" \;
+find i18n/zh/docusaurus-plugin-content-docs -type d -name quick_start -print0 -exec cp _IGNORE/index_pages/Chinese/quick_start.mdx "{}" \;
+
+echo "\nadd placeholders for pages in Chinese docs but not in English docs"
+_IGNORE/add_missing_english_files.sh
+
+echo "\nadding frontmatter for sidebar language"
+_IGNORE/add_chinese_sidebar.sh
+_IGNORE/add_english_sidebar.sh
+
+echo "verifying Markdown"
+npx docusaurus-mdx-checker -c versioned_docs
+npx docusaurus-mdx-checker -c docs
+npx docusaurus-mdx-checker -c i18n

cli.js

@@ -0,0 +1,92 @@
+const args = process.argv;
+const path = require("path");
+const config = require("./src/config");
+const fse = require("fs-extra");
+const fs = require("fs");
+const locales = config.locales;
+const versions = config.versions.filter((i) => i.branch !== "latest");
+const exec = require("child_process").exec;
+const execSync = require("child_process").execSync;
+
+const tempDir = path.join(__dirname, "temp");
+const docsDir = path.join(__dirname, config.docDir);
+const cloneDocs = () => {
+  if (fs.existsSync(tempDir)) {
+    fs.rmSync(tempDir, { recursive: true });
+  }
+  if (fs.existsSync(docsDir)) {
+    fs.rmSync(docsDir, { recursive: true });
+  }
+  fs.mkdirSync(docsDir);
+  fs.mkdirSync(tempDir);
+  Promise.all(
+    locales.map((l) => {
+      const tempLocaleDir = `${tempDir}/${l.id}`;
+      console.log("Cloning docs:" + l.repoUrl);
+      let repoUrl = l.repoUrl;
+      if (process.env.DOCS_GITHUB_USER) {
+        repoUrl =
+          repoUrl.slice(0, 8) +
+          `${process.env.DOCS_GITHUB_USER}:${process.env.DOCS_GITHUB_TOKEN}@` +
+          repoUrl.slice(8);
+      }
+      return new Promise((resolve, reject) => {
+        exec(
+          `git clone ${repoUrl} ${tempLocaleDir}`,
+          { env: process.env },
+          () => {
+            resolve();
+          }
+        );
+      });
+    })
+  ).then(() => {
+    copyDocs();
+  });
+};
+const copyDocs = () => {
+  const deleteExistDir = () => {
+    const enDir = path.join(__dirname, "versioned_docs");
+    const zhDir = path.join(
+      __dirname,
+      "i18n/zh/docusaurus-plugin-content-docs/"
+    );
+    if (fs.existsSync(enDir)) {
+      fs.rmSync(enDir, { recursive: true });
+    }
+    const zhFolders = fs.readdirSync(zhDir);
+    zhFolders.forEach((folder) => {
+      if (folder.startsWith("version-")) {
+        fs.rmSync(zhDir + "/" + folder, { recursive: true });
+      }
+    });
+  };
+  deleteExistDir();
+  locales.map((l) => {
+    const tempLocaleDir = `${tempDir}/${l.id}`;
+    const from = path.join(tempLocaleDir, ("/" + l.docsPath).slice(0, -1));
+    versions.map((v) => {
+      const targetBranch =
+        v.branch === "main" ? "main" : l.branchPrefix + v.branch;
+      let to = path.join(__dirname, `versioned_docs/version-${v.branch}`);
+      if (l.id === "zh-cn") {
+        to = path.join(
+          __dirname,
+          `i18n/zh/docusaurus-plugin-content-docs/version-${v.branch}`
+        );
+      }
+      execSync(`cd ${from} && git checkout ${targetBranch}`);
+      fse.copySync(from, to);
+    });
+  });
+  execSync(`./_IGNORE/clean_up.sh`);
+  console.log("done");
+};
+
+if (args[2] === "clone") {
+  cloneDocs();
+}
+
+if (args[2] === "copy") {
+  copyDocs();
+}

docusaurus.config.js

@@ -18,7 +18,7 @@ const config = {
   url: 'https://danroscigno.github.io/',
   // Set the /<baseUrl>/ pathname under which your site is served
   // For GitHub pages deployment, it is often '/<projectName>/'
-  baseUrl: '/doc/',
+  baseUrl: '/',
   //https://danroscigno.github.io/docusaurusv3/docs/3.0/administration/Authentication.html
 
   // GitHub pages deployment config.

package.json

@@ -10,6 +10,8 @@
     "deploy": "docusaurus deploy",
     "clear": "docusaurus clear",
     "serve": "docusaurus serve",
+    "clone": "node cli.js clone",
+    "copy": "node cli.js copy",
     "write-translations": "docusaurus write-translations",
     "write-heading-ids": "docusaurus write-heading-ids"
   },
@@ -21,6 +23,7 @@
     "@docusaurus/theme-search-algolia": "^2.4.3",
     "@mdx-js/react": "^2.3.0",
     "clsx": "^1.2.1",
+    "fs-extra": "^11.1.1",
     "prism-react-renderer": "^1.3.5",
     "react": "^18.0.0",
     "react-dom": "^18.0.0"
@@ -43,4 +46,4 @@
   "engines": {
     "node": ">=16.14"
   }
-}
+}

src/config.js

@@ -0,0 +1,26 @@
+let locales = require("./locales.json");
+let versions = require("./versions.json");
+
+if (process.env.DOC_LOCALE && process.env.DOC_LOCALE !== "all") {
+  locales = locales.filter((l) => l.id === process.env.DOC_LOCALE);
+}
+if (process.env.DOC_VERSION && process.env.DOC_VERSION !== "all") {
+  versions = versions.filter((v) => v.branch === process.env.DOC_VERSION);
+}
+
+const docDir = "docs";
+
+module.exports = {
+  docDir,
+  locales: locales.map((l) => ({
+    ...l,
+    path: `${docDir}/${l}`,
+    repoUrl:
+      l.id === "en-us"
+        ? `https://github.com/StarRocks/starrocks`
+        : `https://github.com/StarRocks/docs${"." + l.id}`,
+    docsPath: l.id === "en-us" ? "docs/" : "",
+    branchPrefix: l.id === "en-us" ? "branch-" : "",
+  })),
+  versions,
+};

src/locales.json

@@ -0,0 +1,8 @@
+[
+  {
+    "id": "en-us"
+  },
+  {
+    "id": "zh-cn"
+  }
+]

src/versions.json

@@ -0,0 +1,11 @@
+[
+  {
+    "branch": "3.1"
+  },
+  {
+    "branch": "3.0"
+  },
+  {
+    "branch": "2.5"
+  }
+]