feat: add function of generating sitemap under a toggle

requarks · Oct 13, 2023 · a323608 · a323608
1 parent 8932d15
commit a323608
Show file tree

Hide file tree

Showing 5 changed files with 125 additions and 1 deletion.
diff --git a/config.sample.yml b/config.sample.yml
@@ -145,3 +145,9 @@ dataPath: ./data
 # file uploads.
 
 bodyParserLimit: 5mb
+
+
+seo:
+  sitemap:
+    enabled: false # sitemap is experimental feature, do not enable in production
+    cacheExpireTime: 3600 # in seconds
diff --git a/package.json b/package.json
@@ -176,6 +176,7 @@
     "semver": "7.3.8",
     "serve-favicon": "2.5.0",
     "simple-git": "3.16.0",
+    "sitemap": "7.1.1",
     "solr-node": "1.2.1",
     "sqlite3": "5.1.4",
     "ssh2": "1.11.0",

diff --git a/server/controllers/common.js b/server/controllers/common.js
@@ -5,6 +5,7 @@ const _ = require('lodash')
 const CleanCSS = require('clean-css')
 const moment = require('moment')
 const qs = require('querystring')
+const { SitemapStream, streamToPromise } = require('sitemap')
 
 /* global WIKI */
 
@@ -22,6 +23,91 @@ router.get('/robots.txt', (req, res, next) => {
   }
 })
 
+/**
+ * sitemap.xml
+ */
+// TODO: toggle off before the feature is ready
+if(WIKI.config.seo.sitemap.enabled) {
+  WIKI.logger.info(`Experimental feature sitemap is enabled`)
+  router.get('/sitemap.xml', async (req, res
+
+    ) => {
+    const host = WIKI.config.host
+    const {enabled, cacheExpireTime} = WIKI.config.seo.sitemap
+    !enabled && res.status(404).end()
+
+    try {
+
+      res.header('Content-Type', 'application/xml');
+
+      // TODO: if we have a cached entry send it
+      // if (sitemap) {
+      //   res.send(sitemap)
+      //   return
+      // }
+
+      const smStream = new SitemapStream({ hostname: host})
+
+      /**
+         * [
+    Page {
+      id: 1,
+      path: 'home',
+      hash: 'b29b5d2ce62e55412776ab98f05631e0aa96597b',
+      title: 'Untitled Page',
+      description: '',
+      isPrivate: 0,
+      isPublished: 1,
+      privateNS: null,
+      publishStartDate: '',
+      publishEndDate: '',
+      content: '# Header\nYour content here',
+      render: '<h1 class="toc-header" id="header"><a href="#header" class="toc-anchor">¶</a> Header</h1>\n' +
+        '<p>Your content here</p>\n',
+      toc: '[{"title":"Header","anchor":"#header","children":[]}]',
+      contentType: 'markdown',
+      createdAt: '2023-10-12T16:07:42.594Z',
+      updatedAt: '2023-10-12T16:07:44.791Z',
+      editorKey: 'markdown',
+      localeCode: 'en',
+      authorId: 1,
+      creatorId: 1,
+      extra: { js: '', css: '' }
+    }
+  ]
+         */
+      const pages = await WIKI.models.pages.query()
+      .where('isPublished', 1)
+      .where('isPrivate', 0)
+      .orderBy('updatedAt', 'desc')
+
+      for (const page of pages) {
+        // TODO: 
+        /**
+         * TypeError: user.getGlobalPermissions is not a function
+            at Object.checkAccess (/home/xxx/workspace/wiki/server/core/auth.js:218:72)
+            at /home/xxx/workspace/wiki/server/controllers/common.js:92:22
+         */
+        // if (!WIKI.auth.checkAccess(WIKI.auth.guest, ['read:pages'], page)) continue
+
+        smStream.write({ 
+          url: `/${page.localeCode}/${page.path}`, 
+          lastmod: page.updatedAt
+          })
+      }
+
+      smStream.end();
+      smStream.pipe(res).on('error', (e) => { throw e })
+    } catch (e) {
+      console.error(e)
+      res.status(500).end()
+    }
+
+  })
+}
+
+
+
 /**
  * Health Endpoint
  */

diff --git a/server/setup.js b/server/setup.js
@@ -117,7 +117,11 @@ module.exports = () => {
         description: '',
         robots: ['index', 'follow'],
         analyticsService: '',
-        analyticsId: ''
+        analyticsId: '',
+        sitemap: {
+          enabled: false,
+          cacheExpireTime: 3600 // in seconds
+        }
       })
       _.set(WIKI.config, 'sessionSecret', (await crypto.randomBytesAsync(32)).toString('hex'))
       _.set(WIKI.config, 'telemetry', {

diff --git a/yarn.lock b/yarn.lock
@@ -3839,6 +3839,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-12.12.14.tgz#1c1d6e3c75dba466e0326948d56e8bd72a1903d2"
   integrity sha512-u/SJDyXwuihpwjXy7hOOghagLEV1KdAST6syfnOk6QZAMzZuWZqXy5aYYZbh8Jdpd4escVFP0MvftHNDb9pruA==
 
+"@types/node@^17.0.5":
+  version "17.0.45"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-17.0.45.tgz#2c0fafd78705e7a18b7906b5201a522719dc5190"
+  integrity sha512-w+tIMs3rq2afQdsPJlODhoUEKzFP1ayaoyl1CcnwtIlsVe7K7bA1NGm4s3PraqTLlXnbIN84zuBlxBWo1u9BLw==
+
 "@types/node@^8.0.47":
   version "8.10.53"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-8.10.53.tgz#5fa08eef810b08b2c03073e360b54f7bad899db1"
@@ -3890,6 +3895,13 @@
     "@types/node" "*"
     safe-buffer "*"
 
+"@types/sax@^1.2.1":
+  version "1.2.5"
+  resolved "https://registry.yarnpkg.com/@types/sax/-/sax-1.2.5.tgz#4392799e1770d24b6dc8d0c66c8882f8e1c38b3d"
+  integrity sha512-9jWta97bBVC027/MShr3gLab8gPhKy4l6qpb+UJLF5pDm3501NvA7uvqVCW+REFtx00oTi6Cq9JzLwgq6evVgw==
+  dependencies:
+    "@types/node" "*"
+
 "@types/serve-static@*":
   version "1.13.3"
   resolved "https://registry.yarnpkg.com/@types/serve-static/-/serve-static-1.13.3.tgz#eb7e1c41c4468272557e897e9171ded5e2ded9d1"
@@ -5107,6 +5119,11 @@ are-we-there-yet@~1.1.2:
     delegates "^1.0.0"
     readable-stream "^2.0.6"
 
+arg@^5.0.0:
+  version "5.0.2"
+  resolved "https://registry.yarnpkg.com/arg/-/arg-5.0.2.tgz#c81433cc427c92c4dcf4865142dbca6f15acd59c"
+  integrity sha512-PYjyFOLKQ9y57JvQ6QLo8dAgNqswh8M1RMJYdQduT6xbWSgK36P/Z/v+p888pM69jMMfS8Xd8F6I1kQ/I9HUGg==
+
 argparse@^1.0.7:
   version "1.0.10"
   resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
@@ -18189,6 +18206,16 @@ sisteransi@^1.0.3:
   resolved "https://registry.yarnpkg.com/sisteransi/-/sisteransi-1.0.3.tgz#98168d62b79e3a5e758e27ae63c4a053d748f4eb"
   integrity sha512-SbEG75TzH8G7eVXFSN5f9EExILKfly7SUvVY5DhhYLvfhKqhDFY0OzevWa/zwak0RLRfWS5AvfMWpd9gJvr5Yg==
 
+[email protected]:
+  version "7.1.1"
+  resolved "https://registry.yarnpkg.com/sitemap/-/sitemap-7.1.1.tgz#eeed9ad6d95499161a3eadc60f8c6dce4bea2bef"
+  integrity sha512-mK3aFtjz4VdJN0igpIJrinf3EO8U8mxOPsTBzSsy06UtjZQJ3YY3o3Xa7zSc5nMqcMrRwlChHZ18Kxg0caiPBg==
+  dependencies:
+    "@types/node" "^17.0.5"
+    "@types/sax" "^1.2.1"
+    arg "^5.0.0"
+    sax "^1.2.4"
+
 slash@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55"