introduction of server side rendering (#838)

* initial introduction of server side rendering

* update config spec

* ESM cache busting in development with workers

* correctly establish body and template support for SSR routes

* basic metadata support

* frontmatter and graph support

* fix lint

* basic smoke test case for SSR

* fix and clean up specs

* basic smoke testing with HTML optimization for server routes

* handle pre-rendering for SSR routes

* support SSR bundling

* full frontmatter support and custom route data for SSR routes

* document server rendering

* add ssr mode to config docs

* clarify serve command in README

* ESM path to file interop for windows

* rebase fixes

* remove demo code

.c8rc.json

@@ -18,10 +18,10 @@
 
   "checkCoverage": true,
 
-  "statements": 85,
+  "statements": 80,
   "branches": 85,
   "functions": 90,
-  "lines": 85,
+  "lines": 80,
 
   "watermarks": {
     "statements": [75, 85],

README.md

@@ -49,7 +49,7 @@ Then in your _package.json_, add the `type` field and `scripts` for the CLI, lik
 
 - `greenwood build`: Generates a production build of your project
 - `greenwood develop`: Starts a local development server for your project
-- `greenwood serve`: Generates a production build of the project and serves it locally on a simple web server.
+- `greenwood serve`: Generates a production build of your project and runs it on a NodeJS based web server
 
 ## Documentation
 All of our documentation is on our [website](https://www.greenwoodjs.io/) (which itself is built by Greenwood!).  See our website documentation to learn more about:

greenwood.config.js

@@ -12,7 +12,7 @@ const FAVICON_HREF = '/favicon.ico';
 
 export default {
   workspace: fileURLToPath(new URL('./www', import.meta.url)),
-  mode: 'mpa',
+  mode: 'ssr',
   optimization: 'inline',
   title: 'Greenwood',
   meta: [

packages/cli/src/commands/build.js

@@ -1,6 +1,6 @@
 import { bundleCompilation } from '../lifecycles/bundle.js';
 import { copyAssets } from '../lifecycles/copy.js';
-import { devServer } from '../lifecycles/serve.js';
+import { getDevServer } from '../lifecycles/serve.js';
 import fs from 'fs';
 import { generateCompilation } from '../lifecycles/compile.js';
 import { preRenderCompilation, staticRenderCompilation } from '../lifecycles/prerender.js';
@@ -23,7 +23,7 @@ const runProductionBuild = async () => {
       if (prerender) {
         await new Promise(async (resolve, reject) => {
           try {
-            (await devServer(compilation)).listen(port, async () => {
+            (await getDevServer(compilation)).listen(port, async () => {
               console.info(`Started local development server at localhost:${port}`);
 
               const servers = [...compilation.config.plugins.filter((plugin) => {

packages/cli/src/commands/develop.js

@@ -1,6 +1,6 @@
 import { generateCompilation } from '../lifecycles/compile.js';
 import { ServerInterface } from '../lib/server-interface.js';
-import { devServer } from '../lifecycles/serve.js';
+import { getDevServer } from '../lifecycles/serve.js';
 
 const runDevServer = async () => {
 
@@ -10,7 +10,7 @@ const runDevServer = async () => {
       const compilation = await generateCompilation();
       const { port } = compilation.config.devServer;
 
-      (await devServer(compilation)).listen(port, () => {
+      (await getDevServer(compilation)).listen(port, () => {
 
         console.info(`Started local development server at localhost:${port}`);
 

packages/cli/src/commands/serve.js

@@ -1,5 +1,5 @@
 import { generateCompilation } from '../lifecycles/compile.js';
-import { prodServer } from '../lifecycles/serve.js';
+import { getStaticServer, getHybridServer } from '../lifecycles/serve.js';
 
 const runProdServer = async () => {
 
@@ -8,9 +8,10 @@ const runProdServer = async () => {
     try {
       const compilation = await generateCompilation();
       const port = 8080;
-
-      (await prodServer(compilation)).listen(port, () => {
-        console.info(`Started production test server at localhost:${port}`);
+      const server = compilation.config.mode === 'ssr' ? getHybridServer : getStaticServer;
+
+      (await server(compilation)).listen(port, () => {
+        console.info(`Started server at localhost:${port}`);
       });
     } catch (err) {
       reject(err);

packages/cli/src/lib/browser.js

@@ -72,8 +72,6 @@ class BrowserRunner {
     // Serialize page.
     const content = await page.content();
 
-    // console.debug('content????', content);
-
     await page.close();
 
     return content;

packages/cli/src/lib/ssr-route-worker.js

@@ -0,0 +1,28 @@
+import { pathToFileURL } from 'url';
+import { workerData, parentPort } from 'worker_threads';
+
+async function executeRouteModule({ modulePath, compilation, route, label, id }) {
+  const { getTemplate = null, getBody = null, getFrontmatter = null } = await import(pathToFileURL(modulePath)).then(module => module);
+  const parsedCompilation = JSON.parse(compilation);
+  const data = {
+    template: null,
+    body: null,
+    frontmatter: null
+  };
+
+  if (getTemplate) {
+    data.template = await getTemplate(parsedCompilation, route);
+  }
+
+  if (getBody) {
+    data.body = await getBody(parsedCompilation, route);
+  }
+
+  if (getFrontmatter) {
+    data.frontmatter = await getFrontmatter(parsedCompilation, route, label, id);
+  }
+
+  parentPort.postMessage(data);
+}
+
+executeRouteModule(workerData);

packages/cli/src/lifecycles/bundle.js

@@ -5,8 +5,9 @@ const bundleCompilation = async (compilation) => {
 
   return new Promise(async (resolve, reject) => {
     try {
-      // https://rollupjs.org/guide/en/#differences-to-the-javascript-api
+      compilation.graph = compilation.graph.filter(page => !page.isSSR);
 
+      // https://rollupjs.org/guide/en/#differences-to-the-javascript-api
       if (compilation.graph.length > 0) {
         const rollupConfigs = await getRollupConfig(compilation);
         const bundle = await rollup(rollupConfigs[0]);

packages/cli/src/lifecycles/config.js

@@ -29,7 +29,7 @@ const greenwoodPlugins = (await Promise.all([
   };
 });
 
-const modes = ['ssg', 'mpa', 'spa'];
+const modes = ['ssg', 'mpa', 'spa', 'ssr'];
 const optimizations = ['default', 'none', 'static', 'inline'];
 const pluginTypes = ['copy', 'context', 'resource', 'rollup', 'server', 'source'];
 const defaultConfig = {
@@ -47,7 +47,8 @@ const defaultConfig = {
   markdown: { plugins: [], settings: {} },
   prerender: true,
   pagesDirectory: 'pages',
-  templatesDirectory: 'templates'
+  templatesDirectory: 'templates',
+  routesDirectory: 'routes'
 };
 
 const readAndMergeConfig = async() => {

packages/cli/src/lifecycles/context.js

@@ -13,6 +13,7 @@ const initContext = async({ config }) => {
       const userWorkspace = path.join(config.workspace);
       const pagesDir = path.join(userWorkspace, `${config.pagesDirectory}/`);
       const userTemplatesDir = path.join(userWorkspace, `${config.templatesDirectory}/`);
+      const routesDir = path.join(userWorkspace, `${config.routesDirectory}/`);
 
       const context = {
         dataDir,
@@ -21,7 +22,8 @@ const initContext = async({ config }) => {
         pagesDir,
         userTemplatesDir,
         scratchDir,
-        projectDirectory
+        projectDirectory,
+        routesDir
       };
 
       if (!fs.existsSync(scratchDir)) {

packages/cli/src/lifecycles/graph.js

@@ -3,13 +3,14 @@ import fs from 'fs';
 import fm from 'front-matter';
 import path from 'path';
 import toc from 'markdown-toc';
+import { pathToFileURL } from 'url';
 
 const generateGraph = async (compilation) => {
 
   return new Promise(async (resolve, reject) => {
     try {
       const { context, config } = compilation;
-      const { pagesDir, userWorkspace } = context;
+      const { pagesDir, routesDir, userWorkspace } = context;
       let graph = [{
         outputPath: 'index.html',
         filename: 'index.html',
@@ -21,6 +22,91 @@ const generateGraph = async (compilation) => {
         imports: []
       }];
 
+      const walkDirectoryForRoutes = async function(directory, routes = []) {
+        await Promise.all((await fs.promises.readdir(directory)).map(async (filename) => {
+          const fullPath = path.normalize(`${directory}${path.sep}${filename}`);
+
+          if (fs.statSync(fullPath).isDirectory()) {
+            routes = await walkDirectoryForRoutes(fullPath, routes);
+          } else {
+            const { getFrontmatter } = await import(pathToFileURL(fullPath));
+            const relativePagePath = fullPath.substring(routesDir.length - 1, fullPath.length);
+            const id = filename.split(path.sep)[filename.split(path.sep).length - 1].replace('.js', '');
+            const label = id.split('-')
+              .map((idPart) => {
+                return `${idPart.charAt(0).toUpperCase()}${idPart.substring(1)}`;
+              }).join(' ');
+            const route = relativePagePath
+              .replace('.js', '')
+              .replace(/\\/g, '/')
+              .concat('/');
+            let template = 'page';
+            let title = `${compilation.config.title} - ${label}`;
+            let customData = {};
+            let imports = [];
+
+            if (getFrontmatter) {
+              const ssrFmData = await getFrontmatter(compilation, route, label, id);
+
+              template = ssrFmData.template ? ssrFmData.template : template;
+              title = ssrFmData.title ? ssrFmData.title : title;
+              imports = ssrFmData.imports ? ssrFmData.imports : imports;
+              customData = ssrFmData.data ? ssrFmData.data : customData;
+
+              // prune "reserved" attributes that are supported by Greenwood
+              // https://www.greenwoodjs.io/docs/front-matter
+              delete ssrFmData.label;
+              delete ssrFmData.imports;
+              delete ssrFmData.title;
+              delete ssrFmData.template;
+
+              /* Menu Query
+               * Custom front matter - Variable Definitions
+               * --------------------------------------------------
+               * menu: the name of the menu in which this item can be listed and queried
+               * index: the index of this list item within a menu
+               * linkheadings: flag to tell us where to add page's table of contents as menu items
+               * tableOfContents: json object containing page's table of contents(list of headings)
+               */
+              customData.menu = ssrFmData.menu || '';
+              customData.index = ssrFmData.index || '';
+            }
+
+            /*
+             * Graph Properties (per page)
+             *----------------------
+             * data: custom page frontmatter
+             * filename: base filename of the page
+             * id: filename without the extension
+             * label: "pretty" text representation of the filename
+             * imports: per page JS or CSS file imports to be included in HTML output
+             * outputPath: the filename to write to when generating static HTML
+             * path: path to the file relative to the workspace
+             * route: URL route for a given page on outputFilePath
+             * template: page template to use as a base for a generated component
+             * title: a default value that can be used for <title></title>
+             */
+            routes.push({
+              data: customData,
+              filename,
+              id,
+              label,
+              imports,
+              outputPath: route === '/404/'
+                ? '404.html'
+                : `${route}index.html`,
+              path: route,
+              route,
+              template,
+              title,
+              isSSR: true
+            });
+          }
+        }));
+
+        return routes;
+      };
+
       const walkDirectoryForPages = function(directory, pages = []) {
 
         fs.readdirSync(directory).forEach((filename) => {
@@ -149,10 +235,10 @@ const generateGraph = async (compilation) => {
         }];
       } else {
         const oldGraph = graph[0];
+        const pages = fs.existsSync(pagesDir) ? walkDirectoryForPages(pagesDir) : graph;
+        const routes = fs.existsSync(routesDir) ? await walkDirectoryForRoutes(routesDir) : [];
 
-        graph = fs.existsSync(pagesDir)
-          ? walkDirectoryForPages(pagesDir)
-          : graph;
+        graph = [...pages, ...routes];
 
         const has404Page = graph.filter(page => page.route === '/404/').length === 1;
 

packages/cli/src/lifecycles/prerender.js

@@ -52,6 +52,8 @@ async function optimizePage(compilation, contents, route, outputPath, outputDir)
   }
 
   await fs.promises.writeFile(path.join(outputDir, outputPath), htmlOptimized);
+
+  return htmlOptimized;
 }
 
 async function preRenderCompilation(compilation) {
@@ -100,7 +102,7 @@ async function preRenderCompilation(compilation) {
 
   return new Promise(async (resolve, reject) => {
     try {
-      const pages = compilation.graph;
+      const pages = compilation.graph.filter(page => !page.isSSR);
       const port = compilation.config.devServer.port;
       const outputDir = compilation.context.scratchDir;
       const serverAddress = `http://127.0.0.1:${port}`;
@@ -121,7 +123,7 @@ async function preRenderCompilation(compilation) {
 }
 
 async function staticRenderCompilation(compilation) {
-  const pages = compilation.graph;
+  const pages = compilation.graph.filter(page => !page.isSSR);
   const scratchDir = compilation.context.scratchDir;
   const htmlResource = compilation.config.plugins.filter((plugin) => {
     return plugin.name === 'plugin-standard-html';