Package plugin with pkg, chrome and grpc binaries (#19)

* Package plugin via pkg

* Download proper chrome and grpc binaries

* Cleanup build add arch variable

* Updated build and fixes for windows

* Build zip instead of tar.gz

* Update readme with packaging

.gitignore

@@ -37,3 +37,6 @@ dist/**/*
 
 # Ignore output from coverage report
 coverage
+
+# Dirs and tar.gz files generated from build
+plugin-*

Makefile

@@ -1,7 +1,23 @@
+ARCH = darwin-x64-unknown
+
 all: clean build
 
 clean:
-	rm -rf dist
+	rm -rf build
 
 build:
-	tsc
+	./node_modules/.bin/tsc
+
+clean_package:
+	rm -rf ./plugin-${ARCH}
+	rm -f ./plugin-${ARCH}.tar.gz
+
+package:
+	node scripts/pkg.js ${ARCH}
+	node scripts/download_chromium.js ${ARCH}
+	node scripts/download_grpc.js ${ARCH}
+	node scripts/rename_executable.js ${ARCH}
+	cp plugin.json plugin-${ARCH}/
+	zip -yqr plugin-${ARCH}.zip plugin-${ARCH}
+
+build_package: clean clean_package build package

README.md

@@ -33,4 +33,21 @@ cd docker
 docker-compose up
 ```
 
+# Packaging
+This plugin can be packaged into single archive without dependencies.
+```bash
+make build_package ARCH=<arch_string>
+``` 
+
+Where <arch_string> is a combination of 
+- linux, darwin, win32
+- ia32, x64, arm, arm64
+- unknown, glibc, musl
+
+This follows combinations allowed for grpc plugin and you can see options [here](https://console.cloud.google.com/storage/browser/node-precompiled-binaries.grpc.io/grpc/?project=grpc-testing) 
+So far these builds were tested from Mac:
+- darwin-x64-unknown
+- linux-x64-glibc
+- win32-x64-unknown
+
 

package.json

@@ -27,6 +27,7 @@
   "devDependencies": {
     "husky": "^0.14.3",
     "lint-staged": "^6.0.0",
+    "pkg": "4.3.8",
     "prettier": "1.9.2",
     "tsc-watch": "^1.0.21"
   },
@@ -48,5 +49,9 @@
     "trailingComma": "es5",
     "singleQuote": true,
     "printWidth": 120
-  }
+  },
+  "pkg": {
+    "assets": "proto/*"
+  },
+  "bin": "build/app.js"
 }

scripts/download_chromium.js

@@ -0,0 +1,47 @@
+const path = require('path');
+const child_process = require('child_process');
+const Puppeteer = require('puppeteer');
+const puppeteerPackageJson = require('puppeteer/package.json');
+
+const archArg = process.argv[2];
+const pluginDir = `plugin-${archArg}`;
+let [
+  // Should be one of linux, mac, win32, win64 as per options in BrowserFetcher but we reuse the same arch string
+  // as for grpc download (ie darwin-x64-unknown) so we need to transform it a bit
+  platform,
+  arch
+] = archArg.split('-');
+
+if (platform === 'win32' && arch === 'x64') {
+  platform = 'win64'
+}
+
+if (platform === 'darwin') {
+  platform = 'mac'
+}
+
+const browserFetcher = Puppeteer.createBrowserFetcher({ platform });
+const revision = puppeteerPackageJson.puppeteer.chromium_revision;
+
+browserFetcher
+  .download(revision, null)
+  .then(() => {
+    console.log("Chromium downloaded");
+    const parts = browserFetcher.revisionInfo(revision).executablePath.split(path.sep);
+
+      // based on where puppeteer puts the binaries see BrowserFetcher.revisionInfo()
+    while (!parts[parts.length - 1].startsWith('chrome-')) {
+      parts.pop()
+    }
+
+    let execPath = parts.join(path.sep);
+
+    child_process.execSync(`cp -RP ${execPath} ${pluginDir}`);
+
+    console.log(`Chromium moved from ${execPath} to ${pluginDir}/`);
+    process.exit(0);
+  })
+  .catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });

scripts/download_grpc.js

@@ -0,0 +1,75 @@
+const path = require('path');
+const childProcess = require('child_process');
+const https = require('https');
+const fs = require('fs');
+
+// /grpc-precompiled-binaries/node/grpc/v1.11.3/node-v64-darwin-x64-unknown
+const grpcPackageJson = require('grpc/package.json');
+
+// Taken from https://github.com/mapbox/node-pre-gyp/blob/674fda7b28b86bb3b11889db86c15c4d06722d02/lib/util/versioning.js
+// so we can evaluate templates used for node-pre-gyp in grpc package
+function eval_template(template, opts) {
+  Object.keys(opts).forEach(function(key) {
+    var pattern = '{' + key + '}';
+    while (template.indexOf(pattern) > -1) {
+      template = template.replace(pattern, opts[key]);
+    }
+  });
+  return template;
+}
+
+// This is taken from grpc package that uses node-pre-gyp to download or build native grpc core.
+
+// This matches abi version for node10 which is latest version pkg supports (as it is latest LTS)
+// see https://nodejs.org/en/download/releases/
+const node_abi = 'node-v64';
+const name = grpcPackageJson.name;
+const version = grpcPackageJson.version;
+
+const archString = process.argv[2];
+const pluginDir =  `plugin-${archString}`;
+// See https://console.cloud.google.com/storage/browser/node-precompiled-binaries.grpc.io/grpc/?project=grpc-testing
+// for existing prebuild binaries (though there are only ones for newer version).
+const [
+  // linux, darwin, win32
+  platform,
+  // ia32, x64, arm, arm64
+  arch,
+  // unknown, glibc, musl
+  libc,
+] = archString.split('-');
+
+const host = grpcPackageJson.binary.host;
+const remote_path = eval_template(grpcPackageJson.binary.remote_path, { name, version });
+const package_name = eval_template(grpcPackageJson.binary.package_name, { node_abi, platform, arch, libc });
+const url = host + path.join(remote_path, package_name);
+
+console.log(`Getting ${url}`);
+new Promise((resolve, reject) => {
+  const file = fs.createWriteStream(`plugin-${archString}/grpc_node.tar.gz`);
+  https
+    .get(url, function(response) {
+      if (response.statusCode !== 200) {
+        const err = new Error(`response.statusCode = ${response.statusCode}`);
+        err.response = response;
+        reject(err);
+      }
+      response.pipe(file).on('finish', () => {
+        resolve();
+      });
+    })
+    .on('error', e => reject(e));
+})
+  .then(() => {
+    console.log(`Grpc module downloaded`);
+    const dirName = package_name.split('.')[0];
+    childProcess.execSync(`tar -xzf ${pluginDir}/grpc_node.tar.gz --directory ${pluginDir}`);
+    childProcess.execSync(`mv ${pluginDir}/${dirName}/grpc_node.node ${pluginDir}/`);
+    childProcess.execSync(`rm -rf ${pluginDir}/${dirName}`);
+    childProcess.execSync(`rm -rf ${pluginDir}/grpc_node.tar.gz`);
+    process.exit(0);
+  })
+  .catch(err => {
+    console.error(err);
+    process.exit(1);
+  });

scripts/pkg.js

@@ -0,0 +1,26 @@
+const childProcess = require('child_process');
+
+const archArg = process.argv[2];
+let [
+  // linux, darwin, win32
+  platform,
+  // ia32, x64, arm, arm64
+  arch,
+] = archArg.split('-');
+
+const platformTransform = {
+  darwin: 'macos',
+  win32: 'win',
+};
+
+const archTransform = {
+  ia32: 'x84',
+  arm: 'armv6',
+  // I only assume this is correct
+  arm64: 'armv6',
+};
+
+platform = platformTransform[platform] || platform;
+arch = archTransform[arch] || arch;
+
+childProcess.execSync(`./node_modules/.bin/pkg -t node10-${platform}-${arch} . --out-path plugin-${archArg}`);

scripts/rename_executable.js

@@ -0,0 +1,25 @@
+const childProcess = require('child_process');
+
+const archArg = process.argv[2];
+
+let [
+  // linux, darwin, win32
+  platform,
+  // ia32, x64, arm, arm64
+  arch,
+] = archArg.split('-');
+
+const platformTransform = {
+  win32: 'windows'
+};
+
+const archTransform = {
+  x64: 'amd64',
+  ia32: '386'
+};
+
+let ext = platform === 'win32' ? '.exe' : '';
+
+const execFileName = `plugin_start_${platformTransform[platform] || platform}_${archTransform[arch] || arch}${ext}`;
+childProcess.execSync(`mv plugin-${archArg}/renderer${ext} plugin-${archArg}/${execFileName}`);
+

src/app.ts

@@ -31,7 +31,10 @@ async function main() {
   }
 }
 
-main();
+main().catch(err => {
+  console.error(err);
+  process.exit(1);
+});
 
 
 // const puppeteer = require('puppeteer');

src/browser.ts

@@ -1,3 +1,4 @@
+import * as path from 'path';
 import * as os from 'os';
 import * as puppeteer from 'puppeteer';
 import { Logger } from './logger';
@@ -33,7 +34,24 @@ export class Browser {
       // set env timezone
       env.TZ = options.timezone || process.env.TZ;
 
-      browser = await puppeteer.launch({env: env, args: ['--no-sandbox']});
+      if ((process as any).pkg) {
+        const parts = puppeteer.executablePath().split(path.sep);
+        while(!parts[0].startsWith('chrome-')) {
+          parts.shift();
+        }
+        const executablePath = [path.dirname(process.execPath), ...parts].join(path.sep);
+        console.log('executablePath', executablePath);
+        browser = await puppeteer.launch({
+          executablePath,
+          env: env,
+          args: ['--no-sandbox'],
+        });
+      } else {
+        browser = await puppeteer.launch({
+          env: env,
+          args: ['--no-sandbox'],
+        });
+      }
       page = await browser.newPage();
 
       await page.setViewport({

src/http-server.ts

@@ -22,6 +22,7 @@ export class HttpServer {
 
     this.app.get('/render', asyncMiddleware(this.render));
     this.app.use((err, req, res, next) => {
+      console.error(err);
       return res.status(err.output.statusCode).json(err.output.payload);
     });