Add opt-in genome download feature

cli/develop.js

@@ -10,6 +10,7 @@ const version = require('../src/version').version;
 const chalk = require('chalk');
 const generateWebpackConfig = require("../webpack.config.js").default;
 const SUPPRESS = require('argparse').Const.SUPPRESS;
+const bodyParser = require('body-parser');
 
 const addParser = (parser) => {
   const description = `Launch auspice in development mode.
@@ -50,6 +51,11 @@ const run = (args) => {
   process.env.BABEL_ENV = "development";
   process.env.BABEL_EXTENSION_PATH = extensionPath;
 
+  // parse application/json
+  app.use(bodyParser.json({limit: '20mb'}));
+  // parse application/x-www-form-urlencoded
+  app.use(bodyParser.urlencoded({ extended: false }));
+
   /* Redirects / to webpack-generated index */
   app.use((req, res, next) => {
     if (!/^\/__webpack_hmr|^\/charon|\.[A-Za-z0-9]{1,4}$/.test(req.path)) {

cli/server/handleGenomeDbs.js

@@ -0,0 +1,157 @@
+const fs = require('fs');
+const path = require("path");
+const {PassThrough} = require('stream');
+const Engine = require('nedb');
+const fasta = require('bionode-fasta');
+
+const { promisify } = require('util');
+
+const readdir = promisify(fs.readdir);
+
+/*
+  All NeDB database files are stored in the subdirectory 'genomeDbs'
+  at the same level where fasta file and auspice file is located.
+*/
+const getDbPath = (fastaPath) => {
+  const dbRoot = path.join(path.dirname(fastaPath), 'genomeDbs');
+  const dbPath = path.join(dbRoot,
+    path.basename(fastaPath).replace(".fasta", ".db"));
+  return dbPath;
+};
+
+/*
+  @param: ids: an array of fasta sequence ids
+  @param: dbPath: resolvable path to NeDB database of genome sequences
+*/
+const fetchRecords = (ids, dbPath) =>
+  new Promise((resolve, reject) => {
+    console.log("dbPath: " + dbPath);
+    const db = new Engine({filename: dbPath, autoload: true});
+    if (db) {
+      console.log("db connected");
+      db.find({id: {$in: ids}}, (err, docs) => {
+        if (err) {
+          console.log('EE');
+          reject(err);
+        } else if (docs.length === 0) {
+          console.log("No record found!");
+          resolve(docs);
+        } else {
+          console.log("records: " + docs.length);
+          resolve(docs);
+        }
+      });
+    }
+  });
+
+
+/**
+   return response to a POST of fetching genome sequences by an array of ids
+   @param {string} datasetPath same as datasetDir when staring auspice
+*/
+const getGenomeDB = (datasetsPath) => {
+  return async (req, res) => { // eslint-disable-line consistent-return
+    try {
+      const prefix = req.body.prefix
+            .replace(/^\//, '')
+            .replace(/\/$/, '')
+            .split("/")
+            .join("_");
+      const dbPath = datasetsPath + '/genomeDbs/' + prefix + '.db';
+      if (!req.body.ids || req.body.ids.length === 0) {
+        res.setHeader('Content-Type', 'application/json');
+        if (fs.existsSync(dbPath)) {
+          res.end(JSON.stringify({result: true}));
+        } else {
+          res.end(JSON.stringify({result: false}));
+        }
+        return;
+      }
+      res.setHeader('Content-Type', 'text/plain');
+      const db = await fetchRecords(req.body.ids, dbPath);
+      db.forEach((v) => {
+        const wrappedSeq = v.seq.match(/.{1,80}/g).join('\n') + '\n';
+        res.write('>' + v.id + '\n');
+        res.write(wrappedSeq);
+      });
+      res.end();
+    } catch (err) {
+      console.trace(err);
+    }
+  };
+};
+
+/**
+   @param {string} dbRoot Path to directory where genome database should be saved
+   @param {string} fastaPath Path to fasta file to use as input to create database
+
+   Database will overwrite existing database files to avoid duplicates.
+   TODO: Maybe do something else to prevent unexpected data loss
+*/
+const makeDB = (dbRoot, fastaPath) => new Promise((resolve, reject) => {
+
+  process.stdin.setEncoding('utf8');
+
+  if (!fs.existsSync(dbRoot)) {
+    fs.mkdirSync(dbRoot);
+  }
+  const dbPath = getDbPath(fastaPath);
+
+  if (fs.existsSync(dbPath)) {
+    fs.unlink(dbPath, () => { console.log(`Overwrote ${dbPath} with new data!`);});
+  }
+
+  const processRecord = new PassThrough();
+  const db = new Engine({filename: dbPath, autoload: true});
+  let rc = 0;
+
+  processRecord.on('data', (rec) => {
+    const obj = JSON.parse(rec);
+    const outrec = {id: obj.id, seq: obj.seq, source: fastaPath};
+    db.insert(outrec);
+    rc++;
+  });
+
+  processRecord.on('end', () => {
+    console.log(`Total added: ${rc} seqs to ${dbPath}`);
+    if (fs.existsSync(dbPath)) {
+      resolve();
+    } else {
+      reject(`File: ${dbPath} was not created.`);
+    }
+  }
+  );
+  const rs = fs.createReadStream(fastaPath);
+  rs.pipe(fasta())
+    .pipe(processRecord);
+
+});
+
+/**
+   @param {string} path Path to datasetDir so we can create database if corresponding fasta
+   files exists for aupsice input JSON file
+*/
+const prepareDbs = async (localPath) => {
+  try {
+    const files = await readdir(localPath);
+    const v2Files = files.filter((file) => (
+      file.endsWith(".fasta")
+    ));
+    v2Files.forEach((v) => {
+      makeDB(localPath, localPath + '/' + v);
+    });
+
+
+  } catch (err) {
+    // utils.warn(`Couldn't collect available dataset files (path searched: ${locaPath})`);
+    // utils.verbose(err);
+  }
+};
+
+module.exports = {
+  fetchRecords,
+  getDbPath,
+  makeDB,
+  prepareDbs,
+  getGenomeDB
+};

cli/server/parseNarrative.js

@@ -60,7 +60,7 @@ const makeFrontMatterBlock = (frontMatter) => {
       markdown.push(`#### License: ${license}`);
     }
   }
-  
+
   const block = new Proxy({}, blockProxyHandler);
   block.url = frontMatter.dataset;
   block.contents = markdown.join("\n");

cli/view.js

@@ -11,6 +11,7 @@ const utils = require("./utils");
 const version = require('../src/version').version;
 const chalk = require('chalk');
 const SUPPRESS = require('argparse').Const.SUPPRESS;
+const bodyParser = require('body-parser');
 
 
 const addParser = (parser) => {
@@ -58,12 +59,18 @@ const loadAndAddHandlers = ({app, handlersArg, datasetDir, narrativeDir}) => {
       .setUpGetDatasetHandler({datasetsPath});
     handlers.getNarrative = require("./server/getNarrative")
       .setUpGetNarrativeHandler({narrativesPath});
+    handlers.handleGenomeDbs = require("./server/handleGenomeDbs")
+      .prepareDbs(datasetsPath); /* use sanitized datasetPath */
+    handlers.handleGenomeDbs = require("./server/handleGenomeDbs")
+      .getGenomeDB(datasetsPath); /* use sanitized datasetPath */
   }
 
   /* apply handlers */
   app.get("/charon/getAvailable", handlers.getAvailable);
   app.get("/charon/getDataset", handlers.getDataset);
   app.get("/charon/getNarrative", handlers.getNarrative);
+  app.get("/charon/getGenomeData", handlers.handleGenomeDbs);
+  app.post("/charon/getGenomeData", handlers.handleGenomeDbs);
   app.get("/charon*", (req, res) => {
     res.statusMessage = "Query unhandled -- " + req.originalUrl;
     utils.warn(res.statusMessage);
@@ -102,7 +109,13 @@ const run = (args) => {
   const app = express();
   app.set('port', process.env.PORT || 4000);
   app.set('host', process.env.HOST || "localhost");
+  // parse application/json
+  app.use(bodyParser.json({limit: '20mb'}));
   app.use(compression());
+  // parse application/x-www-form-urlencoded
+  app.use(bodyParser.urlencoded({ extended: false }));
+
+
   app.use(nakedRedirect({reverse: true})); /* redirect www.name.org to name.org */
 
   if (args.customBuild) {

docs-src/website/siteConfig.js

@@ -17,7 +17,7 @@ const siteConfig = {
 
   // Header links in the top nav bar
   headerLinks: [
-    {doc: 'introduction/overview', label: 'Docs'},
+    {doc: 'introduction/overview', label: 'Docs'}
     // {doc: 'tutorial/overview', label: 'Tutorial'}
   ],
 

docs/js/scrollSpy.js

@@ -18,7 +18,7 @@
       // throttle
       return;
     }
-    timer = setTimeout(function() {
+    timer = setTimeout(function () {
       timer = null;
       let activeNavFound = false;
       const headings = findHeadings(); // toc nav anchors
@@ -48,7 +48,7 @@
           } else {
             console.error('Can not find header element', {
               id: next,
-              heading,
+              heading
             });
           }
         }
@@ -68,9 +68,9 @@
 
   document.addEventListener('scroll', onScroll);
   document.addEventListener('resize', onScroll);
-  document.addEventListener('DOMContentLoaded', function() {
+  document.addEventListener('DOMContentLoaded', function () {
     // Cache the headings once the page has fully loaded.
     headingsCache = findHeadings();
     onScroll();
   });
-})();
+}());