From f7c5c5f2455d15b6f456c4231dbd5434eb9b2e8e Mon Sep 17 00:00:00 2001
From: Joe Hand <joe@joeahand.com>
Date: Thu, 17 Nov 2016 15:28:08 -0800
Subject: [PATCH] add files

---
 LICENSE              |   0
 cli.js               |  72 ++++++++++++++++++++++++++++
 commands/clone.js    |  17 +++++++
 commands/create.js   |  52 +++++++++++++++++++++
 commands/download.js |  86 ++++++++++++++++++++++++++++++++++
 commands/sync.js     | 109 +++++++++++++++++++++++++++++++++++++++++++
 commands/update.js   |   4 ++
 lib/importFiles.js   |   8 ++++
 lib/initArchive.js   |  40 ++++++++++++++++
 lib/network.js       |  11 +++++
 lib/stats.js         |  12 +++++
 package.json         |  34 ++++++++++++++
 readme.md            |  42 +++++++++++++++++
 ui/bar.js            |  11 +++++
 ui/index.js          |   3 ++
 ui/link.js           |   4 ++
 ui/network.js        |  12 +++++
 17 files changed, 517 insertions(+)
 create mode 100644 LICENSE
 create mode 100755 cli.js
 create mode 100644 commands/clone.js
 create mode 100644 commands/create.js
 create mode 100644 commands/download.js
 create mode 100644 commands/sync.js
 create mode 100644 commands/update.js
 create mode 100644 lib/importFiles.js
 create mode 100644 lib/initArchive.js
 create mode 100644 lib/network.js
 create mode 100644 lib/stats.js
 create mode 100644 package.json
 create mode 100644 readme.md
 create mode 100644 ui/bar.js
 create mode 100644 ui/index.js
 create mode 100644 ui/link.js
 create mode 100644 ui/network.js

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..e69de29
diff --git a/cli.js b/cli.js
new file mode 100755
index 0000000..c140655
--- /dev/null
+++ b/cli.js
@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+
+var subcommand = require('subcommand')
+
+process.title = 'dat'
+
+var commands = [
+  {
+    name: 'clone',
+    command: require('./commands/clone')
+  },
+  {
+    name: 'create',
+    command: require('./commands/create'),
+    options: [
+      {
+        name: 'import',
+        boolean: true,
+        default: true
+      }
+    ]
+  },
+  {
+    name: 'update',
+    command: require('./commands/update')
+  },
+  {
+    name: 'start',
+    command: require('./commands/sync'),
+    options: [
+      {
+        name: 'import',
+        boolean: true,
+        default: true
+      }
+    ]
+  },
+  {
+    name: 'sync',
+    command: require('./commands/sync'),
+    options: [
+      {
+        name: 'import',
+        boolean: true,
+        default: true
+      }
+    ]
+  }
+]
+
+var config = {
+  defaults: [
+    { name: 'dir', default: process.cwd() },
+    { name: 'logspeed', default: 200 }
+  ],
+  none: none,
+  commands: commands
+}
+
+var match = subcommand(config)
+match(process.argv.slice(2))
+
+function none () {
+  console.error('Usage: dat-verb <cmd> [options]')
+  console.error('   dat create                        create a local dat')
+  console.error('   dat sync                          sync updated files for a local dat or update a remote dat')
+  console.error('   dat clone <dat-link> <directory>  clone a remote dat')
+  console.error('   dat update                        download updates from a remote dat and exits')
+  console.error('')
+  console.error('    --dir=<folder>   set directory for all commands')
+  process.exit(1)
+}
diff --git a/commands/clone.js b/commands/clone.js
new file mode 100644
index 0000000..b053ae9
--- /dev/null
+++ b/commands/clone.js
@@ -0,0 +1,17 @@
+var fs = require('fs')
+
+module.exports = function (opts) {
+  opts.key = opts._[0]
+  if (!opts.key) {
+    console.error('key required to clone')
+    process.exit(1)
+  }
+
+  opts.resume = false
+  opts.dir = opts._[1] || opts.dir
+  if (!opts.dir || opts.dir === process.cwd()) opts.dir = opts.key // Don't allow download to cwd for now
+  try {
+    fs.accessSync(opts.dir, fs.F_OK)
+  } catch (e) { fs.mkdirSync(opts.dir) }
+  require('./download')(opts)
+}
diff --git a/commands/create.js b/commands/create.js
new file mode 100644
index 0000000..6b4ab46
--- /dev/null
+++ b/commands/create.js
@@ -0,0 +1,52 @@
+var logger = require('status-logger')
+var prettyBytes = require('pretty-bytes')
+var importFiles = require('../lib/importFiles')
+var initArchive = require('../lib/initArchive')
+var ui = require('../ui')
+
+module.exports = function (opts) {
+  var dir = opts.dir
+  var importStatus = null
+  var bar = ui.bar()
+
+  var output = ['Creating Dat Archive...']
+  var log = logger(output, {debug: false, quiet: false})
+
+  setInterval(function () {
+    if (importStatus) updateProgress()
+    log.print()
+  }, opts.logspeed)
+
+  initArchive(dir, {resume: false}, function (err, archive) {
+    if (err) return exit(err)
+
+    output[0] = `Dat Archive initialized: ${opts.dir}`
+    output.push(ui.link(archive))
+    if (!opts.import) return exit()
+
+    output.push('')
+    output.push('Importing files to archive...')
+
+    importStatus = importFiles(archive, dir, {live: false, resume: false}, function (err) {
+      if (err) return exit(err)
+      output[3] = 'File import finished!'
+      output.push(`Total Size: ${importStatus.fileCount} ${importStatus.fileCount === 1 ? 'file' : 'files'} (${prettyBytes(importStatus.totalSize)})`)
+      exit()
+    })
+  })
+
+  function updateProgress () {
+    var importedFiles = importStatus.fileCount - 1 // TODO: bug in importer?
+    var progress = Math.round(importedFiles * 100 / importStatus.countStats.files)
+    output[3] = bar(progress) + ' ' + importedFiles + ' files imported'
+  }
+
+  function exit (err) {
+    if (err) {
+      console.error(err)
+      process.exit(1)
+    }
+    log.print()
+    process.exit(0)
+  }
+}
diff --git a/commands/download.js b/commands/download.js
new file mode 100644
index 0000000..bd3cf4a
--- /dev/null
+++ b/commands/download.js
@@ -0,0 +1,86 @@
+var logger = require('status-logger')
+var prettyBytes = require('pretty-bytes')
+var initArchive = require('../lib/initArchive')
+var createNetwork = require('../lib/network')
+var initStats = require('../lib/stats')
+var ui = require('../ui')
+
+module.exports = function (opts) {
+  var key = opts.key
+  var dir = opts.dir
+  var resume = opts.resume || false
+  if (!key && !resume) return exit('Key required')
+
+  var bar = ui.bar()
+  var network = null
+  var stats = null
+  var connected = false
+
+  var output = [['Starting Dat...'], []]
+  var log = logger(output, {debug: false, quiet: false})
+  var progressOutput = output[0]
+
+  setInterval(function () {
+    if (stats && connected) updateDownload()
+    if (network) updateNetwork()
+    log.print()
+  }, opts.logspeed)
+
+  initArchive(dir, {resume: resume, key: key}, function (err, archive, db) {
+    if (err) return exit(err)
+
+    // General Archive Info
+    progressOutput[0] = `Cloning Dat Archive: ${dir}`
+    progressOutput.push(ui.link(archive) + '\n')
+
+    // Stats (used for network + download)
+    stats = initStats(archive, db)
+    stats.on('update:blocksProgress', function () {
+      checkDone()
+    })
+    progressOutput.push('Looking for Dat Archive in Network')
+    progressOutput.push('')
+
+    // Network
+    network = createNetwork(archive, opts)
+    network.swarm.once('connection', function (peer) {
+      connected = true
+      progressOutput[2] = 'Starting Download...'
+    })
+  })
+
+  function updateDownload () {
+    var st = stats.get()
+    if (!st.blocksTotal) {
+      progressOutput[2] = '... Fetching Metadata'
+      return
+    }
+
+    var progress = Math.round(st.blocksProgress * 100 / st.blocksTotal)
+    if (progress === 100) return checkDone()
+    progressOutput[2] = bar(progress)
+    progressOutput[3] = `Total size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})\n`
+  }
+
+  function updateNetwork () {
+    output[1] = ui.network(network.swarm.connections.length, stats.get())
+  }
+
+  function checkDone () {
+    var st = stats.get()
+    if (connected && st.blocksTotal && st.blocksProgress === st.blocksTotal) {
+      progressOutput[2] = 'Download Finished!'
+      progressOutput[3] = `Total size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})\n`
+      return exit()
+    }
+  }
+
+  function exit (err) {
+    if (err) {
+      console.error(err)
+      process.exit(1)
+    }
+    log.print()
+    process.exit(0)
+  }
+}
diff --git a/commands/sync.js b/commands/sync.js
new file mode 100644
index 0000000..f661be2
--- /dev/null
+++ b/commands/sync.js
@@ -0,0 +1,109 @@
+var logger = require('status-logger')
+var prettyBytes = require('pretty-bytes')
+var initArchive = require('../lib/initArchive')
+var importFiles = require('../lib/importFiles')
+var createNetwork = require('../lib/network')
+var initStats = require('../lib/stats')
+var ui = require('../ui')
+
+module.exports = function (opts) {
+  var dir = opts.dir
+  var bar = ui.bar()
+  var connected = false
+  var downloader = false
+  var importDone = false
+  var importStatus = null
+  var network = null
+  var stats = null
+
+  var output = [['Starting Dat...'], []]
+  var log = logger(output, {debug: false, quiet: false})
+  var progressOutput = output[0]
+  var netOutput = output[1]
+
+  setInterval(function () {
+    if (stats && downloader) updateDownload()
+    if (importStatus && !importDone) updateImport()
+    if (network) updateNetwork()
+    log.print()
+  }, opts.logspeed)
+
+  initArchive(dir, {resume: true}, function (err, archive, db) {
+    if (err) return exit(err)
+
+    // General Archive Info
+    progressOutput[0] = `Syncing Dat Archive: ${dir}`
+    progressOutput.push(ui.link(archive) + '\n')
+
+    // Stats (used for network + download)
+    stats = initStats(archive, db)
+
+    // Network
+    network = createNetwork(archive, opts)
+    netOutput.push('Waiting for Dat Network connections...')
+
+    if (!archive.owner) {
+      // TODO: Download syncing
+      downloader = true
+      progressOutput.push('Looking for Dat Archive in Network')
+      progressOutput.push('')
+      network.swarm.once('connection', function (peer) {
+        connected = true
+        progressOutput[2] = 'Starting Download...'
+      })
+    }
+
+    if (archive.owner && opts.import) {
+      // File Imports
+      progressOutput.push('Importing new & updated files to archive...')
+      progressOutput.push('')
+
+      // TODO: allow live: true
+      importStatus = importFiles(archive, dir, {live: false, resume: true}, function (err) {
+        if (err) return exit(err)
+        importDone = true
+        progressOutput[2] = 'Archive update finished! Sharing latest files.'
+        progressOutput[3] = `Total Size: ${importStatus.fileCount} ${importStatus.fileCount === 1 ? 'file' : 'files'} (${prettyBytes(importStatus.totalSize)})`
+        progressOutput.push('')
+      })
+    }
+  })
+
+  function updateDownload () {
+    var st = stats.get()
+    if (!st.blocksTotal) {
+      progressOutput[2] = '... Fetching Metadata'
+      return
+    }
+    var completed = st.blocksProgress === st.blocksTotal
+    if (completed && connected) {
+      progressOutput[2] = 'Files updated to latest!'
+    } else if (completed) {
+      progressOutput[2] = 'Waiting for connection to get latest data.'
+    } else {
+      var progress = Math.round(st.blocksProgress * 100 / st.blocksTotal)
+      progressOutput[2] = bar(progress)
+    }
+
+    progressOutput[3] = `Total size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})\n`
+  }
+
+  function updateImport () {
+    var importedFiles = importStatus.fileCount - 1 // TODO: bug in importer?
+    var progress = Math.round(importedFiles * 100 / importStatus.countStats.files)
+    progressOutput[2] = bar(progress) + ' ' + importedFiles + ' files imported\n'
+  }
+
+  function updateNetwork () {
+    netOutput = output[1] = ui.network(network.swarm.connections.length, stats.get())
+  }
+
+  function exit (err) {
+    if (err) {
+      console.error(err)
+      process.exit(1)
+    }
+    log.print()
+    process.exit(0)
+  }
+}
diff --git a/commands/update.js b/commands/update.js
new file mode 100644
index 0000000..29850d1
--- /dev/null
+++ b/commands/update.js
@@ -0,0 +1,4 @@
+module.exports = function (opts) {
+  opts.resume = true
+  require('./download')(opts)
+}
diff --git a/lib/importFiles.js b/lib/importFiles.js
new file mode 100644
index 0000000..26fe94b
--- /dev/null
+++ b/lib/importFiles.js
@@ -0,0 +1,8 @@
+var importer = require('hyperdrive-count-import')
+
+module.exports = function (archive, dir, opts, cb) {
+  if (!opts) opts = {}
+  if (!opts.ignore) opts.ignore = [/^(?:\/.*)?\.dat(?:\/.*)?$/, /[\/\\]\./] // ignore .dat and all hidden
+
+  return importer(archive, dir, opts, cb)
+}
diff --git a/lib/initArchive.js b/lib/initArchive.js
new file mode 100644
index 0000000..ca6aacf
--- /dev/null
+++ b/lib/initArchive.js
@@ -0,0 +1,40 @@
+var path = require('path')
+var datDb = require('dat-folder-db')
+var hyperdrive = require('hyperdrive')
+var raf = require('random-access-file')
+
+module.exports = function (dir, opts, cb) {
+  datDb(dir, opts, function (err, db, key, saveKey) {
+    if (err) return cb(err)
+    if (opts.resume && !key) return cb('No existing archive. Please use create or clone first.')
+    if (!opts.resume && key) return cb(`Archive exists in directory, cannot overwrite. \nExisting key: ${key}`)
+    if (key && opts.key && opts.key !== key) return cb('Error: todo')
+
+    var drive = hyperdrive(db)
+    var archive = drive.createArchive(key || opts.key, {
+      live: true,
+      file: function (name) {
+        return raf(path.join(dir, name))
+      }
+    })
+
+    if (key) return done() // Key already in db
+    saveKey(archive.key, function (err) {
+      // Saves key back to db for resuming.
+      if (err) return cb(err)
+      done()
+    })
+
+    function done () {
+      if (!key && opts.key) {
+        // Downloading a new archive (can't archive.open)
+        archive.owner = false // TODO: HACK!
+        return cb(null, archive, db)
+      }
+      archive.open(function (err) {
+        if (err) return cb(err)
+        cb(null, archive, db)
+      })
+    }
+  })
+}
diff --git a/lib/network.js b/lib/network.js
new file mode 100644
index 0000000..3b21851
--- /dev/null
+++ b/lib/network.js
@@ -0,0 +1,11 @@
+var createSwarm = require('hyperdrive-archive-swarm')
+
+module.exports = function (archive, opts) {
+  var swarm = createSwarm(archive, opts)
+
+  return {
+    swarm: swarm,
+    start: swarm.join(archive.discoveryKey),
+    stop: swarm.leave(archive.discoveryKey)
+  }
+}
diff --git a/lib/stats.js b/lib/stats.js
new file mode 100644
index 0000000..3d12ea0
--- /dev/null
+++ b/lib/stats.js
@@ -0,0 +1,12 @@
+var Stats = require('hyperdrive-stats')
+var sub = require('subleveldown')
+var encoding = require('dat-encoding')
+
+module.exports = function (archive, db) {
+  var stats = Stats({
+    archive: archive,
+    db: sub(db, `${encoding.encode(archive.key)}-stats`)
+  })
+
+  return stats
+}
diff --git a/package.json b/package.json
new file mode 100644
index 0000000..05f5fe4
--- /dev/null
+++ b/package.json
@@ -0,0 +1,34 @@
+{
+  "name": "dat-verb",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "bin": {
+    "dat-verb": "cli.js"
+  },
+  "scripts": {
+    "test": "standard"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/joehand/dat-verb.git"
+  },
+  "author": "Joe Hand <joe@hand.email> (https://joeahand.com/)",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/joehand/dat-verb/issues"
+  },
+  "homepage": "https://github.com/joehand/dat-verb#readme",
+  "dependencies": {
+    "dat-encoding": "^3.0.1",
+    "dat-folder-db": "^2.0.0",
+    "hyperdrive": "^7.8.0",
+    "hyperdrive-archive-swarm": "^5.0.4",
+    "hyperdrive-count-import": "^1.0.0",
+    "pretty-bytes": "^4.0.2",
+    "progress-string": "^1.2.1",
+    "random-access-file": "^1.3.1",
+    "status-logger": "^3.0.0",
+    "subcommand": "^2.0.4"
+  }
+}
diff --git a/readme.md b/readme.md
new file mode 100644
index 0000000..084b5d3
--- /dev/null
+++ b/readme.md
@@ -0,0 +1,42 @@
+# dat-verb
+
+An experimental dat cli client!
+
+## Usage
+
+### Sharing
+
+```
+dat-verb create [--dir=<folder>] [--no-import]
+```
+
+Create a new Dat Archive in the current directory (or specify `--dir`). Will automatically import the files in that directory to the archive.
+
+```
+dat-verb sync [--dir=<folder>] [--no-import]
+dat-verb start
+```
+
+Start sharing your Dat Archive over the network. Will import new or updated files since you ran `create` or `sync` last. 
+
+### Downloading
+
+```
+dat-verb clone <dat-link> [<folder>]
+```
+
+Clone a remote Dat Archive to a local folder. Will create a folder with the key name is no folder is specified.
+
+
+```
+dat-verb update [--dir=<folder>]
+```
+
+Update local Dat Archive to latest files and exit.
+
+```
+dat-verb sync [--dir=<folder>]
+dat-verb start [--dir=<folder>]
+```
+
+Download latest files and keep connection open to continue updating as remote source is updated.
diff --git a/ui/bar.js b/ui/bar.js
new file mode 100644
index 0000000..9154fd1
--- /dev/null
+++ b/ui/bar.js
@@ -0,0 +1,11 @@
+var progress = require('progress-string')
+
+module.exports = function () {
+  return progress({
+    width: 50,
+    total: 100,
+    style: function (complete, incomplete) {
+      return '[' + complete + '>' + incomplete + ']'
+    }
+  })
+}
diff --git a/ui/index.js b/ui/index.js
new file mode 100644
index 0000000..0c7b9ad
--- /dev/null
+++ b/ui/index.js
@@ -0,0 +1,3 @@
+module.exports.bar = require('./bar')
+module.exports.link = require('./link')
+module.exports.network = require('./network')
diff --git a/ui/link.js b/ui/link.js
new file mode 100644
index 0000000..29081d5
--- /dev/null
+++ b/ui/link.js
@@ -0,0 +1,4 @@
+
+module.exports = function (archive) {
+  return `Link: ${archive.key.toString('hex')}`
+}
diff --git a/ui/network.js b/ui/network.js
new file mode 100644
index 0000000..ca8e5cc
--- /dev/null
+++ b/ui/network.js
@@ -0,0 +1,12 @@
+var prettyBytes = require('pretty-bytes')
+
+module.exports = function (peers, stats) {
+  if (!peers) return ['Looking for connections in Dat Network...']
+  var msg = []
+  msg.push(`${peers} ${peers === 1 ? 'peer' : 'peers'} on the Dat Network`)
+  var spdMsg = ''
+  if (stats.downloadSpeed) spdMsg += `Downloading: ${prettyBytes(stats.downloadSpeed)}/s` + '  '
+  if (stats.uploadSpeed) spdMsg += `Uploading: ${prettyBytes(stats.uploadSpeed)}/s`
+  msg.push(spdMsg)
+  return msg
+}