add files

cli.js

@@ -0,0 +1,72 @@
+#!/usr/bin/env node
+
+var subcommand = require('subcommand')
+
+process.title = 'dat'
+
+var commands = [
+  {
+    name: 'clone',
+    command: require('./commands/clone')
+  },
+  {
+    name: 'create',
+    command: require('./commands/create'),
+    options: [
+      {
+        name: 'import',
+        boolean: true,
+        default: true
+      }
+    ]
+  },
+  {
+    name: 'update',
+    command: require('./commands/update')
+  },
+  {
+    name: 'start',
+    command: require('./commands/sync'),
+    options: [
+      {
+        name: 'import',
+        boolean: true,
+        default: true
+      }
+    ]
+  },
+  {
+    name: 'sync',
+    command: require('./commands/sync'),
+    options: [
+      {
+        name: 'import',
+        boolean: true,
+        default: true
+      }
+    ]
+  }
+]
+
+var config = {
+  defaults: [
+    { name: 'dir', default: process.cwd() },
+    { name: 'logspeed', default: 200 }
+  ],
+  none: none,
+  commands: commands
+}
+
+var match = subcommand(config)
+match(process.argv.slice(2))
+
+function none () {
+  console.error('Usage: dat-verb <cmd> [options]')
+  console.error('   dat create                        create a local dat')
+  console.error('   dat sync                          sync updated files for a local dat or update a remote dat')
+  console.error('   dat clone <dat-link> <directory>  clone a remote dat')
+  console.error('   dat update                        download updates from a remote dat and exits')
+  console.error('')
+  console.error('    --dir=<folder>   set directory for all commands')
+  process.exit(1)
+}

commands/clone.js

@@ -0,0 +1,17 @@
+var fs = require('fs')
+
+module.exports = function (opts) {
+  opts.key = opts._[0]
+  if (!opts.key) {
+    console.error('key required to clone')
+    process.exit(1)
+  }
+
+  opts.resume = false
+  opts.dir = opts._[1] || opts.dir
+  if (!opts.dir || opts.dir === process.cwd()) opts.dir = opts.key // Don't allow download to cwd for now
+  try {
+    fs.accessSync(opts.dir, fs.F_OK)
+  } catch (e) { fs.mkdirSync(opts.dir) }
+  require('./download')(opts)
+}

commands/create.js

@@ -0,0 +1,52 @@
+var logger = require('status-logger')
+var prettyBytes = require('pretty-bytes')
+var importFiles = require('../lib/importFiles')
+var initArchive = require('../lib/initArchive')
+var ui = require('../ui')
+
+module.exports = function (opts) {
+  var dir = opts.dir
+  var importStatus = null
+  var bar = ui.bar()
+
+  var output = ['Creating Dat Archive...']
+  var log = logger(output, {debug: false, quiet: false})
+
+  setInterval(function () {
+    if (importStatus) updateProgress()
+    log.print()
+  }, opts.logspeed)
+
+  initArchive(dir, {resume: false}, function (err, archive) {
+    if (err) return exit(err)
+
+    output[0] = `Dat Archive initialized: ${opts.dir}`
+    output.push(ui.link(archive))
+    if (!opts.import) return exit()
+
+    output.push('')
+    output.push('Importing files to archive...')
+
+    importStatus = importFiles(archive, dir, {live: false, resume: false}, function (err) {
+      if (err) return exit(err)
+      output[3] = 'File import finished!'
+      output.push(`Total Size: ${importStatus.fileCount} ${importStatus.fileCount === 1 ? 'file' : 'files'} (${prettyBytes(importStatus.totalSize)})`)
+      exit()
+    })
+  })
+
+  function updateProgress () {
+    var importedFiles = importStatus.fileCount - 1 // TODO: bug in importer?
+    var progress = Math.round(importedFiles * 100 / importStatus.countStats.files)
+    output[3] = bar(progress) + ' ' + importedFiles + ' files imported'
+  }
+
+  function exit (err) {
+    if (err) {
+      console.error(err)
+      process.exit(1)
+    }
+    log.print()
+    process.exit(0)
+  }
+}

commands/download.js

@@ -0,0 +1,86 @@
+var logger = require('status-logger')
+var prettyBytes = require('pretty-bytes')
+var initArchive = require('../lib/initArchive')
+var createNetwork = require('../lib/network')
+var initStats = require('../lib/stats')
+var ui = require('../ui')
+
+module.exports = function (opts) {
+  var key = opts.key
+  var dir = opts.dir
+  var resume = opts.resume || false
+  if (!key && !resume) return exit('Key required')
+
+  var bar = ui.bar()
+  var network = null
+  var stats = null
+  var connected = false
+
+  var output = [['Starting Dat...'], []]
+  var log = logger(output, {debug: false, quiet: false})
+  var progressOutput = output[0]
+
+  setInterval(function () {
+    if (stats && connected) updateDownload()
+    if (network) updateNetwork()
+    log.print()
+  }, opts.logspeed)
+
+  initArchive(dir, {resume: resume, key: key}, function (err, archive, db) {
+    if (err) return exit(err)
+
+    // General Archive Info
+    progressOutput[0] = `Cloning Dat Archive: ${dir}`
+    progressOutput.push(ui.link(archive) + '\n')
+
+    // Stats (used for network + download)
+    stats = initStats(archive, db)
+    stats.on('update:blocksProgress', function () {
+      checkDone()
+    })
+    progressOutput.push('Looking for Dat Archive in Network')
+    progressOutput.push('')
+
+    // Network
+    network = createNetwork(archive, opts)
+    network.swarm.once('connection', function (peer) {
+      connected = true
+      progressOutput[2] = 'Starting Download...'
+    })
+  })
+
+  function updateDownload () {
+    var st = stats.get()
+    if (!st.blocksTotal) {
+      progressOutput[2] = '... Fetching Metadata'
+      return
+    }
+
+    var progress = Math.round(st.blocksProgress * 100 / st.blocksTotal)
+    if (progress === 100) return checkDone()
+    progressOutput[2] = bar(progress)
+    progressOutput[3] = `Total size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})\n`
+  }
+
+  function updateNetwork () {
+    output[1] = ui.network(network.swarm.connections.length, stats.get())
+  }
+
+  function checkDone () {
+    var st = stats.get()
+    if (connected && st.blocksTotal && st.blocksProgress === st.blocksTotal) {
+      progressOutput[2] = 'Download Finished!'
+      progressOutput[3] = `Total size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})\n`
+      return exit()
+    }
+  }
+
+  function exit (err) {
+    if (err) {
+      console.error(err)
+      process.exit(1)
+    }
+    log.print()
+    process.exit(0)
+  }
+}

commands/sync.js

@@ -0,0 +1,109 @@
+var logger = require('status-logger')
+var prettyBytes = require('pretty-bytes')
+var initArchive = require('../lib/initArchive')
+var importFiles = require('../lib/importFiles')
+var createNetwork = require('../lib/network')
+var initStats = require('../lib/stats')
+var ui = require('../ui')
+
+module.exports = function (opts) {
+  var dir = opts.dir
+  var bar = ui.bar()
+  var connected = false
+  var downloader = false
+  var importDone = false
+  var importStatus = null
+  var network = null
+  var stats = null
+
+  var output = [['Starting Dat...'], []]
+  var log = logger(output, {debug: false, quiet: false})
+  var progressOutput = output[0]
+  var netOutput = output[1]
+
+  setInterval(function () {
+    if (stats && downloader) updateDownload()
+    if (importStatus && !importDone) updateImport()
+    if (network) updateNetwork()
+    log.print()
+  }, opts.logspeed)
+
+  initArchive(dir, {resume: true}, function (err, archive, db) {
+    if (err) return exit(err)
+
+    // General Archive Info
+    progressOutput[0] = `Syncing Dat Archive: ${dir}`
+    progressOutput.push(ui.link(archive) + '\n')
+
+    // Stats (used for network + download)
+    stats = initStats(archive, db)
+
+    // Network
+    network = createNetwork(archive, opts)
+    netOutput.push('Waiting for Dat Network connections...')
+
+    if (!archive.owner) {
+      // TODO: Download syncing
+      downloader = true
+      progressOutput.push('Looking for Dat Archive in Network')
+      progressOutput.push('')
+      network.swarm.once('connection', function (peer) {
+        connected = true
+        progressOutput[2] = 'Starting Download...'
+      })
+    }
+
+    if (archive.owner && opts.import) {
+      // File Imports
+      progressOutput.push('Importing new & updated files to archive...')
+      progressOutput.push('')
+
+      // TODO: allow live: true
+      importStatus = importFiles(archive, dir, {live: false, resume: true}, function (err) {
+        if (err) return exit(err)
+        importDone = true
+        progressOutput[2] = 'Archive update finished! Sharing latest files.'
+        progressOutput[3] = `Total Size: ${importStatus.fileCount} ${importStatus.fileCount === 1 ? 'file' : 'files'} (${prettyBytes(importStatus.totalSize)})`
+        progressOutput.push('')
+      })
+    }
+  })
+
+  function updateDownload () {
+    var st = stats.get()
+    if (!st.blocksTotal) {
+      progressOutput[2] = '... Fetching Metadata'
+      return
+    }
+    var completed = st.blocksProgress === st.blocksTotal
+    if (completed && connected) {
+      progressOutput[2] = 'Files updated to latest!'
+    } else if (completed) {
+      progressOutput[2] = 'Waiting for connection to get latest data.'
+    } else {
+      var progress = Math.round(st.blocksProgress * 100 / st.blocksTotal)
+      progressOutput[2] = bar(progress)
+    }
+
+    progressOutput[3] = `Total size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})\n`
+  }
+
+  function updateImport () {
+    var importedFiles = importStatus.fileCount - 1 // TODO: bug in importer?
+    var progress = Math.round(importedFiles * 100 / importStatus.countStats.files)
+    progressOutput[2] = bar(progress) + ' ' + importedFiles + ' files imported\n'
+  }
+
+  function updateNetwork () {
+    netOutput = output[1] = ui.network(network.swarm.connections.length, stats.get())
+  }
+
+  function exit (err) {
+    if (err) {
+      console.error(err)
+      process.exit(1)
+    }
+    log.print()
+    process.exit(0)
+  }
+}

commands/update.js

@@ -0,0 +1,4 @@
+module.exports = function (opts) {
+  opts.resume = true
+  require('./download')(opts)
+}

lib/importFiles.js

@@ -0,0 +1,8 @@
+var importer = require('hyperdrive-count-import')
+
+module.exports = function (archive, dir, opts, cb) {
+  if (!opts) opts = {}
+  if (!opts.ignore) opts.ignore = [/^(?:\/.*)?\.dat(?:\/.*)?$/, /[\/\\]\./] // ignore .dat and all hidden
+
+  return importer(archive, dir, opts, cb)
+}

lib/initArchive.js

@@ -0,0 +1,40 @@
+var path = require('path')
+var datDb = require('dat-folder-db')
+var hyperdrive = require('hyperdrive')
+var raf = require('random-access-file')
+
+module.exports = function (dir, opts, cb) {
+  datDb(dir, opts, function (err, db, key, saveKey) {
+    if (err) return cb(err)
+    if (opts.resume && !key) return cb('No existing archive. Please use create or clone first.')
+    if (!opts.resume && key) return cb(`Archive exists in directory, cannot overwrite. \nExisting key: ${key}`)
+    if (key && opts.key && opts.key !== key) return cb('Error: todo')
+
+    var drive = hyperdrive(db)
+    var archive = drive.createArchive(key || opts.key, {
+      live: true,
+      file: function (name) {
+        return raf(path.join(dir, name))
+      }
+    })
+
+    if (key) return done() // Key already in db
+    saveKey(archive.key, function (err) {
+      // Saves key back to db for resuming.
+      if (err) return cb(err)
+      done()
+    })
+
+    function done () {
+      if (!key && opts.key) {
+        // Downloading a new archive (can't archive.open)
+        archive.owner = false // TODO: HACK!
+        return cb(null, archive, db)
+      }
+      archive.open(function (err) {
+        if (err) return cb(err)
+        cb(null, archive, db)
+      })
+    }
+  })
+}