From 3d16090c9b62e03e27824891450eabd74e83f90b Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Mon, 17 Apr 2017 14:55:39 -0700 Subject: [PATCH] cleanup storage and fix downloads + speed --- cli.js | 35 ++++++++++++++++-------------- index.js | 61 +++++++++++----------------------------------------- package.json | 1 + storage.js | 29 +++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 64 deletions(-) create mode 100644 storage.js diff --git a/cli.js b/cli.js index e52fddf..9e289a1 100755 --- a/cli.js +++ b/cli.js @@ -7,6 +7,7 @@ var progress = require('progress-string') var cliTruncate = require('cli-truncate') var neatLog = require('neat-log') var output = require('neat-log/output') +var networkSpeed = require('hyperdrive-network-speed') var dat = require('./') @@ -62,16 +63,12 @@ function trackProgress (state, bus) { }) function trackDownload () { - // var progress = state.progress + var progress = state.progress state.downloading = true - - // progress.on('put', function (src, dst) { - // state.fileDownload = { - // src: src, - // dst: dst, - // progress: 0 - // } - // }) + state.archive.content.on('sync', function () { + state.nsync = true + bus.emit('render') + }) } function trackImport () { @@ -136,14 +133,13 @@ function trackNetwork (state, bus) { }) }) - archive.content.on('upload', function (index, data) { - state.uploadSpeed = uploadSpeed(data.length) - bus.emit('render') - }) - archive.content.on('download', function (index, data) { - state.downloadSpeed = downloadSpeed(data.length) + var speed = networkSpeed(archive) + + setInterval(function () { + state.uploadSpeed = speed.uploadSpeed + state.downloadSpeed = speed.downloadSpeed bus.emit('render') - }) + }, 500) }) } @@ -192,6 +188,13 @@ function networkUI (state) { } function downloadUI (state) { + if (state.nsync) { + return output` + + Archive up to date with latest. + Exit if you'd like. + ` + } return output` Download progress TODO diff --git a/index.js b/index.js index 7600a70..87103eb 100644 --- a/index.js +++ b/index.js @@ -1,24 +1,30 @@ var fs = require('fs') var path = require('path') var xtend = require('xtend') -var mkdirp = require('mkdirp') var hyperdrive = require('hyperdrive') var network = require('hyperdiscovery') -var ram = require('random-access-memory') -var secretStore = require('dat-secret-storage') var mirror = require('mirror-folder') var count = require('count-files') var datIgnore = require('dat-ignore') var encoding = require('dat-encoding') var debug = require('debug')('dat') +var storage = require('./storage') module.exports = run +/** + * Run dat-next + * @param {string} src directory or key (for downloading) + * @param {string} dest directory to download to, required for download + * @param {object} opts options + * @param {Function} cb callback(err, archive, swarm, progress) + */ function run (src, dest, opts, cb) { if (!opts) opts = {} var key if (dest) { + // Downloading try { // validate key + remove dat:// stuff key = encoding.toStr(src) @@ -26,36 +32,23 @@ function run (src, dest, opts, cb) { return cb(new Error('Invalid dat link')) } src = null - opts.sparse = true + } else { + opts.indexing = true } var progress var ignore = datIgnore(src || dest) - var archive = hyperdrive(storage(), key, opts) + var archive = hyperdrive(storage(src || dest, opts), key, opts) archive.on('ready', function () { if (!archive.metadata.writable && !dest) { return cb(new Error('Archive is not writable and no destination provided.')) } if (archive.metadata.writable) importFiles() - else downloadFiles() var swarm = joinNetwork() cb(null, archive, swarm, progress) }) - function storage () { - if (opts.temp) return ram - if (typeof opts.sleep === 'string') return secretStore(opts.sleep) - if (!src) { - mkdirp.sync(dest) - return path.join(dest, '.dat') // TODO: if dest is file - } - - var isDir = fs.statSync(src).isDirectory() - if (isDir) return secretStore(path.join(src, '.dat')) - return cb(new Error('Specify dir for sleep files: --sleep ')) - } - function importFiles () { if (!archive.metadata.writable) return @@ -77,39 +70,11 @@ function run (src, dest, opts, cb) { return progress } - function downloadFiles () { - if (!archive.metadata.length) { - return archive.metadata.once('append', downloadFiles) - } - var length = archive.metadata.length - var progress = mirror({name: '/', fs: archive}, dest) - var changed = false - - progress.on('put', function (src) { - changed = true - }) - - progress.on('del', function (src) { - changed = true - }) - - progress.on('end', function () { - if (!changed) { - if (length !== archive.metadata.length) downloadFiles() - else archive.metadata.once('append', downloadFiles) - return - } - archive.downloaded = true - }) - - return progress - } - function joinNetwork () { var swarm = network(archive, xtend({ stream: function (peer) { var stream = archive.replicate({ - live: !archive.metadata.writable && opts.sync + live: true // !archive.metadata.writable && opts.sync }) stream.on('error', function (err) { debug('Replication error:', err.message) diff --git a/package.json b/package.json index a3c86cb..abf81c8 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "dat-encoding": "^4.0.2", "dat-ignore": "^1.0.0", "dat-secret-storage": "^1.0.0", + "dat-storage": "github:joehand/dat-storage#fix-write", "debug": "^2.4.5", "hyperdiscovery": "^1.3.0", "hyperdrive": "^8.0.0", diff --git a/storage.js b/storage.js new file mode 100644 index 0000000..c78b3a3 --- /dev/null +++ b/storage.js @@ -0,0 +1,29 @@ +var fs = require('fs') +var ram = require('random-access-memory') +var secretStore = require('dat-secret-storage') +var datStore = require('dat-storage') +var mkdirp = require('mkdirp') +var debug = require('debug')('dat-storage') + +module.exports = storage + +function storage (dir, opts) { + if (opts.temp) return ram + if (typeof opts.sleep === 'string') return datStore(opts.sleep) + // if (typeof opts.sleep === 'string') return secretStore(datStore(opts.sleep)) + + mkdirp.sync(dir) + try { + var isDir = fs.statSync(dir).isDirectory() + if (isDir) return datStore(dir) + // if (isDir) return secretStore(datStore(dir)) + error() + } catch (e) { + throw e + } + + function error() { + // TODO: single file sleep storage + throw new Error('Specify dir for sleep files: --sleep ') + } +} \ No newline at end of file