From 9ff16acf82df649d53e80c2e998b41fc28d6088d Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Sun, 26 Mar 2017 22:40:25 -0700 Subject: [PATCH 01/20] start sleeping --- bin/cli.js | 107 ---------- cli.js | 122 +++++++++++ index.js | 91 ++++++++ lib/commands/auth/login.js | 46 ---- lib/commands/auth/logout.js | 19 -- lib/commands/auth/register.js | 52 ----- lib/commands/auth/whoami.js | 16 -- lib/commands/clone.js | 90 -------- lib/commands/create.js | 119 ----------- lib/commands/doctor.js | 17 -- lib/commands/publish.js | 73 ------- lib/commands/pull.js | 35 --- lib/commands/share.js | 55 ----- lib/commands/snapshot.js | 23 -- lib/commands/sync.js | 69 ------ lib/dat-json.js | 40 ---- lib/download.js | 148 ------------- lib/network.js | 24 +++ lib/share.js | 137 ------------ lib/township.js | 17 -- lib/ui/bar.js | 11 - lib/ui/exit.js | 13 -- lib/ui/import-progress.js | 15 -- lib/ui/index.js | 9 - lib/ui/link.js | 5 - lib/ui/network.js | 12 -- lib/usage.js | 31 --- package.json | 33 ++- readme.md | 32 +-- scripts/auth-server.js | 14 -- tests/auth.js | 229 -------------------- tests/clone.js | 282 ------------------------- tests/create.js | 159 -------------- tests/doctor.js | 37 ---- tests/fixtures/all_hour.csv | 10 - tests/fixtures/folder/nested/hello.txt | 1 - tests/helpers/auth-server.js | 43 ---- tests/helpers/index.js | 98 --------- tests/helpers/spawn.js | 12 -- tests/pull.js | 117 ---------- tests/share.js | 58 ----- tests/snapshot.js | 74 ------- tests/sync-owner.js | 282 ------------------------- tests/sync-remote.js | 101 --------- tests/usage.js | 46 ---- 45 files changed, 254 insertions(+), 2770 deletions(-) delete mode 100755 bin/cli.js create mode 100755 cli.js create mode 100644 index.js delete mode 100644 lib/commands/auth/login.js delete mode 100644 lib/commands/auth/logout.js delete mode 100644 lib/commands/auth/register.js delete mode 100644 lib/commands/auth/whoami.js delete mode 100644 lib/commands/clone.js delete mode 100644 lib/commands/create.js delete mode 100644 lib/commands/doctor.js delete mode 100644 lib/commands/publish.js delete mode 100644 lib/commands/pull.js delete mode 100644 lib/commands/share.js delete mode 100644 lib/commands/snapshot.js delete mode 100644 lib/commands/sync.js delete mode 100644 lib/dat-json.js delete mode 100644 lib/download.js create mode 100644 lib/network.js delete mode 100644 lib/share.js delete mode 100644 lib/township.js delete mode 100644 lib/ui/bar.js delete mode 100644 lib/ui/exit.js delete mode 100644 lib/ui/import-progress.js delete mode 100644 lib/ui/index.js delete mode 100644 lib/ui/link.js delete mode 100644 lib/ui/network.js delete mode 100644 lib/usage.js delete mode 100644 scripts/auth-server.js delete mode 100644 tests/auth.js delete mode 100644 tests/clone.js delete mode 100644 tests/create.js delete mode 100644 tests/doctor.js delete mode 100644 tests/fixtures/all_hour.csv delete mode 100644 tests/fixtures/folder/nested/hello.txt delete mode 100644 tests/helpers/auth-server.js delete mode 100644 tests/helpers/index.js delete mode 100644 tests/helpers/spawn.js delete mode 100644 tests/pull.js delete mode 100644 tests/share.js delete mode 100644 tests/snapshot.js delete mode 100644 tests/sync-owner.js delete mode 100644 tests/sync-remote.js delete mode 100644 tests/usage.js diff --git a/bin/cli.js b/bin/cli.js deleted file mode 100755 index 4ab18c8..0000000 --- a/bin/cli.js +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env node - -var fs = require('fs') -var mkdirp = require('mkdirp') -var subcommand = require('subcommand') -var encoding = require('dat-encoding') -var debug = require('debug')('dat') -var usage = require('../lib/usage') - -process.title = 'dat-next' - -var config = { - defaults: [ - { name: 'dir', default: process.cwd(), help: 'set the directory for Dat' }, - { name: 'logspeed', default: 200 }, - { name: 'port', default: 3282, help: 'port to use for connections' }, - { name: 'utp', default: true, boolean: true, help: 'use utp for discovery' }, - { name: 'debug', default: process.env.DEBUG }, // TODO: does not work right now - { name: 'quiet', default: false, boolean: true }, - { name: 'server', default: 'https://datproject.org/api/v1' } - ], - root: { - options: [ - { - name: 'version', - boolean: true, - default: false, - abbr: 'v' - } - ], - command: usage - }, - none: syncShorthand, - commands: [ - require('../lib/commands/clone'), - require('../lib/commands/create'), - require('../lib/commands/doctor'), - require('../lib/commands/publish'), - require('../lib/commands/pull'), - require('../lib/commands/share'), - require('../lib/commands/snapshot'), - require('../lib/commands/sync'), - require('../lib/commands/auth/register'), - require('../lib/commands/auth/whoami'), - require('../lib/commands/auth/logout'), - require('../lib/commands/auth/login') - ], - usage: { - command: usage, - option: { - name: 'help', - abbr: 'h' - } - }, - aliases: { - 'init': 'create' - } -} - -var match = subcommand(config) -match(alias(process.argv.slice(2))) - -function alias (argv) { - var cmd = argv[0] - if (!config.aliases[cmd]) return argv - argv[0] = config.aliases[cmd] - return argv -} - -function syncShorthand (opts) { - if (!opts._.length) return done() - debug('Sync shortcut command') - debug(opts) - - if (opts._.length > 1) { - // dat {dir} - clone/resume in {dir} - try { - debug('Clone sync') - opts.key = encoding.toStr(opts._[0]) - opts.dir = opts._[1] - // make dir & start download - debug('mkdirp', opts.dir) - // TODO: do I want to mkdirp? or only one child? - mkdirp(opts.dir, function () { - require('../lib/download')('sync', opts) - }) - } catch (e) { return done() } - } else { - // dat {dir} - sync existing dat in {dir} - try { - debug('Share sync') - opts.dir = opts._[0] - fs.stat(opts.dir, function (err, stat) { - if (err || !stat.isDirectory()) return usage(opts) - - // Set default opts. TODO: use default opts in sync - opts.watch = opts.watch || true - opts.import = opts.import || true - require('../lib/commands/sync').command(opts) - }) - } catch (e) { return done() } - } - - function done () { - return usage(opts) - } -} diff --git a/cli.js b/cli.js new file mode 100755 index 0000000..b576f07 --- /dev/null +++ b/cli.js @@ -0,0 +1,122 @@ +#!/usr/bin/env node + +var minimist = require('minimist') +var logger = require('status-logger') +var pretty = require('prettier-bytes') +var speed = require('speedometer') +var progress = require('progress-string') +var dat = require('./') + +process.title = 'dat-next' + +var argv = minimist(process.argv.slice(2), { + alias: {sleep: 's', quiet: 'q'} +}) +var src = argv._[0] || process.cwd() +var dest = argv._[1] + +var output = [['', ''], ['', '', '', '']] +var log = logger(output) +var indexSpeed = speed() +var downloadSpeed = speed() +var hasContent +var imported = 0 +var downloaded = 0 +var total = 0 + +dat(src, dest, argv, function (archive, swarm, importProgress) { + output[0][0] = 'Starting...' + setInterval(function () { + networkUI() + log.print() + if (archive.downloaded) { + console.log('Done! Bye bye.') + process.exit(0) + } + }, 500) + log.print() + + archive.once('content', function () { + output[0].push('') // add space for peers + hasContent = true + + if (!importProgress) { + downloadUI() + } + }) + + if (importProgress) { + output[0][0] = `dat://${archive.key.toString('hex')}` + + var bar + + importProgress.on('count', function (count) { + total = count.bytes + bar = progress({ + total: total, + style: function (a, b) { + return `[${a}${b}] ${pretty(imported)}/${pretty(total)}` + } + }) + output[1][0] = bar(imported) + }) + + importProgress.on('chunk', function (chunk) { + imported += chunk.length + if (bar) output[1][0] = bar(imported) + output[1][1] = pretty(indexSpeed(chunk.length)) + '/s' + }) + + importProgress.on('put', function (src, dst) { + output[1][3] = `ADD: ${dst.name}` + }) + + importProgress.on('del', function (src, dst) { + output[1][3] = `DEL: ${dst.name}` + }) + + importProgress.on('end', function (src, dst) { + output[1] = [`Import complete: ${pretty(total)}`] + }) + } else { + output[0][0] = 'Downloading...' + } + + function downloadUI () { + var bar = downloadBar() + archive.content.ready(function () { + total = archive.content.length + output[0][0] = `Downloading ${pretty(archive.content.byteLength)}` + output[1][0] = bar(downloaded) + for (var i = 0; i < archive.content.length; i++) { + if (archive.content.has(i)) downloaded++ + } + }) + + archive.content.on('download', function (index, data) { + if (archive.content.length !== total) { + output[0][0] = `Downloading ${pretty(archive.content.byteLength)}` + total = archive.content.length + bar = downloadBar() + } + downloaded++ + var per = (downloaded / total * 100).toFixed(2) + if (bar) output[1][0] = bar(downloaded) + ' ' + per + '%' + output[1][1] = pretty(downloadSpeed(data.length)) + '/s' + }) + + function downloadBar () { + return progress({ + total: total, + style: function (a, b) { + return `[${a}${b}]` + } + }) + } + } + + function networkUI () { + if (!swarm.connected || !hasContent) return + output[0][1] = `${archive.content.peers.length} peers` + } +}) diff --git a/index.js b/index.js new file mode 100644 index 0000000..1555fde --- /dev/null +++ b/index.js @@ -0,0 +1,91 @@ +var path = require('path') +var xtend = require('xtend') +var mkdirp = require('mkdirp') +var hyperdrive = require('hyperdrive') +var ram = require('random-access-memory') +var mirror = require('mirror-folder') +var countDir = require('count-files') +var datIgnore = require('dat-ignore') +var debug = require('debug')('dat') +var cast = require('localcast')('dat-next') +var network = require('./lib/network') + +module.exports = run + +function run (src, dest, opts, cb) { + var key + var importProgress + if (dest) { + key = src + src = null + mkdirp.sync(dest) + } + var dir = dest || src + opts = xtend({ + storage: opts.sleep ? path.join(dir, '.dat') : ram + }, opts) + var archive = hyperdrive(opts.storage, key, opts) + + archive.on('ready', function () { + cast.emit('ready') + if (!key) importFiles() + var swarm = joinNetwork() + cb(archive, swarm, importProgress) + }) + + function importFiles () { + var ignore = datIgnore(dir) + importProgress = mirror(dir, {name: '/', fs: archive}, { + ignore: true, + equals: function (a, b, cb) { + // Ignores + if (ignore(a.name)) return cb(null, true) + if (!b.stat) return cb(null, false) + + if (!a.stat.isDirectory() && (a.stat.size !== b.stat.size)) return cb(null, false) + if (a.stat.mtime.getTime() > b.stat.mtime.getTime()) return cb(null, false) + cb(null, true) + } + }) + + countDir(dir, { ignore: ignore }, function (err, data) { + if (err) throw err + cast.emit('import:count', data) + importProgress.emit('count', data) + }) + + importProgress.on('put', function (src, dst) { + cast.emit('import:put', src, dst) + }) + importProgress.on('chunk', function (chunk) { + cast.emit('import:chunk', chunk) + }) + importProgress.on('del', function (dst) { + cast.emit('import:del', dst) + }) + importProgress.on('end', function () { + cast.emit('import:end') + }) + importProgress.on('error', function (err) { + debug('IMPORT ERROR:', err) + }) + } + + function joinNetwork () { + var swarm = network(archive, xtend({ + stream: function (peer) { + var stream = archive.replicate({ + live: !archive.metadata.writable && opts.sync + }) + stream.on('error', function (err) { + debug('Replication error:', err.message) + }) + stream.on('end', function () { + archive.downloaded = true + }) + return stream + } + }, opts)) + return swarm + } +} diff --git a/lib/commands/auth/login.js b/lib/commands/auth/login.js deleted file mode 100644 index c6725df..0000000 --- a/lib/commands/auth/login.js +++ /dev/null @@ -1,46 +0,0 @@ -var prompt = require('prompt') -var ui = require('../../ui') -var TownshipClient = require('../../township') - -module.exports = { - name: 'login', - command: login, - options: [] -} - -function login (opts) { - if (opts.email && opts.password) return makeRequest(opts) - - prompt.message = '' - prompt.colors = false - prompt.start() - prompt.get([{ - name: 'email', - description: 'Email', - required: true - }, - { - name: 'password', - description: 'Password', - required: true, - hidden: true, - replace: '*' - }], function (err, results) { - if (err) return console.log(err.message) - makeRequest(results) - }) - - function makeRequest (user) { - var client = TownshipClient(opts) - - client.login({ - email: user.email, - password: user.password - }, function (err, resp, body) { - if (err && err.message) ui.exitErr(err.message) - else if (err) ui.exitErr(err.toString()) - console.log('Logged in successfully.') - process.exit(0) - }) - } -} diff --git a/lib/commands/auth/logout.js b/lib/commands/auth/logout.js deleted file mode 100644 index b9c2185..0000000 --- a/lib/commands/auth/logout.js +++ /dev/null @@ -1,19 +0,0 @@ -var ui = require('../../ui') -var TownshipClient = require('../../township') - -module.exports = { - name: 'logout', - command: logout, - options: [] -} - -function logout (opts) { - var client = TownshipClient(opts) - - if (!client.getLogin().token) return ui.exitErr('Not logged in.') - client.logout(function (err) { - if (err) ui.exitErr(err) - console.log('Logged out.') - process.exit(0) - }) -} diff --git a/lib/commands/auth/register.js b/lib/commands/auth/register.js deleted file mode 100644 index e839678..0000000 --- a/lib/commands/auth/register.js +++ /dev/null @@ -1,52 +0,0 @@ -var prompt = require('prompt') -var ui = require('../../ui') -var TownshipClient = require('../../township') - -module.exports = { - name: 'register', - command: register, - options: [] -} - -function register (opts) { - if (opts.email && opts.username && opts.password) return makeRequest(opts) - - prompt.message = '' - prompt.colors = false - prompt.start() - prompt.get([{ - name: 'email', - description: 'Email', - required: true - }, - { - name: 'username', - description: 'Username', - required: true - }, - { - name: 'password', - description: 'Password', - required: true, - hidden: true, - replace: '*' - }], function (err, results) { - if (err) return console.log(err.message) - makeRequest(results) - }) - - function makeRequest (user) { - var client = TownshipClient(opts) - - client.register({ - email: user.email, - username: user.username, - password: user.password - }, function (err) { - if (err && err.message) ui.exitErr(err.message) - else if (err) ui.exitErr(err.toString()) - console.log('Registered successfully.') - process.exit(0) - }) - } -} diff --git a/lib/commands/auth/whoami.js b/lib/commands/auth/whoami.js deleted file mode 100644 index a402516..0000000 --- a/lib/commands/auth/whoami.js +++ /dev/null @@ -1,16 +0,0 @@ -var ui = require('../../ui') -var TownshipClient = require('../../township') - -module.exports = { - name: 'whoami', - command: whoami, - options: [] -} - -function whoami (opts) { - var client = TownshipClient(opts) - var login = client.getLogin() - if (!login.token) return ui.exitErr('Not logged in.') - console.log(login.email) - process.exit(0) -} diff --git a/lib/commands/clone.js b/lib/commands/clone.js deleted file mode 100644 index 3070755..0000000 --- a/lib/commands/clone.js +++ /dev/null @@ -1,90 +0,0 @@ -var fs = require('fs') -var nets = require('nets') -var stringKey = require('dat-encoding').toStr -var exit = require('../ui').exitErr -var download = require('../download') -var debug = require('debug')('dat') - -module.exports = { - name: 'clone', - command: clone, - help: [ - 'Clone a remote Dat archive', - '', - 'Usage: dat clone [download-folder]' - ].join('\n'), - options: [ - { - name: 'temp', - boolean: true, - default: false, - help: 'use an in-memory database for metadata' - }, - { - name: 'upload', - boolean: true, - default: false, - help: 'upload data to other peers while cloning' - } - ] -} - -function clone (opts) { - opts.key = opts._[0] - if (!opts.key) return exit('key required to clone') - - // Force these options for clone command - opts.exit = true - opts.newDir = null - - try { - // validates + removes dat:// - opts.key = stringKey(opts.key) - createDir(opts.key, run) - } catch (e) { - var re = /[a-zA-Z0-9-.]\/+[a-zA-Z0-9-]+\/[a-zA-Z0-9-]+$/ // match site.com/username/dataset - if (re.test(opts.key.trim())) return lookup() - if (e.message.indexOf('Invalid key') === -1) return exit(e) - return exit('Link is not a valid Dat link.') - } - - function lookup () { - var url = opts.key.indexOf('http') > -1 ? opts.key : 'http://' + opts.key - - debug('Registry lookup at:', url) - nets({ url: url, json: true }, function (err, resp, body) { - if (err) return exit(err) - if (resp.statusCode !== 200) return exit(body.message) - try { - opts.key = stringKey(body.url) - debug('Received key from registry:', opts.key) - if (opts.key) return createDir(url.split('/').pop(), run) // dirname is name of repo - } catch (e) { - console.error(new Error(e)) - } - exit('Error getting key from registry') - }) - } - - function run () { - download('clone', opts) - } - - function createDir (dir, cb) { - debug('Creating directory for clone', dir) - // Create the directory if it doesn't exist - // If no dir is specified, we put dat in a dir with name = key - opts.dir = opts._[1] || opts.dir - if (!opts.dir || opts.dir === process.cwd()) { // Don't allow download to cwd for now - opts.dir = dir // key or dataset name if using registry shortname - } - try { - fs.accessSync(opts.dir, fs.F_OK) - opts.newDir = false - } catch (e) { - opts.newDir = true - fs.mkdirSync(opts.dir) - } - cb() - } -} diff --git a/lib/commands/create.js b/lib/commands/create.js deleted file mode 100644 index 891bd50..0000000 --- a/lib/commands/create.js +++ /dev/null @@ -1,119 +0,0 @@ -var logger = require('status-logger') -var prettyBytes = require('pretty-bytes') -var Dat = require('dat-node') -var ui = require('../ui') -var datJson = require('../dat-json') -var debug = require('debug')('dat') - -module.exports = { - name: 'create', - command: create, - help: [ - 'Create a local Dat archive to share', - '', - 'Usage: dat create [directory]' - ].join('\n'), - options: [ - { - name: 'import', - boolean: true, - default: false, - help: 'Import files in the given directory' - }, - { - name: 'ignoreHidden', - boolean: true, - default: true, - abbr: 'ignore-hidden' - } - ] -} - -function create (opts) { - opts.errorIfExists = true // cannot resume for create - if (opts._.length && opts.dir === process.cwd()) opts.dir = opts._[0] // use first arg as dir if default set - - var importStatus = null - - // Logging Init - var output = [ - 'Creating Dat Archive...', // Shows Folder - '' // Shows Link - ] - if (opts.import) { - output = output.concat([ - '', // Spacer - '', // Importing Progress - '' // Total Size - ]) - } - var log = logger(output, {debug: opts.verbose, quiet: opts.quiet || opts.debug}) - - // UI Elements - var importUI = ui.importProgress() - var exit = ui.exit(log) - - // Printing Things!! - setInterval(function () { - if (importStatus) updateProgress() - log.print() - }, opts.logspeed) - - debug('Creating Dat archive in', opts.dir) - Dat(opts.dir, opts, function (err, dat) { - if (err && err.message.indexOf('Existing feeds') > -1 || (dat && dat.resumed)) return exit('Archive already exists. Use `dat sync` to resume sharing.') - else if (err) return exit(err) - - // General Archive Info - output[0] = `Dat ${opts.live !== false ? 'Archive' : 'Snapshot Archive'} created: ${dat.path}` - if (dat.key) output[1] = ui.link(dat.key) - else output[1] = 'Creating link...' - if (opts.quiet && dat.key) process.stdout.write(ui.link(dat.key)) - - if (dat.owner) { - datJson.read(dat, function (err, body) { - if (!err) return importFiles() // TODO: if dat.json exists, then what? - if (err.code === 'ENOENT' || !body) { - return datJson.write(dat, function (err) { - if (err) return exit(err) - importFiles() - }) - } - return exit(err) - }) - } - - function importFiles () { - // Not importing files. Just create .dat, print info, and exit. - if (!opts.import) return exit() - debug('Importing files into archive') - - output[3] = 'Importing files to archive...' - importStatus = dat.importFiles({ - // Can't pass through opts here. opts.live has two meanings (archive.live, live file watching) - live: false, // Never live (file watching) for `dat create` - resume: false, // Never resume for `dat create` - ignoreHidden: opts.ignoreHidden - }, function (err) { - if (err) return exit(err) - output[3] = opts.live !== false ? 'File import finished!' : 'Snapshot created!' - output[4] = `Total Size: ${importStatus.fileCount} ${importStatus.fileCount === 1 ? 'file' : 'files'} (${prettyBytes(importStatus.totalSize)})` - - debug('Dat archive created') - if (dat.key) debug(ui.link(dat.key)) - - if (opts.live !== false) return exit() - if (dat.key) output[1] = ui.link(dat.key) - - exit() - }) - importStatus.on('file imported', function (file) { - debug(file) - }) - } - }) - - function updateProgress () { - output[3] = importUI(importStatus) - } -} diff --git a/lib/commands/doctor.js b/lib/commands/doctor.js deleted file mode 100644 index ddbe81e..0000000 --- a/lib/commands/doctor.js +++ /dev/null @@ -1,17 +0,0 @@ -var doctor = require('dat-doctor') - -module.exports = { - name: 'doctor', - help: [ - 'Call the Doctor! Runs two tests:', - ' 1. Check if you can connect to a peer on a public server.', - ' 2. Gives you a link to test direct peer connections.', - '', - 'Usage: dat doctor []' - ].join('\n'), - options: [], - command: function (opts) { - opts.id = opts._[0] - doctor(opts) - } -} diff --git a/lib/commands/publish.js b/lib/commands/publish.js deleted file mode 100644 index 2159015..0000000 --- a/lib/commands/publish.js +++ /dev/null @@ -1,73 +0,0 @@ -var Dat = require('dat-node') -var encoding = require('dat-encoding') -var prompt = require('prompt') -var TownshipClient = require('../township') -var ui = require('../ui') -var datJson = require('../dat-json') - -module.exports = { - name: 'publish', - options: [], - command: publish -} - -function publish (opts) { - var client = TownshipClient(opts) - if (!client.getLogin().token) return ui.exitErr('Please login before publishing.') - - opts.createIfMissing = false // publish must always be a resumed archive - Dat(opts.dir, opts, function (err, dat) { - if (err) return ui.exitErr(err) - - datJson.read(dat, function (err, body) { - if (err && err.code !== 'ENOENT') return ui.exitErr(err) - dat.meta = body || {} - publish(dat) - }) - - function publish (dat) { - var datInfo = { - name: dat.meta.name || opts.name, - url: 'dat://' + encoding.toStr(dat.key), - title: dat.meta.title, - description: dat.meta.description, - keywords: dat.meta.keywords - } - - if (!datInfo.name) { - prompt.message = '' - prompt.colors = false - prompt.start() - prompt.get([{ - name: 'name', - pattern: /^[a-zA-Z\s-]+$/, - message: 'Name must be only letters, spaces, or dashes', - required: true - }], function (err, results) { - if (err) return console.log(err.message) - console.log(results.name) - datInfo.name = results.name - makeRequest(datInfo) - }) - } else { - makeRequest(datInfo) - } - } - - function makeRequest (datInfo) { - console.log(`Publishing archive with name "${datInfo.name}".`) - datJson.write(dat, datInfo, function (err) { - if (err) return ui.exitErr(err) - client.secureRequest({ - method: 'POST', url: '/dats', body: datInfo, json: true - }, function (err, resp, body) { - if (err && err.message) ui.exitErr(err.message) - else if (err) ui.exitErr(err.toString()) - if (body.statusCode === 400) return ui.exitErr(new Error(body.message)) - console.log('Successfully published!') - process.exit(0) - }) - }) - } - }) -} diff --git a/lib/commands/pull.js b/lib/commands/pull.js deleted file mode 100644 index 341c765..0000000 --- a/lib/commands/pull.js +++ /dev/null @@ -1,35 +0,0 @@ -var Dat = require('dat-node') -var download = require('../download') -var exitErr = require('../ui').exitErr -var debug = require('debug')('dat') - -module.exports = { - name: 'pull', - help: [ - 'Pull updates from a cloned Dat archive', - '', - 'Usage: dat pull' - ].join('\n'), - options: [ - { - name: 'upload', - boolean: true, - default: false, - help: 'upload data to other peers while pulling' - } - ], - command: function (opts) { - if (opts._.length && opts.dir === process.cwd()) opts.dir = opts._[0] // use first arg as dir if default set - - // Force these options for pull command - opts.createIfMissing = false - opts.exit = true - - debug('Pulling Dat archive in', opts.dir) - Dat(opts.dir, opts, function (err, dat) { - if (err) return exitErr(err) - if (dat.owner) return exitErr('Cannot pull an archive that you own.') - download('pull', opts, dat) - }) - } -} diff --git a/lib/commands/share.js b/lib/commands/share.js deleted file mode 100644 index f004127..0000000 --- a/lib/commands/share.js +++ /dev/null @@ -1,55 +0,0 @@ -var Dat = require('dat-node') -var share = require('../share') -var exit = require('../ui').exitErr -var debug = require('debug')('dat') - -module.exports = { - name: 'share', - help: [ - 'Create and share a Dat archive', - 'Create a dat, import files, and share to the network.', - '', - 'Usage: dat share' - ].join('\n'), - options: [ - { - name: 'import', - boolean: true, - default: true, - help: 'Import files from the directory to the database.' - }, - { - name: 'ignoreHidden', - boolean: true, - default: true, - abbr: 'ignore-hidden' - }, - { - name: 'watch', - boolean: true, - default: true, - help: 'Watch for changes and import updated files.' - } - ], - command: function (opts) { - if (opts._.length && opts.dir === process.cwd()) opts.dir = opts._[0] // use first arg as dir if default set - - // Gets overwritten by logger. - // Logging starts after Dat cb for lib/download sync - // So we need this to show something right away - if (!opts.quiet && !opts.debug) process.stdout.write('Starting Dat...') - - // Set default options - opts.exit = false - - debug('Reading archive in dir', opts.dir) - - Dat(opts.dir, opts, function (err, dat) { - if (err) return exit(err) - if (!dat.owner) exit('Existing archive that you do not own. Use `dat sync` to download updates.') - - // TODO: dat.json stuff we do in create.js? - share('sync', opts, dat) - }) - } -} diff --git a/lib/commands/snapshot.js b/lib/commands/snapshot.js deleted file mode 100644 index 0ccd730..0000000 --- a/lib/commands/snapshot.js +++ /dev/null @@ -1,23 +0,0 @@ -var create = require('./create') -var debug = require('debug')('dat') - -module.exports = { - name: 'snapshot', - help: [ - 'Create a snapshot Dat archive', - '', - 'Usage: dat snapshot' - ].join('\n'), - options: [], - command: function snapshot (opts) { - if (opts._.length && opts.dir === process.cwd()) opts.dir = opts._[0] // use first arg as dir if default set - - // Force these options for snapshot command - opts.errorIfExists = true - opts.live = false - opts.import = true - - debug('Creating snapshot archive in', opts.dir) - create.command(opts) - } -} diff --git a/lib/commands/sync.js b/lib/commands/sync.js deleted file mode 100644 index d17e836..0000000 --- a/lib/commands/sync.js +++ /dev/null @@ -1,69 +0,0 @@ -var readline = require('readline') -var Dat = require('dat-node') -var download = require('../download') -var share = require('../share') -var exitErr = require('../ui').exitErr -var debug = require('debug')('dat') - -module.exports = { - name: 'sync', - help: [ - 'Sync a Dat archive with the network', - 'Watch and import file changes (if you created the archive)', - '', - 'Usage: dat sync' - ].join('\n'), - options: [ - { - name: 'import', - boolean: true, - default: true, - help: 'Import files from the directory to the database.' - }, - { - name: 'ignoreHidden', - boolean: true, - default: true, - abbr: 'ignore-hidden' - }, - { - name: 'watch', - boolean: true, - default: true, - help: 'Watch for changes and import updated files.' - } - ], - command: function (opts) { - if (opts._.length && opts.dir === process.cwd()) opts.dir = opts._[0] // use first arg as dir if default set - - // Gets overwritten by logger. - // Logging starts after Dat cb for lib/download sync - // So we need this to show something right away - if (!opts.quiet && !opts.debug) process.stdout.write('Starting Dat...') - - // Set default options (some of these may be exposed to CLI eventually) - opts.createIfMissing = false // sync must always be a resumed archive - opts.exit = false - - debug('Reading archive in dir', opts.dir) - - Dat(opts.dir, opts, function (err, dat) { - if (err && (!err.message || err.message.indexOf('No existing') === -1)) return exit(err) - else if (err || !dat.resumed) return exit('No existing archive. Please use `dat create` or `dat clone` first.') - if (dat.owner) debug('Archive owner, syncing local updates to network') - else debug('Not archive owner, syncing remote updates') - - if (!dat.owner) return download('sync', opts, dat) - // TODO: dat.owner is false for snapshot on resume? bug? - // if (!dat.archive.live) opts.import = false - share('sync', opts, dat) - }) - - function exit (err) { - // Get rid of 'Starting Dat..' - readline.cursorTo(process.stdout, 0) - readline.clearLine(process.stdout) - exitErr(err) - } - } -} diff --git a/lib/dat-json.js b/lib/dat-json.js deleted file mode 100644 index 6961ca6..0000000 --- a/lib/dat-json.js +++ /dev/null @@ -1,40 +0,0 @@ -var fs = require('fs') -var stringKey = require('dat-encoding').toStr -var path = require('path') - -module.exports = { - read: function (dat, cb) { - // dat.json - // reads to dat.meta if exists - // (TODO: move to module & validate dat.json) - fs.readFile(datJsonFile(dat), 'utf8', function (err, body) { - if (err) return cb(err) - if (!body) return cb(null, {}) - var meta - try { - meta = JSON.parse(body) - } catch (e) { - return cb(new Error('Error reading the dat.json file.')) - } - cb(null, meta) - }) - }, - write: function (dat, defaults, cb) { - if (typeof defaults === 'function') { - cb = defaults - defaults = {} - } - dat.meta = { - title: defaults.title || path.basename(dat.path), - url: defaults.url, - name: defaults.name, - description: defaults.description || '' - } - if (dat.key) dat.meta.url = 'dat://' + stringKey(dat.key) - fs.writeFile(datJsonFile(dat), JSON.stringify(dat.meta), cb) - } -} - -function datJsonFile (dat) { - return path.join(dat.path, 'dat.json') -} diff --git a/lib/download.js b/lib/download.js deleted file mode 100644 index 8dfa76a..0000000 --- a/lib/download.js +++ /dev/null @@ -1,148 +0,0 @@ -var assert = require('assert') -var logger = require('status-logger') -var prettyBytes = require('pretty-bytes') -var rimraf = require('rimraf') -var memdb = require('memdb') -var Dat = require('dat-node') -var ui = require('./ui') -var debug = require('debug')('dat') - -module.exports = function (type, opts, dat) { - assert.ok(type, 'lib/download download type required') - assert.ok(['sync', 'clone', 'pull'].indexOf(type) > -1, 'lib/download download type must be sync, clone, pull') - - // TODO: clean up this logic - var resume = opts.resume || false - var hasKey = dat && dat.key || opts.key - if (!hasKey && !resume) return ui.exit()('lib/download Key required to download') - - var network = null - var stats = null - var archive = null - var onceConnected = false - - // Logging Init - var output = [ - [ - 'Starting Dat...', // Shows Folder Name - '', // Shows Link - '', // Shows Metadata Progress Bar - '', // Shows Content Progress Bar - '', // Shows Total Size Info - '' // spacer before network info - ], - [] // Shows network information - ] - var progressOutput = output[0] // shortcut for progress output - var log = logger(output, {debug: opts.verbose, quiet: opts.quiet || opts.debug}) // If debug=true we want output to be quiet - - // UI Elements - var bar = ui.bar() - var exit = ui.exit(log) - - // Printing Things!! - if (opts.debug) opts.logspeed = 1000 - setInterval(function () { - if (stats) updateDownload() - if (network) updateNetwork() - log.print() - }, opts.logspeed) - - // Action starts here - if (opts.temp) opts.db = memdb() - if (!dat) Dat(opts.dir, opts, start) - else start(null, dat) - - function start (err, theDat) { - if (err) return exit(err) - - dat = theDat - - debug('Download: ' + type + ' on ' + dat.key.toString('hex')) - - archive = dat.archive - - archive.open(function () { - if (!archive.content) return removeExit() // Not an archive - archive.content.once('download-finished', checkDone) - }) - // TODO: could be optimized for frequent metadata updates - archive.metadata.on('update', updateDownload) - - // General Archive Info - var niceType = (type === 'clone') ? 'Cloning' : type.charAt(0).toUpperCase() + type.slice(1) + 'ing' - progressOutput[0] = `${niceType} Dat Archive: ${dat.path}` - progressOutput[1] = ui.link(dat.key) + '\n' - if (opts.quiet && type !== 'clone') process.stdout.write(ui.link(dat.key)) - - // Stats - stats = dat.trackStats() - stats.on('update:blocksProgress', checkDone) - - // Network - if (!opts.upload && type !== 'sync') opts.upload = false // Do not upload on pull/clone - network = dat.joinNetwork(opts) - network.swarm.once('connection', function (peer) { - debug('Network: first peer connected') - onceConnected = true - progressOutput[2] = 'Starting Download...' - }) - progressOutput[2] = 'Looking for Dat Archive in Network' - - function removeExit () { - output[0] = [''] - output[1] = [''] - log.print() - if (opts.newDir) rimraf.sync(dat.path) - return exit('Link is not a Dat Archive. Please check you have the correct link.') - } - } - - function updateDownload () { - if (!archive.content) { - progressOutput[2] = '... Fetching Metadata' - return - } - // TODO: think about how this could work for empty archives & slow metadata downloads - if (checkDone()) return - var st = stats.get() - - var metaBlocksProgress = archive.metadata.blocks - archive.metadata.blocksRemaining() - var metaProgress = Math.round(metaBlocksProgress * 100 / archive.metadata.blocks) - var contentBlocksProgress = archive.content.blocks - archive.content.blocksRemaining() - var contentProgress = archive.content.blocks === 0 ? 0 : Math.round(contentBlocksProgress * 100 / archive.content.blocks) - // TODO: fix hyperdrive-stats bug where blocksProgress > blocksTotal - // var contentProgress = st.blocksTotal === 0 ? 0 : Math.round(st.blocksProgress * 100 / st.blocksTotal) - progressOutput[2] = 'Metadata: ' + bar(metaProgress) + ' ' + metaProgress + '%' - progressOutput[3] = 'Content: ' + bar(contentProgress) + ' ' + contentProgress + '%' - - if (!onceConnected && dat.live) progressOutput[4] = 'Waiting for connections to check for updates.' - else progressOutput[4] = `Total size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})` - - if (metaProgress < 100) debug('Metadata Download Progress:', metaProgress + '%') - if (contentProgress < 100) debug('Download Progress:', contentProgress + '%') - } - - function updateNetwork () { - output[1] = ui.network(network.connected, stats.network) - } - - function checkDone () { - if (!onceConnected || archive.metadata.blocksRemaining() !== 0) return false - if (!archive.content || archive.content.blocksRemaining() !== 0) return false - if (archive.metadata.blocks > 1 && !archive.content.blocks) return false // If metadata.blocks = 1, content feed is empty so finish - - var st = stats.get() - - progressOutput[2] = '' - progressOutput[3] = (type === 'sync') ? 'Files updated to latest!' : 'Download Finished!' - progressOutput[4] = `Total size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})` - - debug('Download finished') - if (!opts.exit) return true - - // Exit! - output[1] = '' // remove network info - return exit() - } -} diff --git a/lib/network.js b/lib/network.js new file mode 100644 index 0000000..5b9dffb --- /dev/null +++ b/lib/network.js @@ -0,0 +1,24 @@ +var assert = require('assert') +var swarmDefaults = require('dat-swarm-defaults') +var disc = require('discovery-swarm') +var xtend = require('xtend') + +module.exports = function (archive, opts, cb) { + assert.ok(archive, 'dat-node: lib/network archive required') + assert.ok(opts, 'dat-node: lib/network opts required') + + var DEFAULT_PORT = 3282 + var swarmOpts = xtend({ + id: archive.id, + hash: false, + stream: opts.stream + }, opts) + var swarm = disc(swarmDefaults(swarmOpts)) + swarm.once('error', function () { + swarm.listen(0) + }) + swarm.listen(opts.port || DEFAULT_PORT) + swarm.join(archive.discoveryKey, { announce: !(opts.upload === false) }, cb) + swarm.options = swarm._options + return swarm +} diff --git a/lib/share.js b/lib/share.js deleted file mode 100644 index 47334a5..0000000 --- a/lib/share.js +++ /dev/null @@ -1,137 +0,0 @@ -var assert = require('assert') -var basename = require('path').basename -var logger = require('status-logger') -var prettyBytes = require('pretty-bytes') -var Dat = require('dat-node') -var ui = require('./ui') -var debug = require('debug')('dat') - -module.exports = function sync (type, opts, dat) { - assert.ok(type, 'lib/share share type required') - assert.ok(['sync'].indexOf(type) > -1, 'lib/share type must be sync') - - debug('Share: ' + type + ' on ' + dat.key.toString('hex')) - - var importDone = false - var importStatus = null - var network = null - var stats = null - var fileOutput, liveProgressOutput - - // Logging Init. - // After download require because downloader has it's own logger - var output = [ - [ - 'Starting Dat...', // Shows Folder Name - '', // Shows Link - '', // Shows Importing Progress Bar - '', // Shows Total Size Info - '' // spacer before network info - ], - [''] // Shows network information - ] - var progressOutput = output[0] // shortcut for progress output - if (opts.watch) { - output.push(['', '', ''], []) // progress bar, live file list - liveProgressOutput = output[2] - fileOutput = output[3] - } - var log = logger(output, {debug: opts.verbose, quiet: opts.quiet || opts.debug}) // If debug=true we want output to be quiet. - - // UI Elements - var importUI = ui.importProgress() - var exit = ui.exit(log) - - // Printing Things!! - setInterval(function () { - if (importStatus && !importDone) updateImport() - if (network) updateNetwork() - log.print() - }, opts.logspeed) - - // Action starts here - if (!dat) Dat(opts.dir, opts, start) - else start(null, dat) - - function start (err, dat) { - if (err) return exit(err) - - // General Archive Info - progressOutput[0] = `Syncing Dat Archive: ${dat.path}` - progressOutput[1] = ui.link(dat.key) + '\n' - if (opts.quiet) process.stdout.write(ui.link(dat.key)) - - // Stats (used for network + download) - stats = dat.trackStats() - - // Network - network = dat.joinNetwork(opts) - network.swarm.once('connection', function () { - debug('Network: first peer connected') - }) - - if (dat.owner && opts.import) { - debug('Importing updated & new files into archive') - // File Imports - progressOutput[2] = 'Importing files...' - - importStatus = dat.importFiles({ - watch: opts.watch, // TODO: allow live: true. opts.live is used by archive.live though =( - resume: true, - ignoreHidden: opts.ignoreHidden - }, function (err) { - if (err) return exit(err) - debug('Import finished') - var st = stats.get() - importDone = true - progressOutput[2] = opts.watch ? 'Watching for file changes...' : 'Archive update finished! Sharing latest files.' - progressOutput[3] = `Total Size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})` - - if (opts.watch) { - var pending = 0 - importStatus.on('file watch event', function (file) { - pending++ - debug(`File watch ${file.mode}: ${basename(file.path)}`) - // liveProgressOutput.splice(2, 0, `Importing: ${ basename(file.path)}`) - // liveProgressOutput.splice(-2, 1) - }) - importStatus.on('file imported', function (file) { - if (fileOutput.length > 6) { - fileOutput.pop() - } - var fileAction = file.mode === 'created' ? 'Added' : 'Updated' - fileOutput.unshift(`${fileAction}: ${basename(file.path)}`) - pending-- - debug(`File ${file.mode}: ${basename(file.path)}`) - }) - importStatus.on('file skipped', function (file) { - pending-- - debug(`File skipped: ${basename(file.path)}`) - }) - setInterval(function () { - if (pending) liveProgressOutput[1] = `Importing ${pending} files...` - else liveProgressOutput[1] = '' - progressOutput[3] = `Total Size: ${importStatus.fileCount} ${importStatus.fileCount === 1 ? 'file' : 'files'} (${prettyBytes(importStatus.totalSize)})` - }, opts.logspeed) - } - }) - - if (debug.enabled && importStatus) { - importStatus.on('file imported', function (file) { - debug(`Imported ${file.path}`) - }) - setInterval(function () { - if (!importDone) debug(`Imported ${prettyBytes(importStatus.bytesImported)} of ${prettyBytes(importStatus.countStats.bytes)}`) - }, opts.logspeed) - } - } - } - - function updateImport () { - progressOutput[3] = importUI(importStatus) - } - - function updateNetwork () { - output[1] = ui.network(network.connected, stats.network) - } -} diff --git a/lib/township.js b/lib/township.js deleted file mode 100644 index 41addc7..0000000 --- a/lib/township.js +++ /dev/null @@ -1,17 +0,0 @@ -var xtend = require('xtend') -var TownshipClient = require('township-client') - -module.exports = function (opts) { - var townshipOpts = { - server: opts.server, - config: { - filepath: opts.config || '.datrc' - } - } - var defaults = { - // xtend doesn't overwrite when key is present but undefined - // If we want a default, make sure it's not going to passed as undefined - } - var options = xtend(defaults, townshipOpts) - return TownshipClient(options) -} diff --git a/lib/ui/bar.js b/lib/ui/bar.js deleted file mode 100644 index 9154fd1..0000000 --- a/lib/ui/bar.js +++ /dev/null @@ -1,11 +0,0 @@ -var progress = require('progress-string') - -module.exports = function () { - return progress({ - width: 50, - total: 100, - style: function (complete, incomplete) { - return '[' + complete + '>' + incomplete + ']' - } - }) -} diff --git a/lib/ui/exit.js b/lib/ui/exit.js deleted file mode 100644 index 064a545..0000000 --- a/lib/ui/exit.js +++ /dev/null @@ -1,13 +0,0 @@ -var debug = require('debug')('dat') - -module.exports = function (log) { - return function (err) { - if (err) { - if (debug.enabled) console.trace(err) - else console.error(err) - process.exit(1) - } - if (log) log.print() - process.exit(0) - } -} diff --git a/lib/ui/import-progress.js b/lib/ui/import-progress.js deleted file mode 100644 index d754dd6..0000000 --- a/lib/ui/import-progress.js +++ /dev/null @@ -1,15 +0,0 @@ -var Bar = require('./bar') -var prettyBytes = require('pretty-bytes') - -module.exports = function () { - var bar = Bar() - var start = Date.now() - return function (importer) { - if (!importer || !importer.bytesImported) return '' - var importedBytes = importer.bytesImported - var importedFiles = importer.fileCount - 1 // counted before import finishes so -1 TODO: fix? - var speed = importer.bytesImported * 1000 / (Date.now() - start) - var progress = Math.round(importedBytes * 100 / importer.countStats.bytes) - return bar(progress) + ` ${importedFiles} of ${importer.countStats.files} ${importer.countStats.files === 1 ? 'file' : 'files'}` + ' (' + prettyBytes(speed) + '/s' + ')' - } -} diff --git a/lib/ui/index.js b/lib/ui/index.js deleted file mode 100644 index 9895119..0000000 --- a/lib/ui/index.js +++ /dev/null @@ -1,9 +0,0 @@ -module.exports.bar = require('./bar') -module.exports.exit = require('./exit') -module.exports.exitErr = function (err) { - // shortcut - require('./exit')()(err) -} -module.exports.importProgress = require('./import-progress') -module.exports.link = require('./link') -module.exports.network = require('./network') diff --git a/lib/ui/link.js b/lib/ui/link.js deleted file mode 100644 index 1a9d5b5..0000000 --- a/lib/ui/link.js +++ /dev/null @@ -1,5 +0,0 @@ -var stringKey = require('dat-encoding').toStr - -module.exports = function (key) { - return `Link: dat://${stringKey(key)}` -} diff --git a/lib/ui/network.js b/lib/ui/network.js deleted file mode 100644 index ca8e5cc..0000000 --- a/lib/ui/network.js +++ /dev/null @@ -1,12 +0,0 @@ -var prettyBytes = require('pretty-bytes') - -module.exports = function (peers, stats) { - if (!peers) return ['Looking for connections in Dat Network...'] - var msg = [] - msg.push(`${peers} ${peers === 1 ? 'peer' : 'peers'} on the Dat Network`) - var spdMsg = '' - if (stats.downloadSpeed) spdMsg += `Downloading: ${prettyBytes(stats.downloadSpeed)}/s` + ' ' - if (stats.uploadSpeed) spdMsg += `Uploading: ${prettyBytes(stats.uploadSpeed)}/s` - msg.push(spdMsg) - return msg -} diff --git a/lib/usage.js b/lib/usage.js deleted file mode 100644 index 395e1e6..0000000 --- a/lib/usage.js +++ /dev/null @@ -1,31 +0,0 @@ -module.exports = function (opts, help, usage) { - if (opts.version) { - var pkg = require('../package.json') - console.error(pkg.version) - process.exit(1) - } - console.error('Usage: dat [options]') - console.error('') - console.error('Sharing Files:') - console.error(' dat create [] create a local archive') - console.error(' dat sync watch for changes & sync files with the network') - console.error(' dat share create and sync an archive in one command!') - console.error(' dat snapshot create a local snapshot archive') - console.error('') - console.error('Downloading Files:') - console.error(' dat clone [] clone a remote archive') - console.error(' dat pull update from remote archive & exit') - console.error(' dat sync sync files with the network') - console.error('') - console.error('Help & Troubleshooting:') - console.error(' dat doctor run the dat network doctor') - console.error(' dat [] --help, -h print help for a command') - console.error(' dat --version, -v print your dat version') - console.error('') - if (usage) { - console.error('General Options:') - console.error(usage) - } - console.error('Have fun using Dat! Learn more at docs.datproject.org') - process.exit(1) -} diff --git a/package.json b/package.json index d337c68..86340e9 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "2.3.7", "description": "Next version of the Dat CLI client", "bin": { - "dat-next": "bin/cli.js" + "dat-next": "cli.js" }, "scripts": { "auth-server": "DEBUG=* node scripts/auth-server.js", @@ -20,31 +20,24 @@ }, "homepage": "https://github.com/joehand/dat-next#readme", "dependencies": { - "dat-doctor": "^1.2.1", - "dat-encoding": "^4.0.1", - "dat-node": "^1.3.3", + "count-files": "^2.2.1", + "dat-ignore": "0.0.0", + "dat-swarm-defaults": "^1.0.0", "debug": "^2.4.5", - "memdb": "^1.3.1", + "discovery-swarm": "^4.3.0", + "hyperdrive": "github:mafintosh/hyperdrive#v8", + "localcast": "^2.0.1", + "minimist": "^1.2.0", + "mirror-folder": "github:joehand/mirror-folder#add/through", "mkdirp": "^0.5.1", - "nets": "^3.2.0", - "pretty-bytes": "^4.0.2", + "prettier-bytes": "^1.0.3", "progress-string": "^1.2.1", - "prompt": "^1.0.0", - "rimraf": "^2.5.4", + "random-access-memory": "^2.3.0", + "speedometer": "^1.0.0", "status-logger": "^3.0.0", - "subcommand": "^2.1.0", - "township-client": "^1.3.1", "xtend": "^4.0.1" }, "devDependencies": { - "dat-land": "datproject/datfolder", - "hypercore": "^4.15.1", - "hyperdiscovery": "^1.0.1", - "mkdirp": "^0.5.1", - "recursive-readdir-sync": "^1.0.6", - "standard": "^8.6.0", - "tap-spec": "^4.1.1", - "tape": "^4.6.3", - "tape-spawn": "^1.4.2" + "standard": "^8.6.0" } } diff --git a/readme.md b/readme.md index 3edf5c0..9d99cf7 100644 --- a/readme.md +++ b/readme.md @@ -2,36 +2,12 @@ The next version of the `dat` command line tool. -Current version at [datproject/dat](https://github.com/datproject/dat). - -## Currently Deprecated - -dat-next `v2` has been released as `dat` version `12.0.0`. - -### Install Dat: - -``` -npm install -g dat -``` +**WITH SLEEP!!**  😴 -[![Travis](https://img.shields.io/travis/joehand/dat-next.svg?style=flat-square&branch=master)](https://travis-ci.org/joehand/dat-next) [![npm](https://img.shields.io/npm/v/dat-next.svg?style=flat-square)](https://npmjs.org/package/dat-next) - -## For Developers - -This command line library uses [dat-node](https://github.com/datproject/dat-node) to create and manage the archives and networking. -If you'd like to build your own Dat application that is compatible with this command line tool, we suggest using dat-node. - -### Installing from source +Current version at [datproject/dat](https://github.com/datproject/dat). -Clone this repository and in a terminal inside of the folder you cloned run this command: +### Install Dat-Next: ``` -npm link +npm install -g dat-next ``` - -This should add a `dat` command line command to your PATH. Now you can run the dat command to try it out. - -The contribution guide also has more tips on our [development workflow](https://github.com/datproject/dat/blob/master/CONTRIBUTING.md#development-workflow). - -* `npm run test` to run tests -* `npm run auth-server` to run a local auth server for testing diff --git a/scripts/auth-server.js b/scripts/auth-server.js deleted file mode 100644 index 6f07e39..0000000 --- a/scripts/auth-server.js +++ /dev/null @@ -1,14 +0,0 @@ -var createServer = require('../tests/helpers/auth-server') - -createServer(process.env.PORT || 8888, function (err, server, closeServer) { - if (err) throw err - - process.on('exit', close) - process.on('SIGINT', close) - - function close (cb) { - closeServer(function () { - process.exit() - }) - } -}) diff --git a/tests/auth.js b/tests/auth.js deleted file mode 100644 index 5ae4214..0000000 --- a/tests/auth.js +++ /dev/null @@ -1,229 +0,0 @@ -var test = require('tape') -var path = require('path') -var fs = require('fs') -var rimraf = require('rimraf') -var mkdirp = require('mkdirp') -var spawn = require('./helpers/spawn') -var help = require('./helpers') -var authServer = require('./helpers/auth-server') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) -var baseTestDir = help.testFolder() -var fixtures = path.join(__dirname, 'fixtures') - -var port = process.env.PORT || 3000 -var SERVER = 'http://localhost:' + port + '/api/v1' -var config = path.join(__dirname, '.datrc-test') -var opts = ' --server=' + SERVER + ' --config=' + config - -dat += opts - -authServer(port, function (err, server, closeServer) { - if (err) throw err - test('auth - whoami works when not logged in', function (t) { - var cmd = dat + ' whoami ' - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stderr.match(function (output) { - t.same('Not logged in.', output.trim(), 'printed correct output') - return true - }) - st.stdout.empty() - st.end() - }) - - test('auth - register works', function (t) { - var cmd = dat + ' register --email=hello@bob.com --password=joe --username=joe' - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stdout.match(function (output) { - t.same(output.trim(), 'Registered successfully.', 'output success message') - return true - }) - st.stderr.empty() - st.end() - }) - - test('auth - login works', function (t) { - var cmd = dat + ' login --email=hello@bob.com --password=joe' - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stdout.match(function (output) { - t.same(output.trim(), 'Logged in successfully.', 'output success message') - return true - }) - st.stderr.empty() - st.end() - }) - - test('auth - whoami works', function (t) { - var cmd = dat + ' whoami' - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stdout.match(function (output) { - t.same('hello@bob.com', output.trim(), 'email printed') - return true - }) - st.stderr.empty() - st.end() - }) - - test('auth - publish before create fails', function (t) { - var cmd = dat + ' publish' - var st = spawn(t, cmd, {cwd: fixtures}) - st.stdout.empty() - st.stderr.match(function (output) { - t.ok(output.indexOf('existing') > -1, 'Create archive before pub') - return true - }) - st.end() - }) - - test('auth - create dat to publish', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - rimraf.sync(path.join(fixtures, 'dat.json')) - var cmd = dat + ' create --no-import' - var st = spawn(t, cmd, {cwd: fixtures}) - st.stdout.match(function (output) { - var link = help.matchLink(output) - if (!link) return false - t.ok(link, 'prints link') - return true - }) - st.stderr.empty() - st.end() - }) - - test('auth - publish our awesome dat', function (t) { - var cmd = dat + ' publish --name awesome' - var st = spawn(t, cmd, {cwd: fixtures}) - st.stdout.match(function (output) { - var published = output.indexOf('Successfully published') > -1 - if (!published) return false - t.ok(published, 'published') - return true - }) - st.stderr.empty() - st.end() - }) - - test('auth - publish our awesome dat with bad dat.json url', function (t) { - fs.readFile(path.join(fixtures, 'dat.json'), function (err, contents) { - t.ifError(err) - var info = JSON.parse(contents) - var oldUrl = info.url - info.url = info.url.replace('e', 'a') - fs.writeFile(path.join(fixtures, 'dat.json'), JSON.stringify(info), function (err) { - t.ifError(err, 'error after write') - var cmd = dat + ' publish --name awesome' - var st = spawn(t, cmd, {cwd: fixtures}) - st.stdout.match(function (output) { - var published = output.indexOf('Successfully published') > -1 - if (!published) return false - t.ok(published, 'published') - t.same(help.datJson(fixtures).url, oldUrl, 'has dat.json with url') - return true - }) - st.stderr.empty() - st.end() - }) - }) - }) - - test('auth - clone from registry', function (t) { - // MAKE SURE THESE MATCH WHAT is published above - // TODO: be less lazy and make a publish helper - var shortName = 'localhost:' + port + '/joe/awesome' // they'll never guess who wrote these tests - var baseDir = path.join(baseTestDir, 'dat_registry_dir') - mkdirp.sync(baseDir) - var downloadDir = path.join(baseDir, shortName.split('/').pop()) - var cmd = dat + ' clone ' + shortName - var st = spawn(t, cmd, {cwd: baseDir}) - st.stdout.match(function (output) { - var lookingFor = output.indexOf('Looking for') > -1 - if (!lookingFor) return false - t.ok(lookingFor, 'starts looking for peers') - t.ok(output.indexOf(downloadDir) > -1, 'prints dir') - st.kill() - return true - }) - st.stderr.empty() - st.end(function () { - rimraf.sync(downloadDir) - }) - }) - - test('auth - publish our awesome dat without a dat.json file', function (t) { - rimraf(path.join(fixtures, 'dat.json'), function (err) { - t.ifError(err) - var cmd = dat + ' publish --name another-awesome' - var st = spawn(t, cmd, {cwd: fixtures}) - st.stdout.match(function (output) { - var published = output.indexOf('Successfully published') > -1 - if (!published) return false - t.ok(published, 'published') - t.same(help.datJson(fixtures).name, 'another-awesome', 'has dat.json with name') - return true - }) - st.stderr.empty() - st.end(function () { - rimraf.sync(path.join(fixtures, '.dat')) - }) - }) - }) - - test('auth - bad clone from registry', function (t) { - var shortName = 'localhost:' + port + '/joe/not-at-all-awesome' - var baseDir = path.join(baseTestDir, 'dat_registry_dir_too') - mkdirp.sync(baseDir) - var downloadDir = path.join(baseDir, shortName.split('/').pop()) - var cmd = dat + ' clone ' + shortName - var st = spawn(t, cmd, {cwd: baseDir}) - st.stderr.match(function (output) { - t.same(output.trim(), 'Dat with that name not found.', 'not found') - st.kill() - return true - }) - st.stdout.empty() - st.end(function () { - rimraf.sync(downloadDir) - }) - }) - - test('auth - logout works', function (t) { - var cmd = dat + ' logout' - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stdout.match(function (output) { - t.same('Logged out.', output.trim(), 'output correct') - return true - }) - st.stderr.empty() - st.end() - }) - - test('auth - logout prints correctly when trying to log out twice', function (t) { - var cmd = dat + ' logout' - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stderr.match(function (output) { - t.same('Not logged in.', output.trim(), 'output correct') - return true - }) - st.stdout.empty() - st.end() - }) - - test('auth - whoami works after logging out', function (t) { - var cmd = dat + ' whoami' - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stderr.match(function (output) { - t.same('Not logged in.', output.trim()) - return true - }) - st.stdout.empty() - st.end() - }) - - test.onFinish(function () { - closeServer(function () { - fs.unlink(config, function () { - // done! - }) - }) - }) -}) diff --git a/tests/clone.js b/tests/clone.js deleted file mode 100644 index 927d8ca..0000000 --- a/tests/clone.js +++ /dev/null @@ -1,282 +0,0 @@ -var path = require('path') -var test = require('tape') -var rimraf = require('rimraf') -var mkdirp = require('mkdirp') -var spawn = require('./helpers/spawn.js') -var help = require('./helpers') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) -var baseTestDir = help.testFolder() -var shareDat - -test('clone - default opts', function (t) { - // cmd: dat clone - help.shareFixtures(function (_, fixturesDat) { - shareDat = fixturesDat - var key = shareDat.key.toString('hex') - var cmd = dat + ' clone ' + key - var st = spawn(t, cmd, {cwd: baseTestDir}) - var datDir = path.join(baseTestDir, key) - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Download Finished') > -1 - if (!downloadFinished) return false - - var stats = shareDat.stats.get() - var fileRe = new RegExp(stats.filesTotal + ' files') - var bytesRe = new RegExp(/1\.\d{1,2} kB/) - - t.ok(help.matchLink(output), 'prints link') - t.ok(output.indexOf('dat-download-folder/' + key) > -1, 'prints dir') - t.ok(output.match(fileRe), 'total size: files okay') - t.ok(output.match(bytesRe), 'total size: bytes okay') - t.ok(help.isDir(datDir), 'creates download directory') - - var fileList = help.fileList(datDir).join(' ') - var hasCsvFile = fileList.indexOf('all_hour.csv') > -1 - t.ok(hasCsvFile, 'csv file downloaded') - var hasDatFolder = fileList.indexOf('.dat') > -1 - t.ok(hasDatFolder, '.dat folder created') - var hasSubDir = fileList.indexOf('folder') > -1 - t.ok(hasSubDir, 'folder created') - var hasNestedDir = fileList.indexOf('nested') > -1 - t.ok(hasNestedDir, 'nested folder created') - var hasHelloFile = fileList.indexOf('hello.txt') > -1 - t.ok(hasHelloFile, 'hello.txt file downloaded') - - st.kill() - return true - }) - st.succeeds('exits after finishing download') - st.stderr.empty() - st.end() - }) -}) - -test('clone - errors on existing dir', function (t) { - // cmd: dat clone (same thing again as above) - var key = shareDat.key.toString('hex') - var cmd = dat + ' clone ' + key - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stderr.match(function (output) { - t.ok('cannot overwrite', 'Error saying you cannot overwrite') - st.kill() - return true - }) - st.end() -}) - -test('clone - specify dir', function (t) { - // cmd: dat clone my_dir - var key = shareDat.key.toString('hex') - var customDir = 'my_dir' - var cmd = dat + ' clone ' + key + ' ' + customDir - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Download Finished') > -1 - if (!downloadFinished) return false - - t.ok(output.indexOf('dat-download-folder/' + customDir) > -1, 'prints dir') - t.ok(help.isDir(path.join(baseTestDir, customDir)), 'creates download directory') - - st.kill() - return true - }) - st.succeeds('exits after finishing download') - st.stderr.empty() - st.end() -}) - -test('clone - quiet', function (t) { - var key = shareDat.key.toString('hex') - var customDir = 'shhhh_dir' - var cmd = dat + ' clone ' + key + ' ' + customDir + ' --quiet' - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.succeeds('exits after finishing download') - st.stdout.empty() - st.stderr.empty() - st.end() -}) - -test('clone - dat:// link', function (t) { - var key = 'dat://' + shareDat.key.toString('hex') + '/' - var baseDir = path.join(baseTestDir, 'dat_proto_dir') - mkdirp.sync(baseDir) - var downloadDir = path.join(baseDir, shareDat.key.toString('hex')) - var cmd = dat + ' clone ' + key - var st = spawn(t, cmd, {cwd: baseDir}) - st.succeeds('exits after finishing download') - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Download Finished') > -1 - if (!downloadFinished) return false - - t.ok(output.indexOf(downloadDir) > -1, 'prints dir') - t.ok(help.isDir(downloadDir, 'creates download directory')) - - st.kill() - return true - }) - st.stderr.empty() - st.end(function () { - rimraf.sync(downloadDir) - }) -}) - -test('clone - datproject.org link', function (t) { - var key = 'datproject.org/' + shareDat.key.toString('hex') + '/' - var baseDir = path.join(baseTestDir, 'dat_land_dir') - mkdirp.sync(baseDir) - var downloadDir = path.join(baseDir, shareDat.key.toString('hex')) - var cmd = dat + ' clone ' + key - var st = spawn(t, cmd, {cwd: baseDir}) - st.succeeds('exits after finishing download') - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Download Finished') > -1 - if (!downloadFinished) return false - - t.ok(output.indexOf(downloadDir) > -1, 'prints dir') - t.ok(help.isDir(downloadDir, 'creates download directory')) - - st.kill() - return true - }) - st.stderr.empty() - st.end(function () { - rimraf.sync(downloadDir) - }) -}) - -test('close sharer', function (t) { - shareDat.close(function () { - rimraf(path.join(shareDat.path, '.dat'), function () { - t.end() - }) - }) -}) - -test('clone - with --temp', function (t) { - // cmd: dat clone - help.shareFixtures(function (_, fixturesDat) { - shareDat = fixturesDat - var key = shareDat.key.toString('hex') - var cmd = dat + ' clone ' + key + ' --temp' - var st = spawn(t, cmd, {cwd: baseTestDir}) - var datDir = path.join(baseTestDir, key) - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Download Finished') > -1 - if (!downloadFinished) return false - - var stats = shareDat.stats.get() - var fileRe = new RegExp(stats.filesTotal + ' files') - var bytesRe = new RegExp(/1\.\d{1,2} kB/) - - t.ok(help.matchLink(output), 'prints link') - t.ok(output.indexOf('dat-download-folder/' + key) > -1, 'prints dir') - t.ok(output.match(fileRe), 'total size: files okay') - t.ok(output.match(bytesRe), 'total size: bytes okay') - t.ok(help.isDir(datDir), 'creates download directory') - - var fileList = help.fileList(datDir).join(' ') - var hasCsvFile = fileList.indexOf('all_hour.csv') > -1 - t.ok(hasCsvFile, 'csv file downloaded') - var hasDatFolder = fileList.indexOf('.dat') > -1 - t.ok(!hasDatFolder, '.dat folder not created') - var hasSubDir = fileList.indexOf('folder') > -1 - t.ok(hasSubDir, 'folder created') - var hasNestedDir = fileList.indexOf('nested') > -1 - t.ok(hasNestedDir, 'nested folder created') - var hasHelloFile = fileList.indexOf('hello.txt') > -1 - t.ok(hasHelloFile, 'hello.txt file downloaded') - - st.kill() - return true - }) - st.succeeds('exits after finishing download') - st.stderr.empty() - st.end() - }) -}) - -test('clone - hypercore link', function (t) { - help.shareFeed(function (_, key, close) { - var cmd = dat + ' clone ' + key - var st = spawn(t, cmd, {cwd: baseTestDir}) - var datDir = path.join(baseTestDir, key) - st.stderr.match(function (output) { - var error = output.indexOf('not a Dat Archive') > -1 - if (!error) return false - t.ok(error, 'has error') - t.ok(!help.isDir(datDir), 'download dir removed') - st.kill() - return true - }) - st.end(function () { - close() - }) - }) -}) - -test('clone - invalid link', function (t) { - var key = 'best-key-ever' - var cmd = dat + ' clone ' + key - var st = spawn(t, cmd, {cwd: baseTestDir}) - var datDir = path.join(baseTestDir, key) - st.stderr.match(function (output) { - var error = output.indexOf('not a valid Dat link') > -1 - if (!error) return false - t.ok(error, 'has error') - t.ok(!help.isDir(datDir), 'download dir removed') - st.kill() - return true - }) - st.end() -}) - -test('clone - stateless clone `dat {dir}`', function (t) { - var key = shareDat.key.toString('hex') - var customDir = 'stateless-dir' - var cmd = dat + ' ' + key + ' ' + customDir - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Files updated to latest') > -1 - if (!downloadFinished) return false - - t.ok(output.indexOf('dat-download-folder/' + customDir) > -1, 'prints dir') - t.ok(help.isDir(path.join(baseTestDir, customDir)), 'creates download directory') - - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -test('clone - resume stateless clone `dat {dir}`', function (t) { - // same thing as above, with existing dir - var key = shareDat.key.toString('hex') - var customDir = 'stateless-dir' - var cmd = dat + ' ' + key + ' ' + customDir - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Files updated to latest') > -1 - if (!downloadFinished) return false - - t.ok(output.indexOf('dat-download-folder/' + customDir) > -1, 'prints dir') - t.ok(help.isDir(path.join(baseTestDir, customDir)), 'creates download directory') - - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -test('close sharer', function (t) { - shareDat.close(function () { - rimraf.sync(path.join(shareDat.path, '.dat')) - t.end() - }) -}) - -test.onFinish(function () { - rimraf.sync(baseTestDir) -}) diff --git a/tests/create.js b/tests/create.js deleted file mode 100644 index 67643f3..0000000 --- a/tests/create.js +++ /dev/null @@ -1,159 +0,0 @@ -var fs = require('fs') -var path = require('path') -var test = require('tape') -var rimraf = require('rimraf') -var spawn = require('./helpers/spawn.js') -var help = require('./helpers') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) -var fixtures = path.join(__dirname, 'fixtures') - -// os x adds this if you view the fixtures in finder and breaks the file count assertions -try { fs.unlinkSync(path.join(fixtures, '.DS_Store')) } catch (e) { /* ignore error */ } - -// start without dat.json -try { fs.unlinkSync(path.join(fixtures, 'dat.json')) } catch (e) { /* ignore error */ } - -test('create - default opts no import', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - - var cmd = dat + ' create' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stdout.match(function (output) { - var datCreated = output.indexOf('created') > -1 - if (!datCreated) return false - - t.ok(help.isDir(path.join(fixtures, '.dat')), 'creates dat directory') - - st.kill() - return true - }) - st.succeeds('exits after create finishes') - st.stderr.empty() - st.end() -}) - -test('create - default opts with import', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - // cmd: dat create - var cmd = dat + ' create --import' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stdout.match(function (output) { - var importFinished = output.indexOf('import finished') > -1 - if (!importFinished) return false - - t.ok(help.isDir(path.join(fixtures, '.dat')), 'creates dat directory') - - var fileRe = new RegExp('3 files') - var bytesRe = new RegExp(/1\.\d{1,2} kB/) - - t.ok(help.matchLink(output), 'prints link') - t.ok(output.indexOf('tests/fixtures') > -1, 'prints dir') - t.ok(output.match(fileRe), 'total size: files okay') - t.ok(output.match(bytesRe), 'total size: bytes okay') - - t.same(help.datJson(fixtures).title, 'fixtures', 'dat.json: has title') - - st.kill() - return true - }) - st.succeeds('exits after create finishes') - st.stderr.empty() - st.end() -}) - -test('create - errors on existing archive', function (t) { - // cmd: dat create - var cmd = dat + ' create' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stderr.match(function (output) { - t.ok(output.indexOf('Archive already exists') > -1, 'already exists error') - st.kill() - return true - }) - st.end() -}) - -test('pull - pull fails on created archive', function (t) { - // cmd: dat create - var cmd = dat + ' pull' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stderr.match(function (output) { - t.ok(output.indexOf('Cannot pull an archive that you own.') > -1, 'cannot pull on create error') - st.kill() - return true - }) - st.end() -}) - -test('create - sync after create ok', function (t) { - // cmd: dat sync - var cmd = dat + ' sync ' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stdout.match(function (output) { - var connected = output.indexOf('Looking for connections') > -1 - if (!connected) return false - t.ok('Dat Network', 'Shares over dat network') - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -test('create - quiet only prints link', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - var cmd = dat + ' create --quiet' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stderr.empty() - st.stdout.match(function (output) { - var link = help.matchLink(output) - if (!link) return false - t.ok(link, 'prints link') - st.kill() - return true - }) - st.end() -}) - -test('create - init alias okay', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - var cmd = dat + ' init --import' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stderr.empty() - st.stdout.match(function (output) { - var importFinished = output.indexOf('import finished') > -1 - if (!importFinished) return false - t.pass('init did create') - st.kill() - return true - }) - st.end() -}) - -test('create - with path', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - var cmd = dat + ' create --import ' + fixtures - var st = spawn(t, cmd) - - st.stderr.empty() - st.stdout.match(function (output) { - var importFinished = output.indexOf('import finished') > -1 - if (!importFinished) return false - t.ok(output.indexOf('dat-next/tests/fixtures'), 'prints correct dir') - st.kill() - return true - }) - st.end() -}) - -test.onFinish(function () { - rimraf.sync(path.join(fixtures, '.dat')) -}) diff --git a/tests/doctor.js b/tests/doctor.js deleted file mode 100644 index 8a01b3f..0000000 --- a/tests/doctor.js +++ /dev/null @@ -1,37 +0,0 @@ -var path = require('path') -var test = require('tape') -var spawn = require('./helpers/spawn.js') -var help = require('./helpers') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) - -test('misc - doctor option works ', function (t) { - var st = spawn(t, dat + ' doctor', {end: false}) - st.stderr.match(function (output) { - var readyPeer = output.indexOf('Waiting for incoming connections') > -1 - if (!readyPeer) return false - - t.ok(output.indexOf('Phase one success!') > -1, 'doctor connects to public peer') - - var key = help.matchLink(output) - startPhysiciansAssistant(key) - return true - }, 'doctor started') - - function startPhysiciansAssistant (link) { - var assist = spawn(t, dat + ' doctor ' + link, {end: false}) - assist.stderr.match(function (output) { - var readyPeer = output.indexOf('Waiting for incoming connections') > -1 - if (!readyPeer) return false - - t.same(help.matchLink(output), link, 'key of peer matches') - t.ok(readyPeer, 'starts looking for peers') - t.skip(output.indexOf('Remote peer echoed expected data back') > -1, 'echo data back') - st.kill() - return true - }) - assist.end(function () { - t.end() - }) - } -}) diff --git a/tests/fixtures/all_hour.csv b/tests/fixtures/all_hour.csv deleted file mode 100644 index 29cfc71..0000000 --- a/tests/fixtures/all_hour.csv +++ /dev/null @@ -1,10 +0,0 @@ -time,latitude,longitude,depth,mag,magType,nst,gap,dmin,rms,net,id,updated,place,type -2014-04-30T03:34:57.000Z,60.0366,-141.2214,14.6,1.4,ml,,,,1.51,ak,ak11246293,2014-04-30T03:39:27.956Z,"67km E of Cape Yakataga, Alaska",earthquake -2014-04-30T03:16:54.860Z,33.9233322,-117.9376678,0.81,2.4,ml,71,51,0.02487,0.35,ci,ci37218696,2014-04-30T03:35:24.239Z,"1km SE of La Habra, California",earthquake -2014-04-30T03:03:09.000Z,61.126,-149.7035,28.2,1,ml,,,,0.75,ak,ak11246291,2014-04-30T03:09:51.716Z,"14km SE of Anchorage, Alaska",earthquake -2014-04-30T02:57:51.800Z,37.3798,-122.1912,4.5,1.3,Md,8,104.4,0.01796631,0.02,nc,nc72212216,2014-04-30T03:28:05.271Z,"2km SSE of Ladera, California",earthquake -2014-04-30T02:51:26.000Z,60.2062,-147.7298,0,1.8,ml,,,,0.1,ak,ak11246289,2014-04-30T02:54:55.916Z,"82km SE of Whittier, Alaska",earthquake -2014-04-30T02:48:54.000Z,60.4899,-149.4879,28.5,1.4,ml,,,,0.54,ak,ak11246287,2014-04-30T02:54:51.149Z,"36km N of Bear Creek, Alaska",earthquake -2014-04-30T02:47:44.100Z,38.819,-122.7952,3.3,0.8,Md,12,75.6,0.00898315,0.02,nc,nc72212211,2014-04-30T03:17:09.173Z,"5km NW of The Geysers, California",earthquake -2014-04-30T02:46:11.100Z,37.9812,-122.054,14.7,2.1,Md,35,126,0.09881468,0.09,nc,nc72212206,2014-04-30T03:38:06.391Z,"1km E of Pacheco, California",earthquake -2014-04-30T02:44:49.000Z,61.3482,-151.3611,48.5,1.7,ml,,,,0.92,ak,ak11246285,2014-04-30T02:54:49.809Z,"73km N of Nikiski, Alaska",earthquake diff --git a/tests/fixtures/folder/nested/hello.txt b/tests/fixtures/folder/nested/hello.txt deleted file mode 100644 index 5f5e627..0000000 --- a/tests/fixtures/folder/nested/hello.txt +++ /dev/null @@ -1 +0,0 @@ -code for science and society \ No newline at end of file diff --git a/tests/helpers/auth-server.js b/tests/helpers/auth-server.js deleted file mode 100644 index 8d852bf..0000000 --- a/tests/helpers/auth-server.js +++ /dev/null @@ -1,43 +0,0 @@ -var path = require('path') -var Server = require('dat-land/server') -var initDb = require('dat-land/server/database/init') -var rimraf = require('rimraf') - -module.exports = createServer - -function createServer (port, cb) { - var config = { - township: { - secret: 'very secret code', - db: path.join(__dirname, '..', 'test-township.db') - }, - db: { - dialect: 'sqlite3', - connection: { filename: path.join(__dirname, '..', 'test-sqlite.db') }, - useNullAsDefault: true - }, - cachedb: path.join(__dirname, 'test-cache.db'), - whitelist: false, - port: port || 8888 - } - rimraf.sync(config.db.connection.filename) - - initDb(config.db, function (err, db) { - if (err) return cb(err) - - const server = Server(config, db) - server.listen(config.port, function () { - console.log('listening', config.port) - }) - - cb(null, server, close) - - function close (cb) { - server.close(function () { - rimraf.sync(config.township.db) - rimraf.sync(config.db.connection.filename) - process.exit() - }) - } - }) -} diff --git a/tests/helpers/index.js b/tests/helpers/index.js deleted file mode 100644 index d27811f..0000000 --- a/tests/helpers/index.js +++ /dev/null @@ -1,98 +0,0 @@ -var fs = require('fs') -var os = require('os') -var path = require('path') -var mkdirp = require('mkdirp') -var rimraf = require('rimraf') -var encoding = require('dat-encoding') -var recursiveReadSync = require('recursive-readdir-sync') -var Dat = require('dat-node') -var hypercore = require('hypercore') -var memdb = require('memdb') -var swarm = require('hyperdiscovery') - -module.exports.matchLink = matchDatLink -module.exports.isDir = isDir -module.exports.testFolder = newTestFolder -module.exports.datJson = datJson -module.exports.shareFixtures = shareFixtures -module.exports.shareFeed = shareFeed -module.exports.fileList = fileList - -function shareFixtures (opts, cb) { - if (typeof opts === 'function') return shareFixtures(null, opts) - opts = opts || {} - var fixtures = path.join(__dirname, '..', 'fixtures') - // os x adds this if you view the fixtures in finder and breaks the file count assertions - try { fs.unlinkSync(path.join(fixtures, '.DS_Store')) } catch (e) { /* ignore error */ } - - rimraf.sync(path.join(fixtures, '.dat')) - Dat(fixtures, {}, function (err, dat) { - if (err) throw err - dat.trackStats() - dat.joinNetwork() - if (opts.import === false) return cb(null, dat) - dat.importFiles(function (err) { - if (err) throw err - cb(null, dat) - }) - }) -} - -function fileList (dir) { - try { - return recursiveReadSync(dir) - } catch (e) { - return [] - } -} - -function newTestFolder () { - var tmpdir = path.join(os.tmpdir(), 'dat-download-folder') - rimraf.sync(tmpdir) - mkdirp.sync(tmpdir) - return tmpdir -} - -function matchDatLink (str) { - var match = str.match(/[A-Za-z0-9]{64}/) - if (!match) return false - var key - try { - key = encoding.toStr(match[0].trim()) - } catch (e) { - return false - } - return key -} - -function datJson (filepath) { - try { - return JSON.parse(fs.readFileSync(path.join(filepath, 'dat.json'))) - } catch (e) { - return {} - } -} - -function isDir (dir) { - try { - return fs.statSync(dir).isDirectory() - } catch (e) { - return false - } -} - -function shareFeed (cb) { - var core = hypercore(memdb()) - var feed = core.createFeed() - feed.append('hello world', function (err) { - if (err) throw err - cb(null, encoding.toStr(feed.key), close) - }) - var sw = swarm(feed) - - function close (cb) { - feed.close(function () { - sw.close(cb) - }) - } -} diff --git a/tests/helpers/spawn.js b/tests/helpers/spawn.js deleted file mode 100644 index cd9cef7..0000000 --- a/tests/helpers/spawn.js +++ /dev/null @@ -1,12 +0,0 @@ -var spawn = require('tape-spawn') -var fs = require('fs') - -// happens once at require time -// https://github.com/AndreasMadsen/execspawn/issues/2 -var hasBash = fs.existsSync('/bin/bash') - -module.exports = function (t, cmd, opts) { - opts = opts || {} - if (hasBash) opts.shell = '/bin/bash' // override default of /bin/sh - return spawn(t, cmd, opts) -} diff --git a/tests/pull.js b/tests/pull.js deleted file mode 100644 index f6ff66e..0000000 --- a/tests/pull.js +++ /dev/null @@ -1,117 +0,0 @@ -var path = require('path') -var test = require('tape') -var rimraf = require('rimraf') -var spawn = require('./helpers/spawn.js') -var help = require('./helpers') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) -var baseTestDir = help.testFolder() -var shareDat -var shareKey - -test('pull - errors without clone first', function (t) { - // cmd: dat pull - var cmd = dat + ' pull' - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stderr.match(function (output) { - t.ok('No existing archive', 'Error: no existing archive') - st.kill() - return true - }) - st.stdout.empty() - st.end() -}) - -test('pull - clone so we can pull', function (t) { - // cmd: dat clone - // import false so we can pull files later - help.shareFixtures({import: false}, function (_, fixturesDat) { - shareDat = fixturesDat - shareKey = shareDat.key.toString('hex') - var cmd = dat + ' clone ' + shareKey - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Download Finished') > -1 - if (!downloadFinished) return false - st.kill() - return true - }) - st.stderr.empty() - st.end() - }) -}) - -test('pull - default opts', function (t) { - // cmd: dat pull - // import the files to the sharer so we can pull new data - shareDat.importFiles(function (err) { - if (err) throw err - - var datDir = path.join(baseTestDir, shareKey) - var cmd = dat + ' pull' - var st = spawn(t, cmd, {cwd: datDir}) - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Download Finished') > -1 - if (!downloadFinished) return false - - var stats = shareDat.stats.get() - var fileRe = new RegExp(stats.filesTotal + ' files') - var bytesRe = new RegExp(/1\.\d{1,2} kB/) - - t.ok(help.matchLink(output), 'prints link') - t.ok(output.indexOf('dat-download-folder/' + shareKey) > -1, 'prints dir') - t.ok(output.match(fileRe), 'total size: files okay') - t.ok(output.match(bytesRe), 'total size: bytes okay') - t.ok(help.isDir(datDir), 'creates download directory') - - var fileList = help.fileList(datDir).join(' ') - var hasCsvFile = fileList.indexOf('all_hour.csv') > -1 - t.ok(hasCsvFile, 'csv file downloaded') - var hasDatFolder = fileList.indexOf('.dat') > -1 - t.ok(hasDatFolder, '.dat folder created') - var hasSubDir = fileList.indexOf('folder') > -1 - t.ok(hasSubDir, 'folder created') - var hasNestedDir = fileList.indexOf('nested') > -1 - t.ok(hasNestedDir, 'nested folder created') - var hasHelloFile = fileList.indexOf('hello.txt') > -1 - t.ok(hasHelloFile, 'hello.txt file downloaded') - - st.kill() - return true - }) - st.succeeds('exits after finishing download') - st.stderr.empty() - st.end() - }) -}) - -test('pull - with dir arg', function (t) { - var dirName = shareKey - var datDir = path.join(baseTestDir, shareKey) - var cmd = dat + ' pull ' + dirName - var st = spawn(t, cmd, {cwd: baseTestDir}) - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Download Finished') > -1 - if (!downloadFinished) return false - - t.ok(output.indexOf('dat-download-folder/' + dirName) > -1, 'prints dir') - t.ok(help.isDir(datDir), 'creates download directory') - - st.kill() - return true - }) - st.succeeds('exits after finishing download') - st.stderr.empty() - st.end() -}) - -test('close sharer', function (t) { - shareDat.close(function () { - rimraf.sync(path.join(shareDat.path, '.dat')) - t.end() - }) -}) - -test.onFinish(function () { - rimraf.sync(baseTestDir) -}) diff --git a/tests/share.js b/tests/share.js deleted file mode 100644 index 727c7e1..0000000 --- a/tests/share.js +++ /dev/null @@ -1,58 +0,0 @@ -var fs = require('fs') -var path = require('path') -var test = require('tape') -var rimraf = require('rimraf') -var spawn = require('./helpers/spawn.js') -var help = require('./helpers') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) -if (process.env.TRAVIS) dat += ' --no-watch ' -var fixtures = path.join(__dirname, 'fixtures') - -// os x adds this if you view the fixtures in finder and breaks the file count assertions -try { fs.unlinkSync(path.join(fixtures, '.DS_Store')) } catch (e) { /* ignore error */ } - -// start without dat.json -try { fs.unlinkSync(path.join(fixtures, 'dat.json')) } catch (e) { /* ignore error */ } - -test('share - default opts', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - var cmd = dat + ' share' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stdout.match(function (output) { - var importFinished = output.indexOf('Total Size') > -1 - if (!importFinished) return false - - t.ok(help.isDir(path.join(fixtures, '.dat')), 'creates dat directory') - t.ok(output.indexOf('Looking for connections') > -1, 'network') - - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -test('share - with dir arg', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - var cmd = dat + ' share ' + fixtures - var st = spawn(t, cmd) - - st.stdout.match(function (output) { - var importFinished = output.indexOf('Total Size') > -1 - if (!importFinished) return false - - t.ok(help.isDir(path.join(fixtures, '.dat')), 'creates dat directory') - t.ok(output.indexOf('Looking for connections') > -1, 'network') - - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -test.onFinish(function () { - rimraf.sync(path.join(fixtures, '.dat')) -}) diff --git a/tests/snapshot.js b/tests/snapshot.js deleted file mode 100644 index c51f2c5..0000000 --- a/tests/snapshot.js +++ /dev/null @@ -1,74 +0,0 @@ -var fs = require('fs') -var path = require('path') -var test = require('tape') -var rimraf = require('rimraf') -var spawn = require('./helpers/spawn.js') -var help = require('./helpers') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) -var fixtures = path.join(__dirname, 'fixtures') - -// os x adds this if you view the fixtures in finder and breaks the file count assertions -try { fs.unlinkSync(path.join(fixtures, '.DS_Store')) } catch (e) { /* ignore error */ } - -test('snapshot - default opts', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - // cmd: dat snapshot - var cmd = dat + ' snapshot' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stdout.match(function (output) { - var importFinished = output.indexOf('Snapshot created') > -1 - if (!importFinished) return false - - t.ok(help.isDir(path.join(fixtures, '.dat')), 'creates dat directory') - - var fileRe = new RegExp('3 files') - var bytesRe = new RegExp(/1\.\d{1,2} kB/) - - t.ok(help.matchLink(output), 'prints link') - t.ok(output.indexOf('tests/fixtures') > -1, 'prints dir') - t.ok(output.match(fileRe), 'total size: files okay') - t.ok(output.match(bytesRe), 'total size: bytes okay') - st.kill() - return true - }) - st.succeeds('exits after create finishes') - st.stderr.empty() - st.end() -}) - -test('snapshot - errors on existing archive', function (t) { - // cmd: dat snapshot - var cmd = dat + ' snapshot' - var st = spawn(t, cmd, {cwd: fixtures}) - - // st.stdout.empty() Sometimes init statement is getting printed, causing failure - st.stderr.match(function (output) { - t.ok(output.indexOf('Archive already exists') > -1, 'cannot overwrite error') - st.kill() - return true - }) - st.end() -}) - -// TODO: archive.owner is false for snapshot on resume. Hyperdrive bug? -// test('snapshot - sync after create ok', function (t) { -// // cmd: dat sync -// var cmd = dat + ' sync ' -// var st = spawn(t, cmd, {cwd: fixtures}) - -// st.stdout.match(function (output) { -// var connected = output.indexOf('Sharing latest') > -1 -// if (!connected) return false -// t.ok('Dat Network', 'Shares over dat network') -// st.kill() -// return true -// }) -// st.stderr.empty() -// st.end() -// }) - -test.onFinish(function () { - rimraf.sync(path.join(fixtures, '.dat')) -}) diff --git a/tests/sync-owner.js b/tests/sync-owner.js deleted file mode 100644 index cd6cea2..0000000 --- a/tests/sync-owner.js +++ /dev/null @@ -1,282 +0,0 @@ -var fs = require('fs') -var net = require('net') -var path = require('path') -var test = require('tape') -var mkdirp = require('mkdirp') -var rimraf = require('rimraf') -var Dat = require('dat-node') -var spawn = require('./helpers/spawn.js') -var help = require('./helpers') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) -if (process.env.TRAVIS) dat += ' --no-watch ' -var fixtures = path.join(__dirname, 'fixtures') -var downDat - -// os x adds this if you view the fixtures in finder and breaks the file count assertions -try { fs.unlinkSync(path.join(fixtures, '.DS_Store')) } catch (e) { /* ignore error */ } - -test('sync-owner - errors without create first', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - // cmd: dat sync - var cmd = dat + ' sync' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stderr.match(function (output) { - var hasError = output.indexOf('No existing archive') > -1 - t.ok(hasError, 'emits error') - st.kill() - return true - }) - st.end() -}) - -test('sync-owner - create a dat for syncing', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - // cmd: dat create - var cmd = dat + ' create --import' - var st = spawn(t, cmd, {cwd: fixtures}) - st.stdout.match(function (output) { - var importFinished = output.indexOf('import finished') > -1 - if (!importFinished) return false - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -test('sync-owner - default opts', function (t) { - // cmd: dat sync - var cmd = dat + ' sync' - var st = spawn(t, cmd, {cwd: fixtures, end: false}) - - var key - - st.stdout.match(function (output) { - var sharing = output.indexOf('Dat Network') > -1 - if (!sharing) return false - - key = help.matchLink(output) - - t.ok(key, 'prints link') - t.ok(output.indexOf('tests/fixtures') > -1, 'prints dir') - - downloadDat() - return true - }) - st.stderr.empty() - st.end() - - function downloadDat () { - var downloadDir = path.join(help.testFolder(), '' + Date.now()) - mkdirp.sync(downloadDir) - - Dat(downloadDir, { key: key }, function (err, tmpDat) { - if (err) throw err - - downDat = tmpDat - downDat.joinNetwork() - - downDat.network.swarm.once('connection', function () { - t.pass('downloader connects') - downDat.close(function () { - rimraf.sync(downDat.path) - t.end() - }) - }) - }) - } -}) - -test('sync-owner - create without import for syncing', function (t) { - rimraf.sync(path.join(fixtures, '.dat')) - // cmd: dat create - var cmd = dat + ' create' - var st = spawn(t, cmd, {cwd: fixtures}) - st.stdout.match(function (output) { - if (output.indexOf('created') > -1) return true - return false - }) - st.succeeds() - st.end() -}) - -test('sync-owner - imports after no-import create', function (t) { - // cmd: dat sync - var cmd = dat + ' sync' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stdout.match(function (output) { - // have to check both for local test (watching) and travis (sharing) - var sharing = output.indexOf('Watching') > -1 || output.indexOf('Sharing latest') > -1 - if (!sharing) return false - - var fileRe = new RegExp('2 files') - var bytesRe = new RegExp(/1\.\d{1,2} kB/) - - t.ok(help.matchLink(output), 'prints link') - t.ok(output.indexOf('tests/fixtures') > -1, 'prints dir') - t.ok(output.match(fileRe), 'total size: files okay') - t.ok(output.match(bytesRe), 'total size: bytes okay') - - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -// TODO: this test is causing serious memory issues. -// HELP. Maybe related to https://github.com/datproject/dat-node/issues/71 -// test('sync-owner - turn off ignore hidden', function (t) { -// // cmd: dat sync -// var hiddenFile = path.join(fixtures, '.hidden-file') -// var cmd = dat + ' sync --no-ignore-hidden' -// fs.writeFile(hiddenFile, 'You cannot see me', function (err) { -// t.error(err) - -// var st = spawn(t, cmd, {cwd: fixtures, end: false}) -// var key - -// st.stdout.match(function (output) { -// var sharing = output.indexOf('Dat Network') > -1 -// if (!sharing) return false - -// key = help.matchLink(output) - -// downloadDat() -// return true -// }) -// st.stderr.empty() -// st.end() - -// function downloadDat () { -// var downloadDir = path.join(help.testFolder(), '' + Date.now()) -// mkdirp.sync(downloadDir) - -// Dat(downloadDir, { key: key }, function (err, downDat) { -// if (err) throw err - -// downDat.joinNetwork() - -// downDat.network.swarm.once('connection', function () { -// downDat.archive.list({live: false}, function (err, data) { -// t.error(err) -// var hasHiddenFile = data.filter(function (entry) { -// return entry.name === '.hidden-file' -// }) -// t.ok(hasHiddenFile.length, 'hidden file in archive') -// downDat.network.swarm.close(function () { -// process.nextTick(function () { -// downDat.close(function () { -// rimraf(downDat.path, function () { -// fs.unlink(hiddenFile, function () { -// t.end() -// }) -// }) -// }) -// }) -// }) -// }) -// }) -// }) -// } -// }) -// }) - -test('sync-owner - port and utp options', function (t) { - var port = 3281 - var cmd = dat + ' sync --port ' + port + ' --no-utp' - var st = spawn(t, cmd, {cwd: fixtures, end: false}) - st.stderr.empty() - - var server = net.createServer() - server.once('error', function (err) { - if (err.code !== 'EADDRINUSE') return t.error(err) - t.skip('TODO: correct port in use') - done() - }) - server.once('listening', function () { - t.skip(`TODO: port ${server.address().port} should be in use`) - done() - }) - server.listen(port) - - t.skip('TODO: check utp option') // TODO: how to check utp? - - function done () { - server.close(function () { - st.kill() - t.end() - }) - } -}) - -test('sync-owner - shorthand', function (t) { - var cmd = dat + ' .' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stdout.match(function (output) { - var sharing = output.indexOf('Looking for connections') > -1 - if (!sharing) return false - - t.ok(help.matchLink(output), 'prints link') - - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -test('sync-owner - dir argument', function (t) { - var cmd = dat + ' sync ' + fixtures - var st = spawn(t, cmd) - - st.stdout.match(function (output) { - var sharing = output.indexOf('Looking for connections') > -1 - if (!sharing) return false - - t.ok(help.matchLink(output), 'prints link') - - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -if (!process.env.TRAVIS) { - test('sync-owner - live', function (t) { - var liveFile = path.join(fixtures, 'live.txt') - var wroteFile = false - - var cmd = dat + ' sync --watch' - var st = spawn(t, cmd, {cwd: fixtures}) - - st.stdout.match(function (output) { - var watching = output.indexOf('Watching for file changes') > -1 - if (!watching) return false - else if (!wroteFile) { - fs.writeFileSync(liveFile, 'hello') - wroteFile = true - } - var fileImported = output.indexOf('live.txt') > -1 - if (!fileImported) return false - - t.ok(fileImported, 'prints live file output') - t.ok(output.indexOf('3 files') > -1, 'total size: files okay') - - fs.unlinkSync(liveFile) - st.kill() - return true - }) - st.stderr.empty() - st.end() - }) -} - -test.onFinish(function () { - rimraf.sync(path.join(fixtures, '.dat')) -}) diff --git a/tests/sync-remote.js b/tests/sync-remote.js deleted file mode 100644 index a7f89cc..0000000 --- a/tests/sync-remote.js +++ /dev/null @@ -1,101 +0,0 @@ -var path = require('path') -var test = require('tape') -var rimraf = require('rimraf') -var spawn = require('./helpers/spawn.js') -var help = require('./helpers') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) -var baseTestDir = help.testFolder() -var shareDat -var syncDir - -test('sync-remote - default opts', function (t) { - // cmd: dat sync - var key - - help.shareFixtures({import: false}, function (_, fixturesDat) { - shareDat = fixturesDat - key = shareDat.key.toString('hex') - syncDir = path.join(baseTestDir, key) - - makeClone(function () { - shareDat.importFiles(function () { - var cmd = dat + ' sync' - var st = spawn(t, cmd, {cwd: syncDir}) - st.stdout.match(function (output) { - var updated = output.indexOf('Files updated') > -1 - if (!updated) return false - - var fileRe = new RegExp('3 files') - var bytesRe = new RegExp(/1\.\d{1,2} kB/) - - key = help.matchLink(output) - - t.ok(key, 'prints link') - t.ok(output.indexOf('dat-download-folder/' + key) > -1, 'prints dir') - t.ok(output.match(fileRe), 'total size: files okay') - t.ok(output.match(bytesRe), 'total size: bytes okay') - - st.kill() - return true - }) - st.stderr.empty() - st.end() - }) - }) - }) - - function makeClone (cb) { - var cmd = dat + ' clone ' + key - var st = spawn(t, cmd, {cwd: baseTestDir, end: false}) - st.stdout.match(function (output) { - var downloadFinished = output.indexOf('Download Finished') > -1 - if (!downloadFinished) return false - - st.kill() - cb() - return true - }) - st.stderr.empty() - } -}) - -test('sync-remote - shorthand sync', function (t) { - // cmd: dat sync - var cmd = dat + ' .' - var st = spawn(t, cmd, {cwd: syncDir}) - st.stdout.match(function (output) { - var syncing = output.indexOf('Syncing Dat Archive') > -1 - if (!syncing) return false - t.ok(help.matchLink(output), 'prints link') - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -test('sync-remote - dir arg', function (t) { - var cmd = dat + ' ' + syncDir - var st = spawn(t, cmd) - st.stdout.match(function (output) { - var syncing = output.indexOf('Syncing Dat Archive') > -1 - if (!syncing) return false - t.ok(help.matchLink(output), 'prints link') - st.kill() - return true - }) - st.stderr.empty() - st.end() -}) - -test('close sharer', function (t) { - shareDat.close(function () { - rimraf.sync(path.join(shareDat.path, '.dat')) - t.end() - }) -}) - -test.onFinish(function () { - rimraf.sync(baseTestDir) -}) diff --git a/tests/usage.js b/tests/usage.js deleted file mode 100644 index aecdcd3..0000000 --- a/tests/usage.js +++ /dev/null @@ -1,46 +0,0 @@ -var path = require('path') -var test = require('tape') -var spawn = require('./helpers/spawn.js') - -var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js')) -var version = require('../package.json').version - -test('usage - prints usage', function (t) { - var d = spawn(t, dat) - d.stderr.match(function (output) { - var usage = output.indexOf('Usage') > -1 - if (!usage) return false - return true - }) - d.end() -}) - -test('usage - prints version', function (t) { - var d = spawn(t, dat + ' -v') - d.stderr.match(function (output) { - var ver = output.indexOf(version) > -1 - if (!ver) return false - return true - }) - d.end() -}) - -test('usage - also prints version', function (t) { - var d = spawn(t, dat + ' -v') - d.stderr.match(function (output) { - var ver = output.indexOf(version) > -1 - if (!ver) return false - return true - }) - d.end() -}) - -test('usage - help prints usage', function (t) { - var d = spawn(t, dat + ' help') - d.stderr.match(function (output) { - var usage = output.indexOf('Usage') > -1 - if (!usage) return false - return true - }) - d.end() -}) From fc26123b9950205d3988496433529204abbc2385 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Sun, 26 Mar 2017 22:44:17 -0700 Subject: [PATCH 02/20] usage + updates --- .gitignore | 3 --- .travis.yml | 5 ----- LICENSE | 2 +- package.json | 5 ++--- readme.md | 18 +++++++++++++++++- 5 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 9ca7b00..3c3629e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1 @@ node_modules -tests/fixtures/.dat -tests/fixtures/dat.json -tests/**.db \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index c5ec891..9afac73 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,8 +4,3 @@ node_js: - "4" - "6" - "7" - -sudo: false - -script: - - npm test \ No newline at end of file diff --git a/LICENSE b/LICENSE index 4ac95c0..f310f60 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2016 Joe Hand +Copyright (c) 2017 Joe Hand Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/package.json b/package.json index 86340e9..25d1bcb 100644 --- a/package.json +++ b/package.json @@ -6,14 +6,13 @@ "dat-next": "cli.js" }, "scripts": { - "auth-server": "DEBUG=* node scripts/auth-server.js", - "test": "standard && tape tests/*.js | tap-spec" + "test": "standard" }, "repository": { "type": "git", "url": "git+https://github.com/joehand/dat-next.git" }, - "author": "Joe Hand (https://joeahand.com/)", + "author": "Joe Hand ", "license": "MIT", "bugs": { "url": "https://github.com/joehand/dat-next/issues" diff --git a/readme.md b/readme.md index 9d99cf7..08da610 100644 --- a/readme.md +++ b/readme.md @@ -6,8 +6,24 @@ The next version of the `dat` command line tool. Current version at [datproject/dat](https://github.com/datproject/dat). -### Install Dat-Next: +## Install: ``` npm install -g dat-next ``` + +## Usage + +Share `dir`: + +``` +dat-next [options] + --sleep,-s persist sleep files in .dat folder +``` + +Download `key` to `dir`: + +``` +dat-next [options] + --sleep,-s persist sleep files in .dat folder +``` From 2dfdcd3323f433c3f24ac6a262441f8ce4f5c4c7 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Mon, 27 Mar 2017 13:06:28 -0700 Subject: [PATCH 03/20] upgrade dat-ignore --- index.js | 11 +---------- package.json | 2 +- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/index.js b/index.js index 1555fde..790b737 100644 --- a/index.js +++ b/index.js @@ -36,16 +36,7 @@ function run (src, dest, opts, cb) { function importFiles () { var ignore = datIgnore(dir) importProgress = mirror(dir, {name: '/', fs: archive}, { - ignore: true, - equals: function (a, b, cb) { - // Ignores - if (ignore(a.name)) return cb(null, true) - if (!b.stat) return cb(null, false) - - if (!a.stat.isDirectory() && (a.stat.size !== b.stat.size)) return cb(null, false) - if (a.stat.mtime.getTime() > b.stat.mtime.getTime()) return cb(null, false) - cb(null, true) - } + ignore: ignore }) countDir(dir, { ignore: ignore }, function (err, data) { diff --git a/package.json b/package.json index 25d1bcb..fd75e21 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "homepage": "https://github.com/joehand/dat-next#readme", "dependencies": { "count-files": "^2.2.1", - "dat-ignore": "0.0.0", + "dat-ignore": "1.0.0", "dat-swarm-defaults": "^1.0.0", "debug": "^2.4.5", "discovery-swarm": "^4.3.0", From 0ca8b2311adbf8e178d0a4dc18e3b6293cc944a7 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Mon, 27 Mar 2017 16:26:38 -0700 Subject: [PATCH 04/20] update mirror folder to PR api --- cli.js | 2 +- index.js | 25 +++++++++++++++---------- package.json | 2 +- 3 files changed, 17 insertions(+), 12 deletions(-) diff --git a/cli.js b/cli.js index b576f07..2229e83 100755 --- a/cli.js +++ b/cli.js @@ -61,7 +61,7 @@ dat(src, dest, argv, function (archive, swarm, importProgress) { output[1][0] = bar(imported) }) - importProgress.on('chunk', function (chunk) { + importProgress.on('put-data', function (chunk) { imported += chunk.length if (bar) output[1][0] = bar(imported) output[1][1] = pretty(indexSpeed(chunk.length)) + '/s' diff --git a/index.js b/index.js index 790b737..7778ed7 100644 --- a/index.js +++ b/index.js @@ -14,12 +14,12 @@ module.exports = run function run (src, dest, opts, cb) { var key - var importProgress if (dest) { key = src src = null mkdirp.sync(dest) } + var dir = dest || src opts = xtend({ storage: opts.sleep ? path.join(dir, '.dat') : ram @@ -28,38 +28,43 @@ function run (src, dest, opts, cb) { archive.on('ready', function () { cast.emit('ready') - if (!key) importFiles() + var progress = importFiles() var swarm = joinNetwork() - cb(archive, swarm, importProgress) + cb(archive, swarm, progress) }) function importFiles () { + if (!archive.metadata.writable) return + var progress var ignore = datIgnore(dir) - importProgress = mirror(dir, {name: '/', fs: archive}, { + + progress = mirror(dir, {name: '/', fs: archive}, { ignore: ignore }) countDir(dir, { ignore: ignore }, function (err, data) { if (err) throw err cast.emit('import:count', data) - importProgress.emit('count', data) + progress.emit('count', data) }) - importProgress.on('put', function (src, dst) { + progress.on('put', function (src, dst) { cast.emit('import:put', src, dst) }) - importProgress.on('chunk', function (chunk) { + progress.on('put-data', function (chunk) { cast.emit('import:chunk', chunk) }) - importProgress.on('del', function (dst) { + progress.on('del', function (dst) { cast.emit('import:del', dst) }) - importProgress.on('end', function () { + progress.on('end', function () { cast.emit('import:end') }) - importProgress.on('error', function (err) { + progress.on('error', function (err) { debug('IMPORT ERROR:', err) }) + + return progress } function joinNetwork () { diff --git a/package.json b/package.json index fd75e21..852a6f3 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ "hyperdrive": "github:mafintosh/hyperdrive#v8", "localcast": "^2.0.1", "minimist": "^1.2.0", - "mirror-folder": "github:joehand/mirror-folder#add/through", + "mirror-folder": "^1.1.0", "mkdirp": "^0.5.1", "prettier-bytes": "^1.0.3", "progress-string": "^1.2.1", From 051b5c1e34ff15bba7b840b748801bfe4c3a9ef2 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Mon, 27 Mar 2017 16:26:54 -0700 Subject: [PATCH 05/20] use hyperdiscovery --- index.js | 2 +- lib/network.js | 24 ------------------------ package.json | 1 + 3 files changed, 2 insertions(+), 25 deletions(-) delete mode 100644 lib/network.js diff --git a/index.js b/index.js index 7778ed7..da73f98 100644 --- a/index.js +++ b/index.js @@ -2,13 +2,13 @@ var path = require('path') var xtend = require('xtend') var mkdirp = require('mkdirp') var hyperdrive = require('hyperdrive') +var network = require('hyperdiscovery') var ram = require('random-access-memory') var mirror = require('mirror-folder') var countDir = require('count-files') var datIgnore = require('dat-ignore') var debug = require('debug')('dat') var cast = require('localcast')('dat-next') -var network = require('./lib/network') module.exports = run diff --git a/lib/network.js b/lib/network.js deleted file mode 100644 index 5b9dffb..0000000 --- a/lib/network.js +++ /dev/null @@ -1,24 +0,0 @@ -var assert = require('assert') -var swarmDefaults = require('dat-swarm-defaults') -var disc = require('discovery-swarm') -var xtend = require('xtend') - -module.exports = function (archive, opts, cb) { - assert.ok(archive, 'dat-node: lib/network archive required') - assert.ok(opts, 'dat-node: lib/network opts required') - - var DEFAULT_PORT = 3282 - var swarmOpts = xtend({ - id: archive.id, - hash: false, - stream: opts.stream - }, opts) - var swarm = disc(swarmDefaults(swarmOpts)) - swarm.once('error', function () { - swarm.listen(0) - }) - swarm.listen(opts.port || DEFAULT_PORT) - swarm.join(archive.discoveryKey, { announce: !(opts.upload === false) }, cb) - swarm.options = swarm._options - return swarm -} diff --git a/package.json b/package.json index 852a6f3..e23d4fa 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "dat-swarm-defaults": "^1.0.0", "debug": "^2.4.5", "discovery-swarm": "^4.3.0", + "hyperdiscovery": "^1.3.0", "hyperdrive": "github:mafintosh/hyperdrive#v8", "localcast": "^2.0.1", "minimist": "^1.2.0", From 3b2678c726d1bf70f6564786b44ef2f2cf384472 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Mon, 27 Mar 2017 16:40:33 -0700 Subject: [PATCH 06/20] remove cast from index --- index.js | 23 ++++------------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/index.js b/index.js index da73f98..ab5431a 100644 --- a/index.js +++ b/index.js @@ -8,7 +8,6 @@ var mirror = require('mirror-folder') var countDir = require('count-files') var datIgnore = require('dat-ignore') var debug = require('debug')('dat') -var cast = require('localcast')('dat-next') module.exports = run @@ -27,7 +26,6 @@ function run (src, dest, opts, cb) { var archive = hyperdrive(opts.storage, key, opts) archive.on('ready', function () { - cast.emit('ready') var progress = importFiles() var swarm = joinNetwork() cb(archive, swarm, progress) @@ -42,28 +40,15 @@ function run (src, dest, opts, cb) { ignore: ignore }) + progress.on('error', function (err) { + debug('IMPORT ERROR:', err) + }) + countDir(dir, { ignore: ignore }, function (err, data) { if (err) throw err - cast.emit('import:count', data) progress.emit('count', data) }) - progress.on('put', function (src, dst) { - cast.emit('import:put', src, dst) - }) - progress.on('put-data', function (chunk) { - cast.emit('import:chunk', chunk) - }) - progress.on('del', function (dst) { - cast.emit('import:del', dst) - }) - progress.on('end', function () { - cast.emit('import:end') - }) - progress.on('error', function (err) { - debug('IMPORT ERROR:', err) - }) - return progress } From 57fa2e4952983e39b7b0333bcbbff738636ff180 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Mon, 27 Mar 2017 16:53:33 -0700 Subject: [PATCH 07/20] cleanup --- cli.js | 65 +++++++++++++++++++++++++++------------------------------- 1 file changed, 30 insertions(+), 35 deletions(-) diff --git a/cli.js b/cli.js index 2229e83..f199510 100755 --- a/cli.js +++ b/cli.js @@ -16,7 +16,7 @@ var src = argv._[0] || process.cwd() var dest = argv._[1] var output = [['', ''], ['', '', '', '']] -var log = logger(output) +var log = logger(output, { quiet: argv.quiet }) var indexSpeed = speed() var downloadSpeed = speed() var hasContent @@ -25,7 +25,7 @@ var downloaded = 0 var total = 0 dat(src, dest, argv, function (archive, swarm, importProgress) { - output[0][0] = 'Starting...' + output[0][0] = 'Here we go!' setInterval(function () { networkUI() log.print() @@ -39,48 +39,43 @@ dat(src, dest, argv, function (archive, swarm, importProgress) { archive.once('content', function () { output[0].push('') // add space for peers hasContent = true - - if (!importProgress) { - downloadUI() - } + if (!importProgress) downloadUI() }) - if (importProgress) { - output[0][0] = `dat://${archive.key.toString('hex')}` + if (!importProgress) return output[0][0] = 'Connecting...' - var bar + var bar + output[0][0] = `dat://${archive.key.toString('hex')}` - importProgress.on('count', function (count) { - total = count.bytes - bar = progress({ - total: total, - style: function (a, b) { - return `[${a}${b}] ${pretty(imported)}/${pretty(total)}` - } - }) - output[1][0] = bar(imported) + importProgress.on('count', function (count) { + total = count.bytes + bar = progress({ + total: total, + style: function (a, b) { + return `[${a}${b}] ${pretty(imported)}/${pretty(total)}` + } }) + output[1][0] = bar(imported) + }) - importProgress.on('put-data', function (chunk) { - imported += chunk.length - if (bar) output[1][0] = bar(imported) - output[1][1] = pretty(indexSpeed(chunk.length)) + '/s' - }) + importProgress.on('put-data', function (chunk) { + imported += chunk.length + if (bar) output[1][0] = bar(imported) + output[1][1] = pretty(indexSpeed(chunk.length)) + '/s' + }) - importProgress.on('put', function (src, dst) { - output[1][3] = `ADD: ${dst.name}` - }) + importProgress.on('put', function (src, dst) { + output[1][3] = `ADD: ${dst.name}` + }) - importProgress.on('del', function (src, dst) { - output[1][3] = `DEL: ${dst.name}` - }) + importProgress.on('del', function (src, dst) { + output[1][3] = `DEL: ${dst.name}` + }) - importProgress.on('end', function (src, dst) { - output[1] = [`Import complete: ${pretty(total)}`] - }) - } else { - output[0][0] = 'Downloading...' - } + importProgress.on('end', function (src, dst) { + imported = total // TODO: end of put + output[1] = [`Import complete: ${pretty(total)}`] + }) function downloadUI () { var bar = downloadBar() From b69b135aa661486654f996dbd35271f3e5442ece Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Mon, 27 Mar 2017 16:56:10 -0700 Subject: [PATCH 08/20] standard --- cli.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cli.js b/cli.js index f199510..d5706a4 100755 --- a/cli.js +++ b/cli.js @@ -42,7 +42,10 @@ dat(src, dest, argv, function (archive, swarm, importProgress) { if (!importProgress) downloadUI() }) - if (!importProgress) return output[0][0] = 'Connecting...' + if (!importProgress) { + output[0][0] = 'Connecting...' + return + } var bar output[0][0] = `dat://${archive.key.toString('hex')}` From 98be222314896ba88646361adfe39a951d32f5be Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Tue, 28 Mar 2017 14:47:24 -0700 Subject: [PATCH 09/20] support single file sharing --- cli.js | 3 ++- index.js | 29 +++++++++++++++++------------ package.json | 4 ++-- 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/cli.js b/cli.js index d5706a4..f89f6b6 100755 --- a/cli.js +++ b/cli.js @@ -68,7 +68,8 @@ dat(src, dest, argv, function (archive, swarm, importProgress) { }) importProgress.on('put', function (src, dst) { - output[1][3] = `ADD: ${dst.name}` + var name = (dst.name === '/') ? src.name : dst.name // use prettier names if available + output[1][3] = `ADD: ${name}` }) importProgress.on('del', function (src, dst) { diff --git a/index.js b/index.js index ab5431a..b8a0325 100644 --- a/index.js +++ b/index.js @@ -1,3 +1,4 @@ +var fs = require('fs') var path = require('path') var xtend = require('xtend') var mkdirp = require('mkdirp') @@ -5,7 +6,7 @@ var hyperdrive = require('hyperdrive') var network = require('hyperdiscovery') var ram = require('random-access-memory') var mirror = require('mirror-folder') -var countDir = require('count-files') +var count = require('count-files') var datIgnore = require('dat-ignore') var debug = require('debug')('dat') @@ -16,27 +17,31 @@ function run (src, dest, opts, cb) { if (dest) { key = src src = null - mkdirp.sync(dest) } - var dir = dest || src - opts = xtend({ - storage: opts.sleep ? path.join(dir, '.dat') : ram - }, opts) - var archive = hyperdrive(opts.storage, key, opts) - + var archive = hyperdrive(storage(), key, opts) archive.on('ready', function () { var progress = importFiles() var swarm = joinNetwork() cb(archive, swarm, progress) }) + function storage () { + if (!opts.sleep) return ram + if (typeof opts.sleep === 'string') return opts.sleep + if (!src) return path.join(dest, '.dat') // TODO: if dest is file + + var isDir = fs.statSync(src).isDirectory() + if (isDir) return path.join(src, '.dat') + return cb(new Error('Specify dir for sleep files: --sleep ')) + } + function importFiles () { if (!archive.metadata.writable) return var progress - var ignore = datIgnore(dir) + var ignore = datIgnore(src) - progress = mirror(dir, {name: '/', fs: archive}, { + progress = mirror(src, {name: '/', fs: archive}, { ignore: ignore }) @@ -44,8 +49,8 @@ function run (src, dest, opts, cb) { debug('IMPORT ERROR:', err) }) - countDir(dir, { ignore: ignore }, function (err, data) { - if (err) throw err + count(src, { ignore: ignore }, function (err, data) { + if (err) return progress.emit('error', err) progress.emit('count', data) }) diff --git a/package.json b/package.json index e23d4fa..0f2baa0 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ }, "homepage": "https://github.com/joehand/dat-next#readme", "dependencies": { - "count-files": "^2.2.1", + "count-files": "^2.3.0", "dat-ignore": "1.0.0", "dat-swarm-defaults": "^1.0.0", "debug": "^2.4.5", @@ -28,7 +28,7 @@ "hyperdrive": "github:mafintosh/hyperdrive#v8", "localcast": "^2.0.1", "minimist": "^1.2.0", - "mirror-folder": "^1.1.0", + "mirror-folder": "^1.1.1", "mkdirp": "^0.5.1", "prettier-bytes": "^1.0.3", "progress-string": "^1.2.1", From 9f3eedf0a5287d56d302dcc0bd4044041db6aa98 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Tue, 28 Mar 2017 14:51:31 -0700 Subject: [PATCH 10/20] parse dat key --- index.js | 8 +++++++- package.json | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index b8a0325..34ace94 100644 --- a/index.js +++ b/index.js @@ -8,6 +8,7 @@ var ram = require('random-access-memory') var mirror = require('mirror-folder') var count = require('count-files') var datIgnore = require('dat-ignore') +var encoding = require('dat-encoding') var debug = require('debug')('dat') module.exports = run @@ -15,7 +16,12 @@ module.exports = run function run (src, dest, opts, cb) { var key if (dest) { - key = src + try { + // validate key + remove dat:// stuff + key = encoding.toStr(src) + } catch (e) { + return cb(new Error('Invalid dat link')) + } src = null } diff --git a/package.json b/package.json index 0f2baa0..d94c091 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "homepage": "https://github.com/joehand/dat-next#readme", "dependencies": { "count-files": "^2.3.0", + "dat-encoding": "^4.0.2", "dat-ignore": "1.0.0", "dat-swarm-defaults": "^1.0.0", "debug": "^2.4.5", From 3fe59a10800db999ea8af9ee6743809a0ebad088 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Tue, 28 Mar 2017 15:07:42 -0700 Subject: [PATCH 11/20] forgot mkdrip --- index.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/index.js b/index.js index 34ace94..e04fa68 100644 --- a/index.js +++ b/index.js @@ -35,7 +35,10 @@ function run (src, dest, opts, cb) { function storage () { if (!opts.sleep) return ram if (typeof opts.sleep === 'string') return opts.sleep - if (!src) return path.join(dest, '.dat') // TODO: if dest is file + if (!src) { + mkdirp.sync(dest) + return path.join(dest, '.dat') // TODO: if dest is file + } var isDir = fs.statSync(src).isDirectory() if (isDir) return path.join(src, '.dat') From 5c955085d40d8a742a5388f0d44fb49c46f9bfee Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Wed, 29 Mar 2017 19:21:09 -0700 Subject: [PATCH 12/20] add live option --- cli.js | 3 ++- index.js | 3 +++ readme.md | 20 ++++++++++++++++++-- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cli.js b/cli.js index f89f6b6..f54a1cd 100755 --- a/cli.js +++ b/cli.js @@ -10,7 +10,7 @@ var dat = require('./') process.title = 'dat-next' var argv = minimist(process.argv.slice(2), { - alias: {sleep: 's', quiet: 'q'} + alias: {sleep: 's', quiet: 'q', watch: 'w'} }) var src = argv._[0] || process.cwd() var dest = argv._[1] @@ -65,6 +65,7 @@ dat(src, dest, argv, function (archive, swarm, importProgress) { imported += chunk.length if (bar) output[1][0] = bar(imported) output[1][1] = pretty(indexSpeed(chunk.length)) + '/s' + if (argv.watch) output[1][2] = 'Watching for changes...' }) importProgress.on('put', function (src, dst) { diff --git a/index.js b/index.js index e04fa68..6f4e4b2 100644 --- a/index.js +++ b/index.js @@ -14,6 +14,8 @@ var debug = require('debug')('dat') module.exports = run function run (src, dest, opts, cb) { + if (!opts) opts = {} + var key if (dest) { try { @@ -51,6 +53,7 @@ function run (src, dest, opts, cb) { var ignore = datIgnore(src) progress = mirror(src, {name: '/', fs: archive}, { + live: opts.watch, ignore: ignore }) diff --git a/readme.md b/readme.md index 08da610..8da8f5d 100644 --- a/readme.md +++ b/readme.md @@ -14,14 +14,30 @@ npm install -g dat-next ## Usage -Share `dir`: +### Share files: + +``` +❯ dat-next my-data/ --sleep +dat://22607c7238e0cd74faa239d4119f32f455bb5bd555987090b2696dd0b090db38 +``` + +### Download files: + +``` +❯ dat-next my-data/ --sleep + +``` + + files + +### Share a directory: ``` dat-next [options] --sleep,-s persist sleep files in .dat folder ``` -Download `key` to `dir`: +### Download `key` to `dir`: ``` dat-next [options] From 3ccecd4f51d201d104f0a27a2f26afc19fe178d9 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Thu, 30 Mar 2017 15:51:23 -0700 Subject: [PATCH 13/20] use secret store --- index.js | 5 +++-- package.json | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/index.js b/index.js index 6f4e4b2..745e7a1 100644 --- a/index.js +++ b/index.js @@ -5,6 +5,7 @@ var mkdirp = require('mkdirp') var hyperdrive = require('hyperdrive') var network = require('hyperdiscovery') var ram = require('random-access-memory') +var secretStore = require('dat-secret-storage') var mirror = require('mirror-folder') var count = require('count-files') var datIgnore = require('dat-ignore') @@ -36,14 +37,14 @@ function run (src, dest, opts, cb) { function storage () { if (!opts.sleep) return ram - if (typeof opts.sleep === 'string') return opts.sleep + if (typeof opts.sleep === 'string') return secretStore(opts.sleep) if (!src) { mkdirp.sync(dest) return path.join(dest, '.dat') // TODO: if dest is file } var isDir = fs.statSync(src).isDirectory() - if (isDir) return path.join(src, '.dat') + if (isDir) return secretStore(path.join(src, '.dat')) return cb(new Error('Specify dir for sleep files: --sleep ')) } diff --git a/package.json b/package.json index d94c091..a7aa769 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,7 @@ "count-files": "^2.3.0", "dat-encoding": "^4.0.2", "dat-ignore": "1.0.0", + "dat-secret-storage": "0.0.0", "dat-swarm-defaults": "^1.0.0", "debug": "^2.4.5", "discovery-swarm": "^4.3.0", From b1ebbf4f119013bc033c67f6d9bb751ae2cbb498 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Mon, 3 Apr 2017 10:27:52 -0700 Subject: [PATCH 14/20] add ui for live file watching --- cli.js | 126 +++++++++++++++++++++++++++++++++++++++++++++---------- index.js | 18 ++++++++ 2 files changed, 121 insertions(+), 23 deletions(-) diff --git a/cli.js b/cli.js index f54a1cd..1ffb668 100755 --- a/cli.js +++ b/cli.js @@ -15,7 +15,11 @@ var argv = minimist(process.argv.slice(2), { var src = argv._[0] || process.cwd() var dest = argv._[1] -var output = [['', ''], ['', '', '', '']] +var output = [ + ['', ''], // Key/Msg + Peer Count + ['', ''], // Total Import/Download Progress + ['', ''] // File Import Progress +] var log = logger(output, { quiet: argv.quiet }) var indexSpeed = speed() var downloadSpeed = speed() @@ -23,6 +27,10 @@ var hasContent var imported = 0 var downloaded = 0 var total = 0 +var fileImported = 0 +var bar +var totalBar +var watchTimeout dat(src, dest, argv, function (archive, swarm, importProgress) { output[0][0] = 'Here we go!' @@ -30,58 +38,130 @@ dat(src, dest, argv, function (archive, swarm, importProgress) { networkUI() log.print() if (archive.downloaded) { + log.clear() console.log('Done! Bye bye.') process.exit(0) } - }, 500) + }, 200) log.print() archive.once('content', function () { - output[0].push('') // add space for peers hasContent = true + imported = archive.content.byteLength if (!importProgress) downloadUI() }) + swarm.once('connection', function () { + output[0].push('') // add space for peers + }) if (!importProgress) { output[0][0] = 'Connecting...' return } - var bar output[0][0] = `dat://${archive.key.toString('hex')}` - importProgress.on('count', function (count) { + importProgress.once('count', function (count) { total = count.bytes - bar = progress({ - total: total, - style: function (a, b) { - return `[${a}${b}] ${pretty(imported)}/${pretty(total)}` - } - }) - output[1][0] = bar(imported) + if (!argv.watch) { + totalBar = progress({ + total: total, + style: function (a, b) { + return `[${a}${b}] ${pretty(imported)} / ${pretty(total)}` + } + }) + output[1][1] = totalBar(imported) + output[1].push('') // Import Speed + output[1].push('') // Spacer + } + updateImportTotal() }) - importProgress.on('put-data', function (chunk) { + importProgress.on('put', function (src, dst) { + // Show progress for files only + if (src.stat.isDirectory()) return + clearTimeout(watchTimeout) + + var name = (dst.name === '/') ? src.name : dst.name // use prettier names if available + output[2][0] = `ADD: ${name}` + fileImported = 0 + + // Avoid flashing progress bar of small files + if (src.stat.size > Math.pow(10,7) ) { + bar = progress({ + total: src.stat.size, + style: function (a, b) { + return `[${a}${b}] ${pretty(fileImported)} / ${pretty(src.stat.size)}` + } + }) + output[2][1] = bar(fileImported) + } + }) + + importProgress.on('put-data', function (chunk, src, dst) { imported += chunk.length - if (bar) output[1][0] = bar(imported) - output[1][1] = pretty(indexSpeed(chunk.length)) + '/s' - if (argv.watch) output[1][2] = 'Watching for changes...' + fileImported += chunk.length + + if (bar) { + output[2][1] = bar(fileImported) + if (!totalBar) output[2][2] = `${pretty(indexSpeed(chunk.length))}/s` + } + updateImportTotal(chunk.length) }) - importProgress.on('put', function (src, dst) { - var name = (dst.name === '/') ? src.name : dst.name // use prettier names if available - output[1][3] = `ADD: ${name}` + importProgress.on('put-end', function (src, dst) { + // Remove put file progress + if (bar) { + output[2][1] = '' + if (!totalBar) output[2][2] = '' + } + fileImported = 0 + bar = null + updateImportTotal() + + if (argv.watch) { + watchTimeout = setTimeout(function () { + if (argv.watch) output[2] = [''] // clear output for idle watching + }, 1200) + } }) - importProgress.on('del', function (src, dst) { - output[1][3] = `DEL: ${dst.name}` + importProgress.on('del', function (dst) { + output[2][0] = `DEL: ${dst.name}` + clearTimeout(watchTimeout) + + if (argv.watch) { + watchTimeout = setTimeout(function () { + if (argv.watch) output[2] = [''] // clear output for idle watching + }, 1200) + } }) importProgress.on('end', function (src, dst) { - imported = total // TODO: end of put - output[1] = [`Import complete: ${pretty(total)}`] + // Only fires if argv.watch === false + totalBar = null + output[1] = [output[1][0]] // Clear total bar + import speed + output[2] = [`\nImport complete`] + setTimeout(function () { + output.pop() + }, 5000) }) + function updateImportTotal (size) { + size = size || 0 + var verb = !argv.watch + ? imported === total + ? 'Sharing' + : 'Importing to' + : 'Syncing' + + output[1][0] = `${verb} Archive: ${archive.metadata.length - 1} files (${pretty(archive.content.byteLength)})` + if (totalBar) { + output[1][1] = totalBar(imported) + output[1][2] = `${pretty(indexSpeed(size))}/s` + } + } + function downloadUI () { var bar = downloadBar() archive.content.ready(function () { diff --git a/index.js b/index.js index 745e7a1..ab52795 100644 --- a/index.js +++ b/index.js @@ -35,6 +35,24 @@ function run (src, dest, opts, cb) { cb(archive, swarm, progress) }) + if (opts.sparse) { + if (typeof opts.sparse === 'string') { + downloadFiles(opts.sparse) + } else { + var syncFile = path.join(dest, '.datsync') + fs.readFile(syncFile, 'utf-8', function (err, data) { + if (err) throw err + data = data.split('\n') + downloadFiles(data) + }) + } + + function downloadFiles (list) { + console.log('downloading files', list) + // TODO + } + } + function storage () { if (!opts.sleep) return ram if (typeof opts.sleep === 'string') return secretStore(opts.sleep) From 74740dff5c444f30633b5291fa0d89fa03eecd1f Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Tue, 4 Apr 2017 09:57:34 -0700 Subject: [PATCH 15/20] update status-logger --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a7aa769..3da7f75 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,7 @@ "progress-string": "^1.2.1", "random-access-memory": "^2.3.0", "speedometer": "^1.0.0", - "status-logger": "^3.0.0", + "status-logger": "^3.1.0", "xtend": "^4.0.1" }, "devDependencies": { From 47ea8d86423307af1f01c8188eb42e46f573e151 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Wed, 5 Apr 2017 10:30:39 -0700 Subject: [PATCH 16/20] add dereference options --- index.js | 5 +++-- package.json | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index ab52795..faa460c 100644 --- a/index.js +++ b/index.js @@ -73,14 +73,15 @@ function run (src, dest, opts, cb) { progress = mirror(src, {name: '/', fs: archive}, { live: opts.watch, - ignore: ignore + ignore: ignore, + dereference: true }) progress.on('error', function (err) { debug('IMPORT ERROR:', err) }) - count(src, { ignore: ignore }, function (err, data) { + count(src, { ignore: ignore, dereference: true }, function (err, data) { if (err) return progress.emit('error', err) progress.emit('count', data) }) diff --git a/package.json b/package.json index 3da7f75..0b97180 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ }, "homepage": "https://github.com/joehand/dat-next#readme", "dependencies": { - "count-files": "^2.3.0", + "count-files": "^2.4.0", "dat-encoding": "^4.0.2", "dat-ignore": "1.0.0", "dat-secret-storage": "0.0.0", From be5750a39c704af4837ab8ebab9dbacb1a35bb2a Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Wed, 5 Apr 2017 13:35:29 -0700 Subject: [PATCH 17/20] update cli to try neat-log --- cli-old.js | 202 ++++++++++++++++++++++++++++++++ cli.js | 337 +++++++++++++++++++++++++++++------------------------ 2 files changed, 384 insertions(+), 155 deletions(-) create mode 100755 cli-old.js diff --git a/cli-old.js b/cli-old.js new file mode 100755 index 0000000..f107ad2 --- /dev/null +++ b/cli-old.js @@ -0,0 +1,202 @@ +#!/usr/bin/env node + +var minimist = require('minimist') +var logger = require('status-logger') +var pretty = require('prettier-bytes') +var speed = require('speedometer') +var progressBar = require('progress-string') +var dat = require('./') + +process.title = 'dat-next' + +var argv = minimist(process.argv.slice(2), { + alias: {sleep: 's', quiet: 'q', watch: 'w'} +}) +var src = argv._[0] || process.cwd() +var dest = argv._[1] + +var output = [ + ['', ''], // Key/Msg + Peer Count + ['', ''], // Total Import/Download Progress + ['', ''] // File Import Progress +] +var log = logger(output, { quiet: argv.quiet }) +var indexSpeed = speed() +var downloadSpeed = speed() +var hasContent +var imported = 0 +var downloaded = 0 +var total = 0 +var fileImported = 0 +var bar +var totalBar +var watchTimeout + +dat(src, dest, argv, function (archive, swarm, progress) { + output[0][0] = 'Here we go!' + setInterval(function () { + networkUI() + log.print() + if (archive.downloaded) { + log.clear() + console.log('Done! Bye bye.') + process.exit(0) + } + }, 200) + log.print() + + archive.once('content', function () { + hasContent = true + imported = archive.content.byteLength + if (!archive.metadata.writable) downloadUI() + }) + swarm.once('connection', function () { + output[0].push('') // add space for peers + }) + + if (!archive.metadata.writable) { + output[0][0] = 'Connecting...' + return + } + + output[0][0] = `dat://${archive.key.toString('hex')}` + + progress.once('count', function (count) { + total = count.bytes + if (!argv.watch) { + totalBar = progressBar({ + total: total, + style: function (a, b) { + return `[${a}${b}] ${pretty(imported)} / ${pretty(total)}` + } + }) + output[1][1] = totalBar(imported) + output[1].push('') // Import Speed + output[1].push('') // Spacer + } + updateImportTotal() + }) + + progress.on('put', function (src, dst) { + // Show progress for files only + if (src.stat.isDirectory()) return + clearTimeout(watchTimeout) + + var name = (dst.name === '/') ? src.name : dst.name // use prettier names if available + output[2][0] = `ADD: ${name}` + fileImported = 0 + + // Avoid flashing progress bar of small files + if (src.stat.size > Math.pow(10,7) ) { + bar = progressBar({ + total: src.stat.size, + style: function (a, b) { + return `[${a}${b}] ${pretty(fileImported)} / ${pretty(src.stat.size)}` + } + }) + output[2][1] = bar(fileImported) + } + }) + + progress.on('put-data', function (chunk, src, dst) { + imported += chunk.length + fileImported += chunk.length + + if (bar) { + output[2][1] = bar(fileImported) + if (!totalBar) output[2][2] = `${pretty(indexSpeed(chunk.length))}/s` + } + updateImportTotal(chunk.length) + }) + + progress.on('put-end', function (src, dst) { + // Remove put file progress + if (bar) { + output[2][1] = '' + if (!totalBar) output[2][2] = '' + } + fileImported = 0 + bar = null + updateImportTotal() + + if (argv.watch) { + watchTimeout = setTimeout(function () { + if (argv.watch) output[2] = [''] // clear output for idle watching + }, 1200) + } + }) + + progress.on('del', function (dst) { + output[2][0] = `DEL: ${dst.name}` + clearTimeout(watchTimeout) + + if (argv.watch) { + watchTimeout = setTimeout(function () { + if (argv.watch) output[2] = [''] // clear output for idle watching + }, 1200) + } + }) + + progress.on('end', function (src, dst) { + // Only fires if argv.watch === false + totalBar = null + output[1] = [output[1][0]] // Clear total bar + import speed + output[2] = [`\nImport complete`] + setTimeout(function () { + output.pop() + }, 5000) + }) + + function updateImportTotal (size) { + size = size || 0 + var verb = !argv.watch + ? imported === total + ? 'Sharing' + : 'Importing to' + : 'Syncing' + + output[1][0] = `${verb} Archive: ${archive.metadata.length - 1} files (${pretty(archive.content.byteLength)})` + if (totalBar) { + output[1][1] = totalBar(imported) + output[1][2] = `${pretty(indexSpeed(size))}/s` + } + } + + function downloadUI () { + var bar = downloadBar() + archive.content.ready(function () { + total = archive.content.length + output[0][0] = `Downloading ${pretty(archive.content.byteLength)}` + output[1][0] = bar(downloaded) + for (var i = 0; i < archive.content.length; i++) { + if (archive.content.has(i)) downloaded++ + } + }) + + archive.content.on('download', function (index, data) { + if (archive.content.length !== total) { + output[0][0] = `Downloading ${pretty(archive.content.byteLength)}` + total = archive.content.length + bar = downloadBar() + } + downloaded++ + var per = (downloaded / total * 100).toFixed(2) + if (bar) output[1][0] = bar(downloaded) + ' ' + per + '%' + output[1][1] = pretty(downloadSpeed(data.length)) + '/s' + }) + + function downloadBar () { + return progressBar({ + total: total, + style: function (a, b) { + return `[${a}${b}]` + } + }) + } + } + + function networkUI () { + if (!swarm.connected || !hasContent) return + output[0][1] = `${archive.content.peers.length} peers` + } +}) diff --git a/cli.js b/cli.js index 1ffb668..d227fa4 100755 --- a/cli.js +++ b/cli.js @@ -1,202 +1,229 @@ #!/usr/bin/env node var minimist = require('minimist') -var logger = require('status-logger') var pretty = require('prettier-bytes') var speed = require('speedometer') var progress = require('progress-string') +var neatLog = require('neat-log') +var output = require('neat-log/output') + var dat = require('./') process.title = 'dat-next' var argv = minimist(process.argv.slice(2), { - alias: {sleep: 's', quiet: 'q', watch: 'w'} + alias: {temp: 't', quiet: 'q', watch: 'w', sleep: 's'} }) var src = argv._[0] || process.cwd() var dest = argv._[1] - -var output = [ - ['', ''], // Key/Msg + Peer Count - ['', ''], // Total Import/Download Progress - ['', ''] // File Import Progress -] -var log = logger(output, { quiet: argv.quiet }) var indexSpeed = speed() var downloadSpeed = speed() -var hasContent -var imported = 0 -var downloaded = 0 -var total = 0 -var fileImported = 0 -var bar -var totalBar -var watchTimeout - -dat(src, dest, argv, function (archive, swarm, importProgress) { - output[0][0] = 'Here we go!' - setInterval(function () { - networkUI() - log.print() - if (archive.downloaded) { - log.clear() - console.log('Done! Bye bye.') - process.exit(0) - } - }, 200) - log.print() +var uploadSpeed = speed() + +var neat = neatLog([mainView, progressView], {logspeed: 200}) // todo: opts.debug +neat.use(runDat) +neat.use(trackNetwork) +neat.use(trackProgress) + +function runDat (state, bus) { + state.title = 'Starting Dat program...' + bus.emit('render') + + dat(src, dest, argv, function (archive, network, progress) { + state.archive = archive + state.network = network + state.progress = progress + state.writable = archive.metadata.writable + archive.once('content', function () { + bus.emit('archive:content') + bus.emit('render') + }) + archive.metadata.on('append', function () { + bus.emit('render') + }) + + if (state.writable) state.title = `dat://${archive.key.toString('hex')}` + else state.title = 'Dat Download' - archive.once('content', function () { - hasContent = true - imported = archive.content.byteLength - if (!importProgress) downloadUI() + bus.emit('archive') + bus.emit('render') }) - swarm.once('connection', function () { - output[0].push('') // add space for peers +} + +function trackProgress (state, bus) { + bus.on('archive:content', function () { + if (state.writable) trackImport() + else trackDownload() }) - if (!importProgress) { - output[0][0] = 'Connecting...' - return + function trackDownload () { + var progress = state.progress + state.downloading = true + + // progress.on('put', function (src, dst) { + // state.fileDownload = { + // src: src, + // dst: dst, + // progress: 0 + // } + // }) } - output[0][0] = `dat://${archive.key.toString('hex')}` + function trackImport () { + var progress = state.progress - importProgress.once('count', function (count) { - total = count.bytes - if (!argv.watch) { - totalBar = progress({ - total: total, - style: function (a, b) { - return `[${a}${b}] ${pretty(imported)} / ${pretty(total)}` - } - }) - output[1][1] = totalBar(imported) - output[1].push('') // Import Speed - output[1].push('') // Spacer + state.importing = true + state.import = { + progress: state.archive.content.byteLength // ? is this what I want } - updateImportTotal() - }) - importProgress.on('put', function (src, dst) { - // Show progress for files only - if (src.stat.isDirectory()) return - clearTimeout(watchTimeout) + progress.on('count', function (count) { + state.count = count + bus.emit('render') + }) - var name = (dst.name === '/') ? src.name : dst.name // use prettier names if available - output[2][0] = `ADD: ${name}` - fileImported = 0 + progress.on('put', function (src, dst) { + if (src.stat.isDirectory()) return + var name = (dst.name === '/') ? src.name : dst.name // use prettier names if available + state.fileImport = { + src: src, + dst: dst, + progress: 0, + type: 'put' + } + bus.emit('render') + }) - // Avoid flashing progress bar of small files - if (src.stat.size > Math.pow(10,7) ) { - bar = progress({ - total: src.stat.size, - style: function (a, b) { - return `[${a}${b}] ${pretty(fileImported)} / ${pretty(src.stat.size)}` - } - }) - output[2][1] = bar(fileImported) - } - }) + progress.on('put-data', function (chunk, src, dst) { + state.fileImport.progress += chunk.length + state.import.progress += chunk.length + indexSpeed(chunk.length) + bus.emit('render') + }) - importProgress.on('put-data', function (chunk, src, dst) { - imported += chunk.length - fileImported += chunk.length + progress.on('put-end', function (src, dst) { + state.fileImport = null + bus.emit('render') + }) - if (bar) { - output[2][1] = bar(fileImported) - if (!totalBar) output[2][2] = `${pretty(indexSpeed(chunk.length))}/s` - } - updateImportTotal(chunk.length) - }) + progress.on('end', function (src, dst) { + state.fileImport = null + // state.importing = false + bus.emit('render') + }) + } +} - importProgress.on('put-end', function (src, dst) { - // Remove put file progress - if (bar) { - output[2][1] = '' - if (!totalBar) output[2][2] = '' - } - fileImported = 0 - bar = null - updateImportTotal() - - if (argv.watch) { - watchTimeout = setTimeout(function () { - if (argv.watch) output[2] = [''] // clear output for idle watching - }, 1200) - } - }) +function trackNetwork (state, bus) { + bus.on('archive:content', function () { + var archive = state.archive + var network = state.network - importProgress.on('del', function (dst) { - output[2][0] = `DEL: ${dst.name}` - clearTimeout(watchTimeout) + network.on('connection', function (peer) { + bus.emit('render') + peer.on('close', function () { + bus.emit('render') + }) + }) - if (argv.watch) { - watchTimeout = setTimeout(function () { - if (argv.watch) output[2] = [''] // clear output for idle watching - }, 1200) - } + archive.content.on('upload', function (index, data) { + state.uploadSpeed = uploadSpeed(data.length) + bus.emit('render') + }) + archive.content.on('download', function (index, data) { + state.downloadSpeed = downloadSpeed(data.length) + bus.emit('render') + }) }) +} + +function mainView (state) { + return output` + ${state.title} + ${archiveUI(state)} + ${networkUI(state)} + ` +} + +function progressView (state) { + if (state.downloading) return downloadUI(state) + else if (state.importing) return importUI(state) + return '' +} + +function archiveUI (state) { + if (!state.archive) return `Starting...` + var archive = state.archive + var size = archive.content ? archive.content.byteLength : 0 + return output` + ${state.downloading ? 'Downloading' : 'Syncing'} Archive: ${archive.metadata.length - 1} files (${pretty(size)}) + ` +} + +function networkUI (state) { + if (!state.network) return '' + if (!state.network.connected || !state.archive.content) { + if (state.writable) return '\nWaiting for Connections...' + return '\nConnecting...' + } + return output` - importProgress.on('end', function (src, dst) { - // Only fires if argv.watch === false - totalBar = null - output[1] = [output[1][0]] // Clear total bar + import speed - output[2] = [`\nImport complete`] - setTimeout(function () { - output.pop() - }, 5000) - }) + ${state.archive.content.peers.length} peers + ${speed()} + ` - function updateImportTotal (size) { - size = size || 0 - var verb = !argv.watch - ? imported === total - ? 'Sharing' - : 'Importing to' - : 'Syncing' - - output[1][0] = `${verb} Archive: ${archive.metadata.length - 1} files (${pretty(archive.content.byteLength)})` - if (totalBar) { - output[1][1] = totalBar(imported) - output[1][2] = `${pretty(indexSpeed(size))}/s` - } + function speed () { + var output = '' + if (state.uploadSpeed) output += `Uploading ${pretty(state.uploadSpeed)}/s` + if (state.downloadSpeed) output += `Downloading ${pretty(state.downloadSpeed)}/s` + return output } - - function downloadUI () { - var bar = downloadBar() - archive.content.ready(function () { - total = archive.content.length - output[0][0] = `Downloading ${pretty(archive.content.byteLength)}` - output[1][0] = bar(downloaded) - for (var i = 0; i < archive.content.length; i++) { - if (archive.content.has(i)) downloaded++ +} + +function downloadUI (state) { + return output` + + Download progress TODO + ` +} + +function importUI (state) { + if (!state.count) return + if (state.import.progress === state.count.bytes) return '\nAll files imported.' + if (!state.totalBar) { + var total = state.count.bytes + state.totalBar = progress({ + total: total, + style: function (a, b) { + return `[${a}${b}] ${pretty(state.import.progress)} / ${pretty(total)}` } }) + } - archive.content.on('download', function (index, data) { - if (archive.content.length !== total) { - output[0][0] = `Downloading ${pretty(archive.content.byteLength)}` - total = archive.content.length - bar = downloadBar() - } - downloaded++ - var per = (downloaded / total * 100).toFixed(2) - if (bar) output[1][0] = bar(downloaded) + ' ' + per + '%' - output[1][1] = pretty(downloadSpeed(data.length)) + '/s' - }) + return output` + + Importing ${state.count.files} files to Archive + ${state.totalBar(state.import.progress)} + ${pretty(indexSpeed())}/s + ${fileImport()} + ` - function downloadBar () { - return progress({ + function fileImport () { + if (!state.fileImport) return `` + if (state.fileImport.type === 'del') return `\nDEL: ${state.fileImport.src.name}` + if (!state.fileImport.bar) { + var total = state.fileImport.src.stat.size + state.fileImport.bar = progress({ total: total, style: function (a, b) { - return `[${a}${b}]` + return `[${a}${b}] ${pretty(state.fileImport.progress)} / ${pretty(total)}` } }) } - } + return output` - function networkUI () { - if (!swarm.connected || !hasContent) return - output[0][1] = `${archive.content.peers.length} peers` + ADD: ${state.fileImport.src.name} + ${state.fileImport.bar(state.fileImport.progress)} + ` } -}) +} From 868112720c3b2e0bd873d306031b3528a1e67332 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Wed, 5 Apr 2017 13:35:52 -0700 Subject: [PATCH 18/20] use hyperdrive 8 release and add downloads --- index.js | 58 ++++++++++++++++++++++++++++++++-------------------- package.json | 7 ++++--- 2 files changed, 40 insertions(+), 25 deletions(-) diff --git a/index.js b/index.js index faa460c..e669429 100644 --- a/index.js +++ b/index.js @@ -26,35 +26,22 @@ function run (src, dest, opts, cb) { return cb(new Error('Invalid dat link')) } src = null + opts.sparse = true } + var progress + var ignore = datIgnore(src || dest) var archive = hyperdrive(storage(), key, opts) + archive.on('ready', function () { - var progress = importFiles() + if (archive.metadata.writable) importFiles() + else downloadFiles() var swarm = joinNetwork() cb(archive, swarm, progress) }) - if (opts.sparse) { - if (typeof opts.sparse === 'string') { - downloadFiles(opts.sparse) - } else { - var syncFile = path.join(dest, '.datsync') - fs.readFile(syncFile, 'utf-8', function (err, data) { - if (err) throw err - data = data.split('\n') - downloadFiles(data) - }) - } - - function downloadFiles (list) { - console.log('downloading files', list) - // TODO - } - } - function storage () { - if (!opts.sleep) return ram + if (opts.temp) return ram if (typeof opts.sleep === 'string') return secretStore(opts.sleep) if (!src) { mkdirp.sync(dest) @@ -68,8 +55,6 @@ function run (src, dest, opts, cb) { function importFiles () { if (!archive.metadata.writable) return - var progress - var ignore = datIgnore(src) progress = mirror(src, {name: '/', fs: archive}, { live: opts.watch, @@ -89,6 +74,35 @@ function run (src, dest, opts, cb) { return progress } + function downloadFiles () { + if (!archive.metadata.length) { + return archive.metadata.once('append', downloadFiles) + } + + var length = archive.metadata.length + var progress = mirror({name: '/', fs: archive}, dest) + var changed = false + + progress.on('put', function (src) { + changed = true + }) + + progress.on('del', function (src) { + changed = true + }) + + progress.on('end', function () { + if (!changed) { + if (length !== archive.metadata.length) downloadFiles() + else archive.metadata.once('append', downloadFiles) + return + } + archive.downloaded = true + }) + + return progress + } + function joinNetwork () { var swarm = network(archive, xtend({ stream: function (peer) { diff --git a/package.json b/package.json index 0b97180..143cd69 100644 --- a/package.json +++ b/package.json @@ -21,17 +21,18 @@ "dependencies": { "count-files": "^2.4.0", "dat-encoding": "^4.0.2", - "dat-ignore": "1.0.0", + "dat-ignore": "^1.0.0", "dat-secret-storage": "0.0.0", "dat-swarm-defaults": "^1.0.0", "debug": "^2.4.5", "discovery-swarm": "^4.3.0", "hyperdiscovery": "^1.3.0", - "hyperdrive": "github:mafintosh/hyperdrive#v8", + "hyperdrive": "^8.0.0", "localcast": "^2.0.1", "minimist": "^1.2.0", - "mirror-folder": "^1.1.1", + "mirror-folder": "^1.2.2", "mkdirp": "^0.5.1", + "neat-log": "^1.0.0", "prettier-bytes": "^1.0.3", "progress-string": "^1.2.1", "random-access-memory": "^2.3.0", From 44ebb85f2654c21c6d1c28d2db74cefcd45bcc1f Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Wed, 5 Apr 2017 13:37:47 -0700 Subject: [PATCH 19/20] update readme --- readme.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index 8da8f5d..d0b1800 100644 --- a/readme.md +++ b/readme.md @@ -17,29 +17,28 @@ npm install -g dat-next ### Share files: ``` -❯ dat-next my-data/ --sleep +❯ dat-next my-data/ dat://22607c7238e0cd74faa239d4119f32f455bb5bd555987090b2696dd0b090db38 ``` ### Download files: ``` -❯ dat-next my-data/ --sleep +❯ dat-next my-data/ ``` - files - ### Share a directory: ``` dat-next [options] - --sleep,-s persist sleep files in .dat folder + --temp,-t keep sleep data in memory + --watch, -w watch folder for changes + auto import ``` ### Download `key` to `dir`: ``` dat-next [options] - --sleep,-s persist sleep files in .dat folder + --temp,-t keep sleep data in memory ``` From ebcd3fa85baa76cbe3db8d409b1b0007b5367352 Mon Sep 17 00:00:00 2001 From: Joe Hand Date: Wed, 5 Apr 2017 13:38:50 -0700 Subject: [PATCH 20/20] standard --- cli-old.js | 2 +- cli.js | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cli-old.js b/cli-old.js index f107ad2..6126dbb 100755 --- a/cli-old.js +++ b/cli-old.js @@ -87,7 +87,7 @@ dat(src, dest, argv, function (archive, swarm, progress) { fileImported = 0 // Avoid flashing progress bar of small files - if (src.stat.size > Math.pow(10,7) ) { + if (src.stat.size > Math.pow(10, 7)) { bar = progressBar({ total: src.stat.size, style: function (a, b) { diff --git a/cli.js b/cli.js index d227fa4..c30875d 100755 --- a/cli.js +++ b/cli.js @@ -57,7 +57,7 @@ function trackProgress (state, bus) { }) function trackDownload () { - var progress = state.progress + // var progress = state.progress state.downloading = true // progress.on('put', function (src, dst) { @@ -84,7 +84,6 @@ function trackProgress (state, bus) { progress.on('put', function (src, dst) { if (src.stat.isDirectory()) return - var name = (dst.name === '/') ? src.name : dst.name // use prettier names if available state.fileImport = { src: src, dst: dst,