Live file watching with ui (#77)

* start live file watching with ui

* update total size for live file watching

* standard

* add live imports debug

* small test fixes for live

* add basic live test

* remove unused deps

* upgrade dat-node

* add note about deleting

* add docs

* fix auth clone test

* fix share test for travis

lib/commands/sync.js

@@ -17,6 +17,11 @@ module.exports = {
       boolean: true,
       default: true,
       abbr: 'ignore-hidden'
+    },
+    {
+      name: 'watch',
+      boolean: true,
+      default: true
     }
   ],
   command: function (opts) {

lib/download.js

@@ -90,7 +90,7 @@ module.exports = function (type, opts, dat) {
       output[0] = ['']
       output[1] = ['']
       log.print()
-      rimraf.sync(dat.path)
+      rimraf.sync(dat.path) // (TODO: only do this if we made the dir. Otherwise may delete users files on an error.)
       return exit('Link is not a Dat Archive. Please check you have the correct link.')
     }
   }

lib/share.js

@@ -1,4 +1,5 @@
 var assert = require('assert')
+var basename = require('path').basename
 var logger = require('status-logger')
 var prettyBytes = require('pretty-bytes')
 var Dat = require('dat-node')
@@ -15,6 +16,7 @@ module.exports = function sync (type, opts, dat) {
   var importStatus = null
   var network = null
   var stats = null
+  var fileOutput, liveProgressOutput
 
   // Logging Init.
   // After download require because downloader has it's own logger
@@ -26,9 +28,14 @@ module.exports = function sync (type, opts, dat) {
       '', // Shows Total Size Info
       '' //  spacer before network info
     ],
-    [] // Shows network information
+    [''] // Shows network information
   ]
   var progressOutput = output[0] // shortcut for progress output
+  if (opts.watch) {
+    output.push(['', '', ''], []) // progress bar, live file list
+    liveProgressOutput = output[2]
+    fileOutput = output[3]
+  }
   var log = logger(output, {debug: opts.verbose, quiet: opts.quiet || opts.debug}) // If debug=true we want output to be quiet.
 
   // UI Elements
@@ -69,16 +76,44 @@ module.exports = function sync (type, opts, dat) {
       progressOutput[2] = 'Importing new & updated files to archive...'
 
       importStatus = dat.importFiles({
-        live: false, // TODO: allow live: true. opts.live is used by archive.live though =(
+        watch: opts.watch, // TODO: allow live: true. opts.live is used by archive.live though =(
         resume: true,
         ignoreHidden: opts.ignoreHidden
       }, function (err) {
         if (err) return exit(err)
         debug('Import finished')
         var st = stats.get()
         importDone = true
-        progressOutput[2] = 'Archive update finished! Sharing latest files.'
+        progressOutput[2] = opts.watch ? 'Watching for file changes...' : 'Archive update finished! Sharing latest files.'
         progressOutput[3] = `Total Size: ${st.filesTotal} ${st.filesTotal === 1 ? 'file' : 'files'} (${prettyBytes(st.bytesTotal)})`
+
+        if (opts.watch) {
+          var pending = 0
+          importStatus.on('file watch event', function (file) {
+            pending++
+            debug(`File watch ${file.mode}: ${basename(file.path)}`)
+            // liveProgressOutput.splice(2, 0, `Importing: ${ basename(file.path)}`)
+            // liveProgressOutput.splice(-2, 1)
+          })
+          importStatus.on('file imported', function (file) {
+            if (fileOutput.length > 6) {
+              fileOutput.pop()
+            }
+            var fileAction = file.mode === 'created' ? 'Added' : 'Updated'
+            fileOutput.unshift(`${fileAction}: ${basename(file.path)}`)
+            pending--
+            debug(`File ${file.mode}: ${basename(file.path)}`)
+          })
+          importStatus.on('file skipped', function (file) {
+            pending--
+            debug(`File skipped: ${basename(file.path)}`)
+          })
+          setInterval(function () {
+            if (pending) liveProgressOutput[1] = `Importing ${pending} files...`
+            else liveProgressOutput[1] = ''
+            progressOutput[3] = `Total Size: ${importStatus.fileCount} ${importStatus.fileCount === 1 ? 'file' : 'files'} (${prettyBytes(importStatus.totalSize)})`
+          }, opts.logspeed)
+        }
       })
     }
   }

package.json

@@ -22,11 +22,8 @@
   "dependencies": {
     "dat-doctor": "^1.0.1",
     "dat-encoding": "^4.0.1",
-    "dat-node": "^1.1.1",
+    "dat-node": "^1.1.2",
     "debug": "^2.4.5",
-    "hyperdrive": "^7.12.2",
-    "hyperdrive-count-import": "^1.0.2",
-    "hyperdrive-network-speed": "^1.1.0",
     "memdb": "^1.3.1",
     "nets": "^3.2.0",
     "pretty-bytes": "^4.0.2",

readme.md

@@ -69,12 +69,14 @@ dat-next sync
 Syncing Dat Archive: /path/to/my-files-to-share
 Link: dat://56c1977328c94c988137c9ff3cbeaab217d765772299840e7be7172b18dbb151
 
-Archive update finished! Sharing latest files.
+Watching files for changes...
 Total Size: 50 files (40.5 MB)
 
 Looking for connections in Dat Network...
 ```
 
+The `sync` command will also watch your directory for changes and share any updated files.
+
 #### Downloading Files
 
 Once the files are shared somewhere, you can clone them to another location:
@@ -161,12 +163,15 @@ Will automatically import the files in that directory to the archive.
 
 
 ```
-dat sync [--dir=<folder>] [--no-import]
+dat sync [--dir=<folder>] [--no-import] [--no-watch]
 ```
 
 Start sharing your Dat Archive over the network.
 Sync will import new or updated files since you ran `create` or `sync` last.
+Sync watched files for changes and imports updated files.
 
+* Use `--no-import` to not import any new or updated files.
+* Use `--no-watch` to not watch directory for changes. `--import` must be true for `--watch` to work.
 
 #### Snapshot
 

tests/auth.js

@@ -112,7 +112,9 @@ authServer(port, function (err, server, closeServer) {
     var cmd = dat + ' clone ' + shortName
     var st = spawn(t, cmd, {cwd: baseDir})
     st.stdout.match(function (output) {
-      t.ok(output.indexOf('Looking for') > -1, 'starts looking for peers')
+      var lookingFor = output.indexOf('Looking for') > -1
+      if (!lookingFor) return false
+      t.ok(lookingFor, 'starts looking for peers')
       t.ok(output.indexOf(downloadDir) > -1, 'prints dir')
       st.kill()
       return true

tests/create.js

@@ -76,7 +76,7 @@ test('create - sync after create ok', function (t) {
   var st = spawn(t, cmd, {cwd: fixtures})
 
   st.stdout.match(function (output) {
-    var connected = output.indexOf('Sharing latest') > -1
+    var connected = output.indexOf('Looking for connections') > -1
     if (!connected) return false
     t.ok('Dat Network', 'Shares over dat network')
     st.kill()

tests/sync-owner.js

@@ -9,6 +9,7 @@ var spawn = require('./helpers/spawn.js')
 var help = require('./helpers')
 
 var dat = path.resolve(path.join(__dirname, '..', 'bin', 'cli.js'))
+if (process.env.TRAVIS) dat += ' --no-watch '
 var fixtures = path.join(__dirname, 'fixtures')
 var downDat
 
@@ -107,7 +108,8 @@ test('sync-owner - imports after no-import create', function (t) {
   var st = spawn(t, cmd, {cwd: fixtures})
 
   st.stdout.match(function (output) {
-    var sharing = output.indexOf('Sharing latest') > -1
+    // have to check both for local test (watching) and travis (sharing)
+    var sharing = output.indexOf('Watching') > -1 || output.indexOf('Sharing latest') > -1
     if (!sharing) return false
 
     var fileRe = new RegExp('2 files')
@@ -216,7 +218,7 @@ test('sync-owner - shorthand', function (t) {
   var st = spawn(t, cmd, {cwd: fixtures})
 
   st.stdout.match(function (output) {
-    var sharing = output.indexOf('Sharing latest') > -1
+    var sharing = output.indexOf('Looking for connections') > -1
     if (!sharing) return false
 
     t.ok(help.matchLink(output), 'prints link')
@@ -228,6 +230,36 @@ test('sync-owner - shorthand', function (t) {
   st.end()
 })
 
+if (!process.env.TRAVIS) {
+  test('sync-owner - live', function (t) {
+    var liveFile = path.join(fixtures, 'live.txt')
+    var wroteFile = false
+
+    var cmd = dat + ' sync --watch'
+    var st = spawn(t, cmd, {cwd: fixtures})
+
+    st.stdout.match(function (output) {
+      var watching = output.indexOf('Watching for file changes') > -1
+      if (!watching) return false
+      else if (!wroteFile) {
+        fs.writeFileSync(liveFile, 'hello')
+        wroteFile = true
+      }
+      var fileImported = output.indexOf('live.txt') > -1
+      if (!fileImported) return false
+
+      t.ok(fileImported, 'prints live file output')
+      t.ok(output.indexOf('3 files') > -1, 'total size: files okay')
+
+      fs.unlinkSync(liveFile)
+      st.kill()
+      return true
+    })
+    st.stderr.empty()
+    st.end()
+  })
+}
+
 test.onFinish(function () {
   rimraf.sync(path.join(fixtures, '.dat'))
 })