Implement Time-To-Live Feature

actions/describeTimeToLive.js

@@ -1,8 +1,13 @@
 
 module.exports = function describeTimeToLive(store, data, cb) {
-  store.getTable(data.TableName, false, function(err) {
+  store.getTable(data.TableName, false, function(err, table) {
     if (err) return cb(err)
 
-    cb(null, {TimeToLiveDescription: {TimeToLiveStatus: 'DISABLED'}})
+    if (table.TimeToLiveDescription !== null && typeof table.TimeToLiveDescription === 'object') {
+      cb(null, {TimeToLiveDescription: table.TimeToLiveDescription})
+    } else {
+      cb(null, {TimeToLiveDescription: {TimeToLiveStatus: 'DISABLED'}})
+    }
+
   })
 }

actions/updateTimeToLive.js

@@ -0,0 +1,38 @@
+const db = require('../db');
+
+module.exports = function updateTimeToLive(store, data, cb) {
+    var key = data.TableName,
+     TimeToLiveSpecification = data.TimeToLiveSpecification,
+     tableDb = store.tableDb,
+        returnValue;
+
+    store.getTable(key, false, function(err, table) {
+        if (err) return cb(err)
+
+        if (TimeToLiveSpecification.Enabled) {
+            if (table.TimeToLiveDescription && table.TimeToLiveDescription.TimeToLiveStatus === 'ENABLED') {
+                return cb(db.validationError('TimeToLive is already enabled'))
+            }
+            table.TimeToLiveDescription = {
+                AttributeName: TimeToLiveSpecification.AttributeName,
+                TimeToLiveStatus: 'ENABLED',
+            }
+            returnValue = TimeToLiveSpecification
+        } else {
+            if (table.TimeToLiveDescription == null || table.TimeToLiveDescription.TimeToLiveStatus === 'DISABLED') {
+                return cb(db.validationError('TimeToLive is already disabled'))
+            }
+
+            table.TimeToLiveDescription = {
+                TimeToLiveStatus: 'DISABLED',
+            }
+            returnValue = {Enabled: false}
+        }
+
+        tableDb.put(key, table, function(err) {
+            if (err) return cb(err)
+
+            cb(null, {TimeToLiveSpecification: returnValue})
+        })
+    })
+}

db/index.js

@@ -127,6 +127,38 @@ function create(options) {
     })
   }
 
+  var timerIdTtlScanner = setInterval(function() {
+    const currentUnixSeconds = Math.round(Date.now() / 1000)
+    function logError(err, result) {
+      if (err) console.error("@@@", err)
+    }
+    lazyStream(tableDb.createKeyStream({}), logError)
+        .join(function(tableNames) {
+          tableNames.forEach(function(name) {
+            getTable(name, false, function(err, table) {
+              if (err) return
+              if (!table.TimeToLiveDescription || table.TimeToLiveDescription.TimeToLiveStatus !== 'ENABLED') return
+
+              var itemDb = getItemDb(table.TableName)
+              var kvStream = lazyStream(itemDb.createReadStream({}), logError())
+              kvStream = kvStream.filter(function(item){
+                var ttl = item.value[table.TimeToLiveDescription.AttributeName]
+                return ttl && typeof ttl.N === 'string' && currentUnixSeconds > Number(ttl.N)
+              })
+              kvStream.join(function(items){
+                items.forEach(function(item) {
+                  itemDb.del(item.key)
+                })
+              })
+            })
+          })
+        })
+  }, 1000)
+
+  function stopBackgroundJobs() {
+    clearInterval(timerIdTtlScanner)
+  }
+
   return {
     options: options,
     db: db,
@@ -139,6 +171,7 @@ function create(options) {
     deleteTagDb: deleteTagDb,
     getTable: getTable,
     recreate: recreate,
+    stopBackgroundJobs: stopBackgroundJobs,
   }
 }
 

index.js

@@ -13,7 +13,7 @@ var MAX_REQUEST_BYTES = 16 * 1024 * 1024
 var validApis = ['DynamoDB_20111205', 'DynamoDB_20120810'],
     validOperations = ['BatchGetItem', 'BatchWriteItem', 'CreateTable', 'DeleteItem', 'DeleteTable',
       'DescribeTable', 'DescribeTimeToLive', 'GetItem', 'ListTables', 'PutItem', 'Query', 'Scan', 'TagResource',
-      'UntagResource', 'ListTagsOfResource', 'UpdateItem', 'UpdateTable'],
+      'UntagResource', 'ListTagsOfResource', 'UpdateItem', 'UpdateTable', 'UpdateTimeToLive'],
     actions = {},
     actionValidations = {}
 
@@ -35,6 +35,7 @@ function dynalite(options) {
   // Ensure we close DB when we're closing the server too
   var httpServerClose = server.close, httpServerListen = server.listen
   server.close = function(cb) {
+    store.stopBackgroundJobs()
     store.db.close(function(err) {
       if (err) return cb(err)
       // Recreate the store if the user wants to listen again
@@ -46,6 +47,7 @@ function dynalite(options) {
     })
   }
 
+
   return server
 }
 

test/updateTimeToLive.js

@@ -0,0 +1,198 @@
+var helpers = require('./helpers')
+
+var target = 'UpdateTimeToLive',
+    request = helpers.request,
+    opts = helpers.opts.bind(null, target),
+    assertType = helpers.assertType.bind(null, target),
+    assertValidation = helpers.assertValidation.bind(null, target),
+    assertNotFound = helpers.assertNotFound.bind(null, target)
+
+describe('updateTimeToLive', function() {
+
+  describe('serializations', function() {
+
+    it('should return SerializationException when TableName is not a string', function(done) {
+      assertType('TableName', 'String', done)
+    })
+
+    it('should return SerializationException when TimeToLiveSpecification is not a struct', function(done) {
+      assertType('TimeToLiveSpecification', 'FieldStruct<TimeToLiveSpecification>', done)
+    })
+
+    it('should return SerializationException when TimeToLiveSpecification.AttributeName is not a string', function(done) {
+      assertType('TimeToLiveSpecification.AttributeName', 'String', done)
+    })
+
+    it('should return SerializationException when TimeToLiveSpecification.Enabled is not a boolean', function(done) {
+      assertType('TimeToLiveSpecification.Enabled', 'Boolean', done)
+    })
+
+  })
+
+  describe('validations', function() {
+
+    it('should return ValidationException for no TableName', function(done) {
+      assertValidation({},
+        'The parameter \'TableName\' is required but was not present in the request', done)
+    })
+
+    it('should return ValidationException for empty TableName', function(done) {
+      assertValidation({TableName: ''},
+        'TableName must be at least 3 characters long and at most 255 characters long', done)
+    })
+
+    it('should return ValidationException for short TableName', function(done) {
+      assertValidation({TableName: 'a;'},
+        'TableName must be at least 3 characters long and at most 255 characters long', done)
+    })
+
+    it('should return ValidationException for long TableName', function(done) {
+      var name = new Array(256 + 1).join('a')
+      assertValidation({TableName: name},
+        'TableName must be at least 3 characters long and at most 255 characters long', done)
+    })
+
+    it('should return ValidationException for invalid chars', function(done) {
+      assertValidation({TableName: 'abc;'},
+        '1 validation error detected: ' +
+        'Value \'abc;\' at \'tableName\' failed to satisfy constraint: ' +
+        'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', done)
+    })
+
+    it('should return ValidationException for empty TimeToLiveSpecification', function(done) {
+      assertValidation({TableName: 'abc', TimeToLiveSpecification: {}}, [
+        'Value null at \'timeToLiveSpecification.enabled\' failed to satisfy constraint: ' +
+        'Member must not be null',
+        'Value null at \'timeToLiveSpecification.attributeName\' failed to satisfy constraint: ' +
+        'Member must not be null',
+      ], done)
+    })
+
+    it('should return ValidationException for null members in TimeToLiveSpecification', function(done) {
+      assertValidation({TableName: 'abc', TimeToLiveSpecification: {AttributeName: null, Enabled: null}}, [
+        'Value null at \'timeToLiveSpecification.attributeName\' failed to satisfy constraint: ' +
+        'Member must not be null',
+        'Value null at \'timeToLiveSpecification.enabled\' failed to satisfy constraint: ' +
+        'Member must not be null',
+      ], done)
+    })
+
+    it('should return ValidationException for empty TimeToLiveSpecification.AttributeName', function(done) {
+      assertValidation({TableName: 'abc',
+        TimeToLiveSpecification: {AttributeName: "", Enabled: true}},
+        'TimeToLiveSpecification.AttributeName must be non empty', done)
+    })
+
+    it('should return ResourceNotFoundException if table does not exist', function(done) {
+      var name = helpers.randomString()
+      assertNotFound({TableName: name,
+        TimeToLiveSpecification: {AttributeName: "id", Enabled: true}},
+        'Requested resource not found: Table: ' + name + ' not found', done)
+    })
+
+    it('should return ValidationException for false TimeToLiveSpecification.Enabled when already disabled', function(done) {
+      assertValidation({TableName: helpers.testHashTable,
+        TimeToLiveSpecification: {AttributeName: "a", Enabled: false}},
+        'TimeToLive is already disabled', done)
+    })
+
+    it('should return ValidationException for true TimeToLiveSpecification.Enabled when already enabled', function(done) {
+      request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: true}}), function(err, res) {
+        if (err) return done(err)
+        res.statusCode.should.equal(200)
+
+        assertValidation({TableName: helpers.testHashTable,
+          TimeToLiveSpecification: {AttributeName: "a", Enabled: true}},
+          'TimeToLive is already enabled', function(err){
+            if (err) return done(err)
+            // teardown
+            request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: false}}), function(err, res) {
+              if (err) return done(err)
+              res.statusCode.should.equal(200)
+              done()
+            })
+          })
+      })
+    })
+  })
+
+  describe('functionality', function() {
+    it('should enable when disabled', function(done) {
+      request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: true}}), function(err, res) {
+        if (err) return done(err)
+        res.statusCode.should.equal(200)
+        res.body.should.eql({TimeToLiveSpecification: {AttributeName: "a", Enabled: true}})
+
+        request(helpers.opts('DescribeTimeToLive', {TableName: helpers.testHashTable}), function(err, res) {
+          if (err) return done(err)
+          res.statusCode.should.equal(200)
+          res.body.should.eql({TimeToLiveDescription: {TimeToLiveStatus: "ENABLED", AttributeName: "a"}})
+
+          // teardown
+          request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: false}}), function(err, res) {
+            if (err) return done(err)
+            res.statusCode.should.equal(200)
+            done()
+          })
+        })
+      })
+    })
+
+    it('should disable when enabled', function(done) {
+      request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: true}}), function(err, res) {
+        if (err) return done(err)
+        res.statusCode.should.equal(200)
+        res.body.should.eql({TimeToLiveSpecification: {AttributeName: "a", Enabled: true}})
+
+        request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: false}}), function(err, res) {
+          if (err) return done(err)
+          res.statusCode.should.equal(200)
+          res.body.should.eql({TimeToLiveSpecification: {Enabled: false}})
+
+
+          request(helpers.opts('DescribeTimeToLive', {TableName: helpers.testHashTable}), function(err, res) {
+            if (err) return done(err)
+            res.statusCode.should.equal(200)
+            res.body.should.eql({TimeToLiveDescription: {TimeToLiveStatus: "DISABLED"}})
+            done()
+          })
+        })
+      })
+    })
+
+    it('should delete the expired items when TTL is enabled', function(done) {
+      request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "TTL", Enabled: true}}), function(err, res) {
+        if (err) return done(err)
+        res.statusCode.should.equal(200)
+        res.body.should.eql({TimeToLiveSpecification: {AttributeName: "TTL", Enabled: true}})
+
+        var timestampOneSecondLater = Math.round(Date.now() / 1000) + 1;
+        var item = {
+          a: {S: helpers.randomString()},
+          TTL: {N: timestampOneSecondLater.toString()},
+        }
+
+        request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) {
+          if (err) return done(err)
+          res.statusCode.should.equal(200)
+
+          setTimeout(function(){
+            request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: { a: item.a }}), function(err, res) {
+              if (err) return done(err)
+              res.statusCode.should.equal(200)
+              // Item should be deleted
+              res.body.should.eql({})
+
+              // teardown
+              request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "TTL", Enabled: false}}), function(err, res) {
+                if (err) return done(err)
+                res.statusCode.should.equal(200)
+                done()
+              })
+            })
+          }, 3000)
+        })
+      })
+    })
+  })
+})

validations/updateTimeToLive.js

@@ -0,0 +1,30 @@
+exports.types = {
+  TableName: {
+    type: 'String',
+    required: true,
+    tableName: true,
+    regex: '[a-zA-Z0-9_.-]+',
+  },
+  TimeToLiveSpecification: {
+    type: 'FieldStruct<TimeToLiveSpecification>',
+    children: {
+      AttributeName: {
+        type: 'String',
+        required: true,
+        notNull: true,
+      },
+      Enabled: {
+        type: 'Boolean',
+        required: true,
+        notNull: true,
+      },
+    },
+  },
+}
+
+
+exports.custom = function(data) {
+  if (data.TimeToLiveSpecification.AttributeName === '') {
+    return 'TimeToLiveSpecification.AttributeName must be non empty';
+  }
+}