Skip to content

Commit a0dae69

Browse files
exoegoexoego
committedAug 22, 2021
Implement Time-To-Live Feature
1 parent 58d0955 commit a0dae69

File tree

6 files changed

+309
-3
lines changed

6 files changed

+309
-3
lines changed
 

‎actions/describeTimeToLive.js

+7-2
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11

22
module.exports = function describeTimeToLive(store, data, cb) {
3-
store.getTable(data.TableName, false, function(err) {
3+
store.getTable(data.TableName, false, function(err, table) {
44
if (err) return cb(err)
55

6-
cb(null, {TimeToLiveDescription: {TimeToLiveStatus: 'DISABLED'}})
6+
if (table.TimeToLiveDescription !== null && typeof table.TimeToLiveDescription === 'object') {
7+
cb(null, {TimeToLiveDescription: table.TimeToLiveDescription})
8+
} else {
9+
cb(null, {TimeToLiveDescription: {TimeToLiveStatus: 'DISABLED'}})
10+
}
11+
712
})
813
}

‎actions/updateTimeToLive.js

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
var db = require('../db');
2+
3+
module.exports = function updateTimeToLive(store, data, cb) {
4+
var key = data.TableName,
5+
TimeToLiveSpecification = data.TimeToLiveSpecification,
6+
tableDb = store.tableDb,
7+
returnValue;
8+
9+
store.getTable(key, false, function(err, table) {
10+
if (err) return cb(err)
11+
12+
if (TimeToLiveSpecification.Enabled) {
13+
if (table.TimeToLiveDescription && table.TimeToLiveDescription.TimeToLiveStatus === 'ENABLED') {
14+
return cb(db.validationError('TimeToLive is already enabled'))
15+
}
16+
table.TimeToLiveDescription = {
17+
AttributeName: TimeToLiveSpecification.AttributeName,
18+
TimeToLiveStatus: 'ENABLED',
19+
}
20+
returnValue = TimeToLiveSpecification
21+
} else {
22+
if (table.TimeToLiveDescription == null || table.TimeToLiveDescription.TimeToLiveStatus === 'DISABLED') {
23+
return cb(db.validationError('TimeToLive is already disabled'))
24+
}
25+
26+
table.TimeToLiveDescription = {
27+
TimeToLiveStatus: 'DISABLED',
28+
}
29+
returnValue = {Enabled: false}
30+
}
31+
32+
tableDb.put(key, table, function(err) {
33+
if (err) return cb(err)
34+
35+
cb(null, {TimeToLiveSpecification: returnValue})
36+
})
37+
})
38+
}

‎db/index.js

+33
Original file line numberDiff line numberDiff line change
@@ -127,6 +127,38 @@ function create(options) {
127127
})
128128
}
129129

130+
var timerIdTtlScanner = setInterval(function() {
131+
var currentUnixSeconds = Math.round(Date.now() / 1000)
132+
function logError(err, result) {
133+
if (err) console.error("@@@", err)
134+
}
135+
lazyStream(tableDb.createKeyStream({}), logError)
136+
.join(function(tableNames) {
137+
tableNames.forEach(function(name) {
138+
getTable(name, false, function(err, table) {
139+
if (err) return
140+
if (!table.TimeToLiveDescription || table.TimeToLiveDescription.TimeToLiveStatus !== 'ENABLED') return
141+
142+
var itemDb = getItemDb(table.TableName)
143+
var kvStream = lazyStream(itemDb.createReadStream({}), logError())
144+
kvStream = kvStream.filter(function(item){
145+
var ttl = item.value[table.TimeToLiveDescription.AttributeName]
146+
return ttl && typeof ttl.N === 'string' && currentUnixSeconds > Number(ttl.N)
147+
})
148+
kvStream.join(function(items){
149+
items.forEach(function(item) {
150+
itemDb.del(item.key)
151+
})
152+
})
153+
})
154+
})
155+
})
156+
}, 1000)
157+
158+
function stopBackgroundJobs() {
159+
clearInterval(timerIdTtlScanner)
160+
}
161+
130162
return {
131163
options: options,
132164
db: db,
@@ -139,6 +171,7 @@ function create(options) {
139171
deleteTagDb: deleteTagDb,
140172
getTable: getTable,
141173
recreate: recreate,
174+
stopBackgroundJobs: stopBackgroundJobs,
142175
}
143176
}
144177

‎index.js

+3-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ var MAX_REQUEST_BYTES = 16 * 1024 * 1024
1313
var validApis = ['DynamoDB_20111205', 'DynamoDB_20120810'],
1414
validOperations = ['BatchGetItem', 'BatchWriteItem', 'CreateTable', 'DeleteItem', 'DeleteTable',
1515
'DescribeTable', 'DescribeTimeToLive', 'GetItem', 'ListTables', 'PutItem', 'Query', 'Scan', 'TagResource',
16-
'UntagResource', 'ListTagsOfResource', 'UpdateItem', 'UpdateTable'],
16+
'UntagResource', 'ListTagsOfResource', 'UpdateItem', 'UpdateTable', 'UpdateTimeToLive'],
1717
actions = {},
1818
actionValidations = {}
1919

@@ -35,6 +35,7 @@ function dynalite(options) {
3535
// Ensure we close DB when we're closing the server too
3636
var httpServerClose = server.close, httpServerListen = server.listen
3737
server.close = function(cb) {
38+
store.stopBackgroundJobs()
3839
store.db.close(function(err) {
3940
if (err) return cb(err)
4041
// Recreate the store if the user wants to listen again
@@ -46,6 +47,7 @@ function dynalite(options) {
4647
})
4748
}
4849

50+
4951
return server
5052
}
5153

‎test/updateTimeToLive.js

+198
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
var helpers = require('./helpers')
2+
3+
var target = 'UpdateTimeToLive',
4+
request = helpers.request,
5+
opts = helpers.opts.bind(null, target),
6+
assertType = helpers.assertType.bind(null, target),
7+
assertValidation = helpers.assertValidation.bind(null, target),
8+
assertNotFound = helpers.assertNotFound.bind(null, target)
9+
10+
describe('updateTimeToLive', function() {
11+
12+
describe('serializations', function() {
13+
14+
it('should return SerializationException when TableName is not a string', function(done) {
15+
assertType('TableName', 'String', done)
16+
})
17+
18+
it('should return SerializationException when TimeToLiveSpecification is not a struct', function(done) {
19+
assertType('TimeToLiveSpecification', 'FieldStruct<TimeToLiveSpecification>', done)
20+
})
21+
22+
it('should return SerializationException when TimeToLiveSpecification.AttributeName is not a string', function(done) {
23+
assertType('TimeToLiveSpecification.AttributeName', 'String', done)
24+
})
25+
26+
it('should return SerializationException when TimeToLiveSpecification.Enabled is not a boolean', function(done) {
27+
assertType('TimeToLiveSpecification.Enabled', 'Boolean', done)
28+
})
29+
30+
})
31+
32+
describe('validations', function() {
33+
34+
it('should return ValidationException for no TableName', function(done) {
35+
assertValidation({},
36+
'The parameter \'TableName\' is required but was not present in the request', done)
37+
})
38+
39+
it('should return ValidationException for empty TableName', function(done) {
40+
assertValidation({TableName: ''},
41+
'TableName must be at least 3 characters long and at most 255 characters long', done)
42+
})
43+
44+
it('should return ValidationException for short TableName', function(done) {
45+
assertValidation({TableName: 'a;'},
46+
'TableName must be at least 3 characters long and at most 255 characters long', done)
47+
})
48+
49+
it('should return ValidationException for long TableName', function(done) {
50+
var name = new Array(256 + 1).join('a')
51+
assertValidation({TableName: name},
52+
'TableName must be at least 3 characters long and at most 255 characters long', done)
53+
})
54+
55+
it('should return ValidationException for invalid chars', function(done) {
56+
assertValidation({TableName: 'abc;'},
57+
'1 validation error detected: ' +
58+
'Value \'abc;\' at \'tableName\' failed to satisfy constraint: ' +
59+
'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', done)
60+
})
61+
62+
it('should return ValidationException for empty TimeToLiveSpecification', function(done) {
63+
assertValidation({TableName: 'abc', TimeToLiveSpecification: {}}, [
64+
'Value null at \'timeToLiveSpecification.enabled\' failed to satisfy constraint: ' +
65+
'Member must not be null',
66+
'Value null at \'timeToLiveSpecification.attributeName\' failed to satisfy constraint: ' +
67+
'Member must not be null',
68+
], done)
69+
})
70+
71+
it('should return ValidationException for null members in TimeToLiveSpecification', function(done) {
72+
assertValidation({TableName: 'abc', TimeToLiveSpecification: {AttributeName: null, Enabled: null}}, [
73+
'Value null at \'timeToLiveSpecification.attributeName\' failed to satisfy constraint: ' +
74+
'Member must not be null',
75+
'Value null at \'timeToLiveSpecification.enabled\' failed to satisfy constraint: ' +
76+
'Member must not be null',
77+
], done)
78+
})
79+
80+
it('should return ValidationException for empty TimeToLiveSpecification.AttributeName', function(done) {
81+
assertValidation({TableName: 'abc',
82+
TimeToLiveSpecification: {AttributeName: "", Enabled: true}},
83+
'TimeToLiveSpecification.AttributeName must be non empty', done)
84+
})
85+
86+
it('should return ResourceNotFoundException if table does not exist', function(done) {
87+
var name = helpers.randomString()
88+
assertNotFound({TableName: name,
89+
TimeToLiveSpecification: {AttributeName: "id", Enabled: true}},
90+
'Requested resource not found: Table: ' + name + ' not found', done)
91+
})
92+
93+
it('should return ValidationException for false TimeToLiveSpecification.Enabled when already disabled', function(done) {
94+
assertValidation({TableName: helpers.testHashTable,
95+
TimeToLiveSpecification: {AttributeName: "a", Enabled: false}},
96+
'TimeToLive is already disabled', done)
97+
})
98+
99+
it('should return ValidationException for true TimeToLiveSpecification.Enabled when already enabled', function(done) {
100+
request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: true}}), function(err, res) {
101+
if (err) return done(err)
102+
res.statusCode.should.equal(200)
103+
104+
assertValidation({TableName: helpers.testHashTable,
105+
TimeToLiveSpecification: {AttributeName: "a", Enabled: true}},
106+
'TimeToLive is already enabled', function(err){
107+
if (err) return done(err)
108+
// teardown
109+
request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: false}}), function(err, res) {
110+
if (err) return done(err)
111+
res.statusCode.should.equal(200)
112+
done()
113+
})
114+
})
115+
})
116+
})
117+
})
118+
119+
describe('functionality', function() {
120+
it('should enable when disabled', function(done) {
121+
request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: true}}), function(err, res) {
122+
if (err) return done(err)
123+
res.statusCode.should.equal(200)
124+
res.body.should.eql({TimeToLiveSpecification: {AttributeName: "a", Enabled: true}})
125+
126+
request(helpers.opts('DescribeTimeToLive', {TableName: helpers.testHashTable}), function(err, res) {
127+
if (err) return done(err)
128+
res.statusCode.should.equal(200)
129+
res.body.should.eql({TimeToLiveDescription: {TimeToLiveStatus: "ENABLED", AttributeName: "a"}})
130+
131+
// teardown
132+
request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: false}}), function(err, res) {
133+
if (err) return done(err)
134+
res.statusCode.should.equal(200)
135+
done()
136+
})
137+
})
138+
})
139+
})
140+
141+
it('should disable when enabled', function(done) {
142+
request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: true}}), function(err, res) {
143+
if (err) return done(err)
144+
res.statusCode.should.equal(200)
145+
res.body.should.eql({TimeToLiveSpecification: {AttributeName: "a", Enabled: true}})
146+
147+
request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "a", Enabled: false}}), function(err, res) {
148+
if (err) return done(err)
149+
res.statusCode.should.equal(200)
150+
res.body.should.eql({TimeToLiveSpecification: {Enabled: false}})
151+
152+
153+
request(helpers.opts('DescribeTimeToLive', {TableName: helpers.testHashTable}), function(err, res) {
154+
if (err) return done(err)
155+
res.statusCode.should.equal(200)
156+
res.body.should.eql({TimeToLiveDescription: {TimeToLiveStatus: "DISABLED"}})
157+
done()
158+
})
159+
})
160+
})
161+
})
162+
163+
it('should delete the expired items when TTL is enabled', function(done) {
164+
request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "TTL", Enabled: true}}), function(err, res) {
165+
if (err) return done(err)
166+
res.statusCode.should.equal(200)
167+
res.body.should.eql({TimeToLiveSpecification: {AttributeName: "TTL", Enabled: true}})
168+
169+
var timestampOneSecondLater = Math.round(Date.now() / 1000) + 1;
170+
var item = {
171+
a: {S: helpers.randomString()},
172+
TTL: {N: timestampOneSecondLater.toString()},
173+
}
174+
175+
request(helpers.opts('PutItem', {TableName: helpers.testHashTable, Item: item}), function(err, res) {
176+
if (err) return done(err)
177+
res.statusCode.should.equal(200)
178+
179+
setTimeout(function(){
180+
request(helpers.opts('GetItem', {TableName: helpers.testHashTable, Key: { a: item.a }}), function(err, res) {
181+
if (err) return done(err)
182+
res.statusCode.should.equal(200)
183+
// Item should be deleted
184+
res.body.should.eql({})
185+
186+
// teardown
187+
request(opts({TableName: helpers.testHashTable, TimeToLiveSpecification: {AttributeName: "TTL", Enabled: false}}), function(err, res) {
188+
if (err) return done(err)
189+
res.statusCode.should.equal(200)
190+
done()
191+
})
192+
})
193+
}, 3000)
194+
})
195+
})
196+
})
197+
})
198+
})

‎validations/updateTimeToLive.js

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
exports.types = {
2+
TableName: {
3+
type: 'String',
4+
required: true,
5+
tableName: true,
6+
regex: '[a-zA-Z0-9_.-]+',
7+
},
8+
TimeToLiveSpecification: {
9+
type: 'FieldStruct<TimeToLiveSpecification>',
10+
children: {
11+
AttributeName: {
12+
type: 'String',
13+
required: true,
14+
notNull: true,
15+
},
16+
Enabled: {
17+
type: 'Boolean',
18+
required: true,
19+
notNull: true,
20+
},
21+
},
22+
},
23+
}
24+
25+
26+
exports.custom = function(data) {
27+
if (data.TimeToLiveSpecification.AttributeName === '') {
28+
return 'TimeToLiveSpecification.AttributeName must be non empty';
29+
}
30+
}

0 commit comments

Comments
 (0)