Skip to content
This repository has been archived by the owner on Jul 17, 2019. It is now read-only.

Commit

Permalink
extend tests to use item with a range key.
Browse files Browse the repository at this point in the history 
This exposes a bug (previously filed as mapbox#90) which occurs when items with a range key are read
from the DDB event stream, an md5 hash of the key is computed and the item written to S3.

The issue is that the DDB event stream handler does not (and should not) do a 'describe_table' to
know which key is the HASH and which is the RANGE and therefor simply generates the md5 hash of
the item keys in whatever order they happen to appear in the stream event.
The s3-backfill util does do a 'describe_table' and does order the keys by declaration
order which DDB requires to be HASH first, RANGE second.

The different ordering of the item keys will produce a distinct md5 hash value and different S3
path/keys will result in some items appearing twice in S3 effectively corrupting the incremental
backups since two valid versions will be present at the same time.
  • Loading branch information
@btalbot
btalbot committed May 9, 2017
1 parent 89b4fd5 commit 0c065a5
Show file tree
Hide file tree
Showing 9 changed files with 144 additions and 43 deletions.
79 changes: 65 additions & 14 deletions test/fixtures/events/adjust-many.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -19,6 +22,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand All @@ -32,8 +38,11 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"N": "11"
"data": {
"S": "data-2"
},
"arange": {
"N": "1"
},
"id": {
"S": "record-1"
Expand All @@ -43,7 +52,10 @@
"StreamViewType":"NEW_AND_OLD_IMAGES",
"SequenceNumber":"222",
"OldImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -53,6 +65,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand All @@ -66,7 +81,10 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -79,6 +97,9 @@
"Keys":{
"id": {
"S": "record-3"
},
"arange": {
"N": "1"
}
}
},
Expand All @@ -92,7 +113,10 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -105,6 +129,9 @@
"Keys":{
"id": {
"S": "record-2"
},
"arange": {
"N": "1"
}
}
},
Expand All @@ -118,8 +145,11 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"N": "22"
"data": {
"S": "data-2"
},
"arange": {
"N": "1"
},
"id": {
"S": "record-2"
Expand All @@ -129,7 +159,10 @@
"StreamViewType":"NEW_AND_OLD_IMAGES",
"SequenceNumber":"222",
"OldImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -139,6 +172,9 @@
"Keys":{
"id": {
"S": "record-2"
},
"arange": {
"N": "1"
}
}
},
Expand All @@ -155,8 +191,11 @@
"StreamViewType":"NEW_AND_OLD_IMAGES",
"SequenceNumber":"333",
"OldImage":{
"range": {
"N": "11"
"data": {
"S": "data-2"
},
"arange": {
"N": "1"
},
"id": {
"S": "record-1"
Expand All @@ -165,6 +204,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand All @@ -178,8 +220,11 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"N": "33"
"data": {
"S": "data-2"
},
"arange": {
"N": "1"
},
"id": {
"S": "record-3"
Expand All @@ -189,7 +234,10 @@
"StreamViewType":"NEW_AND_OLD_IMAGES",
"SequenceNumber":"222",
"OldImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -199,6 +247,9 @@
"Keys":{
"id": {
"S": "record-3"
},
"arange": {
"N": "1"
}
}
},
Expand Down
5 changes: 4 additions & 1 deletion test/fixtures/events/insert-buffer.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"arange": {
"N": "1"
},
"id": {
Expand Down Expand Up @@ -44,6 +44,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand Down
33 changes: 27 additions & 6 deletions test/fixtures/events/insert-modify-delete.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -19,6 +22,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand All @@ -32,8 +38,11 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"N": "2"
"data": {
"S": "data-2"
},
"arange": {
"N": "1"
},
"id": {
"S": "record-1"
Expand All @@ -43,7 +52,10 @@
"StreamViewType":"NEW_AND_OLD_IMAGES",
"SequenceNumber":"222",
"OldImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -53,6 +65,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand All @@ -69,8 +84,11 @@
"StreamViewType":"NEW_AND_OLD_IMAGES",
"SequenceNumber":"333",
"OldImage":{
"range": {
"N": "2"
"data": {
"S": "data-2"
},
"arange": {
"N": "1"
},
"id": {
"S": "record-1"
Expand All @@ -79,6 +97,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand Down
23 changes: 19 additions & 4 deletions test/fixtures/events/insert-modify.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -19,6 +22,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand All @@ -32,8 +38,11 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"N": "2"
"data": {
"S": "data-2"
},
"arange": {
"N": "1"
},
"id": {
"S": "record-1"
Expand All @@ -43,7 +52,10 @@
"StreamViewType":"NEW_AND_OLD_IMAGES",
"SequenceNumber":"222",
"OldImage":{
"range": {
"data": {
"S": "data-1"
},
"arange": {
"N": "1"
},
"id": {
Expand All @@ -53,6 +65,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand Down
5 changes: 4 additions & 1 deletion test/fixtures/events/insert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"eventSource":"aws:dynamodb",
"dynamodb": {
"NewImage":{
"range": {
"arange": {
"N": "1"
},
"id": {
Expand All @@ -19,6 +19,9 @@
"Keys":{
"id": {
"S": "record-1"
},
"arange": {
"N": "1"
}
}
},
Expand Down
1 change: 1 addition & 0 deletions test/fixtures/records.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ module.exports = function(num) {
return _.range(0, num).map(function() {
return {
id: crypto.randomBytes(16).toString('hex'),
arange: Math.random(),
data: crypto.randomBytes(256).toString('base64')
};
});
Expand Down
6 changes: 4 additions & 2 deletions test/fixtures/table.js
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
module.exports = {
AttributeDefinitions: [
{AttributeName: 'id', AttributeType: 'S'}
{AttributeName: 'id', AttributeType: 'S'},
{AttributeName: 'arange', AttributeType: 'N'}
],
KeySchema: [
{AttributeName: 'id', KeyType: 'HASH'}
{AttributeName: 'id', KeyType: 'HASH'},
{AttributeName: 'arange', KeyType: 'RANGE'}
],
ProvisionedThroughput: {
ReadCapacityUnits: 1,
Expand Down
Loading

0 comments on commit 0c065a5

Please sign in to comment.