Skip to content
This repository has been archived by the owner on Apr 27, 2023. It is now read-only.

Commit

Permalink
Retain user record partition keys (a8m#1)
Browse files Browse the repository at this point in the history
  • Loading branch information
jawang35 committed Aug 22, 2019
1 parent ef47e4c commit 4f21b05
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 9 deletions.
18 changes: 10 additions & 8 deletions aggregator.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,16 @@ func (a *Aggregator) Count() int {

// Put record using `data` and `partitionKey`. This method is thread-safe.
func (a *Aggregator) Put(data []byte, partitionKey string) {
// For now, all records in the aggregated record will have
// the same partition key.
// later, we will add shard-mapper same as the KPL use.
// see: https://github.com/a8m/kinesis-producer/issues/1
if len(a.pkeys) == 0 {
a.pkeys = []string{partitionKey}
a.nbytes += len([]byte(partitionKey))
}
// a8m: For now, all records in the aggregated record will have
// the same partition key.
// later, we will add shard-mapper same as the KPL use.
// see: https://github.com/a8m/kinesis-producer/issues/1
// jawang35: In this fork I'm allowing user records to retain
// their individual partition keys with the
// understanding that shard-mapping is not implemented
// like it is in the KPL.
a.pkeys = append(a.pkeys, partitionKey)
a.nbytes += len([]byte(partitionKey))
keyIndex := uint64(len(a.pkeys) - 1)

a.nbytes += partitionKeyIndexSize
Expand Down
2 changes: 1 addition & 1 deletion aggregator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ func TestSizeAndCount(t *testing.T) {
for i := 0; i < n; i++ {
a.Put(data, pkey)
}
assert(t, a.Size() == 5*n+5+8*n, "size should equal to the data and the partition-key")
assert(t, a.Size() == 5*n+5*n+8*n, "size should equal to the data and the partition-keys")
assert(t, a.Count() == n, "count should be equal to the number of Put calls")
}

Expand Down

0 comments on commit 4f21b05

Please # to comment.