From 0ab33c1695c79728aed8d77ecbd16306192d6894 Mon Sep 17 00:00:00 2001 From: Vivek Menezes Date: Thu, 11 Jun 2015 15:47:00 -0400 Subject: [PATCH 1/3] Keys for the table data --- keys/constants.go | 3 +++ keys/keys.go | 55 ++++++++++++++++++++++++++++++++++++----- keys/keys_test.go | 62 +++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 6 deletions(-) diff --git a/keys/constants.go b/keys/constants.go index bc32e9f3bb11..180539ef4f5c 100644 --- a/keys/constants.go +++ b/keys/constants.go @@ -175,4 +175,7 @@ var ( StatusStorePrefix = MakeKey(StatusPrefix, proto.Key("store-")) // StatusNodePrefix stores all status info for nodes. StatusNodePrefix = MakeKey(StatusPrefix, proto.Key("node-")) + // TableDataPrefix prefixes all Table data to aid in transitioning + // key:value data to Table data, and for ease of debugging. + TableDataPrefix = proto.Key("table-") ) diff --git a/keys/keys.go b/keys/keys.go index ede46a976cf4..b1434148a52d 100644 --- a/keys/keys.go +++ b/keys/keys.go @@ -20,6 +20,7 @@ package keys import ( "bytes" + "encoding/binary" "fmt" "github.com/cockroachdb/cockroach/proto" @@ -27,10 +28,6 @@ import ( "github.com/cockroachdb/cockroach/util/log" ) -const ( - maxVarintSize = 10 -) - // MakeKey makes a new key which is the concatenation of the // given inputs, in order. func MakeKey(keys ...proto.Key) proto.Key { @@ -62,7 +59,7 @@ func NodeStatusKey(nodeID int32) proto.Key { // MakeNameMetadataKey returns the key for the namespace. func MakeNameMetadataKey(parentID uint32, name string) proto.Key { - k := make([]byte, 0, len(NameMetadataPrefix)+maxVarintSize+len(name)) + k := make([]byte, 0, len(NameMetadataPrefix)+binary.MaxVarintLen64+len(name)) k = append(k, NameMetadataPrefix...) k = encoding.EncodeUvarint(k, uint64(parentID)) k = append(k, name...) @@ -71,12 +68,58 @@ func MakeNameMetadataKey(parentID uint32, name string) proto.Key { // MakeDescMetadataKey returns the key for the table in namespaceID. func MakeDescMetadataKey(descID uint32) proto.Key { - k := make([]byte, 0, len(DescMetadataPrefix)+maxVarintSize) + k := make([]byte, 0, len(DescMetadataPrefix)+binary.MaxVarintLen64) k = append(k, DescMetadataPrefix...) k = encoding.EncodeUvarint(k, uint64(descID)) return k } +// indexKeyBufferWidth returns a likely cap on the width of the index key. +// The buffer width can likely accomodate the encoded constant prefix, tableID, +// indexID, and column values. +// +// This cap is inaccurate because the size of the encoding varies, depending +// on the ints and the bytes being encoded. We really don't care, as long as +// a value is chosen such that the append() builtin used to populate the +// buffer, infrequently reallocates more space. +func indexKeyMaxBufferWidth(columnValues ...[]byte) (width int) { + // Accomodate the constant prefix, tableID, and indexID. + width += len(TableDataPrefix) + 2*binary.MaxVarintLen64 + for _, value := range columnValues { + // Add 2 for encoding + width += len(value) + 2 + } + return +} + +// populateTableIndexKey populates the key passed in with the +// order encoded values forming the index key. +func populateTableIndexKey(key []byte, tableID, indexID uint32, columnValues ...[]byte) []byte { + key = append(key, TableDataPrefix...) + key = encoding.EncodeUvarint(key, uint64(tableID)) + key = encoding.EncodeUvarint(key, uint64(indexID)) + for _, value := range columnValues { + key = encoding.EncodeBytes(key, value) + } + return key +} + +// MakeTableIndexKey returns a primary or a secondary index key. +func MakeTableIndexKey(tableID, indexID uint32, columnValues ...[]byte) proto.Key { + k := make([]byte, 0, indexKeyMaxBufferWidth(columnValues...)) + k = populateTableIndexKey(k, tableID, indexID, columnValues...) + return k +} + +// MakeTableDataKey returns a key to a value at a specific row and column +// in the table. +func MakeTableDataKey(tableID, indexID, columnID uint32, columnValues ...[]byte) proto.Key { + k := make([]byte, 0, indexKeyMaxBufferWidth(columnValues...)+binary.MaxVarintLen64) + k = populateTableIndexKey(k, tableID, indexID, columnValues...) + k = encoding.EncodeUvarint(k, uint64(columnID)) + return k +} + // MakeRangeIDKey creates a range-local key based on the range's // Raft ID, metadata key suffix, and optional detail (e.g. the // encoded command ID for a response cache entry, etc.). diff --git a/keys/keys_test.go b/keys/keys_test.go index a67049369f92..36012ba87970 100644 --- a/keys/keys_test.go +++ b/keys/keys_test.go @@ -23,6 +23,7 @@ import ( "github.com/cockroachdb/cockroach/proto" "github.com/cockroachdb/cockroach/util" + "github.com/cockroachdb/cockroach/util/encoding" "github.com/cockroachdb/cockroach/util/leaktest" ) @@ -75,6 +76,67 @@ func TestKeyAddress(t *testing.T) { } } +func TestMakeTableIndexKey(t *testing.T) { + defer leaktest.AfterTest(t) + key := MakeTableIndexKey(12, 345, []byte("foo"), []byte("bar")) + expKey := MakeKey(TableDataPrefix, encoding.EncodeUvarint(nil, 12), encoding.EncodeUvarint(nil, 345), encoding.EncodeBytes(nil, []byte("foo")), encoding.EncodeBytes(nil, []byte("bar"))) + if !key.Equal(expKey) { + t.Errorf("key %q doesn't match expected %q", key, expKey) + } + // Check that keys are ordered + keys := []proto.Key{ + MakeTableIndexKey(0, 0, []byte("foo")), + MakeTableIndexKey(0, 0, []byte("fooo")), + MakeTableIndexKey(0, 1, []byte("bar")), + MakeTableIndexKey(1, 0, []byte("bar")), + MakeTableIndexKey(1, 0, []byte("bar"), []byte("foo")), + MakeTableIndexKey(1, 1, []byte("bar"), []byte("fo")), + MakeTableIndexKey(1, 1, []byte("bar"), []byte("foo")), + MakeTableIndexKey(1, 2, []byte("bar")), + MakeTableIndexKey(2, 2, []byte("ba")), + } + for i := 1; i < len(keys); i++ { + if bytes.Compare(keys[i-1], keys[i]) >= 0 { + t.Errorf("key %d >= key %d", i-1, i) + } + } +} + +func TestMakeTableDataKey(t *testing.T) { + defer leaktest.AfterTest(t) + key := MakeTableDataKey(12, 345, 6, []byte("foo"), []byte("bar")) + // Expected key is the TableIndexKey + ColumnID + expKey := MakeKey(MakeTableIndexKey(12, 345, []byte("foo"), []byte("bar")), encoding.EncodeUvarint(nil, 6)) + if !key.Equal(expKey) { + t.Errorf("key %q doesn't match expected %q", key, expKey) + } + // Check that keys are ordered + keys := []proto.Key{ + // Data-key follows Index key order + MakeTableDataKey(0, 0, 0, []byte("foo")), + MakeTableDataKey(0, 0, 8, []byte("fooo")), + MakeTableDataKey(0, 1, 10, []byte("bar")), + MakeTableDataKey(1, 0, 3, []byte("bar")), + MakeTableDataKey(1, 0, 5, []byte("bar"), []byte("foo")), + MakeTableDataKey(1, 1, 7, []byte("bar"), []byte("fo")), + MakeTableDataKey(1, 1, 4, []byte("bar"), []byte("foo")), + MakeTableDataKey(1, 2, 89, []byte("bar")), + MakeTableDataKey(2, 2, 23, []byte("ba")), + // For the same Index key, Data-key follows column ID order. + MakeTableDataKey(2, 2, 0, []byte("bar"), []byte("foo")), + MakeTableDataKey(2, 2, 7, []byte("bar"), []byte("foo")), + MakeTableDataKey(2, 2, 23, []byte("bar"), []byte("foo")), + MakeTableDataKey(2, 2, 45, []byte("bar"), []byte("foo")), + } + + for i := 1; i < len(keys); i++ { + if bytes.Compare(keys[i-1], keys[i]) >= 0 { + t.Errorf("key %d >= key %d", i-1, i) + } + } + +} + func TestRangeMetaKey(t *testing.T) { defer leaktest.AfterTest(t) testCases := []struct { From 8faf3fd0421441d924f64d77dd910a598d7440ed Mon Sep 17 00:00:00 2001 From: Vivek Menezes Date: Tue, 16 Jun 2015 10:24:41 -0400 Subject: [PATCH 2/3] Added MaxUvarintSize to encoding --- keys/keys.go | 9 ++++----- util/encoding/encoding.go | 5 +++++ 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/keys/keys.go b/keys/keys.go index b1434148a52d..6b8198ab5846 100644 --- a/keys/keys.go +++ b/keys/keys.go @@ -20,7 +20,6 @@ package keys import ( "bytes" - "encoding/binary" "fmt" "github.com/cockroachdb/cockroach/proto" @@ -59,7 +58,7 @@ func NodeStatusKey(nodeID int32) proto.Key { // MakeNameMetadataKey returns the key for the namespace. func MakeNameMetadataKey(parentID uint32, name string) proto.Key { - k := make([]byte, 0, len(NameMetadataPrefix)+binary.MaxVarintLen64+len(name)) + k := make([]byte, 0, len(NameMetadataPrefix)+encoding.MaxUvarintSize+len(name)) k = append(k, NameMetadataPrefix...) k = encoding.EncodeUvarint(k, uint64(parentID)) k = append(k, name...) @@ -68,7 +67,7 @@ func MakeNameMetadataKey(parentID uint32, name string) proto.Key { // MakeDescMetadataKey returns the key for the table in namespaceID. func MakeDescMetadataKey(descID uint32) proto.Key { - k := make([]byte, 0, len(DescMetadataPrefix)+binary.MaxVarintLen64) + k := make([]byte, 0, len(DescMetadataPrefix)+encoding.MaxUvarintSize) k = append(k, DescMetadataPrefix...) k = encoding.EncodeUvarint(k, uint64(descID)) return k @@ -84,7 +83,7 @@ func MakeDescMetadataKey(descID uint32) proto.Key { // buffer, infrequently reallocates more space. func indexKeyMaxBufferWidth(columnValues ...[]byte) (width int) { // Accomodate the constant prefix, tableID, and indexID. - width += len(TableDataPrefix) + 2*binary.MaxVarintLen64 + width += len(TableDataPrefix) + 2*encoding.MaxUvarintSize for _, value := range columnValues { // Add 2 for encoding width += len(value) + 2 @@ -114,7 +113,7 @@ func MakeTableIndexKey(tableID, indexID uint32, columnValues ...[]byte) proto.Ke // MakeTableDataKey returns a key to a value at a specific row and column // in the table. func MakeTableDataKey(tableID, indexID, columnID uint32, columnValues ...[]byte) proto.Key { - k := make([]byte, 0, indexKeyMaxBufferWidth(columnValues...)+binary.MaxVarintLen64) + k := make([]byte, 0, indexKeyMaxBufferWidth(columnValues...)+encoding.MaxUvarintSize) k = populateTableIndexKey(k, tableID, indexID, columnValues...) k = encoding.EncodeUvarint(k, uint64(columnID)) return k diff --git a/util/encoding/encoding.go b/util/encoding/encoding.go index e19bb518c7a7..da300b567d6d 100644 --- a/util/encoding/encoding.go +++ b/util/encoding/encoding.go @@ -287,6 +287,11 @@ func DecodeVarintDecreasing(b []byte) ([]byte, int64) { return leftover, ^v } +const ( + // MaxUvarintSize is the maximum size in bytes of an encoded uvarint. + MaxUvarintSize = 10 +) + // EncodeUvarint encodes the uint64 value using a variable length // (length-prefixed) representation. The length is encoded as a single // byte indicating the number of encoded bytes (-8) to follow. See From 0f06ca505c5f1a8aec33f8ca2e42a59e1ac61fb7 Mon Sep 17 00:00:00 2001 From: Vivek Menezes Date: Tue, 16 Jun 2015 10:33:47 -0400 Subject: [PATCH 3/3] Update MaxUvarintSize to 9 --- util/encoding/encoding.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/encoding/encoding.go b/util/encoding/encoding.go index da300b567d6d..5b53bf462a7c 100644 --- a/util/encoding/encoding.go +++ b/util/encoding/encoding.go @@ -289,7 +289,7 @@ func DecodeVarintDecreasing(b []byte) ([]byte, int64) { const ( // MaxUvarintSize is the maximum size in bytes of an encoded uvarint. - MaxUvarintSize = 10 + MaxUvarintSize = 9 ) // EncodeUvarint encodes the uint64 value using a variable length