From 02f228bac9f17646a16633c7251bdd6af9b9b507 Mon Sep 17 00:00:00 2001 From: qupeng Date: Wed, 21 Jun 2017 18:05:10 +0800 Subject: [PATCH 1/2] codec: simplify json codec. --- util/codec/codec.go | 15 ++++++--------- util/types/json/json_test.go | 22 +++++++++++++-------- util/types/json/serdes.go | 37 ++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 17 deletions(-) diff --git a/util/codec/codec.go b/util/codec/codec.go index ee18e5baa59f8..b035418b1c651 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -80,9 +80,8 @@ func encode(b []byte, vals []types.Datum, comparable bool) ([]byte, error) { case types.KindMysqlSet: b = encodeUnsignedInt(b, uint64(val.GetMysqlSet().ToNumber()), comparable) case types.KindMysqlJSON: - bytes := json.Serialize(val.GetMysqlJSON()) b = append(b, jsonFlag) - b = EncodeCompactBytes(b, bytes) + b = append(b, json.Serialize(val.GetMysqlJSON())...) case types.KindNull: b = append(b, NilFlag) case types.KindMinNotNull: @@ -216,12 +215,8 @@ func DecodeOne(b []byte) (remain []byte, d types.Datum, err error) { d.SetValue(v) } case jsonFlag: - var v []byte - if b, v, err = DecodeCompactBytes(b); err == nil { - var j json.JSON - if j, err = json.Deserialize(v); err == nil { - d.SetMysqlJSON(j) - } + if j, err := json.Deserialize(b); err == nil { + d.SetMysqlJSON(j) } case NilFlag: default: @@ -272,7 +267,7 @@ func peek(b []byte) (length int, err error) { l = 8 case bytesFlag: l, err = peekBytes(b, false) - case compactBytesFlag, jsonFlag: + case compactBytesFlag: l, err = peekCompactBytes(b) case decimalFlag: l, err = types.DecimalPeak(b) @@ -280,6 +275,8 @@ func peek(b []byte) (length int, err error) { l, err = peekVarint(b) case uvarintFlag: l, err = peekUvarint(b) + case jsonFlag: + l, err = json.PeekBytesAsJSON(b) default: return 0, errors.Errorf("invalid encoded key flag %v", flag) } diff --git a/util/types/json/json_test.go b/util/types/json/json_test.go index 9c4b972938d08..b374e4b1a5ff7 100644 --- a/util/types/json/json_test.go +++ b/util/types/json/json_test.go @@ -54,15 +54,16 @@ func (s *testJSONSuite) TestSerializeAndDeserialize(c *C) { j2 := mustParseFromString(`[{"a": 1, "b": true}, 3, 3.5, "hello, world", null, true]`) var testcses = []struct { - In JSON - Out JSON + In JSON + Out JSON + size int }{ - {In: jsonNilValue, Out: jsonNilValue}, - {In: jsonBoolValue, Out: jsonBoolValue}, - {In: jsonDoubleValue, Out: jsonDoubleValue}, - {In: jsonStringValue, Out: jsonStringValue}, - {In: j1, Out: j1}, - {In: j2, Out: j2}, + {In: jsonNilValue, Out: jsonNilValue, size: 2}, + {In: jsonBoolValue, Out: jsonBoolValue, size: 2}, + {In: jsonDoubleValue, Out: jsonDoubleValue, size: 9}, + {In: jsonStringValue, Out: jsonStringValue, size: 15}, + {In: j1, Out: j1, size: 144}, + {In: j2, Out: j2, size: 108}, } for _, s := range testcses { @@ -73,6 +74,11 @@ func (s *testJSONSuite) TestSerializeAndDeserialize(c *C) { v1 := t.String() v2 := s.Out.String() c.Assert(v1, Equals, v2) + + size, err := PeekBytesAsJSON(data) + c.Assert(err, IsNil) + c.Assert(len(data), Equals, size) + c.Assert(len(data), Equals, s.size) } } diff --git a/util/types/json/serdes.go b/util/types/json/serdes.go index 68365cde7bdc6..c41ccb4faa8e4 100644 --- a/util/types/json/serdes.go +++ b/util/types/json/serdes.go @@ -93,6 +93,43 @@ import ( // lengths up to 16383, and so on... */ +const ( + typeCodeLen int = 1 + compoundCountLen = 4 + compoundSizeLen = 4 +) + +// PeekBytesAsJSON trys to peek some bytes from b, until +// we can deserialize a JSON from those bytes. +func PeekBytesAsJSON(b []byte) (n int, err error) { + if len(b) > 0 { + switch c := b[0]; c { + case byte(typeCodeObject), byte(typeCodeArray): + if len(b) >= typeCodeLen+compoundCountLen+compoundSizeLen { + var size uint32 + start := typeCodeLen + compoundCountLen + end := typeCodeLen + compoundCountLen + compoundSizeLen + binary.Read(bytes.NewReader(b[start:end]), binary.LittleEndian, &size) + n = int(size) + typeCodeLen + return + } + case byte(typeCodeString): + var size uint64 + reader := bytes.NewReader(b[typeCodeLen:]) + size, err = binary.ReadUvarint(reader) + if err == nil { + n = int(size) + int(reader.Size()) - int(reader.Len()) + typeCodeLen + return + } + case byte(typeCodeInt64), byte(typeCodeFloat64), byte(typeCodeLiteral): + n = jsonTypeCodeLength[TypeCode(c)] + typeCodeLen + return + } + } + err = errors.New("Invalid JSON bytes") + return +} + // Serialize means serialize itself into bytes. func Serialize(j JSON) []byte { var buffer = new(bytes.Buffer) From 4ceee88deb7b67569e5a9c929e363f0cd4eb74fd Mon Sep 17 00:00:00 2001 From: qupeng Date: Fri, 23 Jun 2017 12:00:29 +0800 Subject: [PATCH 2/2] address comments. --- util/types/json/json_test.go | 4 +-- util/types/json/serdes.go | 69 +++++++++++++++++++----------------- 2 files changed, 39 insertions(+), 34 deletions(-) diff --git a/util/types/json/json_test.go b/util/types/json/json_test.go index b374e4b1a5ff7..8eab7001aab07 100644 --- a/util/types/json/json_test.go +++ b/util/types/json/json_test.go @@ -53,7 +53,7 @@ func (s *testJSONSuite) TestSerializeAndDeserialize(c *C) { j1 := mustParseFromString(`{"aaaaaaaaaaa": [1, "2", {"aa": "bb"}, 4.0], "bbbbbbbbbb": true, "ccccccccc": "d"}`) j2 := mustParseFromString(`[{"a": 1, "b": true}, 3, 3.5, "hello, world", null, true]`) - var testcses = []struct { + var testcases = []struct { In JSON Out JSON size int @@ -66,7 +66,7 @@ func (s *testJSONSuite) TestSerializeAndDeserialize(c *C) { {In: j2, Out: j2, size: 108}, } - for _, s := range testcses { + for _, s := range testcases { data := Serialize(s.In) t, err := Deserialize(data) c.Assert(err, IsNil) diff --git a/util/types/json/serdes.go b/util/types/json/serdes.go index c41ccb4faa8e4..4fd07a3adef89 100644 --- a/util/types/json/serdes.go +++ b/util/types/json/serdes.go @@ -97,34 +97,39 @@ const ( typeCodeLen int = 1 compoundCountLen = 4 compoundSizeLen = 4 + keyOffsetLen = 4 + keyLengthLen = 2 + valueInlineLen = 4 ) // PeekBytesAsJSON trys to peek some bytes from b, until // we can deserialize a JSON from those bytes. func PeekBytesAsJSON(b []byte) (n int, err error) { - if len(b) > 0 { - switch c := b[0]; c { - case byte(typeCodeObject), byte(typeCodeArray): - if len(b) >= typeCodeLen+compoundCountLen+compoundSizeLen { - var size uint32 - start := typeCodeLen + compoundCountLen - end := typeCodeLen + compoundCountLen + compoundSizeLen - binary.Read(bytes.NewReader(b[start:end]), binary.LittleEndian, &size) - n = int(size) + typeCodeLen - return - } - case byte(typeCodeString): - var size uint64 - reader := bytes.NewReader(b[typeCodeLen:]) - size, err = binary.ReadUvarint(reader) - if err == nil { - n = int(size) + int(reader.Size()) - int(reader.Len()) + typeCodeLen - return - } - case byte(typeCodeInt64), byte(typeCodeFloat64), byte(typeCodeLiteral): - n = jsonTypeCodeLength[TypeCode(c)] + typeCodeLen + if len(b) <= 0 { + err = errors.New("Cant peek from empty bytes") + return + } + switch c := TypeCode(b[0]); c { + case typeCodeObject, typeCodeArray: + if len(b) >= typeCodeLen+compoundCountLen+compoundSizeLen { + var size uint32 + start := typeCodeLen + compoundCountLen + end := typeCodeLen + compoundCountLen + compoundSizeLen + binary.Read(bytes.NewReader(b[start:end]), binary.LittleEndian, &size) + n = int(size) + typeCodeLen + return + } + case typeCodeString: + var size uint64 + reader := bytes.NewReader(b[typeCodeLen:]) + size, err = binary.ReadUvarint(reader) + if err == nil { + n = int(size) + int(reader.Size()) - int(reader.Len()) + typeCodeLen return } + case typeCodeInt64, typeCodeFloat64, typeCodeLiteral: + n = jsonTypeCodeLength[TypeCode(c)] + typeCodeLen + return } err = errors.New("Invalid JSON bytes") return @@ -239,11 +244,11 @@ func encodeJSONObject(m map[string]JSON, buffer *bytes.Buffer) { // object ::= element-count size key-entry* value-entry* key* value* // key-entry ::= key-offset key-length var countAndSize = make([]uint32, 2) - var countAndSizeLen = len(countAndSize) * 4 + var countAndSizeLen = compoundCountLen + compoundSizeLen var keySlice = getSortedKeys(m) - var keyEntrysLen = (4 + 2) * len(m) - var valueEntrysLen = (1 + 4) * len(m) + var keyEntrysLen = (keyOffsetLen + keyLengthLen) * len(m) + var valueEntrysLen = (typeCodeLen + valueInlineLen) * len(m) var keyEntrys = new(bytes.Buffer) var valueEntrys = new(bytes.Buffer) var keys = new(bytes.Buffer) @@ -305,9 +310,9 @@ func decodeJSONObject(m *map[string]JSON, data []byte) (err error) { var key = string(keyBuffer) var value JSON typeLen, _ := jsonTypeCodeLength[TypeCode(valueTypes[i])] - if typeLen >= 0 && typeLen <= 4 { + if typeLen >= 0 && typeLen <= valueInlineLen { var inline = valueOffsets[i] - var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: 4, Cap: 4} + var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: valueInlineLen, Cap: valueInlineLen} var buf = *(*[]byte)(unsafe.Pointer(&hdr)) value, err = decode(valueTypes[i], buf) } else { @@ -324,9 +329,9 @@ func decodeJSONObject(m *map[string]JSON, data []byte) (err error) { func encodeJSONArray(a []JSON, buffer *bytes.Buffer) { // array ::= element-count size value-entry* value* var countAndSize = make([]uint32, 2) - var countAndSizeLen = len(countAndSize) * 4 + var countAndSizeLen = compoundCountLen + compoundSizeLen - var valueEntrysLen = (1 + 4) * len(a) + var valueEntrysLen = (typeCodeLen + valueInlineLen) * len(a) var valueEntrys = new(bytes.Buffer) var values = new(bytes.Buffer) for _, value := range a { @@ -360,9 +365,9 @@ func decodeJSONArray(a *[]JSON, data []byte) (err error) { for i := 0; i < int(countAndSize[0]); i++ { var value JSON typeLen, _ := jsonTypeCodeLength[TypeCode(valueTypes[i])] - if typeLen >= 0 && typeLen <= 4 { + if typeLen >= 0 && typeLen <= valueInlineLen { var inline = valueOffsets[i] - var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: 4, Cap: 4} + var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: valueInlineLen, Cap: valueInlineLen} var buf = *(*[]byte)(unsafe.Pointer(&hdr)) value, err = decode(valueTypes[i], buf) } else { @@ -402,7 +407,7 @@ func pushValueEntry(value JSON, valueEntrys *bytes.Buffer, values *bytes.Buffer, valueEntrys.WriteByte(byte(typeCode)) typeLen, _ := jsonTypeCodeLength[typeCode] - if typeLen > 0 && typeLen <= 4 { + if typeLen > 0 && typeLen <= valueInlineLen { // If the value has length in (0, 4], it could be inline here. // And padding 0x00 to 4 bytes if needed. pushInlineValue(valueEntrys, value) @@ -427,7 +432,7 @@ func pushInlineValue(buffer *bytes.Buffer, value JSON) { panic(msg) } var newLen = buffer.Len() - for i := 0; i < 4-(newLen-oldLen); i++ { + for i := 0; i < valueInlineLen-(newLen-oldLen); i++ { buffer.WriteByte(0x00) } }