diff --git a/util/codec/codec.go b/util/codec/codec.go index ee18e5baa59f8..b035418b1c651 100644 --- a/util/codec/codec.go +++ b/util/codec/codec.go @@ -80,9 +80,8 @@ func encode(b []byte, vals []types.Datum, comparable bool) ([]byte, error) { case types.KindMysqlSet: b = encodeUnsignedInt(b, uint64(val.GetMysqlSet().ToNumber()), comparable) case types.KindMysqlJSON: - bytes := json.Serialize(val.GetMysqlJSON()) b = append(b, jsonFlag) - b = EncodeCompactBytes(b, bytes) + b = append(b, json.Serialize(val.GetMysqlJSON())...) case types.KindNull: b = append(b, NilFlag) case types.KindMinNotNull: @@ -216,12 +215,8 @@ func DecodeOne(b []byte) (remain []byte, d types.Datum, err error) { d.SetValue(v) } case jsonFlag: - var v []byte - if b, v, err = DecodeCompactBytes(b); err == nil { - var j json.JSON - if j, err = json.Deserialize(v); err == nil { - d.SetMysqlJSON(j) - } + if j, err := json.Deserialize(b); err == nil { + d.SetMysqlJSON(j) } case NilFlag: default: @@ -272,7 +267,7 @@ func peek(b []byte) (length int, err error) { l = 8 case bytesFlag: l, err = peekBytes(b, false) - case compactBytesFlag, jsonFlag: + case compactBytesFlag: l, err = peekCompactBytes(b) case decimalFlag: l, err = types.DecimalPeak(b) @@ -280,6 +275,8 @@ func peek(b []byte) (length int, err error) { l, err = peekVarint(b) case uvarintFlag: l, err = peekUvarint(b) + case jsonFlag: + l, err = json.PeekBytesAsJSON(b) default: return 0, errors.Errorf("invalid encoded key flag %v", flag) } diff --git a/util/types/json/json_test.go b/util/types/json/json_test.go index 9c4b972938d08..8eab7001aab07 100644 --- a/util/types/json/json_test.go +++ b/util/types/json/json_test.go @@ -53,19 +53,20 @@ func (s *testJSONSuite) TestSerializeAndDeserialize(c *C) { j1 := mustParseFromString(`{"aaaaaaaaaaa": [1, "2", {"aa": "bb"}, 4.0], "bbbbbbbbbb": true, "ccccccccc": "d"}`) j2 := mustParseFromString(`[{"a": 1, "b": true}, 3, 3.5, "hello, world", null, true]`) - var testcses = []struct { - In JSON - Out JSON + var testcases = []struct { + In JSON + Out JSON + size int }{ - {In: jsonNilValue, Out: jsonNilValue}, - {In: jsonBoolValue, Out: jsonBoolValue}, - {In: jsonDoubleValue, Out: jsonDoubleValue}, - {In: jsonStringValue, Out: jsonStringValue}, - {In: j1, Out: j1}, - {In: j2, Out: j2}, + {In: jsonNilValue, Out: jsonNilValue, size: 2}, + {In: jsonBoolValue, Out: jsonBoolValue, size: 2}, + {In: jsonDoubleValue, Out: jsonDoubleValue, size: 9}, + {In: jsonStringValue, Out: jsonStringValue, size: 15}, + {In: j1, Out: j1, size: 144}, + {In: j2, Out: j2, size: 108}, } - for _, s := range testcses { + for _, s := range testcases { data := Serialize(s.In) t, err := Deserialize(data) c.Assert(err, IsNil) @@ -73,6 +74,11 @@ func (s *testJSONSuite) TestSerializeAndDeserialize(c *C) { v1 := t.String() v2 := s.Out.String() c.Assert(v1, Equals, v2) + + size, err := PeekBytesAsJSON(data) + c.Assert(err, IsNil) + c.Assert(len(data), Equals, size) + c.Assert(len(data), Equals, s.size) } } diff --git a/util/types/json/serdes.go b/util/types/json/serdes.go index 68365cde7bdc6..4fd07a3adef89 100644 --- a/util/types/json/serdes.go +++ b/util/types/json/serdes.go @@ -93,6 +93,48 @@ import ( // lengths up to 16383, and so on... */ +const ( + typeCodeLen int = 1 + compoundCountLen = 4 + compoundSizeLen = 4 + keyOffsetLen = 4 + keyLengthLen = 2 + valueInlineLen = 4 +) + +// PeekBytesAsJSON trys to peek some bytes from b, until +// we can deserialize a JSON from those bytes. +func PeekBytesAsJSON(b []byte) (n int, err error) { + if len(b) <= 0 { + err = errors.New("Cant peek from empty bytes") + return + } + switch c := TypeCode(b[0]); c { + case typeCodeObject, typeCodeArray: + if len(b) >= typeCodeLen+compoundCountLen+compoundSizeLen { + var size uint32 + start := typeCodeLen + compoundCountLen + end := typeCodeLen + compoundCountLen + compoundSizeLen + binary.Read(bytes.NewReader(b[start:end]), binary.LittleEndian, &size) + n = int(size) + typeCodeLen + return + } + case typeCodeString: + var size uint64 + reader := bytes.NewReader(b[typeCodeLen:]) + size, err = binary.ReadUvarint(reader) + if err == nil { + n = int(size) + int(reader.Size()) - int(reader.Len()) + typeCodeLen + return + } + case typeCodeInt64, typeCodeFloat64, typeCodeLiteral: + n = jsonTypeCodeLength[TypeCode(c)] + typeCodeLen + return + } + err = errors.New("Invalid JSON bytes") + return +} + // Serialize means serialize itself into bytes. func Serialize(j JSON) []byte { var buffer = new(bytes.Buffer) @@ -202,11 +244,11 @@ func encodeJSONObject(m map[string]JSON, buffer *bytes.Buffer) { // object ::= element-count size key-entry* value-entry* key* value* // key-entry ::= key-offset key-length var countAndSize = make([]uint32, 2) - var countAndSizeLen = len(countAndSize) * 4 + var countAndSizeLen = compoundCountLen + compoundSizeLen var keySlice = getSortedKeys(m) - var keyEntrysLen = (4 + 2) * len(m) - var valueEntrysLen = (1 + 4) * len(m) + var keyEntrysLen = (keyOffsetLen + keyLengthLen) * len(m) + var valueEntrysLen = (typeCodeLen + valueInlineLen) * len(m) var keyEntrys = new(bytes.Buffer) var valueEntrys = new(bytes.Buffer) var keys = new(bytes.Buffer) @@ -268,9 +310,9 @@ func decodeJSONObject(m *map[string]JSON, data []byte) (err error) { var key = string(keyBuffer) var value JSON typeLen, _ := jsonTypeCodeLength[TypeCode(valueTypes[i])] - if typeLen >= 0 && typeLen <= 4 { + if typeLen >= 0 && typeLen <= valueInlineLen { var inline = valueOffsets[i] - var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: 4, Cap: 4} + var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: valueInlineLen, Cap: valueInlineLen} var buf = *(*[]byte)(unsafe.Pointer(&hdr)) value, err = decode(valueTypes[i], buf) } else { @@ -287,9 +329,9 @@ func decodeJSONObject(m *map[string]JSON, data []byte) (err error) { func encodeJSONArray(a []JSON, buffer *bytes.Buffer) { // array ::= element-count size value-entry* value* var countAndSize = make([]uint32, 2) - var countAndSizeLen = len(countAndSize) * 4 + var countAndSizeLen = compoundCountLen + compoundSizeLen - var valueEntrysLen = (1 + 4) * len(a) + var valueEntrysLen = (typeCodeLen + valueInlineLen) * len(a) var valueEntrys = new(bytes.Buffer) var values = new(bytes.Buffer) for _, value := range a { @@ -323,9 +365,9 @@ func decodeJSONArray(a *[]JSON, data []byte) (err error) { for i := 0; i < int(countAndSize[0]); i++ { var value JSON typeLen, _ := jsonTypeCodeLength[TypeCode(valueTypes[i])] - if typeLen >= 0 && typeLen <= 4 { + if typeLen >= 0 && typeLen <= valueInlineLen { var inline = valueOffsets[i] - var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: 4, Cap: 4} + var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: valueInlineLen, Cap: valueInlineLen} var buf = *(*[]byte)(unsafe.Pointer(&hdr)) value, err = decode(valueTypes[i], buf) } else { @@ -365,7 +407,7 @@ func pushValueEntry(value JSON, valueEntrys *bytes.Buffer, values *bytes.Buffer, valueEntrys.WriteByte(byte(typeCode)) typeLen, _ := jsonTypeCodeLength[typeCode] - if typeLen > 0 && typeLen <= 4 { + if typeLen > 0 && typeLen <= valueInlineLen { // If the value has length in (0, 4], it could be inline here. // And padding 0x00 to 4 bytes if needed. pushInlineValue(valueEntrys, value) @@ -390,7 +432,7 @@ func pushInlineValue(buffer *bytes.Buffer, value JSON) { panic(msg) } var newLen = buffer.Len() - for i := 0; i < 4-(newLen-oldLen); i++ { + for i := 0; i < valueInlineLen-(newLen-oldLen); i++ { buffer.WriteByte(0x00) } }