Skip to content

Commit

Permalink
codec: simplify json codec. (#3524)
Browse files Browse the repository at this point in the history
  • Loading branch information
@hicqu
hicqu authored Jun 23, 2017
1 parent 8a31944 commit b9088c2
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 30 deletions.
15 changes: 6 additions & 9 deletions util/codec/codec.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,8 @@ func encode(b []byte, vals []types.Datum, comparable bool) ([]byte, error) {
case types.KindMysqlSet:
b = encodeUnsignedInt(b, uint64(val.GetMysqlSet().ToNumber()), comparable)
case types.KindMysqlJSON:
bytes := json.Serialize(val.GetMysqlJSON())
b = append(b, jsonFlag)
b = EncodeCompactBytes(b, bytes)
b = append(b, json.Serialize(val.GetMysqlJSON())...)
case types.KindNull:
b = append(b, NilFlag)
case types.KindMinNotNull:
Expand Down Expand Up @@ -216,12 +215,8 @@ func DecodeOne(b []byte) (remain []byte, d types.Datum, err error) {
d.SetValue(v)
}
case jsonFlag:
var v []byte
if b, v, err = DecodeCompactBytes(b); err == nil {
var j json.JSON
if j, err = json.Deserialize(v); err == nil {
d.SetMysqlJSON(j)
}
if j, err := json.Deserialize(b); err == nil {
d.SetMysqlJSON(j)
}
case NilFlag:
default:
Expand Down Expand Up @@ -272,14 +267,16 @@ func peek(b []byte) (length int, err error) {
l = 8
case bytesFlag:
l, err = peekBytes(b, false)
case compactBytesFlag, jsonFlag:
case compactBytesFlag:
l, err = peekCompactBytes(b)
case decimalFlag:
l, err = types.DecimalPeak(b)
case varintFlag:
l, err = peekVarint(b)
case uvarintFlag:
l, err = peekUvarint(b)
case jsonFlag:
l, err = json.PeekBytesAsJSON(b)
default:
return 0, errors.Errorf("invalid encoded key flag %v", flag)
}
Expand Down
26 changes: 16 additions & 10 deletions util/types/json/json_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,26 +53,32 @@ func (s *testJSONSuite) TestSerializeAndDeserialize(c *C) {
j1 := mustParseFromString(`{"aaaaaaaaaaa": [1, "2", {"aa": "bb"}, 4.0], "bbbbbbbbbb": true, "ccccccccc": "d"}`)
j2 := mustParseFromString(`[{"a": 1, "b": true}, 3, 3.5, "hello, world", null, true]`)

var testcses = []struct {
In JSON
Out JSON
var testcases = []struct {
In JSON
Out JSON
size int
}{
{In: jsonNilValue, Out: jsonNilValue},
{In: jsonBoolValue, Out: jsonBoolValue},
{In: jsonDoubleValue, Out: jsonDoubleValue},
{In: jsonStringValue, Out: jsonStringValue},
{In: j1, Out: j1},
{In: j2, Out: j2},
{In: jsonNilValue, Out: jsonNilValue, size: 2},
{In: jsonBoolValue, Out: jsonBoolValue, size: 2},
{In: jsonDoubleValue, Out: jsonDoubleValue, size: 9},
{In: jsonStringValue, Out: jsonStringValue, size: 15},
{In: j1, Out: j1, size: 144},
{In: j2, Out: j2, size: 108},
}

for _, s := range testcses {
for _, s := range testcases {
data := Serialize(s.In)
t, err := Deserialize(data)
c.Assert(err, IsNil)

v1 := t.String()
v2 := s.Out.String()
c.Assert(v1, Equals, v2)

size, err := PeekBytesAsJSON(data)
c.Assert(err, IsNil)
c.Assert(len(data), Equals, size)
c.Assert(len(data), Equals, s.size)
}
}

Expand Down
64 changes: 53 additions & 11 deletions util/types/json/serdes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,48 @@ import (
// lengths up to 16383, and so on...
*/

const (
typeCodeLen int = 1
compoundCountLen = 4
compoundSizeLen = 4
keyOffsetLen = 4
keyLengthLen = 2
valueInlineLen = 4
)

// PeekBytesAsJSON trys to peek some bytes from b, until
// we can deserialize a JSON from those bytes.
func PeekBytesAsJSON(b []byte) (n int, err error) {
if len(b) <= 0 {
err = errors.New("Cant peek from empty bytes")
return
}
switch c := TypeCode(b[0]); c {
case typeCodeObject, typeCodeArray:
if len(b) >= typeCodeLen+compoundCountLen+compoundSizeLen {
var size uint32
start := typeCodeLen + compoundCountLen
end := typeCodeLen + compoundCountLen + compoundSizeLen
binary.Read(bytes.NewReader(b[start:end]), binary.LittleEndian, &size)
n = int(size) + typeCodeLen
return
}
case typeCodeString:
var size uint64
reader := bytes.NewReader(b[typeCodeLen:])
size, err = binary.ReadUvarint(reader)
if err == nil {
n = int(size) + int(reader.Size()) - int(reader.Len()) + typeCodeLen
return
}
case typeCodeInt64, typeCodeFloat64, typeCodeLiteral:
n = jsonTypeCodeLength[TypeCode(c)] + typeCodeLen
return
}
err = errors.New("Invalid JSON bytes")
return
}

// Serialize means serialize itself into bytes.
func Serialize(j JSON) []byte {
var buffer = new(bytes.Buffer)
Expand Down Expand Up @@ -202,11 +244,11 @@ func encodeJSONObject(m map[string]JSON, buffer *bytes.Buffer) {
// object ::= element-count size key-entry* value-entry* key* value*
// key-entry ::= key-offset key-length
var countAndSize = make([]uint32, 2)
var countAndSizeLen = len(countAndSize) * 4
var countAndSizeLen = compoundCountLen + compoundSizeLen
var keySlice = getSortedKeys(m)

var keyEntrysLen = (4 + 2) * len(m)
var valueEntrysLen = (1 + 4) * len(m)
var keyEntrysLen = (keyOffsetLen + keyLengthLen) * len(m)
var valueEntrysLen = (typeCodeLen + valueInlineLen) * len(m)
var keyEntrys = new(bytes.Buffer)
var valueEntrys = new(bytes.Buffer)
var keys = new(bytes.Buffer)
Expand Down Expand Up @@ -268,9 +310,9 @@ func decodeJSONObject(m *map[string]JSON, data []byte) (err error) {
var key = string(keyBuffer)
var value JSON
typeLen, _ := jsonTypeCodeLength[TypeCode(valueTypes[i])]
if typeLen >= 0 && typeLen <= 4 {
if typeLen >= 0 && typeLen <= valueInlineLen {
var inline = valueOffsets[i]
var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: 4, Cap: 4}
var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: valueInlineLen, Cap: valueInlineLen}
var buf = *(*[]byte)(unsafe.Pointer(&hdr))
value, err = decode(valueTypes[i], buf)
} else {
Expand All @@ -287,9 +329,9 @@ func decodeJSONObject(m *map[string]JSON, data []byte) (err error) {
func encodeJSONArray(a []JSON, buffer *bytes.Buffer) {
// array ::= element-count size value-entry* value*
var countAndSize = make([]uint32, 2)
var countAndSizeLen = len(countAndSize) * 4
var countAndSizeLen = compoundCountLen + compoundSizeLen

var valueEntrysLen = (1 + 4) * len(a)
var valueEntrysLen = (typeCodeLen + valueInlineLen) * len(a)
var valueEntrys = new(bytes.Buffer)
var values = new(bytes.Buffer)
for _, value := range a {
Expand Down Expand Up @@ -323,9 +365,9 @@ func decodeJSONArray(a *[]JSON, data []byte) (err error) {
for i := 0; i < int(countAndSize[0]); i++ {
var value JSON
typeLen, _ := jsonTypeCodeLength[TypeCode(valueTypes[i])]
if typeLen >= 0 && typeLen <= 4 {
if typeLen >= 0 && typeLen <= valueInlineLen {
var inline = valueOffsets[i]
var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: 4, Cap: 4}
var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: valueInlineLen, Cap: valueInlineLen}
var buf = *(*[]byte)(unsafe.Pointer(&hdr))
value, err = decode(valueTypes[i], buf)
} else {
Expand Down Expand Up @@ -365,7 +407,7 @@ func pushValueEntry(value JSON, valueEntrys *bytes.Buffer, values *bytes.Buffer,
valueEntrys.WriteByte(byte(typeCode))

typeLen, _ := jsonTypeCodeLength[typeCode]
if typeLen > 0 && typeLen <= 4 {
if typeLen > 0 && typeLen <= valueInlineLen {
// If the value has length in (0, 4], it could be inline here.
// And padding 0x00 to 4 bytes if needed.
pushInlineValue(valueEntrys, value)
Expand All @@ -390,7 +432,7 @@ func pushInlineValue(buffer *bytes.Buffer, value JSON) {
panic(msg)
}
var newLen = buffer.Len()
for i := 0; i < 4-(newLen-oldLen); i++ {
for i := 0; i < valueInlineLen-(newLen-oldLen); i++ {
buffer.WriteByte(0x00)
}
}

0 comments on commit b9088c2

Please sign in to comment.