Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

codec: simplify json codec. #3524

Merged
merged 3 commits into from
Jun 23, 2017
Merged

codec: simplify json codec. #3524

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
02f228b
codec: simplify json codec.
hicqu Jun 21, 2017
4ceee88
address comments.
hicqu Jun 23, 2017
31a373b
Merge branch 'master' into qupeng/json-codec-improve
hicqu Jun 23, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 6 additions & 9 deletions util/codec/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,9 +80,8 @@ func encode(b []byte, vals []types.Datum, comparable bool) ([]byte, error) {
case types.KindMysqlSet:
b = encodeUnsignedInt(b, uint64(val.GetMysqlSet().ToNumber()), comparable)
case types.KindMysqlJSON:
bytes := json.Serialize(val.GetMysqlJSON())
b = append(b, jsonFlag)
b = EncodeCompactBytes(b, bytes)
b = append(b, json.Serialize(val.GetMysqlJSON())...)
case types.KindNull:
b = append(b, NilFlag)
case types.KindMinNotNull:
Expand Down Expand Up @@ -216,12 +215,8 @@ func DecodeOne(b []byte) (remain []byte, d types.Datum, err error) {
d.SetValue(v)
}
case jsonFlag:
var v []byte
if b, v, err = DecodeCompactBytes(b); err == nil {
var j json.JSON
if j, err = json.Deserialize(v); err == nil {
d.SetMysqlJSON(j)
}
if j, err := json.Deserialize(b); err == nil {
d.SetMysqlJSON(j)
}
case NilFlag:
default:
Expand Down Expand Up @@ -272,14 +267,16 @@ func peek(b []byte) (length int, err error) {
l = 8
case bytesFlag:
l, err = peekBytes(b, false)
case compactBytesFlag, jsonFlag:
case compactBytesFlag:
l, err = peekCompactBytes(b)
case decimalFlag:
l, err = types.DecimalPeak(b)
case varintFlag:
l, err = peekVarint(b)
case uvarintFlag:
l, err = peekUvarint(b)
case jsonFlag:
l, err = json.PeekBytesAsJSON(b)
default:
return 0, errors.Errorf("invalid encoded key flag %v", flag)
}
Expand Down
26 changes: 16 additions & 10 deletions util/types/json/json_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,26 +53,32 @@ func (s *testJSONSuite) TestSerializeAndDeserialize(c *C) {
j1 := mustParseFromString(`{"aaaaaaaaaaa": [1, "2", {"aa": "bb"}, 4.0], "bbbbbbbbbb": true, "ccccccccc": "d"}`)
j2 := mustParseFromString(`[{"a": 1, "b": true}, 3, 3.5, "hello, world", null, true]`)

var testcses = []struct {
In JSON
Out JSON
var testcases = []struct {
In JSON
Out JSON
size int
}{
{In: jsonNilValue, Out: jsonNilValue},
{In: jsonBoolValue, Out: jsonBoolValue},
{In: jsonDoubleValue, Out: jsonDoubleValue},
{In: jsonStringValue, Out: jsonStringValue},
{In: j1, Out: j1},
{In: j2, Out: j2},
{In: jsonNilValue, Out: jsonNilValue, size: 2},
{In: jsonBoolValue, Out: jsonBoolValue, size: 2},
{In: jsonDoubleValue, Out: jsonDoubleValue, size: 9},
{In: jsonStringValue, Out: jsonStringValue, size: 15},
{In: j1, Out: j1, size: 144},
{In: j2, Out: j2, size: 108},
}

for _, s := range testcses {
for _, s := range testcases {
data := Serialize(s.In)
t, err := Deserialize(data)
c.Assert(err, IsNil)

v1 := t.String()
v2 := s.Out.String()
c.Assert(v1, Equals, v2)

size, err := PeekBytesAsJSON(data)
c.Assert(err, IsNil)
c.Assert(len(data), Equals, size)
c.Assert(len(data), Equals, s.size)
}
}

Expand Down
64 changes: 53 additions & 11 deletions util/types/json/serdes.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,48 @@ import (
// lengths up to 16383, and so on...
*/

const (
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add comments for these constants.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The constant 4 is used somewhere else in this file, we could use a const variable for all of them.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

fixed.

typeCodeLen int = 1
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

comment for these constant?

compoundCountLen = 4
compoundSizeLen = 4
keyOffsetLen = 4
keyLengthLen = 2
valueInlineLen = 4
)

// PeekBytesAsJSON trys to peek some bytes from b, until
// we can deserialize a JSON from those bytes.
func PeekBytesAsJSON(b []byte) (n int, err error) {
if len(b) <= 0 {
err = errors.New("Cant peek from empty bytes")
return
}
switch c := TypeCode(b[0]); c {
case typeCodeObject, typeCodeArray:
if len(b) >= typeCodeLen+compoundCountLen+compoundSizeLen {
var size uint32
start := typeCodeLen + compoundCountLen
end := typeCodeLen + compoundCountLen + compoundSizeLen
binary.Read(bytes.NewReader(b[start:end]), binary.LittleEndian, &size)
n = int(size) + typeCodeLen
return
}
case typeCodeString:
var size uint64
reader := bytes.NewReader(b[typeCodeLen:])
size, err = binary.ReadUvarint(reader)
if err == nil {
n = int(size) + int(reader.Size()) - int(reader.Len()) + typeCodeLen
return
}
case typeCodeInt64, typeCodeFloat64, typeCodeLiteral:
n = jsonTypeCodeLength[TypeCode(c)] + typeCodeLen
return
}
err = errors.New("Invalid JSON bytes")
Copy link
Contributor

@hhkbp2 hhkbp2 Jun 23, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it better to return different error message for 1. len(b) == 0, 2. b[0] is invalid ?

return
}

// Serialize means serialize itself into bytes.
func Serialize(j JSON) []byte {
var buffer = new(bytes.Buffer)
Expand Down Expand Up @@ -202,11 +244,11 @@ func encodeJSONObject(m map[string]JSON, buffer *bytes.Buffer) {
// object ::= element-count size key-entry* value-entry* key* value*
// key-entry ::= key-offset key-length
var countAndSize = make([]uint32, 2)
var countAndSizeLen = len(countAndSize) * 4
var countAndSizeLen = compoundCountLen + compoundSizeLen
var keySlice = getSortedKeys(m)

var keyEntrysLen = (4 + 2) * len(m)
var valueEntrysLen = (1 + 4) * len(m)
var keyEntrysLen = (keyOffsetLen + keyLengthLen) * len(m)
var valueEntrysLen = (typeCodeLen + valueInlineLen) * len(m)
var keyEntrys = new(bytes.Buffer)
var valueEntrys = new(bytes.Buffer)
var keys = new(bytes.Buffer)
Expand Down Expand Up @@ -268,9 +310,9 @@ func decodeJSONObject(m *map[string]JSON, data []byte) (err error) {
var key = string(keyBuffer)
var value JSON
typeLen, _ := jsonTypeCodeLength[TypeCode(valueTypes[i])]
if typeLen >= 0 && typeLen <= 4 {
if typeLen >= 0 && typeLen <= valueInlineLen {
var inline = valueOffsets[i]
var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: 4, Cap: 4}
var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: valueInlineLen, Cap: valueInlineLen}
var buf = *(*[]byte)(unsafe.Pointer(&hdr))
value, err = decode(valueTypes[i], buf)
} else {
Expand All @@ -287,9 +329,9 @@ func decodeJSONObject(m *map[string]JSON, data []byte) (err error) {
func encodeJSONArray(a []JSON, buffer *bytes.Buffer) {
// array ::= element-count size value-entry* value*
var countAndSize = make([]uint32, 2)
var countAndSizeLen = len(countAndSize) * 4
var countAndSizeLen = compoundCountLen + compoundSizeLen

var valueEntrysLen = (1 + 4) * len(a)
var valueEntrysLen = (typeCodeLen + valueInlineLen) * len(a)
var valueEntrys = new(bytes.Buffer)
var values = new(bytes.Buffer)
for _, value := range a {
Expand Down Expand Up @@ -323,9 +365,9 @@ func decodeJSONArray(a *[]JSON, data []byte) (err error) {
for i := 0; i < int(countAndSize[0]); i++ {
var value JSON
typeLen, _ := jsonTypeCodeLength[TypeCode(valueTypes[i])]
if typeLen >= 0 && typeLen <= 4 {
if typeLen >= 0 && typeLen <= valueInlineLen {
var inline = valueOffsets[i]
var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: 4, Cap: 4}
var hdr = reflect.SliceHeader{Data: uintptr(unsafe.Pointer(&inline)), Len: valueInlineLen, Cap: valueInlineLen}
var buf = *(*[]byte)(unsafe.Pointer(&hdr))
value, err = decode(valueTypes[i], buf)
} else {
Expand Down Expand Up @@ -365,7 +407,7 @@ func pushValueEntry(value JSON, valueEntrys *bytes.Buffer, values *bytes.Buffer,
valueEntrys.WriteByte(byte(typeCode))

typeLen, _ := jsonTypeCodeLength[typeCode]
if typeLen > 0 && typeLen <= 4 {
if typeLen > 0 && typeLen <= valueInlineLen {
// If the value has length in (0, 4], it could be inline here.
// And padding 0x00 to 4 bytes if needed.
pushInlineValue(valueEntrys, value)
Expand All @@ -390,7 +432,7 @@ func pushInlineValue(buffer *bytes.Buffer, value JSON) {
panic(msg)
}
var newLen = buffer.Len()
for i := 0; i < 4-(newLen-oldLen); i++ {
for i := 0; i < valueInlineLen-(newLen-oldLen); i++ {
buffer.WriteByte(0x00)
}
}