diff --git a/spanner/go.mod b/spanner/go.mod index 8912cb5a0cba..587674a3b8b7 100644 --- a/spanner/go.mod +++ b/spanner/go.mod @@ -9,6 +9,8 @@ require ( github.com/golang/protobuf v1.5.3 github.com/google/go-cmp v0.6.0 github.com/googleapis/gax-go/v2 v2.12.0 + github.com/json-iterator/go v1.1.12 + github.com/stretchr/testify v1.8.3 go.opencensus.io v0.24.0 golang.org/x/oauth2 v0.13.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 @@ -27,15 +29,20 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/envoyproxy/go-control-plane v0.11.1 // indirect github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/crypto v0.14.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/sync v0.4.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/appengine v1.6.7 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/spanner/go.sum b/spanner/go.sum index 02c5d41dec3b..4bae31351e78 100644 --- a/spanner/go.sum +++ b/spanner/go.sum @@ -23,6 +23,7 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -59,6 +60,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -67,14 +69,24 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -156,8 +168,10 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/spanner/integration_test.go b/spanner/integration_test.go index 84d3872e8e22..9356c0f9d046 100644 --- a/spanner/integration_test.go +++ b/spanner/integration_test.go @@ -45,6 +45,7 @@ import ( v1 "cloud.google.com/go/spanner/apiv1" sppb "cloud.google.com/go/spanner/apiv1/spannerpb" "cloud.google.com/go/spanner/internal" + "github.com/stretchr/testify/require" "go.opencensus.io/stats/view" "go.opencensus.io/tag" "google.golang.org/api/iterator" @@ -2017,29 +2018,40 @@ func TestIntegration_BasicTypes(t *testing.T) { n2 := *n2p type Message struct { - Name string - Body string - Time int64 + Name string + Body string + Time int64 + FloatValue interface{} + } + msg := Message{"Alice", "Hello", 145688415796432520, json.Number("0.39240506000000003")} + unmarshalledJSONStructUsingNumber := map[string]interface{}{ + "Name": "Alice", + "Body": "Hello", + "Time": json.Number("145688415796432520"), + "FloatValue": json.Number("0.39240506000000003"), + } + unmarshalledJSONStruct := map[string]interface{}{ + "Name": "Alice", + "Body": "Hello", + "Time": 1.456884157964325e+17, + "FloatValue": 0.39240506, } - msg := Message{"Alice", "Hello", 1294706395881547000} - jsonStr := `{"Name":"Alice","Body":"Hello","Time":1294706395881547000}` - var unmarshalledJSONstruct interface{} - json.Unmarshal([]byte(jsonStr), &unmarshalledJSONstruct) tests := []struct { - col string - val interface{} - want interface{} + col string + val interface{} + wantWithDefaultConfig interface{} + wantWithNumber interface{} }{ {col: "String", val: ""}, - {col: "String", val: "", want: NullString{"", true}}, + {col: "String", val: "", wantWithDefaultConfig: NullString{"", true}, wantWithNumber: NullString{"", true}}, {col: "String", val: "foo"}, - {col: "String", val: "foo", want: NullString{"foo", true}}, - {col: "String", val: NullString{"bar", true}, want: "bar"}, - {col: "String", val: NullString{"bar", false}, want: NullString{"", false}}, - {col: "StringArray", val: []string(nil), want: []NullString(nil)}, - {col: "StringArray", val: []string{}, want: []NullString{}}, - {col: "StringArray", val: []string{"foo", "bar"}, want: []NullString{{"foo", true}, {"bar", true}}}, + {col: "String", val: "foo", wantWithDefaultConfig: NullString{"foo", true}, wantWithNumber: NullString{"foo", true}}, + {col: "String", val: NullString{"bar", true}, wantWithDefaultConfig: "bar", wantWithNumber: "bar"}, + {col: "String", val: NullString{"bar", false}, wantWithDefaultConfig: NullString{"", false}, wantWithNumber: NullString{"", false}}, + {col: "StringArray", val: []string(nil), wantWithDefaultConfig: []NullString(nil), wantWithNumber: []NullString(nil)}, + {col: "StringArray", val: []string{}, wantWithDefaultConfig: []NullString{}, wantWithNumber: []NullString{}}, + {col: "StringArray", val: []string{"foo", "bar"}, wantWithDefaultConfig: []NullString{{"foo", true}, {"bar", true}}, wantWithNumber: []NullString{{"foo", true}, {"bar", true}}}, {col: "StringArray", val: []NullString(nil)}, {col: "StringArray", val: []NullString{}}, {col: "StringArray", val: []NullString{{"foo", true}, {}}}, @@ -2049,32 +2061,32 @@ func TestIntegration_BasicTypes(t *testing.T) { {col: "BytesArray", val: [][]byte(nil)}, {col: "BytesArray", val: [][]byte{}}, {col: "BytesArray", val: [][]byte{{1}, {2, 3}}}, - {col: "Int64a", val: 0, want: int64(0)}, - {col: "Int64a", val: -1, want: int64(-1)}, - {col: "Int64a", val: 2, want: int64(2)}, + {col: "Int64a", val: 0, wantWithDefaultConfig: int64(0), wantWithNumber: int64(0)}, + {col: "Int64a", val: -1, wantWithDefaultConfig: int64(-1), wantWithNumber: int64(-1)}, + {col: "Int64a", val: 2, wantWithDefaultConfig: int64(2), wantWithNumber: int64(2)}, {col: "Int64a", val: int64(3)}, - {col: "Int64a", val: 4, want: NullInt64{4, true}}, - {col: "Int64a", val: NullInt64{5, true}, want: int64(5)}, - {col: "Int64a", val: NullInt64{6, true}, want: int64(6)}, - {col: "Int64a", val: NullInt64{7, false}, want: NullInt64{0, false}}, - {col: "Int64Array", val: []int(nil), want: []NullInt64(nil)}, - {col: "Int64Array", val: []int{}, want: []NullInt64{}}, - {col: "Int64Array", val: []int{1, 2}, want: []NullInt64{{1, true}, {2, true}}}, - {col: "Int64Array", val: []int64(nil), want: []NullInt64(nil)}, - {col: "Int64Array", val: []int64{}, want: []NullInt64{}}, - {col: "Int64Array", val: []int64{1, 2}, want: []NullInt64{{1, true}, {2, true}}}, + {col: "Int64a", val: 4, wantWithDefaultConfig: NullInt64{4, true}, wantWithNumber: NullInt64{4, true}}, + {col: "Int64a", val: NullInt64{5, true}, wantWithDefaultConfig: int64(5), wantWithNumber: int64(5)}, + {col: "Int64a", val: NullInt64{6, true}, wantWithDefaultConfig: int64(6), wantWithNumber: int64(6)}, + {col: "Int64a", val: NullInt64{7, false}, wantWithDefaultConfig: NullInt64{0, false}, wantWithNumber: NullInt64{0, false}}, + {col: "Int64Array", val: []int(nil), wantWithDefaultConfig: []NullInt64(nil), wantWithNumber: []NullInt64(nil)}, + {col: "Int64Array", val: []int{}, wantWithDefaultConfig: []NullInt64{}, wantWithNumber: []NullInt64{}}, + {col: "Int64Array", val: []int{1, 2}, wantWithDefaultConfig: []NullInt64{{1, true}, {2, true}}, wantWithNumber: []NullInt64{{1, true}, {2, true}}}, + {col: "Int64Array", val: []int64(nil), wantWithDefaultConfig: []NullInt64(nil), wantWithNumber: []NullInt64(nil)}, + {col: "Int64Array", val: []int64{}, wantWithDefaultConfig: []NullInt64{}, wantWithNumber: []NullInt64{}}, + {col: "Int64Array", val: []int64{1, 2}, wantWithDefaultConfig: []NullInt64{{1, true}, {2, true}}, wantWithNumber: []NullInt64{{1, true}, {2, true}}}, {col: "Int64Array", val: []NullInt64(nil)}, {col: "Int64Array", val: []NullInt64{}}, {col: "Int64Array", val: []NullInt64{{1, true}, {}}}, {col: "Bool", val: false}, {col: "Bool", val: true}, - {col: "Bool", val: false, want: NullBool{false, true}}, - {col: "Bool", val: true, want: NullBool{true, true}}, + {col: "Bool", val: false, wantWithDefaultConfig: NullBool{false, true}, wantWithNumber: NullBool{false, true}}, + {col: "Bool", val: true, wantWithDefaultConfig: NullBool{true, true}, wantWithNumber: NullBool{true, true}}, {col: "Bool", val: NullBool{true, true}}, {col: "Bool", val: NullBool{false, false}}, - {col: "BoolArray", val: []bool(nil), want: []NullBool(nil)}, - {col: "BoolArray", val: []bool{}, want: []NullBool{}}, - {col: "BoolArray", val: []bool{true, false}, want: []NullBool{{true, true}, {false, true}}}, + {col: "BoolArray", val: []bool(nil), wantWithDefaultConfig: []NullBool(nil), wantWithNumber: []NullBool(nil)}, + {col: "BoolArray", val: []bool{}, wantWithDefaultConfig: []NullBool{}, wantWithNumber: []NullBool{}}, + {col: "BoolArray", val: []bool{true, false}, wantWithDefaultConfig: []NullBool{{true, true}, {false, true}}, wantWithNumber: []NullBool{{true, true}, {false, true}}}, {col: "BoolArray", val: []NullBool(nil)}, {col: "BoolArray", val: []NullBool{}}, {col: "BoolArray", val: []NullBool{{false, true}, {true, true}, {}}}, @@ -2083,150 +2095,162 @@ func TestIntegration_BasicTypes(t *testing.T) { {col: "Float64", val: math.NaN()}, {col: "Float64", val: math.Inf(1)}, {col: "Float64", val: math.Inf(-1)}, - {col: "Float64", val: 2.78, want: NullFloat64{2.78, true}}, - {col: "Float64", val: NullFloat64{2.71, true}, want: 2.71}, - {col: "Float64", val: NullFloat64{1.41, true}, want: NullFloat64{1.41, true}}, + {col: "Float64", val: 2.78, wantWithDefaultConfig: NullFloat64{2.78, true}, wantWithNumber: NullFloat64{2.78, true}}, + {col: "Float64", val: NullFloat64{2.71, true}, wantWithDefaultConfig: 2.71, wantWithNumber: 2.71}, + {col: "Float64", val: NullFloat64{1.41, true}, wantWithDefaultConfig: NullFloat64{1.41, true}, wantWithNumber: NullFloat64{1.41, true}}, {col: "Float64", val: NullFloat64{0, false}}, - {col: "Float64Array", val: []float64(nil), want: []NullFloat64(nil)}, - {col: "Float64Array", val: []float64{}, want: []NullFloat64{}}, - {col: "Float64Array", val: []float64{2.72, 3.14, math.Inf(1)}, want: []NullFloat64{{2.72, true}, {3.14, true}, {math.Inf(1), true}}}, + {col: "Float64Array", val: []float64(nil), wantWithDefaultConfig: []NullFloat64(nil), wantWithNumber: []NullFloat64(nil)}, + {col: "Float64Array", val: []float64{}, wantWithDefaultConfig: []NullFloat64{}, wantWithNumber: []NullFloat64{}}, + {col: "Float64Array", val: []float64{2.72, 3.14, math.Inf(1)}, wantWithDefaultConfig: []NullFloat64{{2.72, true}, {3.14, true}, {math.Inf(1), true}}, wantWithNumber: []NullFloat64{{2.72, true}, {3.14, true}, {math.Inf(1), true}}}, {col: "Float64Array", val: []NullFloat64(nil)}, {col: "Float64Array", val: []NullFloat64{}}, {col: "Float64Array", val: []NullFloat64{{2.72, true}, {math.Inf(1), true}, {}}}, {col: "Date", val: d1}, - {col: "Date", val: d1, want: NullDate{d1, true}}, + {col: "Date", val: d1, wantWithDefaultConfig: NullDate{d1, true}, wantWithNumber: NullDate{d1, true}}, {col: "Date", val: NullDate{d1, true}}, - {col: "Date", val: NullDate{d1, true}, want: d1}, + {col: "Date", val: NullDate{d1, true}, wantWithDefaultConfig: d1, wantWithNumber: d1}, {col: "Date", val: NullDate{civil.Date{}, false}}, - {col: "DateArray", val: []civil.Date(nil), want: []NullDate(nil)}, - {col: "DateArray", val: []civil.Date{}, want: []NullDate{}}, - {col: "DateArray", val: []civil.Date{d1, d2, d3}, want: []NullDate{{d1, true}, {d2, true}, {d3, true}}}, + {col: "DateArray", val: []civil.Date(nil), wantWithDefaultConfig: []NullDate(nil), wantWithNumber: []NullDate(nil)}, + {col: "DateArray", val: []civil.Date{}, wantWithDefaultConfig: []NullDate{}, wantWithNumber: []NullDate{}}, + {col: "DateArray", val: []civil.Date{d1, d2, d3}, wantWithDefaultConfig: []NullDate{{d1, true}, {d2, true}, {d3, true}}, wantWithNumber: []NullDate{{d1, true}, {d2, true}, {d3, true}}}, {col: "Timestamp", val: t1}, - {col: "Timestamp", val: t1, want: NullTime{t1, true}}, + {col: "Timestamp", val: t1, wantWithDefaultConfig: NullTime{t1, true}, wantWithNumber: NullTime{t1, true}}, {col: "Timestamp", val: NullTime{t1, true}}, - {col: "Timestamp", val: NullTime{t1, true}, want: t1}, + {col: "Timestamp", val: NullTime{t1, true}, wantWithDefaultConfig: t1, wantWithNumber: t1}, {col: "Timestamp", val: NullTime{}}, - {col: "TimestampArray", val: []time.Time(nil), want: []NullTime(nil)}, - {col: "TimestampArray", val: []time.Time{}, want: []NullTime{}}, - {col: "TimestampArray", val: []time.Time{t1, t2, t3}, want: []NullTime{{t1, true}, {t2, true}, {t3, true}}}, + {col: "TimestampArray", val: []time.Time(nil), wantWithDefaultConfig: []NullTime(nil), wantWithNumber: []NullTime(nil)}, + {col: "TimestampArray", val: []time.Time{}, wantWithDefaultConfig: []NullTime{}, wantWithNumber: []NullTime{}}, + {col: "TimestampArray", val: []time.Time{t1, t2, t3}, wantWithDefaultConfig: []NullTime{{t1, true}, {t2, true}, {t3, true}}, wantWithNumber: []NullTime{{t1, true}, {t2, true}, {t3, true}}}, {col: "Numeric", val: n1}, {col: "Numeric", val: n2}, - {col: "Numeric", val: n1, want: NullNumeric{n1, true}}, - {col: "Numeric", val: n2, want: NullNumeric{n2, true}}, - {col: "Numeric", val: NullNumeric{n1, true}, want: n1}, - {col: "Numeric", val: NullNumeric{n1, true}, want: NullNumeric{n1, true}}, + {col: "Numeric", val: n1, wantWithDefaultConfig: NullNumeric{n1, true}, wantWithNumber: NullNumeric{n1, true}}, + {col: "Numeric", val: n2, wantWithDefaultConfig: NullNumeric{n2, true}, wantWithNumber: NullNumeric{n2, true}}, + {col: "Numeric", val: NullNumeric{n1, true}, wantWithDefaultConfig: n1, wantWithNumber: n1}, + {col: "Numeric", val: NullNumeric{n1, true}, wantWithDefaultConfig: NullNumeric{n1, true}, wantWithNumber: NullNumeric{n1, true}}, {col: "Numeric", val: NullNumeric{n0, false}}, - {col: "NumericArray", val: []big.Rat(nil), want: []NullNumeric(nil)}, - {col: "NumericArray", val: []big.Rat{}, want: []NullNumeric{}}, - {col: "NumericArray", val: []big.Rat{n1, n2}, want: []NullNumeric{{n1, true}, {n2, true}}}, + {col: "NumericArray", val: []big.Rat(nil), wantWithDefaultConfig: []NullNumeric(nil), wantWithNumber: []NullNumeric(nil)}, + {col: "NumericArray", val: []big.Rat{}, wantWithDefaultConfig: []NullNumeric{}, wantWithNumber: []NullNumeric{}}, + {col: "NumericArray", val: []big.Rat{n1, n2}, wantWithDefaultConfig: []NullNumeric{{n1, true}, {n2, true}}, wantWithNumber: []NullNumeric{{n1, true}, {n2, true}}}, {col: "NumericArray", val: []NullNumeric(nil)}, {col: "NumericArray", val: []NullNumeric{}}, {col: "NumericArray", val: []NullNumeric{{n1, true}, {n2, true}, {}}}, - {col: "JSON", val: NullJSON{msg, true}, want: NullJSON{unmarshalledJSONstruct, true}}, - {col: "JSON", val: NullJSON{msg, false}, want: NullJSON{}}, + {col: "JSON", val: NullJSON{msg, true}, wantWithDefaultConfig: NullJSON{unmarshalledJSONStruct, true}, wantWithNumber: NullJSON{unmarshalledJSONStructUsingNumber, true}}, + {col: "JSON", val: NullJSON{msg, false}, wantWithDefaultConfig: NullJSON{}, wantWithNumber: NullJSON{}}, {col: "JSONArray", val: []NullJSON(nil)}, {col: "JSONArray", val: []NullJSON{}}, - {col: "JSONArray", val: []NullJSON{{msg, true}, {msg, true}, {}}, want: []NullJSON{{unmarshalledJSONstruct, true}, {unmarshalledJSONstruct, true}, {}}}, - {col: "String", val: nil, want: NullString{}}, - {col: "StringArray", val: nil, want: []NullString(nil)}, - {col: "Bytes", val: nil, want: []byte(nil)}, - {col: "BytesArray", val: nil, want: [][]byte(nil)}, - {col: "Int64a", val: nil, want: NullInt64{}}, - {col: "Int64Array", val: nil, want: []NullInt64(nil)}, - {col: "Bool", val: nil, want: NullBool{}}, - {col: "BoolArray", val: nil, want: []NullBool(nil)}, - {col: "Float64", val: nil, want: NullFloat64{}}, - {col: "Float64Array", val: nil, want: []NullFloat64(nil)}, - {col: "Numeric", val: nil, want: NullNumeric{}}, - {col: "NumericArray", val: nil, want: []NullNumeric(nil)}, - {col: "JSON", val: nil, want: NullJSON{}}, - {col: "JSONArray", val: nil, want: []NullJSON(nil)}, + {col: "JSONArray", val: []NullJSON{{msg, true}, {msg, true}, {}}, wantWithDefaultConfig: []NullJSON{{unmarshalledJSONStruct, true}, {unmarshalledJSONStruct, true}, {}}, wantWithNumber: []NullJSON{{unmarshalledJSONStructUsingNumber, true}, {unmarshalledJSONStructUsingNumber, true}, {}}}, + {col: "String", val: nil, wantWithDefaultConfig: NullString{}, wantWithNumber: NullString{}}, + {col: "StringArray", val: nil, wantWithDefaultConfig: []NullString(nil), wantWithNumber: []NullString(nil)}, + {col: "Bytes", val: nil, wantWithDefaultConfig: []byte(nil), wantWithNumber: []byte(nil)}, + {col: "BytesArray", val: nil, wantWithDefaultConfig: [][]byte(nil), wantWithNumber: [][]byte(nil)}, + {col: "Int64a", val: nil, wantWithDefaultConfig: NullInt64{}, wantWithNumber: NullInt64{}}, + {col: "Int64Array", val: nil, wantWithDefaultConfig: []NullInt64(nil), wantWithNumber: []NullInt64(nil)}, + {col: "Bool", val: nil, wantWithDefaultConfig: NullBool{}, wantWithNumber: NullBool{}}, + {col: "BoolArray", val: nil, wantWithDefaultConfig: []NullBool(nil), wantWithNumber: []NullBool(nil)}, + {col: "Float64", val: nil, wantWithDefaultConfig: NullFloat64{}, wantWithNumber: NullFloat64{}}, + {col: "Float64Array", val: nil, wantWithDefaultConfig: []NullFloat64(nil), wantWithNumber: []NullFloat64(nil)}, + {col: "Numeric", val: nil, wantWithDefaultConfig: NullNumeric{}, wantWithNumber: NullNumeric{}}, + {col: "NumericArray", val: nil, wantWithDefaultConfig: []NullNumeric(nil), wantWithNumber: []NullNumeric(nil)}, + {col: "JSON", val: nil, wantWithDefaultConfig: NullJSON{}, wantWithNumber: NullJSON{}}, + {col: "JSONArray", val: nil, wantWithDefaultConfig: []NullJSON(nil), wantWithNumber: []NullJSON(nil)}, } // See https://github.com/GoogleCloudPlatform/cloud-spanner-emulator/issues/31 if !isEmulatorEnvSet() { tests = append(tests, []struct { - col string - val interface{} - want interface{} + col string + val interface{} + wantWithDefaultConfig interface{} + wantWithNumber interface{} }{ - {col: "Date", val: nil, want: NullDate{}}, - {col: "Timestamp", val: nil, want: NullTime{}}, + {col: "Date", val: nil, wantWithDefaultConfig: NullDate{}, wantWithNumber: NullDate{}}, + {col: "Timestamp", val: nil, wantWithDefaultConfig: NullTime{}, wantWithNumber: NullTime{}}, }...) } - // Write rows into table first using DML. - statements := make([]Statement, 0) - for i, test := range tests { - stmt := NewStatement(fmt.Sprintf("INSERT INTO Types (RowId, `%s`) VALUES (@id, @value)", test.col)) - // Note: We are not setting the parameter type here to ensure that it - // can be automatically recognized when it is actually needed. - stmt.Params["id"] = i - stmt.Params["value"] = test.val - statements = append(statements, stmt) - } - _, err := client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *ReadWriteTransaction) error { - rowCounts, err := tx.BatchUpdate(ctx, statements) - if err != nil { - return err + for _, withNumberConfigOption := range []bool{false, true} { + if withNumberConfigOption { + UseNumberWithJSONDecoderEncoder(withNumberConfigOption) + defer UseNumberWithJSONDecoderEncoder(!withNumberConfigOption) } - if len(rowCounts) != len(tests) { - return fmt.Errorf("rowCounts length mismatch\nGot: %v\nWant: %v", len(rowCounts), len(tests)) + // Write rows into table first using DML. + statements := make([]Statement, 0) + for i, test := range tests { + stmt := NewStatement(fmt.Sprintf("INSERT INTO Types (RowId, `%s`) VALUES (@id, @value)", test.col)) + // Note: We are not setting the parameter type here to ensure that it + // can be automatically recognized when it is actually needed. + stmt.Params["id"] = i + stmt.Params["value"] = test.val + statements = append(statements, stmt) } - for i, c := range rowCounts { - if c != 1 { - return fmt.Errorf("row count mismatch for row %v:\nGot: %v\nWant: %v", i, c, 1) + _, err := client.ReadWriteTransaction(ctx, func(ctx context.Context, tx *ReadWriteTransaction) error { + rowCounts, err := tx.BatchUpdate(ctx, statements) + if err != nil { + return err + } + if len(rowCounts) != len(tests) { + return fmt.Errorf("rowCounts length mismatch\nGot: %v\nWant: %v", len(rowCounts), len(tests)) + } + for i, c := range rowCounts { + if c != 1 { + return fmt.Errorf("row count mismatch for row %v:\nGot: %v\nWant: %v", i, c, 1) + } } + return nil + }) + if err != nil { + t.Fatalf("failed to insert values using DML: %v", err) } - return nil - }) - if err != nil { - t.Fatalf("failed to insert values using DML: %v", err) - } - // Delete all the rows so we can insert them using mutations as well. - _, err = client.Apply(ctx, []*Mutation{Delete("Types", AllKeys())}) - if err != nil { - t.Fatalf("failed to delete all rows: %v", err) - } - - // Verify that we can insert the rows using mutations. - var muts []*Mutation - for i, test := range tests { - muts = append(muts, InsertOrUpdate("Types", []string{"RowID", test.col}, []interface{}{i, test.val})) - } - if _, err := client.Apply(ctx, muts, ApplyAtLeastOnce()); err != nil { - t.Fatal(err) - } - - for i, test := range tests { - row, err := client.Single().ReadRow(ctx, "Types", []interface{}{i}, []string{test.col}) + // Delete all the rows so we can insert them using mutations as well. + _, err = client.Apply(ctx, []*Mutation{Delete("Types", AllKeys())}) if err != nil { - t.Fatalf("Unable to fetch row %v: %v", i, err) + t.Fatalf("failed to delete all rows: %v", err) } - verifyDirectPathRemoteAddress(t) - // Create new instance of type of test.want. - want := test.want - if want == nil { - want = test.val + + // Verify that we can insert the rows using mutations. + var muts []*Mutation + for i, test := range tests { + muts = append(muts, InsertOrUpdate("Types", []string{"RowID", test.col}, []interface{}{i, test.val})) } - gotp := reflect.New(reflect.TypeOf(want)) - if err := row.Column(0, gotp.Interface()); err != nil { - t.Errorf("%d: col:%v val:%#v, %v", i, test.col, test.val, err) - continue + if _, err := client.Apply(ctx, muts, ApplyAtLeastOnce()); err != nil { + t.Fatal(err) } - got := reflect.Indirect(gotp).Interface() - // One of the test cases is checking NaN handling. Given - // NaN!=NaN, we can't use reflect to test for it. - if isNaN(got) && isNaN(want) { - continue - } + for i, test := range tests { + row, err := client.Single().ReadRow(ctx, "Types", []interface{}{i}, []string{test.col}) + if err != nil { + t.Fatalf("Unable to fetch row %v: %v", i, err) + } + verifyDirectPathRemoteAddress(t) + want := test.wantWithDefaultConfig + if withNumberConfigOption { + want = test.wantWithNumber + } + if want == nil { + want = test.val + } + gotp := reflect.New(reflect.TypeOf(want)) + if err := row.Column(0, gotp.Interface()); err != nil { + t.Errorf("%v-%d: col:%v val:%#v, %v", withNumberConfigOption, i, test.col, test.val, err) + continue + } + got := reflect.Indirect(gotp).Interface() - // Check non-NaN cases. - if !testEqual(got, want) { - t.Errorf("%d: col:%v val:%#v, got %#v, want %#v", i, test.col, test.val, got, want) - continue + // One of the test cases is checking NaN handling. Given + // NaN!=NaN, we can't use reflect to test for it. + if isNaN(got) && isNaN(want) { + continue + } + + // Check non-NaN cases. + if !testEqual(got, want) { + t.Errorf("%v-%d: col:%v val:%#v, got %#v, want %#v", withNumberConfigOption, i, test.col, test.val, got, want) + continue + } } + // cleanup + _, err = client.Apply(ctx, []*Mutation{Delete("Types", AllKeys())}) + require.NoError(t, err) } } diff --git a/spanner/value.go b/spanner/value.go index 28436eb00c31..d0360c5c16c0 100644 --- a/spanner/value.go +++ b/spanner/value.go @@ -21,7 +21,6 @@ import ( "database/sql" "database/sql/driver" "encoding/base64" - "encoding/json" "fmt" "math" "math/big" @@ -35,6 +34,7 @@ import ( sppb "cloud.google.com/go/spanner/apiv1/spannerpb" "github.com/golang/protobuf/proto" proto3 "github.com/golang/protobuf/ptypes/struct" + jsoniter "github.com/json-iterator/go" "google.golang.org/grpc/codes" ) @@ -113,8 +113,24 @@ var ( commitTimestamp = time.Unix(0, 0).In(time.FixedZone("CommitTimestamp placeholder", 0xDB)) jsonNullBytes = []byte("null") + + jsonProvider = jsoniter.ConfigCompatibleWithStandardLibrary ) +// UseNumberWithJSONDecoderEncoder specifies whether Cloud Spanner JSON numbers are decoded +// as Number (preserving precision) or float64 (risking loss). +// Defaults to the same behavior as the standard Go library, which means decoding to float64. +// Call this method to enable lossless precision. +// NOTE 1: Calling this method affects the behavior of all clients created by this library, both existing and future instances. +// NOTE 2: This method sets a global variable that is used by the client to encode/decode JSON numbers. Access to the global variable is not synchronized. You should only call this method when there are no goroutines encoding/decoding Cloud Spanner JSON values. It is recommended to only call this method during the initialization of your application, and preferably before you create any Cloud Spanner clients, and/or in tests when there are no queries being executed. +func UseNumberWithJSONDecoderEncoder(useNumber bool) { + jsonProvider = jsoniter.Config{ + EscapeHTML: true, + SortMapKeys: true, // Sort map keys to ensure deterministic output, to be consistent with encoding. + UseNumber: useNumber, + }.Froze() +} + // Encoder is the interface implemented by a custom type that can be encoded to // a supported type by Spanner. A code example: // @@ -281,7 +297,7 @@ func (n *NullString) UnmarshalJSON(payload []byte) error { return nil } var s *string - if err := json.Unmarshal(payload, &s); err != nil { + if err := jsonProvider.Unmarshal(payload, &s); err != nil { return err } if s != nil { @@ -682,10 +698,7 @@ func (n NullNumeric) String() string { // MarshalJSON implements json.Marshaler.MarshalJSON for NullNumeric. func (n NullNumeric) MarshalJSON() ([]byte, error) { - if n.Valid { - return json.Marshal(NumericString(&n.Numeric)) - } - return jsonNullBytes, nil + return nulljson(n.Valid, NumericString(&n.Numeric)) } // UnmarshalJSON implements json.Unmarshaler.UnmarshalJSON for NullNumeric. @@ -773,7 +786,7 @@ func (n NullJSON) String() string { if !n.Valid { return nullString } - b, err := json.Marshal(n.Value) + b, err := jsonProvider.Marshal(n.Value) if err != nil { return fmt.Sprintf("error: %v", err) } @@ -795,7 +808,7 @@ func (n *NullJSON) UnmarshalJSON(payload []byte) error { return nil } var v interface{} - err := json.Unmarshal(payload, &v) + err := jsonProvider.Unmarshal(payload, &v) if err != nil { return fmt.Errorf("payload cannot be converted to a struct: got %v, err: %w", string(payload), err) } @@ -879,7 +892,7 @@ func (n PGJsonB) String() string { if !n.Valid { return nullString } - b, err := json.Marshal(n.Value) + b, err := jsonProvider.Marshal(n.Value) if err != nil { return fmt.Sprintf("error: %v", err) } @@ -901,7 +914,7 @@ func (n *PGJsonB) UnmarshalJSON(payload []byte) error { return nil } var v interface{} - err := json.Unmarshal(payload, &v) + err := jsonProvider.Unmarshal(payload, &v) if err != nil { return fmt.Errorf("payload cannot be converted to a struct: got %v, err: %w", string(payload), err) } @@ -914,7 +927,7 @@ func nulljson(valid bool, v interface{}) ([]byte, error) { if !valid { return jsonNullBytes, nil } - return json.Marshal(v) + return jsonProvider.Marshal(v) } // GenericColumnValue represents the generic encoded value and type of the @@ -1525,7 +1538,7 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}, opts ...decodeO } x := v.GetStringValue() var y interface{} - err := json.Unmarshal([]byte(x), &y) + err := jsonProvider.Unmarshal([]byte(x), &y) if err != nil { return err } @@ -1684,7 +1697,7 @@ func decodeValue(v *proto3.Value, t *sppb.Type, ptr interface{}, opts ...decodeO } x := v.GetStringValue() var y interface{} - err := json.Unmarshal([]byte(x), &y) + err := jsonProvider.Unmarshal([]byte(x), &y) if err != nil { return err } @@ -2345,7 +2358,7 @@ func (dsc decodableSpannerType) decodeValueToCustomType(v *proto3.Value, t *sppb } x := v.GetStringValue() var y interface{} - err := json.Unmarshal([]byte(x), &y) + err := jsonProvider.Unmarshal([]byte(x), &y) if err != nil { return err } @@ -2360,7 +2373,7 @@ func (dsc decodableSpannerType) decodeValueToCustomType(v *proto3.Value, t *sppb } x := v.GetStringValue() var y interface{} - err := json.Unmarshal([]byte(x), &y) + err := jsonProvider.Unmarshal([]byte(x), &y) if err != nil { return err } @@ -2958,7 +2971,7 @@ func decodeNullJSONArrayToNullJSON(pb *proto3.ListValue) (*NullJSON, error) { } s := fmt.Sprintf("[%s]", strings.Join(strs, ",")) var y interface{} - err := json.Unmarshal([]byte(s), &y) + err := jsonProvider.Unmarshal([]byte(s), &y) if err != nil { return nil, err } @@ -3573,7 +3586,7 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { pt = listType(pgNumericType()) case NullJSON: if v.Valid { - b, err := json.Marshal(v.Value) + b, err := jsonProvider.Marshal(v.Value) if err != nil { return nil, nil, err } @@ -3590,7 +3603,7 @@ func encodeValue(v interface{}) (*proto3.Value, *sppb.Type, error) { pt = listType(jsonType()) case PGJsonB: if v.Valid { - b, err := json.Marshal(v.Value) + b, err := jsonProvider.Marshal(v.Value) if err != nil { return nil, nil, err }