Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(orm): add module db #10991

Merged
merged 14 commits into from
Jan 22, 2022
25 changes: 9 additions & 16 deletions orm/encoding/ormkv/entry.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,10 @@ import (
"fmt"
"strings"

"google.golang.org/protobuf/encoding/protojson"
"github.com/cosmos/cosmos-sdk/orm/internal/stablejson"

"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protoreflect"
"google.golang.org/protobuf/types/known/structpb"
)

// Entry defines a logical representation of a kv-store entry for ORM instances.
Expand Down Expand Up @@ -45,7 +45,12 @@ func (p *PrimaryKeyEntry) String() string {
if p.Value == nil {
return fmt.Sprintf("PK %s %s -> _", p.TableName, fmtValues(p.Key))
} else {
return fmt.Sprintf("PK %s %s -> %s", p.TableName, fmtValues(p.Key), p.Value)
valBz, err := stablejson.Marshal(p.Value)
valStr := string(valBz)
if err != nil {
valStr = fmt.Sprintf("ERR %v", err)
}
return fmt.Sprintf("PK %s %s -> %s", p.TableName, fmtValues(p.Key), valStr)
aaronc marked this conversation as resolved.
Show resolved Hide resolved
}
}

Expand All @@ -56,19 +61,7 @@ func fmtValues(values []protoreflect.Value) string {

parts := make([]string, len(values))
for i, v := range values {
val, err := structpb.NewValue(v.Interface())
if err != nil {
parts[i] = "ERR"
continue
}

bz, err := protojson.Marshal(val)
if err != nil {
parts[i] = "ERR"
continue
}

parts[i] = string(bz)
parts[i] = fmt.Sprintf("%v", v.Interface())
aaronc marked this conversation as resolved.
Show resolved Hide resolved
}

return strings.Join(parts, "/")
Expand Down
10 changes: 9 additions & 1 deletion orm/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@ go 1.17

require (
github.com/cosmos/cosmos-proto v1.0.0-alpha6
github.com/cosmos/cosmos-sdk/api v0.1.0-alpha2
github.com/cosmos/cosmos-sdk/api v0.1.0-alpha3
github.com/cosmos/cosmos-sdk/errors v1.0.0-beta.2
github.com/stretchr/testify v1.7.0
github.com/tendermint/tm-db v0.6.6
google.golang.org/protobuf v1.27.1
gotest.tools/v3 v3.1.0
Expand All @@ -15,26 +16,33 @@ require (
require (
github.com/DataDog/zstd v1.4.5 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgraph-io/badger/v2 v2.2007.2 // indirect
github.com/dgraph-io/ristretto v0.0.3 // indirect
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
github.com/dustin/go-humanize v1.0.0 // indirect
github.com/facebookgo/ensure v0.0.0-20160127193407-b4ab57deab51 // indirect
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/subset v0.0.0-20150612182917-8dac2c3c4870 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.3-0.20201103224600-674baa8c7fc3 // indirect
github.com/google/btree v1.0.0 // indirect
github.com/google/go-cmp v0.5.5 // indirect
github.com/jmhodges/levigo v1.0.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca // indirect
github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect
go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f // indirect
golang.org/x/sys v0.0.0-20210903071746-97244b99971b // indirect
golang.org/x/text v0.3.6 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
google.golang.org/grpc v1.43.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
)

replace github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
11 changes: 8 additions & 3 deletions orm/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -23,11 +23,10 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
github.com/cosmos/cosmos-proto v1.0.0-alpha4/go.mod h1:msdDWOvfStHLG+Z2y2SJ0dcqimZ2vc8M1MPnZ4jOF7U=
github.com/cosmos/cosmos-proto v1.0.0-alpha6 h1:N2BvV2AyzGAXCJnvlw1pMzEQ+76tj5FDBrkYQYIDCdU=
github.com/cosmos/cosmos-proto v1.0.0-alpha6/go.mod h1:msdDWOvfStHLG+Z2y2SJ0dcqimZ2vc8M1MPnZ4jOF7U=
github.com/cosmos/cosmos-sdk/api v0.1.0-alpha2 h1:47aK2mZ8oh3wyr5Q4OiZxyrMkQZRW67Ah/HfC8dW8hs=
github.com/cosmos/cosmos-sdk/api v0.1.0-alpha2/go.mod h1:xWm3hne2f6upv80eIS+fJnnUaed/R2EJno1It4Zb9aw=
github.com/cosmos/cosmos-sdk/api v0.1.0-alpha3 h1:tqpedvX/1UgQyX3W2hlW6Xg801FyojYT/NOHbW0oG+s=
github.com/cosmos/cosmos-sdk/api v0.1.0-alpha3/go.mod h1:Ht15guGn9F8b0lv8NkjXE9/asAvVUOt2n4gvQ4q5MyU=
github.com/cosmos/cosmos-sdk/errors v1.0.0-beta.2 h1:bBglNlra8ZHb4dmbEE8V85ihLA+DkriSm7tcx6x/JWo=
github.com/cosmos/cosmos-sdk/errors v1.0.0-beta.2/go.mod h1:Gi7pzVRnvZ1N16JAXpLADzng0ePoE7YeEHaULSFB2Ts=
github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
Expand Down Expand Up @@ -100,8 +99,10 @@ github.com/jmhodges/levigo v1.0.0 h1:q5EC36kV79HWeTBWsod3mG11EgStG3qArTKcvlksN1U
github.com/jmhodges/levigo v1.0.0/go.mod h1:Q6Qx+uH3RAqyK4rFQroq9RL7mdkABMcfhEI+nNuzMJQ=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
Expand All @@ -122,6 +123,7 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/regen-network/protobuf v1.3.3-alpha.regen.1 h1:OHEc+q5iIAXpqiqFKeLpu5NwTIkVXUs48vFMwzqpqY4=
github.com/regen-network/protobuf v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
Expand Down Expand Up @@ -237,6 +239,7 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98
google.golang.org/genproto v0.0.0-20200324203455-a04cca1dde73/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb h1:ZrsicilzPCS/Xr8qtBZZLpy4P9TYXAfl49ctG1/5tgw=
google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
Expand All @@ -247,6 +250,7 @@ google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTp
google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/grpc v1.43.0 h1:Eeu7bZtDZ2DpRCsLhUlcrLnvYaMK1Gz86a+hMVvELmM=
google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
Expand All @@ -263,6 +267,7 @@ google.golang.org/protobuf v1.27.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
Expand Down
93 changes: 93 additions & 0 deletions orm/internal/stablejson/encode.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
package stablejson

import (
"bytes"
"fmt"

"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/reflect/protopath"
"google.golang.org/protobuf/reflect/protorange"
"google.golang.org/protobuf/reflect/protoreflect"
)

// Marshal marshals the provided message to JSON with a stable ordering based
// on ascending field numbers.
func Marshal(message proto.Message) ([]byte, error) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

haven't done a deep check, but maybe there are some protobuf well-known types that need to be json marshalled in a custom way, examples are: timestamp, struct, etc.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes, you're right... hopefully it's not too bad. will take a look soon

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think for now, maybe this is okay just because the main use case is to do stable marshaling for tests. I'm mainly trying to prevent spurious CI failures rather than a full fledged encoder

buf := &bytes.Buffer{}
firstStack := []bool{true}
err := protorange.Options{
Stable: true,
}.Range(message.ProtoReflect(),
func(p protopath.Values) error {
// Starting printing the value.
if !firstStack[len(firstStack)-1] {
_, _ = fmt.Fprintf(buf, ",")
}
firstStack[len(firstStack)-1] = false

// Print the key.
var fd protoreflect.FieldDescriptor
last := p.Index(-1)
beforeLast := p.Index(-2)
switch last.Step.Kind() {
case protopath.FieldAccessStep:
fd = last.Step.FieldDescriptor()
_, _ = fmt.Fprintf(buf, "%q:", fd.Name())
case protopath.ListIndexStep:
fd = beforeLast.Step.FieldDescriptor() // lists always appear in the context of a repeated field
case protopath.MapIndexStep:
fd = beforeLast.Step.FieldDescriptor() // maps always appear in the context of a repeated field
_, _ = fmt.Fprintf(buf, "%v:", last.Step.MapIndex().Interface())
case protopath.AnyExpandStep:
_, _ = fmt.Fprintf(buf, `"@type":%q`, last.Value.Message().Descriptor().FullName())
return nil
case protopath.UnknownAccessStep:
_, _ = fmt.Fprintf(buf, "?: ")
}

switch v := last.Value.Interface().(type) {
case protoreflect.Message:
_, _ = fmt.Fprintf(buf, "{")
firstStack = append(firstStack, true)
case protoreflect.List:
_, _ = fmt.Fprintf(buf, "[")
firstStack = append(firstStack, true)
case protoreflect.Map:
_, _ = fmt.Fprintf(buf, "{")
firstStack = append(firstStack, true)
case protoreflect.EnumNumber:
var ev protoreflect.EnumValueDescriptor
if fd != nil {
ev = fd.Enum().Values().ByNumber(v)
}
if ev != nil {
_, _ = fmt.Fprintf(buf, "%v", ev.Name())
} else {
_, _ = fmt.Fprintf(buf, "%v", v)
}
case string, []byte:
_, _ = fmt.Fprintf(buf, "%q", v)
default:
_, _ = fmt.Fprintf(buf, "%v", v)
}
return nil
},
func(p protopath.Values) error {
last := p.Index(-1)
switch last.Value.Interface().(type) {
case protoreflect.Message:
if last.Step.Kind() != protopath.AnyExpandStep {
_, _ = fmt.Fprintf(buf, "}")
}
case protoreflect.List:
_, _ = fmt.Fprintf(buf, "]")
firstStack = firstStack[:len(firstStack)-1]
case protoreflect.Map:
_, _ = fmt.Fprintf(buf, "}")
firstStack = firstStack[:len(firstStack)-1]
}
return nil
},
)
return buf.Bytes(), err
}
37 changes: 37 additions & 0 deletions orm/internal/stablejson/encode_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package stablejson_test

import (
"testing"

"github.com/stretchr/testify/require"

"google.golang.org/protobuf/types/known/anypb"

bankv1beta1 "github.com/cosmos/cosmos-sdk/api/cosmos/bank/v1beta1"
basev1beta1 "github.com/cosmos/cosmos-sdk/api/cosmos/base/v1beta1"
txv1beta1 "github.com/cosmos/cosmos-sdk/api/cosmos/tx/v1beta1"
"github.com/cosmos/cosmos-sdk/orm/internal/stablejson"
)

func TestStableJSON(t *testing.T) {
msg, err := anypb.New(&bankv1beta1.MsgSend{
FromAddress: "foo213325",
ToAddress: "foo32t5sdfh",
Amount: []*basev1beta1.Coin{
{
Denom: "bar",
Amount: "1234",
},
{
Denom: "baz",
Amount: "321",
},
},
})
require.NoError(t, err)
bz, err := stablejson.Marshal(&txv1beta1.TxBody{Messages: []*anypb.Any{msg}})
require.NoError(t, err)
require.Equal(t,
`{"messages":[{"@type":"cosmos.bank.v1beta1.MsgSend","from_address":"foo213325","to_address":"foo32t5sdfh","amount":[{"denom":"bar","amount":"1234"},{"denom":"baz","amount":"321"}]}]}`,
string(bz))
}
30 changes: 26 additions & 4 deletions orm/internal/testkv/debug.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ package testkv
import (
"fmt"

"github.com/cosmos/cosmos-sdk/orm/internal/stablejson"

"google.golang.org/protobuf/proto"

"github.com/cosmos/cosmos-sdk/orm/encoding/ormkv"
Expand Down Expand Up @@ -210,10 +212,15 @@ type debugHooks struct {
}

func (d debugHooks) OnInsert(message proto.Message) error {
jsonBz, err := stablejson.Marshal(message)
if err != nil {
return err
}

d.debugger.Log(fmt.Sprintf(
"ORM INSERT %s %s",
message.ProtoReflect().Descriptor().FullName(),
message,
jsonBz,
))
if d.hooks != nil {
return d.hooks.OnInsert(message)
Expand All @@ -222,11 +229,21 @@ func (d debugHooks) OnInsert(message proto.Message) error {
}

func (d debugHooks) OnUpdate(existing, new proto.Message) error {
existingJson, err := stablejson.Marshal(existing)
if err != nil {
return err
}

newJson, err := stablejson.Marshal(new)
if err != nil {
return err
}

d.debugger.Log(fmt.Sprintf(
"ORM UPDATE %s %s -> %s",
existing.ProtoReflect().Descriptor().FullName(),
existing,
new,
existingJson,
newJson,
))
if d.hooks != nil {
return d.hooks.OnUpdate(existing, new)
Expand All @@ -235,10 +252,15 @@ func (d debugHooks) OnUpdate(existing, new proto.Message) error {
}

func (d debugHooks) OnDelete(message proto.Message) error {
jsonBz, err := stablejson.Marshal(message)
if err != nil {
return err
}

d.debugger.Log(fmt.Sprintf(
"ORM DELETE %s %s",
message.ProtoReflect().Descriptor().FullName(),
message,
jsonBz,
))
if d.hooks != nil {
return d.hooks.OnDelete(message)
Expand Down
31 changes: 31 additions & 0 deletions orm/internal/testpb/bank.proto
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
syntax = "proto3";

package testpb;

import "cosmos/orm/v1alpha1/orm.proto";

option go_package = "github.com/cosmos/cosmos-sdk/orm/internal/testpb";

// This is a simulated bank schema used for testing.

message Balance {
option (cosmos.orm.v1alpha1.table) = {
id: 1;
primary_key:{fields: "address,denom"}
index: {id: 1 fields: "denom"}
};

string address = 1;
string denom = 2;
uint64 amount = 3;
}

message Supply {
option (cosmos.orm.v1alpha1.table) = {
id: 2;
primary_key:{fields: "denom"}
};

string denom = 1;
uint64 amount = 2;
}
Loading