From 1b764e57768573ef8dcf32508051c504fe749153 Mon Sep 17 00:00:00 2001 From: brennan lamey Date: Fri, 11 Aug 2023 00:59:34 -0500 Subject: [PATCH 1/8] added a new theoretical structure for transactions, signatures, and types --- go.mod | 1 + go.sum | 9 ++ pkg/crypto/crypto.go | 22 +++++ pkg/crypto/ed25519.go | 61 +++++++++++++ pkg/crypto/keys.go | 24 +++++ pkg/crypto/secp256k1.go | 72 +++++++++++++++ pkg/crypto/signature.go | 38 ++++---- pkg/serialize/rlp/encode.go | 64 ++++++++++++++ pkg/serialize/rlp/encode_test.go | 109 +++++++++++++++++++++++ pkg/serialize/rlp/value.go | 59 +++++++++++++ pkg/transactions/errors.go | 7 ++ pkg/transactions/message.go | 55 ++++++++++++ pkg/types/execute.go | 43 +++++++++ pkg/types/schema.go | 127 +++++++++++++++++++++++++++ pkg/types/types_test.go | 146 +++++++++++++++++++++++++++++++ pkg/types/validators.go | 1 + 16 files changed, 819 insertions(+), 19 deletions(-) create mode 100644 pkg/crypto/ed25519.go create mode 100644 pkg/crypto/keys.go create mode 100644 pkg/crypto/secp256k1.go create mode 100644 pkg/serialize/rlp/encode.go create mode 100644 pkg/serialize/rlp/encode_test.go create mode 100644 pkg/serialize/rlp/value.go create mode 100644 pkg/transactions/errors.go create mode 100644 pkg/transactions/message.go create mode 100644 pkg/types/execute.go create mode 100644 pkg/types/schema.go create mode 100644 pkg/types/types_test.go create mode 100644 pkg/types/validators.go diff --git a/go.mod b/go.mod index e59bc4d27..2382735aa 100644 --- a/go.mod +++ b/go.mod @@ -117,6 +117,7 @@ require ( github.com/moby/sys/symlink v0.2.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/oasisprotocol/curve25519-voi v0.0.0-20230110094441-db37f07504ce // indirect github.com/onsi/gomega v1.16.0 // indirect github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/prometheus/client_golang v1.14.0 // indirect diff --git a/go.sum b/go.sum index 65a7e4819..bd1dcef5e 100644 --- a/go.sum +++ b/go.sum @@ -94,7 +94,9 @@ github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrd github.com/adlio/schema v1.3.3 h1:oBJn8I02PyTB466pZO1UZEn1TV5XLlifBSyMrmHl/1I= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alexliesenfeld/health v0.6.0 h1:HRBTCgybNSe4lqGEk7nU82c3bjwh9W+3b46W6UvD4CQ= github.com/alexliesenfeld/health v0.6.0/go.mod h1:N4NDIeQtlWumG+6z1ne1v62eQxktz5ylEgGgH9emdMw= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -492,6 +494,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaW github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5 h1:3IZOAnD058zZllQTZNBioTlrzrBG/IjpiZ133IEtusM= github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.0.0-rc.5/go.mod h1:xbKERva94Pw2cPen0s79J3uXmGzbbpDYFBFDlZ4mV/w= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.12.0 h1:kr3j8iIMR4ywO/O0rvksXaJvauGGCMg2zAZIiNZ9uIQ= @@ -552,6 +555,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= @@ -660,11 +664,14 @@ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7P github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/oasisprotocol/curve25519-voi v0.0.0-20230110094441-db37f07504ce h1:/pEpMk55wH0X+E5zedGEMOdLuWmV8P4+4W3+LZaM6kg= +github.com/oasisprotocol/curve25519-voi v0.0.0-20230110094441-db37f07504ce/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -752,6 +759,7 @@ github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZV github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= @@ -806,6 +814,7 @@ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= diff --git a/pkg/crypto/crypto.go b/pkg/crypto/crypto.go index 808577f1b..32aa699fa 100644 --- a/pkg/crypto/crypto.go +++ b/pkg/crypto/crypto.go @@ -1,9 +1,14 @@ package crypto import ( + "crypto/ecdsa" c256 "crypto/sha256" c512 "crypto/sha512" "encoding/hex" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + ec "github.com/ethereum/go-ethereum/crypto" ) // Sha384 returns the sha384 hash of the data. @@ -36,3 +41,20 @@ func Sha256(data []byte) []byte { func Sha256Hex(data []byte) string { return hex.EncodeToString(Sha256(data)) } + +func ECDSAFromHex(hex string) (*ecdsa.PrivateKey, error) { + return ec.HexToECDSA(hex) +} + +func AddressFromPrivateKey(key *ecdsa.PrivateKey) string { + caddr := ec.PubkeyToAddress(key.PublicKey) + return caddr.Hex() +} + +func IsValidAddress(addr string) bool { + return common.IsHexAddress(addr) +} + +func HexFromECDSAPrivateKey(key *ecdsa.PrivateKey) string { + return hexutil.Encode(ec.FromECDSA(key))[2:] +} diff --git a/pkg/crypto/ed25519.go b/pkg/crypto/ed25519.go new file mode 100644 index 000000000..b8ded3082 --- /dev/null +++ b/pkg/crypto/ed25519.go @@ -0,0 +1,61 @@ +package crypto + +import ( + oasisEd25519 "github.com/oasisprotocol/curve25519-voi/primitives/ed25519" +) + +const Ed25519 KeyType = "Ed25519" + +type Ed25519PrivateKey struct { + privateKey []byte +} + +func (s *Ed25519PrivateKey) Bytes() []byte { + return s.privateKey +} + +func (s *Ed25519PrivateKey) PubKey() PublicKey { + panic("implement me") +} + +func (s *Ed25519PrivateKey) Sign(msg []byte, signatureType SignatureType) ([]byte, error) { + return oasisEd25519.Sign(oasisEd25519.PrivateKey(s.privateKey), msg), nil +} + +func (s *Ed25519PrivateKey) Type() KeyType { + return Ed25519 +} + +type Ed25519PublicKey struct { +} + +func (s *Ed25519PublicKey) Address() Address { + panic("implement me") +} + +func (s *Ed25519PublicKey) Bytes() []byte { + panic("implement me") +} + +func (s *Ed25519PublicKey) Type() KeyType { + return Ed25519 +} + +func (s *Ed25519PublicKey) Verify(sig *Signature) error { + panic("implement me") +} + +type Ed25519Address struct { +} + +func (s *Ed25519Address) Bytes() []byte { + panic("implement me") +} + +func (s *Ed25519Address) Type() KeyType { + return Ed25519 +} + +func (s *Ed25519Address) String() string { + panic("implement me") +} diff --git a/pkg/crypto/keys.go b/pkg/crypto/keys.go new file mode 100644 index 000000000..155041ed1 --- /dev/null +++ b/pkg/crypto/keys.go @@ -0,0 +1,24 @@ +package crypto + +type KeyType string + +type PrivateKey interface { + Bytes() []byte + Type() KeyType + Sign(msg []byte, signatureType SignatureType) ([]byte, error) + PubKey() PublicKey +} + +type PublicKey interface { + Bytes() []byte + Type() KeyType + Verify(sign *Signature) error + Address() Address +} + +type Address interface { + Bytes() []byte + String() string + // do we need to know the key type? + Type() KeyType +} diff --git a/pkg/crypto/secp256k1.go b/pkg/crypto/secp256k1.go new file mode 100644 index 000000000..8258ad9cf --- /dev/null +++ b/pkg/crypto/secp256k1.go @@ -0,0 +1,72 @@ +package crypto + +import ( + "crypto/ecdsa" + + "github.com/ethereum/go-ethereum/common" + ethCrypto "github.com/ethereum/go-ethereum/crypto" +) + +const Secp256k1 KeyType = "secp256k1" + +type Secp256k1PrivateKey struct { + privateKey *ecdsa.PrivateKey +} + +func (s *Secp256k1PrivateKey) Bytes() []byte { + return ethCrypto.FromECDSA(s.privateKey) +} + +func (s *Secp256k1PrivateKey) PubKey() PublicKey { + return &Secp256k1PublicKey{ + publicKey: &s.privateKey.PublicKey, + } +} + +func (s *Secp256k1PrivateKey) Sign(msg []byte, signatureType SignatureType) ([]byte, error) { + // TODO: implement + panic("TODO") +} + +func (s *Secp256k1PrivateKey) Type() KeyType { + return Secp256k1 +} + +type Secp256k1PublicKey struct { + publicKey *ecdsa.PublicKey +} + +func (s *Secp256k1PublicKey) Address() Address { + return &Secp256k1Address{ + address: ethCrypto.PubkeyToAddress(*s.publicKey), + } +} + +func (s *Secp256k1PublicKey) Bytes() []byte { + return ethCrypto.FromECDSAPub(s.publicKey) +} + +func (s *Secp256k1PublicKey) Type() KeyType { + return Secp256k1 +} + +func (s *Secp256k1PublicKey) Verify(sig *Signature) error { + // TODO: implement + panic("TODO") +} + +type Secp256k1Address struct { + address common.Address +} + +func (s *Secp256k1Address) Bytes() []byte { + return s.address.Bytes() +} + +func (s *Secp256k1Address) Type() KeyType { + return Secp256k1 +} + +func (s *Secp256k1Address) String() string { + return s.address.String() +} diff --git a/pkg/crypto/signature.go b/pkg/crypto/signature.go index 177fa5a23..a77b8dfdb 100644 --- a/pkg/crypto/signature.go +++ b/pkg/crypto/signature.go @@ -9,7 +9,6 @@ import ( cmtjson "github.com/cometbft/cometbft/libs/json" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" ec "github.com/ethereum/go-ethereum/crypto" ) @@ -19,6 +18,12 @@ type SignatureType int32 const ( SIGNATURE_TYPE_INVALID SignatureType = iota + + // all of these different SECP256k1 signatures should + // be revisited. If they are warranted, then at the very least + // they should be renamed to be more descriptive + // it could also be worth giving them string representations, + // instead of just numbers PK_SECP256K1_UNCOMPRESSED ACCOUNT_SECP256K1_UNCOMPRESSED ACCOUNT_SECP256K1_UNCOMPRESSED_STRUCTURED_v1 @@ -38,10 +43,21 @@ func (s SignatureType) Int32() int32 { } type Signature struct { - Signature []byte `json:"signature_bytes"` + Message []byte `json:"message"` + Signature []byte `json:"signature_bytes"` + Sender PublicKey Type SignatureType `json:"signature_type"` } +func (s *Signature) Verify() error { + return s.Sender.Verify(s) +} + +/* + everything below this is considered "deprecated", and is only left for backwards compatibility + while the new signature is being implemented +*/ + func Sign(data []byte, k *ecdsa.PrivateKey) (*Signature, error) { signature := Signature{ Type: PK_SECP256K1_UNCOMPRESSED, @@ -56,6 +72,7 @@ func Sign(data []byte, k *ecdsa.PrivateKey) (*Signature, error) { return &signature, nil } +// Deprecated: use Verify instead func (s *Signature) Check(sender string, data []byte) error { ok, err := CheckSignature(sender, s, data) if err != nil { @@ -69,23 +86,6 @@ func (s *Signature) Check(sender string, data []byte) error { return nil } -func ECDSAFromHex(hex string) (*ecdsa.PrivateKey, error) { - return ec.HexToECDSA(hex) -} - -func AddressFromPrivateKey(key *ecdsa.PrivateKey) string { - caddr := ec.PubkeyToAddress(key.PublicKey) - return caddr.Hex() -} - -func IsValidAddress(addr string) bool { - return common.IsHexAddress(addr) -} - -func HexFromECDSAPrivateKey(key *ecdsa.PrivateKey) string { - return hexutil.Encode(ec.FromECDSA(key))[2:] -} - func CheckSignature(addr string, sig *Signature, data []byte) (bool, error) { if addr == "" { return false, fmt.Errorf("transaction does not have a sender address") diff --git a/pkg/serialize/rlp/encode.go b/pkg/serialize/rlp/encode.go new file mode 100644 index 000000000..f90d7d255 --- /dev/null +++ b/pkg/serialize/rlp/encode.go @@ -0,0 +1,64 @@ +package rlp + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/rlp" +) + +type encodingType uint16 + +const ( + // it is very important that the order of the encoding types is not changed + encodingTypeInvalid encodingType = iota + encodingTypeRLP +) + +var currentEncodingType = encodingTypeRLP + +func Encode(val any) ([]byte, error) { + var btsVal []byte + var err error + switch currentEncodingType { + case encodingTypeRLP: + btsVal, err = encodeRLP(val) + default: + return nil, fmt.Errorf("invalid encoding type: %d", currentEncodingType) + } + if err != nil { + return nil, err + } + + return serializeEncodedValue(&encodedValue{ + EncodingType: currentEncodingType, + Data: btsVal, + }) +} + +func Decode[T any](bts []byte) (*T, error) { + encVal, err := deserializeEncodedValue(bts) + if err != nil { + return nil, err + } + + switch encVal.EncodingType { + case encodingTypeRLP: + return decodeRLP[T](encVal.Data) + default: + return nil, fmt.Errorf("invalid encoding type: %d", encVal.EncodingType) + } +} + +func encodeRLP(val any) ([]byte, error) { + return rlp.EncodeToBytes(val) +} + +func decodeRLP[T any](bts []byte) (*T, error) { + var val T + err := rlp.DecodeBytes(bts, &val) + if err != nil { + return nil, err + } + + return &val, nil +} diff --git a/pkg/serialize/rlp/encode_test.go b/pkg/serialize/rlp/encode_test.go new file mode 100644 index 000000000..dc61e97c0 --- /dev/null +++ b/pkg/serialize/rlp/encode_test.go @@ -0,0 +1,109 @@ +package rlp_test + +import ( + "testing" + + "github.com/kwilteam/kwil-db/pkg/serialize/rlp" + "github.com/stretchr/testify/assert" +) + +func Test_Encoding(t *testing.T) { + type testCase struct { + name string + input any + output any + encodingErr bool + } + + testCases := []testCase{ + { + name: "valid struct", + input: TestStruct1{ + Val1: 1, + Val3: []byte("test"), + Val2: "test", + Val4: true, + }, + output: TestStruct1{ + Val1: 1, + Val2: "test", + Val3: []byte("test"), + Val4: true, + }, + }, + { + name: "unexported field", + input: StructUnexportedField{ + val1: 1, + Val2: "test", + }, + output: StructUnexportedField{ + Val2: "test", + }, + }, + { + name: "invalid struct - signed int", + input: InvalidStructSignedInt{ + Val1: -1, + }, + encodingErr: true, + }, + { + name: "invalid struct - map", + input: InvalidStructMap{ + Val1: map[string]string{ + "test": "test", + }, + }, + encodingErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + output, err := rlp.Encode(tc.input) + if tc.encodingErr && err == nil { + t.Errorf("Expected error, got nil") + } + if !tc.encodingErr && err != nil { + t.Errorf("Expected no error, got %v", err) + } + if tc.encodingErr { + return + } + + decoded, err := rlp.Decode[any](output) + if err != nil { + t.Errorf("Expected no error, got %v", err) + } + if err != nil { + return + } + + if *decoded == tc.output { + assert.Equal(t, tc.output, *decoded) + } + }) + } +} + +type TestStruct1 struct { + Val1 uint64 + Val2 string + Val3 []byte + Val4 bool +} + +type InvalidStructSignedInt struct { + Val1 int64 // RLPEncode only supports unsigned integers +} + +type InvalidStructMap struct { + Val1 map[string]string // RLPEncode does not support maps +} + +// will not error, but will not encode +type StructUnexportedField struct { + val1 uint64 // RLPEncode only supports exported fields + Val2 string +} diff --git a/pkg/serialize/rlp/value.go b/pkg/serialize/rlp/value.go new file mode 100644 index 000000000..cbaa4197c --- /dev/null +++ b/pkg/serialize/rlp/value.go @@ -0,0 +1,59 @@ +package rlp + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +type encodedValue struct { + EncodingType encodingType + Data []byte +} + +func serializeEncodedValue(value *encodedValue) ([]byte, error) { + result, err := uint16ToBytes(uint16(value.EncodingType)) + if err != nil { + return nil, err + } + + return append(result, value.Data...), nil +} + +func deserializeEncodedValue(data []byte) (*encodedValue, error) { + if len(data) == 0 { + return nil, fmt.Errorf("cannot deserialize encoded value: data is empty") + } + typ, err := bytesToUint16(data[:2]) + if err != nil { + return nil, err + } + + return &encodedValue{ + EncodingType: encodingType(typ), + Data: data[2:], + }, nil +} + +func uint16ToBytes(n uint16) ([]byte, error) { + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.BigEndian, n) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func bytesToUint16(b []byte) (uint16, error) { + if len(b) < 2 { + return 0, fmt.Errorf("cannot convert bytes to uint16: bytes are too short") + } + + buf := bytes.NewReader(b) + var n uint16 + err := binary.Read(buf, binary.BigEndian, &n) + if err != nil { + return 0, err + } + return n, nil +} diff --git a/pkg/transactions/errors.go b/pkg/transactions/errors.go new file mode 100644 index 000000000..4be7f22d7 --- /dev/null +++ b/pkg/transactions/errors.go @@ -0,0 +1,7 @@ +package transactions + +import "errors" + +var ( + ErrFailedHashReconstruction = errors.New("failed to reconstruct hash") +) diff --git a/pkg/transactions/message.go b/pkg/transactions/message.go new file mode 100644 index 000000000..6763ba3ba --- /dev/null +++ b/pkg/transactions/message.go @@ -0,0 +1,55 @@ +/* +Package transactions contains all the logic for creating and validating +transactions and signed messages. +*/ +package transactions + +import ( + "bytes" + "math/big" + + "github.com/kwilteam/kwil-db/pkg/crypto" +) + +// SignedMessage is any message that has been signed by a private key +// It contains a signature and a payload. The message in the signature +// should be the hash of the payload. +type SignedMessage struct { + Signature *crypto.Signature + Message Serializable +} + +// Verify verifies the authenticity of a signed message. +// It does this by reconstructing the hash of the payload and comparing +// it to the message in the signature. +// It then uses the public key in the signature to verify the signature. +func (s *SignedMessage) Verify() error { + messageBytes, err := s.Message.Bytes() + if err != nil { + return err + } + + if !bytes.Equal(s.Signature.Message, crypto.Sha256(messageBytes)) { + return ErrFailedHashReconstruction + } + + return s.Verify() +} + +// Serializable is any message that can be hashed +// This hash is used to both sign the message, as well as reconstruct the +// hash (if we are checking the message) to verify the signature +type Serializable interface { + Bytes() ([]byte, error) +} + +// TransactionMessageis the payload of a transaction +// It contains information on the nonce, fee, and the transaction payload +// The type of the transaction can be derived from the payload +type TransactionMessage struct { + Nonce int64 + Fee *big.Int +} + +// AuthenticatedMessage is a message that has been signed by a private key +// it is not a valid blockchain transaction, but can be used to authenticate diff --git a/pkg/types/execute.go b/pkg/types/execute.go new file mode 100644 index 000000000..32b24445a --- /dev/null +++ b/pkg/types/execute.go @@ -0,0 +1,43 @@ +package types + +import "github.com/kwilteam/kwil-db/pkg/serialize/rlp" + +type ActionExecution struct { + DBID string + Action string + Arguments [][]string +} + +func (a *ActionExecution) Bytes() ([]byte, error) { + return rlp.Encode(a) +} + +func (s *ActionExecution) FromBytes(b []byte) error { + res, err := rlp.Decode[ActionExecution](b) + if err != nil { + return err + } + + *s = *res + return nil +} + +type ActionCall struct { + DBID string + Action string + Arguments []string +} + +func (a *ActionCall) Bytes() ([]byte, error) { + return rlp.Encode(a) +} + +func (s *ActionCall) FromBytes(b []byte) error { + res, err := rlp.Decode[ActionCall](b) + if err != nil { + return err + } + + *s = *res + return nil +} diff --git a/pkg/types/schema.go b/pkg/types/schema.go new file mode 100644 index 000000000..dd4e79477 --- /dev/null +++ b/pkg/types/schema.go @@ -0,0 +1,127 @@ +package types + +import "github.com/kwilteam/kwil-db/pkg/serialize/rlp" + +type Schema struct { + Owner string + Name string + Tables []*Table + Actions []*Action + Extensions []*Extension +} + +func (s *Schema) Bytes() ([]byte, error) { + return rlp.Encode(s) +} + +func (s *Schema) FromBytes(b []byte) error { + result, err := rlp.Decode[Schema](b) + if err != nil { + return err + } + + *s = *result + return nil +} + +type Extension struct { + Name string + Config []*ExtensionConfig + Alias string +} + +type ExtensionConfig struct { + Argument string + Value string +} + +type Table struct { + Name string + Columns []*Column + Indexes []*Index + ForeignKeys []*ForeignKey +} + +type Column struct { + Name string + Type string + Attributes []*Attribute +} + +type Attribute struct { + Type string + Value string +} + +type Action struct { + Name string + Inputs []string + // Mutability could be empty if the abi is generated by legacy version of kuneiform, + // default to "update" for backward compatibility + Mutability string + // Auxiliaries are the auxiliary types that are required for the action, specifying extra characteristics of the action + Auxiliaries []string + Public bool + Statements []string +} + +type Index struct { + Name string + Columns []string + Type string +} + +type ForeignKey struct { + // ChildKeys are the columns that are referencing another. + // For example, in FOREIGN KEY (a) REFERENCES tbl2(b), "a" is the child key + ChildKeys []string + + // ParentKeys are the columns that are being referred to. + // For example, in FOREIGN KEY (a) REFERENCES tbl2(b), "tbl2.b" is the parent key + ParentKeys []string + + // ParentTable is the table that holds the parent columns. + // For example, in FOREIGN KEY (a) REFERENCES tbl2(b), "tbl2.b" is the parent table + ParentTable string + + // Action refers to what the foreign key should do when the parent is altered. + // This is NOT the same as a database action; + // however sqlite's docs refer to these as actions, + // so we should be consistent with that. + // For example, ON DELETE CASCADE is a foreign key action + Actions []*ForeignKeyAction +} + +// ForeignKeyAction is used to specify what should occur +// if a parent key is updated or deleted +type ForeignKeyAction struct { + // On can be either "UPDATE" or "DELETE" + On string + + // Do specifies what a foreign key action should do + Do string +} + +// MutabilityType is the type of mutability +type MutabilityType string + +func (t MutabilityType) String() string { + return string(t) +} + +const ( + MutabilityUpdate MutabilityType = "update" + MutabilityView MutabilityType = "view" +) + +// AuxiliaryType is the type of auxiliary +type AuxiliaryType string + +func (t AuxiliaryType) String() string { + return string(t) +} + +const ( + // AuxiliaryTypeMustSign is used to specify that an action need signature, it is used for `view` action. + AuxiliaryTypeMustSign AuxiliaryType = "mustsign" +) diff --git a/pkg/types/types_test.go b/pkg/types/types_test.go new file mode 100644 index 000000000..9b243113c --- /dev/null +++ b/pkg/types/types_test.go @@ -0,0 +1,146 @@ +package types_test + +import ( + "testing" + + "github.com/kwilteam/kwil-db/pkg/types" + "github.com/stretchr/testify/assert" +) + +type serializable interface { + Bytes() ([]byte, error) + FromBytes([]byte) error +} + +// this simply test that they all serialize and comply with RLP +func Test_Types(t *testing.T) { + type testCase struct { + name string + obj serializable + } + + testCases := []testCase{ + { + name: "schema", + obj: &types.Schema{ + Owner: "user", + Name: "test_schema", + Tables: []*types.Table{ + { + Name: "users", + Columns: []*types.Column{ + { + Name: "id", + Type: "integer", + Attributes: []*types.Attribute{ + { + Type: "primary_key", + Value: "true", + }, + }, + }, + }, + ForeignKeys: []*types.ForeignKey{ + { + ChildKeys: []string{"child_id"}, + ParentKeys: []string{"parent_id"}, + ParentTable: "parent_table", + Actions: []*types.ForeignKeyAction{ + { + On: "delete", + Do: "cascade", + }, + }, + }, + }, + Indexes: []*types.Index{ + { + Name: "index_name", + Columns: []string{"id", "name"}, + Type: "btree", + }, + }, + }, + }, + Actions: []*types.Action{ + { + Name: "get_user", + Inputs: []string{"user_id"}, + Mutability: types.MutabilityUpdate.String(), + Auxiliaries: []string{types.AuxiliaryTypeMustSign.String()}, + Public: true, + Statements: []string{"SELECT * FROM users WHERE id = $user_id"}, + }, + }, + Extensions: []*types.Extension{ + { + Name: "auth", + Config: []*types.ExtensionConfig{ + { + Argument: "token", + Value: "abc123", + }, + }, + Alias: "authentication", + }, + }, + }, + }, + { + name: "action_execution", + obj: &types.ActionExecution{ + DBID: "db_id", + Action: "action", + Arguments: [][]string{ + { + "arg1", + "arg2", + }, + { + "arg3", + "arg4", + }, + }, + }, + }, + { + name: "action_call", + obj: &types.ActionCall{ + DBID: "db_id", + Action: "action", + Arguments: []string{ + "arg1", + "arg2", + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + bts, err := tc.obj.Bytes() + if err != nil { + t.Fatal(err) + } + + var obj serializable + switch tc.obj.(type) { + case *types.Schema: + obj = &types.Schema{} + case *types.ActionExecution: + obj = &types.ActionExecution{} + case *types.ActionCall: + obj = &types.ActionCall{} + default: + t.Fatal("unknown type") + } + + if err := obj.FromBytes(bts); err != nil { + t.Fatal(err) + } + + // reflect + assert.EqualValuesf(t, tc.obj, obj, "objects are not equal") + }) + } +} diff --git a/pkg/types/validators.go b/pkg/types/validators.go new file mode 100644 index 000000000..ab1254f4c --- /dev/null +++ b/pkg/types/validators.go @@ -0,0 +1 @@ +package types From e27570948e1f0156e6e5177563c65594f42e5c98 Mon Sep 17 00:00:00 2001 From: brennan lamey Date: Fri, 11 Aug 2023 01:49:55 -0500 Subject: [PATCH 2/8] added drop schema payload --- pkg/types/schema.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pkg/types/schema.go b/pkg/types/schema.go index dd4e79477..0927e8dfc 100644 --- a/pkg/types/schema.go +++ b/pkg/types/schema.go @@ -125,3 +125,23 @@ const ( // AuxiliaryTypeMustSign is used to specify that an action need signature, it is used for `view` action. AuxiliaryTypeMustSign AuxiliaryType = "mustsign" ) + +type DropSchema struct { + Owner string + Name string +} + +func (s *DropSchema) Bytes() ([]byte, error) { + return rlp.Encode(s) +} + +func (s *DropSchema) FromBytes(b []byte) error { + res, err := rlp.Decode[DropSchema](b) + if err != nil { + return err + } + + *s = *res + + return nil +} From a747d9c56778bb4e8e1fe2ea92c08a03a90d5489 Mon Sep 17 00:00:00 2001 From: brennan lamey Date: Fri, 11 Aug 2023 11:30:15 -0500 Subject: [PATCH 3/8] made changes discussed this morning --- pkg/{crypto => _crypto}/crypto_test.go | 0 pkg/_crypto/ed25519.go | 61 +++++++ pkg/_crypto/keys.go | 24 +++ pkg/_crypto/secp256k1.go | 72 +++++++++ pkg/_crypto/signature.go | 150 ++++++++++++++++++ pkg/{crypto => _crypto}/signature_test.go | 0 pkg/crypto/crypto.go | 22 --- pkg/crypto/ed25519.go | 2 +- pkg/crypto/keys.go | 2 +- pkg/crypto/secp256k1.go | 2 +- pkg/crypto/signature.go | 128 +-------------- pkg/serialize/rlp/encode.go | 6 +- pkg/transactions/errors.go | 7 - pkg/transactions/message.go | 36 +---- .../schema.go => transactions/payloads.go} | 77 ++++++++- .../payloads_test.go} | 67 ++++---- pkg/transactions/transaction.go | 47 ++++++ pkg/types/execute.go | 43 ----- pkg/types/validators.go | 1 - 19 files changed, 481 insertions(+), 266 deletions(-) rename pkg/{crypto => _crypto}/crypto_test.go (100%) create mode 100644 pkg/_crypto/ed25519.go create mode 100644 pkg/_crypto/keys.go create mode 100644 pkg/_crypto/secp256k1.go create mode 100644 pkg/_crypto/signature.go rename pkg/{crypto => _crypto}/signature_test.go (100%) delete mode 100644 pkg/transactions/errors.go rename pkg/{types/schema.go => transactions/payloads.go} (63%) rename pkg/{types/types_test.go => transactions/payloads_test.go} (58%) create mode 100644 pkg/transactions/transaction.go delete mode 100644 pkg/types/execute.go delete mode 100644 pkg/types/validators.go diff --git a/pkg/crypto/crypto_test.go b/pkg/_crypto/crypto_test.go similarity index 100% rename from pkg/crypto/crypto_test.go rename to pkg/_crypto/crypto_test.go diff --git a/pkg/_crypto/ed25519.go b/pkg/_crypto/ed25519.go new file mode 100644 index 000000000..7ce344c0c --- /dev/null +++ b/pkg/_crypto/ed25519.go @@ -0,0 +1,61 @@ +package crypto + +import ( + oasisEd25519 "github.com/oasisprotocol/curve25519-voi/primitives/ed25519" +) + +const Ed25519 KeyType = "Ed25519" + +type Ed25519PrivateKey struct { + privateKey []byte +} + +func (s *Ed25519PrivateKey) Bytes() []byte { + return s.privateKey +} + +func (s *Ed25519PrivateKey) PubKey() PublicKey { + panic("implement me") +} + +func (s *Ed25519PrivateKey) Sign(msg []byte, signatureType SignatureType) ([]byte, error) { + return oasisEd25519.Sign(oasisEd25519.PrivateKey(s.privateKey), msg), nil +} + +func (s *Ed25519PrivateKey) Type() KeyType { + return Ed25519 +} + +type Ed25519PublicKey struct { +} + +func (s *Ed25519PublicKey) Address() Address { + panic("implement me") +} + +func (s *Ed25519PublicKey) Bytes() []byte { + panic("implement me") +} + +func (s *Ed25519PublicKey) Type() KeyType { + return Ed25519 +} + +func (s *Ed25519PublicKey) Verify(sig *Signature2, data []byte) error { + panic("implement me") +} + +type Ed25519Address struct { +} + +func (s *Ed25519Address) Bytes() []byte { + panic("implement me") +} + +func (s *Ed25519Address) Type() KeyType { + return Ed25519 +} + +func (s *Ed25519Address) String() string { + panic("implement me") +} diff --git a/pkg/_crypto/keys.go b/pkg/_crypto/keys.go new file mode 100644 index 000000000..6e0b7ec4c --- /dev/null +++ b/pkg/_crypto/keys.go @@ -0,0 +1,24 @@ +package crypto + +type KeyType string + +type PrivateKey interface { + Bytes() []byte + Type() KeyType + Sign(msg []byte, signatureType SignatureType) ([]byte, error) + PubKey() PublicKey +} + +type PublicKey interface { + Bytes() []byte + Type() KeyType + Verify(sig *Signature2, data []byte) error + Address() Address +} + +type Address interface { + Bytes() []byte + String() string + // do we need to know the key type? + Type() KeyType +} diff --git a/pkg/_crypto/secp256k1.go b/pkg/_crypto/secp256k1.go new file mode 100644 index 000000000..ceb25629a --- /dev/null +++ b/pkg/_crypto/secp256k1.go @@ -0,0 +1,72 @@ +package crypto + +import ( + "crypto/ecdsa" + + "github.com/ethereum/go-ethereum/common" + ethCrypto "github.com/ethereum/go-ethereum/crypto" +) + +const Secp256k1 KeyType = "secp256k1" + +type Secp256k1PrivateKey struct { + privateKey *ecdsa.PrivateKey +} + +func (s *Secp256k1PrivateKey) Bytes() []byte { + return ethCrypto.FromECDSA(s.privateKey) +} + +func (s *Secp256k1PrivateKey) PubKey() PublicKey { + return &Secp256k1PublicKey{ + publicKey: &s.privateKey.PublicKey, + } +} + +func (s *Secp256k1PrivateKey) Sign(msg []byte, signatureType SignatureType) ([]byte, error) { + // TODO: implement + panic("TODO") +} + +func (s *Secp256k1PrivateKey) Type() KeyType { + return Secp256k1 +} + +type Secp256k1PublicKey struct { + publicKey *ecdsa.PublicKey +} + +func (s *Secp256k1PublicKey) Address() Address { + return &Secp256k1Address{ + address: ethCrypto.PubkeyToAddress(*s.publicKey), + } +} + +func (s *Secp256k1PublicKey) Bytes() []byte { + return ethCrypto.FromECDSAPub(s.publicKey) +} + +func (s *Secp256k1PublicKey) Type() KeyType { + return Secp256k1 +} + +func (s *Secp256k1PublicKey) Verify(sig *Signature2, data []byte) error { + // TODO: implement + panic("TODO") +} + +type Secp256k1Address struct { + address common.Address +} + +func (s *Secp256k1Address) Bytes() []byte { + return s.address.Bytes() +} + +func (s *Secp256k1Address) Type() KeyType { + return Secp256k1 +} + +func (s *Secp256k1Address) String() string { + return s.address.String() +} diff --git a/pkg/_crypto/signature.go b/pkg/_crypto/signature.go new file mode 100644 index 000000000..a77b8dfdb --- /dev/null +++ b/pkg/_crypto/signature.go @@ -0,0 +1,150 @@ +package crypto + +import ( + "crypto/ecdsa" + "errors" + "fmt" + + cmtCrypto "github.com/cometbft/cometbft/crypto" + cmtjson "github.com/cometbft/cometbft/libs/json" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + ec "github.com/ethereum/go-ethereum/crypto" +) + +var ErrInvalidSignature = errors.New("invalid signature") + +type SignatureType int32 + +const ( + SIGNATURE_TYPE_INVALID SignatureType = iota + + // all of these different SECP256k1 signatures should + // be revisited. If they are warranted, then at the very least + // they should be renamed to be more descriptive + // it could also be worth giving them string representations, + // instead of just numbers + PK_SECP256K1_UNCOMPRESSED + ACCOUNT_SECP256K1_UNCOMPRESSED + ACCOUNT_SECP256K1_UNCOMPRESSED_STRUCTURED_v1 + PK_ED25519 + END_SIGNATURE_TYPE +) + +func (s *SignatureType) IsValid() error { + if *s < SIGNATURE_TYPE_INVALID || *s >= END_SIGNATURE_TYPE { + return fmt.Errorf("invalid signature type '%d'", *s) + } + return nil +} + +func (s SignatureType) Int32() int32 { + return int32(s) +} + +type Signature struct { + Message []byte `json:"message"` + Signature []byte `json:"signature_bytes"` + Sender PublicKey + Type SignatureType `json:"signature_type"` +} + +func (s *Signature) Verify() error { + return s.Sender.Verify(s) +} + +/* + everything below this is considered "deprecated", and is only left for backwards compatibility + while the new signature is being implemented +*/ + +func Sign(data []byte, k *ecdsa.PrivateKey) (*Signature, error) { + signature := Signature{ + Type: PK_SECP256K1_UNCOMPRESSED, + } + hash := ec.Keccak256Hash(data) + sig, err := ec.Sign(hash.Bytes(), k) + if err != nil { + return &signature, err + } + + signature.Signature = sig + return &signature, nil +} + +// Deprecated: use Verify instead +func (s *Signature) Check(sender string, data []byte) error { + ok, err := CheckSignature(sender, s, data) + if err != nil { + return err + } + + if !ok { + return ErrInvalidSignature + } + + return nil +} + +func CheckSignature(addr string, sig *Signature, data []byte) (bool, error) { + if addr == "" { + return false, fmt.Errorf("transaction does not have a sender address") + } + switch sig.Type { + case PK_SECP256K1_UNCOMPRESSED: + return checkSignaturePkSECP256k1Uncompressed(addr, sig.Signature, data) + case ACCOUNT_SECP256K1_UNCOMPRESSED: + return checkSignatureAccountSECP256k1Uncompressed(addr, sig.Signature, data) + case PK_ED25519: + return checkSignatureED25519(addr, sig.Signature, data) + default: + return false, fmt.Errorf("unknown signature indicator: %d", sig.Type) + } +} + +func checkSignatureED25519(addr string, sig []byte, data []byte) (bool, error) { + var publicKey cmtCrypto.PubKey + key := fmt.Sprintf(`{"type":"tendermint/PubKeyEd25519","value":%s}`, addr) + fmt.Println("Key:", key) + + err := cmtjson.Unmarshal([]byte(key), &publicKey) + if err != nil { + return false, fmt.Errorf("failed to unmarshal validator public key: %w", err) + } + fmt.Println("publicKey: ", publicKey) + + valid := publicKey.VerifySignature(data, sig) + if valid { + return true, nil + } else { + return false, fmt.Errorf("signature verification failed") + } +} + +func checkSigHash(addr string, sig, hash []byte) (bool, error) { + recoveredPubKey, err := ec.SigToPub(hash, sig) + if err != nil { + return false, fmt.Errorf("failed to recover public key: %w", err) + } + + derAddr := ec.PubkeyToAddress(*recoveredPubKey) + return derAddr == common.HexToAddress(addr), nil + // NOTE: We compare the address [N]byte to be robust to string formatting + // details, namely deviations from EIP55 and prefix (or not). +} + +// This would be for a signature generated by a private key. Likely EVM chains +func checkSignaturePkSECP256k1Uncompressed(addr string, sig []byte, data []byte) (bool, error) { + hash := ec.Keccak256Hash(data) + return checkSigHash(addr, sig, hash[:]) +} + +// checkSignatureAccount checks a signature that was generated by an config instead of a private key +// generally this would mean a signature from MetaMask / equivalent for EVM chains +func checkSignatureAccountSECP256k1Uncompressed(addr string, sig []byte, data []byte) (bool, error) { + msg := accounts.TextHash(data) + if sig[ec.RecoveryIDOffset] == 27 || sig[ec.RecoveryIDOffset] == 28 { + sig[ec.RecoveryIDOffset] -= 27 // Transform yellow paper V from 27/28 to 0/1 + } + return checkSigHash(addr, sig, msg) +} diff --git a/pkg/crypto/signature_test.go b/pkg/_crypto/signature_test.go similarity index 100% rename from pkg/crypto/signature_test.go rename to pkg/_crypto/signature_test.go diff --git a/pkg/crypto/crypto.go b/pkg/crypto/crypto.go index 32aa699fa..808577f1b 100644 --- a/pkg/crypto/crypto.go +++ b/pkg/crypto/crypto.go @@ -1,14 +1,9 @@ package crypto import ( - "crypto/ecdsa" c256 "crypto/sha256" c512 "crypto/sha512" "encoding/hex" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - ec "github.com/ethereum/go-ethereum/crypto" ) // Sha384 returns the sha384 hash of the data. @@ -41,20 +36,3 @@ func Sha256(data []byte) []byte { func Sha256Hex(data []byte) string { return hex.EncodeToString(Sha256(data)) } - -func ECDSAFromHex(hex string) (*ecdsa.PrivateKey, error) { - return ec.HexToECDSA(hex) -} - -func AddressFromPrivateKey(key *ecdsa.PrivateKey) string { - caddr := ec.PubkeyToAddress(key.PublicKey) - return caddr.Hex() -} - -func IsValidAddress(addr string) bool { - return common.IsHexAddress(addr) -} - -func HexFromECDSAPrivateKey(key *ecdsa.PrivateKey) string { - return hexutil.Encode(ec.FromECDSA(key))[2:] -} diff --git a/pkg/crypto/ed25519.go b/pkg/crypto/ed25519.go index b8ded3082..d119bc7fe 100644 --- a/pkg/crypto/ed25519.go +++ b/pkg/crypto/ed25519.go @@ -41,7 +41,7 @@ func (s *Ed25519PublicKey) Type() KeyType { return Ed25519 } -func (s *Ed25519PublicKey) Verify(sig *Signature) error { +func (s *Ed25519PublicKey) Verify(sig *Signature, data []byte) error { panic("implement me") } diff --git a/pkg/crypto/keys.go b/pkg/crypto/keys.go index 155041ed1..1c9656a09 100644 --- a/pkg/crypto/keys.go +++ b/pkg/crypto/keys.go @@ -12,7 +12,7 @@ type PrivateKey interface { type PublicKey interface { Bytes() []byte Type() KeyType - Verify(sign *Signature) error + Verify(sig *Signature, data []byte) error Address() Address } diff --git a/pkg/crypto/secp256k1.go b/pkg/crypto/secp256k1.go index 8258ad9cf..b8b76d1af 100644 --- a/pkg/crypto/secp256k1.go +++ b/pkg/crypto/secp256k1.go @@ -50,7 +50,7 @@ func (s *Secp256k1PublicKey) Type() KeyType { return Secp256k1 } -func (s *Secp256k1PublicKey) Verify(sig *Signature) error { +func (s *Secp256k1PublicKey) Verify(sig *Signature, data []byte) error { // TODO: implement panic("TODO") } diff --git a/pkg/crypto/signature.go b/pkg/crypto/signature.go index a77b8dfdb..4cd10c622 100644 --- a/pkg/crypto/signature.go +++ b/pkg/crypto/signature.go @@ -1,36 +1,17 @@ package crypto import ( - "crypto/ecdsa" - "errors" "fmt" - - cmtCrypto "github.com/cometbft/cometbft/crypto" - cmtjson "github.com/cometbft/cometbft/libs/json" - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common" - ec "github.com/ethereum/go-ethereum/crypto" ) -var ErrInvalidSignature = errors.New("invalid signature") - type SignatureType int32 const ( SIGNATURE_TYPE_INVALID SignatureType = iota - - // all of these different SECP256k1 signatures should - // be revisited. If they are warranted, then at the very least - // they should be renamed to be more descriptive - // it could also be worth giving them string representations, - // instead of just numbers - PK_SECP256K1_UNCOMPRESSED - ACCOUNT_SECP256K1_UNCOMPRESSED - ACCOUNT_SECP256K1_UNCOMPRESSED_STRUCTURED_v1 - PK_ED25519 END_SIGNATURE_TYPE ) +// IsValid returns an error if the signature type is invalid. func (s *SignatureType) IsValid() error { if *s < SIGNATURE_TYPE_INVALID || *s >= END_SIGNATURE_TYPE { return fmt.Errorf("invalid signature type '%d'", *s) @@ -38,113 +19,18 @@ func (s *SignatureType) IsValid() error { return nil } +// Int32 returns the signature type as an int32. func (s SignatureType) Int32() int32 { return int32(s) } +// Signature is a cryptographic signature. type Signature struct { - Message []byte `json:"message"` - Signature []byte `json:"signature_bytes"` - Sender PublicKey + Signature []byte `json:"signature_bytes"` Type SignatureType `json:"signature_type"` } -func (s *Signature) Verify() error { - return s.Sender.Verify(s) -} - -/* - everything below this is considered "deprecated", and is only left for backwards compatibility - while the new signature is being implemented -*/ - -func Sign(data []byte, k *ecdsa.PrivateKey) (*Signature, error) { - signature := Signature{ - Type: PK_SECP256K1_UNCOMPRESSED, - } - hash := ec.Keccak256Hash(data) - sig, err := ec.Sign(hash.Bytes(), k) - if err != nil { - return &signature, err - } - - signature.Signature = sig - return &signature, nil -} - -// Deprecated: use Verify instead -func (s *Signature) Check(sender string, data []byte) error { - ok, err := CheckSignature(sender, s, data) - if err != nil { - return err - } - - if !ok { - return ErrInvalidSignature - } - - return nil -} - -func CheckSignature(addr string, sig *Signature, data []byte) (bool, error) { - if addr == "" { - return false, fmt.Errorf("transaction does not have a sender address") - } - switch sig.Type { - case PK_SECP256K1_UNCOMPRESSED: - return checkSignaturePkSECP256k1Uncompressed(addr, sig.Signature, data) - case ACCOUNT_SECP256K1_UNCOMPRESSED: - return checkSignatureAccountSECP256k1Uncompressed(addr, sig.Signature, data) - case PK_ED25519: - return checkSignatureED25519(addr, sig.Signature, data) - default: - return false, fmt.Errorf("unknown signature indicator: %d", sig.Type) - } -} - -func checkSignatureED25519(addr string, sig []byte, data []byte) (bool, error) { - var publicKey cmtCrypto.PubKey - key := fmt.Sprintf(`{"type":"tendermint/PubKeyEd25519","value":%s}`, addr) - fmt.Println("Key:", key) - - err := cmtjson.Unmarshal([]byte(key), &publicKey) - if err != nil { - return false, fmt.Errorf("failed to unmarshal validator public key: %w", err) - } - fmt.Println("publicKey: ", publicKey) - - valid := publicKey.VerifySignature(data, sig) - if valid { - return true, nil - } else { - return false, fmt.Errorf("signature verification failed") - } -} - -func checkSigHash(addr string, sig, hash []byte) (bool, error) { - recoveredPubKey, err := ec.SigToPub(hash, sig) - if err != nil { - return false, fmt.Errorf("failed to recover public key: %w", err) - } - - derAddr := ec.PubkeyToAddress(*recoveredPubKey) - return derAddr == common.HexToAddress(addr), nil - // NOTE: We compare the address [N]byte to be robust to string formatting - // details, namely deviations from EIP55 and prefix (or not). -} - -// This would be for a signature generated by a private key. Likely EVM chains -func checkSignaturePkSECP256k1Uncompressed(addr string, sig []byte, data []byte) (bool, error) { - hash := ec.Keccak256Hash(data) - return checkSigHash(addr, sig, hash[:]) -} - -// checkSignatureAccount checks a signature that was generated by an config instead of a private key -// generally this would mean a signature from MetaMask / equivalent for EVM chains -func checkSignatureAccountSECP256k1Uncompressed(addr string, sig []byte, data []byte) (bool, error) { - msg := accounts.TextHash(data) - if sig[ec.RecoveryIDOffset] == 27 || sig[ec.RecoveryIDOffset] == 28 { - sig[ec.RecoveryIDOffset] -= 27 // Transform yellow paper V from 27/28 to 0/1 - } - return checkSigHash(addr, sig, msg) +// Verify verifies the signature against the given public key and data. +func (s *Signature) Verify(publicKey PublicKey, data []byte) error { + return publicKey.Verify(s, data) } diff --git a/pkg/serialize/rlp/encode.go b/pkg/serialize/rlp/encode.go index f90d7d255..e8fbd82d4 100644 --- a/pkg/serialize/rlp/encode.go +++ b/pkg/serialize/rlp/encode.go @@ -6,6 +6,8 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +type SerializedData []byte + type encodingType uint16 const ( @@ -16,7 +18,7 @@ const ( var currentEncodingType = encodingTypeRLP -func Encode(val any) ([]byte, error) { +func Encode(val any) (SerializedData, error) { var btsVal []byte var err error switch currentEncodingType { @@ -35,7 +37,7 @@ func Encode(val any) ([]byte, error) { }) } -func Decode[T any](bts []byte) (*T, error) { +func Decode[T any](bts SerializedData) (*T, error) { encVal, err := deserializeEncodedValue(bts) if err != nil { return nil, err diff --git a/pkg/transactions/errors.go b/pkg/transactions/errors.go deleted file mode 100644 index 4be7f22d7..000000000 --- a/pkg/transactions/errors.go +++ /dev/null @@ -1,7 +0,0 @@ -package transactions - -import "errors" - -var ( - ErrFailedHashReconstruction = errors.New("failed to reconstruct hash") -) diff --git a/pkg/transactions/message.go b/pkg/transactions/message.go index 6763ba3ba..26dbce311 100644 --- a/pkg/transactions/message.go +++ b/pkg/transactions/message.go @@ -5,10 +5,8 @@ transactions and signed messages. package transactions import ( - "bytes" - "math/big" - "github.com/kwilteam/kwil-db/pkg/crypto" + "github.com/kwilteam/kwil-db/pkg/serialize/rlp" ) // SignedMessage is any message that has been signed by a private key @@ -16,7 +14,8 @@ import ( // should be the hash of the payload. type SignedMessage struct { Signature *crypto.Signature - Message Serializable + Message rlp.SerializedData + Sender crypto.PublicKey } // Verify verifies the authenticity of a signed message. @@ -24,32 +23,5 @@ type SignedMessage struct { // it to the message in the signature. // It then uses the public key in the signature to verify the signature. func (s *SignedMessage) Verify() error { - messageBytes, err := s.Message.Bytes() - if err != nil { - return err - } - - if !bytes.Equal(s.Signature.Message, crypto.Sha256(messageBytes)) { - return ErrFailedHashReconstruction - } - - return s.Verify() -} - -// Serializable is any message that can be hashed -// This hash is used to both sign the message, as well as reconstruct the -// hash (if we are checking the message) to verify the signature -type Serializable interface { - Bytes() ([]byte, error) -} - -// TransactionMessageis the payload of a transaction -// It contains information on the nonce, fee, and the transaction payload -// The type of the transaction can be derived from the payload -type TransactionMessage struct { - Nonce int64 - Fee *big.Int + return s.Sender.Verify(s.Signature, s.Message) } - -// AuthenticatedMessage is a message that has been signed by a private key -// it is not a valid blockchain transaction, but can be used to authenticate diff --git a/pkg/types/schema.go b/pkg/transactions/payloads.go similarity index 63% rename from pkg/types/schema.go rename to pkg/transactions/payloads.go index 0927e8dfc..002d9a7f8 100644 --- a/pkg/types/schema.go +++ b/pkg/transactions/payloads.go @@ -1,6 +1,19 @@ -package types +package transactions -import "github.com/kwilteam/kwil-db/pkg/serialize/rlp" +import ( + "encoding" + + "github.com/kwilteam/kwil-db/pkg/serialize/rlp" +) + +type PayloadType string + +const ( + PayloadTypeDeploySchema PayloadType = "deploy_schema" + PayloadTypeDropSchema PayloadType = "drop_schema" + PayloadTypeExecuteAction PayloadType = "execute_action" + PayloadTypeCallAction PayloadType = "call_action" +) type Schema struct { Owner string @@ -10,11 +23,14 @@ type Schema struct { Extensions []*Extension } -func (s *Schema) Bytes() ([]byte, error) { +var _ encoding.BinaryMarshaler = (*Schema)(nil) +var _ encoding.BinaryUnmarshaler = (*Schema)(nil) + +func (s *Schema) MarshalBinary() ([]byte, error) { return rlp.Encode(s) } -func (s *Schema) FromBytes(b []byte) error { +func (s *Schema) UnmarshalBinary(b []byte) error { result, err := rlp.Decode[Schema](b) if err != nil { return err @@ -131,11 +147,14 @@ type DropSchema struct { Name string } -func (s *DropSchema) Bytes() ([]byte, error) { +var _ encoding.BinaryMarshaler = (*DropSchema)(nil) +var _ encoding.BinaryUnmarshaler = (*DropSchema)(nil) + +func (s *DropSchema) MarshalBinary() ([]byte, error) { return rlp.Encode(s) } -func (s *DropSchema) FromBytes(b []byte) error { +func (s *DropSchema) UnmarshalBinary(b []byte) error { res, err := rlp.Decode[DropSchema](b) if err != nil { return err @@ -145,3 +164,49 @@ func (s *DropSchema) FromBytes(b []byte) error { return nil } + +type ActionExecution struct { + DBID string + Action string + Arguments [][]string +} + +var _ encoding.BinaryMarshaler = (*ActionExecution)(nil) +var _ encoding.BinaryUnmarshaler = (*ActionExecution)(nil) + +func (a *ActionExecution) MarshalBinary() ([]byte, error) { + return rlp.Encode(a) +} + +func (s *ActionExecution) UnmarshalBinary(b []byte) error { + res, err := rlp.Decode[ActionExecution](b) + if err != nil { + return err + } + + *s = *res + return nil +} + +type ActionCall struct { + DBID string + Action string + Arguments []string +} + +var _ encoding.BinaryMarshaler = (*ActionCall)(nil) +var _ encoding.BinaryUnmarshaler = (*ActionCall)(nil) + +func (a *ActionCall) MarshalBinary() ([]byte, error) { + return rlp.Encode(a) +} + +func (s *ActionCall) UnmarshalBinary(b []byte) error { + res, err := rlp.Decode[ActionCall](b) + if err != nil { + return err + } + + *s = *res + return nil +} diff --git a/pkg/types/types_test.go b/pkg/transactions/payloads_test.go similarity index 58% rename from pkg/types/types_test.go rename to pkg/transactions/payloads_test.go index 9b243113c..50ff292bb 100644 --- a/pkg/types/types_test.go +++ b/pkg/transactions/payloads_test.go @@ -1,38 +1,38 @@ -package types_test +package transactions_test import ( "testing" - "github.com/kwilteam/kwil-db/pkg/types" + "github.com/kwilteam/kwil-db/pkg/transactions" "github.com/stretchr/testify/assert" ) -type serializable interface { - Bytes() ([]byte, error) - FromBytes([]byte) error +type marshallable interface { + MarshalBinary() ([]byte, error) + UnmarshalBinary([]byte) error } // this simply test that they all serialize and comply with RLP func Test_Types(t *testing.T) { type testCase struct { name string - obj serializable + obj marshallable } testCases := []testCase{ { name: "schema", - obj: &types.Schema{ + obj: &transactions.Schema{ Owner: "user", Name: "test_schema", - Tables: []*types.Table{ + Tables: []*transactions.Table{ { Name: "users", - Columns: []*types.Column{ + Columns: []*transactions.Column{ { Name: "id", Type: "integer", - Attributes: []*types.Attribute{ + Attributes: []*transactions.Attribute{ { Type: "primary_key", Value: "true", @@ -40,12 +40,12 @@ func Test_Types(t *testing.T) { }, }, }, - ForeignKeys: []*types.ForeignKey{ + ForeignKeys: []*transactions.ForeignKey{ { ChildKeys: []string{"child_id"}, ParentKeys: []string{"parent_id"}, ParentTable: "parent_table", - Actions: []*types.ForeignKeyAction{ + Actions: []*transactions.ForeignKeyAction{ { On: "delete", Do: "cascade", @@ -53,7 +53,7 @@ func Test_Types(t *testing.T) { }, }, }, - Indexes: []*types.Index{ + Indexes: []*transactions.Index{ { Name: "index_name", Columns: []string{"id", "name"}, @@ -62,20 +62,20 @@ func Test_Types(t *testing.T) { }, }, }, - Actions: []*types.Action{ + Actions: []*transactions.Action{ { Name: "get_user", Inputs: []string{"user_id"}, - Mutability: types.MutabilityUpdate.String(), - Auxiliaries: []string{types.AuxiliaryTypeMustSign.String()}, + Mutability: transactions.MutabilityUpdate.String(), + Auxiliaries: []string{transactions.AuxiliaryTypeMustSign.String()}, Public: true, Statements: []string{"SELECT * FROM users WHERE id = $user_id"}, }, }, - Extensions: []*types.Extension{ + Extensions: []*transactions.Extension{ { Name: "auth", - Config: []*types.ExtensionConfig{ + Config: []*transactions.ExtensionConfig{ { Argument: "token", Value: "abc123", @@ -88,7 +88,7 @@ func Test_Types(t *testing.T) { }, { name: "action_execution", - obj: &types.ActionExecution{ + obj: &transactions.ActionExecution{ DBID: "db_id", Action: "action", Arguments: [][]string{ @@ -105,7 +105,7 @@ func Test_Types(t *testing.T) { }, { name: "action_call", - obj: &types.ActionCall{ + obj: &transactions.ActionCall{ DBID: "db_id", Action: "action", Arguments: []string{ @@ -114,28 +114,37 @@ func Test_Types(t *testing.T) { }, }, }, + { + name: "drop_schema", + obj: &transactions.DropSchema{ + Owner: "user", + Name: "test_schema", + }, + }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - bts, err := tc.obj.Bytes() + bts, err := tc.obj.MarshalBinary() if err != nil { t.Fatal(err) } - var obj serializable + var obj marshallable switch tc.obj.(type) { - case *types.Schema: - obj = &types.Schema{} - case *types.ActionExecution: - obj = &types.ActionExecution{} - case *types.ActionCall: - obj = &types.ActionCall{} + case *transactions.Schema: + obj = &transactions.Schema{} + case *transactions.ActionExecution: + obj = &transactions.ActionExecution{} + case *transactions.ActionCall: + obj = &transactions.ActionCall{} + case *transactions.DropSchema: + obj = &transactions.DropSchema{} default: t.Fatal("unknown type") } - if err := obj.FromBytes(bts); err != nil { + if err := obj.UnmarshalBinary(bts); err != nil { t.Fatal(err) } diff --git a/pkg/transactions/transaction.go b/pkg/transactions/transaction.go new file mode 100644 index 000000000..ff0a9b258 --- /dev/null +++ b/pkg/transactions/transaction.go @@ -0,0 +1,47 @@ +package transactions + +import ( + "math/big" + + "github.com/kwilteam/kwil-db/pkg/crypto" + "github.com/kwilteam/kwil-db/pkg/serialize/rlp" +) + +type Transaction struct { + Signature *crypto.Signature + Payload *TransactionPayload + Sender crypto.PublicKey +} + +// Verify verifies the signature of the transaction +func (t *Transaction) Verify() error { + data, err := t.Payload.MarshalBinary() + if err != nil { + return err + } + + return t.Sender.Verify(t.Signature, data) +} + +// TransactionPayload is a generic payload type that can be used to send binary data +type TransactionPayload struct { + // Payload are the raw bytes of the payload data + Payload rlp.SerializedData + + // PayloadType is the type of the payload + // This can be used to determine how to decode the payload + PayloadType PayloadType + + // Fee is the fee the sender is willing to pay for the transaction + Fee *big.Int + + // Nonce is the next nonce of the sender + Nonce uint64 + + // Salt is a random number used to prevent replay attacks, as well as help guarantee uniqueness + Salt uint64 +} + +func (t *TransactionPayload) MarshalBinary() ([]byte, error) { + return rlp.Encode(t) +} diff --git a/pkg/types/execute.go b/pkg/types/execute.go deleted file mode 100644 index 32b24445a..000000000 --- a/pkg/types/execute.go +++ /dev/null @@ -1,43 +0,0 @@ -package types - -import "github.com/kwilteam/kwil-db/pkg/serialize/rlp" - -type ActionExecution struct { - DBID string - Action string - Arguments [][]string -} - -func (a *ActionExecution) Bytes() ([]byte, error) { - return rlp.Encode(a) -} - -func (s *ActionExecution) FromBytes(b []byte) error { - res, err := rlp.Decode[ActionExecution](b) - if err != nil { - return err - } - - *s = *res - return nil -} - -type ActionCall struct { - DBID string - Action string - Arguments []string -} - -func (a *ActionCall) Bytes() ([]byte, error) { - return rlp.Encode(a) -} - -func (s *ActionCall) FromBytes(b []byte) error { - res, err := rlp.Decode[ActionCall](b) - if err != nil { - return err - } - - *s = *res - return nil -} diff --git a/pkg/types/validators.go b/pkg/types/validators.go deleted file mode 100644 index ab1254f4c..000000000 --- a/pkg/types/validators.go +++ /dev/null @@ -1 +0,0 @@ -package types From d1f50d2a29cffd3ea6dffc5ebf43afc23bc073d6 Mon Sep 17 00:00:00 2001 From: brennan lamey Date: Fri, 11 Aug 2023 11:38:18 -0500 Subject: [PATCH 4/8] minor changes --- pkg/transactions/transaction.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/transactions/transaction.go b/pkg/transactions/transaction.go index ff0a9b258..9b5954295 100644 --- a/pkg/transactions/transaction.go +++ b/pkg/transactions/transaction.go @@ -9,13 +9,13 @@ import ( type Transaction struct { Signature *crypto.Signature - Payload *TransactionPayload + Body *TransactionBody Sender crypto.PublicKey } // Verify verifies the signature of the transaction func (t *Transaction) Verify() error { - data, err := t.Payload.MarshalBinary() + data, err := t.Body.MarshalBinary() if err != nil { return err } @@ -23,8 +23,8 @@ func (t *Transaction) Verify() error { return t.Sender.Verify(t.Signature, data) } -// TransactionPayload is a generic payload type that can be used to send binary data -type TransactionPayload struct { +// TransactionBody is the body of a transaction that gets included in the signature +type TransactionBody struct { // Payload are the raw bytes of the payload data Payload rlp.SerializedData @@ -38,10 +38,10 @@ type TransactionPayload struct { // Nonce is the next nonce of the sender Nonce uint64 - // Salt is a random number used to prevent replay attacks, as well as help guarantee uniqueness - Salt uint64 + // Salt is a random value that is used to prevent replay attacks and hash collisions + Salt []byte } -func (t *TransactionPayload) MarshalBinary() ([]byte, error) { +func (t *TransactionBody) MarshalBinary() ([]byte, error) { return rlp.Encode(t) } From 2d200434c6cf29a6a3b057b9f44d1d3d013131b9 Mon Sep 17 00:00:00 2001 From: brennan lamey Date: Mon, 14 Aug 2023 00:12:07 -0500 Subject: [PATCH 5/8] modified repo to reflect new transaction structure --- api/openapi-spec/api/v1/api.swagger.json | 166 ++++++++++------ cmd/kwil-cli/cmds/common/display/response.go | 8 +- cmd/kwil-cli/cmds/database/batch.go | 28 ++- cmd/kwil-cli/cmds/database/call.go | 27 ++- cmd/kwil-cli/cmds/database/deploy.go | 12 +- cmd/kwil-cli/cmds/database/execute.go | 42 +++- cmd/kwil-cli/cmds/database/read_schema.go | 2 +- cmd/kwil-cli/cmds/utils/node_cfg.go | 79 -------- cmd/kwil-cli/cmds/utils/utils.go | 1 - internal/app/kwild/client/client.go | 6 +- internal/app/kwild/config/config_test.go | 7 +- internal/app/kwild/driver.go | 35 ++-- internal/app/kwild/server/root.go | 23 --- internal/app/kwild/server/utils.go | 23 +++ .../controller/grpc/txsvc/v1/broadcast.go | 65 ++----- internal/controller/grpc/txsvc/v1/call.go | 37 ++-- internal/controller/grpc/txsvc/v1/convert.go | 182 ++++++++++++++++++ internal/controller/grpc/txsvc/v1/pricing.go | 66 ++++--- internal/controller/grpc/txsvc/v1/schema.go | 113 +---------- internal/controller/grpc/txsvc/v1/service.go | 8 +- pkg/_crypto/crypto.go | 26 +++ pkg/{serialize => _serialize}/consensus.go | 0 pkg/{serialize => _serialize}/schema.go | 0 pkg/{serialize => _serialize}/serial.go | 0 pkg/{tx => _tx}/message.go | 4 +- pkg/{tx => _tx}/payload.go | 0 pkg/{tx => _tx}/response.go | 0 pkg/{tx => _tx}/sign_test.go | 0 pkg/{tx => _tx}/tx.go | 0 pkg/abci/abci.go | 77 +++++--- pkg/abci/interfaces.go | 12 +- pkg/abci/payloads.go | 38 ---- pkg/abci/utils.go | 1 - pkg/client/client.go | 159 ++++++++------- pkg/client/opts.go | 19 +- pkg/client/tx.go | 125 ++++++------ pkg/client/types.go | 19 -- pkg/config/config_test.go | 7 +- pkg/crypto/crypto.go | 19 ++ pkg/crypto/ed25519.go | 8 +- pkg/crypto/keys.go | 2 +- pkg/crypto/secp256k1.go | 2 +- pkg/crypto/signature.go | 2 + pkg/engine/dataset/dataset.go | 29 +-- pkg/engine/dataset/dataset_test.go | 88 ++++----- pkg/engine/engine.go | 9 +- pkg/engine/interfaces.go | 4 +- pkg/grpc/client/v1/broadcast.go | 37 +--- pkg/grpc/client/v1/call.go | 26 +-- pkg/grpc/client/v1/convert.go | 50 +++++ pkg/grpc/client/v1/schema.go | 40 ++-- pkg/grpc/client/v1/validator.go | 16 +- pkg/modules/datasets/convert.go | 180 +++++++++++++++++ pkg/modules/datasets/databases.go | 4 +- pkg/modules/datasets/execution.go | 103 ++++++---- pkg/modules/datasets/interfaces.go | 2 +- pkg/modules/datasets/pricing.go | 2 +- pkg/modules/datasets/read.go | 18 +- pkg/serialize/{rlp => }/encode.go | 15 +- pkg/serialize/{rlp => }/encode_test.go | 8 +- pkg/serialize/prefix.go | 55 ++++++ pkg/serialize/rlp/value.go | 59 ------ pkg/transactions/message.go | 28 ++- pkg/transactions/payloads.go | 106 ++++++---- pkg/transactions/payloads_test.go | 12 +- pkg/transactions/transaction.go | 150 ++++++++++++++- proto | 2 +- 67 files changed, 1514 insertions(+), 979 deletions(-) delete mode 100644 cmd/kwil-cli/cmds/utils/node_cfg.go create mode 100644 internal/controller/grpc/txsvc/v1/convert.go create mode 100644 pkg/_crypto/crypto.go rename pkg/{serialize => _serialize}/consensus.go (100%) rename pkg/{serialize => _serialize}/schema.go (100%) rename pkg/{serialize => _serialize}/serial.go (100%) rename pkg/{tx => _tx}/message.go (95%) rename pkg/{tx => _tx}/payload.go (100%) rename pkg/{tx => _tx}/response.go (100%) rename pkg/{tx => _tx}/sign_test.go (100%) rename pkg/{tx => _tx}/tx.go (100%) delete mode 100644 pkg/abci/payloads.go delete mode 100644 pkg/abci/utils.go create mode 100644 pkg/grpc/client/v1/convert.go create mode 100644 pkg/modules/datasets/convert.go rename pkg/serialize/{rlp => }/encode.go (75%) rename pkg/serialize/{rlp => }/encode_test.go (91%) create mode 100644 pkg/serialize/prefix.go delete mode 100644 pkg/serialize/rlp/value.go diff --git a/api/openapi-spec/api/v1/api.swagger.json b/api/openapi-spec/api/v1/api.swagger.json index 82e01e797..3fbf8ae6c 100644 --- a/api/openapi-spec/api/v1/api.swagger.json +++ b/api/openapi-spec/api/v1/api.swagger.json @@ -415,6 +415,40 @@ } }, "definitions": { + "ExtensionsExtensionConfig": { + "type": "object", + "properties": { + "argument": { + "type": "string" + }, + "value": { + "type": "string" + } + } + }, + "TransactionBody": { + "type": "object", + "properties": { + "payload": { + "type": "string", + "format": "byte" + }, + "payloadType": { + "type": "string" + }, + "fee": { + "type": "string" + }, + "nonce": { + "type": "string", + "format": "uint64" + }, + "salt": { + "type": "string", + "format": "byte" + } + } + }, "protobufAny": { "type": "object", "properties": { @@ -504,15 +538,15 @@ "type": "object", "properties": { "tx": { - "$ref": "#/definitions/txTx" + "$ref": "#/definitions/txTransaction" } } }, "txBroadcastResponse": { "type": "object", "properties": { - "receipt": { - "$ref": "#/definitions/txTxReceipt" + "status": { + "$ref": "#/definitions/txTransactionStatus" } } }, @@ -527,7 +561,8 @@ "$ref": "#/definitions/txSignature" }, "sender": { - "type": "string" + "type": "string", + "format": "byte" } } }, @@ -557,41 +592,35 @@ } } }, - "txDataset": { + "txEstimatePriceRequest": { "type": "object", "properties": { - "owner": { - "type": "string" - }, - "name": { - "type": "string" - }, - "tables": { - "type": "array", - "items": { - "$ref": "#/definitions/txTable" - } - }, - "actions": { - "type": "array", - "items": { - "$ref": "#/definitions/txAction" - } + "tx": { + "$ref": "#/definitions/txTransaction" } } }, - "txEstimatePriceRequest": { + "txEstimatePriceResponse": { "type": "object", "properties": { - "tx": { - "$ref": "#/definitions/txTx" + "price": { + "type": "string" } } }, - "txEstimatePriceResponse": { + "txExtensions": { "type": "object", "properties": { - "price": { + "name": { + "type": "string" + }, + "initialization": { + "type": "array", + "items": { + "$ref": "#/definitions/ExtensionsExtensionConfig" + } + }, + "alias": { "type": "string" } } @@ -622,8 +651,8 @@ "txGetSchemaResponse": { "type": "object", "properties": { - "dataset": { - "$ref": "#/definitions/txDataset" + "schema": { + "$ref": "#/definitions/txSchema" } } }, @@ -702,6 +731,35 @@ ], "default": "OK" }, + "txSchema": { + "type": "object", + "properties": { + "owner": { + "type": "string" + }, + "name": { + "type": "string" + }, + "tables": { + "type": "array", + "items": { + "$ref": "#/definitions/txTable" + } + }, + "actions": { + "type": "array", + "items": { + "$ref": "#/definitions/txAction" + } + }, + "extensions": { + "type": "array", + "items": { + "$ref": "#/definitions/txExtensions" + } + } + } + }, "txSignature": { "type": "object", "properties": { @@ -735,49 +793,39 @@ } } }, - "txTx": { + "txTransaction": { "type": "object", "properties": { - "hash": { - "type": "string", - "format": "byte" - }, - "payload_type": { - "type": "integer", - "format": "int32" - }, - "payload": { - "type": "string", - "format": "byte" - }, - "nonce": { - "type": "string", - "format": "int64" + "body": { + "$ref": "#/definitions/TransactionBody" }, "signature": { "$ref": "#/definitions/txSignature" }, - "fee": { - "type": "string" - }, "sender": { - "type": "string" + "type": "string", + "format": "byte" } } }, - "txTxReceipt": { + "txTransactionStatus": { "type": "object", "properties": { - "txHash": { + "id": { "type": "string", "format": "byte" }, "fee": { "type": "string" }, - "body": { - "type": "string", - "format": "byte" + "status": { + "type": "string" + }, + "errors": { + "type": "array", + "items": { + "type": "string" + } } } }, @@ -805,16 +853,16 @@ "type": "object", "properties": { "tx": { - "$ref": "#/definitions/txTx" + "$ref": "#/definitions/txTransaction" } }, - "title": "If power is 0, the validator will be removed from the validator set\nIf power is \u003e 0, if the validator is in the approved validator list, validator will be added to the validator set\nBase64 encoded both pubkey and validator power" + "title": "TODO: these should not wrap Transactions; all of these should just get forwarded to the broadcast endpoint" }, "txValidatorJoinResponse": { "type": "object", "properties": { "receipt": { - "$ref": "#/definitions/txTxReceipt" + "$ref": "#/definitions/txTransactionStatus" } } }, @@ -854,7 +902,7 @@ "type": "object", "properties": { "tx": { - "$ref": "#/definitions/txTx" + "$ref": "#/definitions/txTransaction" } } }, @@ -862,7 +910,7 @@ "type": "object", "properties": { "receipt": { - "$ref": "#/definitions/txTxReceipt" + "$ref": "#/definitions/txTransactionStatus" } } } diff --git a/cmd/kwil-cli/cmds/common/display/response.go b/cmd/kwil-cli/cmds/common/display/response.go index 7e8cda638..6fde649aa 100644 --- a/cmd/kwil-cli/cmds/common/display/response.go +++ b/cmd/kwil-cli/cmds/common/display/response.go @@ -2,17 +2,17 @@ package display import ( "fmt" - txTypes "github.com/kwilteam/kwil-db/pkg/tx" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/kwilteam/kwil-db/pkg/transactions" ) -func PrintTxResponse(res *txTypes.Receipt) { - if res.TxHash != nil { +func PrintTxResponse(res *transactions.TransactionStatus) { + if res.ID != nil { fmt.Println("Success!") } fmt.Println("Response:") - fmt.Println(" Hash:", hexutil.Encode(res.TxHash)) + fmt.Println(" Hash:", hexutil.Encode(res.ID)) fmt.Println(" Fee:", res.Fee) } diff --git a/cmd/kwil-cli/cmds/database/batch.go b/cmd/kwil-cli/cmds/database/batch.go index d6e532154..04b0b7ee3 100644 --- a/cmd/kwil-cli/cmds/database/batch.go +++ b/cmd/kwil-cli/cmds/database/batch.go @@ -11,6 +11,7 @@ import ( "github.com/kwilteam/kwil-db/cmd/kwil-cli/config" "github.com/kwilteam/kwil-db/pkg/client" "github.com/kwilteam/kwil-db/pkg/csv" + "github.com/kwilteam/kwil-db/pkg/transactions" "github.com/spf13/cobra" ) @@ -59,7 +60,17 @@ The execution is treated as a single transaction, and will either succeed or fai return fmt.Errorf("error building inputs: %w", err) } - receipt, err := client.ExecuteAction(ctx, dbid, strings.ToLower(action), inputs) + actionStructure, err := getAction(ctx, client, dbid, action) + if err != nil { + return fmt.Errorf("error getting action: %w", err) + } + + tuples, err := createActionInputs(inputs, actionStructure) + if err != nil { + return fmt.Errorf("error creating action inputs: %w", err) + } + + receipt, err := client.ExecuteAction(ctx, dbid, strings.ToLower(action), tuples...) if err != nil { return fmt.Errorf("error executing action: %w", err) } @@ -85,6 +96,21 @@ The execution is treated as a single transaction, and will either succeed or fai return cmd } +func getAction(ctx context.Context, c *client.Client, dbid, action string) (*transactions.Action, error) { + schema, err := c.GetSchema(context.Background(), dbid) + if err != nil { + return nil, fmt.Errorf("error getting schema: %w", err) + } + + for _, a := range schema.Actions { + if a.Name == action { + return a, nil + } + } + + return nil, fmt.Errorf("action not found: %s", action) +} + // buildInputs builds the inputs for the file func buildInputs(file *os.File, fileType string, columnMappingFlag []string, inputMappings []string) ([]map[string]any, error) { switch fileType { diff --git a/cmd/kwil-cli/cmds/database/call.go b/cmd/kwil-cli/cmds/database/call.go index 234244296..f216cfa65 100644 --- a/cmd/kwil-cli/cmds/database/call.go +++ b/cmd/kwil-cli/cmds/database/call.go @@ -12,7 +12,7 @@ import ( ) func callCmd() *cobra.Command { - var actionName string + var action string cmd := &cobra.Command{ Use: "call", @@ -41,22 +41,33 @@ OR '$name:satoshi' '$age:32' --dbid=x1234 --action=create_user `, RunE: func(cmd *cobra.Command, args []string) error { return common.DialClient(cmd.Context(), 0, func(ctx context.Context, client *client.Client, conf *config.KwilCliConfig) error { - dbId, err := getSelectedDbid(cmd, conf) + dbid, err := getSelectedDbid(cmd, conf) if err != nil { return fmt.Errorf("target database not properly specified: %w", err) } - lowerName := strings.ToLower(actionName) + lowerName := strings.ToLower(action) - inputs, err := GetInputs(args) + inputs, err := parseInputs(args) if err != nil { return fmt.Errorf("error getting inputs: %w", err) } - if len(inputs) == 0 { - inputs = append(inputs, map[string]any{}) + + actionStructure, err := getAction(ctx, client, dbid, lowerName) + if err != nil { + return fmt.Errorf("error getting action: %w", err) + } + + tuples, err := createActionInputs(inputs, actionStructure) + if err != nil { + return fmt.Errorf("error creating action inputs: %w", err) + } + + if len(tuples) == 0 { + tuples = append(tuples, []any{}) } - res, err := client.CallAction(ctx, dbId, lowerName, inputs[0]) + res, err := client.CallAction(ctx, dbid, lowerName, tuples[0]) if err != nil { return fmt.Errorf("error executing action: %w", err) } @@ -78,7 +89,7 @@ OR cmd.Flags().StringP(nameFlag, "n", "", "the database name") cmd.Flags().StringP(ownerFlag, "o", "", "the database owner") cmd.Flags().StringP(dbidFlag, "i", "", "the database id") - cmd.Flags().StringVarP(&actionName, actionNameFlag, "a", "", "the action name (required)") + cmd.Flags().StringVarP(&action, actionNameFlag, "a", "", "the action name (required)") cmd.MarkFlagRequired(actionNameFlag) return cmd diff --git a/cmd/kwil-cli/cmds/database/deploy.go b/cmd/kwil-cli/cmds/database/deploy.go index a7f50b94d..f468a90cc 100644 --- a/cmd/kwil-cli/cmds/database/deploy.go +++ b/cmd/kwil-cli/cmds/database/deploy.go @@ -17,7 +17,7 @@ import ( "github.com/kwilteam/kwil-db/cmd/kwil-cli/config" "github.com/kwilteam/kwil-db/pkg/client" "github.com/kwilteam/kwil-db/pkg/crypto" - "github.com/kwilteam/kwil-db/pkg/serialize" + "github.com/kwilteam/kwil-db/pkg/transactions" "github.com/spf13/cobra" ) @@ -37,7 +37,7 @@ func deployCmd() *cobra.Command { } defer file.Close() - var db *serialize.Schema + var db *transactions.Schema if fileType == "kf" { db, err = UnmarshalKf(file) } else if fileType == "json" { @@ -68,7 +68,7 @@ func deployCmd() *cobra.Command { return cmd } -func UnmarshalKf(file *os.File) (*serialize.Schema, error) { +func UnmarshalKf(file *os.File) (*transactions.Schema, error) { source, err := io.ReadAll(file) if err != nil { return nil, fmt.Errorf("failed to read Kuneiform source file: %w", err) @@ -84,7 +84,7 @@ func UnmarshalKf(file *os.File) (*serialize.Schema, error) { return nil, fmt.Errorf("failed to marshal schema: %w", err) } - var db serialize.Schema + var db transactions.Schema err = json.Unmarshal(schemaJson, &db) if err != nil { return nil, fmt.Errorf("failed to unmarshal schema json: %w", err) @@ -93,13 +93,13 @@ func UnmarshalKf(file *os.File) (*serialize.Schema, error) { return &db, nil } -func UnmarshalJson(file *os.File) (*serialize.Schema, error) { +func UnmarshalJson(file *os.File) (*transactions.Schema, error) { bts, err := io.ReadAll(file) if err != nil { return nil, fmt.Errorf("failed to read file: %w", err) } - var db serialize.Schema + var db transactions.Schema err = json.Unmarshal(bts, &db) if err != nil { return nil, fmt.Errorf("failed to unmarshal file: %w", err) diff --git a/cmd/kwil-cli/cmds/database/execute.go b/cmd/kwil-cli/cmds/database/execute.go index 1c8af7ce9..13d642df1 100644 --- a/cmd/kwil-cli/cmds/database/execute.go +++ b/cmd/kwil-cli/cmds/database/execute.go @@ -9,6 +9,7 @@ import ( "github.com/kwilteam/kwil-db/cmd/kwil-cli/cmds/common/display" "github.com/kwilteam/kwil-db/cmd/kwil-cli/config" "github.com/kwilteam/kwil-db/pkg/client" + "github.com/kwilteam/kwil-db/pkg/transactions" "github.com/spf13/cobra" ) @@ -50,12 +51,17 @@ OR lowerName := strings.ToLower(actionName) - inputs, err := GetInputs(args) + actionStructure, err := getAction(ctx, client, dbId, lowerName) + if err != nil { + return fmt.Errorf("error getting action: %w", err) + } + + inputs, err := GetInputs(args, actionStructure) if err != nil { return fmt.Errorf("error getting inputs: %w", err) } - receipt, err := client.ExecuteAction(ctx, dbId, lowerName, inputs) + receipt, err := client.ExecuteAction(ctx, dbId, lowerName, inputs...) if err != nil { return fmt.Errorf("error executing database: %w", err) } @@ -83,7 +89,7 @@ OR // inputs will be received as args. The args will be in the form of // $argname:value. Example $username:satoshi $age:32 -func GetInputs(args []string) ([]map[string]any, error) { +func parseInputs(args []string) ([]map[string]any, error) { inputs := make(map[string]any) for _, arg := range args { @@ -101,6 +107,36 @@ func GetInputs(args []string) ([]map[string]any, error) { return []map[string]any{inputs}, nil } +func GetInputs(args []string, action *transactions.Action) ([][]any, error) { + inputs, err := parseInputs(args) + if err != nil { + return nil, fmt.Errorf("error getting inputs: %w", err) + } + + return createActionInputs(inputs, action) +} + +// createActionInputs takes a []map[string]any and an action, and converts it to [][]any +func createActionInputs(inputs []map[string]any, action *transactions.Action) ([][]any, error) { + + tuples := [][]any{} + for _, input := range inputs { + newTuple := []any{} + for _, inputField := range action.Inputs { + value, ok := input[inputField] + if !ok { + return nil, fmt.Errorf("missing input: %s", inputField) + } + + newTuple = append(newTuple, value) + } + + tuples = append(tuples, newTuple) + } + + return tuples, nil +} + func printActionResults(results []map[string]any) { for _, row := range results { for k, v := range row { diff --git a/cmd/kwil-cli/cmds/database/read_schema.go b/cmd/kwil-cli/cmds/database/read_schema.go index 69271544a..89e8c3cb3 100644 --- a/cmd/kwil-cli/cmds/database/read_schema.go +++ b/cmd/kwil-cli/cmds/database/read_schema.go @@ -40,7 +40,7 @@ func readSchemaCmd() *cobra.Command { fmt.Printf(" Type: %s\n", c.Type) for _, a := range c.Attributes { fmt.Printf(" %s\n", a.Type) - if a.Value != nil { + if a.Value != "" { fmt.Printf(" %s\n", fmt.Sprint(a.Value)) } } diff --git a/cmd/kwil-cli/cmds/utils/node_cfg.go b/cmd/kwil-cli/cmds/utils/node_cfg.go deleted file mode 100644 index 07b7893ac..000000000 --- a/cmd/kwil-cli/cmds/utils/node_cfg.go +++ /dev/null @@ -1,79 +0,0 @@ -package utils - -import ( - "context" - "fmt" - "html/template" - "os" - "text/tabwriter" - - "github.com/kwilteam/kwil-db/cmd/kwil-cli/cmds/common" - "github.com/kwilteam/kwil-db/cmd/kwil-cli/config" - "github.com/kwilteam/kwil-db/pkg/client" - grpc "github.com/kwilteam/kwil-db/pkg/grpc/client/v1" - - "github.com/pkg/errors" - "github.com/spf13/cobra" -) - -var cfgYamlTemplate = ` -Funding: - ChainCode: {{.Funding.ChainCode}} - PoolAddress: {{.Funding.PoolAddress}} - ProviderAddress: {{.Funding.ProviderAddress}} - RpcUrl: {{.Funding.RpcUrl}} -Gateway: - GraphqlUrl: {{.Gateway.GraphqlUrl}} -` - -type cfgOptions struct { - format string -} - -func NewServerCfgCmd() *cobra.Command { - var opts cfgOptions - - var cmd = &cobra.Command{ - Use: "node-config [OPTIONS]", - Short: "Show connected node configuration", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - return common.DialClient(cmd.Context(), common.WithoutPrivateKey, func(ctx context.Context, client *client.Client, config *config.KwilCliConfig) error { - tmpl := template.New("version") - // load different template according to the opts.format - cfgTemplate := cfgYamlTemplate - tmpl, err := tmpl.Parse(cfgTemplate) - if err != nil { - return errors.Wrap(err, "template parsing error") - } - - cfg, err := client.GetConfig(ctx) - if err != nil { - return errors.Wrap(err, "error getting node configuration") - } - - printCfg(cfg) - return nil - }) - }, - } - - flags := cmd.Flags() - flags.StringVarP(&opts.format, "format", "f", "", "Format the output using the given Go template") - - return cmd -} - -func printCfg(cfg *grpc.SvcConfig) { - fmt.Printf("ChainCode: %d\n", cfg.ChainCode) - fmt.Printf("PoolAddress: %s\n", cfg.PoolAddress) - fmt.Printf("ProviderAddress: %s\n", cfg.ProviderAddress) -} - -func prettyPrint(svcCfg *grpc.SvcConfig, tmpl *template.Template) error { - t := tabwriter.NewWriter(os.Stdout, 20, 1, 1, ' ', 0) - err := tmpl.Execute(t, svcCfg) - _, _ = t.Write([]byte("\n")) - t.Flush() - return err -} diff --git a/cmd/kwil-cli/cmds/utils/utils.go b/cmd/kwil-cli/cmds/utils/utils.go index 011522640..96e7ec6a3 100644 --- a/cmd/kwil-cli/cmds/utils/utils.go +++ b/cmd/kwil-cli/cmds/utils/utils.go @@ -16,7 +16,6 @@ func NewCmdUtils() *cobra.Command { signCmd(), pingCmd(), printConfigCmd(), - NewServerCfgCmd(), ) return cmd diff --git a/internal/app/kwild/client/client.go b/internal/app/kwild/client/client.go index 01d843e84..82d6bc883 100644 --- a/internal/app/kwild/client/client.go +++ b/internal/app/kwild/client/client.go @@ -19,7 +19,11 @@ func NewClient(ctx context.Context, cfg *KwildConfig) (*client.Client, error) { options = append(options, client.WithCometBftUrl(cfg.ClientChainRPCURL)) } if cfg.PrivateKey != "" { - key, _ := crypto.ECDSAFromHex(cfg.PrivateKey) + key, err := crypto.PrivateKeyFromHex(cfg.PrivateKey) + if err != nil { + return nil, err + } + options = append(options, client.WithPrivateKey(key)) } clt, err := client.New(ctx, cfg.GrpcURL, options...) diff --git a/internal/app/kwild/config/config_test.go b/internal/app/kwild/config/config_test.go index 05d4eac4e..040dcf63a 100644 --- a/internal/app/kwild/config/config_test.go +++ b/internal/app/kwild/config/config_test.go @@ -1,12 +1,10 @@ package config_test import ( - "fmt" "os" "testing" config "github.com/kwilteam/kwil-db/internal/app/kwild/config" - "github.com/kwilteam/kwil-db/pkg/crypto" ) func Test_Config(t *testing.T) { @@ -14,11 +12,8 @@ func Test_Config(t *testing.T) { os.Setenv("KWILD_PORT", "8081") os.Setenv("KWILD_DEPOSITS_POOL_ADDRESS", "0xabc") os.Setenv("KWILD_EXTENSION_ENDPOINTS", "localhost:8080,localhost:8081, localhost:8082") - cfg, err := config.LoadKwildConfig() + _, err := config.LoadKwildConfig() if err != nil { t.Fatal(err) } - - addr := crypto.AddressFromPrivateKey(cfg.PrivateKey) - fmt.Println(addr) } diff --git a/internal/app/kwild/driver.go b/internal/app/kwild/driver.go index 3af8d4b90..41bc0c309 100644 --- a/internal/app/kwild/driver.go +++ b/internal/app/kwild/driver.go @@ -10,8 +10,7 @@ import ( "github.com/kwilteam/kwil-db/pkg/client" "github.com/kwilteam/kwil-db/pkg/engine/utils" "github.com/kwilteam/kwil-db/pkg/log" - "github.com/kwilteam/kwil-db/pkg/serialize" - kTx "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" types "github.com/cometbft/cometbft/abci/types" ec "github.com/ethereum/go-ethereum/crypto" @@ -42,14 +41,15 @@ func (d *KwildDriver) GetUserAddress() string { // TODO: this likely needs to change; the old Kwild driver is not compatible, since deploy, drop, and execute are asynchronous -func (d *KwildDriver) DeployDatabase(ctx context.Context, db *serialize.Schema) error { +func (d *KwildDriver) DeployDatabase(ctx context.Context, db *transactions.Schema) error { rec, err := d.clt.DeployDatabase(ctx, db) if err != nil { fmt.Println("Error deploying database: ", err.Error()) return fmt.Errorf("error deploying database: %w", err) } - res, err := d.clt.CometBftClient.Tx(ctx, rec.TxHash, false) + // this likely does not work; cometbft uses its own transaction hash, not the one generated from Kwil + res, err := d.clt.CometBftClient.Tx(ctx, rec.ID, false) if err != nil { return fmt.Errorf("error getting transaction: %w", err) } @@ -76,31 +76,27 @@ func (d *KwildDriver) DatabaseShouldExists(ctx context.Context, owner string, db return fmt.Errorf("database does not exist") } -func (d *KwildDriver) ExecuteAction(ctx context.Context, dbid string, actionName string, actionInputs []map[string]any) (*kTx.Receipt, []map[string]any, error) { - rec, err := d.clt.ExecuteAction(ctx, dbid, actionName, actionInputs) +func (d *KwildDriver) ExecuteAction(ctx context.Context, dbid string, actionName string, actionInputs ...[]any) (*transactions.TransactionStatus, error) { + rec, err := d.clt.ExecuteAction(ctx, dbid, actionName, actionInputs...) if err != nil { - return nil, nil, fmt.Errorf("error executing query: %w", err) + return nil, fmt.Errorf("error executing query: %w", err) } - res, err := d.clt.CometBftClient.Tx(ctx, rec.TxHash, false) + // this likely does not work; cometbft uses its own transaction hash, not the one generated from Kwil + res, err := d.clt.CometBftClient.Tx(ctx, rec.ID, false) if err != nil { - return nil, nil, fmt.Errorf("error getting transaction: %w", err) + return nil, fmt.Errorf("error getting transaction: %w", err) } data := res.TxResult.Data - var updated_rec *kTx.Receipt + var updated_rec *transactions.TransactionStatus err = json.Unmarshal(data, &updated_rec) if err != nil { - return nil, nil, err - } - - outputs, err := client.DecodeOutputs(updated_rec.Body) - if err != nil { - return nil, nil, err + return nil, err } d.logger.Debug("execute action", zap.String("database", dbid), zap.String("action", actionName)) - return rec, outputs, nil + return rec, nil } func (d *KwildDriver) DropDatabase(ctx context.Context, dbName string) error { @@ -109,7 +105,8 @@ func (d *KwildDriver) DropDatabase(ctx context.Context, dbName string) error { return fmt.Errorf("error dropping database: %w", err) } - res, err := d.clt.CometBftClient.Tx(ctx, rec.TxHash, false) + // this likely does not work; cometbft uses its own transaction hash, not the one generated from Kwil + res, err := d.clt.CometBftClient.Tx(ctx, rec.ID, false) if err != nil { return fmt.Errorf("error getting transaction: %w", err) } @@ -126,7 +123,7 @@ func (d *KwildDriver) QueryDatabase(ctx context.Context, dbid, query string) (*c return d.clt.Query(ctx, dbid, query) } -func (d *KwildDriver) Call(ctx context.Context, dbid, action string, inputs map[string]any, opts ...client.CallOpt) ([]map[string]any, error) { +func (d *KwildDriver) Call(ctx context.Context, dbid, action string, inputs []any, opts ...client.CallOpt) ([]map[string]any, error) { return d.clt.CallAction(ctx, dbid, action, inputs, opts...) } diff --git a/internal/app/kwild/server/root.go b/internal/app/kwild/server/root.go index 85b33c14c..fdcf9c061 100644 --- a/internal/app/kwild/server/root.go +++ b/internal/app/kwild/server/root.go @@ -2,7 +2,6 @@ package server import ( "context" - "encoding/json" "fmt" "net" "os" @@ -25,19 +24,16 @@ import ( "github.com/kwilteam/kwil-db/pkg/log" "github.com/kwilteam/kwil-db/pkg/modules/datasets" "github.com/kwilteam/kwil-db/pkg/sql" - "github.com/kwilteam/kwil-db/pkg/tx" "github.com/spf13/viper" "google.golang.org/grpc/health/grpc_health_v1" txSvc "github.com/kwilteam/kwil-db/internal/controller/grpc/txsvc/v1" cmtcfg "github.com/cometbft/cometbft/config" - "github.com/cometbft/cometbft/crypto/tmhash" cmtflags "github.com/cometbft/cometbft/libs/cli/flags" cmtlog "github.com/cometbft/cometbft/libs/log" nm "github.com/cometbft/cometbft/node" cmtlocal "github.com/cometbft/cometbft/rpc/client/local" - cmttypes "github.com/cometbft/cometbft/types" ) // BuildKwildServer builds the kwild server @@ -66,25 +62,6 @@ func BuildKwildServer(ctx context.Context) (svr *Server, err error) { return buildServer(deps), nil } -// wrappedCometBFTClient satisfies the generic txsvc.BlockchainBroadcaster -// interface, hiding the details of cometBFT. -type wrappedCometBFTClient struct { - *cmtlocal.Local -} - -func (wc *wrappedCometBFTClient) BroadcastTxAsync(ctx context.Context, tx *tx.Transaction) ([]byte, error) { - bts, err := json.Marshal(tx) - if err != nil { - return nil, fmt.Errorf("failed to serialize transaction data: %w", err) - } - hash := tmhash.Sum(bts) - _, err = wc.Local.BroadcastTxAsync(ctx, cmttypes.Tx(bts)) - if err != nil { - return nil, err - } - return hash, err -} - func buildServer(d *coreDependencies) *Server { // engine e := buildEngine(d) diff --git a/internal/app/kwild/server/utils.go b/internal/app/kwild/server/utils.go index c80c9e0a8..d6eb93d4a 100644 --- a/internal/app/kwild/server/utils.go +++ b/internal/app/kwild/server/utils.go @@ -5,11 +5,14 @@ import ( "fmt" "os" + cmtlocal "github.com/cometbft/cometbft/rpc/client/local" + cmttypes "github.com/cometbft/cometbft/types" "github.com/kwilteam/kwil-db/pkg/engine" "github.com/kwilteam/kwil-db/pkg/extensions" "github.com/kwilteam/kwil-db/pkg/log" "github.com/kwilteam/kwil-db/pkg/sql" "github.com/kwilteam/kwil-db/pkg/sql/client" + "github.com/kwilteam/kwil-db/pkg/transactions" ) var defaultFilePath string @@ -83,3 +86,23 @@ func (s *sqliteOpener) Open(fileName string, logger log.Logger) (sql.Database, e client.WithPath(s.sqliteFilePath), ) } + +// wrappedCometBFTClient satisfies the generic txsvc.BlockchainBroadcaster +// interface, hiding the details of cometBFT. +type wrappedCometBFTClient struct { + *cmtlocal.Local +} + +func (wc *wrappedCometBFTClient) BroadcastTxAsync(ctx context.Context, tx *transactions.Transaction) error { + bts, err := tx.MarshalBinary() + if err != nil { + return fmt.Errorf("failed to serialize transaction data: %w", err) + } + + _, err = wc.Local.BroadcastTxAsync(ctx, cmttypes.Tx(bts)) + if err != nil { + return err + } + + return err +} diff --git a/internal/controller/grpc/txsvc/v1/broadcast.go b/internal/controller/grpc/txsvc/v1/broadcast.go index ecb5efd15..e68ace830 100644 --- a/internal/controller/grpc/txsvc/v1/broadcast.go +++ b/internal/controller/grpc/txsvc/v1/broadcast.go @@ -8,13 +8,12 @@ import ( "google.golang.org/grpc/status" txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" - "github.com/kwilteam/kwil-db/pkg/crypto" - kTx "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" "go.uber.org/zap" ) func (s *Service) Broadcast(ctx context.Context, req *txpb.BroadcastRequest) (*txpb.BroadcastResponse, error) { - tx, err := convertTx(req.Tx) + tx, err := convertTransaction(req.Tx) if err != nil { return nil, status.Errorf(codes.Internal, "failed to convert transaction: %s", err) } @@ -24,15 +23,21 @@ func (s *Service) Broadcast(ctx context.Context, req *txpb.BroadcastRequest) (*t return nil, status.Errorf(codes.Unauthenticated, "failed to verify transaction: %s", err) } - hash, err := s.chainClient.BroadcastTxAsync(ctx, tx) + err = s.chainClient.BroadcastTxAsync(ctx, tx) if err != nil { return nil, fmt.Errorf("failed to broadcast transaction with error: %s", err) } - s.log.Info("broadcasted transaction ", zap.String("payload_type", tx.PayloadType.String())) + txHash, err := tx.GetHash() + if err != nil { + return nil, fmt.Errorf("failed to get transaction hash with error: %s", err) + } + + s.log.Info("broadcasted transaction ", zap.String("payload_type", tx.Body.PayloadType.String()), zap.ByteString("ID", txHash)) return &txpb.BroadcastResponse{ - Receipt: &txpb.TxReceipt{ - TxHash: hash, + Status: &txpb.TransactionStatus{ + Id: txHash, + Status: transactions.StatusPending.String(), }, }, nil } @@ -50,49 +55,3 @@ func (s *Service) Broadcast(ctx context.Context, req *txpb.BroadcastRequest) (*t // }, // }, nil // } - -func convertTx(incoming *txpb.Tx) (*kTx.Transaction, error) { - payloadType := kTx.PayloadType(incoming.PayloadType) - if err := payloadType.IsValid(); err != nil { - return nil, err - } - - convSignature, err := convertSignature(incoming.Signature) - if err != nil { - return nil, err - } - - return &kTx.Transaction{ - Hash: incoming.Hash, - PayloadType: payloadType, - Payload: incoming.Payload, - Fee: incoming.Fee, - Nonce: incoming.Nonce, - Signature: convSignature, - Sender: incoming.Sender, - }, nil -} - -func newEmptySignature() (bytes []byte, sigType crypto.SignatureType) { - return []byte{}, crypto.PK_SECP256K1_UNCOMPRESSED -} - -func convertSignature(sig *txpb.Signature) (*crypto.Signature, error) { - if sig == nil { - emptyBts, emptyType := newEmptySignature() - return &crypto.Signature{ - Signature: emptyBts, - Type: emptyType, - }, nil - } - - sigType := crypto.SignatureType(sig.SignatureType) - if err := sigType.IsValid(); err != nil { - return nil, err - } - - return &crypto.Signature{ - Signature: sig.SignatureBytes, - Type: sigType, - }, nil -} diff --git a/internal/controller/grpc/txsvc/v1/call.go b/internal/controller/grpc/txsvc/v1/call.go index 93367103f..f6bc3df18 100644 --- a/internal/controller/grpc/txsvc/v1/call.go +++ b/internal/controller/grpc/txsvc/v1/call.go @@ -8,7 +8,8 @@ import ( "google.golang.org/grpc/status" txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" - "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/crypto" + "github.com/kwilteam/kwil-db/pkg/transactions" ) func (s *Service) Call(ctx context.Context, req *txpb.CallRequest) (*txpb.CallResponse, error) { @@ -18,14 +19,19 @@ func (s *Service) Call(ctx context.Context, req *txpb.CallRequest) (*txpb.CallRe return nil, status.Errorf(codes.InvalidArgument, "failed to convert action call: %s", err.Error()) } - if msg.Sender != "" { + if msg.Sender == nil { err = msg.Verify() if err != nil { return nil, status.Errorf(codes.InvalidArgument, "failed to verify signed message: %s", err.Error()) } } - executeResult, err := s.engine.Call(ctx, body, msg) + args := make([]any, len(body.Arguments)) + for i, arg := range body.Arguments { + args[i] = arg + } + + executeResult, err := s.engine.Call(ctx, body.DBID, body.Action, args, msg) if err != nil { return nil, status.Errorf(codes.Internal, "failed to execution action: %s", err.Error()) } @@ -40,9 +46,14 @@ func (s *Service) Call(ctx context.Context, req *txpb.CallRequest) (*txpb.CallRe }, nil } -func convertActionCall(req *txpb.CallRequest) (*tx.CallActionPayload, *tx.SignedMessage[tx.JsonPayload], error) { - var actionPayload *tx.CallActionPayload - err := json.Unmarshal(req.Payload, &actionPayload) +func convertActionCall(req *txpb.CallRequest) (*transactions.ActionCall, *transactions.SignedMessage, error) { + var actionPayload transactions.ActionCall + err := actionPayload.UnmarshalBinary(req.Payload) + if err != nil { + return nil, nil, err + } + + sender, err := crypto.PublicKeyFromBytes(req.Sender) if err != nil { return nil, nil, err } @@ -52,13 +63,9 @@ func convertActionCall(req *txpb.CallRequest) (*tx.CallActionPayload, *tx.Signed return nil, nil, err } - return &tx.CallActionPayload{ - Action: actionPayload.Action, - DBID: actionPayload.DBID, - Params: actionPayload.Params, - }, &tx.SignedMessage[tx.JsonPayload]{ - Payload: tx.JsonPayload(req.Payload), - Signature: convSignature, - Sender: req.Sender, - }, nil + return &actionPayload, &transactions.SignedMessage{ + Message: req.Payload, + Signature: convSignature, + Sender: sender, + }, nil } diff --git a/internal/controller/grpc/txsvc/v1/convert.go b/internal/controller/grpc/txsvc/v1/convert.go new file mode 100644 index 000000000..1f2aa2916 --- /dev/null +++ b/internal/controller/grpc/txsvc/v1/convert.go @@ -0,0 +1,182 @@ +package txsvc + +import ( + "fmt" + + "math/big" + + txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" + "github.com/kwilteam/kwil-db/pkg/crypto" + engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" + "github.com/kwilteam/kwil-db/pkg/transactions" +) + +func convertTransaction(incoming *txpb.Transaction) (*transactions.Transaction, error) { + payloadType := transactions.PayloadType(incoming.Body.PayloadType) + if !payloadType.Valid() { + return nil, fmt.Errorf("invalid payload type: %s", incoming.Body.PayloadType) + } + + if incoming.Signature == nil { + return nil, fmt.Errorf("transaction signature cannot be nil") + } + + sender, err := crypto.PublicKeyFromBytes(incoming.Sender) + if err != nil { + return nil, fmt.Errorf("invalid sender public key: %s", err.Error()) + } + + convSignature, err := convertSignature(incoming.Signature) + if err != nil { + return nil, err + } + + bigFee, ok := big.NewInt(0).SetString(incoming.Body.Fee, 10) + if !ok { + return nil, fmt.Errorf("invalid fee: %s", incoming.Body.Fee) + } + + return &transactions.Transaction{ + Signature: convSignature, + Body: &transactions.TransactionBody{ + PayloadType: payloadType, + Payload: incoming.Body.Payload, + Nonce: incoming.Body.Nonce, + Fee: bigFee, + Salt: incoming.Body.Salt, + }, + Sender: sender, + }, nil +} + +func newEmptySignature() (bytes []byte, sigType crypto.SignatureType) { + return []byte{}, crypto.SIGNATURE_TYPE_EMPTY +} + +func convertSignature(sig *txpb.Signature) (*crypto.Signature, error) { + if sig == nil { + emptyBts, emptyType := newEmptySignature() + return &crypto.Signature{ + Signature: emptyBts, + Type: emptyType, + }, nil + } + + sigType := crypto.SignatureType(sig.SignatureType) + if err := sigType.IsValid(); err != nil { + return nil, err + } + + return &crypto.Signature{ + Signature: sig.SignatureBytes, + Type: sigType, + }, nil +} + +func convertSchemaFromEngine(schema *engineTypes.Schema) (*txpb.Schema, error) { + actions, err := convertActionsFromEngine(schema.Procedures) + if err != nil { + return nil, err + } + return &txpb.Schema{ + Owner: schema.Owner, + Name: schema.Name, + Tables: convertTablesFromEngine(schema.Tables), + Actions: actions, + }, nil +} + +func convertTablesFromEngine(tables []*engineTypes.Table) []*txpb.Table { + convTables := make([]*txpb.Table, len(tables)) + for i, table := range tables { + convTable := &txpb.Table{ + Name: table.Name, + Columns: convertColumnsFromEngine(table.Columns), + Indexes: convertIndexesFromEngine(table.Indexes), + } + convTables[i] = convTable + } + + return convTables +} + +func convertColumnsFromEngine(columns []*engineTypes.Column) []*txpb.Column { + convColumns := make([]*txpb.Column, len(columns)) + for i, column := range columns { + convColumn := &txpb.Column{ + Name: column.Name, + Type: column.Type.String(), + Attributes: convertAttributesFromEngine(column.Attributes), + } + convColumns[i] = convColumn + } + + return convColumns +} + +func convertAttributesFromEngine(attributes []*engineTypes.Attribute) []*txpb.Attribute { + convAttributes := make([]*txpb.Attribute, len(attributes)) + for i, attribute := range attributes { + convAttribute := &txpb.Attribute{ + Type: attribute.Type.String(), + Value: fmt.Sprint(attribute.Value), + } + convAttributes[i] = convAttribute + } + + return convAttributes +} + +func convertIndexesFromEngine(indexes []*engineTypes.Index) []*txpb.Index { + convIndexes := make([]*txpb.Index, len(indexes)) + for i, index := range indexes { + convIndexes[i] = &txpb.Index{ + Name: index.Name, + Columns: index.Columns, + Type: index.Type.String(), + } + } + + return convIndexes +} + +func convertActionsFromEngine(actions []*engineTypes.Procedure) ([]*txpb.Action, error) { + + convActions := make([]*txpb.Action, len(actions)) + for i, action := range actions { + mutability, auxiliaries, err := convertModifiersFromEngine(action.Modifiers) + if err != nil { + return nil, err + } + + convActions[i] = &txpb.Action{ + Name: action.Name, + Public: action.Public, + Mutability: mutability, + Auxiliaries: auxiliaries, + Inputs: action.Args, + Statements: action.Statements, + } + } + + return convActions, nil +} + +func convertModifiersFromEngine(mods []engineTypes.Modifier) (mutability string, auxiliaries []string, err error) { + auxiliaries = make([]string, 0) + mutability = "UPDATE" + for _, mod := range mods { + switch mod { + case engineTypes.ModifierAuthenticated: + auxiliaries = append(auxiliaries, "AUTHENTICATED") + case engineTypes.ModifierView: + mutability = "VIEW" + case engineTypes.ModifierOwner: + auxiliaries = append(auxiliaries, "OWNER") + default: + return "", nil, fmt.Errorf("unknown modifier type: %v", mod) + } + } + + return mutability, auxiliaries, nil +} diff --git a/internal/controller/grpc/txsvc/v1/pricing.go b/internal/controller/grpc/txsvc/v1/pricing.go index 142457965..43f316d3b 100644 --- a/internal/controller/grpc/txsvc/v1/pricing.go +++ b/internal/controller/grpc/txsvc/v1/pricing.go @@ -6,31 +6,31 @@ import ( "math/big" txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" - "github.com/kwilteam/kwil-db/pkg/serialize" - kTx "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/modules/datasets" + "github.com/kwilteam/kwil-db/pkg/transactions" ) func (s *Service) EstimatePrice(ctx context.Context, req *txpb.EstimatePriceRequest) (*txpb.EstimatePriceResponse, error) { - tx, err := convertTx(req.Tx) + tx, err := convertTransaction(req.Tx) if err != nil { return nil, fmt.Errorf("failed to convert transaction: %w", err) } var price *big.Int - switch tx.PayloadType { - case kTx.DEPLOY_DATABASE: + switch tx.Body.PayloadType { + case transactions.PayloadTypeDeploySchema: price, err = s.priceDeploy(ctx, tx) - case kTx.DROP_DATABASE: + case transactions.PayloadTypeDropSchema: price, err = s.priceDrop(ctx, tx) - case kTx.EXECUTE_ACTION: + case transactions.PayloadTypeExecuteAction: price, err = s.priceAction(ctx, tx) - case kTx.VALIDATOR_JOIN: + case transactions.PayloadTypeValidatorJoin: price, err = s.priceValidatorJoin(ctx, tx) - case kTx.VALIDATOR_LEAVE: + case transactions.PayloadTypeValidatorApprove: price, err = s.priceValidatorLeave(ctx, tx) default: - price, err = nil, fmt.Errorf("invalid payload type") + price, err = nil, fmt.Errorf("invalid transaction payload type %s", tx.Body.PayloadType) } if err != nil { return nil, fmt.Errorf("failed to estimate price: %w", err) @@ -41,37 +41,55 @@ func (s *Service) EstimatePrice(ctx context.Context, req *txpb.EstimatePriceRequ }, nil } -func (s *Service) priceDeploy(ctx context.Context, tx *kTx.Transaction) (*big.Int, error) { - ds, err := serialize.DeserializeSchema(tx.Payload) +func (s *Service) priceDeploy(ctx context.Context, tx *transactions.Transaction) (*big.Int, error) { + schema := &transactions.Schema{} + err := schema.UnmarshalBinary(tx.Body.Payload) if err != nil { - return nil, fmt.Errorf("failed to deserialize schema: %w", err) + return nil, fmt.Errorf("failed to unmarshal schema: %w", err) } - return s.engine.PriceDeploy(ctx, ds) + convertedSchema, err := datasets.ConvertSchemaToEngine(schema) + if err != nil { + return nil, fmt.Errorf("failed to convert schema: %w", err) + } + + return s.engine.PriceDeploy(ctx, convertedSchema) } -func (s *Service) priceDrop(ctx context.Context, tx *kTx.Transaction) (*big.Int, error) { - dbid, err := serialize.DeserializeDBID(tx.Payload) +func (s *Service) priceDrop(ctx context.Context, tx *transactions.Transaction) (*big.Int, error) { + dropSchema := &transactions.DropSchema{} + err := dropSchema.UnmarshalBinary(tx.Body.Payload) if err != nil { - return nil, fmt.Errorf("failed to deserialize DBID: %w", err) + return nil, fmt.Errorf("failed to unmarshal drop schema: %w", err) } - return s.engine.PriceDrop(ctx, dbid) + return s.engine.PriceDrop(ctx, dropSchema.DBID) } -func (s *Service) priceAction(ctx context.Context, tx *kTx.Transaction) (*big.Int, error) { - executionBody, err := serialize.DeserializeActionPayload(tx.Payload) +func (s *Service) priceAction(ctx context.Context, tx *transactions.Transaction) (*big.Int, error) { + executionBody := &transactions.ActionExecution{} + err := executionBody.UnmarshalBinary(tx.Body.Payload) if err != nil { - return nil, fmt.Errorf("failed to deserialize action execution: %w", err) + return nil, fmt.Errorf("failed to unmarshal action execution: %w", err) + } + + var tuples [][]any + for _, tuple := range executionBody.Arguments { + newTuple := make([]any, len(tuple)) + for i, arg := range tuple { + newTuple[i] = arg + } + + tuples = append(tuples, newTuple) } - return s.engine.PriceExecute(ctx, executionBody.DBID, executionBody.Action, executionBody.Params) + return s.engine.PriceExecute(ctx, executionBody.DBID, executionBody.Action, tuples) } -func (s *Service) priceValidatorJoin(ctx context.Context, tx *kTx.Transaction) (*big.Int, error) { +func (s *Service) priceValidatorJoin(ctx context.Context, tx *transactions.Transaction) (*big.Int, error) { return big.NewInt(10000000000000), nil } -func (s *Service) priceValidatorLeave(ctx context.Context, tx *kTx.Transaction) (*big.Int, error) { +func (s *Service) priceValidatorLeave(ctx context.Context, tx *transactions.Transaction) (*big.Int, error) { return big.NewInt(10000000000000), nil } diff --git a/internal/controller/grpc/txsvc/v1/schema.go b/internal/controller/grpc/txsvc/v1/schema.go index 91351c48a..1c568eb0f 100644 --- a/internal/controller/grpc/txsvc/v1/schema.go +++ b/internal/controller/grpc/txsvc/v1/schema.go @@ -2,10 +2,8 @@ package txsvc import ( "context" - "fmt" txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" - engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" ) func (s *Service) GetSchema(ctx context.Context, req *txpb.GetSchemaRequest) (*txpb.GetSchemaResponse, error) { @@ -14,119 +12,12 @@ func (s *Service) GetSchema(ctx context.Context, req *txpb.GetSchemaRequest) (*t return nil, err } - txSchema, err := convertSchema(schema) + txSchema, err := convertSchemaFromEngine(schema) if err != nil { return nil, err } return &txpb.GetSchemaResponse{ - Dataset: txSchema, + Schema: txSchema, }, nil } - -func convertSchema(schema *engineTypes.Schema) (*txpb.Dataset, error) { - actions, err := convertActions(schema.Procedures) - if err != nil { - return nil, err - } - return &txpb.Dataset{ - Owner: schema.Owner, - Name: schema.Name, - Tables: convertTables(schema.Tables), - Actions: actions, - }, nil -} - -func convertTables(tables []*engineTypes.Table) []*txpb.Table { - convTables := make([]*txpb.Table, len(tables)) - for i, table := range tables { - convTable := &txpb.Table{ - Name: table.Name, - Columns: convertColumns(table.Columns), - Indexes: convertIndexes(table.Indexes), - } - convTables[i] = convTable - } - - return convTables -} - -func convertColumns(columns []*engineTypes.Column) []*txpb.Column { - convColumns := make([]*txpb.Column, len(columns)) - for i, column := range columns { - convColumn := &txpb.Column{ - Name: column.Name, - Type: column.Type.String(), - Attributes: convertAttributes(column.Attributes), - } - convColumns[i] = convColumn - } - - return convColumns -} - -func convertAttributes(attributes []*engineTypes.Attribute) []*txpb.Attribute { - convAttributes := make([]*txpb.Attribute, len(attributes)) - for i, attribute := range attributes { - convAttribute := &txpb.Attribute{ - Type: attribute.Type.String(), - Value: fmt.Sprint(attribute.Value), - } - convAttributes[i] = convAttribute - } - - return convAttributes -} - -func convertIndexes(indexes []*engineTypes.Index) []*txpb.Index { - convIndexes := make([]*txpb.Index, len(indexes)) - for i, index := range indexes { - convIndexes[i] = &txpb.Index{ - Name: index.Name, - Columns: index.Columns, - Type: index.Type.String(), - } - } - - return convIndexes -} - -func convertActions(actions []*engineTypes.Procedure) ([]*txpb.Action, error) { - - convActions := make([]*txpb.Action, len(actions)) - for i, action := range actions { - mutability, auxiliaries, err := convertModifiers(action.Modifiers) - if err != nil { - return nil, err - } - - convActions[i] = &txpb.Action{ - Name: action.Name, - Public: action.Public, - Mutability: mutability, - Auxiliaries: auxiliaries, - Inputs: action.Args, - Statements: action.Statements, - } - } - - return convActions, nil -} - -func convertModifiers(mods []engineTypes.Modifier) (mutability string, auxiliaries []string, err error) { - auxiliaries = make([]string, 0) - mutability = "UPDATE" - for _, mod := range mods { - switch mod { - case engineTypes.ModifierAuthenticated: - auxiliaries = append(auxiliaries, "AUTHENTICATED") - case engineTypes.ModifierView: - mutability = "VIEW" - // TODO: add modifier owner once merged - default: - return "", nil, fmt.Errorf("unknown modifier type: %v", mod) - } - } - - return mutability, auxiliaries, nil -} diff --git a/internal/controller/grpc/txsvc/v1/service.go b/internal/controller/grpc/txsvc/v1/service.go index 1e1c1fe1f..e7d8ce2e8 100644 --- a/internal/controller/grpc/txsvc/v1/service.go +++ b/internal/controller/grpc/txsvc/v1/service.go @@ -8,7 +8,7 @@ import ( "github.com/kwilteam/kwil-db/pkg/balances" engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" "github.com/kwilteam/kwil-db/pkg/log" - "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" ) type Service struct { @@ -38,12 +38,12 @@ func NewService(engine EngineReader, accountStore AccountReader, chainClient Blo } type EngineReader interface { - Call(ctx context.Context, call *tx.CallActionPayload, msg *tx.SignedMessage[tx.JsonPayload]) ([]map[string]any, error) + Call(ctx context.Context, dbid string, action string, args []any, msg *transactions.SignedMessage) ([]map[string]any, error) GetSchema(ctx context.Context, dbid string) (*engineTypes.Schema, error) ListOwnedDatabases(ctx context.Context, owner string) ([]string, error) PriceDeploy(ctx context.Context, schema *engineTypes.Schema) (price *big.Int, err error) PriceDrop(ctx context.Context, dbid string) (price *big.Int, err error) - PriceExecute(ctx context.Context, dbid string, action string, params []map[string]any) (price *big.Int, err error) + PriceExecute(ctx context.Context, dbid string, action string, args [][]any) (price *big.Int, err error) Query(ctx context.Context, dbid string, query string) ([]map[string]any, error) } @@ -52,5 +52,5 @@ type AccountReader interface { } type BlockchainBroadcaster interface { - BroadcastTxAsync(ctx context.Context, tx *tx.Transaction) (hash []byte, err error) + BroadcastTxAsync(ctx context.Context, tx *transactions.Transaction) (err error) } diff --git a/pkg/_crypto/crypto.go b/pkg/_crypto/crypto.go new file mode 100644 index 000000000..4501d6d6c --- /dev/null +++ b/pkg/_crypto/crypto.go @@ -0,0 +1,26 @@ +package crypto + +import ( + "crypto/ecdsa" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + ec "github.com/ethereum/go-ethereum/crypto" +) + +func ECDSAFromHex(hex string) (*ecdsa.PrivateKey, error) { + return ec.HexToECDSA(hex) +} + +func AddressFromPrivateKey(key *ecdsa.PrivateKey) string { + caddr := ec.PubkeyToAddress(key.PublicKey) + return caddr.Hex() +} + +func IsValidAddress(addr string) bool { + return common.IsHexAddress(addr) +} + +func HexFromECDSAPrivateKey(key *ecdsa.PrivateKey) string { + return hexutil.Encode(ec.FromECDSA(key))[2:] +} diff --git a/pkg/serialize/consensus.go b/pkg/_serialize/consensus.go similarity index 100% rename from pkg/serialize/consensus.go rename to pkg/_serialize/consensus.go diff --git a/pkg/serialize/schema.go b/pkg/_serialize/schema.go similarity index 100% rename from pkg/serialize/schema.go rename to pkg/_serialize/schema.go diff --git a/pkg/serialize/serial.go b/pkg/_serialize/serial.go similarity index 100% rename from pkg/serialize/serial.go rename to pkg/_serialize/serial.go diff --git a/pkg/tx/message.go b/pkg/_tx/message.go similarity index 95% rename from pkg/tx/message.go rename to pkg/_tx/message.go index b9fb2c32e..139a5ad39 100644 --- a/pkg/tx/message.go +++ b/pkg/_tx/message.go @@ -1,7 +1,7 @@ package tx import ( - "crypto/ecdsa" + "crypto" "encoding/json" kwilCrypto "github.com/kwilteam/kwil-db/pkg/crypto" @@ -44,7 +44,7 @@ func (s *SignedMessage[T]) Verify() error { } // CreateSignedMessage creates and signs a SignedMessage -func CreateSignedMessage[T Serializable](message T, privateKey *ecdsa.PrivateKey) (*SignedMessage[T], error) { +func CreateSignedMessage[T Serializable](message T, privateKey crypto.PrivateKey) (*SignedMessage[T], error) { msg := &SignedMessage[T]{ Payload: message, } diff --git a/pkg/tx/payload.go b/pkg/_tx/payload.go similarity index 100% rename from pkg/tx/payload.go rename to pkg/_tx/payload.go diff --git a/pkg/tx/response.go b/pkg/_tx/response.go similarity index 100% rename from pkg/tx/response.go rename to pkg/_tx/response.go diff --git a/pkg/tx/sign_test.go b/pkg/_tx/sign_test.go similarity index 100% rename from pkg/tx/sign_test.go rename to pkg/_tx/sign_test.go diff --git a/pkg/tx/tx.go b/pkg/_tx/tx.go similarity index 100% rename from pkg/tx/tx.go rename to pkg/_tx/tx.go diff --git a/pkg/abci/abci.go b/pkg/abci/abci.go index 7e890403a..f352c0f16 100644 --- a/pkg/abci/abci.go +++ b/pkg/abci/abci.go @@ -2,11 +2,14 @@ package abci import ( "context" + "fmt" "sync" abciTypes "github.com/cometbft/cometbft/abci/types" + engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" "github.com/kwilteam/kwil-db/pkg/log" - transactions "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/modules/datasets" + "github.com/kwilteam/kwil-db/pkg/transactions" "go.uber.org/zap" ) @@ -38,9 +41,6 @@ type AbciApp struct { // committer is the atomic committer that handles atomic commits across multiple stores committer AtomicCommitter - // payloadEncoder is the encoder that encodes and decodes payloads - payloadEncoder PayloadEncoder - log log.Logger // commitWaiter is a waitgroup that waits for the commit to finish @@ -99,7 +99,7 @@ func (a *AbciApp) DeliverTx(req abciTypes.RequestDeliverTx) abciTypes.ResponseDe ctx := context.Background() tx := &transactions.Transaction{} - err := a.payloadEncoder.Decode(req.Tx, tx) + err := tx.UnmarshalBinary(req.Tx) if err != nil { return abciTypes.ResponseDeliverTx{ Code: 1, @@ -107,42 +107,55 @@ func (a *AbciApp) DeliverTx(req abciTypes.RequestDeliverTx) abciTypes.ResponseDe } } - var res *transactions.ExecutionResponse + var res *transactions.TransactionStatus - switch tx.PayloadType { - case transactions.DEPLOY_DATABASE: - payload := &PayloadDeployDatabase{} - err = a.payloadEncoder.Decode(tx.Payload, payload) + switch tx.Body.PayloadType { + case transactions.PayloadTypeDeploySchema: + var schemaPayload transactions.Schema + err = schemaPayload.UnmarshalBinary(tx.Body.Payload) if err != nil { break } - res, err = a.database.Deploy(ctx, payload.Schema, tx) - case transactions.DROP_DATABASE: - datasetIdentifier := &PayloadDropDatabase{} - err = a.payloadEncoder.Decode(tx.Payload, datasetIdentifier) + var schema *engineTypes.Schema + schema, err = datasets.ConvertSchemaToEngine(&schemaPayload) if err != nil { break } - res, err = a.database.Drop(ctx, datasetIdentifier.Name, tx) - case transactions.EXECUTE_ACTION: - execution := &PayloadExecuteAction{} - err = a.payloadEncoder.Decode(tx.Payload, execution) + res, err = a.database.Deploy(ctx, schema, tx) + case transactions.PayloadTypeDropSchema: + drop := &transactions.DropSchema{} + err = drop.UnmarshalBinary(tx.Body.Payload) if err != nil { break } - res, err = a.database.Execute(ctx, execution.DBID, execution.Action, execution.Params, tx) - case transactions.VALIDATOR_JOIN: - validatorJoin := &PayloadValidatorJoin{} - err = a.payloadEncoder.Decode(tx.Payload, validatorJoin) + res, err = a.database.Drop(ctx, drop.DBID, tx) + case transactions.PayloadTypeExecuteAction: + execution := &transactions.ActionExecution{} + err = execution.UnmarshalBinary(tx.Body.Payload) if err != nil { break } - res, err = a.validators.ValidatorJoin(ctx, validatorJoin.Address, tx) - case transactions.VALIDATOR_APPROVE: + res, err = a.database.Execute(ctx, execution.DBID, execution.Action, convertArgs(execution.Arguments), tx) + case transactions.PayloadTypeValidatorJoin: + // TODO: update this with validator payload + panic("TODO") + /* + validatorJoin := &PayloadValidatorJoin{} + err = a.payloadEncoder.Decode(tx.Payload, validatorJoin) + if err != nil { + break + } + + res, err = a.validators.ValidatorJoin(ctx, validatorJoin.Address, tx) + */ + case transactions.PayloadTypeValidatorApprove: + // TODO: update this with validator payload + panic("TODO") + /* validatorApprove := &PayloadValidatorApprove{} err = a.payloadEncoder.Decode(tx.Payload, validatorApprove) if err != nil { @@ -150,6 +163,9 @@ func (a *AbciApp) DeliverTx(req abciTypes.RequestDeliverTx) abciTypes.ResponseDe } res, err = a.validators.ValidatorApprove(ctx, validatorApprove.ValidatorToApprove, validatorApprove.ApprovedBy, tx) + */ + default: + err = fmt.Errorf("unknown payload type: %s", tx.Body.PayloadType.String()) } if err != nil { return abciTypes.ResponseDeliverTx{ @@ -198,3 +214,16 @@ func (a *AbciApp) ProcessProposal(p0 abciTypes.RequestProcessProposal) abciTypes func (a *AbciApp) Query(p0 abciTypes.RequestQuery) abciTypes.ResponseQuery { panic("TODO") } + +// convertArgs converts the string args to type any. +func convertArgs(args [][]string) [][]any { + converted := make([][]any, len(args)) + for i, arg := range args { + converted[i] = make([]any, len(arg)) + for j, a := range arg { + converted[i][j] = a + } + } + + return converted +} diff --git a/pkg/abci/interfaces.go b/pkg/abci/interfaces.go index 86d9537f0..38ef8aa70 100644 --- a/pkg/abci/interfaces.go +++ b/pkg/abci/interfaces.go @@ -4,19 +4,19 @@ import ( "context" "github.com/kwilteam/kwil-db/pkg/engine/types" - "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" ) type DatasetsModule interface { - Deploy(ctx context.Context, schema *types.Schema, tx *tx.Transaction) (*tx.ExecutionResponse, error) - Drop(ctx context.Context, dbid string, tx *tx.Transaction) (*tx.ExecutionResponse, error) - Execute(ctx context.Context, dbid string, action string, params []map[string]any, tx *tx.Transaction) (*tx.ExecutionResponse, error) + Deploy(ctx context.Context, schema *types.Schema, tx *transactions.Transaction) (*transactions.TransactionStatus, error) + Drop(ctx context.Context, dbid string, tx *transactions.Transaction) (*transactions.TransactionStatus, error) + Execute(ctx context.Context, dbid string, action string, args [][]any, tx *transactions.Transaction) (*transactions.TransactionStatus, error) } // Should be implemented in pkg/modules/validators type ValidatorModule interface { - ValidatorJoin(ctx context.Context, address string, tx *tx.Transaction) (*tx.ExecutionResponse, error) - ValidatorApprove(ctx context.Context, address string, approvedBy string, tx *tx.Transaction) (*tx.ExecutionResponse, error) + ValidatorJoin(ctx context.Context, address string, tx *transactions.Transaction) (*transactions.TransactionStatus, error) + ValidatorApprove(ctx context.Context, address string, approvedBy string, tx *transactions.Transaction) (*transactions.TransactionStatus, error) } // AtomicCommitter is an interface for a struct that implements atomic commits across multiple stores diff --git a/pkg/abci/payloads.go b/pkg/abci/payloads.go deleted file mode 100644 index 6b0d7823f..000000000 --- a/pkg/abci/payloads.go +++ /dev/null @@ -1,38 +0,0 @@ -package abci - -import engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" - -type PayloadDeployDatabase struct { - Schema *engineTypes.Schema -} - -type PayloadDropDatabase struct { - Name string - Owner string -} - -type PayloadExecuteAction struct { - Action string - DBID string - Params []map[string]any -} - -type PayloadCallAction struct { - Action string - DBID string - Params map[string]any -} - -type PayloadValidatorJoin struct { - Address string -} - -type PayloadValidatorApprove struct { - ValidatorToApprove string - ApprovedBy string -} - -type PayloadEncoder interface { - Encode(v any) ([]byte, error) - Decode(data []byte, v any) error -} diff --git a/pkg/abci/utils.go b/pkg/abci/utils.go deleted file mode 100644 index 57a89e324..000000000 --- a/pkg/abci/utils.go +++ /dev/null @@ -1 +0,0 @@ -package abci diff --git a/pkg/client/client.go b/pkg/client/client.go index 3ed2f2584..9819c0ec2 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -2,18 +2,20 @@ package client import ( "context" - "crypto/ecdsa" "encoding/json" "fmt" "strings" + "github.com/cstockton/go-conv" + "github.com/kwilteam/kwil-db/pkg/crypto" + cmtCrypto "github.com/cometbft/cometbft/crypto" cmtjson "github.com/cometbft/cometbft/libs/json" rpchttp "github.com/cometbft/cometbft/rpc/client/http" "github.com/kwilteam/kwil-db/pkg/balances" + engineUtils "github.com/kwilteam/kwil-db/pkg/engine/utils" grpcClient "github.com/kwilteam/kwil-db/pkg/grpc/client/v1" - "github.com/kwilteam/kwil-db/pkg/serialize" - kTx "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" ) @@ -21,8 +23,8 @@ import ( type Client struct { client *grpcClient.Client CometBftClient *rpchttp.HTTP - datasets map[string]*serialize.Schema - PrivateKey *ecdsa.PrivateKey + datasets map[string]*transactions.Schema + PrivateKey crypto.PrivateKey cometBftRpcUrl string } @@ -30,7 +32,7 @@ type Client struct { // New creates a new client func New(ctx context.Context, target string, opts ...ClientOpt) (c *Client, err error) { c = &Client{ - datasets: make(map[string]*serialize.Schema), + datasets: make(map[string]*transactions.Schema), cometBftRpcUrl: "tcp://localhost:26657", } @@ -54,7 +56,7 @@ func New(ctx context.Context, target string, opts ...ClientOpt) (c *Client, err } // GetSchema returns the entity of a database -func (c *Client) GetSchema(ctx context.Context, dbid string) (*serialize.Schema, error) { +func (c *Client) GetSchema(ctx context.Context, dbid string) (*transactions.Schema, error) { ds, ok := c.datasets[dbid] if ok { return ds, nil @@ -70,7 +72,7 @@ func (c *Client) GetSchema(ctx context.Context, dbid string) (*serialize.Schema, } // DeployDatabase deploys a schema -func (c *Client) DeployDatabase(ctx context.Context, ds *serialize.Schema) (*kTx.Receipt, error) { +func (c *Client) DeployDatabase(ctx context.Context, ds *transactions.Schema) (*transactions.TransactionStatus, error) { address, err := c.getAddress() if err != nil { return nil, fmt.Errorf("failed to get address from private key: %w", err) @@ -80,7 +82,7 @@ func (c *Client) DeployDatabase(ctx context.Context, ds *serialize.Schema) (*kTx return nil, fmt.Errorf("dataset owner is not the same as the address") } - tx, err := c.deploySchemaTx(ctx, ds) + tx, err := c.newTx(ctx, ds) if err != nil { return nil, err } @@ -88,24 +90,18 @@ func (c *Client) DeployDatabase(ctx context.Context, ds *serialize.Schema) (*kTx return c.client.Broadcast(ctx, tx) } -// deploySchemaTx creates a new transaction to deploy a schema -func (c *Client) deploySchemaTx(ctx context.Context, ds *serialize.Schema) (*kTx.Transaction, error) { - return c.newTx(ctx, kTx.DEPLOY_DATABASE, ds) -} - // DropDatabase drops a database -func (c *Client) DropDatabase(ctx context.Context, name string) (*kTx.Receipt, error) { +func (c *Client) DropDatabase(ctx context.Context, name string) (*transactions.TransactionStatus, error) { address, err := c.getAddress() if err != nil { return nil, fmt.Errorf("failed to get address from private key: %w", err) } - identifier := &datasetIdentifier{ - Owner: address, - Name: name, + identifier := &transactions.DropSchema{ + DBID: engineUtils.GenerateDBID(address, name), } - tx, err := c.dropDatabaseTx(ctx, identifier) + tx, err := c.newTx(ctx, identifier) if err != nil { return nil, err } @@ -115,26 +111,27 @@ func (c *Client) DropDatabase(ctx context.Context, name string) (*kTx.Receipt, e return nil, err } - delete(c.datasets, identifier.Dbid()) + delete(c.datasets, identifier.DBID) return res, nil } -// dropDatabaseTx creates a new transaction to drop a database -func (c *Client) dropDatabaseTx(ctx context.Context, dbIdent *datasetIdentifier) (*kTx.Transaction, error) { - return c.newTx(ctx, kTx.DROP_DATABASE, dbIdent) -} - // ExecuteAction executes an action. // It returns the receipt, as well as outputs which is the decoded body of the receipt. -func (c *Client) ExecuteAction(ctx context.Context, dbid string, action string, inputs []map[string]any) (*kTx.Receipt, error) { - executionBody := &actionExecution{ - Action: action, - DBID: dbid, - Params: inputs, +// It can take any number of inputs, and if multiple tuples of inputs are passed, it will execute them transactionally. +func (c *Client) ExecuteAction(ctx context.Context, dbid string, action string, tuples ...[]any) (*transactions.TransactionStatus, error) { + stringTuples, err := convertTuples(tuples) + if err != nil { + return nil, err } - tx, err := c.executeActionTx(ctx, executionBody) + executionBody := &transactions.ActionExecution{ + Action: action, + DBID: dbid, + Arguments: stringTuples, + } + + tx, err := c.newTx(ctx, executionBody) if err != nil { return nil, err } @@ -144,50 +141,54 @@ func (c *Client) ExecuteAction(ctx context.Context, dbid string, action string, return nil, err } - /* outputs, err := DecodeOutputs(res.Body) - if err != nil { - return nil, err - } - */ return res, nil } // CallAction call an action, if auxiliary `mustsign` is set, need to sign the action payload. It returns the records. -func (c *Client) CallAction(ctx context.Context, dbid string, action string, inputs map[string]any, opts ...CallOpt) ([]map[string]any, error) { +func (c *Client) CallAction(ctx context.Context, dbid string, action string, inputs []any, opts ...CallOpt) ([]map[string]any, error) { callOpts := &callOptions{} for _, opt := range opts { opt(callOpts) } - payload := &kTx.CallActionPayload{ - DBID: dbid, - Action: action, - Params: inputs, + stringInputs, err := convertTuple(inputs) + if err != nil { + return nil, err + } + + payload := &transactions.ActionCall{ + DBID: dbid, + Action: action, + Arguments: stringInputs, } - var signedMsg *kTx.SignedMessage[*kTx.CallActionPayload] + var signedMsg *transactions.SignedMessage shouldSign, err := shouldAuthenticate(c.PrivateKey, callOpts.forceAuthenticated) if err != nil { return nil, err } - if shouldSign { - signedMsg, err = kTx.CreateSignedMessage(payload, c.PrivateKey) - } else { - signedMsg = kTx.CreateEmptySignedMessage(payload) - } + msg, err := transactions.CreateSignedMessage(payload) if err != nil { return nil, fmt.Errorf("failed to create signed message: %w", err) } + if shouldSign { + err = msg.Sign(c.PrivateKey) + + if err != nil { + return nil, fmt.Errorf("failed to create signed message: %w", err) + } + } + return c.client.Call(ctx, signedMsg) } // shouldAuthenticate decides whether the client should authenticate or not // if enforced is not nil, it will be used instead of the default value // otherwise, if the private key is not nil, it will authenticate -func shouldAuthenticate(privateKey *ecdsa.PrivateKey, enforced *bool) (bool, error) { +func shouldAuthenticate(privateKey crypto.PrivateKey, enforced *bool) (bool, error) { if enforced != nil { if !*enforced { return false, nil @@ -217,16 +218,6 @@ func DecodeOutputs(bts []byte) ([]map[string]any, error) { return outputs, nil } -// executeActionTx creates a new transaction to execute an action -func (c *Client) executeActionTx(ctx context.Context, executionBody *actionExecution) (*kTx.Transaction, error) { - return c.newTx(ctx, kTx.EXECUTE_ACTION, executionBody) -} - -// GetConfig returns the provider config -func (c *Client) GetConfig(ctx context.Context) (*grpcClient.SvcConfig, error) { - return c.client.GetConfig(ctx) -} - // Query executes a query func (c *Client) Query(ctx context.Context, dbid string, query string) (*Records, error) { res, err := c.client.Query(ctx, dbid, query) @@ -251,7 +242,8 @@ func (c *Client) GetAccount(ctx context.Context, address string) (*balances.Acco } func (c *Client) ApproveValidator(ctx context.Context, approver string, joiner string) ([]byte, error) { - tx, err := c.NewNodeTx(ctx, kTx.VALIDATOR_APPROVE, joiner, approver) + // TODO: change this PayloadTypeExecuteAction to just be a validator update payload (or some equivalent) + tx, err := c.NewNodeTx(ctx, transactions.PayloadTypeExecuteAction, joiner, approver) if err != nil { return nil, err } @@ -268,15 +260,17 @@ func (c *Client) ApproveValidator(ctx context.Context, approver string, joiner s return res.Hash, nil } +// TODO: update once validator module is included func (c *Client) ValidatorJoin(ctx context.Context, joiner string, power int64) ([]byte, error) { - return c.ValidatorUpdate(ctx, joiner, power, kTx.VALIDATOR_JOIN) + return c.ValidatorUpdate(ctx, joiner, power, transactions.PayloadTypeValidatorJoin) } +// TODO: update once validator module is included func (c *Client) ValidatorLeave(ctx context.Context, joiner string, power int64) ([]byte, error) { - return c.ValidatorUpdate(ctx, joiner, 0, kTx.VALIDATOR_LEAVE) + return c.ValidatorUpdate(ctx, joiner, 0, transactions.PayloadTypeValidatorApprove) } -func (c *Client) ValidatorUpdate(ctx context.Context, joinerPrivKey string, power int64, payloadtype kTx.PayloadType) ([]byte, error) { +func (c *Client) ValidatorUpdate(ctx context.Context, joinerPrivKey string, power int64, payloadtype transactions.PayloadType) ([]byte, error) { var nodeKey cmtCrypto.PrivKey key := fmt.Sprintf(`{"type":"tendermint/PrivKeyEd25519","value":"%s"}`, joinerPrivKey) err := cmtjson.Unmarshal([]byte(key), &nodeKey) @@ -284,16 +278,18 @@ func (c *Client) ValidatorUpdate(ctx context.Context, joinerPrivKey string, powe return nil, err } - fmt.Println("Node PublicKey: ", nodeKey.PubKey()) - bts, _ := json.Marshal(nodeKey.PubKey()) - fmt.Println("Node PublicKey: ", string(bts)) + bts, err := json.Marshal(nodeKey.PubKey()) + if err != nil { + return nil, err + } vldtr := &validator{ PubKey: string(bts), Power: power, } - tx, err := c.NewNodeTx(ctx, payloadtype, vldtr, joinerPrivKey) + // TODO: change this PayloadTypeExecuteAction to just be a validator update payload (or some equivalent) + tx, err := c.NewNodeTx(ctx, transactions.PayloadTypeExecuteAction, vldtr, joinerPrivKey) if err != nil { return nil, err } @@ -309,3 +305,34 @@ func (c *Client) ValidatorUpdate(ctx context.Context, joinerPrivKey string, powe } return res.Hash, nil } + +// convertTuples converts user passed tuples to strings. +// this is necessary for RLP encoding +func convertTuples(tuples [][]any) ([][]string, error) { + ins := [][]string{} + for _, tuple := range tuples { + stringTuple, err := convertTuple(tuple) + if err != nil { + return nil, err + } + ins = append(ins, stringTuple) + } + + return ins, nil +} + +// convertTuple converts user passed tuple to strings. +func convertTuple(tuple []any) ([]string, error) { + stringTuple := []string{} + for _, val := range tuple { + + stringVal, err := conv.String(val) + if err != nil { + return nil, err + } + + stringTuple = append(stringTuple, stringVal) + } + + return stringTuple, nil +} diff --git a/pkg/client/opts.go b/pkg/client/opts.go index f7b009a5b..e228eb476 100644 --- a/pkg/client/opts.go +++ b/pkg/client/opts.go @@ -1,17 +1,22 @@ package client -import ( - "crypto/ecdsa" -) +import "github.com/kwilteam/kwil-db/pkg/crypto" type ClientOpt func(*Client) -func WithPrivateKey(key *ecdsa.PrivateKey) ClientOpt { +func WithPrivateKey(key crypto.PrivateKey) ClientOpt { return func(c *Client) { c.PrivateKey = key } } +// TODO: replace this, since we should not be using cometBFT RPCs +func WithCometBftUrl(url string) ClientOpt { + return func(c *Client) { + c.cometBftRpcUrl = url + } +} + type callOptions struct { // forceAuthenticated is used to force the client to authenticate // if nil, the client will use the default value @@ -31,9 +36,3 @@ func Authenticated(shouldSign bool) CallOpt { o.forceAuthenticated = &copied } } - -func WithCometBftUrl(url string) ClientOpt { - return func(c *Client) { - c.cometBftRpcUrl = url - } -} diff --git a/pkg/client/tx.go b/pkg/client/tx.go index 6e709f9eb..78051afb2 100644 --- a/pkg/client/tx.go +++ b/pkg/client/tx.go @@ -2,29 +2,19 @@ package client import ( "context" - "encoding/json" "fmt" "math/big" - cmtCrypto "github.com/cometbft/cometbft/crypto" - cmtjson "github.com/cometbft/cometbft/libs/json" "github.com/kwilteam/kwil-db/pkg/balances" - "github.com/kwilteam/kwil-db/pkg/crypto" - kTx "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" ) // Transaction signed by the client -func (c *Client) newTx(ctx context.Context, payloadType kTx.PayloadType, data any) (*kTx.Transaction, error) { +func (c *Client) newTx(ctx context.Context, data transactions.Payload) (*transactions.Transaction, error) { if c.PrivateKey == nil { return nil, fmt.Errorf("private key is nil") } - // serialize data - bts, err := json.Marshal(data) - if err != nil { - return nil, fmt.Errorf("failed to serialize data: %w", err) - } - address, err := c.getAddress() if err != nil { return nil, fmt.Errorf("failed to get address from private key: %w", err) @@ -41,7 +31,10 @@ func (c *Client) newTx(ctx context.Context, payloadType kTx.PayloadType, data an } // build transaction - tx := kTx.NewTx(payloadType, bts, acc.Nonce+1) + tx, err := transactions.CreateTransaction(data, uint64(acc.Nonce+1)) + if err != nil { + return nil, fmt.Errorf("failed to create transaction: %w", err) + } // estimate price price, err := c.client.EstimateCost(ctx, tx) @@ -50,7 +43,7 @@ func (c *Client) newTx(ctx context.Context, payloadType kTx.PayloadType, data an } // set fee - tx.Fee = price.String() + tx.Body.Fee = price // sign transaction err = tx.Sign(c.PrivateKey) @@ -62,66 +55,70 @@ func (c *Client) newTx(ctx context.Context, payloadType kTx.PayloadType, data an } // Tx Signed by the Validator Node -func (c *Client) NewNodeTx(ctx context.Context, payloadType kTx.PayloadType, data any, privKey string) (*kTx.Transaction, error) { - var nodeKey cmtCrypto.PrivKey - key := fmt.Sprintf(`{"type":"tendermint/PrivKeyEd25519","value":"%s"}`, privKey) - err := cmtjson.Unmarshal([]byte(key), &nodeKey) - if err != nil { - return nil, err - } +// TODO: this needs to be updated once validator store (containing the validator payload) is merged in +func (c *Client) NewNodeTx(ctx context.Context, payloadType transactions.PayloadType, data any, privKey string) (*transactions.TransactionStatus, error) { + panic("implement me") + /* + var nodeKey cmtCrypto.PrivKey + key := fmt.Sprintf(`{"type":"tendermint/PrivKeyEd25519","value":"%s"}`, privKey) + err := cmtjson.Unmarshal([]byte(key), &nodeKey) + if err != nil { + return nil, err + } - // serialize data - bts, err := json.Marshal(data) - if err != nil { - return nil, fmt.Errorf("failed to serialize data: %w", err) - } + // serialize data + bts, err := json.Marshal(data) + if err != nil { + return nil, fmt.Errorf("failed to serialize data: %w", err) + } - address := nodeKey.PubKey().Address().String() - acc, err := c.client.GetAccount(ctx, address) - if err != nil { - acc = &balances.Account{ - Address: address, - Nonce: 0, - Balance: big.NewInt(0), + address := nodeKey.PubKey().Address().String() + acc, err := c.client.GetAccount(ctx, address) + if err != nil { + acc = &balances.Account{ + Address: address, + Nonce: 0, + Balance: big.NewInt(0), + } } - } - // build transaction - tx := kTx.NewTx(payloadType, bts, acc.Nonce+1) - // sign transaction - tx.Fee = "0" + // build transaction + tx := kTx.NewTx(payloadType, bts, acc.Nonce+1) + // sign transaction + tx.Fee = "0" - hash := tx.GenerateHash() - sign, err := nodeKey.Sign(hash) - if err != nil { - return nil, fmt.Errorf("failed to sign tx: %w", err) - } + hash := tx.GenerateHash() + sign, err := nodeKey.Sign(hash) + if err != nil { + return nil, fmt.Errorf("failed to sign tx: %w", err) + } - var keytype crypto.SignatureType - if nodeKey.Type() == "ed25519" { - keytype = crypto.PK_ED25519 - } + var keytype crypto.SignatureType + if nodeKey.Type() == "ed25519" { + keytype = crypto.SIGNATURE_TYPE_ED25519 + } - tx.Signature = &crypto.Signature{ - Signature: sign, - Type: keytype, - } + tx.Signature = &crypto.Signature{ + Signature: sign, + Type: keytype, + } - tx.Hash = hash + tx.Hash = hash - keybts, err := json.Marshal(nodeKey.PubKey()) - if err != nil { - return nil, fmt.Errorf("failed to marshal pubkey: %w", err) - } + keybts, err := json.Marshal(nodeKey.PubKey()) + if err != nil { + return nil, fmt.Errorf("failed to marshal pubkey: %w", err) + } - tx.Sender = string(keybts) - fmt.Println("tx hash", tx.Hash) - fmt.Println("tx sender", tx.Sender) - fmt.Println("tx signature", tx.Signature) - fmt.Println("tx payload", tx.Payload) - fmt.Println("tx payload type", tx.PayloadType) + tx.Sender = string(keybts) + fmt.Println("tx hash", tx.Hash) + fmt.Println("tx sender", tx.Sender) + fmt.Println("tx signature", tx.Signature) + fmt.Println("tx payload", tx.Payload) + fmt.Println("tx payload type", tx.PayloadType) - return tx, nil + return tx, nil + */ } func (c *Client) getAddress() (string, error) { @@ -129,5 +126,5 @@ func (c *Client) getAddress() (string, error) { return "", fmt.Errorf("private key is nil") } - return crypto.AddressFromPrivateKey(c.PrivateKey), nil + return c.PrivateKey.PubKey().Address().String(), nil } diff --git a/pkg/client/types.go b/pkg/client/types.go index 621790509..8cd9ef696 100644 --- a/pkg/client/types.go +++ b/pkg/client/types.go @@ -1,24 +1,5 @@ package client -import "github.com/kwilteam/kwil-db/pkg/engine/utils" - -// some of these mimic internal/entity - -type datasetIdentifier struct { - Owner string `json:"owner"` - Name string `json:"name"` -} - -func (d *datasetIdentifier) Dbid() string { - return utils.GenerateDBID(d.Name, d.Owner) -} - -type actionExecution struct { - Action string `json:"action"` - DBID string `json:"dbid"` - Params []map[string]any `json:"params"` -} - type validator struct { PubKey string `json:"pubKey"` Power int64 `json:"power"` diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 16b952938..287d76081 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -2,11 +2,12 @@ package config_test import ( "crypto/ecdsa" - "github.com/kwilteam/kwil-db/pkg/config" - "github.com/kwilteam/kwil-db/pkg/crypto" "os" "testing" + "github.com/ethereum/go-ethereum/crypto" + "github.com/kwilteam/kwil-db/pkg/config" + "github.com/cstockton/go-conv" ) @@ -133,7 +134,7 @@ var ( return nil, err } - return crypto.ECDSAFromHex(strVal) + return crypto.HexToECDSA(strVal) // TODO: we should rethink this since we support ed25519 now }, Required: true, } diff --git a/pkg/crypto/crypto.go b/pkg/crypto/crypto.go index 808577f1b..d4c59e850 100644 --- a/pkg/crypto/crypto.go +++ b/pkg/crypto/crypto.go @@ -1,9 +1,12 @@ package crypto import ( + "crypto/ecdsa" c256 "crypto/sha256" c512 "crypto/sha512" "encoding/hex" + + ec "github.com/ethereum/go-ethereum/crypto" ) // Sha384 returns the sha384 hash of the data. @@ -36,3 +39,19 @@ func Sha256(data []byte) []byte { func Sha256Hex(data []byte) string { return hex.EncodeToString(Sha256(data)) } + +func PublicKeyFromBytes(key []byte) (PublicKey, error) { + panic("not implemented") +} + +func PrivateKeyFromHex(h string) (PrivateKey, error) { + panic("not implemented") +} + +func PrivateKeyFromBytes(key []byte) (PrivateKey, error) { + panic("not implemented") +} + +func ECDSAFromHex(hex string) (*ecdsa.PrivateKey, error) { + return ec.HexToECDSA(hex) +} diff --git a/pkg/crypto/ed25519.go b/pkg/crypto/ed25519.go index d119bc7fe..64a974eea 100644 --- a/pkg/crypto/ed25519.go +++ b/pkg/crypto/ed25519.go @@ -1,9 +1,5 @@ package crypto -import ( - oasisEd25519 "github.com/oasisprotocol/curve25519-voi/primitives/ed25519" -) - const Ed25519 KeyType = "Ed25519" type Ed25519PrivateKey struct { @@ -18,8 +14,8 @@ func (s *Ed25519PrivateKey) PubKey() PublicKey { panic("implement me") } -func (s *Ed25519PrivateKey) Sign(msg []byte, signatureType SignatureType) ([]byte, error) { - return oasisEd25519.Sign(oasisEd25519.PrivateKey(s.privateKey), msg), nil +func (s *Ed25519PrivateKey) Sign(msg []byte) (*Signature, error) { + panic("implement me") } func (s *Ed25519PrivateKey) Type() KeyType { diff --git a/pkg/crypto/keys.go b/pkg/crypto/keys.go index 1c9656a09..af8d382d1 100644 --- a/pkg/crypto/keys.go +++ b/pkg/crypto/keys.go @@ -5,7 +5,7 @@ type KeyType string type PrivateKey interface { Bytes() []byte Type() KeyType - Sign(msg []byte, signatureType SignatureType) ([]byte, error) + Sign(msg []byte) (*Signature, error) PubKey() PublicKey } diff --git a/pkg/crypto/secp256k1.go b/pkg/crypto/secp256k1.go index b8b76d1af..8d3d1ec6a 100644 --- a/pkg/crypto/secp256k1.go +++ b/pkg/crypto/secp256k1.go @@ -23,7 +23,7 @@ func (s *Secp256k1PrivateKey) PubKey() PublicKey { } } -func (s *Secp256k1PrivateKey) Sign(msg []byte, signatureType SignatureType) ([]byte, error) { +func (s *Secp256k1PrivateKey) Sign(msg []byte) (*Signature, error) { // TODO: implement panic("TODO") } diff --git a/pkg/crypto/signature.go b/pkg/crypto/signature.go index 4cd10c622..3aebcf010 100644 --- a/pkg/crypto/signature.go +++ b/pkg/crypto/signature.go @@ -8,6 +8,8 @@ type SignatureType int32 const ( SIGNATURE_TYPE_INVALID SignatureType = iota + SIGNATURE_TYPE_EMPTY + SIGNATURE_TYPE_ED25519 END_SIGNATURE_TYPE ) diff --git a/pkg/engine/dataset/dataset.go b/pkg/engine/dataset/dataset.go index 1affd5616..6b52fa8ee 100644 --- a/pkg/engine/dataset/dataset.go +++ b/pkg/engine/dataset/dataset.go @@ -4,10 +4,11 @@ import ( "context" "errors" "fmt" - "go.uber.org/zap" "io" "strings" + "go.uber.org/zap" + "github.com/kwilteam/kwil-db/pkg/log" "github.com/kwilteam/kwil-db/pkg/sql" @@ -69,7 +70,7 @@ func OpenDataset(ctx context.Context, ds Datastore, opts ...OpenOpt) (*Dataset, func (d *Dataset) execConstructor(ctx context.Context, opts *TxOpts) error { for _, procedure := range d.metadata.Procedures { if strings.EqualFold(procedure.Name, constructorName) { - _, err := d.executeOnce(ctx, procedure, make(map[string]any), d.getExecutionOpts(procedure, opts)...) + _, err := d.executeOnce(ctx, procedure, []any{}, d.getExecutionOpts(procedure, opts)...) if err != nil { return err } @@ -84,7 +85,7 @@ func (d *Dataset) execConstructor(ctx context.Context, opts *TxOpts) error { const constructorName = "init" // Execute executes a procedure. -func (d *Dataset) Execute(ctx context.Context, action string, args []map[string]any, opts *TxOpts) ([]map[string]any, error) { +func (d *Dataset) Execute(ctx context.Context, action string, args [][]any, opts *TxOpts) ([]map[string]any, error) { if opts == nil { opts = &TxOpts{} } @@ -110,7 +111,7 @@ func (d *Dataset) Execute(ctx context.Context, action string, args []map[string] defer savepoint.Rollback() if len(args) == 0 { // if no args, add an empty arg map so we can execute once - args = append(args, make(map[string]any)) + args = append(args, []any{}) } var result []map[string]any @@ -130,15 +131,11 @@ func (d *Dataset) Execute(ctx context.Context, action string, args []map[string] } // Call is like execute, but it is non-mutative. -func (d *Dataset) Call(ctx context.Context, action string, args map[string]any, opts *TxOpts) ([]map[string]any, error) { +func (d *Dataset) Call(ctx context.Context, action string, args []any, opts *TxOpts) ([]map[string]any, error) { if opts == nil { opts = &TxOpts{} } - if len(args) == 0 { // if no args, add an empty arg map so we can execute once - args = make(map[string]any) - } - proc, err := d.getProcedure(action) if err != nil { return nil, err @@ -187,15 +184,9 @@ func (d *Dataset) getExecutionOpts(proc *types.Procedure, opts *TxOpts) []execut return execOpts } -func (d *Dataset) executeOnce(ctx context.Context, proc *types.Procedure, args map[string]any, opts ...execution.ExecutionOpt) ([]map[string]any, error) { - var argArr []any - for _, arg := range proc.Args { - val, ok := args[arg] - if !ok { - return nil, fmt.Errorf("missing argument %s", arg) - } - - argArr = append(argArr, val) +func (d *Dataset) executeOnce(ctx context.Context, proc *types.Procedure, args []any, opts ...execution.ExecutionOpt) ([]map[string]any, error) { + if len(args) != len(proc.Args) { + return nil, fmt.Errorf("expected %d args, got %d", len(proc.Args), len(args)) } sp, err := d.db.Savepoint() @@ -204,7 +195,7 @@ func (d *Dataset) executeOnce(ctx context.Context, proc *types.Procedure, args m } defer sp.Rollback() - results, err := d.engine.ExecuteProcedure(ctx, proc.Name, argArr, + results, err := d.engine.ExecuteProcedure(ctx, proc.Name, args, opts..., ) if err != nil { diff --git a/pkg/engine/dataset/dataset_test.go b/pkg/engine/dataset/dataset_test.go index c9f8b6ed1..90851843d 100644 --- a/pkg/engine/dataset/dataset_test.go +++ b/pkg/engine/dataset/dataset_test.go @@ -36,7 +36,7 @@ func Test_Execute(t *testing.T) { type args struct { procedure string - inputs []map[string]interface{} + inputs [][]any finisher func(*dataset.Dataset) error } @@ -55,11 +55,11 @@ func Test_Execute(t *testing.T) { fields: defaultFields, args: args{ procedure: "create_user", - inputs: []map[string]interface{}{ + inputs: [][]any{ { - "$id": "1", - "$username": "test_username", - "$age": 20, + "1", + "test_username", + 20, }, }, }, @@ -71,7 +71,7 @@ func Test_Execute(t *testing.T) { fields: defaultFields, args: args{ procedure: "get_time", - inputs: []map[string]interface{}{}, + inputs: [][]any{}, }, expectedOutputs: nil, wantErr: false, @@ -81,12 +81,12 @@ func Test_Execute(t *testing.T) { fields: defaultFields, args: args{ procedure: "create_post", - inputs: []map[string]interface{}{ + inputs: [][]any{ { - "$id": "1", - "$title": "test_title", - "$content": "test_content", - "$author_id": "20485", + "1", + "test_title", + "test_content", + "20485", }, }, }, @@ -98,14 +98,14 @@ func Test_Execute(t *testing.T) { fields: defaultFields, args: args{ procedure: "create_post_and_user", - inputs: []map[string]interface{}{ + inputs: [][]any{ { - "$id": "1", - "$title": "test_title", - "$content": "test_content", - "$author_id": "1", - "$username": "test_username", - "$age": 20, + "1", + "test_title", + "test_content", + "1", + "test_username", + 20, }, }, }, @@ -140,18 +140,18 @@ func Test_Execute(t *testing.T) { }, args: args{ procedure: "create_user_manual", - inputs: []map[string]interface{}{ + inputs: [][]any{ { - "$id": "1", - "$username": "test_username", - "$age": 20, - "$address": "0x123", + "1", + "test_username", + 20, + "0x123", }, { - "$id": "2", - "$username": "test_username2", - "$age": 20, - "$address": "0x456", + "2", + "test_username2", + 20, + "0x456", }, }, }, @@ -182,18 +182,18 @@ func Test_Execute(t *testing.T) { }, args: args{ procedure: "create_user_manual", - inputs: []map[string]interface{}{ + inputs: [][]any{ { - "$id": "1", - "$username": "test_username", - "$age": 20, - "$address": "0x123", + "1", + "test_username", + 20, + "0x123", }, { - "$id": "2abc", // this will fail - "$username": "test_username2", - "$age": 20, - "$address": "0x456", + "2abc", // this will fail + "test_username2", + 20, + "0x456", }, }, finisher: func(database *dataset.Dataset) error { @@ -231,9 +231,9 @@ func Test_Execute(t *testing.T) { }, args: args{ procedure: "use_ext", - inputs: []map[string]interface{}{ + inputs: [][]any{ { - "$name": "satoshi", + "satoshi", }, }, }, @@ -265,9 +265,9 @@ func Test_Execute(t *testing.T) { }, args: args{ procedure: "use_ext", - inputs: []map[string]interface{}{ + inputs: [][]any{ { - "$name": "satoshi", + "satoshi", }, }, }, @@ -295,7 +295,7 @@ func Test_Execute(t *testing.T) { }, args: args{ procedure: "create_user", - inputs: []map[string]interface{}{}, + inputs: [][]any{}, }, expectedOutputs: nil, isCall: true, @@ -324,7 +324,7 @@ func Test_Execute(t *testing.T) { }, args: args{ procedure: "create_user", - inputs: []map[string]interface{}{}, + inputs: [][]any{}, }, expectedOutputs: []map[string]interface{}{}, isCall: true, @@ -353,7 +353,7 @@ func Test_Execute(t *testing.T) { }, args: args{ procedure: "create_user", - inputs: []map[string]interface{}{}, + inputs: [][]any{}, }, expectedOutputs: nil, isCall: false, @@ -403,7 +403,7 @@ func Test_Execute(t *testing.T) { var outputs []map[string]interface{} if tt.isCall { if len(tt.args.inputs) == 0 { - tt.args.inputs = []map[string]interface{}{nil} + tt.args.inputs = append(tt.args.inputs, []any{}) } outputs, err = ds.Call(ctx, tt.args.procedure, tt.args.inputs[0], &dataset.TxOpts{ diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go index 93c118090..de1b2df1e 100644 --- a/pkg/engine/engine.go +++ b/pkg/engine/engine.go @@ -130,7 +130,7 @@ func (e *Engine) getInitializers() map[string]dataset.Initializer { // Execute executes a procedure on a database. // It returns the result of the procedure. // It takes a context, the database id, the procedure name, the arguments, and optionally some options. -func (e *Engine) Execute(ctx context.Context, dbid string, procedure string, args []map[string]any, opts ...ExecutionOpt) ([]map[string]any, error) { +func (e *Engine) Execute(ctx context.Context, dbid string, procedure string, args [][]any, opts ...ExecutionOpt) ([]map[string]any, error) { options := &executionConfig{} for _, opt := range opts { opt(options) @@ -141,10 +141,11 @@ func (e *Engine) Execute(ctx context.Context, dbid string, procedure string, arg return nil, ErrDatasetNotFound } + if len(args) == 0 { + args = append(args, []any{}) + } + if options.ReadOnly { - if len(args) == 0 { - args = append(args, make(map[string]any)) - } return ds.Call(ctx, procedure, args[0], &dataset.TxOpts{ Caller: options.Sender, diff --git a/pkg/engine/interfaces.go b/pkg/engine/interfaces.go index 0ac680214..d7fe921e5 100644 --- a/pkg/engine/interfaces.go +++ b/pkg/engine/interfaces.go @@ -45,11 +45,11 @@ type Dataset interface { Metadata() (name, owner string) Delete() error Query(ctx context.Context, stmt string, args map[string]any) ([]map[string]any, error) - Execute(ctx context.Context, procedure string, args []map[string]any, opts *dataset.TxOpts) ([]map[string]any, error) + Execute(ctx context.Context, procedure string, args [][]any, opts *dataset.TxOpts) ([]map[string]any, error) ApplyChangeset(changeset io.Reader) error Savepoint() (sql.Savepoint, error) CreateSession() (sql.Session, error) - Call(ctx context.Context, procedure string, args map[string]any, opts *dataset.TxOpts) ([]map[string]any, error) + Call(ctx context.Context, procedure string, args []any, opts *dataset.TxOpts) ([]map[string]any, error) } type MasterDB interface { diff --git a/pkg/grpc/client/v1/broadcast.go b/pkg/grpc/client/v1/broadcast.go index 372e8d656..5f584d1e8 100644 --- a/pkg/grpc/client/v1/broadcast.go +++ b/pkg/grpc/client/v1/broadcast.go @@ -6,21 +6,24 @@ import ( "math/big" txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" - kTx "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" ) -func (c *Client) Broadcast(ctx context.Context, tx *kTx.Transaction) (*kTx.Receipt, error) { - pbTx := ConvertTx(tx) +func (c *Client) Broadcast(ctx context.Context, tx *transactions.Transaction) (*transactions.TransactionStatus, error) { + pbTx := convertTx(tx) res, err := c.txClient.Broadcast(ctx, &txpb.BroadcastRequest{Tx: pbTx}) if err != nil { return nil, fmt.Errorf("TxServiceClient failed to Broadcast transaction: %w", err) } - if res.Receipt == nil { + if res.Status == nil { return nil, fmt.Errorf("TxServiceClient failed to Broadcast transaction: receipt is nil") } - txRes := ConvertReceipt(res.Receipt) + txRes, err := convertTransactionStatus(res.Status) + if err != nil { + return nil, fmt.Errorf("TxServiceClient failed to convert transaction status: %w", err) + } return txRes, nil } @@ -34,9 +37,9 @@ func (c *Client) Ping(ctx context.Context) (string, error) { return res.Message, nil } -func (c *Client) EstimateCost(ctx context.Context, tx *kTx.Transaction) (*big.Int, error) { +func (c *Client) EstimateCost(ctx context.Context, tx *transactions.Transaction) (*big.Int, error) { // convert transaction to proto - pbTx := ConvertTx(tx) + pbTx := convertTx(tx) res, err := c.txClient.EstimatePrice(ctx, &txpb.EstimatePriceRequest{ Tx: pbTx, @@ -53,23 +56,3 @@ func (c *Client) EstimateCost(ctx context.Context, tx *kTx.Transaction) (*big.In fmt.Println("Estimated cost:", bigCost) return bigCost, nil } - -func ConvertTx(incoming *kTx.Transaction) *txpb.Tx { - return &txpb.Tx{ - Hash: incoming.Hash, - PayloadType: incoming.PayloadType.Int32(), - Payload: incoming.Payload, - Fee: incoming.Fee, - Nonce: incoming.Nonce, - Signature: convertActionSignature(incoming.Signature), - Sender: incoming.Sender, - } -} - -func ConvertReceipt(incoming *txpb.TxReceipt) *kTx.Receipt { - return &kTx.Receipt{ - TxHash: incoming.TxHash, - Fee: incoming.Fee, - Body: incoming.Body, - } -} diff --git a/pkg/grpc/client/v1/call.go b/pkg/grpc/client/v1/call.go index 4b1b31de7..9d97e3d71 100644 --- a/pkg/grpc/client/v1/call.go +++ b/pkg/grpc/client/v1/call.go @@ -6,20 +6,15 @@ import ( "fmt" txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" - "github.com/kwilteam/kwil-db/pkg/crypto" - "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" ) -func (c *Client) Call(ctx context.Context, req *tx.SignedMessage[*tx.CallActionPayload]) ([]map[string]any, error) { - payload, err := req.Payload.Bytes() - if err != nil { - return nil, fmt.Errorf("failed to convert payload to bytes: %w", err) - } +func (c *Client) Call(ctx context.Context, req *transactions.SignedMessage) ([]map[string]any, error) { grpcMsg := &txpb.CallRequest{ - Payload: payload, + Payload: req.Message, Signature: convertActionSignature(req.Signature), - Sender: req.Sender, + Sender: req.Sender.Bytes(), } res, err := c.txClient.Call(ctx, grpcMsg) @@ -36,16 +31,3 @@ func (c *Client) Call(ctx context.Context, req *tx.SignedMessage[*tx.CallActionP return result, nil } - -func convertActionSignature(oldSig *crypto.Signature) *txpb.Signature { - if oldSig == nil { - return &txpb.Signature{} - } - - newSig := &txpb.Signature{ - SignatureBytes: oldSig.Signature, - SignatureType: oldSig.Type.Int32(), - } - - return newSig -} diff --git a/pkg/grpc/client/v1/convert.go b/pkg/grpc/client/v1/convert.go new file mode 100644 index 000000000..185ab096d --- /dev/null +++ b/pkg/grpc/client/v1/convert.go @@ -0,0 +1,50 @@ +package client + +import ( + "fmt" + "math/big" + + txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" + "github.com/kwilteam/kwil-db/pkg/crypto" + "github.com/kwilteam/kwil-db/pkg/transactions" +) + +func convertTx(incoming *transactions.Transaction) *txpb.Transaction { + return &txpb.Transaction{ + Body: &txpb.Transaction_Body{ + Payload: incoming.Body.Payload, + PayloadType: incoming.Body.PayloadType.String(), + Fee: incoming.Body.Fee.String(), + Nonce: incoming.Body.Nonce, + Salt: incoming.Body.Salt, + }, + Signature: convertActionSignature(incoming.Signature), + Sender: incoming.Sender.Bytes(), + } +} + +func convertActionSignature(oldSig *crypto.Signature) *txpb.Signature { + if oldSig == nil { + return &txpb.Signature{} + } + + newSig := &txpb.Signature{ + SignatureBytes: oldSig.Signature, + SignatureType: oldSig.Type.Int32(), + } + + return newSig +} + +func convertTransactionStatus(incoming *txpb.TransactionStatus) (*transactions.TransactionStatus, error) { + bigFee, ok := big.NewInt(0).SetString(incoming.Fee, 10) + if !ok { + return nil, fmt.Errorf("failed to convert fee to big.Int") + } + + return &transactions.TransactionStatus{ + ID: incoming.Id, + Fee: bigFee, + Status: transactions.Status(incoming.Status), + }, nil +} diff --git a/pkg/grpc/client/v1/schema.go b/pkg/grpc/client/v1/schema.go index ff52ab679..9325db47a 100644 --- a/pkg/grpc/client/v1/schema.go +++ b/pkg/grpc/client/v1/schema.go @@ -4,10 +4,10 @@ import ( "context" txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" - "github.com/kwilteam/kwil-db/pkg/serialize" + "github.com/kwilteam/kwil-db/pkg/transactions" ) -func (c *Client) GetSchema(ctx context.Context, dbid string) (*serialize.Schema, error) { +func (c *Client) GetSchema(ctx context.Context, dbid string) (*transactions.Schema, error) { res, err := c.txClient.GetSchema(ctx, &txpb.GetSchemaRequest{ Dbid: dbid, }) @@ -15,11 +15,11 @@ func (c *Client) GetSchema(ctx context.Context, dbid string) (*serialize.Schema, return nil, err } - return convertSchema(res.Dataset), nil + return convertSchema(res.Schema), nil } -func convertSchema(dataset *txpb.Dataset) *serialize.Schema { - return &serialize.Schema{ +func convertSchema(dataset *txpb.Schema) *transactions.Schema { + return &transactions.Schema{ Owner: dataset.Owner, Name: dataset.Name, Tables: convertTables(dataset.Tables), @@ -27,10 +27,10 @@ func convertSchema(dataset *txpb.Dataset) *serialize.Schema { } } -func convertTables(tables []*txpb.Table) []*serialize.Table { - convTables := make([]*serialize.Table, len(tables)) +func convertTables(tables []*txpb.Table) []*transactions.Table { + convTables := make([]*transactions.Table, len(tables)) for i, table := range tables { - convTables[i] = &serialize.Table{ + convTables[i] = &transactions.Table{ Name: table.Name, Columns: convertColumns(table.Columns), Indexes: convertIndexes(table.Indexes), @@ -40,10 +40,10 @@ func convertTables(tables []*txpb.Table) []*serialize.Table { return convTables } -func convertColumns(columns []*txpb.Column) []*serialize.Column { - convColumns := make([]*serialize.Column, len(columns)) +func convertColumns(columns []*txpb.Column) []*transactions.Column { + convColumns := make([]*transactions.Column, len(columns)) for i, column := range columns { - convColumns[i] = &serialize.Column{ + convColumns[i] = &transactions.Column{ Name: column.Name, Type: column.Type, Attributes: convertAttributes(column.Attributes), @@ -53,10 +53,10 @@ func convertColumns(columns []*txpb.Column) []*serialize.Column { return convColumns } -func convertAttributes(attributes []*txpb.Attribute) []*serialize.Attribute { - convAttributes := make([]*serialize.Attribute, len(attributes)) +func convertAttributes(attributes []*txpb.Attribute) []*transactions.Attribute { + convAttributes := make([]*transactions.Attribute, len(attributes)) for i, attribute := range attributes { - convAttributes[i] = &serialize.Attribute{ + convAttributes[i] = &transactions.Attribute{ Type: attribute.Type, Value: attribute.Value, } @@ -65,10 +65,10 @@ func convertAttributes(attributes []*txpb.Attribute) []*serialize.Attribute { return convAttributes } -func convertIndexes(indexes []*txpb.Index) []*serialize.Index { - convIndexes := make([]*serialize.Index, len(indexes)) +func convertIndexes(indexes []*txpb.Index) []*transactions.Index { + convIndexes := make([]*transactions.Index, len(indexes)) for i, index := range indexes { - convIndexes[i] = &serialize.Index{ + convIndexes[i] = &transactions.Index{ Name: index.Name, Columns: index.Columns, Type: index.Type, @@ -78,10 +78,10 @@ func convertIndexes(indexes []*txpb.Index) []*serialize.Index { return convIndexes } -func convertActions(actions []*txpb.Action) []*serialize.Action { - convActions := make([]*serialize.Action, len(actions)) +func convertActions(actions []*txpb.Action) []*transactions.Action { + convActions := make([]*transactions.Action, len(actions)) for i, action := range actions { - convActions[i] = &serialize.Action{ + convActions[i] = &transactions.Action{ Name: action.Name, Public: action.Public, Mutability: action.Mutability, diff --git a/pkg/grpc/client/v1/validator.go b/pkg/grpc/client/v1/validator.go index e1d0b715f..2bb483cc4 100644 --- a/pkg/grpc/client/v1/validator.go +++ b/pkg/grpc/client/v1/validator.go @@ -5,7 +5,7 @@ import ( "fmt" txpb "github.com/kwilteam/kwil-db/api/protobuf/tx/v1" - kTx "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" ) func (c *Client) ApproveValidator(ctx context.Context, pubKey []byte) error { @@ -23,8 +23,8 @@ func (c *Client) ApproveValidator(ctx context.Context, pubKey []byte) error { return nil } -func (c *Client) ValidatorJoin(ctx context.Context, tx *kTx.Transaction) (*kTx.Receipt, error) { - pbTx := ConvertTx(tx) +func (c *Client) ValidatorJoin(ctx context.Context, tx *transactions.Transaction) (*transactions.TransactionStatus, error) { + pbTx := convertTx(tx) fmt.Println("Broadcasting ValidatorJoin transaction") resp, err := c.txClient.ValidatorJoin(ctx, &txpb.ValidatorJoinRequest{Tx: pbTx}) if err != nil { @@ -37,12 +37,11 @@ func (c *Client) ValidatorJoin(ctx context.Context, tx *kTx.Transaction) (*kTx.R return nil, fmt.Errorf("TxServiceClient failed to join Validator: receipt is nil") } - txRes := ConvertReceipt(resp.Receipt) - return txRes, nil + return convertTransactionStatus(resp.Receipt) } -func (c *Client) ValidatorLeave(ctx context.Context, tx *kTx.Transaction) (*kTx.Receipt, error) { - pbTx := ConvertTx(tx) +func (c *Client) ValidatorLeave(ctx context.Context, tx *transactions.Transaction) (*transactions.TransactionStatus, error) { + pbTx := convertTx(tx) fmt.Println("Broadcasting ValidatorLeave transaction") resp, err := c.txClient.ValidatorLeave(ctx, &txpb.ValidatorLeaveRequest{Tx: pbTx}) if err != nil { @@ -55,8 +54,7 @@ func (c *Client) ValidatorLeave(ctx context.Context, tx *kTx.Transaction) (*kTx. return nil, fmt.Errorf("TxServiceClient failed to leave Validator: receipt is nil") } - txRes := ConvertReceipt(resp.Receipt) - return txRes, nil + return convertTransactionStatus(resp.Receipt) } func (c *Client) ValidatorJoinStatus(ctx context.Context, pubKey []byte) (*txpb.ValidatorJoinStatusResponse, error) { diff --git a/pkg/modules/datasets/convert.go b/pkg/modules/datasets/convert.go new file mode 100644 index 000000000..9092b5069 --- /dev/null +++ b/pkg/modules/datasets/convert.go @@ -0,0 +1,180 @@ +package datasets + +import ( + "fmt" + + engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" + "github.com/kwilteam/kwil-db/pkg/transactions" +) + +func ConvertSchemaToEngine(old *transactions.Schema) (*engineTypes.Schema, error) { + procedures, err := convertActionsToEngine(old.Actions) + if err != nil { + return nil, err + } + + tables, err := convertTablesToEngine(old.Tables) + if err != nil { + return nil, err + } + + extensions, err := convertExtensionsToEngine(old.Extensions) + if err != nil { + return nil, err + } + + return &engineTypes.Schema{ + Owner: old.Owner, + Name: old.Name, + Tables: tables, + Procedures: procedures, + Extensions: extensions, + }, nil +} + +func convertTablesToEngine(tables []*transactions.Table) ([]*engineTypes.Table, error) { + convTables := make([]*engineTypes.Table, len(tables)) + for i, table := range tables { + columns, err := convertColumnsToEngine(table.Columns) + if err != nil { + return nil, err + } + + indexes, err := convertIndexesToEngine(table.Indexes) + if err != nil { + return nil, err + } + + convTables[i] = &engineTypes.Table{ + Name: table.Name, + Columns: columns, + Indexes: indexes, + } + } + + return convTables, nil +} + +func convertColumnsToEngine(columns []*transactions.Column) ([]*engineTypes.Column, error) { + convColumns := make([]*engineTypes.Column, len(columns)) + for i, column := range columns { + colType := engineTypes.DataType(column.Type) + if err := colType.Clean(); err != nil { + return nil, err + } + + attributes, err := convertAttributesToEngine(column.Attributes) + if err != nil { + return nil, err + } + + convColumns[i] = &engineTypes.Column{ + Name: column.Name, + Type: colType, + Attributes: attributes, + } + } + + return convColumns, nil +} + +func convertAttributesToEngine(attributes []*transactions.Attribute) ([]*engineTypes.Attribute, error) { + convAttributes := make([]*engineTypes.Attribute, len(attributes)) + for i, attribute := range attributes { + attrType := engineTypes.AttributeType(attribute.Type) + if err := attrType.Clean(); err != nil { + return nil, err + } + + convAttributes[i] = &engineTypes.Attribute{ + Type: attrType, + Value: attribute.Value, // Assuming you have a function to parse the value based on its type + } + } + + return convAttributes, nil +} + +func convertIndexesToEngine(indexes []*transactions.Index) ([]*engineTypes.Index, error) { + convIndexes := make([]*engineTypes.Index, len(indexes)) + for i, index := range indexes { + indexType := engineTypes.IndexType(index.Type) + if err := indexType.Clean(); err != nil { + return nil, err + } + + convIndexes[i] = &engineTypes.Index{ + Name: index.Name, + Columns: index.Columns, + Type: indexType, + } + } + + return convIndexes, nil +} + +func convertActionsToEngine(actions []*transactions.Action) ([]*engineTypes.Procedure, error) { + convActions := make([]*engineTypes.Procedure, len(actions)) + for i, action := range actions { + mods, err := convertModifiersToEngine(action.Mutability, action.Auxiliaries) + if err != nil { + return nil, err + } + + convActions[i] = &engineTypes.Procedure{ + Name: action.Name, + Public: action.Public, + Modifiers: mods, + Args: action.Inputs, + Statements: action.Statements, + } + } + + return convActions, nil +} + +func convertModifiersToEngine(mutability string, auxiliaries []string) ([]engineTypes.Modifier, error) { + mods := make([]engineTypes.Modifier, len(auxiliaries)+1) + switch mutability { + case "UPDATE": + break + case "VIEW": + mods[0] = engineTypes.ModifierView + default: + return nil, fmt.Errorf("unknown mutability type: %v", mutability) + } + for i, aux := range auxiliaries { + switch aux { + case "AUTHENTICATED": + mods[i+1] = engineTypes.ModifierAuthenticated + case "OWNER": + mods[i+1] = engineTypes.ModifierOwner + default: + return nil, fmt.Errorf("unknown auxiliary type: %v", aux) + } + } + + return mods, nil +} + +func convertExtensionsToEngine(extensions []*transactions.Extension) ([]*engineTypes.Extension, error) { + convExtensions := make([]*engineTypes.Extension, len(extensions)) + for i, extension := range extensions { + convExtensions[i] = &engineTypes.Extension{ + Name: extension.Name, + Initialization: convertExtensionConfigToEngine(extension.Config), + Alias: extension.Alias, + } + } + + return convExtensions, nil +} + +func convertExtensionConfigToEngine(configs []*transactions.ExtensionConfig) map[string]string { + conf := make(map[string]string) + for _, param := range configs { + conf[param.Argument] = param.Value + } + + return conf +} diff --git a/pkg/modules/datasets/databases.go b/pkg/modules/datasets/databases.go index 2e1120de6..c40eb45bb 100644 --- a/pkg/modules/datasets/databases.go +++ b/pkg/modules/datasets/databases.go @@ -1,6 +1,8 @@ package datasets -import "github.com/kwilteam/kwil-db/pkg/log" +import ( + "github.com/kwilteam/kwil-db/pkg/log" +) type DatasetModule struct { engine Engine diff --git a/pkg/modules/datasets/execution.go b/pkg/modules/datasets/execution.go index c67129a73..8837ce5ad 100644 --- a/pkg/modules/datasets/execution.go +++ b/pkg/modules/datasets/execution.go @@ -8,7 +8,7 @@ import ( "github.com/kwilteam/kwil-db/pkg/balances" "github.com/kwilteam/kwil-db/pkg/engine" engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" - transaction "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" ) /* @@ -22,88 +22,119 @@ type ExecutionResponse struct { } // Deploy deploys a database. -func (u *DatasetModule) Deploy(ctx context.Context, schema *engineTypes.Schema, tx *transaction.Transaction) (*transaction.ExecutionResponse, error) { +func (u *DatasetModule) Deploy(ctx context.Context, schema *engineTypes.Schema, tx *transactions.Transaction) (*transactions.TransactionStatus, error) { price, err := u.PriceDeploy(ctx, schema) if err != nil { - return nil, err + if price == nil { + price = big.NewInt(0) + } + return failure(tx, price, err) } err = u.compareAndSpend(ctx, price, tx) if err != nil { - return nil, err + return failure(tx, price, err) } _, err = u.engine.CreateDataset(ctx, schema) if err != nil { - return nil, fmt.Errorf("failed to create dataset: %w", err) + return failure(tx, price, fmt.Errorf("failed to create dataset: %w", err)) } - return &transaction.ExecutionResponse{ - Fee: price, - }, nil + return success(tx, price) } // Drop drops a database. -func (u *DatasetModule) Drop(ctx context.Context, dbid string, tx *transaction.Transaction) (*transaction.ExecutionResponse, error) { +func (u *DatasetModule) Drop(ctx context.Context, dbid string, tx *transactions.Transaction) (*transactions.TransactionStatus, error) { price, err := u.PriceDrop(ctx, dbid) if err != nil { - return nil, err + if price == nil { + price = big.NewInt(0) + } + return failure(tx, price, err) } err = u.compareAndSpend(ctx, price, tx) if err != nil { - return nil, err + return failure(tx, price, err) } - err = u.engine.DropDataset(ctx, tx.Sender, dbid) + err = u.engine.DropDataset(ctx, tx.Sender.Address().String(), dbid) if err != nil { - return nil, fmt.Errorf("failed to drop dataset: %w", err) + return failure(tx, price, fmt.Errorf("failed to drop dataset: %w", err)) } - return &transaction.ExecutionResponse{ - Fee: price, - }, nil + return success(tx, price) } // Execute executes an action against a database. -func (u *DatasetModule) Execute(ctx context.Context, dbid string, action string, params []map[string]any, tx *transaction.Transaction) (*transaction.ExecutionResponse, error) { - price, err := u.PriceExecute(ctx, dbid, action, params) +// TODO: I think args should be [][]any, not [][]string +func (u *DatasetModule) Execute(ctx context.Context, dbid string, action string, args [][]any, tx *transactions.Transaction) (*transactions.TransactionStatus, error) { + price, err := u.PriceExecute(ctx, dbid, action, args) if err != nil { - return nil, err + if price == nil { + price = big.NewInt(0) + } + return failure(tx, price, err) } err = u.compareAndSpend(ctx, price, tx) if err != nil { - return nil, err + return failure(tx, price, err) } - _, err = u.engine.Execute(ctx, dbid, action, params, - engine.WithCaller(tx.Sender), + _, err = u.engine.Execute(ctx, dbid, action, args, + engine.WithCaller(tx.Sender.Address().String()), ) if err != nil { - return nil, fmt.Errorf("failed to execute action: %w", err) + return failure(tx, price, fmt.Errorf("failed to execute action: %w", err)) } - return &transaction.ExecutionResponse{ - Fee: price, - }, nil + return success(tx, price) } // compareAndSpend compares the calculated price to the transaction's fee, and spends the price if the fee is sufficient. -func (u *DatasetModule) compareAndSpend(ctx context.Context, price *big.Int, tx *transaction.Transaction) error { - bigFee := new(big.Int) - _, ok := bigFee.SetString(tx.Fee, 10) - if !ok { - return fmt.Errorf("failed to parse fee %s", tx.Fee) - } +func (u *DatasetModule) compareAndSpend(ctx context.Context, price *big.Int, tx *transactions.Transaction) error { - if bigFee.Cmp(price) < 0 { - return fmt.Errorf(`%w: fee %s is less than price %s`, ErrInsufficientFee, tx.Fee, price.String()) + if tx.Body.Fee.Cmp(price) < 0 { + return fmt.Errorf(`%w: fee %s is less than price %s`, ErrInsufficientFee, tx.Body.Fee.String(), price.String()) } return u.accountStore.Spend(ctx, &balances.Spend{ - AccountAddress: tx.Sender, + AccountAddress: tx.Sender.Address().String(), Amount: price, - Nonce: tx.Nonce, + Nonce: int64(tx.Body.Nonce), }) } + +func success(tx *transactions.Transaction, fee *big.Int) (*transactions.TransactionStatus, error) { + txid, err := tx.GetHash() + if err != nil { + return nil, fmt.Errorf("failed to get tx hash: %w", err) + } + + return &transactions.TransactionStatus{ + ID: txid, + Fee: fee, + Status: transactions.StatusSuccess, + }, nil +} + +func failure(tx *transactions.Transaction, fee *big.Int, errs ...error) (*transactions.TransactionStatus, error) { + txid, err := tx.GetHash() + if err != nil { + return nil, fmt.Errorf("failed to get tx hash: %w", err) + } + + var errStrings []string + for _, err := range errs { + errStrings = append(errStrings, err.Error()) + } + + return &transactions.TransactionStatus{ + ID: txid, + Fee: fee, + Status: transactions.StatusFailed, + Errors: errStrings, + }, nil +} diff --git a/pkg/modules/datasets/interfaces.go b/pkg/modules/datasets/interfaces.go index 946c734b5..18b7b9621 100644 --- a/pkg/modules/datasets/interfaces.go +++ b/pkg/modules/datasets/interfaces.go @@ -15,7 +15,7 @@ type AccountStore interface { type Engine interface { CreateDataset(ctx context.Context, schema *engineTypes.Schema) (dbid string, finalErr error) DropDataset(ctx context.Context, sender, dbid string) error - Execute(ctx context.Context, dbid string, procedure string, args []map[string]any, opts ...engine.ExecutionOpt) ([]map[string]any, error) + Execute(ctx context.Context, dbid string, procedure string, args [][]any, opts ...engine.ExecutionOpt) ([]map[string]any, error) ListDatasets(ctx context.Context, owner string) ([]string, error) Query(ctx context.Context, dbid string, query string) ([]map[string]any, error) GetSchema(ctx context.Context, dbid string) (*engineTypes.Schema, error) diff --git a/pkg/modules/datasets/pricing.go b/pkg/modules/datasets/pricing.go index 8f931a422..849853c80 100644 --- a/pkg/modules/datasets/pricing.go +++ b/pkg/modules/datasets/pricing.go @@ -24,7 +24,7 @@ func (d *DatasetModule) PriceDrop(ctx context.Context, dbid string) (price *big. } // PriceExecute returns the price of executing an action. -func (d *DatasetModule) PriceExecute(ctx context.Context, dbid string, action string, params []map[string]any) (price *big.Int, err error) { +func (d *DatasetModule) PriceExecute(ctx context.Context, dbid string, action string, args [][]any) (price *big.Int, err error) { return d.applyFeeMultiplier(defaultExecutePrice), nil } diff --git a/pkg/modules/datasets/read.go b/pkg/modules/datasets/read.go index ffe2089c6..1cbb62205 100644 --- a/pkg/modules/datasets/read.go +++ b/pkg/modules/datasets/read.go @@ -6,33 +6,27 @@ import ( "github.com/kwilteam/kwil-db/pkg/engine" engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" - transaction "github.com/kwilteam/kwil-db/pkg/tx" + "github.com/kwilteam/kwil-db/pkg/transactions" ) // Call executes a call action on a database. It is a read-only action. // It returns the result of the call. // If a message caller is specified, then it will check the signature of the message and use the caller as the caller of the action. -func (u *DatasetModule) Call(ctx context.Context, call *transaction.CallActionPayload, msg *transaction.SignedMessage[transaction.JsonPayload]) ([]map[string]any, error) { +// TODO: this should not use the action payload type +func (u *DatasetModule) Call(ctx context.Context, dbid string, action string, args []any, msg *transactions.SignedMessage) ([]map[string]any, error) { executionOpts := []engine.ExecutionOpt{ engine.ReadOnly(true), } - if msg.Sender != "" { + if msg.Sender != nil { err := msg.Verify() if err != nil { return nil, fmt.Errorf(`%w: failed to verify signed message: %s`, ErrAuthenticationFailed, err.Error()) } - executionOpts = append(executionOpts, engine.WithCaller(msg.Sender)) + executionOpts = append(executionOpts, engine.WithCaller(msg.Sender.Address().String())) } - params := []map[string]any{} - if call.Params != nil { - params = append(params, call.Params) - } else { - params = append(params, map[string]any{}) - } - - return u.engine.Execute(ctx, call.DBID, call.Action, params, executionOpts...) + return u.engine.Execute(ctx, dbid, action, [][]any{args}, executionOpts...) } // Query executes an ad-hoc query on a database. It is a read-only action. diff --git a/pkg/serialize/rlp/encode.go b/pkg/serialize/encode.go similarity index 75% rename from pkg/serialize/rlp/encode.go rename to pkg/serialize/encode.go index e8fbd82d4..2d4e518a6 100644 --- a/pkg/serialize/rlp/encode.go +++ b/pkg/serialize/encode.go @@ -1,4 +1,4 @@ -package rlp +package serialize import ( "fmt" @@ -31,23 +31,20 @@ func Encode(val any) (SerializedData, error) { return nil, err } - return serializeEncodedValue(&encodedValue{ - EncodingType: currentEncodingType, - Data: btsVal, - }) + return addSerializedTypePrefix(currentEncodingType, btsVal) } func Decode[T any](bts SerializedData) (*T, error) { - encVal, err := deserializeEncodedValue(bts) + encType, val, err := removeSerializedTypePrefix(bts) if err != nil { return nil, err } - switch encVal.EncodingType { + switch encType { case encodingTypeRLP: - return decodeRLP[T](encVal.Data) + return decodeRLP[T](val) default: - return nil, fmt.Errorf("invalid encoding type: %d", encVal.EncodingType) + return nil, fmt.Errorf("invalid encoding type: %d", val) } } diff --git a/pkg/serialize/rlp/encode_test.go b/pkg/serialize/encode_test.go similarity index 91% rename from pkg/serialize/rlp/encode_test.go rename to pkg/serialize/encode_test.go index dc61e97c0..0d7dda7e3 100644 --- a/pkg/serialize/rlp/encode_test.go +++ b/pkg/serialize/encode_test.go @@ -1,9 +1,9 @@ -package rlp_test +package serialize_test import ( "testing" - "github.com/kwilteam/kwil-db/pkg/serialize/rlp" + serialize "github.com/kwilteam/kwil-db/pkg/serialize" "github.com/stretchr/testify/assert" ) @@ -61,7 +61,7 @@ func Test_Encoding(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - output, err := rlp.Encode(tc.input) + output, err := serialize.Encode(tc.input) if tc.encodingErr && err == nil { t.Errorf("Expected error, got nil") } @@ -72,7 +72,7 @@ func Test_Encoding(t *testing.T) { return } - decoded, err := rlp.Decode[any](output) + decoded, err := serialize.Decode[any](output) if err != nil { t.Errorf("Expected no error, got %v", err) } diff --git a/pkg/serialize/prefix.go b/pkg/serialize/prefix.go new file mode 100644 index 000000000..25fd7a89e --- /dev/null +++ b/pkg/serialize/prefix.go @@ -0,0 +1,55 @@ +package serialize + +import ( + "bytes" + "encoding/binary" + "fmt" +) + +// addSerializedTypePrefix adds a prefix to the encoded value to indicate the encoding type. +func addSerializedTypePrefix(encoding encodingType, encodedValue []byte) (SerializedData, error) { + result, err := uint16ToBytes(uint16(encoding)) + if err != nil { + return nil, err + } + + return append(result, encodedValue...), nil +} + +// removeSerializedTypePrefix removes the prefix from the encoded value. +func removeSerializedTypePrefix(data SerializedData) (encodingType, []byte, error) { + if len(data) == 0 { + return encodingTypeInvalid, nil, fmt.Errorf("cannot deserialize encoded value: data is empty") + } + typ, err := bytesToUint16(data[:2]) + if err != nil { + return encodingTypeInvalid, nil, err + } + + return encodingType(typ), data[2:], nil +} + +// uint16ToBytes converts a uint16 to a byte slice. +func uint16ToBytes(n uint16) ([]byte, error) { + buf := new(bytes.Buffer) + err := binary.Write(buf, binary.BigEndian, n) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// bytesToUint16 converts a byte slice to a uint16. +func bytesToUint16(b []byte) (uint16, error) { + if len(b) < 2 { + return 0, fmt.Errorf("cannot convert bytes to uint16: bytes are too short") + } + + buf := bytes.NewReader(b) + var n uint16 + err := binary.Read(buf, binary.BigEndian, &n) + if err != nil { + return 0, err + } + return n, nil +} diff --git a/pkg/serialize/rlp/value.go b/pkg/serialize/rlp/value.go deleted file mode 100644 index cbaa4197c..000000000 --- a/pkg/serialize/rlp/value.go +++ /dev/null @@ -1,59 +0,0 @@ -package rlp - -import ( - "bytes" - "encoding/binary" - "fmt" -) - -type encodedValue struct { - EncodingType encodingType - Data []byte -} - -func serializeEncodedValue(value *encodedValue) ([]byte, error) { - result, err := uint16ToBytes(uint16(value.EncodingType)) - if err != nil { - return nil, err - } - - return append(result, value.Data...), nil -} - -func deserializeEncodedValue(data []byte) (*encodedValue, error) { - if len(data) == 0 { - return nil, fmt.Errorf("cannot deserialize encoded value: data is empty") - } - typ, err := bytesToUint16(data[:2]) - if err != nil { - return nil, err - } - - return &encodedValue{ - EncodingType: encodingType(typ), - Data: data[2:], - }, nil -} - -func uint16ToBytes(n uint16) ([]byte, error) { - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.BigEndian, n) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} - -func bytesToUint16(b []byte) (uint16, error) { - if len(b) < 2 { - return 0, fmt.Errorf("cannot convert bytes to uint16: bytes are too short") - } - - buf := bytes.NewReader(b) - var n uint16 - err := binary.Read(buf, binary.BigEndian, &n) - if err != nil { - return 0, err - } - return n, nil -} diff --git a/pkg/transactions/message.go b/pkg/transactions/message.go index 26dbce311..8f1de68bd 100644 --- a/pkg/transactions/message.go +++ b/pkg/transactions/message.go @@ -6,15 +6,28 @@ package transactions import ( "github.com/kwilteam/kwil-db/pkg/crypto" - "github.com/kwilteam/kwil-db/pkg/serialize/rlp" + "github.com/kwilteam/kwil-db/pkg/serialize" ) +// CreateSignedMessage creates a signed message from a message. +// This message is used for non-transactional messages. +func CreateSignedMessage(message Payload) (*SignedMessage, error) { + bts, err := message.MarshalBinary() + if err != nil { + return nil, err + } + + return &SignedMessage{ + Message: bts, + }, nil +} + // SignedMessage is any message that has been signed by a private key // It contains a signature and a payload. The message in the signature // should be the hash of the payload. type SignedMessage struct { Signature *crypto.Signature - Message rlp.SerializedData + Message serialize.SerializedData Sender crypto.PublicKey } @@ -25,3 +38,14 @@ type SignedMessage struct { func (s *SignedMessage) Verify() error { return s.Sender.Verify(s.Signature, s.Message) } + +// Sign signs a message with a private key. +func (s *SignedMessage) Sign(privateKey crypto.PrivateKey) error { + signature, err := privateKey.Sign(s.Message) + if err != nil { + return err + } + s.Signature = signature + s.Sender = privateKey.PubKey() + return nil +} diff --git a/pkg/transactions/payloads.go b/pkg/transactions/payloads.go index 002d9a7f8..a5788b02a 100644 --- a/pkg/transactions/payloads.go +++ b/pkg/transactions/payloads.go @@ -1,20 +1,46 @@ package transactions import ( - "encoding" - - "github.com/kwilteam/kwil-db/pkg/serialize/rlp" + "github.com/kwilteam/kwil-db/pkg/serialize" ) +// PayloadType is the type of payload type PayloadType string +func (p PayloadType) String() string { + return string(p) +} + +func (p PayloadType) Valid() bool { + switch p { + case PayloadTypeDeploySchema, + PayloadTypeDropSchema, + PayloadTypeExecuteAction, + PayloadTypeCallAction: + return true + default: + return false + } +} + const ( - PayloadTypeDeploySchema PayloadType = "deploy_schema" - PayloadTypeDropSchema PayloadType = "drop_schema" - PayloadTypeExecuteAction PayloadType = "execute_action" - PayloadTypeCallAction PayloadType = "call_action" + PayloadTypeDeploySchema PayloadType = "deploy_schema" + PayloadTypeDropSchema PayloadType = "drop_schema" + PayloadTypeExecuteAction PayloadType = "execute_action" + PayloadTypeCallAction PayloadType = "call_action" + PayloadTypeValidatorJoin PayloadType = "validator_join" + PayloadTypeValidatorApprove PayloadType = "validator_approve" ) +// Payload is the interface that all payloads must implement +// Implementations should use Kwil's serialization package to encode and decode themselves +type Payload interface { + MarshalBinary() (serialize.SerializedData, error) + UnmarshalBinary(serialize.SerializedData) error + Type() PayloadType +} + +// Schema is the payload that is used to deploy a schema type Schema struct { Owner string Name string @@ -23,15 +49,14 @@ type Schema struct { Extensions []*Extension } -var _ encoding.BinaryMarshaler = (*Schema)(nil) -var _ encoding.BinaryUnmarshaler = (*Schema)(nil) +var _ Payload = (*Schema)(nil) -func (s *Schema) MarshalBinary() ([]byte, error) { - return rlp.Encode(s) +func (s *Schema) MarshalBinary() (serialize.SerializedData, error) { + return serialize.Encode(s) } -func (s *Schema) UnmarshalBinary(b []byte) error { - result, err := rlp.Decode[Schema](b) +func (s *Schema) UnmarshalBinary(b serialize.SerializedData) error { + result, err := serialize.Decode[Schema](b) if err != nil { return err } @@ -40,6 +65,10 @@ func (s *Schema) UnmarshalBinary(b []byte) error { return nil } +func (s *Schema) Type() PayloadType { + return PayloadTypeDeploySchema +} + type Extension struct { Name string Config []*ExtensionConfig @@ -142,20 +171,19 @@ const ( AuxiliaryTypeMustSign AuxiliaryType = "mustsign" ) +// DropSchema is the payload that is used to drop a schema type DropSchema struct { - Owner string - Name string + DBID string } -var _ encoding.BinaryMarshaler = (*DropSchema)(nil) -var _ encoding.BinaryUnmarshaler = (*DropSchema)(nil) +var _ Payload = (*DropSchema)(nil) -func (s *DropSchema) MarshalBinary() ([]byte, error) { - return rlp.Encode(s) +func (s *DropSchema) MarshalBinary() (serialize.SerializedData, error) { + return serialize.Encode(s) } -func (s *DropSchema) UnmarshalBinary(b []byte) error { - res, err := rlp.Decode[DropSchema](b) +func (s *DropSchema) UnmarshalBinary(b serialize.SerializedData) error { + res, err := serialize.Decode[DropSchema](b) if err != nil { return err } @@ -165,21 +193,25 @@ func (s *DropSchema) UnmarshalBinary(b []byte) error { return nil } +func (s *DropSchema) Type() PayloadType { + return PayloadTypeDropSchema +} + +// ActionExecution is the payload that is used to execute an action type ActionExecution struct { DBID string Action string Arguments [][]string } -var _ encoding.BinaryMarshaler = (*ActionExecution)(nil) -var _ encoding.BinaryUnmarshaler = (*ActionExecution)(nil) +var _ Payload = (*ActionExecution)(nil) -func (a *ActionExecution) MarshalBinary() ([]byte, error) { - return rlp.Encode(a) +func (a *ActionExecution) MarshalBinary() (serialize.SerializedData, error) { + return serialize.Encode(a) } -func (s *ActionExecution) UnmarshalBinary(b []byte) error { - res, err := rlp.Decode[ActionExecution](b) +func (s *ActionExecution) UnmarshalBinary(b serialize.SerializedData) error { + res, err := serialize.Decode[ActionExecution](b) if err != nil { return err } @@ -188,21 +220,25 @@ func (s *ActionExecution) UnmarshalBinary(b []byte) error { return nil } +func (a *ActionExecution) Type() PayloadType { + return PayloadTypeExecuteAction +} + +// ActionCall is the payload that is used to call an action type ActionCall struct { DBID string Action string Arguments []string } -var _ encoding.BinaryMarshaler = (*ActionCall)(nil) -var _ encoding.BinaryUnmarshaler = (*ActionCall)(nil) +var _ Payload = (*ActionCall)(nil) -func (a *ActionCall) MarshalBinary() ([]byte, error) { - return rlp.Encode(a) +func (a *ActionCall) MarshalBinary() (serialize.SerializedData, error) { + return serialize.Encode(a) } -func (s *ActionCall) UnmarshalBinary(b []byte) error { - res, err := rlp.Decode[ActionCall](b) +func (s *ActionCall) UnmarshalBinary(b serialize.SerializedData) error { + res, err := serialize.Decode[ActionCall](b) if err != nil { return err } @@ -210,3 +246,7 @@ func (s *ActionCall) UnmarshalBinary(b []byte) error { *s = *res return nil } + +func (a *ActionCall) Type() PayloadType { + return PayloadTypeCallAction +} diff --git a/pkg/transactions/payloads_test.go b/pkg/transactions/payloads_test.go index 50ff292bb..8a1f1822f 100644 --- a/pkg/transactions/payloads_test.go +++ b/pkg/transactions/payloads_test.go @@ -7,16 +7,11 @@ import ( "github.com/stretchr/testify/assert" ) -type marshallable interface { - MarshalBinary() ([]byte, error) - UnmarshalBinary([]byte) error -} - // this simply test that they all serialize and comply with RLP func Test_Types(t *testing.T) { type testCase struct { name string - obj marshallable + obj transactions.Payload } testCases := []testCase{ @@ -117,8 +112,7 @@ func Test_Types(t *testing.T) { { name: "drop_schema", obj: &transactions.DropSchema{ - Owner: "user", - Name: "test_schema", + DBID: "db_id", }, }, } @@ -130,7 +124,7 @@ func Test_Types(t *testing.T) { t.Fatal(err) } - var obj marshallable + var obj transactions.Payload switch tc.obj.(type) { case *transactions.Schema: obj = &transactions.Schema{} diff --git a/pkg/transactions/transaction.go b/pkg/transactions/transaction.go index 9b5954295..5ffabc24c 100644 --- a/pkg/transactions/transaction.go +++ b/pkg/transactions/transaction.go @@ -1,16 +1,52 @@ package transactions import ( + "fmt" "math/big" "github.com/kwilteam/kwil-db/pkg/crypto" - "github.com/kwilteam/kwil-db/pkg/serialize/rlp" + "github.com/kwilteam/kwil-db/pkg/serialize" + "github.com/kwilteam/kwil-db/pkg/utils/random" ) +// CreateTransaction creates a new unsigned transaction. +func CreateTransaction(contents Payload, nonce uint64) (*Transaction, error) { + data, err := contents.MarshalBinary() + if err != nil { + return nil, err + } + + salt, err := generateRandomSalt() + if err != nil { + return nil, err + } + + return &Transaction{ + Body: &TransactionBody{ + Payload: data, + PayloadType: contents.Type(), + Fee: big.NewInt(0), + Nonce: nonce, + Salt: salt[:], + }, + }, nil +} + type Transaction struct { + // Signature is the signature of the transaction + // It can be nil if the transaction is unsigned Signature *crypto.Signature - Body *TransactionBody - Sender crypto.PublicKey + + // Body is the body of the transaction + // It gets serialized and signed + Body *TransactionBody + + // Sender is the public key of the sender + // It is not included in the signature + Sender crypto.PublicKey + + // hash of the transaction that is signed. it is kept here as a cache + hash []byte } // Verify verifies the signature of the transaction @@ -23,10 +59,64 @@ func (t *Transaction) Verify() error { return t.Sender.Verify(t.Signature, data) } +func (t *Transaction) Sign(privateKey crypto.PrivateKey) error { + data, err := t.Body.MarshalBinary() + if err != nil { + return err + } + + // TODO: we need to figure out if we hash the payload before or after signing + signature, err := privateKey.Sign(data) + if err != nil { + return err + } + + t.Signature = signature + t.Sender = privateKey.PubKey() + + return nil +} + +// GetHash gets the hash for the transaction +// If a hash has already been generated, it is returned +func (t *Transaction) GetHash() ([]byte, error) { + if t.hash != nil { + return t.hash, nil + } + + return t.SetHash() +} + +// SetHash re-hashes the transaction and caches the new hash +func (t *Transaction) SetHash() ([]byte, error) { + bts, err := t.Body.MarshalBinary() + if err != nil { + return nil, err + } + + t.hash = crypto.Sha256(bts) + + return t.hash, nil +} + +func (t *Transaction) MarshalBinary() (serialize.SerializedData, error) { + return serialize.Encode(t) +} + +func (t *Transaction) UnmarshalBinary(data serialize.SerializedData) error { + res, err := serialize.Decode[Transaction](data) + if err != nil { + return err + } + + *t = *res + return nil +} + // TransactionBody is the body of a transaction that gets included in the signature type TransactionBody struct { // Payload are the raw bytes of the payload data - Payload rlp.SerializedData + Payload serialize.SerializedData // PayloadType is the type of the payload // This can be used to determine how to decode the payload @@ -42,6 +132,56 @@ type TransactionBody struct { Salt []byte } +func (t *TransactionBody) Verify() error { + if !t.PayloadType.Valid() { + return fmt.Errorf("invalid payload type: %s", t.PayloadType) + } + + if t.Fee == nil { + t.Fee = big.NewInt(0) + } + + return nil +} + func (t *TransactionBody) MarshalBinary() ([]byte, error) { - return rlp.Encode(t) + return serialize.Encode(t) +} + +// generateRandomSalt generates a new random salt +// this salt is not used for any sort of security purpose; +// rather, it is just to prevent hash collisions +// therefore, we only need a small amount of entropy +func generateRandomSalt() ([8]byte, error) { + var s [8]byte + + _, err := random.New().Read(s[:]) + if err != nil { + return s, err + } + return s, nil } + +// TransactionStatus is used to show the status of a transaction +// It is returned to the client after a tx is submitted, +// and can also be used for querying the status of a tx +type TransactionStatus struct { + ID []byte + Fee *big.Int + Status Status + Errors []string +} + +// Status is the status of a transaction +type Status string + +func (s Status) String() string { + return string(s) +} + +// we can add / modify these as needed +const ( + StatusPending Status = "pending" + StatusSuccess Status = "success" + StatusFailed Status = "failed" +) diff --git a/proto b/proto index 5775ca794..c9de21008 160000 --- a/proto +++ b/proto @@ -1 +1 @@ -Subproject commit 5775ca794b57b6c556e4acdc3c618753b0987caf +Subproject commit c9de21008bf3a44896bf2924d7037a5492f76141 From d324858d088aec43eb42c6445a02852fed00199f Mon Sep 17 00:00:00 2001 From: brennan lamey Date: Mon, 14 Aug 2023 10:10:52 -0500 Subject: [PATCH 6/8] updated CLI to use stubbed out methods. Should now be building --- cmd/kwil-cli/cmds/configure/configure.go | 4 ++-- cmd/kwil-cli/cmds/database/deploy.go | 3 +-- cmd/kwil-cli/cmds/database/flags.go | 6 +----- cmd/kwil-cli/cmds/utils/sign.go | 3 +-- cmd/kwil-cli/config/config.go | 7 +++---- pkg/crypto/keys.go | 1 + 6 files changed, 9 insertions(+), 15 deletions(-) diff --git a/cmd/kwil-cli/cmds/configure/configure.go b/cmd/kwil-cli/cmds/configure/configure.go index 44b1d3164..cec9e8bc8 100644 --- a/cmd/kwil-cli/cmds/configure/configure.go +++ b/cmd/kwil-cli/cmds/configure/configure.go @@ -73,7 +73,7 @@ func promptGRPCURL(conf *config.KwilCliConfig) error { func promptPrivateKey(conf *config.KwilCliConfig) error { prompt := &common.Prompter{ Label: "Private Key", - Default: crypto.HexFromECDSAPrivateKey(conf.PrivateKey), + Default: conf.PrivateKey.Hex(), } res, err := prompt.Run() if err != nil { @@ -85,7 +85,7 @@ func promptPrivateKey(conf *config.KwilCliConfig) error { return nil } - pk, err := crypto.ECDSAFromHex(res) + pk, err := crypto.PrivateKeyFromHex(res) if err != nil { fmt.Println(`invalid private key. key could not be converted to hex. received: `, res) promptAskAgain := &common.Prompter{ diff --git a/cmd/kwil-cli/cmds/database/deploy.go b/cmd/kwil-cli/cmds/database/deploy.go index f468a90cc..ca8cd8666 100644 --- a/cmd/kwil-cli/cmds/database/deploy.go +++ b/cmd/kwil-cli/cmds/database/deploy.go @@ -16,7 +16,6 @@ import ( "github.com/kwilteam/kwil-db/cmd/kwil-cli/cmds/common/display" "github.com/kwilteam/kwil-db/cmd/kwil-cli/config" "github.com/kwilteam/kwil-db/pkg/client" - "github.com/kwilteam/kwil-db/pkg/crypto" "github.com/kwilteam/kwil-db/pkg/transactions" "github.com/spf13/cobra" ) @@ -49,7 +48,7 @@ func deployCmd() *cobra.Command { return fmt.Errorf("failed to unmarshal file: %w", err) } - db.Owner = crypto.AddressFromPrivateKey(conf.PrivateKey) + db.Owner = conf.PrivateKey.PubKey().Address().String() res, err := client.DeployDatabase(ctx, db) if err != nil { diff --git a/cmd/kwil-cli/cmds/database/flags.go b/cmd/kwil-cli/cmds/database/flags.go index 6b085099f..5f51aa533 100644 --- a/cmd/kwil-cli/cmds/database/flags.go +++ b/cmd/kwil-cli/cmds/database/flags.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/kwilteam/kwil-db/cmd/kwil-cli/config" - "github.com/kwilteam/kwil-db/pkg/crypto" "github.com/kwilteam/kwil-db/pkg/engine/utils" "github.com/spf13/cobra" @@ -33,15 +32,12 @@ func getSelectedOwner(cmd *cobra.Command, conf *config.KwilCliConfig) (string, e return address, fmt.Errorf("no address provided") } - if !crypto.IsValidAddress(address) { - return address, fmt.Errorf("invalid address provided: %s", address) - } } else { if conf.PrivateKey == nil { return address, fmt.Errorf("no address provided") } - address = crypto.AddressFromPrivateKey(conf.PrivateKey) + address = conf.PrivateKey.PubKey().Address().String() } return address, nil diff --git a/cmd/kwil-cli/cmds/utils/sign.go b/cmd/kwil-cli/cmds/utils/sign.go index feb7e4758..583b3eeae 100644 --- a/cmd/kwil-cli/cmds/utils/sign.go +++ b/cmd/kwil-cli/cmds/utils/sign.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/kwilteam/kwil-db/cmd/kwil-cli/config" - "github.com/kwilteam/kwil-db/pkg/crypto" "github.com/spf13/cobra" ) @@ -26,7 +25,7 @@ func signCmd() *cobra.Command { } // generate signature - sig, err := crypto.Sign([]byte(args[0]), conf.PrivateKey) + sig, err := conf.PrivateKey.Sign([]byte(args[0])) if err != nil { return fmt.Errorf("error generating signature: %w", err) } diff --git a/cmd/kwil-cli/config/config.go b/cmd/kwil-cli/config/config.go index 87957fa7f..d4e64bbda 100644 --- a/cmd/kwil-cli/config/config.go +++ b/cmd/kwil-cli/config/config.go @@ -1,7 +1,6 @@ package config import ( - "crypto/ecdsa" "encoding/json" "fmt" "os" @@ -13,14 +12,14 @@ import ( ) type KwilCliConfig struct { - PrivateKey *ecdsa.PrivateKey + PrivateKey crypto.PrivateKey GrpcURL string ClientChainRPCURL string } func (c *KwilCliConfig) ToPeristedConfig() *kwilCliPersistedConfig { return &kwilCliPersistedConfig{ - PrivateKey: crypto.HexFromECDSAPrivateKey(c.PrivateKey), + PrivateKey: c.PrivateKey.Hex(), GrpcURL: c.GrpcURL, ClientChainRPCURL: c.ClientChainRPCURL, } @@ -42,7 +41,7 @@ func (c *kwilCliPersistedConfig) toKwilCliConfig() (*KwilCliConfig, error) { ClientChainRPCURL: c.ClientChainRPCURL, } - privateKey, err := crypto.ECDSAFromHex(c.PrivateKey) + privateKey, err := crypto.PrivateKeyFromHex(c.PrivateKey) if err != nil { return kwilConfig, nil } diff --git a/pkg/crypto/keys.go b/pkg/crypto/keys.go index af8d382d1..32aaab104 100644 --- a/pkg/crypto/keys.go +++ b/pkg/crypto/keys.go @@ -7,6 +7,7 @@ type PrivateKey interface { Type() KeyType Sign(msg []byte) (*Signature, error) PubKey() PublicKey + Hex() string } type PublicKey interface { From 47e0f8647598c9b6d8588020b54944b7134ff398 Mon Sep 17 00:00:00 2001 From: brennan lamey Date: Mon, 14 Aug 2023 10:13:56 -0500 Subject: [PATCH 7/8] removed old packages --- pkg/_crypto/crypto.go | 26 --- pkg/_crypto/crypto_test.go | 87 --------- pkg/_crypto/ed25519.go | 61 ------ pkg/_crypto/keys.go | 24 --- pkg/_crypto/secp256k1.go | 72 ------- pkg/_crypto/signature.go | 150 -------------- pkg/_crypto/signature_test.go | 131 ------------- pkg/_serialize/consensus.go | 6 - pkg/_serialize/schema.go | 357 ---------------------------------- pkg/_serialize/serial.go | 46 ----- pkg/_tx/message.go | 131 ------------- pkg/_tx/payload.go | 52 ----- pkg/_tx/response.go | 9 - pkg/_tx/sign_test.go | 69 ------- pkg/_tx/tx.go | 101 ---------- 15 files changed, 1322 deletions(-) delete mode 100644 pkg/_crypto/crypto.go delete mode 100644 pkg/_crypto/crypto_test.go delete mode 100644 pkg/_crypto/ed25519.go delete mode 100644 pkg/_crypto/keys.go delete mode 100644 pkg/_crypto/secp256k1.go delete mode 100644 pkg/_crypto/signature.go delete mode 100644 pkg/_crypto/signature_test.go delete mode 100644 pkg/_serialize/consensus.go delete mode 100644 pkg/_serialize/schema.go delete mode 100644 pkg/_serialize/serial.go delete mode 100644 pkg/_tx/message.go delete mode 100644 pkg/_tx/payload.go delete mode 100644 pkg/_tx/response.go delete mode 100644 pkg/_tx/sign_test.go delete mode 100644 pkg/_tx/tx.go diff --git a/pkg/_crypto/crypto.go b/pkg/_crypto/crypto.go deleted file mode 100644 index 4501d6d6c..000000000 --- a/pkg/_crypto/crypto.go +++ /dev/null @@ -1,26 +0,0 @@ -package crypto - -import ( - "crypto/ecdsa" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - ec "github.com/ethereum/go-ethereum/crypto" -) - -func ECDSAFromHex(hex string) (*ecdsa.PrivateKey, error) { - return ec.HexToECDSA(hex) -} - -func AddressFromPrivateKey(key *ecdsa.PrivateKey) string { - caddr := ec.PubkeyToAddress(key.PublicKey) - return caddr.Hex() -} - -func IsValidAddress(addr string) bool { - return common.IsHexAddress(addr) -} - -func HexFromECDSAPrivateKey(key *ecdsa.PrivateKey) string { - return hexutil.Encode(ec.FromECDSA(key))[2:] -} diff --git a/pkg/_crypto/crypto_test.go b/pkg/_crypto/crypto_test.go deleted file mode 100644 index 390761a2e..000000000 --- a/pkg/_crypto/crypto_test.go +++ /dev/null @@ -1,87 +0,0 @@ -package crypto_test - -import ( - "encoding/hex" - "github.com/kwilteam/kwil-db/pkg/crypto" - "reflect" - "testing" -) - -// Using private key 4bb214b1f3a0737d758bc3828cdff371e3769fe84a2678da34700cb18d50770e -// Public: -// Public Bytes: [4 197 141 51 158 16 36 14 57 147 17 68 175 224 209 17 1 128 241 107 124 249 138 4 140 195 17 175 251 164 131 87 37 187 20 25 78 94 105 159 107 221 221 213 105 170 169 248 255 206 112 253 139 14 195 102 158 15 104 246 110 146 154 137 171] -// Address: 0x995d95245698212D4Af52c8031F614C3D3127994 - -func TestSha384(t *testing.T) { - res, err := hex.DecodeString("82835f0f3732e85736f1372184640199c9155a81980f562b4418aadabe2a21f57cb580b48f2f06b439bdf204f4b3dcb7") - if err != nil { - t.Errorf("Error with TestSha384") - } - - type args struct { - data []byte - } - tests := []struct { - name string - args args - want []byte - }{ - { - name: "test1", - args: args{ - data: []byte("kwil"), - }, - want: res, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := crypto.Sha384(tt.args.data); !reflect.DeepEqual(got, tt.want) { - t.Errorf("Sha384() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestSha384Str(t *testing.T) { - - type args struct { - data []byte - } - tests := []struct { - name string - args args - want string - }{ - { - name: "test1", - args: args{ - data: []byte("kwil"), - }, - want: "82835f0f3732e85736f1372184640199c9155a81980f562b4418aadabe2a21f57cb580b48f2f06b439bdf204f4b3dcb7", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := crypto.Sha384Hex(tt.args.data); got != tt.want { - t.Errorf("Sha384Str() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_Sha256(t *testing.T) { - res := "0c9e4969977f81d845cb959915b563a09a857e8d16911ce5a2780f38a5985410" - hash := crypto.Sha256Hex([]byte("kwil")) - if hash != res { - t.Errorf("Error with Sha256") - } -} - -func Test_Sha224(t *testing.T) { - res := "9651608f63f583b7684c3c856d3ef1e3c9d70e2c05b4fa0080989c4d" - hash := crypto.Sha224Hex([]byte("kwil")) - if hash != res { - t.Errorf("Error with Sha224") - } -} diff --git a/pkg/_crypto/ed25519.go b/pkg/_crypto/ed25519.go deleted file mode 100644 index 7ce344c0c..000000000 --- a/pkg/_crypto/ed25519.go +++ /dev/null @@ -1,61 +0,0 @@ -package crypto - -import ( - oasisEd25519 "github.com/oasisprotocol/curve25519-voi/primitives/ed25519" -) - -const Ed25519 KeyType = "Ed25519" - -type Ed25519PrivateKey struct { - privateKey []byte -} - -func (s *Ed25519PrivateKey) Bytes() []byte { - return s.privateKey -} - -func (s *Ed25519PrivateKey) PubKey() PublicKey { - panic("implement me") -} - -func (s *Ed25519PrivateKey) Sign(msg []byte, signatureType SignatureType) ([]byte, error) { - return oasisEd25519.Sign(oasisEd25519.PrivateKey(s.privateKey), msg), nil -} - -func (s *Ed25519PrivateKey) Type() KeyType { - return Ed25519 -} - -type Ed25519PublicKey struct { -} - -func (s *Ed25519PublicKey) Address() Address { - panic("implement me") -} - -func (s *Ed25519PublicKey) Bytes() []byte { - panic("implement me") -} - -func (s *Ed25519PublicKey) Type() KeyType { - return Ed25519 -} - -func (s *Ed25519PublicKey) Verify(sig *Signature2, data []byte) error { - panic("implement me") -} - -type Ed25519Address struct { -} - -func (s *Ed25519Address) Bytes() []byte { - panic("implement me") -} - -func (s *Ed25519Address) Type() KeyType { - return Ed25519 -} - -func (s *Ed25519Address) String() string { - panic("implement me") -} diff --git a/pkg/_crypto/keys.go b/pkg/_crypto/keys.go deleted file mode 100644 index 6e0b7ec4c..000000000 --- a/pkg/_crypto/keys.go +++ /dev/null @@ -1,24 +0,0 @@ -package crypto - -type KeyType string - -type PrivateKey interface { - Bytes() []byte - Type() KeyType - Sign(msg []byte, signatureType SignatureType) ([]byte, error) - PubKey() PublicKey -} - -type PublicKey interface { - Bytes() []byte - Type() KeyType - Verify(sig *Signature2, data []byte) error - Address() Address -} - -type Address interface { - Bytes() []byte - String() string - // do we need to know the key type? - Type() KeyType -} diff --git a/pkg/_crypto/secp256k1.go b/pkg/_crypto/secp256k1.go deleted file mode 100644 index ceb25629a..000000000 --- a/pkg/_crypto/secp256k1.go +++ /dev/null @@ -1,72 +0,0 @@ -package crypto - -import ( - "crypto/ecdsa" - - "github.com/ethereum/go-ethereum/common" - ethCrypto "github.com/ethereum/go-ethereum/crypto" -) - -const Secp256k1 KeyType = "secp256k1" - -type Secp256k1PrivateKey struct { - privateKey *ecdsa.PrivateKey -} - -func (s *Secp256k1PrivateKey) Bytes() []byte { - return ethCrypto.FromECDSA(s.privateKey) -} - -func (s *Secp256k1PrivateKey) PubKey() PublicKey { - return &Secp256k1PublicKey{ - publicKey: &s.privateKey.PublicKey, - } -} - -func (s *Secp256k1PrivateKey) Sign(msg []byte, signatureType SignatureType) ([]byte, error) { - // TODO: implement - panic("TODO") -} - -func (s *Secp256k1PrivateKey) Type() KeyType { - return Secp256k1 -} - -type Secp256k1PublicKey struct { - publicKey *ecdsa.PublicKey -} - -func (s *Secp256k1PublicKey) Address() Address { - return &Secp256k1Address{ - address: ethCrypto.PubkeyToAddress(*s.publicKey), - } -} - -func (s *Secp256k1PublicKey) Bytes() []byte { - return ethCrypto.FromECDSAPub(s.publicKey) -} - -func (s *Secp256k1PublicKey) Type() KeyType { - return Secp256k1 -} - -func (s *Secp256k1PublicKey) Verify(sig *Signature2, data []byte) error { - // TODO: implement - panic("TODO") -} - -type Secp256k1Address struct { - address common.Address -} - -func (s *Secp256k1Address) Bytes() []byte { - return s.address.Bytes() -} - -func (s *Secp256k1Address) Type() KeyType { - return Secp256k1 -} - -func (s *Secp256k1Address) String() string { - return s.address.String() -} diff --git a/pkg/_crypto/signature.go b/pkg/_crypto/signature.go deleted file mode 100644 index a77b8dfdb..000000000 --- a/pkg/_crypto/signature.go +++ /dev/null @@ -1,150 +0,0 @@ -package crypto - -import ( - "crypto/ecdsa" - "errors" - "fmt" - - cmtCrypto "github.com/cometbft/cometbft/crypto" - cmtjson "github.com/cometbft/cometbft/libs/json" - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common" - ec "github.com/ethereum/go-ethereum/crypto" -) - -var ErrInvalidSignature = errors.New("invalid signature") - -type SignatureType int32 - -const ( - SIGNATURE_TYPE_INVALID SignatureType = iota - - // all of these different SECP256k1 signatures should - // be revisited. If they are warranted, then at the very least - // they should be renamed to be more descriptive - // it could also be worth giving them string representations, - // instead of just numbers - PK_SECP256K1_UNCOMPRESSED - ACCOUNT_SECP256K1_UNCOMPRESSED - ACCOUNT_SECP256K1_UNCOMPRESSED_STRUCTURED_v1 - PK_ED25519 - END_SIGNATURE_TYPE -) - -func (s *SignatureType) IsValid() error { - if *s < SIGNATURE_TYPE_INVALID || *s >= END_SIGNATURE_TYPE { - return fmt.Errorf("invalid signature type '%d'", *s) - } - return nil -} - -func (s SignatureType) Int32() int32 { - return int32(s) -} - -type Signature struct { - Message []byte `json:"message"` - Signature []byte `json:"signature_bytes"` - Sender PublicKey - Type SignatureType `json:"signature_type"` -} - -func (s *Signature) Verify() error { - return s.Sender.Verify(s) -} - -/* - everything below this is considered "deprecated", and is only left for backwards compatibility - while the new signature is being implemented -*/ - -func Sign(data []byte, k *ecdsa.PrivateKey) (*Signature, error) { - signature := Signature{ - Type: PK_SECP256K1_UNCOMPRESSED, - } - hash := ec.Keccak256Hash(data) - sig, err := ec.Sign(hash.Bytes(), k) - if err != nil { - return &signature, err - } - - signature.Signature = sig - return &signature, nil -} - -// Deprecated: use Verify instead -func (s *Signature) Check(sender string, data []byte) error { - ok, err := CheckSignature(sender, s, data) - if err != nil { - return err - } - - if !ok { - return ErrInvalidSignature - } - - return nil -} - -func CheckSignature(addr string, sig *Signature, data []byte) (bool, error) { - if addr == "" { - return false, fmt.Errorf("transaction does not have a sender address") - } - switch sig.Type { - case PK_SECP256K1_UNCOMPRESSED: - return checkSignaturePkSECP256k1Uncompressed(addr, sig.Signature, data) - case ACCOUNT_SECP256K1_UNCOMPRESSED: - return checkSignatureAccountSECP256k1Uncompressed(addr, sig.Signature, data) - case PK_ED25519: - return checkSignatureED25519(addr, sig.Signature, data) - default: - return false, fmt.Errorf("unknown signature indicator: %d", sig.Type) - } -} - -func checkSignatureED25519(addr string, sig []byte, data []byte) (bool, error) { - var publicKey cmtCrypto.PubKey - key := fmt.Sprintf(`{"type":"tendermint/PubKeyEd25519","value":%s}`, addr) - fmt.Println("Key:", key) - - err := cmtjson.Unmarshal([]byte(key), &publicKey) - if err != nil { - return false, fmt.Errorf("failed to unmarshal validator public key: %w", err) - } - fmt.Println("publicKey: ", publicKey) - - valid := publicKey.VerifySignature(data, sig) - if valid { - return true, nil - } else { - return false, fmt.Errorf("signature verification failed") - } -} - -func checkSigHash(addr string, sig, hash []byte) (bool, error) { - recoveredPubKey, err := ec.SigToPub(hash, sig) - if err != nil { - return false, fmt.Errorf("failed to recover public key: %w", err) - } - - derAddr := ec.PubkeyToAddress(*recoveredPubKey) - return derAddr == common.HexToAddress(addr), nil - // NOTE: We compare the address [N]byte to be robust to string formatting - // details, namely deviations from EIP55 and prefix (or not). -} - -// This would be for a signature generated by a private key. Likely EVM chains -func checkSignaturePkSECP256k1Uncompressed(addr string, sig []byte, data []byte) (bool, error) { - hash := ec.Keccak256Hash(data) - return checkSigHash(addr, sig, hash[:]) -} - -// checkSignatureAccount checks a signature that was generated by an config instead of a private key -// generally this would mean a signature from MetaMask / equivalent for EVM chains -func checkSignatureAccountSECP256k1Uncompressed(addr string, sig []byte, data []byte) (bool, error) { - msg := accounts.TextHash(data) - if sig[ec.RecoveryIDOffset] == 27 || sig[ec.RecoveryIDOffset] == 28 { - sig[ec.RecoveryIDOffset] -= 27 // Transform yellow paper V from 27/28 to 0/1 - } - return checkSigHash(addr, sig, msg) -} diff --git a/pkg/_crypto/signature_test.go b/pkg/_crypto/signature_test.go deleted file mode 100644 index 3f38e6018..000000000 --- a/pkg/_crypto/signature_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package crypto_test - -import ( - "bytes" - "crypto/ecdsa" - "testing" - - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/common/hexutil" - ec "github.com/ethereum/go-ethereum/crypto" - "github.com/kwilteam/kwil-db/pkg/crypto" -) - -func TestSign(t *testing.T) { - pk, err := ec.HexToECDSA("4bb214b1f3a0737d758bc3828cdff371e3769fe84a2678da34700cb18d50770e") - if err != nil { - t.Errorf("failed to get test private key: %d", err) - } - - sig, err := crypto.Sign([]byte("kwil"), pk) - if err != nil { - t.Errorf("failed to sign: %d", err) - } - - expected, err := hexutil.Decode("0x39fd0a5551cd0008eb45244ad3eea11fb960ff6d8d13aaad9651632b61d26ee20da867cf4f53564bc7bfa795d1efb2bb1169209d1e6f42a2d9e88cfce556b42501") - if err != nil { - t.Errorf("failed to decode expected signature: %d", err) - } - - if !bytes.Equal(sig.Signature, expected) { - t.Errorf("expected %s, got %s", expected, sig.Signature) - } -} - -// signPrefixedLegacy emulates how an account wallet such as MetaMask would -// prefix the message with a pre-determined string to prevent tx signing in -// malicious phishing attempts. This function is only used to test -// CheckSignature with the ACCOUNT_SECP256K1_UNCOMPRESSED Type. -func signPrefixedLegacy(t *testing.T, msg []byte, pk *ecdsa.PrivateKey) *crypto.Signature { - hash := accounts.TextHash(msg) - sig, err := ec.Sign(hash[:], pk) - if err != nil { - t.Fatalf("Sign: %v", err) - } - sig[ec.RecoveryIDOffset] += 27 - - return &crypto.Signature{ - Type: crypto.ACCOUNT_SECP256K1_UNCOMPRESSED, - Signature: sig, - } -} - -func TestCheckSignature(t *testing.T) { - pk, err := ec.HexToECDSA("4bb214b1f3a0737d758bc3828cdff371e3769fe84a2678da34700cb18d50770e") - if err != nil { - t.Errorf("failed to get test private key: %d", err) - } - - msg := []byte("kwil") - - sig, err := crypto.Sign(msg, pk) - if err != nil { - t.Errorf("failed to sign: %d", err) - } - - ok, err := crypto.CheckSignature("0x995d95245698212D4Af52c8031F614C3D3127994", sig, msg) - if err != nil { - t.Errorf("failed to check signature: %d", err) - } - - if !ok { - t.Errorf("expected signature to be valid") - } - - // permit case variation from EIP55 - ok, err = crypto.CheckSignature("0x995d95245698212D4Af52C8031F614C3D3127994", sig, msg) - if err != nil { - t.Errorf("failed to check signature: %d", err) - } - - if !ok { - t.Errorf("expected signature to be valid") - } - - // no prefix OK - ok, err = crypto.CheckSignature("995d95245698212D4Af52C8031F614C3D3127994", sig, msg) - if err != nil { - t.Errorf("failed to check signature: %d", err) - } - - if !ok { - t.Errorf("expected signature to be valid") - } - - // Test ACCOUNT_SECP256K1_UNCOMPRESSED i.e. text signature with auto-prefix and - sig = signPrefixedLegacy(t, msg, pk) - - ok, err = crypto.CheckSignature("0x995d95245698212D4Af52c8031F614C3D3127994", sig, msg) - if err != nil { - t.Errorf("failed to check signature: %d", err) - } - - if !ok { - t.Errorf("expected signature to be valid") - } - - // "account uncompressed" with string case deviation from EIP 55. - ok, err = crypto.CheckSignature("0x995d95245698212D4Af52C8031F614C3D3127994", sig, msg) - if err != nil { - t.Errorf("failed to check signature: %d", err) - } - - if !ok { - t.Errorf("expected signature to be valid") - } -} - -func TestMiscCrypto(t *testing.T) { - pk := "4bb214b1f3a0737d758bc3828cdff371e3769fe84a2678da34700cb18d50770e" - - ecdsaPk, err := crypto.ECDSAFromHex(pk) - if err != nil { - t.Errorf("error getting ecdsa private key from hex") - } - - addr := crypto.AddressFromPrivateKey(ecdsaPk) - - if addr != "0x995d95245698212D4Af52c8031F614C3D3127994" { - t.Errorf("received unexpected address") - } -} diff --git a/pkg/_serialize/consensus.go b/pkg/_serialize/consensus.go deleted file mode 100644 index 60fa4c60b..000000000 --- a/pkg/_serialize/consensus.go +++ /dev/null @@ -1,6 +0,0 @@ -package serialize - -type Validator struct { - PubKey string `json:"pubKey"` - Power int64 `json:"power"` -} diff --git a/pkg/_serialize/schema.go b/pkg/_serialize/schema.go deleted file mode 100644 index 4501d9679..000000000 --- a/pkg/_serialize/schema.go +++ /dev/null @@ -1,357 +0,0 @@ -package serialize - -import ( - "encoding/json" - "fmt" - "strings" - - engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" -) - -type DatasetIdentifier struct { - Owner string `json:"owner"` - Name string `json:"name"` -} - -type Schema struct { - Owner string `json:"owner"` - Name string `json:"name"` - Tables []*Table `json:"tables"` - Actions []*Action `json:"actions"` - Extensions []*Extension `json:"extensions"` -} - -type Extension struct { - Name string `json:"name"` - Config map[string]string `json:"config"` - Alias string `json:"alias"` -} - -type Table struct { - Name string `json:"name"` - Columns []*Column `json:"columns"` - Indexes []*Index `json:"indexes"` - ForeignKeys []*ForeignKey `json:"foreign_keys"` -} - -type Column struct { - Name string `json:"name"` - Type string `json:"type"` - Attributes []*Attribute `json:"attributes,omitempty"` -} - -type Attribute struct { - Type string `json:"type"` - Value any `json:"value"` -} - -type Action struct { - Name string `json:"name"` - Inputs []string `json:"inputs"` - // Mutability could be empty if the abi is generated by legacy version of kuneiform, - // default to "update" for backward compatibility - Mutability string `json:"mutability"` - // Auxiliaries are the auxiliary types that are required for the action, specifying extra characteristics of the action - Auxiliaries []string `json:"auxiliaries"` - Public bool `json:"public"` - Statements []string `json:"statements"` -} - -type Index struct { - Name string `json:"name"` - Columns []string `json:"columns"` - Type string `json:"type"` -} - -type ForeignKey struct { - // ChildKeys are the columns that are referencing another. - // For example, in FOREIGN KEY (a) REFERENCES tbl2(b), "a" is the child key - ChildKeys []string `json:"child_keys"` - - // ParentKeys are the columns that are being referred to. - // For example, in FOREIGN KEY (a) REFERENCES tbl2(b), "tbl2.b" is the parent key - ParentKeys []string `json:"parent_keys"` - - // ParentTable is the table that holds the parent columns. - // For example, in FOREIGN KEY (a) REFERENCES tbl2(b), "tbl2.b" is the parent table - ParentTable string `json:"parent_table"` - - // Action refers to what the foreign key should do when the parent is altered. - // This is NOT the same as a database action; - // however sqlite's docs refer to these as actions, - // so we should be consistent with that. - // For example, ON DELETE CASCADE is a foreign key action - Actions []*ForeignKeyAction `json:"actions"` -} - -// ForeignKeyAction is used to specify what should occur -// if a parent key is updated or deleted -type ForeignKeyAction struct { - // On can be either "UPDATE" or "DELETE" - On string `json:"on"` - - // Do specifies what a foreign key action should do - Do string `json:"do"` -} - -// MutabilityType is the type of mutability -type MutabilityType string - -func (t MutabilityType) String() string { - return string(t) -} - -const ( - MutabilityUpdate MutabilityType = "update" - MutabilityView MutabilityType = "view" -) - -// AuxiliaryType is the type of auxiliary -type AuxiliaryType string - -func (t AuxiliaryType) String() string { - return string(t) -} - -const ( - // AuxiliaryTypeMustSign is used to specify that an action need signature, it is used for `view` action. - AuxiliaryTypeMustSign AuxiliaryType = "mustsign" -) - -func convertSchema(schema *Schema) (*engineTypes.Schema, error) { - actions, err := convertActionsToDto(schema.Actions) - if err != nil { - return nil, err - } - - tables, err := convertTablesToDto(schema.Tables) - if err != nil { - return nil, err - } - - return &engineTypes.Schema{ - Name: schema.Name, - Owner: schema.Owner, - Procedures: actions, - Tables: tables, - Extensions: convertExtensionsToDto(schema.Extensions), - }, nil -} - -func convertActions(actions []*engineTypes.Procedure) []*Action { - entityActions := make([]*Action, len(actions)) - for i, action := range actions { - entityActions[i] = &Action{ - Name: action.Name, - Inputs: action.Args, - Public: action.Public, - Statements: action.Statements, - } - } - - return entityActions -} - -func convertTables(tables []*engineTypes.Table) []*Table { - entityTables := make([]*Table, len(tables)) - for i, table := range tables { - entityTables[i] = &Table{ - Name: table.Name, - Columns: convertColumns(table.Columns), - Indexes: convertIndexes(table.Indexes), - } - } - - return entityTables -} - -func convertColumns(columns []*engineTypes.Column) []*Column { - entityColumns := make([]*Column, len(columns)) - for i, column := range columns { - entityColumns[i] = &Column{ - Name: column.Name, - Type: column.Type.String(), - Attributes: convertAttributes(column.Attributes), - } - } - - return entityColumns -} - -func convertAttributes(attributes []*engineTypes.Attribute) []*Attribute { - entityAttributes := make([]*Attribute, len(attributes)) - for i, attribute := range attributes { - entityAttributes[i] = &Attribute{ - Type: attribute.Type.String(), - Value: attribute.Value, - } - } - - return entityAttributes -} - -func convertIndexes(indexes []*engineTypes.Index) []*Index { - entityIndexes := make([]*Index, len(indexes)) - for i, index := range indexes { - entityIndexes[i] = &Index{ - Name: index.Name, - Columns: index.Columns, - Type: index.Type.String(), - } - } - - return entityIndexes -} - -func convertActionsToDto(actions []*Action) ([]*engineTypes.Procedure, error) { - entityActions := make([]*engineTypes.Procedure, len(actions)) - for i, action := range actions { - mods, err := getModifiers(action) - if err != nil { - return nil, err - } - - entityActions[i] = &engineTypes.Procedure{ - Name: action.Name, - Args: action.Inputs, - Public: action.Public, - Modifiers: mods, - Statements: action.Statements, - } - } - - for i := range entityActions { - err := entityActions[i].Clean() - if err != nil { - return nil, err - } - } - - return entityActions, nil -} - -func getModifiers(action *Action) ([]engineTypes.Modifier, error) { - mods := make([]engineTypes.Modifier, 0) - - if strings.EqualFold(action.Mutability, "view") { - mods = append(mods, engineTypes.ModifierView) - } - - for _, aux := range action.Auxiliaries { - switch aux { - case "mustsign": - mods = append(mods, engineTypes.ModifierAuthenticated) - default: - return nil, fmt.Errorf("modifier %s not supported", aux) - } - } - - return mods, nil -} - -func convertExtensionsToDto(extensions []*Extension) []*engineTypes.Extension { - entityExtensions := make([]*engineTypes.Extension, len(extensions)) - for i, extension := range extensions { - entityExtensions[i] = &engineTypes.Extension{ - Name: extension.Name, - Initialization: extension.Config, - Alias: extension.Alias, - } - } - - return entityExtensions -} - -func convertTablesToDto(tables []*Table) ([]*engineTypes.Table, error) { - entityTables := make([]*engineTypes.Table, len(tables)) - for i, table := range tables { - entityTables[i] = &engineTypes.Table{ - Name: table.Name, - Columns: convertColumnsToDto(table.Columns), - Indexes: convertIndexesToDto(table.Indexes), - ForeignKeys: convertForeignKeysToDto(table.ForeignKeys), - } - } - - for i := range entityTables { - err := entityTables[i].Clean() - if err != nil { - return nil, err - } - } - - return entityTables, nil -} - -func convertForeignKeysToDto(foreignKeys []*ForeignKey) []*engineTypes.ForeignKey { - entityForeignKeys := make([]*engineTypes.ForeignKey, len(foreignKeys)) - for i, foreignKey := range foreignKeys { - entityForeignKeys[i] = &engineTypes.ForeignKey{ - ChildKeys: foreignKey.ChildKeys, - ParentKeys: foreignKey.ParentKeys, - ParentTable: foreignKey.ParentTable, - Actions: convertForeignKeyActionsToDto(foreignKey.Actions), - } - } - - return entityForeignKeys -} - -func convertForeignKeyActionsToDto(foreignKeyActions []*ForeignKeyAction) []*engineTypes.ForeignKeyAction { - entityForeignKeyActions := make([]*engineTypes.ForeignKeyAction, len(foreignKeyActions)) - for i, foreignKeyAction := range foreignKeyActions { - entityForeignKeyActions[i] = &engineTypes.ForeignKeyAction{ - On: engineTypes.ForeignKeyActionOn(foreignKeyAction.On), - Do: engineTypes.ForeignKeyActionDo(foreignKeyAction.Do), - } - } - - return entityForeignKeyActions -} - -func convertColumnsToDto(columns []*Column) []*engineTypes.Column { - entityColumns := make([]*engineTypes.Column, len(columns)) - for i, column := range columns { - entityColumns[i] = &engineTypes.Column{ - Name: column.Name, - Type: engineTypes.DataType(column.Type), - Attributes: convertAttributesToDto(column.Attributes), - } - } - - return entityColumns -} - -func convertAttributesToDto(attributes []*Attribute) []*engineTypes.Attribute { - entityAttributes := make([]*engineTypes.Attribute, len(attributes)) - for i, attribute := range attributes { - entityAttributes[i] = &engineTypes.Attribute{ - Type: engineTypes.AttributeType(attribute.Type), - Value: attribute.Value, - } - } - - return entityAttributes -} - -func convertIndexesToDto(indexes []*Index) []*engineTypes.Index { - entityIndexes := make([]*engineTypes.Index, len(indexes)) - for i, index := range indexes { - entityIndexes[i] = &engineTypes.Index{ - Name: index.Name, - Columns: index.Columns, - Type: engineTypes.IndexType(index.Type), - } - } - - return entityIndexes -} - -func UnmarshalDatasetIdentifier(b []byte) (*DatasetIdentifier, error) { - var dsIdent DatasetIdentifier - err := json.Unmarshal(b, &dsIdent) - if err != nil { - return nil, err - } - return &dsIdent, nil -} diff --git a/pkg/_serialize/serial.go b/pkg/_serialize/serial.go deleted file mode 100644 index 0c2ff5159..000000000 --- a/pkg/_serialize/serial.go +++ /dev/null @@ -1,46 +0,0 @@ -/* - The serialize package makes the old way of serializing / deserializing transaction payloads compatible - with the refactored codebase. This will likely be deleted, but it is going here now to isolate it. -*/ - -package serialize - -import ( - "encoding/json" - "fmt" - - engineTypes "github.com/kwilteam/kwil-db/pkg/engine/types" - "github.com/kwilteam/kwil-db/pkg/engine/utils" - "github.com/kwilteam/kwil-db/pkg/tx" -) - -func DeserializeSchema(bts []byte) (*engineTypes.Schema, error) { - schma := &Schema{} - err := json.Unmarshal(bts, schma) - if err != nil { - return nil, err - } - - return convertSchema(schma) -} - -func DeserializeDBID(bts []byte) (string, error) { - di := &DatasetIdentifier{} - err := json.Unmarshal(bts, di) - if err != nil { - return "", err - } - - return utils.GenerateDBID(di.Name, di.Owner), nil -} - -func DeserializeActionPayload(payload []byte) (*tx.ExecuteActionPayload, error) { - exec := tx.ExecuteActionPayload{} - - err := json.Unmarshal(payload, &exec) - if err != nil { - return nil, fmt.Errorf("failed to unmarshal action execution: %w", err) - } - - return &exec, nil -} diff --git a/pkg/_tx/message.go b/pkg/_tx/message.go deleted file mode 100644 index 139a5ad39..000000000 --- a/pkg/_tx/message.go +++ /dev/null @@ -1,131 +0,0 @@ -package tx - -import ( - "crypto" - "encoding/json" - - kwilCrypto "github.com/kwilteam/kwil-db/pkg/crypto" - "github.com/kwilteam/kwil-db/pkg/engine/types" -) - -// SignedMessage is a signed message. -// This was made after Transaction, and is made to be more general. -// Unlike Transaction, SignedMessage contains a deserialized payload -type SignedMessage[T Serializable] struct { - Payload T // we use generic here to give access to the underlying struct fields - Signature *kwilCrypto.Signature - Sender string -} - -type Serializable interface { - Bytes() ([]byte, error) -} - -type Verifiable interface { - Verify() error -} - -func (s *SignedMessage[T]) generateHash() ([]byte, error) { - data, err := s.Payload.Bytes() - if err != nil { - return nil, err - } - - return kwilCrypto.Sha384(data), nil -} - -func (s *SignedMessage[T]) Verify() error { - hash, err := s.generateHash() - if err != nil { - return err - } - - return s.Signature.Check(s.Sender, hash) -} - -// CreateSignedMessage creates and signs a SignedMessage -func CreateSignedMessage[T Serializable](message T, privateKey crypto.PrivateKey) (*SignedMessage[T], error) { - msg := &SignedMessage[T]{ - Payload: message, - } - - hash, err := msg.generateHash() - if err != nil { - return nil, err - } - - msg.Sender = kwilCrypto.AddressFromPrivateKey(privateKey) - - sig, err := kwilCrypto.Sign(hash, privateKey) - if err != nil { - return nil, err - } - - msg.Signature = sig - - return msg, nil -} - -// CreateEmptySignedMessage creates a SignedMessage and does not sign it -func CreateEmptySignedMessage[T Serializable](payload T) *SignedMessage[T] { - return &SignedMessage[T]{ - Payload: payload, - Signature: &kwilCrypto.Signature{}, - Sender: "", - } -} - -// AccountTransaction is a struct that is used when a signed message should alter the state of an account -type AccountTransaction[T any] struct{} - -// CallActionPayload is a struct that represents the action call -type CallActionPayload struct { - Action string `json:"action"` - DBID string `json:"dbid"` - Params map[string]any `json:"params"` -} - -func (c *CallActionPayload) Bytes() ([]byte, error) { - return json.Marshal(c) -} - -type JsonPayload []byte - -func (j JsonPayload) Bytes() ([]byte, error) { - return j, nil -} - -// ExecuteActionPayload is a struct that represents the action execution -type ExecuteActionPayload struct { - Action string `json:"action"` - DBID string `json:"dbid"` - Params []map[string]any `json:"params"` -} - -func (e *ExecuteActionPayload) Bytes() ([]byte, error) { - return json.Marshal(e) -} - -func UnmarshalExecuteAction(b []byte) (*ExecuteActionPayload, error) { - var exec ExecuteActionPayload - err := json.Unmarshal(b, &exec) - if err != nil { - return nil, err - } - return &exec, nil -} - -type DatasetIdentifierPayload struct { - Owner string `json:"owner"` - Name string `json:"name"` -} - -func (d *DatasetIdentifierPayload) Bytes() ([]byte, error) { - return json.Marshal(d) -} - -type DeployDatabasePayload types.Schema - -func (d *DeployDatabasePayload) Bytes() ([]byte, error) { - return json.Marshal(d) -} diff --git a/pkg/_tx/payload.go b/pkg/_tx/payload.go deleted file mode 100644 index 0b279b842..000000000 --- a/pkg/_tx/payload.go +++ /dev/null @@ -1,52 +0,0 @@ -package tx - -import ( - "fmt" -) - -type PayloadType int32 - -const ( - INVALID_PAYLOAD_TYPE PayloadType = iota + 100 - DEPLOY_DATABASE - DROP_DATABASE - EXECUTE_ACTION - VALIDATOR_JOIN - VALIDATOR_LEAVE - VALIDATOR_APPROVE - END_PAYLOAD_TYPE -) - -func (x PayloadType) String() string { - switch x { - case INVALID_PAYLOAD_TYPE: - return "INVALID_PAYLOAD_TYPE" - case DEPLOY_DATABASE: - return "DEPLOY_DATABASE" - case DROP_DATABASE: - return "DROP_DATABASE" - case EXECUTE_ACTION: - return "EXECUTE_QUERY" - case END_PAYLOAD_TYPE: - return "END_PAYLOAD_TYPE" - case VALIDATOR_JOIN: - return "VALIDATOR_JOIN" - case VALIDATOR_LEAVE: - return "VALIDATOR_LEAVE" - case VALIDATOR_APPROVE: - return "VALIDATOR_APPROVE" - default: - return fmt.Sprintf("PayloadType(%d)", x) - } -} - -func (x PayloadType) IsValid() error { - if x < INVALID_PAYLOAD_TYPE || x >= END_PAYLOAD_TYPE { - return fmt.Errorf("invalid payload type '%d'", x) - } - return nil -} - -func (x PayloadType) Int32() int32 { - return int32(x) -} diff --git a/pkg/_tx/response.go b/pkg/_tx/response.go deleted file mode 100644 index ea08eae9a..000000000 --- a/pkg/_tx/response.go +++ /dev/null @@ -1,9 +0,0 @@ -package tx - -import "math/big" - -// ExecutionResponse is the response from any interaction that modifies state. -type ExecutionResponse struct { - // Fee is the amount of tokens spent on the execution - Fee *big.Int -} diff --git a/pkg/_tx/sign_test.go b/pkg/_tx/sign_test.go deleted file mode 100644 index 7bd1e1d9c..000000000 --- a/pkg/_tx/sign_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package tx_test - -import ( - "testing" - - "github.com/kwilteam/kwil-db/pkg/crypto" - "github.com/kwilteam/kwil-db/pkg/tx" -) - -const defaultPrivateKey = "f1aa5a7966c3863ccde3047f6a1e266cdc0c76b399e256b8fede92b1c69e4f4e" - -func Test_Sign(t *testing.T) { - type testCase struct { - name string - payload tx.Serializable - signer string - checkedSigner string - wantErr bool - } - - testCases := []testCase{ - { - name: "call payload", - payload: &tx.CallActionPayload{ - Action: "get_post", - DBID: "dbid", - Params: map[string]any{ - "$id": 1, - }, - }, - signer: defaultPrivateKey, - checkedSigner: defaultPrivateKey, - wantErr: false, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - signer, err := crypto.ECDSAFromHex(tc.signer) - if err != nil { - t.Fatalf("failed to create private key: %s", err.Error()) - } - - checkSigner, err := crypto.ECDSAFromHex(tc.checkedSigner) - if err != nil { - t.Fatalf("failed to create private key: %s", err.Error()) - } - - signedMsg, err := tx.CreateSignedMessage(tc.payload, signer) - if err != nil { - t.Fatalf("failed to sign payload: %s", err.Error()) - } - - signedMsg.Sender = crypto.AddressFromPrivateKey(checkSigner) - - err = signedMsg.Verify() - if tc.wantErr { - if err == nil { - t.Errorf("expected error, got nil") - } - } else { - if err != nil { - t.Errorf("expected no error, got %s", err.Error()) - } - } - }) - - } -} diff --git a/pkg/_tx/tx.go b/pkg/_tx/tx.go deleted file mode 100644 index 9d48ee1e6..000000000 --- a/pkg/_tx/tx.go +++ /dev/null @@ -1,101 +0,0 @@ -package tx - -import ( - "bytes" - "crypto/ecdsa" - "encoding/binary" - "encoding/hex" - "encoding/json" - "fmt" - - kwilCrypto "github.com/kwilteam/kwil-db/pkg/crypto" -) - -type Transaction struct { - Hash []byte `json:"hash"` - PayloadType PayloadType `json:"payload_type"` - Payload []byte `json:"payload"` - Fee string `json:"fee"` - Nonce int64 `json:"nonce"` - Signature *kwilCrypto.Signature `json:"signature"` - Sender string `json:"sender"` -} - -func NewTx(txType PayloadType, data []byte, nonce int64) *Transaction { - return &Transaction{ - PayloadType: txType, - Payload: data, - Fee: "0", - Nonce: nonce, - } -} - -func (t *Transaction) Verify() error { - if !bytes.Equal(t.Hash, t.GenerateHash()) { - return fmt.Errorf("invalid hash. received %s, expected %s", hex.EncodeToString(t.Hash), hex.EncodeToString(t.GenerateHash())) - } - - // verify valid payload type - if t.PayloadType <= INVALID_PAYLOAD_TYPE || t.PayloadType >= END_PAYLOAD_TYPE { - return fmt.Errorf("invalid payload type") - } - - // Not returning this function directly since I want specific error messages. - err := t.Signature.Check(t.Sender, t.Hash) - if err != nil { - return fmt.Errorf("failed to verify signed message: %v", err) - } - - return nil -} - -func (t *Transaction) Bytes() ([]byte, error) { - return json.Marshal(t) -} - -// generateHash generates a hash of the transaction -// it does this by hashing the payload type, payload, fee, and nonce -func (t *Transaction) GenerateHash() []byte { - var data []byte - - // convert payload type to bytes - payloadTypeBytes := make([]byte, 4) - binary.LittleEndian.PutUint32(payloadTypeBytes, uint32(t.PayloadType)) - data = append(data, payloadTypeBytes...) - - // hash payload - payloadHash := kwilCrypto.Sha384(t.Payload) - data = append(data, payloadHash...) - - // add fee - data = append(data, []byte(t.Fee)...) - - // convert nonce to bytes - nonceBytes := make([]byte, 8) - binary.LittleEndian.PutUint64(nonceBytes, uint64(t.Nonce)) - data = append(data, nonceBytes...) - - return kwilCrypto.Sha384(data) -} - -func (t *Transaction) Sign(p *ecdsa.PrivateKey) error { - hash := t.GenerateHash() - sig, err := kwilCrypto.Sign(hash, p) - if err != nil { - return fmt.Errorf("failed to sign transaction: %v", err) - } - - address := kwilCrypto.AddressFromPrivateKey(p) - - t.Hash = hash - t.Signature = sig - t.Sender = address - - return nil -} - -type Receipt struct { - TxHash []byte `json:"tx_hash"` - Fee string `json:"fee"` - Body []byte `json:"body"` -} From 661278404e905df5ed0e3950ea6fe80a9f2185ce Mon Sep 17 00:00:00 2001 From: brennan lamey Date: Mon, 14 Aug 2023 10:14:25 -0500 Subject: [PATCH 8/8] removed old packages --- pkg/balances/_chain-syncer/builder.go | 134 -------------- pkg/balances/_chain-syncer/empty.go | 13 -- pkg/balances/_chain-syncer/sync.go | 232 ------------------------- pkg/utils/_serialize/serialize.go | 81 --------- pkg/utils/_serialize/serialize_test.go | 62 ------- 5 files changed, 522 deletions(-) delete mode 100644 pkg/balances/_chain-syncer/builder.go delete mode 100644 pkg/balances/_chain-syncer/empty.go delete mode 100644 pkg/balances/_chain-syncer/sync.go delete mode 100644 pkg/utils/_serialize/serialize.go delete mode 100644 pkg/utils/_serialize/serialize_test.go diff --git a/pkg/balances/_chain-syncer/builder.go b/pkg/balances/_chain-syncer/builder.go deleted file mode 100644 index 9e877e0aa..000000000 --- a/pkg/balances/_chain-syncer/builder.go +++ /dev/null @@ -1,134 +0,0 @@ -package chainsyncer - -import ( - "fmt" - - "github.com/kwilteam/kwil-db/pkg/chain/contracts/escrow" - "github.com/kwilteam/kwil-db/pkg/log" -) - -// trying a new builder pattern type here -// reference can be found at https://devcharmander.medium.com/design-patterns-in-golang-the-builder-dac468a71194 - -// post-mortem: I like it because it is pretty readable and gives a nice API, however unlike the options pattern, -// it doesn't show what is required until you call build, which is a bit annoying. However, given the amount of -// configuration that is required (and the fact that the consumer will probably want to use most optional configs in ChainSyncer), -// I think it is a good tradeoff. - -func Builder() *ChainSyncBuilder { - return &ChainSyncBuilder{ - syncer: &ChainSyncer{ - log: log.NewNoOp(), - height: default_start_height, - chunkSize: default_chunk_size, - receiverAddress: "", - escrowAddress: "", - }, - } -} - -type ChainSyncBuilder struct { - syncer *ChainSyncer -} - -type ChainSyncAccountRepoBuilder struct { - *ChainSyncBuilder -} - -type ChainSyncChainClientBuilder struct { - *ChainSyncBuilder -} - -// WithLogger sets the logger for the chain syncer -func (c *ChainSyncBuilder) WithLogger(logger log.Logger) *ChainSyncBuilder { - c.syncer.log = logger - return c -} - -// WritesTo specifies the account repository that the chain syncer will write to -func (c *ChainSyncBuilder) WritesTo(repo accountRepository) *ChainSyncAccountRepoBuilder { - c.syncer.accountRepository = repo - return &ChainSyncAccountRepoBuilder{c} -} - -// ListensTo specifies the address of the escrow contract that the chain syncer will listen to -func (c *ChainSyncBuilder) ListensTo(address string) *ChainSyncChainClientBuilder { - c.syncer.escrowAddress = address - return &ChainSyncChainClientBuilder{c} -} - -// WithChainClient sets the chain client that the chain syncer will use -func (c *ChainSyncChainClientBuilder) WithChainClient(client chainClient) *ChainSyncChainClientBuilder { - c.syncer.chainClient = client - return c -} - -// WithStartingHeight sets the starting height of the chain syncer -func (c *ChainSyncChainClientBuilder) WithStartingHeight(height int64) *ChainSyncChainClientBuilder { - c.syncer.height = height - return c -} - -// WithChunkSize sets the chunk size of the chain syncer -func (c *ChainSyncChainClientBuilder) WithChunkSize(size int64) *ChainSyncChainClientBuilder { - if size <= 0 { - panic("chunk size must be greater than 0") - } - - c.syncer.chunkSize = size - return c -} - -// WithReceiverAddress sets the receiver address of the chain syncer -func (c *ChainSyncChainClientBuilder) WithReceiverAddress(address string) *ChainSyncChainClientBuilder { - c.syncer.receiverAddress = address - return c -} - -// Build builds the chain syncer -func (c *ChainSyncBuilder) Build() (*ChainSyncer, error) { - if c.syncer.accountRepository == nil { - return nil, fmt.Errorf("account repository not set") - } - if c.syncer.chainClient == nil { - return nil, fmt.Errorf("chain client not set") - } - if c.syncer.escrowAddress == "" { - return nil, fmt.Errorf("escrow address not set") - } - if c.syncer.receiverAddress == "" { - return nil, fmt.Errorf("deposit receiver address not set") - } - - escrowCtr, err := c.syncer.chainClient.Contracts().Escrow(c.syncer.escrowAddress, escrow.WithLogger(c.syncer.log)) - if err != nil { - return nil, err - } - - c.syncer.escrowContract = escrowCtr - c.syncer.chainCode = c.syncer.chainClient.ChainCode() - - err = c.ensureChainExistsInDB(c.syncer.chainCode.Int32(), c.syncer.height) - if err != nil { - return nil, err - } - - return c.syncer, nil -} - -// ensureChainExistsInDB ensures that the chain exists in the database -func (c *ChainSyncBuilder) ensureChainExistsInDB(chainCode int32, height int64) error { - exists, err := c.syncer.accountRepository.ChainExists(chainCode) - if err != nil { - return err - } - - if !exists { - err = c.syncer.accountRepository.CreateChain(chainCode, height) - if err != nil { - return err - } - } - - return nil -} diff --git a/pkg/balances/_chain-syncer/empty.go b/pkg/balances/_chain-syncer/empty.go deleted file mode 100644 index 906800d0f..000000000 --- a/pkg/balances/_chain-syncer/empty.go +++ /dev/null @@ -1,13 +0,0 @@ -package chainsyncer - -import "context" - -type EmptyChainsyncer struct{} - -func (c *EmptyChainsyncer) Start(ctx context.Context) error { - return nil -} - -func NewEmptyChainSyncer() *EmptyChainsyncer { - return &EmptyChainsyncer{} -} diff --git a/pkg/balances/_chain-syncer/sync.go b/pkg/balances/_chain-syncer/sync.go deleted file mode 100644 index b57eb3f38..000000000 --- a/pkg/balances/_chain-syncer/sync.go +++ /dev/null @@ -1,232 +0,0 @@ -package chainsyncer - -import ( - "context" - "fmt" - "math/big" - "time" - - "github.com/kwilteam/kwil-db/pkg/balances" - "github.com/kwilteam/kwil-db/pkg/chain/contracts" - "github.com/kwilteam/kwil-db/pkg/chain/contracts/escrow" - provider "github.com/kwilteam/kwil-db/pkg/chain/provider/dto" - chainCodes "github.com/kwilteam/kwil-db/pkg/chain/types" - "github.com/kwilteam/kwil-db/pkg/log" - - "go.uber.org/zap" -) - -const ( - default_start_height = 0 - default_chunk_size = 100000 -) - -type ChainSyncer struct { - log log.Logger - - // accountRepository is the interface that is used to interact with the account store - accountRepository accountRepository - - // chainClient is the client that is used to interact with the chain - chainClient chainClient - - // height is the height of the last block that has been synced - height int64 - - // chunkSize is the number of blocks that are synced in a single batch - chunkSize int64 - - // chainCode is the chain code of the chain that is being synced - chainCode chainCodes.ChainCode - - // escrowContract is the escrow contract interface - escrowContract escrow.EscrowContract - - // escrowAddress is the address of the escrow contract - escrowAddress string - - // receiverAddress is the address of the deposit receiver - // this will almost always be the address of the node's wallet - receiverAddress string -} - -type accountRepository interface { - BatchCredit([]*balances.Credit, *balances.ChainConfig) error - GetHeight(int32) (int64, error) - CreateChain(chainCode int32, height int64) error - ChainExists(chainCode int32) (bool, error) -} - -type chainClient interface { - ChainCode() chainCodes.ChainCode - GetLatestBlock(ctx context.Context) (*provider.Header, error) - Contracts() contracts.Contracter - Listen(ctx context.Context, blocks chan<- int64) error -} - -// Start starts the chain syncer -func (cs *ChainSyncer) Start(ctx context.Context) error { - var err error - cs.height, err = cs.getStartingHeight() - if err != nil { - return err - } - - latestBlock, err := cs.chainClient.GetLatestBlock(ctx) - if err != nil { - return err - } - - cs.log.Info("Starting sync from height", zap.Int64("starting_height", cs.height), zap.Int64("latest_height", latestBlock.Height)) - chunks := splitBlocks(cs.height, latestBlock.Height, cs.chunkSize) - - for _, chunk := range chunks { - err = cs.syncChunk(ctx, chunk) - if err != nil { - return err - } - - time.Sleep(100 * time.Millisecond) // alchemy doesn't like it when we make too many requests in a short period of time - } - - return cs.listen(ctx) -} - -// getLastHeight retrieves the last synced height from the account repository -func (cs *ChainSyncer) getLastHeight() (int64, error) { - return cs.accountRepository.GetHeight(cs.chainCode.Int32()) -} - -// getStartingHeight will return either the last synced height or the configured start height, -// depending on which is greater -func (cs *ChainSyncer) getStartingHeight() (int64, error) { - lastHeight, err := cs.getLastHeight() - if err != nil { - return 0, err - } - - if lastHeight > cs.height { - return lastHeight, nil - } - - return cs.height, nil -} - -type chunkRange [2]int64 - -func splitBlocks(start, end, chunkSize int64) []chunkRange { - if start == end { - return []chunkRange{{start, start}} - } - var chunks []chunkRange - for i := start; i < end; i += chunkSize { - chunkEnd := i + chunkSize - if chunkEnd > end { - chunkEnd = end - } - chunks = append(chunks, chunkRange{i, chunkEnd - 1}) - } - - if len(chunks) == 0 { - return []chunkRange{{start, end}} - } - - if chunks[len(chunks)-1][1] != end { - chunks[len(chunks)-1][1] = end - } - return chunks -} - -// getCreditsForRange retrieves all deposits for a given range of blocks, and returns them as credits -func (cs *ChainSyncer) getCreditsForRange(ctx context.Context, start, end int64) ([]*balances.Credit, error) { - deposits, err := cs.escrowContract.GetDeposits(ctx, start, end, cs.receiverAddress) - if err != nil { - return nil, err - } - - credits := make([]*balances.Credit, len(deposits)) - for i, deposit := range deposits { - bigAmount, ok := new(big.Int).SetString(deposit.Amount, 10) - if !ok { - return nil, fmt.Errorf("failed to parse amount %s", deposit.Amount) - } - - credits[i] = &balances.Credit{ - AccountAddress: deposit.Caller, - Amount: bigAmount, - } - } - - return credits, nil -} - -// syncChunk syncs a chunk of blocks -func (cs *ChainSyncer) syncChunk(ctx context.Context, chunk chunkRange) error { - if ctx.Err() != nil { - return ctx.Err() - } - - cs.log.Debug("Syncing chunk", zap.Int64("start", chunk[0]), zap.Int64("end", chunk[1])) - - credits, err := cs.getCreditsForRange(ctx, chunk[0], chunk[1]) - if err != nil { - return err - } - - err = cs.accountRepository.BatchCredit(credits, &balances.ChainConfig{ - ChainCode: cs.chainCode.Int32(), - Height: chunk[1], - }) - if err != nil { - return err - } - - cs.log.Debug("Synced chunk", zap.Int64("start", chunk[0]), zap.Int64("end", chunk[1]), zap.Int64("deposits in chunk", int64(len(credits)))) - return nil -} - -// listen listens for new blocks on the chain and syncs them as they come in -func (cs *ChainSyncer) listen(ctx context.Context) error { - blockChan := make(chan int64) - err := cs.chainClient.Listen(ctx, blockChan) - if err != nil { - return err - } - - go func(blockChan <-chan int64) { - defer func() { - if err := recover(); err != nil { - cs.log.Error("Chain syncer panic", zap.Any("error", err)) - } - cs.log.Info("Chain syncer stopped") - }() - for { - select { - case <-ctx.Done(): - cs.log.Info("stop Chain syncer") - return - case block := <-blockChan: - cs.log.Debug("Received block", zap.Int64("block", block)) - - credits, err := cs.getCreditsForRange(ctx, block, block) - if err != nil { - cs.log.Error("Failed to get credits for block", zap.Int64("block", block), zap.Error(err)) - return - } - - cs.log.Debug("Syncing deposits from block", zap.Int64("block", block), zap.Int64("number of deposits:", int64(len(credits)))) - - err = cs.accountRepository.BatchCredit(credits, &balances.ChainConfig{ - ChainCode: cs.chainCode.Int32(), - Height: block, - }) - if err != nil { - cs.log.Error("Failed to credit block", zap.Int64("block", block), zap.Error(err)) - return - } - } - } - }(blockChan) - - return nil -} diff --git a/pkg/utils/_serialize/serialize.go b/pkg/utils/_serialize/serialize.go deleted file mode 100644 index 5630f6051..000000000 --- a/pkg/utils/_serialize/serialize.go +++ /dev/null @@ -1,81 +0,0 @@ -package serialize - -import ( - "encoding/json" - "reflect" -) - -type Marshaler[T interface{}] struct { - KnownImplementations []T -} - -func (m Marshaler[T]) Marshal(v T) ([]byte, error) { - return json.Marshal(container[T]{ - Value: v, - KnownImplementations: m.KnownImplementations, - }) -} - -func (m Marshaler[T]) Unmarshal(bytes []byte) (T, error) { - - var v T - cont := container[T]{ - Value: v, - KnownImplementations: m.KnownImplementations, - } - - if err := json.Unmarshal(bytes, &cont); err != nil { - return v, err - } - - return cont.Value, nil -} - -type container[T interface{}] struct { - Value T `json:"value"` - KnownImplementations []T -} - -// exporting this since it gets called recursively -func (c *container[T]) UnmarshalJSON(bytes []byte) error { - var data struct { - Type string - Value json.RawMessage - } - if err := json.Unmarshal(bytes, &data); err != nil { - return err - } - - for _, knownImplementation := range c.KnownImplementations { - knownType := reflect.TypeOf(knownImplementation) - if knownType.String() == data.Type { - // Create a new pointer to a value of the concrete message type - target := reflect.New(knownType) - - if err := c.UnmarshalJSON(data.Value); err != nil { - return err - } - - // Unmarshal the data to an interface to the concrete value (which will act as a pointer, don't ask why) - if err := json.Unmarshal(data.Value, target.Interface()); err != nil { - return err - } - // Now we get the element value of the target and convert it to the interface type (this is to get rid of a pointer type instead of a plain struct value) - c.Value = target.Elem().Interface().(T) - return nil - } - } - - return nil -} - -func (c container[T]) MarshalJSON() ([]byte, error) { - // Marshal to type and actual data to handle unmarshaling to specific interface type - return json.Marshal(struct { - Type string - Value any - }{ - Type: reflect.TypeOf(c.Value).String(), - Value: c.Value, - }) -} diff --git a/pkg/utils/_serialize/serialize_test.go b/pkg/utils/_serialize/serialize_test.go deleted file mode 100644 index 5b1b1d11c..000000000 --- a/pkg/utils/_serialize/serialize_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package serialize_test - -import ( - "testing" - - "github.com/kwilteam/kwil-db/pkg/utils/serialize" -) - -func Test_Serialize(t *testing.T) { - marshaler := serialize.Marshaler[MyInter]{ - KnownImplementations: []MyInter{ - MyStruct{}, - MyStruct2{}, - }, - } - - marshaler2 := serialize.Marshaler[MyInter]{ - KnownImplementations: []MyInter{ - MyStruct{}, - MyStruct2{}, - }, - } - - bytes, err := marshaler.Marshal(MyStruct2{Val: "testy", Inter: MyStruct{Val: "testy2"}}) - if err != nil { - t.Fatal(err) - } - - retVal, err := marshaler2.Unmarshal(bytes) - if err != nil { - t.Fatal(err) - } - - if retVal.(MyStruct2).Val != "testy" { - t.Fatal("wrong value") - } - - if retVal.(MyStruct2).Inter.(MyStruct).Val != "testy2" { - t.Fatal("wrong value") - } -} - -type MyInter interface { - Do() -} - -type MyStruct struct { - Val string `json:"val"` -} - -func (m MyStruct) Do() { - println("do") -} - -type MyStruct2 struct { - Val string `json:"val"` - Inter MyInter `json:"inter"` -} - -func (m MyStruct2) Do() { - println("do") -}