Skip to content

Commit

Permalink
add kvstore (#1004)
Browse files Browse the repository at this point in the history
* add kvstore

* use a fork of substrate rpc client

* fix lint

* use client decode function

* move decodeSecondKey to utils.go
  • Loading branch information
Eslam-Nawara authored Sep 16, 2024
1 parent ea6fdf3 commit 10a6bc1
Show file tree
Hide file tree
Showing 7 changed files with 249 additions and 586 deletions.
4 changes: 2 additions & 2 deletions clients/tfchain-client-go/account.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ type AccountID types.AccountID
// Balance
type Balance struct {
Free types.U128 `json:"free"`
Reserved types.U128 `json:"reserverd"`
Reserved types.U128 `json:"reserved"`
MiscFrozen types.U128 `json:"misc_frozen"`
FreeFrozen types.U128 `json:"free_frozen"`
}
Expand All @@ -54,7 +54,7 @@ type AccountInfo struct {

// TODO: Add service response status code if avialble
type ActivationServiceError struct {
StatusCode int // 0 or a valid HTTP status code. 0 indicates that a response was not recived due to an error such host unreachable.
StatusCode int // 0 or a valid HTTP status code. 0 indicates that a response was not received due to an error such host unreachable.
Err error
}

Expand Down
20 changes: 12 additions & 8 deletions clients/tfchain-client-go/go.mod
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
module github.com/threefoldtech/tfchain/clients/tfchain-client-go

go 1.17
go 1.21

require (
github.com/cenkalti/backoff v2.2.1+incompatible
github.com/centrifuge/go-substrate-rpc-client/v4 v4.0.12
github.com/jbenet/go-base58 v0.0.0-20150317085156-6237cf65f3a6
github.com/pkg/errors v0.9.1
github.com/rs/zerolog v1.26.0
github.com/stretchr/testify v1.7.0
github.com/stretchr/testify v1.7.2
github.com/vedhavyas/go-subkey v1.0.3
golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871
golang.org/x/crypto v0.7.0
)

require (
Expand All @@ -19,20 +19,24 @@ require (
github.com/cosmos/go-bip39 v1.0.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/deckarep/golang-set v1.8.0 // indirect
github.com/decred/base58 v1.0.3 // indirect
github.com/decred/base58 v1.0.4 // indirect
github.com/decred/dcrd/crypto/blake256 v1.0.0 // indirect
github.com/ethereum/go-ethereum v1.10.17 // indirect
github.com/ethereum/go-ethereum v1.10.20 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/go-stack/stack v1.8.1 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
github.com/gtank/merlin v0.1.1 // indirect
github.com/gtank/ristretto255 v0.1.2 // indirect
github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect
github.com/pierrec/xxHash v0.1.5 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rs/cors v1.8.2 // indirect
github.com/tklauser/go-sysconf v0.3.9 // indirect
golang.org/x/sys v0.0.0-20211124211545-fe61309f8881 // indirect
github.com/vedhavyas/go-subkey/v2 v2.0.0 // indirect
golang.org/x/sys v0.6.0 // indirect
gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

replace github.com/centrifuge/go-substrate-rpc-client/v4 => github.com/threefoldtech/go-substrate-rpc-client/v4 v4.0.0-20240916115924-b6e9bfa88b8a
595 changes: 25 additions & 570 deletions clients/tfchain-client-go/go.sum

Large diffs are not rendered by default.

133 changes: 133 additions & 0 deletions clients/tfchain-client-go/kvstore.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package substrate

import (
"github.com/centrifuge/go-substrate-rpc-client/v4/types"

"github.com/pkg/errors"
)

func (s *Substrate) KVStoreSet(identity Identity, key string, value string) error {
cl, meta, err := s.GetClient()
if err != nil {
return err
}

c, err := types.NewCall(meta, "TFKVStore.set",
key, value,
)
if err != nil {
return errors.Wrap(err, "failed to create call")
}

res, err := s.Call(cl, meta, identity, c)
if err != nil {
return errors.Wrap(err, "failed to create contract")
}

if err := s.checkForError(res); err != nil {
return err
}

return nil
}

func (s *Substrate) KVStoreDelete(identity Identity, key string) error {
cl, meta, err := s.GetClient()
if err != nil {
return err
}

c, err := types.NewCall(meta, "TFKVStore.delete",
key,
)
if err != nil {
return errors.Wrap(err, "failed to create call")
}

res, err := s.Call(cl, meta, identity, c)
if err != nil {
return errors.Wrap(err, "failed to create contract")
}

if err := s.checkForError(res); err != nil {
return err
}
return nil
}

func (s *Substrate) KVStoreGet(identity Identity, key string) ([]byte, error) {
cl, meta, err := s.GetClient()
if err != nil {
return nil, err
}

bytes, err := Encode(key)
if err != nil {
return nil, err
}

storageKey, err := types.CreateStorageKey(meta, "TFKVStore", "TFKVStore", identity.PublicKey(), bytes)
if err != nil {
return nil, errors.Wrap(err, "failed to create substrate query key")
}

var value []byte
ok, err := cl.RPC.State.GetStorageLatest(storageKey, &value)
if err != nil {
return nil, errors.Wrap(err, "failed to lookup entity")
}

if !ok {
return nil, errors.Wrap(ErrNotFound, "key not found")
}

return value, nil
}

type Val struct {
Id string
Key string
}

func (s *Substrate) KVStoreList(identity Identity) (map[string]string, error) {
cl, meta, err := s.GetClient()
if err != nil {
return nil, err
}

storageKey, err := types.CreateStorageKey(meta, "TFKVStore", "TFKVStore", identity.PublicKey())
if err != nil {
return nil, errors.Wrap(err, "failed to create substrate query key")
}

keys, err := cl.RPC.State.GetKeysLatest(storageKey)
if err != nil {
return nil, errors.Wrap(err, "failed to lookup entity")
}

query, err := cl.RPC.State.QueryStorageAtLatest(keys)
if err != nil {
return nil, err
}

pairs := make(map[string]string)
for _, q := range query {
for _, c := range q.Changes {
var key, val []byte

key, err = decodeSecondKey(c.StorageKey, identity)
if err != nil {
return nil, err
}

err = Decode(c.StorageData, &val)
if err != nil {
return nil, errors.Wrapf(err, "failed to decode value %v", string(c.StorageData))
}

pairs[string(key)] = string(val)
}
}

return pairs, nil
}
51 changes: 51 additions & 0 deletions clients/tfchain-client-go/kvstore_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
package substrate

import (
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestKVStore(t *testing.T) {
pairs := map[string]string{"key1": "value1", "key2": "value2", "key3": "value3"}

// Alice identity
id, err := NewIdentityFromSr25519Phrase("//Alice")
require.NoError(t, err)

sub := startLocalConnection(t)
defer sub.Close()

t.Run("kvstore set values", func(t *testing.T) {
for key, val := range pairs {
err = sub.KVStoreSet(id, key, val)
assert.NoError(t, err)
}
})

t.Run("kvstore get values", func(t *testing.T) {
for key, val := range pairs {
value, err := sub.KVStoreGet(id, key)
assert.NoError(t, err)
assert.Equal(t, []byte(val), value)
}
})

t.Run("kvstore list keys", func(t *testing.T) {
values, err := sub.KVStoreList(id)
assert.NoError(t, err)
assert.EqualValues(t, values, pairs)
})

t.Run("kvstore delete", func(t *testing.T) {
for key := range pairs {
err = sub.KVStoreDelete(id, key)
assert.NoError(t, err)
}

values, err := sub.KVStoreList(id)
assert.NoError(t, err)
assert.Empty(t, values)
})
}
2 changes: 1 addition & 1 deletion clients/tfchain-client-go/service_contract_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ func TestServiceContract(t *testing.T) {
})

// 4. Bill consumer for service contract
// should be able to go to future block to test varaible amount greater than 0
// should be able to go to future block to test variable amount greater than 0

err = cl.ServiceContractBill(serviceIdentity, serviceContractID, variableAmount, billMetadata)
require.NoError(t, err)
Expand Down
30 changes: 25 additions & 5 deletions clients/tfchain-client-go/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,6 @@ func (s *Substrate) sign(e *types.Extrinsic, signer Identity, o types.SignatureO
}

signerPubKey, err := types.NewMultiAddressFromAccountID(signer.PublicKey())

if err != nil {
return err
}
Expand All @@ -346,7 +345,6 @@ func (s *Substrate) sign(e *types.Extrinsic, signer Identity, o types.SignatureO
}

sig, err := signer.Sign(b)

if err != nil {
return err
}
Expand Down Expand Up @@ -412,7 +410,7 @@ func (s *Substrate) CallOnce(cl Conn, meta Meta, identity Identity, call types.C
return hash, err
}

//node.Address =identity.PublicKey
// node.Address =identity.PublicKey
account, err := s.getAccount(cl, meta, identity)
if err != nil {
return hash, errors.Wrap(err, "failed to get account")
Expand Down Expand Up @@ -536,15 +534,37 @@ func (s *Substrate) checkForError(callResponse *CallResponse) error {
if *accId == who {
if int(e.DispatchError.ModuleError.Index) < len(moduleErrors) {
if int(errIndex) >= len(moduleErrors[e.DispatchError.ModuleError.Index]) || moduleErrors[e.DispatchError.ModuleError.Index] == nil {
return fmt.Errorf("module error (%d) with unknown code %d occured, please update the module error list", e.DispatchError.ModuleError.Index, e.DispatchError.ModuleError.Error)
return fmt.Errorf("module error (%d) with unknown code %d occurred, please update the module error list", e.DispatchError.ModuleError.Index, e.DispatchError.ModuleError.Error)
}
return errors.New(moduleErrors[e.DispatchError.ModuleError.Index][errIndex])
} else {
return fmt.Errorf("unknown module error (%d) with code %d occured, please create the module error list", e.DispatchError.ModuleError.Index, e.DispatchError.ModuleError.Error)
return fmt.Errorf("unknown module error (%d) with code %d occurred, please create the module error list", e.DispatchError.ModuleError.Index, e.DispatchError.ModuleError.Error)
}
}
}
}

return nil
}

// decodeSecondKey extracts and decodes the second key(Vec<u8>) from the storage key.
func decodeSecondKey(storageKey types.StorageKey, identity Identity) (key []byte, err error) {
// remove 16 bytes(32 in hex) pallet and map prefixes.
// pallet prefix (8 bytes): twox64(pallet_name)
// map prefix (8bytes) twox64(map_name)
prefixLen := 32

// the storage key contains two keys (AccountID and Vec<u8>)
// remove the length of the first key(AccountID)
// the hasher `Blake2_128Concat` includes a 16-byte hash followed by the AccountID
firstKeyLen := 32 + len(identity.PublicKey())

offset := prefixLen + firstKeyLen

if len(storageKey) < offset {
return nil, errors.New(fmt.Sprintf("failed to decode second key, storage key len should not be less than %d bytes", offset))
}

err = Decode(storageKey[offset:], &key)
return key, err
}

0 comments on commit 10a6bc1

Please sign in to comment.