From 811ffb79112992ca423c8c957eaf127ad4874208 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 14:36:55 -0400 Subject: [PATCH 01/10] sharding: begin shard local storage --- sharding/database/database.go | 21 +++++++++++++++++++++ sharding/interfaces.go | 10 ++++++++++ sharding/notary/service.go | 1 + sharding/shard.go | 9 --------- 4 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 sharding/database/database.go diff --git a/sharding/database/database.go b/sharding/database/database.go new file mode 100644 index 000000000000..fe2bb7438090 --- /dev/null +++ b/sharding/database/database.go @@ -0,0 +1,21 @@ +package database + +import ( + "path/filepath" + + "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/sharding" + "github.com/micro/cli" +) + +// CreateShardDB initializes a shardDB that writes to local disk. +func CreateShardDB(ctx *cli.Context, name string) (sharding.ShardBackend, error) { + + dataDir := ctx.GlobalString(utils.DataDir.Name) + path := filepath.Join(dataDir, name) + + // Uses default cache and handles values. + // TODO: fix interface. + return ethdb.NewLDBDatabase(path, 16, 16) +} diff --git a/sharding/interfaces.go b/sharding/interfaces.go index a75a04ca6256..8795e8eb6b8d 100644 --- a/sharding/interfaces.go +++ b/sharding/interfaces.go @@ -1,6 +1,7 @@ package sharding import ( + "github.com/ethereum/go-ethereum/common" cli "gopkg.in/urfave/cli.v1" ) @@ -22,3 +23,12 @@ type Service interface { // ServiceConstructor defines the callback passed in when registering a service // to a sharding node. type ServiceConstructor func(ctx *cli.Context) (Service, error) + +// ShardBackend defines an interface for a shardDB's necessary method +// signatures. +type ShardBackend interface { + Get(k common.Hash) (*[]byte, error) + Has(k common.Hash) bool + Put(k common.Hash, val []byte) error + Delete(k common.Hash) error +} diff --git a/sharding/notary/service.go b/sharding/notary/service.go index f1123f2985d0..7281be0cf26a 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -3,6 +3,7 @@ package notary import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/sharding/node" + cli "gopkg.in/urfave/cli.v1" ) diff --git a/sharding/shard.go b/sharding/shard.go index 3408fbe75c28..9e84ca0f10d6 100644 --- a/sharding/shard.go +++ b/sharding/shard.go @@ -9,15 +9,6 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// ShardBackend defines an interface for a shardDB's necessary method -// signatures. -type ShardBackend interface { - Get(k common.Hash) (*[]byte, error) - Has(k common.Hash) bool - Put(k common.Hash, val []byte) error - Delete(k common.Hash) error -} - // Shard base struct. type Shard struct { shardDB ShardBackend From 7f3811ebfbd15eeb2855064ca0f32ccd0748cc37 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Tue, 22 May 2018 14:49:59 -0400 Subject: [PATCH 02/10] sharding: using eth leveldb, interface mismatch --- sharding/database/database.go | 9 +++++---- sharding/notary/service.go | 14 ++++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/sharding/database/database.go b/sharding/database/database.go index fe2bb7438090..ee1aebc23c18 100644 --- a/sharding/database/database.go +++ b/sharding/database/database.go @@ -9,13 +9,14 @@ import ( "github.com/micro/cli" ) -// CreateShardDB initializes a shardDB that writes to local disk. -func CreateShardDB(ctx *cli.Context, name string) (sharding.ShardBackend, error) { +// NewShardDB initializes a shardDB that writes to local disk. +func NewShardDB(ctx *cli.Context, name string) (sharding.ShardBackend, error) { - dataDir := ctx.GlobalString(utils.DataDir.Name) + dataDir := ctx.GlobalString(utils.DataDirFlag.Name) path := filepath.Join(dataDir, name) // Uses default cache and handles values. - // TODO: fix interface. + // TODO: allow these to be set based on cli context. + // TODO: fix interface - lots of methods do not match. return ethdb.NewLDBDatabase(path, 16, 16) } diff --git a/sharding/notary/service.go b/sharding/notary/service.go index 7281be0cf26a..2f52e3e8cf8c 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -2,6 +2,8 @@ package notary import ( "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/sharding" + "github.com/ethereum/go-ethereum/sharding/database" "github.com/ethereum/go-ethereum/sharding/node" cli "gopkg.in/urfave/cli.v1" @@ -11,12 +13,20 @@ import ( // in a sharded system. Must satisfy the Service interface defined in // sharding/service.go. type Notary struct { - node node.Node + node node.Node + shardDB sharding.ShardBackend } // NewNotary creates a new notary instance. func NewNotary(ctx *cli.Context, node node.Node) (*Notary, error) { - return &Notary{node}, nil + // Initializes a shardDB that writes to disk at /path/to/datadir/shardchaindata. + // This DB can be used by the Notary service to create Shard struct + // instances. + shardDB, err := database.NewShardDB(node.Context(), "shardchaindata") + if err != nil { + return nil, err + } + return &Notary{node, shardDB}, nil } // Start the main routine for a notary. From eff22d315f4680427f453b5c9df6df9fd7762717 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 24 May 2018 17:36:20 -0600 Subject: [PATCH 03/10] sharding: fix datadir and import cycle --- sharding/database/database.go | 9 ++------- sharding/node/node.go | 6 ++++++ sharding/notary/service.go | 8 ++++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/sharding/database/database.go b/sharding/database/database.go index d61a7b01525a..a97d04cfabf9 100644 --- a/sharding/database/database.go +++ b/sharding/database/database.go @@ -5,7 +5,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/micro/cli" ) // ShardBackend defines an interface for a shardDB's necessary method @@ -19,13 +18,9 @@ type ShardBackend interface { // NewShardDB initializes a shardDB that writes to local disk. // TODO: make it return ShardBackend but modify interface methods. -func NewShardDB(ctx *cli.Context, name string) (*ethdb.LDBDatabase, error) { - - dataDir := "" - path := filepath.Join(dataDir, name) - +func NewShardDB(dataDir string, name string) (*ethdb.LDBDatabase, error) { // Uses default cache and handles values. // TODO: allow these to be set based on cli context. // TODO: fix interface - lots of methods do not match. - return ethdb.NewLDBDatabase(path, 16, 16) + return ethdb.NewLDBDatabase(filepath.Join(dataDir, name), 16, 16) } diff --git a/sharding/node/node.go b/sharding/node/node.go index a5eec20fe45d..4a370e955eb9 100644 --- a/sharding/node/node.go +++ b/sharding/node/node.go @@ -44,6 +44,7 @@ type Node interface { SMCCaller() *contracts.SMCCaller SMCTransactor() *contracts.SMCTransactor DepositFlagSet() bool + DataDirFlag() string } // General node for a sharding-enabled system. @@ -213,6 +214,11 @@ func (n *shardingNode) DepositFlagSet() bool { return n.ctx.GlobalBool(utils.DepositFlag.Name) } +// DataDirFlag returns the datadir flag as a string. +func (n *shardingNode) DataDirFlag() string { + return n.ctx.GlobalString(utils.DataDirFlag.Name) +} + // Client to interact with a geth node via JSON-RPC. func (n *shardingNode) ethereumClient() *ethclient.Client { return n.client diff --git a/sharding/notary/service.go b/sharding/notary/service.go index e08c827134d1..bbd5d00c43c9 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -2,7 +2,6 @@ package notary import ( "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/sharding" "github.com/ethereum/go-ethereum/sharding/database" "github.com/ethereum/go-ethereum/sharding/node" ) @@ -12,7 +11,7 @@ import ( // sharding/service.go. type Notary struct { node node.Node - shardDB sharding.ShardBackend + shardDB database.ShardBackend } // NewNotary creates a new notary instance. @@ -20,11 +19,12 @@ func NewNotary(node node.Node) (*Notary, error) { // Initializes a shardDB that writes to disk at /path/to/datadir/shardchaindata. // This DB can be used by the Notary service to create Shard struct // instances. - shardDB, err := database.NewShardDB(node.Context(), "shardchaindata") + shardDB, err := database.NewShardDB(node.DataDirFlag(), "shardchaindata") if err != nil { return nil, err } - return &Notary{node, shardDB}, nil + // return &Notary{node, shardDB}, nil + return &Notary{node: node}, nil } // Start the main routine for a notary. From 7377447126298c70ba93320f501fd0e0829315a9 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 24 May 2018 18:03:24 -0600 Subject: [PATCH 04/10] sharding: generalized shardbackend, refactor database package and tests --- sharding/database/database.go | 11 +++--- sharding/database/inmemory.go | 24 ++++++------- sharding/database/inmemory_test.go | 54 ++++++++++++++++-------------- 3 files changed, 46 insertions(+), 43 deletions(-) diff --git a/sharding/database/database.go b/sharding/database/database.go index a97d04cfabf9..9e30b5fa92a5 100644 --- a/sharding/database/database.go +++ b/sharding/database/database.go @@ -3,22 +3,21 @@ package database import ( "path/filepath" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" ) // ShardBackend defines an interface for a shardDB's necessary method // signatures. type ShardBackend interface { - Get(k common.Hash) (*[]byte, error) - Has(k common.Hash) bool - Put(k common.Hash, val []byte) error - Delete(k common.Hash) error + Get(k []byte) ([]byte, error) + Has(k []byte) (bool, error) + Put(k []byte, val []byte) error + Delete(k []byte) error } // NewShardDB initializes a shardDB that writes to local disk. // TODO: make it return ShardBackend but modify interface methods. -func NewShardDB(dataDir string, name string) (*ethdb.LDBDatabase, error) { +func NewShardDB(dataDir string, name string) (ShardBackend, error) { // Uses default cache and handles values. // TODO: allow these to be set based on cli context. // TODO: fix interface - lots of methods do not match. diff --git a/sharding/database/inmemory.go b/sharding/database/inmemory.go index 6d7ca35fe0e6..515bcf976176 100644 --- a/sharding/database/inmemory.go +++ b/sharding/database/inmemory.go @@ -12,48 +12,48 @@ import ( // ShardKV is an in-memory mapping of hashes to RLP encoded values. type ShardKV struct { - kv map[common.Hash]*[]byte + kv map[common.Hash][]byte lock sync.RWMutex } // NewShardKV initializes a keyval store in memory. func NewShardKV() *ShardKV { - return &ShardKV{kv: make(map[common.Hash]*[]byte)} + return &ShardKV{kv: make(map[common.Hash][]byte)} } // Get fetches a val from the mappping by key. -func (sb *ShardKV) Get(k common.Hash) (*[]byte, error) { +func (sb *ShardKV) Get(k []byte) ([]byte, error) { sb.lock.RLock() defer sb.lock.RUnlock() - v, ok := sb.kv[k] + v, ok := sb.kv[common.BytesToHash(k)] if !ok { - return nil, fmt.Errorf("key not found: %v", k) + return []byte{}, fmt.Errorf("key not found: %v", k) } return v, nil } // Has checks if the key exists in the mapping. -func (sb *ShardKV) Has(k common.Hash) bool { +func (sb *ShardKV) Has(k []byte) (bool, error) { sb.lock.RLock() defer sb.lock.RUnlock() - v := sb.kv[k] - return v != nil + v := sb.kv[common.BytesToHash(k)] + return v != nil, nil } // Put updates a key's value in the mapping. -func (sb *ShardKV) Put(k common.Hash, v []byte) error { +func (sb *ShardKV) Put(k []byte, v []byte) error { sb.lock.Lock() defer sb.lock.Unlock() // there is no error in a simple setting of a value in a go map. - sb.kv[k] = &v + sb.kv[common.BytesToHash(k)] = v return nil } // Delete removes the key and value from the mapping. -func (sb *ShardKV) Delete(k common.Hash) error { +func (sb *ShardKV) Delete(k []byte) error { sb.lock.Lock() defer sb.lock.Unlock() // There is no return value for deleting a simple key in a go map. - delete(sb.kv, k) + delete(sb.kv, common.BytesToHash(k)) return nil } diff --git a/sharding/database/inmemory_test.go b/sharding/database/inmemory_test.go index 2f9a54dae92a..4043c20d4496 100644 --- a/sharding/database/inmemory_test.go +++ b/sharding/database/inmemory_test.go @@ -2,73 +2,77 @@ package database import ( "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/sharding" ) // Verifies that ShardKV implements the ShardBackend interface. -var _ = sharding.ShardBackend(&ShardKV{}) +var _ = ShardBackend(&ShardKV{}) func Test_ShardKVPut(t *testing.T) { kv := NewShardKV() - hash := common.BytesToHash([]byte("ralph merkle")) - if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { + if err := kv.Put([]byte("ralph merkle"), []byte{1, 2, 3}); err != nil { t.Errorf("could not save value in kv store: %v", err) } } func Test_ShardKVHas(t *testing.T) { kv := NewShardKV() - hash := common.BytesToHash([]byte("ralph merkle")) + key := []byte("ralph merkle") - if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { + if err := kv.Put(key, []byte{1, 2, 3}); err != nil { t.Fatalf("could not save value in kv store: %v", err) } - if !kv.Has(hash) { - t.Errorf("kv store does not have hash: %v", hash) + has, err := kv.Has(key) + if err != nil { + t.Errorf("could not check if kv store has key: %v", err) + } + if !has { + t.Errorf("kv store should have key: %v", key) } - hash2 := common.BytesToHash([]byte{}) - if kv.Has(hash2) { - t.Errorf("kv store should not contain unset key: %v", hash2) + key2 := []byte{} + has2, err := kv.Has(key2) + if err != nil { + t.Errorf("could not check if kv store has key: %v", err) + } + if has2 { + t.Errorf("kv store should not have non-existent key: %v", key2) } } func Test_ShardKVGet(t *testing.T) { kv := NewShardKV() - hash := common.BytesToHash([]byte("ralph merkle")) + key := []byte("ralph merkle") - if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { + if err := kv.Put(key, []byte{1, 2, 3}); err != nil { t.Fatalf("could not save value in kv store: %v", err) } - val, err := kv.Get(hash) + val, err := kv.Get(key) if err != nil { t.Errorf("get failed: %v", err) } - if val == nil { + if len(val) == 0 { t.Errorf("no value stored for key") } - hash2 := common.BytesToHash([]byte{}) - val2, err := kv.Get(hash2) - if val2 != nil { - t.Errorf("non-existent key should not have a value. key=%v, value=%v", hash2, val2) + key2 := []byte{} + val2, err := kv.Get(key2) + if len(val2) != 0 { + t.Errorf("non-existent key should not have a value. key=%v, value=%v", key2, val2) } } func Test_ShardKVDelete(t *testing.T) { kv := NewShardKV() - hash := common.BytesToHash([]byte("ralph merkle")) + key := []byte("ralph merkle") - if err := kv.Put(hash, []byte{1, 2, 3}); err != nil { + if err := kv.Put(key, []byte{1, 2, 3}); err != nil { t.Fatalf("could not save value in kv store: %v", err) } - if err := kv.Delete(hash); err != nil { - t.Errorf("could not delete key: %v", hash) + if err := kv.Delete(key); err != nil { + t.Errorf("could not delete key: %v", key) } } From 594db620c8ceac9ec7d924833ea429e2f26c1580 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Thu, 24 May 2018 18:09:42 -0600 Subject: [PATCH 05/10] sharding: refactor shard.go for new interface changes --- sharding/shard.go | 31 +++++++++++++++---------------- sharding/shard_test.go | 5 +++-- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/sharding/shard.go b/sharding/shard.go index 66a90d043f4b..ca2905ee8b61 100644 --- a/sharding/shard.go +++ b/sharding/shard.go @@ -39,17 +39,17 @@ func (s *Shard) ValidateShardID(h *CollationHeader) error { // HeaderByHash looks up a collation header from the shardDB using the header's hash. func (s *Shard) HeaderByHash(hash *common.Hash) (*CollationHeader, error) { - encoded, err := s.shardDB.Get(*hash) + encoded, err := s.shardDB.Get(hash.Bytes()) if err != nil { return nil, fmt.Errorf("get failed: %v", err) } - if encoded == nil { + if len(encoded) == 0 { return nil, fmt.Errorf("no value set for header hash: %vs", hash.Hex()) } var header CollationHeader - stream := rlp.NewStream(bytes.NewReader(*encoded), uint64(len(*encoded))) + stream := rlp.NewStream(bytes.NewReader(encoded), uint64(len(encoded))) if err := header.DecodeRLP(stream); err != nil { return nil, fmt.Errorf("could not decode RLP header: %v", err) } @@ -80,18 +80,18 @@ func (s *Shard) CanonicalHeaderHash(shardID *big.Int, period *big.Int) (*common. key := canonicalCollationLookupKey(shardID, period) // fetches the RLP encoded collation header corresponding to the key. - encoded, err := s.shardDB.Get(key) + encoded, err := s.shardDB.Get(key.Bytes()) if err != nil { return nil, err } - if encoded == nil { + if len(encoded) == 0 { return nil, fmt.Errorf("no canonical collation header set for period=%v, shardID=%v pair: %v", shardID, period, err) } // RLP decodes the header, computes its hash. var header CollationHeader - stream := rlp.NewStream(bytes.NewReader(*encoded), uint64(len(*encoded))) + stream := rlp.NewStream(bytes.NewReader(encoded), uint64(len(encoded))) if err := header.DecodeRLP(stream); err != nil { return nil, fmt.Errorf("could not decode RLP header: %v", err) } @@ -112,27 +112,26 @@ func (s *Shard) CanonicalCollation(shardID *big.Int, period *big.Int) (*Collatio // BodyByChunkRoot fetches a collation body. func (s *Shard) BodyByChunkRoot(chunkRoot *common.Hash) ([]byte, error) { - body, err := s.shardDB.Get(*chunkRoot) + body, err := s.shardDB.Get(chunkRoot.Bytes()) if err != nil { return nil, err } - if body == nil { + if len(body) == 0 { return nil, fmt.Errorf("no corresponding body with chunk root found: %s", chunkRoot) } - return *body, nil + return body, nil } // CheckAvailability is used by notaries to confirm a header's data availability. func (s *Shard) CheckAvailability(header *CollationHeader) (bool, error) { key := dataAvailabilityLookupKey(header.ChunkRoot()) - val, err := s.shardDB.Get(key) + availability, err := s.shardDB.Get(key.Bytes()) if err != nil { return false, err } - if val == nil { + if len(availability) == 0 { return false, fmt.Errorf("availability not set for header") } - availability := *val // availability is a byte array of length 1. return availability[0] != 0, nil } @@ -146,7 +145,7 @@ func (s *Shard) SetAvailability(chunkRoot *common.Hash, availability bool) error } else { encoded = []byte{0} } - return s.shardDB.Put(key, encoded) + return s.shardDB.Put(key.Bytes(), encoded) } // SaveHeader adds the collation header to shardDB. @@ -162,7 +161,7 @@ func (s *Shard) SaveHeader(header *CollationHeader) error { } // uses the hash of the header as the key. - return s.shardDB.Put(header.Hash(), encoded) + return s.shardDB.Put(header.Hash().Bytes(), encoded) } // SaveBody adds the collation body to the shardDB and sets availability. @@ -173,7 +172,7 @@ func (s *Shard) SaveBody(body []byte) error { // right now we will just take the raw keccak256 of the body until #92 is merged. chunkRoot := common.BytesToHash(body) s.SetAvailability(&chunkRoot, true) - return s.shardDB.Put(chunkRoot, body) + return s.shardDB.Put(chunkRoot.Bytes(), body) } // SaveCollation adds the collation's header and body to shardDB. @@ -214,7 +213,7 @@ func (s *Shard) SetCanonical(header *CollationHeader) error { } // sets the key to be the canonical collation lookup key and val as RLP encoded // collation header. - return s.shardDB.Put(key, encoded) + return s.shardDB.Put(key.Bytes(), encoded) } // dataAvailabilityLookupKey formats a string that will become a lookup diff --git a/sharding/shard_test.go b/sharding/shard_test.go index e4258c38d280..32ccc462a387 100644 --- a/sharding/shard_test.go +++ b/sharding/shard_test.go @@ -16,8 +16,9 @@ type mockShardDB struct { kv map[common.Hash]*[]byte } -func (m *mockShardDB) Get(k common.Hash) (*[]byte, error) { - return nil, nil +// TOOD: FINISH MOCK CLIENT +func (m *mockShardDB) Get(k common.Hash) ([]byte, error) { + return []byte{, nil } func (m *mockShardDB) Has(k common.Hash) bool { From f6f70806a8b3dbaf1dd11c69848464386c0d7470 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Fri, 25 May 2018 09:06:39 -0600 Subject: [PATCH 06/10] sharding: refactor all tests, travis passes --- sharding/database/database.go | 4 +--- sharding/shard.go | 2 +- sharding/shard_test.go | 26 +++++++++++++------------- 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/sharding/database/database.go b/sharding/database/database.go index 9e30b5fa92a5..a06b34f18fae 100644 --- a/sharding/database/database.go +++ b/sharding/database/database.go @@ -16,10 +16,8 @@ type ShardBackend interface { } // NewShardDB initializes a shardDB that writes to local disk. -// TODO: make it return ShardBackend but modify interface methods. func NewShardDB(dataDir string, name string) (ShardBackend, error) { // Uses default cache and handles values. - // TODO: allow these to be set based on cli context. - // TODO: fix interface - lots of methods do not match. + // TODO: allow these arguments to be set based on cli context. return ethdb.NewLDBDatabase(filepath.Join(dataDir, name), 16, 16) } diff --git a/sharding/shard.go b/sharding/shard.go index ca2905ee8b61..218cbfbcc533 100644 --- a/sharding/shard.go +++ b/sharding/shard.go @@ -114,7 +114,7 @@ func (s *Shard) CanonicalCollation(shardID *big.Int, period *big.Int) (*Collatio func (s *Shard) BodyByChunkRoot(chunkRoot *common.Hash) ([]byte, error) { body, err := s.shardDB.Get(chunkRoot.Bytes()) if err != nil { - return nil, err + return []byte{}, err } if len(body) == 0 { return nil, fmt.Errorf("no corresponding body with chunk root found: %s", chunkRoot) diff --git a/sharding/shard_test.go b/sharding/shard_test.go index 32ccc462a387..58b6823e238f 100644 --- a/sharding/shard_test.go +++ b/sharding/shard_test.go @@ -13,23 +13,23 @@ import ( ) type mockShardDB struct { - kv map[common.Hash]*[]byte + kv map[common.Hash][]byte } // TOOD: FINISH MOCK CLIENT -func (m *mockShardDB) Get(k common.Hash) ([]byte, error) { - return []byte{, nil +func (m *mockShardDB) Get(k []byte) ([]byte, error) { + return []byte{}, nil } -func (m *mockShardDB) Has(k common.Hash) bool { - return false +func (m *mockShardDB) Has(k []byte) (bool, error) { + return false, nil } -func (m *mockShardDB) Put(k common.Hash, v []byte) error { +func (m *mockShardDB) Put(k []byte, v []byte) error { return fmt.Errorf("error updating db") } -func (m *mockShardDB) Delete(k common.Hash) error { +func (m *mockShardDB) Delete(k []byte) error { return fmt.Errorf("error deleting value in db") } @@ -65,7 +65,7 @@ func TestShard_HeaderByHash(t *testing.T) { header := NewCollationHeader(big.NewInt(1), &emptyHash, big.NewInt(1), &emptyAddr, []byte{}) // creates a mockDB that always returns nil values from .Get and errors in every other method. - mockDB := &mockShardDB{kv: make(map[common.Hash]*[]byte)} + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} // creates a well-functioning shardDB. shardDB := database.NewShardKV() @@ -296,10 +296,10 @@ func TestShard_BodyByChunkRoot(t *testing.T) { } // setting the val of the key to nil. - if err := shard.shardDB.Put(emptyHash, nil); err != nil { + if err := shard.shardDB.Put([]byte{}, nil); err != nil { t.Fatalf("could not update shardDB: %v", err) } - if _, err := shard.BodyByChunkRoot(&emptyHash); err != nil { + if _, err := shard.BodyByChunkRoot(&emptyHash); err == nil { t.Errorf("value set as nil in shardDB should return error from BodyByChunkRoot") } @@ -347,7 +347,7 @@ func TestShard_SetAvailability(t *testing.T) { header := NewCollationHeader(big.NewInt(1), &chunkRoot, big.NewInt(1), nil, []byte{}) // creates a mockDB that always returns nil values from .Get and errors in every other method. - mockDB := &mockShardDB{kv: make(map[common.Hash]*[]byte)} + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} // creates a well-functioning shardDB. shardDB := database.NewShardKV() @@ -402,7 +402,7 @@ func TestShard_SaveCollation(t *testing.T) { func TestShard_SaveHeader(t *testing.T) { // creates a mockDB that always returns nil values from .Get and errors in every other method. - mockDB := &mockShardDB{kv: make(map[common.Hash]*[]byte)} + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} emptyHash := common.BytesToHash([]byte{}) errorShard := NewShard(big.NewInt(1), mockDB) @@ -414,7 +414,7 @@ func TestShard_SaveHeader(t *testing.T) { func TestShard_SaveBody(t *testing.T) { // creates a mockDB that always returns nil values from .Get and errors in every other method. - mockDB := &mockShardDB{kv: make(map[common.Hash]*[]byte)} + mockDB := &mockShardDB{kv: make(map[common.Hash][]byte)} errorShard := NewShard(big.NewInt(1), mockDB) if err := errorShard.SaveBody([]byte{1, 2, 3}); err == nil { From 7d95123ac2450e1d8d02e35673860b09de9885ba Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Fri, 25 May 2018 09:52:18 -0600 Subject: [PATCH 07/10] sharding: tests pass, services updated to include shardDB --- sharding/database/database_test.go | 92 ++++++++++++++++++++++++++++++ sharding/notary/service.go | 3 +- sharding/proposer/service.go | 10 +++- 3 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 sharding/database/database_test.go diff --git a/sharding/database/database_test.go b/sharding/database/database_test.go new file mode 100644 index 000000000000..80e4409f6a94 --- /dev/null +++ b/sharding/database/database_test.go @@ -0,0 +1,92 @@ +package database + +import ( + "strconv" + "testing" +) + +var db ShardBackend + +func init() { + shardDB, err := NewShardDB("/tmp/datadir", "shardchaindata") + if err != nil { + panic(err) + } + db = shardDB +} + +// Testing the concurrency of the shardDB with multiple processes attempting to write. +func Test_DBConcurrent(t *testing.T) { + for i := 0; i < 100; i++ { + go func(val string) { + if err := db.Put([]byte("ralph merkle"), []byte(val)); err != nil { + t.Errorf("could not save value in db: %v", err) + } + }(strconv.Itoa(i)) + } +} + +func Test_DBPut(t *testing.T) { + if err := db.Put([]byte("ralph merkle"), []byte{1, 2, 3}); err != nil { + t.Errorf("could not save value in db: %v", err) + } +} + +func Test_DBHas(t *testing.T) { + key := []byte("ralph merkle") + + if err := db.Put(key, []byte{1, 2, 3}); err != nil { + t.Fatalf("could not save value in db: %v", err) + } + + has, err := db.Has(key) + if err != nil { + t.Errorf("could not check if db has key: %v", err) + } + if !has { + t.Errorf("db should have key: %v", key) + } + + key2 := []byte{} + has2, err := db.Has(key2) + if err != nil { + t.Errorf("could not check if db has key: %v", err) + } + if has2 { + t.Errorf("db should not have non-existent key: %v", key2) + } +} + +func Test_DBGet(t *testing.T) { + key := []byte("ralph merkle") + + if err := db.Put(key, []byte{1, 2, 3}); err != nil { + t.Fatalf("could not save value in db: %v", err) + } + + val, err := db.Get(key) + if err != nil { + t.Errorf("get failed: %v", err) + } + if len(val) == 0 { + t.Errorf("no value stored for key") + } + + key2 := []byte{} + val2, err := db.Get(key2) + if len(val2) != 0 { + t.Errorf("non-existent key should not have a value. key=%v, value=%v", key2, val2) + } +} + +func Test_DBDelete(t *testing.T) { + key := []byte("ralph merkle") + + if err := db.Put(key, []byte{1, 2, 3}); err != nil { + t.Fatalf("could not save value in db: %v", err) + } + + if err := db.Delete(key); err != nil { + t.Errorf("could not delete key: %v", key) + } +} diff --git a/sharding/notary/service.go b/sharding/notary/service.go index bbd5d00c43c9..bfd0e0f82232 100644 --- a/sharding/notary/service.go +++ b/sharding/notary/service.go @@ -23,8 +23,7 @@ func NewNotary(node node.Node) (*Notary, error) { if err != nil { return nil, err } - // return &Notary{node, shardDB}, nil - return &Notary{node: node}, nil + return &Notary{node, shardDB}, nil } // Start the main routine for a notary. diff --git a/sharding/proposer/service.go b/sharding/proposer/service.go index afa490bbaee4..f1dd818283ae 100644 --- a/sharding/proposer/service.go +++ b/sharding/proposer/service.go @@ -2,6 +2,7 @@ package proposer import ( "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/sharding/database" "github.com/ethereum/go-ethereum/sharding/node" ) @@ -9,14 +10,19 @@ import ( // in a sharded system. Must satisfy the Service interface defined in // sharding/service.go. type Proposer struct { - node node.Node + node node.Node + shardDB database.ShardBackend } // NewProposer creates a struct instance. It is initialized and // registered as a service upon start of a sharding node. // Has access to the public methods of this node. func NewProposer(node node.Node) (*Proposer, error) { - return &Proposer{node}, nil + shardDB, err := database.NewShardDB(node.DataDirFlag(), "shardchaindata") + if err != nil { + return nil, err + } + return &Proposer{node, shardDB}, nil } // Start the main loop for proposing collations. From 73cad370aa0c233cf64bc0514bd09ceb5f7e22df Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Fri, 25 May 2018 20:16:29 -0600 Subject: [PATCH 08/10] sharding: revert master changes - travis was broken --- sharding/proposer/service.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sharding/proposer/service.go b/sharding/proposer/service.go index f1dd818283ae..da672f34b269 100644 --- a/sharding/proposer/service.go +++ b/sharding/proposer/service.go @@ -18,6 +18,7 @@ type Proposer struct { // registered as a service upon start of a sharding node. // Has access to the public methods of this node. func NewProposer(node node.Node) (*Proposer, error) { + // Initializes a shardchaindata directory persistent db. shardDB, err := database.NewShardDB(node.DataDirFlag(), "shardchaindata") if err != nil { return nil, err From a07cf375262041a572ba41103962816e48e0fa78 Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Fri, 25 May 2018 21:19:04 -0600 Subject: [PATCH 09/10] sharding: fix comment --- sharding/shard_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/sharding/shard_test.go b/sharding/shard_test.go index 58b6823e238f..49247fcda278 100644 --- a/sharding/shard_test.go +++ b/sharding/shard_test.go @@ -16,7 +16,6 @@ type mockShardDB struct { kv map[common.Hash][]byte } -// TOOD: FINISH MOCK CLIENT func (m *mockShardDB) Get(k []byte) ([]byte, error) { return []byte{}, nil } From cd6dd8fd1c77bc93913a09bcd96f441b6e2e6e3c Mon Sep 17 00:00:00 2001 From: Raul Jordan Date: Sat, 26 May 2018 12:25:43 -0600 Subject: [PATCH 10/10] sharding: fix lint --- sharding/notary/service_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/sharding/notary/service_test.go b/sharding/notary/service_test.go index 52fb00e37882..683ef9c63756 100644 --- a/sharding/notary/service_test.go +++ b/sharding/notary/service_test.go @@ -63,6 +63,10 @@ func (m *mockNode) DepositFlagSet() bool { return m.DepositFlag } +func (m *mockNode) DataDirFlag() string { + return "/tmp/datadir" +} + // Unused mockClient methods. func (m *mockNode) Start() error { m.t.Fatal("Start called")