From f1c53488b9b0f1f558fa129b97a45c2cba4387bd Mon Sep 17 00:00:00 2001 From: Terence Tsao Date: Tue, 22 May 2018 20:36:38 -0700 Subject: [PATCH 01/10] sharding/contracts: tests for event logs --- sharding/contracts/sharding_manager_test.go | 64 ++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/sharding/contracts/sharding_manager_test.go b/sharding/contracts/sharding_manager_test.go index 0ca772287b27..bad4d7431982 100644 --- a/sharding/contracts/sharding_manager_test.go +++ b/sharding/contracts/sharding_manager_test.go @@ -92,6 +92,21 @@ func (s *smcTestHelper) registerNotaries(deposit *big.Int, params ...int) error "Got - deposited:%v, index:%v, period:%v ", i, notary.Deposited, notary.PoolIndex, notary.DeregisteredPeriod) } } + // Filter SMC logs by notaryRegistered. + log, err := s.smc.FilterNotaryRegistered(&bind.FilterOpts{}) + if err != nil { + return err + } + // Iterate notaryRegistered logs, compare each address and poolIndex. + for i := 0; i < params[1]; i++ { + log.Next() + if log.Event.Notary != s.testAccounts[i].addr { + return fmt.Errorf("incorrect address in notaryRegistered log. Want: %v Got: %v", s.testAccounts[i].addr, log.Event.Notary) + } + if log.Event.PoolIndex.Cmp(big.NewInt(int64(i))) != 0 { + return fmt.Errorf("incorrect index in notaryRegistered log. Want: %v Got: %v", i, log.Event.Notary) + } + } return nil } @@ -110,6 +125,24 @@ func (s *smcTestHelper) deregisterNotaries(params ...int) error { return fmt.Errorf("Degistered period can not be 0 right after deregistration") } } + // Filter SMC logs by notaryDeregistered. + log, err := s.smc.FilterNotaryDeregistered(&bind.FilterOpts{}) + if err != nil { + return err + } + // Iterate notaryDeregistered logs, compare each address, poolIndex and verify period is set. + for i := 0; i < params[1]; i++ { + log.Next() + if log.Event.Notary != s.testAccounts[i].addr { + return fmt.Errorf("incorrect address in notaryDeregistered log. Want: %v Got: %v", s.testAccounts[i].addr, log.Event.Notary) + } + if log.Event.PoolIndex.Cmp(big.NewInt(int64(i))) != 0 { + return fmt.Errorf("incorrect index in notaryDeregistered log. Want: %v Got: %v", i, log.Event.Notary) + } + if log.Event.DeregisteredPeriod.Cmp(big.NewInt(0)) == 0 { + return fmt.Errorf("incorrect period in notaryDeregistered log. Got: %v", log.Event.DeregisteredPeriod) + } + } return nil } @@ -134,10 +167,25 @@ func (s *smcTestHelper) addHeader(a *testAccount, shard *big.Int, period *big.In if cr.ChunkRoot != [32]byte{chunkRoot} { return fmt.Errorf("Chunkroot mismatched. Want: %v, Got: %v", chunkRoot, cr) } + + // Filter SMC logs by headerAdded. + shardIndex := []*big.Int{shard} + logPeriod := uint64(period.Int64() * sharding.PeriodLength) + log, err := s.smc.FilterHeaderAdded(&bind.FilterOpts{Start: logPeriod}, shardIndex) + if err != nil { + return err + } + log.Next() + if log.Event.ProposerAddress != s.testAccounts[0].addr { + return fmt.Errorf("incorrect proposer address in addHeader log. Want: %v Got: %v", s.testAccounts[0].addr, log.Event.ProposerAddress) + } + if log.Event.ChunkRoot != [32]byte{chunkRoot} { + return fmt.Errorf("chunk root missmatch in addHeader log. Want: %v Got: %v", [32]byte{chunkRoot}, log.Event.ChunkRoot) + } return nil } -// addHeader is a helper function for notary to submit vote on a given header, +// submitVote is a helper function for notary to submit vote on a given header, // it also verifies vote is properly submitted and casted. func (s *smcTestHelper) submitVote(a *testAccount, shard *big.Int, period *big.Int, index *big.Int, chunkRoot uint8) error { _, err := s.smc.SubmitVote(a.txOpts, shard, period, index, [32]byte{chunkRoot}) @@ -153,6 +201,20 @@ func (s *smcTestHelper) submitVote(a *testAccount, shard *big.Int, period *big.I if !v { return fmt.Errorf("Notary's indexd bit did not cast to 1 in index %v", index) } + // Filter SMC logs by submitVote. + shardIndex := []*big.Int{shard} + logPeriod := uint64(period.Int64() * sharding.PeriodLength) + log, err := s.smc.FilterVoteSubmitted(&bind.FilterOpts{Start: logPeriod}, shardIndex) + if err != nil { + return err + } + log.Next() + if log.Event.NotaryAddress != a.addr { + return fmt.Errorf("incorrect notary address in submitVote log. Want: %v Got: %v", s.testAccounts[0].addr, a.addr) + } + if log.Event.ChunkRoot != [32]byte{chunkRoot} { + return fmt.Errorf("chunk root missmatch in submitVote log. Want: %v Got: %v", [32]byte{chunkRoot}, log.Event.ChunkRoot) + } return nil } From d5186573631f4691b155101c90d8832d2ef06b8f Mon Sep 17 00:00:00 2001 From: Terence Tsao Date: Tue, 22 May 2018 21:20:56 -0700 Subject: [PATCH 02/10] sharding/contracts: comment for comparison --- sharding/contracts/sharding_manager_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/sharding/contracts/sharding_manager_test.go b/sharding/contracts/sharding_manager_test.go index bad4d7431982..d4d0f4b958de 100644 --- a/sharding/contracts/sharding_manager_test.go +++ b/sharding/contracts/sharding_manager_test.go @@ -103,6 +103,7 @@ func (s *smcTestHelper) registerNotaries(deposit *big.Int, params ...int) error if log.Event.Notary != s.testAccounts[i].addr { return fmt.Errorf("incorrect address in notaryRegistered log. Want: %v Got: %v", s.testAccounts[i].addr, log.Event.Notary) } + // Verify notaryPoolIndex is incremental starting from 1st registered Notary. if log.Event.PoolIndex.Cmp(big.NewInt(int64(i))) != 0 { return fmt.Errorf("incorrect index in notaryRegistered log. Want: %v Got: %v", i, log.Event.Notary) } From a1941432f7e318eeb2552dcf554b2e53067588c1 Mon Sep 17 00:00:00 2001 From: Terence Tsao Date: Thu, 24 May 2018 14:58:34 -0700 Subject: [PATCH 03/10] sharding: Serialize function shouldn't take pointer receiver --- sharding/collation.go | 41 ++++++++++---------------------------- sharding/collation_test.go | 2 +- 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 3ce4b55e3387..6c9c45340b7c 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -111,26 +111,6 @@ func (c *Collation) CalculateChunkRoot() { c.header.data.ChunkRoot = &chunkRoot } -// CreateRawBlobs creates raw blobs from transactions. -func (c Collation) CreateRawBlobs() ([]*utils.RawBlob, error) { - - // It does not skip evm execution by default - blobs := make([]*utils.RawBlob, len(c.transactions)) - for i := 0; i < len(c.transactions); i++ { - - err := error(nil) - blobs[i], err = utils.NewRawBlob(c.transactions[i], false) - - if err != nil { - return nil, fmt.Errorf("Creation of raw blobs from transactions failed: %v", err) - } - - } - - return blobs, nil - -} - // ConvertBackToTx converts raw blobs back to their original transactions. func ConvertBackToTx(rawBlobs []utils.RawBlob) ([]*types.Transaction, error) { @@ -149,29 +129,28 @@ func ConvertBackToTx(rawBlobs []utils.RawBlob) ([]*types.Transaction, error) { } -// Serialize method serializes the collation body to a byte array. -func (c *Collation) Serialize() ([]byte, error) { - - blobs, err := c.CreateRawBlobs() +// Serialize method serializes the input transactions +// and returns the chunks in byte arrays. +func Serialize(txs []*types.Transaction) ([]byte, error) { - if err != nil { - return nil, fmt.Errorf("%v", err) + blobs := make([]*utils.RawBlob, len(txs)) + for i := 0; i < len(txs); i++ { + err := error(nil) + blobs[i], err = utils.NewRawBlob(txs[i], false) + if err != nil { + return nil, fmt.Errorf("%v", err) + } } - serializedTx, err := utils.Serialize(blobs) - if err != nil { return nil, fmt.Errorf("%v", err) } if int64(len(serializedTx)) > collationSizelimit { - return nil, fmt.Errorf("The serialized body exceeded the collation size limit: %v", serializedTx) - } return serializedTx, nil - } // Deserialize takes a byte array and converts its back to its original transactions. diff --git a/sharding/collation_test.go b/sharding/collation_test.go index c0e4cb7355b0..cca9d904d124 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -56,7 +56,7 @@ func TestSerialize_Deserialize(t *testing.T) { tx := c.transactions - results, err := c.Serialize() + results, err := Serialize(tx) if err != nil { t.Errorf("Unable to Serialize transactions, %v", err) From 5e0596bd7a89599a5b5afd5418c6a5d4fbe06109 Mon Sep 17 00:00:00 2001 From: Terence Tsao Date: Thu, 24 May 2018 17:18:55 -0700 Subject: [PATCH 04/10] sharding: changed names to be sharding specific --- sharding/collation.go | 11 ++++++----- sharding/collation_test.go | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/sharding/collation.go b/sharding/collation.go index 6c9c45340b7c..2275e89ab188 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -129,9 +129,9 @@ func ConvertBackToTx(rawBlobs []utils.RawBlob) ([]*types.Transaction, error) { } -// Serialize method serializes the input transactions -// and returns the chunks in byte arrays. -func Serialize(txs []*types.Transaction) ([]byte, error) { +// Serialize method serializes the input tx +// and returns the blobs in byte array. +func SerializeTxToBlob(txs []*types.Transaction) ([]byte, error) { blobs := make([]*utils.RawBlob, len(txs)) for i := 0; i < len(txs); i++ { @@ -153,8 +153,9 @@ func Serialize(txs []*types.Transaction) ([]byte, error) { return serializedTx, nil } -// Deserialize takes a byte array and converts its back to its original transactions. -func Deserialize(serialisedBlob []byte) (*[]*types.Transaction, error) { +// DeserializeBlobToTx takes byte array blob and converts it back +// to original txs and returns the txs in tx array. +func DeserializeBlobToTx(serialisedBlob []byte) (*[]*types.Transaction, error) { deserializedBlobs, err := utils.Deserialize(serialisedBlob) if err != nil { diff --git a/sharding/collation_test.go b/sharding/collation_test.go index cca9d904d124..f3c896e2bfef 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -56,13 +56,13 @@ func TestSerialize_Deserialize(t *testing.T) { tx := c.transactions - results, err := Serialize(tx) + results, err := SerializeTxToBlob(tx) if err != nil { t.Errorf("Unable to Serialize transactions, %v", err) } - deserializedTxs, err := Deserialize(results) + deserializedTxs, err := DeserializeBlobToTx(results) if err != nil { t.Errorf("Unable to deserialize collation body, %v", err) From 6ecf1249378619effb637901bcbd4df4f3fa5bff Mon Sep 17 00:00:00 2001 From: Terence Tsao Date: Thu, 24 May 2018 17:24:58 -0700 Subject: [PATCH 05/10] sharding: fixed starting name to comply with GoDoc --- sharding/collation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharding/collation.go b/sharding/collation.go index 2275e89ab188..e7ec5104a286 100644 --- a/sharding/collation.go +++ b/sharding/collation.go @@ -129,7 +129,7 @@ func ConvertBackToTx(rawBlobs []utils.RawBlob) ([]*types.Transaction, error) { } -// Serialize method serializes the input tx +// SerializeTxToBlob method serializes the input tx // and returns the blobs in byte array. func SerializeTxToBlob(txs []*types.Transaction) ([]byte, error) { From cd9cde1a03561829e85521bb2f5c515e4e2d3cf9 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Thu, 24 May 2018 20:45:50 -0400 Subject: [PATCH 06/10] Add basic benchmark tests --- sharding/collation_test.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index f3c896e2bfef..542e6b672fed 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -2,6 +2,7 @@ package sharding import ( "bytes" + "crypto/rand" "math/big" "reflect" "testing" @@ -126,3 +127,38 @@ func TestSerialize_Deserialize(t *testing.T) { func makeTxWithGasLimit(gl uint64) *types.Transaction { return types.NewTransaction(0 /*nonce*/, common.HexToAddress("0x0") /*to*/, nil /*amount*/, gl, nil /*gasPrice*/, nil /*data*/) } + +// BENCHMARK TESTS + +// Helper function to generate test that completes round trip transaction tests for a specific number of transactions. +func runBenchTest(b *testing.B, numTransactions int) { + var txs []*types.Transaction + for i := 0; i < 10; i++ { + data := make([]byte, 650) + rand.Read(data) + txs = append(txs, types.NewTransaction(0 /*nonce*/, common.HexToAddress("0x0") /*to*/, nil /*amount*/, 0 /*gasLimit*/, nil /*gasPrice*/, data)) + } + b.ResetTimer() + + for i := 0; i < b.N; i++ { + results, _ := SerializeTxToBlob(txs) + _, _ = DeserializeBlobToTx(results) + } + +} + +func BenchmarkSerialization10(b *testing.B) { + runBenchTest(b, 10) +} + +func BenchmarkSerialization100(b *testing.B) { + runBenchTest(b, 100) +} + +func BenchmarkSerialization1000(b *testing.B) { + runBenchTest(b, 1000) +} + +func BenchmarkSerialization10000(b *testing.B) { + runBenchTest(b, 10000) +} From 68aa0568b6425a99794efeb1d179d4dec6c30fa1 Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Fri, 25 May 2018 08:22:14 -0400 Subject: [PATCH 07/10] Sharding: update comment. --- sharding/collation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 542e6b672fed..5e9347e09fee 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -130,7 +130,7 @@ func makeTxWithGasLimit(gl uint64) *types.Transaction { // BENCHMARK TESTS -// Helper function to generate test that completes round trip transaction tests for a specific number of transactions. +// Helper function to generate test that completes round trip serialization tests for a specific number of transactions. func runBenchTest(b *testing.B, numTransactions int) { var txs []*types.Transaction for i := 0; i < 10; i++ { From 8bd3053bbd1f39d2f9448f37dd8e91eed44bd4de Mon Sep 17 00:00:00 2001 From: Preston Van Loon Date: Fri, 25 May 2018 08:23:14 -0400 Subject: [PATCH 08/10] Sharding: 10 -> numTransactions. Oops! --- sharding/collation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sharding/collation_test.go b/sharding/collation_test.go index 5e9347e09fee..297cdb3f235d 100644 --- a/sharding/collation_test.go +++ b/sharding/collation_test.go @@ -133,7 +133,7 @@ func makeTxWithGasLimit(gl uint64) *types.Transaction { // Helper function to generate test that completes round trip serialization tests for a specific number of transactions. func runBenchTest(b *testing.B, numTransactions int) { var txs []*types.Transaction - for i := 0; i < 10; i++ { + for i := 0; i < numTransactions; i++ { data := make([]byte, 650) rand.Read(data) txs = append(txs, types.NewTransaction(0 /*nonce*/, common.HexToAddress("0x0") /*to*/, nil /*amount*/, 0 /*gasLimit*/, nil /*gasPrice*/, data)) From 5c05da0376f75823900eda673781e5310e298930 Mon Sep 17 00:00:00 2001 From: Terence Tsao Date: Fri, 25 May 2018 10:20:51 -0700 Subject: [PATCH 09/10] sharding/contracts: hash digest instead of byte array --- sharding/contracts/sharding_manager_test.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/sharding/contracts/sharding_manager_test.go b/sharding/contracts/sharding_manager_test.go index d4d0f4b958de..ed94bf954efc 100644 --- a/sharding/contracts/sharding_manager_test.go +++ b/sharding/contracts/sharding_manager_test.go @@ -73,8 +73,7 @@ func (s *smcTestHelper) fastForward(p int) { } } -// registerNotaries is a helper function register notaries in batch, -// it also verifies notary registration is successful. +// registerNotaries is a helper function register notaries in batch. func (s *smcTestHelper) registerNotaries(deposit *big.Int, params ...int) error { for i := params[0]; i < params[1]; i++ { s.testAccounts[i].txOpts.Value = deposit @@ -111,8 +110,7 @@ func (s *smcTestHelper) registerNotaries(deposit *big.Int, params ...int) error return nil } -// deregisterNotaries is a helper function that deregister notaries in batch, -// it also verifies notary deregistration is successful. +// deregisterNotaries is a helper function that deregister notaries in batch. func (s *smcTestHelper) deregisterNotaries(params ...int) error { for i := params[0]; i < params[1]; i++ { s.testAccounts[i].txOpts.Value = big.NewInt(0) @@ -147,8 +145,7 @@ func (s *smcTestHelper) deregisterNotaries(params ...int) error { return nil } -// addHeader is a helper function to add header to smc, -// it also verifies header is correctly added to the SMC. +// addHeader is a helper function to add header to smc. func (s *smcTestHelper) addHeader(a *testAccount, shard *big.Int, period *big.Int, chunkRoot uint8) error { _, err := s.smc.AddHeader(a.txOpts, shard, period, [32]byte{chunkRoot}) if err != nil { @@ -186,8 +183,7 @@ func (s *smcTestHelper) addHeader(a *testAccount, shard *big.Int, period *big.In return nil } -// submitVote is a helper function for notary to submit vote on a given header, -// it also verifies vote is properly submitted and casted. +// submitVote is a helper function for notary to submit vote on a given header. func (s *smcTestHelper) submitVote(a *testAccount, shard *big.Int, period *big.Int, index *big.Int, chunkRoot uint8) error { _, err := s.smc.SubmitVote(a.txOpts, shard, period, index, [32]byte{chunkRoot}) if err != nil { @@ -214,7 +210,7 @@ func (s *smcTestHelper) submitVote(a *testAccount, shard *big.Int, period *big.I return fmt.Errorf("incorrect notary address in submitVote log. Want: %v Got: %v", s.testAccounts[0].addr, a.addr) } if log.Event.ChunkRoot != [32]byte{chunkRoot} { - return fmt.Errorf("chunk root missmatch in submitVote log. Want: %v Got: %v", [32]byte{chunkRoot}, log.Event.ChunkRoot) + return fmt.Errorf("chunk root missmatch in submitVote log. Want: %v Got: %v", common.BytesToHash([]byte{chunkRoot}), common.BytesToHash(log.Event.ChunkRoot[:])) } return nil } From c680bf55b5916dbe83eb3131438fb18112672e6e Mon Sep 17 00:00:00 2001 From: Terence Tsao Date: Fri, 25 May 2018 15:22:30 -0700 Subject: [PATCH 10/10] set gometallinter deadline to 10m --- build/ci.go | 1 + 1 file changed, 1 insertion(+) diff --git a/build/ci.go b/build/ci.go index 79dcc146c309..e721934f30e3 100644 --- a/build/ci.go +++ b/build/ci.go @@ -338,6 +338,7 @@ func doLint(cmdline []string) { "--enable=misspell", "--enable=goconst", "--min-occurrences=6", // for goconst + "--deadline=10m", } build.MustRunCommand(filepath.Join(GOBIN, "gometalinter.v2"), append(configs, packages...)...)