Use FastSSZ Marshal/Unmarshal for DB Encodings in v0.11.1 (#5351)

* try * use marshaler structure for db instead of proto * white list types * attempt * revert * testutil.NewBeaconState() * Fully populate fields for round trip ssz marshal * fix //beacon-chain/db/kv:go_default_test * more passing tests * another test target passed * fixed stategen * blockchain tests green * passing sync * more targets fixed * more test fixes in rpc/validator * most rpc val * validators test fixes * skip round robin old * aggregate test * whitelist done * Update beacon-chain/rpc/validator/attester_test.go * edit baz * Fixed tests * Fixed getblock test * Add back init * reduce test size * fix broken build * tests pass Co-authored-by: Preston Van Loon <[email protected]> Co-authored-by: prylabs-bulldozer[bot] <58059840+prylabs-bulldozer[bot]@users.noreply.github.com> Co-authored-by: terence tsao <[email protected]>
prysmaticlabs · Apr 10, 2020 · 31b6fc7 · 31b6fc7
1 parent 4256321
commit 31b6fc7
Show file tree

Hide file tree

Showing 38 changed files with 593 additions and 671 deletions.
diff --git a/beacon-chain/archiver/BUILD.bazel b/beacon-chain/archiver/BUILD.bazel
@@ -39,7 +39,6 @@ go_test(
         "//shared/testutil:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
-        "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_sirupsen_logrus//hooks/test:go_default_library",
     ],

diff --git a/beacon-chain/archiver/service_test.go b/beacon-chain/archiver/service_test.go
@@ -9,7 +9,6 @@ import (
 
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
-	"github.com/prysmaticlabs/go-bitfield"
 	mock "github.com/prysmaticlabs/prysm/beacon-chain/blockchain/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/epoch/precompute"
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/feed"
@@ -34,12 +33,8 @@ func TestArchiverService_ReceivesBlockProcessedEvent(t *testing.T) {
 	hook := logTest.NewGlobal()
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
-	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
-		Slot: 1,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
+	st := testutil.NewBeaconState()
+	st.SetSlot(1)
 	svc.headFetcher = &mock.ChainService{
 		State: st,
 	}
@@ -61,12 +56,8 @@ func TestArchiverService_OnlyArchiveAtEpochEnd(t *testing.T) {
 	svc, beaconDB := setupService(t)
 	defer dbutil.TeardownDB(t, beaconDB)
 	// The head state is NOT an epoch end.
-	st, err := stateTrie.InitializeFromProto(&pb.BeaconState{
-		Slot: params.BeaconConfig().SlotsPerEpoch - 2,
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
+	st := testutil.NewBeaconState()
+	st.SetSlot(params.BeaconConfig().SlotsPerEpoch - 2)
 	svc.headFetcher = &mock.ChainService{
 		State: st,
 	}
@@ -433,18 +424,12 @@ func setupState(validatorCount uint64) (*stateTrie.BeaconState, error) {
 
 	// We initialize a head state that has attestations from participated
 	// validators in a simulated fashion.
-	return stateTrie.InitializeFromProto(&pb.BeaconState{
-		Slot:                       (2 * params.BeaconConfig().SlotsPerEpoch) - 1,
-		Validators:                 validators,
-		Balances:                   balances,
-		BlockRoots:                 make([][]byte, params.BeaconConfig().SlotsPerHistoricalRoot),
-		Slashings:                  []uint64{0, 1e9, 1e9},
-		RandaoMixes:                make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
-		CurrentEpochAttestations:   atts,
-		FinalizedCheckpoint:        &ethpb.Checkpoint{},
-		JustificationBits:          bitfield.Bitvector4{0x00},
-		CurrentJustifiedCheckpoint: &ethpb.Checkpoint{},
-	})
+	st := testutil.NewBeaconState()
+	st.SetSlot((2 * params.BeaconConfig().SlotsPerEpoch) - 1)
+	st.SetValidators(validators)
+	st.SetBalances(balances)
+	st.SetCurrentEpochAttestations(atts)
+	return st, nil
 }
 
 func setupService(t *testing.T) (*Service, db.Database) {

diff --git a/beacon-chain/blockchain/head_test.go b/beacon-chain/blockchain/head_test.go
@@ -9,8 +9,7 @@ import (
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func TestSaveHead_Same(t *testing.T) {
@@ -46,7 +45,8 @@ func TestSaveHead_Different(t *testing.T) {
 	newHeadSignedBlock := &ethpb.SignedBeaconBlock{Block: newHeadBlock}
 	service.beaconDB.SaveBlock(context.Background(), newHeadSignedBlock)
 	newRoot, _ := ssz.HashTreeRoot(newHeadBlock)
-	headState, _ := state.InitializeFromProto(&pb.BeaconState{Slot: 1})
+	headState := testutil.NewBeaconState()
+	headState.SetSlot(1)
 	service.beaconDB.SaveState(context.Background(), headState, newRoot)
 	if err := service.saveHead(context.Background(), newRoot); err != nil {
 		t.Fatal(err)

diff --git a/beacon-chain/blockchain/process_attestation_test.go b/beacon-chain/blockchain/process_attestation_test.go
@@ -13,7 +13,6 @@ import (
 	"github.com/prysmaticlabs/prysm/beacon-chain/core/state"
 	testDB "github.com/prysmaticlabs/prysm/beacon-chain/db/testing"
 	"github.com/prysmaticlabs/prysm/beacon-chain/forkchoice/protoarray"
-	beaconstate "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	stateTrie "github.com/prysmaticlabs/prysm/beacon-chain/state"
 	"github.com/prysmaticlabs/prysm/beacon-chain/state/stategen"
 	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
@@ -54,7 +53,7 @@ func TestStore_OnAttestation(t *testing.T) {
 	}
 	BlkWithStateBadAttRoot, _ := ssz.HashTreeRoot(BlkWithStateBadAtt.Block)
 
-	s, err := beaconstate.InitializeFromProto(&pb.BeaconState{})
+	s := testutil.NewBeaconState()
 	if err := s.SetSlot(100 * params.BeaconConfig().SlotsPerEpoch); err != nil {
 		t.Fatal(err)
 	}
@@ -67,13 +66,11 @@ func TestStore_OnAttestation(t *testing.T) {
 		t.Fatal(err)
 	}
 	BlkWithValidStateRoot, _ := ssz.HashTreeRoot(BlkWithValidState.Block)
-	s, _ = stateTrie.InitializeFromProto(&pb.BeaconState{
-		Fork: &pb.Fork{
-			Epoch:           0,
-			CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
-			PreviousVersion: params.BeaconConfig().GenesisForkVersion,
-		},
-		RandaoMixes: make([][]byte, params.BeaconConfig().EpochsPerHistoricalVector),
+	s = testutil.NewBeaconState()
+	s.SetFork(&pb.Fork{
+		Epoch:           0,
+		CurrentVersion:  params.BeaconConfig().GenesisForkVersion,
+		PreviousVersion: params.BeaconConfig().GenesisForkVersion,
 	})
 	if err := service.beaconDB.SaveState(ctx, s, BlkWithValidStateRoot); err != nil {
 		t.Fatal(err)

diff --git a/beacon-chain/blockchain/process_block_test.go b/beacon-chain/blockchain/process_block_test.go
@@ -46,10 +46,7 @@ func TestStore_OnBlock(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
+	st := testutil.NewBeaconState()
 	if err := service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot); err != nil {
 		t.Fatal(err)
 	}
@@ -187,7 +184,8 @@ func TestRemoveStateSinceLastFinalized(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: uint64(i)})
+		s := testutil.NewBeaconState()
+		s.SetSlot(uint64(i))
 		if err := service.beaconDB.SaveState(ctx, s, r); err != nil {
 			t.Fatal(err)
 		}
@@ -397,16 +395,17 @@ func TestSaveInitState_CanSaveDelete(t *testing.T) {
 
 	for i := uint64(0); i < 64; i++ {
 		b := &ethpb.BeaconBlock{Slot: i}
-		s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{Slot: i})
+		s := testutil.NewBeaconState()
+		s.SetSlot(i)
 		r, _ := ssz.HashTreeRoot(b)
 		service.initSyncState[r] = s
 	}
 
 	// Set finalized root as slot 32
 	finalizedRoot, _ := ssz.HashTreeRoot(&ethpb.BeaconBlock{Slot: 32})
-
-	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{FinalizedCheckpoint: &ethpb.Checkpoint{
-		Epoch: 1, Root: finalizedRoot[:]}})
+	s := testutil.NewBeaconState()
+	s.SetFinalizedCheckpoint(&ethpb.Checkpoint{
+		Epoch: 1, Root: finalizedRoot[:]})
 	if err := service.saveInitState(ctx, s); err != nil {
 		t.Fatal(err)
 	}
@@ -442,17 +441,15 @@ func TestUpdateJustified_CouldUpdateBest(t *testing.T) {
 	}
 	service.justifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
 	service.bestJustifiedCheckpt = &ethpb.Checkpoint{Root: []byte{'A'}}
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		t.Fatal(err)
-	}
+	st := testutil.NewBeaconState()
 	service.initSyncState[r] = st.Copy()
 	if err := db.SaveState(ctx, st.Copy(), r); err != nil {
 		t.Fatal(err)
 	}
 
 	// Could update
-	s, _ := stateTrie.InitializeFromProto(&pb.BeaconState{CurrentJustifiedCheckpoint: &ethpb.Checkpoint{Epoch: 1, Root: r[:]}})
+	s := testutil.NewBeaconState()
+	s.SetCurrentJustifiedCheckpoint(&ethpb.Checkpoint{Epoch: 1, Root: r[:]})
 	if err := service.updateJustified(context.Background(), s); err != nil {
 		t.Fatal(err)
 	}
@@ -487,7 +484,7 @@ func TestFilterBlockRoots_CanFilter(t *testing.T) {
 	fRoot, _ := ssz.HashTreeRoot(fBlock)
 	hBlock := &ethpb.BeaconBlock{Slot: 1}
 	headRoot, _ := ssz.HashTreeRoot(hBlock)
-	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	st := testutil.NewBeaconState()
 	if err := service.beaconDB.SaveBlock(ctx, &ethpb.SignedBeaconBlock{Block: fBlock}); err != nil {
 		t.Fatal(err)
 	}
@@ -530,7 +527,7 @@ func TestPersistCache_CanSave(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	st := testutil.NewBeaconState()
 
 	for i := uint64(0); i < initialSyncCacheSize; i++ {
 		st.SetSlot(i)
@@ -582,7 +579,7 @@ func TestFillForkChoiceMissingBlocks_CanSave(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	st := testutil.NewBeaconState()
 	if err := service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot); err != nil {
 		t.Fatal(err)
 	}
@@ -636,7 +633,7 @@ func TestFillForkChoiceMissingBlocks_FilterFinalized(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	st, _ := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
+	st := testutil.NewBeaconState()
 	if err := service.beaconDB.SaveState(ctx, st.Copy(), validGenesisRoot); err != nil {
 		t.Fatal(err)
 	}
@@ -695,10 +692,7 @@ func blockTree1(db db.Database, genesisRoot []byte) ([][]byte, error) {
 	r7, _ := ssz.HashTreeRoot(b7)
 	b8 := &ethpb.BeaconBlock{Slot: 8, ParentRoot: r6[:]}
 	r8, _ := ssz.HashTreeRoot(b8)
-	st, err := stateTrie.InitializeFromProtoUnsafe(&pb.BeaconState{})
-	if err != nil {
-		return nil, err
-	}
+	st := testutil.NewBeaconState()
 	for _, b := range []*ethpb.BeaconBlock{b0, b1, b3, b4, b5, b6, b7, b8} {
 		if err := db.SaveBlock(context.Background(), &ethpb.SignedBeaconBlock{Block: b}); err != nil {
 			return nil, err

diff --git a/beacon-chain/blockchain/service_test.go b/beacon-chain/blockchain/service_test.go
@@ -235,10 +235,8 @@ func TestChainStartStop_Initialized(t *testing.T) {
 	if err := db.SaveBlock(ctx, genesisBlk); err != nil {
 		t.Fatal(err)
 	}
-	s, err := beaconstate.InitializeFromProto(&pb.BeaconState{Slot: 1})
-	if err != nil {
-		t.Fatal(err)
-	}
+	s := testutil.NewBeaconState()
+	s.SetSlot(1)
 	if err := db.SaveState(ctx, s, blkRoot); err != nil {
 		t.Fatal(err)
 	}
@@ -348,10 +346,9 @@ func TestChainService_InitializeChainInfo(t *testing.T) {
 
 	finalizedSlot := params.BeaconConfig().SlotsPerEpoch*2 + 1
 	headBlock := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: finalizedSlot, ParentRoot: genesisRoot[:]}}
-	headState, err := beaconstate.InitializeFromProto(&pb.BeaconState{Slot: finalizedSlot, GenesisValidatorsRoot: params.BeaconConfig().ZeroHash[:]})
-	if err != nil {
-		t.Fatal(err)
-	}
+	headState := testutil.NewBeaconState()
+	headState.SetSlot(finalizedSlot)
+	headState.SetGenesisValidatorRoot(params.BeaconConfig().ZeroHash[:])
 	headRoot, _ := ssz.HashTreeRoot(headBlock.Block)
 	if err := db.SaveState(ctx, headState, headRoot); err != nil {
 		t.Fatal(err)
@@ -414,8 +411,7 @@ func TestChainService_SaveHeadNoDB(t *testing.T) {
 	}
 	b := &ethpb.SignedBeaconBlock{Block: &ethpb.BeaconBlock{Slot: 1}}
 	r, _ := ssz.HashTreeRoot(b)
-	state := &pb.BeaconState{}
-	newState, err := beaconstate.InitializeFromProto(state)
+	newState := testutil.NewBeaconState()
 	s.stateGen.SaveState(ctx, r, newState)
 	if err := s.saveHeadNoDB(ctx, b, r); err != nil {
 		t.Fatal(err)
@@ -447,11 +443,8 @@ func TestChainService_PruneOldStates(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
-		state := &pb.BeaconState{Slot: uint64(i)}
-		newState, err := beaconstate.InitializeFromProto(state)
-		if err != nil {
-			t.Fatal(err)
-		}
+		newState := testutil.NewBeaconState()
+		newState.SetSlot(uint64(i))
 		if err := s.beaconDB.SaveState(ctx, newState, r); err != nil {
 			t.Fatal(err)
 		}

diff --git a/beacon-chain/core/blocks/block_operations.go b/beacon-chain/core/blocks/block_operations.go
@@ -456,7 +456,7 @@ func VerifyProposerSlashing(
 		return fmt.Errorf("validator with key %#x is not slashable", proposer.PublicKey)
 	}
 	// Using headerEpoch1 here because both of the headers should have the same epoch.
-	domain, err := helpers.Domain(beaconState.Fork(), helpers.StartSlot(slashing.Header_1.Header.Slot), params.BeaconConfig().DomainBeaconProposer, beaconState.GenesisValidatorRoot())
+	domain, err := helpers.Domain(beaconState.Fork(), helpers.SlotToEpoch(slashing.Header_1.Header.Slot), params.BeaconConfig().DomainBeaconProposer, beaconState.GenesisValidatorRoot())
 	if err != nil {
 		return err
 	}

diff --git a/beacon-chain/db/kv/BUILD.bazel b/beacon-chain/db/kv/BUILD.bazel
@@ -44,6 +44,7 @@ go_library(
         "//shared/traceutil:go_default_library",
         "@com_github_dgraph_io_ristretto//:go_default_library",
         "@com_github_ethereum_go_ethereum//common:go_default_library",
+        "@com_github_ferranbt_fastssz//:go_default_library",
         "@com_github_gogo_protobuf//proto:go_default_library",
         "@com_github_golang_snappy//:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
@@ -80,7 +81,6 @@ go_test(
     deps = [
         "//beacon-chain/cache:go_default_library",
         "//beacon-chain/db/filters:go_default_library",
-        "//beacon-chain/state:go_default_library",
         "//proto/beacon/p2p/v1:go_default_library",
         "//proto/testing:go_default_library",
         "//shared/bytesutil:go_default_library",
@@ -91,5 +91,6 @@ go_test(
         "@com_github_prysmaticlabs_ethereumapis//eth/v1alpha1:go_default_library",
         "@com_github_prysmaticlabs_go_bitfield//:go_default_library",
         "@com_github_prysmaticlabs_go_ssz//:go_default_library",
+        "@in_gopkg_d4l3k_messagediff_v1//:go_default_library",
     ],
 )
diff --git a/beacon-chain/db/kv/backup_test.go b/beacon-chain/db/kv/backup_test.go
@@ -8,8 +8,7 @@ import (
 
 	eth "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func TestStore_Backup(t *testing.T) {
@@ -26,7 +25,7 @@ func TestStore_Backup(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	st, err := state.InitializeFromProto(&pb.BeaconState{})
+	st := testutil.NewBeaconState()
 	if err := db.SaveState(ctx, st, root); err != nil {
 		t.Fatal(err)
 	}

diff --git a/beacon-chain/db/kv/checkpoint_test.go b/beacon-chain/db/kv/checkpoint_test.go
@@ -8,9 +8,8 @@ import (
 	"github.com/gogo/protobuf/proto"
 	ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
 	"github.com/prysmaticlabs/go-ssz"
-	"github.com/prysmaticlabs/prysm/beacon-chain/state"
-	pb "github.com/prysmaticlabs/prysm/proto/beacon/p2p/v1"
 	"github.com/prysmaticlabs/prysm/shared/bytesutil"
+	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
 func TestStore_JustifiedCheckpoint_CanSaveRetrieve(t *testing.T) {
@@ -22,10 +21,9 @@ func TestStore_JustifiedCheckpoint_CanSaveRetrieve(t *testing.T) {
 		Epoch: 10,
 		Root:  root[:],
 	}
-	st, err := state.InitializeFromProto(&pb.BeaconState{Slot: 1})
-	if err != nil {
-		t.Fatal(err)
-	}
+	st := testutil.NewBeaconState()
+	st.SetSlot(1)
+
 	if err := db.SaveState(ctx, st, root); err != nil {
 		t.Fatal(err)
 	}
@@ -74,10 +72,8 @@ func TestStore_FinalizedCheckpoint_CanSaveRetrieve(t *testing.T) {
 	if err := db.SaveBlock(ctx, blk); err != nil {
 		t.Fatal(err)
 	}
-	st, err := state.InitializeFromProto(&pb.BeaconState{Slot: 1})
-	if err != nil {
-		t.Fatal(err)
-	}
+	st := testutil.NewBeaconState()
+	st.SetSlot(1)
 	// a state is required to save checkpoint
 	if err := db.SaveState(ctx, st, root); err != nil {
 		t.Fatal(err)