Add post-genesis deposit testing to long-running E2E (#5449)

* WIP add deposits
* Modify validator component
* Merge branch 'master' of https://github.com/prysmaticlabs/prysm into e2e-add-depsoits
* Fix e2e
* Merge branch 'master' of https://github.com/prysmaticlabs/prysm into e2e-add-depsoits
* Merge branch 'master' of https://github.com/prysmaticlabs/prysm into e2e-add-depsoits
* Start running with extra deposits
* Begin adding evluator for e2e deposit
* Get deposit E2E working
* Add more rigorous testing for deposits
* Merge branch 'master' of https://github.com/prysmaticlabs/prysm into e2e-add-depsoits
* Improve policy for deposits
* Fix build
* Remove sync testing for long running e2e
* Undo shard change
* Undo unneeded changes
* Adjust for comments
* Merge branch 'master' into e2e-add-depsoits
* Fix bug where long running E2E would always run
* Merge branch 'master' of https://github.com/prysmaticlabs/prysm into e2e-add-depsoits
* Merge branch 'e2e-add-depsoits' of https://github.com/0xKiwi/Prysm into e2e-add-depsoits
* Merge branch 'master' into e2e-add-depsoits

endtoend/BUILD.bazel

@@ -6,6 +6,7 @@ go_test(
     testonly = True,
     srcs = [
         "endtoend_test.go",
+        "long_minimal_e2e_test.go",
         "minimal_antiflake_e2e_1_test.go",
         "minimal_antiflake_e2e_2_test.go",
         "minimal_e2e_test.go",

endtoend/components/BUILD.bazel

@@ -24,6 +24,7 @@ go_library(
         "@com_github_ethereum_go_ethereum//core/types:go_default_library",
         "@com_github_ethereum_go_ethereum//ethclient:go_default_library",
         "@com_github_ethereum_go_ethereum//rpc:go_default_library",
+        "@com_github_pkg_errors//:go_default_library",
         "@io_bazel_rules_go//go/tools/bazel:go_default_library",
     ],
 )

endtoend/components/validator.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/pkg/errors"
 	contracts "github.com/prysmaticlabs/prysm/contracts/deposit-contract"
 	"github.com/prysmaticlabs/prysm/endtoend/helpers"
 	e2e "github.com/prysmaticlabs/prysm/endtoend/params"
@@ -24,104 +25,133 @@ import (
 	"github.com/prysmaticlabs/prysm/shared/testutil"
 )
 
-// StartValidators sends the deposits to the eth1 chain and starts the validator clients.
-func StartValidators(
-	t *testing.T,
-	config *types.E2EConfig,
-	keystorePath string,
-) []int {
-	binaryPath, found := bazel.FindBinary("validator", "validator")
-	if !found {
-		t.Fatal("validator binary not found")
-	}
-
+// StartValidatorClients starts the configured amount of validators, also sending and mining their validator deposits.
+// Should only be used on initialization.
+func StartValidatorClients(t *testing.T, config *types.E2EConfig, keystorePath string) []int {
 	// Always using genesis count since using anything else would be difficult to test for.
 	validatorNum := int(params.BeaconConfig().MinGenesisActiveValidatorCount)
 	beaconNodeNum := e2e.TestParams.BeaconNodeCount
 	if validatorNum%beaconNodeNum != 0 {
 		t.Fatal("Validator count is not easily divisible by beacon node count.")
 	}
-
 	processIDs := make([]int, beaconNodeNum)
 	validatorsPerNode := validatorNum / beaconNodeNum
-	for n := 0; n < beaconNodeNum; n++ {
-		file, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, fmt.Sprintf(e2e.ValidatorLogFileName, n))
-		if err != nil {
-			t.Fatal(err)
-		}
-		args := []string{
-			fmt.Sprintf("--datadir=%s/eth2-val-%d", e2e.TestParams.TestPath, n),
-			fmt.Sprintf("--log-file=%s", file.Name()),
-			fmt.Sprintf("--interop-num-validators=%d", validatorsPerNode),
-			fmt.Sprintf("--interop-start-index=%d", validatorsPerNode*n),
-			fmt.Sprintf("--monitoring-port=%d", e2e.TestParams.ValidatorMetricsPort+n),
-			fmt.Sprintf("--beacon-rpc-provider=localhost:%d", e2e.TestParams.BeaconNodeRPCPort+n),
-			"--grpc-headers=dummy=value,foo=bar", // Sending random headers shouldn't break anything.
-			"--force-clear-db",
-		}
-		args = append(args, featureconfig.E2EValidatorFlags...)
-		args = append(args, config.ValidatorFlags...)
+	for i := 0; i < beaconNodeNum; i++ {
+		pID := StartNewValidatorClient(t, config, validatorsPerNode, i)
+		processIDs[i] = pID
+	}
 
-		cmd := exec.Command(binaryPath, args...)
-		t.Logf("Starting validator client %d with flags: %s", n, strings.Join(args[2:], " "))
-		if err := cmd.Start(); err != nil {
-			t.Fatal(err)
-		}
-		processIDs[n] = cmd.Process.Pid
+	SendAndMineDeposits(t, keystorePath, validatorNum, 0)
+
+	return processIDs
+}
+
+// StartNewValidatorClient starts a validator client with the passed in configuration.
+func StartNewValidatorClient(t *testing.T, config *types.E2EConfig, validatorNum int, index int) int {
+	validatorsPerClient := int(params.BeaconConfig().MinGenesisActiveValidatorCount) / e2e.TestParams.BeaconNodeCount
+	// Only allow validatorsPerClient count for each validator client.
+	if validatorNum != validatorsPerClient {
+		return 0
+	}
+	binaryPath, found := bazel.FindBinary("validator", "validator")
+	if !found {
+		t.Fatal("validator binary not found")
+	}
+
+	beaconRPCPort := e2e.TestParams.BeaconNodeRPCPort + index
+	if beaconRPCPort >= e2e.TestParams.BeaconNodeRPCPort+e2e.TestParams.BeaconNodeCount {
+		// Point any extra validator clients to a node we know is running.
+		beaconRPCPort = e2e.TestParams.BeaconNodeRPCPort
 	}
 
+	file, err := helpers.DeleteAndCreateFile(e2e.TestParams.LogPath, fmt.Sprintf(e2e.ValidatorLogFileName, index))
+	if err != nil {
+		t.Fatal(err)
+	}
+	args := []string{
+		fmt.Sprintf("--datadir=%s/eth2-val-%d", e2e.TestParams.TestPath, index),
+		fmt.Sprintf("--log-file=%s", file.Name()),
+		fmt.Sprintf("--interop-num-validators=%d", validatorNum),
+		fmt.Sprintf("--interop-start-index=%d", validatorNum*index),
+		fmt.Sprintf("--monitoring-port=%d", e2e.TestParams.ValidatorMetricsPort+index),
+		fmt.Sprintf("--beacon-rpc-provider=localhost:%d", beaconRPCPort),
+		"--grpc-headers=dummy=value,foo=bar", // Sending random headers shouldn't break anything.
+		"--force-clear-db",
+	}
+	args = append(args, featureconfig.E2EValidatorFlags...)
+	args = append(args, config.ValidatorFlags...)
+
+	cmd := exec.Command(binaryPath, args...)
+	t.Logf("Starting validator client %d with flags: %s", index, strings.Join(args[2:], " "))
+	if err := cmd.Start(); err != nil {
+		t.Fatal(err)
+	}
+
+	return cmd.Process.Pid
+}
+
+// SendAndMineDeposits sends the requested amount of deposits and mines the chain after to ensure the deposits are seen.
+func SendAndMineDeposits(t *testing.T, keystorePath string, validatorNum int, offset int) {
 	client, err := rpc.DialHTTP(fmt.Sprintf("http://127.0.0.1:%d", e2e.TestParams.Eth1RPCPort))
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer client.Close()
 	web3 := ethclient.NewClient(client)
 
-	jsonBytes, err := ioutil.ReadFile(keystorePath)
+	keystoreBytes, err := ioutil.ReadFile(keystorePath)
 	if err != nil {
 		t.Fatal(err)
 	}
-	txOps, err := bind.NewTransactor(bytes.NewReader(jsonBytes), "" /*password*/)
+	if err := SendDeposits(web3, keystoreBytes, validatorNum, offset); err != nil {
+		t.Fatal(err)
+	}
+	mineKey, err := keystore.DecryptKey(keystoreBytes, "" /*password*/)
 	if err != nil {
 		t.Fatal(err)
 	}
+	if err := mineBlocks(web3, mineKey, params.BeaconConfig().Eth1FollowDistance); err != nil {
+		t.Fatalf("failed to mine blocks %v", err)
+	}
+}
+
+// SendDeposits uses the passed in web3 and keystore bytes to send the requested deposits.
+func SendDeposits(web3 *ethclient.Client, keystoreBytes []byte, num int, offset int) error {
+	txOps, err := bind.NewTransactor(bytes.NewReader(keystoreBytes), "" /*password*/)
+	if err != nil {
+		return err
+	}
 	depositInGwei := big.NewInt(int64(params.BeaconConfig().MaxEffectiveBalance))
 	txOps.Value = depositInGwei.Mul(depositInGwei, big.NewInt(int64(params.BeaconConfig().GweiPerEth)))
 	txOps.GasLimit = 4000000
 	nonce, err := web3.PendingNonceAt(context.Background(), txOps.From)
 	if err != nil {
-		t.Fatal(err)
+		return err
 	}
 	txOps.Nonce = big.NewInt(int64(nonce))
 
 	contract, err := contracts.NewDepositContract(e2e.TestParams.ContractAddress, web3)
 	if err != nil {
-		t.Fatal(err)
+		return err
 	}
 
-	deposits, _, err := testutil.DeterministicDepositsAndKeys(uint64(validatorNum))
+	deposits, _, err := testutil.DeterministicDepositsAndKeys(uint64(num + offset))
 	if err != nil {
-		t.Fatal(err)
+		return err
 	}
 	_, roots, err := testutil.DeterministicDepositTrie(len(deposits))
 	if err != nil {
-		t.Fatal(err)
+		return err
 	}
 	for index, dd := range deposits {
+		if index < offset {
+			continue
+		}
 		_, err = contract.Deposit(txOps, dd.Data.PublicKey, dd.Data.WithdrawalCredentials, dd.Data.Signature, roots[index])
 		if err != nil {
-			t.Errorf("unable to send transaction to contract: %v", err)
+			return errors.Wrap(err, "unable to send transaction to contract")
 		}
 		txOps.Nonce = txOps.Nonce.Add(txOps.Nonce, big.NewInt(1))
 	}
-
-	keystore, err := keystore.DecryptKey(jsonBytes, "" /*password*/)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	if err := mineBlocks(web3, keystore, params.BeaconConfig().Eth1FollowDistance); err != nil {
-		t.Fatalf("failed to mine blocks %v", err)
-	}
-
-	return processIDs
+	return nil
 }

endtoend/endtoend_test.go

@@ -32,7 +32,7 @@ func runEndToEndTest(t *testing.T, config *types.E2EConfig) {
 	keystorePath, eth1PID := components.StartEth1Node(t)
 	bootnodeENR, _ := components.StartBootnode(t)
 	bProcessIDs := components.StartBeaconNodes(t, config, bootnodeENR)
-	valProcessIDs := components.StartValidators(t, config, keystorePath)
+	valProcessIDs := components.StartValidatorClients(t, config, keystorePath)
 	processIDs := append(valProcessIDs, bProcessIDs...)
 	processIDs = append(processIDs, eth1PID)
 	defer helpers.LogOutput(t, config)
@@ -77,6 +77,12 @@ func runEndToEndTest(t *testing.T, config *types.E2EConfig) {
 		slasherPIDs := components.StartSlashers(t)
 		defer helpers.KillProcesses(t, slasherPIDs)
 	}
+	if config.TestDeposits {
+		valCount := int(params.BeaconConfig().MinGenesisActiveValidatorCount) / e2e.TestParams.BeaconNodeCount
+		valPid := components.StartNewValidatorClient(t, config, valCount, e2e.TestParams.BeaconNodeCount)
+		defer helpers.KillProcesses(t, []int{valPid})
+		components.SendAndMineDeposits(t, keystorePath, valCount, int(params.BeaconConfig().MinGenesisActiveValidatorCount))
+	}
 
 	ticker := helpers.GetEpochTicker(genesisTime, epochSeconds)
 	for currentEpoch := range ticker.C() {
@@ -113,8 +119,8 @@ func runEndToEndTest(t *testing.T, config *types.E2EConfig) {
 	}
 	conns = append(conns, syncConn)
 
-	// Sleep until the next epoch to give time for the newly started node to sync.
-	extraTimeToSync := (config.EpochsToRun+3)*epochSeconds + 60
+	// Sleep for a few epochs to give time for the newly started node to sync.
+	extraTimeToSync := (config.EpochsToRun+config.EpochsToRun/2)*epochSeconds + 60
 	genesisTime.Add(time.Duration(extraTimeToSync) * time.Second)
 	// Wait until middle of epoch to request to prevent conflicts.
 	time.Sleep(time.Until(genesisTime))

endtoend/evaluators/slashing.go

@@ -2,6 +2,7 @@ package evaluators
 
 import (
 	"context"
+	"fmt"
 
 	ptypes "github.com/gogo/protobuf/types"
 	eth "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1"
@@ -22,6 +23,20 @@ var InjectDoubleVote = types.Evaluator{
 	Evaluation: insertDoubleAttestationIntoPool,
 }
 
+// ValidatorsSlashed ensures the expected amount of validators are slashed.
+var ValidatorsSlashed = types.Evaluator{
+	Name:       "validators_slashed_epoch_%d",
+	Policy:     afterNthEpoch(0),
+	Evaluation: validatorsSlashed,
+}
+
+// SlashedValidatorsLoseBalance checks if the validators slashed lose the right balance.
+var SlashedValidatorsLoseBalance = types.Evaluator{
+	Name:       "slashed_validators_lose_valance_epoch_%d",
+	Policy:     afterNthEpoch(0),
+	Evaluation: validatorsLoseBalance,
+}
+
 var slashedIndices []uint64
 
 // Not including first epoch because of issues with genesis.
@@ -31,6 +46,52 @@ func beforeEpoch(epoch uint64) func(uint64) bool {
 	}
 }
 
+func validatorsSlashed(conns ...*grpc.ClientConn) error {
+	conn := conns[0]
+	ctx := context.Background()
+	client := eth.NewBeaconChainClient(conn)
+	req := &eth.GetValidatorActiveSetChangesRequest{}
+	changes, err := client.GetValidatorActiveSetChanges(ctx, req)
+	if err != nil {
+		return err
+	}
+	if len(changes.SlashedIndices) != 2 && len(changes.SlashedIndices) != 4 {
+		return fmt.Errorf("expected 2 or 4 indices to be slashed, received %d", len(changes.SlashedIndices))
+	}
+	return nil
+}
+
+func validatorsLoseBalance(conns ...*grpc.ClientConn) error {
+	conn := conns[0]
+	ctx := context.Background()
+	client := eth.NewBeaconChainClient(conn)
+
+	for i, indice := range slashedIndices {
+		req := &eth.GetValidatorRequest{
+			QueryFilter: &eth.GetValidatorRequest_Index{
+				Index: indice,
+			},
+		}
+		valResp, err := client.GetValidator(ctx, req)
+		if err != nil {
+			return err
+		}
+
+		slashedPenalty := params.BeaconConfig().MaxEffectiveBalance / params.BeaconConfig().MinSlashingPenaltyQuotient
+		slashedBal := params.BeaconConfig().MaxEffectiveBalance - slashedPenalty + params.BeaconConfig().EffectiveBalanceIncrement/10
+		if valResp.EffectiveBalance >= slashedBal {
+			return fmt.Errorf(
+				"expected slashed validator %d to balance less than %d, received %d",
+				i,
+				slashedBal,
+				valResp.EffectiveBalance,
+			)
+		}
+
+	}
+	return nil
+}
+
 func insertDoubleAttestationIntoPool(conns ...*grpc.ClientConn) error {
 	conn := conns[0]
 	valClient := eth.NewBeaconNodeValidatorClient(conn)