Merge branch 'develop' into withdrawal-bug-electra

CHANGELOG.md

@@ -18,6 +18,7 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 - Updated the `beacon-chain/monitor` package to Electra. [PR](https://github.com/prysmaticlabs/prysm/pull/14562)
 - Added ListAttestationsV2 endpoint.
 - Add ability to rollback node's internal state during processing.
+- Simplified `EjectedValidatorIndices`.
 
 ### Changed
 
@@ -32,6 +33,9 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 - Use read only validator for core processing to avoid unnecessary copying.
 - Use ROBlock across block processing pipeline.
 - Added missing Eth-Consensus-Version headers to GetBlockAttestationsV2 and GetAttesterSlashingsV2 endpoints.
+- Updated pgo profile for beacon chain with holesky data. This improves the profile guided
+  optimizations in the go compiler.
+- Use read only state when computing the active validator list.
 
 ### Deprecated
 
@@ -49,7 +53,11 @@ The format is based on Keep a Changelog, and this project adheres to Semantic Ve
 - Certain deb files were returning a 404 which made building new docker images without an existing
   cache impossible. This has been fixed with updates to rules_oci and bazel-lib.
 - Fixed an issue where the length check between block body KZG commitments and the existing cache from the database was incompatible.
-- EIP7521 - Fixes withdrawal bug by accounting for pending partial withdrawals and deducting already withdrawn amounts from the sweep balance.
+- Fix `--backfill-oldest-slot` handling - this flag was totally broken, the code would always backfill to the default slot [pr](https://github.com/prysmaticlabs/prysm/pull/14584)
+- Fix keymanager API should return corrected error format for malformed tokens
+- Fix keymanager API so that get keys returns an empty response instead of a 500 error when using an unsupported keystore.
+- Small log imporvement, removing some redundant or duplicate logs
+- EIP7521 - Fixes withdrawal bug by accounting for pending partial withdrawals and deducting already withdrawn amounts from the sweep balance. [PR](https://github.com/prysmaticlabs/prysm/pull/14578)
 
 
 ### Security

beacon-chain/blockchain/chain_info.go

@@ -243,7 +243,7 @@ func (s *Service) HeadValidatorsIndices(ctx context.Context, epoch primitives.Ep
 	if !s.hasHeadState() {
 		return []primitives.ValidatorIndex{}, nil
 	}
-	return helpers.ActiveValidatorIndices(ctx, s.headState(ctx), epoch)
+	return helpers.ActiveValidatorIndices(ctx, s.headStateReadOnly(ctx), epoch)
 }
 
 // HeadGenesisValidatorsRoot returns genesis validators root of the head state.

beacon-chain/core/validators/validator.go

@@ -279,38 +279,19 @@ func ExitedValidatorIndices(epoch primitives.Epoch, validators []*ethpb.Validato
 	return exited, nil
 }
 
-// EjectedValidatorIndices determines the indices ejected during the given epoch.
-func EjectedValidatorIndices(epoch primitives.Epoch, validators []*ethpb.Validator, activeValidatorCount uint64) ([]primitives.ValidatorIndex, error) {
+// EjectedValidatorIndices returns the indices of validators who were ejected during the specified epoch.
+//
+// A validator is considered ejected during an epoch if:
+// - Their ExitEpoch equals the epoch.
+// - Their EffectiveBalance is less than or equal to the EjectionBalance threshold.
+//
+// This function simplifies the ejection determination by directly checking the validator's ExitEpoch
+// and EffectiveBalance, avoiding the complexities and potential inaccuracies of calculating
+// withdrawable epochs.
+func EjectedValidatorIndices(epoch primitives.Epoch, validators []*ethpb.Validator) ([]primitives.ValidatorIndex, error) {
 	ejected := make([]primitives.ValidatorIndex, 0)
-	exitEpochs := make([]primitives.Epoch, 0)
-	for i := 0; i < len(validators); i++ {
-		val := validators[i]
-		if val.ExitEpoch != params.BeaconConfig().FarFutureEpoch {
-			exitEpochs = append(exitEpochs, val.ExitEpoch)
-		}
-	}
-	exitQueueEpoch := primitives.Epoch(0)
-	for _, i := range exitEpochs {
-		if exitQueueEpoch < i {
-			exitQueueEpoch = i
-		}
-	}
-
-	// We use the exit queue churn to determine if we have passed a churn limit.
-	exitQueueChurn := uint64(0)
-	for _, val := range validators {
-		if val.ExitEpoch == exitQueueEpoch {
-			exitQueueChurn++
-		}
-	}
-	churn := helpers.ValidatorExitChurnLimit(activeValidatorCount)
-	if churn < exitQueueChurn {
-		exitQueueEpoch++
-	}
-	withdrawableEpoch := exitQueueEpoch + params.BeaconConfig().MinValidatorWithdrawabilityDelay
 	for i, val := range validators {
-		if val.ExitEpoch == epoch && val.WithdrawableEpoch == withdrawableEpoch &&
-			val.EffectiveBalance <= params.BeaconConfig().EjectionBalance {
+		if val.ExitEpoch == epoch && val.EffectiveBalance <= params.BeaconConfig().EjectionBalance {
 			ejected = append(ejected, primitives.ValidatorIndex(i))
 		}
 	}

beacon-chain/rpc/core/validator.go

@@ -886,7 +886,7 @@ func (s *Service) ValidatorActiveSetChanges(
 		}
 	}
 	slashedIndices := validators.SlashedValidatorIndices(coreTime.CurrentEpoch(requestedState), vs)
-	ejectedIndices, err := validators.EjectedValidatorIndices(coreTime.CurrentEpoch(requestedState), vs, activeValidatorCount)
+	ejectedIndices, err := validators.EjectedValidatorIndices(coreTime.CurrentEpoch(requestedState), vs)
 	if err != nil {
 		return nil, &RpcError{
 			Err:    errors.Wrap(err, "could not determine ejected validator indices"),

cmd/beacon-chain/sync/backfill/options.go

@@ -19,7 +19,7 @@ func BeaconNodeOptions(c *cli.Context) ([]node.Option, error) {
 		}
 		// The zero value of this uint flag would be genesis, so we use IsSet to differentiate nil from zero case.
 		if c.IsSet(flags.BackfillOldestSlot.Name) {
-			uv := c.Uint64(flags.BackfillBatchSize.Name)
+			uv := c.Uint64(flags.BackfillOldestSlot.Name)
 			bno = append(bno, backfill.WithMinimumSlot(primitives.Slot(uv)))
 		}
 		node.BackfillOpts = bno

config/proposer/loader/loader.go

@@ -142,10 +142,6 @@ func (psl *settingsLoader) Load(cliCtx *cli.Context) (*proposer.Settings, error)
 	for _, method := range psl.loadMethods {
 		switch method {
 		case defaultFlag:
-			if psl.existsInDB && len(psl.loadMethods) == 1 {
-				// only log the below if default flag is the only load method
-				log.Warn("Previously saved proposer settings were loaded from the DB, only default settings will be updated. Please provide new proposer settings or clear DB to reset proposer settings.")
-			}
 			suggestedFeeRecipient := cliCtx.String(flags.SuggestedFeeRecipientFlag.Name)
 			if !common.IsHexAddress(suggestedFeeRecipient) {
 				return nil, errors.Errorf("--%s is not a valid Ethereum address", flags.SuggestedFeeRecipientFlag.Name)
@@ -159,6 +155,10 @@ func (psl *settingsLoader) Load(cliCtx *cli.Context) (*proposer.Settings, error)
 			if psl.options.builderConfig != nil {
 				defaultConfig.Builder = psl.options.builderConfig.ToConsensus()
 			}
+			if psl.existsInDB && len(psl.loadMethods) == 1 {
+				// only log the below if default flag is the only load method
+				log.Debug("Overriding previously saved proposer default settings.")
+			}
 			loadConfig.DefaultConfig = defaultConfig
 		case fileFlag:
 			var settingFromFile *validatorpb.ProposerSettingsPayload

validator/client/validator.go

@@ -1327,6 +1327,11 @@ func (v *validator) buildSignedRegReqs(
 	if v.genesisTime > uint64(time.Now().UTC().Unix()) {
 		return signedValRegRequests
 	}
+
+	if v.ProposerSettings().DefaultConfig != nil && v.ProposerSettings().DefaultConfig.FeeRecipientConfig == nil && v.ProposerSettings().DefaultConfig.BuilderConfig != nil {
+		log.Warn("Builder is `enabled` in default config but will be ignored because no fee recipient was provided!")
+	}
+
 	for i, k := range activePubkeys {
 		// map is populated before this function in buildPrepProposerReq
 		_, ok := v.pubkeyToStatus[k]
@@ -1338,10 +1343,6 @@ func (v *validator) buildSignedRegReqs(
 		gasLimit := params.BeaconConfig().DefaultBuilderGasLimit
 		enabled := false
 
-		if v.ProposerSettings().DefaultConfig != nil && v.ProposerSettings().DefaultConfig.FeeRecipientConfig == nil && v.ProposerSettings().DefaultConfig.BuilderConfig != nil {
-			log.Warn("Builder is `enabled` in default config but will be ignored because no fee recipient was provided!")
-		}
-
 		if v.ProposerSettings().DefaultConfig != nil && v.ProposerSettings().DefaultConfig.FeeRecipientConfig != nil {
 			defaultConfig := v.ProposerSettings().DefaultConfig
 			feeRecipient = defaultConfig.FeeRecipientConfig.FeeRecipient // Use cli defaultBuilderConfig for fee recipient.

validator/rpc/BUILD.bazel

@@ -117,6 +117,7 @@ go_test(
         "//encoding/bytesutil:go_default_library",
         "//io/file:go_default_library",
         "//io/logs/mock:go_default_library",
+        "//network/httputil:go_default_library",
         "//proto/prysm/v1alpha1:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",

validator/rpc/handlers_keymanager.go

@@ -50,7 +50,11 @@ func (s *Server) ListKeystores(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if s.wallet.KeymanagerKind() != keymanager.Derived && s.wallet.KeymanagerKind() != keymanager.Local {
-		httputil.HandleError(w, errors.Wrap(err, "Prysm validator keys are not stored locally with this keymanager type").Error(), http.StatusInternalServerError)
+		log.Debugf("List keystores keymanager api expected wallet type %s but got %s", s.wallet.KeymanagerKind().String(), keymanager.Local.String())
+		response := &ListKeystoresResponse{
+			Data: make([]*Keystore, 0),
+		}
+		httputil.WriteJson(w, response)
 		return
 	}
 	pubKeys, err := km.FetchValidatingPublicKeys(ctx)
@@ -402,7 +406,11 @@ func (s *Server) ListRemoteKeys(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if s.wallet.KeymanagerKind() != keymanager.Web3Signer {
-		httputil.HandleError(w, "Prysm Wallet is not of type Web3Signer. Please execute validator client with web3signer flags.", http.StatusInternalServerError)
+		log.Debugf("List remote keys keymanager api expected wallet type %s but got %s", s.wallet.KeymanagerKind().String(), keymanager.Web3Signer.String())
+		response := &ListKeystoresResponse{
+			Data: make([]*Keystore, 0),
+		}
+		httputil.WriteJson(w, response)
 		return
 	}
 	pubKeys, err := km.FetchValidatingPublicKeys(ctx)

validator/rpc/handlers_keymanager_test.go

@@ -119,6 +119,16 @@ func TestServer_ListKeystores(t *testing.T) {
 			)
 		}
 	})
+	t.Run("calling list remote while using a local wallet returns empty", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/eth/v1/remotekeys"), nil)
+		wr := httptest.NewRecorder()
+		wr.Body = &bytes.Buffer{}
+		s.ListRemoteKeys(wr, req)
+		require.Equal(t, http.StatusOK, wr.Code)
+		resp := &ListRemoteKeysResponse{}
+		require.NoError(t, json.Unmarshal(wr.Body.Bytes(), resp))
+		require.Equal(t, 0, len(resp.Data))
+	})
 }
 
 func TestServer_ImportKeystores(t *testing.T) {
@@ -1366,6 +1376,16 @@ func TestServer_ListRemoteKeys(t *testing.T) {
 			require.DeepEqual(t, hexutil.Encode(expectedKeys[i][:]), resp.Data[i].Pubkey)
 		}
 	})
+	t.Run("calling list keystores while using a remote wallet returns empty", func(t *testing.T) {
+		req := httptest.NewRequest(http.MethodGet, fmt.Sprintf("/eth/v1/keystores"), nil)
+		wr := httptest.NewRecorder()
+		wr.Body = &bytes.Buffer{}
+		s.ListKeystores(wr, req)
+		require.Equal(t, http.StatusOK, wr.Code)
+		resp := &ListKeystoresResponse{}
+		require.NoError(t, json.Unmarshal(wr.Body.Bytes(), resp))
+		require.Equal(t, 0, len(resp.Data))
+	})
 }
 
 func TestServer_ImportRemoteKeys(t *testing.T) {

validator/rpc/intercepter.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/prysmaticlabs/prysm/v5/api"
+	"github.com/prysmaticlabs/prysm/v5/network/httputil"
 	"github.com/sirupsen/logrus"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -41,18 +42,18 @@ func (s *Server) AuthTokenHandler(next http.Handler) http.Handler {
 			// ignore some routes
 			reqToken := r.Header.Get("Authorization")
 			if reqToken == "" {
-				http.Error(w, "unauthorized: no Authorization header passed. Please use an Authorization header with the jwt created in the prysm wallet", http.StatusUnauthorized)
+				httputil.HandleError(w, "Unauthorized: no Authorization header passed. Please use an Authorization header with the jwt created in the prysm wallet", http.StatusUnauthorized)
 				return
 			}
 			tokenParts := strings.Split(reqToken, "Bearer ")
 			if len(tokenParts) != 2 {
-				http.Error(w, "Invalid token format", http.StatusBadRequest)
+				httputil.HandleError(w, "Invalid token format", http.StatusBadRequest)
 				return
 			}
 
 			token := tokenParts[1]
 			if strings.TrimSpace(token) != s.authToken || strings.TrimSpace(s.authToken) == "" {
-				http.Error(w, "Forbidden: token value is invalid", http.StatusForbidden)
+				httputil.HandleError(w, "Forbidden: token value is invalid", http.StatusForbidden)
 				return
 			}
 		}

validator/rpc/intercepter_test.go

@@ -2,11 +2,13 @@ package rpc
 
 import (
 	"context"
+	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"testing"
 
 	"github.com/prysmaticlabs/prysm/v5/api"
+	"github.com/prysmaticlabs/prysm/v5/network/httputil"
 	"github.com/prysmaticlabs/prysm/v5/testing/require"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
@@ -72,6 +74,9 @@ func TestServer_AuthTokenHandler(t *testing.T) {
 		require.NoError(t, err)
 		testHandler.ServeHTTP(rr, req)
 		require.Equal(t, http.StatusUnauthorized, rr.Code)
+		errJson := &httputil.DefaultJsonError{}
+		require.NoError(t, json.Unmarshal(rr.Body.Bytes(), errJson))
+		require.StringContains(t, "Unauthorized", errJson.Message)
 	})
 	t.Run("wrong auth token was sent", func(t *testing.T) {
 		rr := httptest.NewRecorder()
@@ -80,6 +85,9 @@ func TestServer_AuthTokenHandler(t *testing.T) {
 		req.Header.Set("Authorization", "Bearer YOUR_JWT_TOKEN") // Replace with a valid JWT token
 		testHandler.ServeHTTP(rr, req)
 		require.Equal(t, http.StatusForbidden, rr.Code)
+		errJson := &httputil.DefaultJsonError{}
+		require.NoError(t, json.Unmarshal(rr.Body.Bytes(), errJson))
+		require.StringContains(t, "token value is invalid", errJson.Message)
 	})
 	t.Run("good auth token was sent", func(t *testing.T) {
 		rr := httptest.NewRecorder()
@@ -95,6 +103,9 @@ func TestServer_AuthTokenHandler(t *testing.T) {
 		require.NoError(t, err)
 		testHandler.ServeHTTP(rr, req)
 		require.Equal(t, http.StatusUnauthorized, rr.Code)
+		errJson := &httputil.DefaultJsonError{}
+		require.NoError(t, json.Unmarshal(rr.Body.Bytes(), errJson))
+		require.StringContains(t, "Unauthorized", errJson.Message)
 	})
 	t.Run("initialize does not need auth", func(t *testing.T) {
 		rr := httptest.NewRecorder()