(BIDS-2261) Merge branch 'NOBIDS/fix_sync_page' into BIDS-2261/stats_etl

cmd/explorer/main.go

@@ -458,7 +458,7 @@ func main() {
 			router.HandleFunc("/validator/{pubkey}/deposits", handlers.ValidatorDeposits).Methods("GET")
 			router.HandleFunc("/validator/{index}/slashings", handlers.ValidatorSlashings).Methods("GET")
 			router.HandleFunc("/validator/{index}/effectiveness", handlers.ValidatorAttestationInclusionEffectiveness).Methods("GET")
-			router.HandleFunc("/validator/{pubkey}/save", handlers.ValidatorSave).Methods("POST")
+			router.HandleFunc("/validator/{pubkey}/name", handlers.SaveValidatorName).Methods("POST")
 			router.HandleFunc("/watchlist/add", handlers.UsersModalAddValidator).Methods("POST")
 			router.HandleFunc("/validator/{pubkey}/remove", handlers.UserValidatorWatchlistRemove).Methods("POST")
 			router.HandleFunc("/validator/{index}/stats", handlers.ValidatorStatsTable).Methods("GET")
@@ -555,7 +555,6 @@ func main() {
 
 			oauthRouter := router.PathPrefix("/user").Subrouter()
 			oauthRouter.HandleFunc("/authorize", handlers.UserAuthorizeConfirm).Methods("GET")
-			oauthRouter.HandleFunc("/cancel", handlers.UserAuthorizationCancel).Methods("GET")
 			oauthRouter.Use(csrfHandler)
 
 			authRouter := router.PathPrefix("/user").Subrouter()

db/db.go

@@ -1632,7 +1632,7 @@ func saveBlocks(blocks map[uint64]map[string]*types.Block, tx *sqlx.Tx, forceSlo
 				b.Signature,
 				b.RandaoReveal,
 				b.Graffiti,
-				utils.GraffitiToSring(b.Graffiti),
+				utils.GraffitiToString(b.Graffiti),
 				b.Eth1Data.DepositRoot,
 				b.Eth1Data.DepositCount,
 				b.Eth1Data.BlockHash,
@@ -2965,7 +2965,7 @@ func GetBLSChangesCountForQuery(query string) (uint64, error) {
 		if decErr != nil {
 			return 0, decErr
 		}
-		err = ReaderDb.Select(&count, fmt.Sprintf(blsQuery, searchQuery, BlsChangeQueryLimit),
+		err = ReaderDb.Get(&count, fmt.Sprintf(blsQuery, searchQuery, BlsChangeQueryLimit),
 			pubkey)
 	} else if uiQuery, parseErr := strconv.ParseUint(query, 10, 64); parseErr == nil {
 		// Check whether the query can be used for a validator, slot or epoch search

db/ens.go

@@ -308,11 +308,10 @@ func (bigtable *Bigtable) TransformEnsNameRegistered(blk *types.Eth1Block, cache
 	return bulkData, bulkMetadataUpdates, nil
 }
 
-// Ens names are only supported to a length of 40
-// Source: https://github.com/wealdtech/go-ens/blob/5b323a4ef0472f06c515723b060b166843b9db08/resolver.go#L190
 func verifyName(name string) error {
-	if (strings.HasPrefix(name, "0x") && len(name) > 42) || (!strings.HasPrefix(name, "0x") && len(name) > 40) {
-		return fmt.Errorf("name too long")
+	// limited by max capacity of db (caused by btrees of indexes); tests showed maximum of 2684 (added buffer)
+	if len(name) > 2048 {
+		return fmt.Errorf("name too long: %v", name)
 	}
 	return nil
 }

exporter/exporter.go

@@ -32,7 +32,7 @@ var Client *rpc.Client
 func Start(client rpc.Client) error {
 	go networkLivenessUpdater(client)
 	go eth1DepositsExporter()
-	go genesisDepositsExporter()
+	go genesisDepositsExporter(client)
 	go checkSubscriptions()
 	go syncCommitteesExporter(client)
 	go syncCommitteesCountExporter()
@@ -678,7 +678,7 @@ func networkLivenessUpdater(client rpc.Client) {
 	}
 }
 
-func genesisDepositsExporter() {
+func genesisDepositsExporter(client rpc.Client) {
 	for {
 		// check if the beaconchain has started
 		var latestEpoch uint64
@@ -696,7 +696,7 @@ func genesisDepositsExporter() {
 
 		// check if genesis-deposits have already been exported
 		var genesisDepositsCount uint64
-		err = db.WriterDb.Get(&genesisDepositsCount, "SELECT COUNT(*) FROM blocks_deposits INNER JOIN blocks ON blocks_deposits.block_root = blocks.blockroot AND blocks.status = '1' WHERE block_slot=0")
+		err = db.WriterDb.Get(&genesisDepositsCount, "SELECT COUNT(*) FROM blocks_deposits WHERE block_slot=0")
 		if err != nil {
 			logger.Errorf("error retrieving genesis-deposits-count when exporting genesis-deposits: %v", err)
 			time.Sleep(time.Second * 60)
@@ -708,32 +708,9 @@ func genesisDepositsExporter() {
 			return
 		}
 
-		// get genesis-validators-count
-		var genesisValidatorsCount uint64
-		err = db.WriterDb.Get(&genesisValidatorsCount, "SELECT validatorscount FROM epochs WHERE epoch=0")
+		genesisValidators, err := client.GetValidatorState(0)
 		if err != nil {
-			logger.Errorf("error retrieving validatorscount for genesis-epoch when exporting genesis-deposits: %v", err)
-			time.Sleep(time.Second * 60)
-			continue
-		}
-
-		// check if eth1-deposits have already been exported
-		var missingEth1Deposits uint64
-		err = db.WriterDb.Get(&missingEth1Deposits, `
-			SELECT COUNT(*)
-			FROM validators v
-			LEFT JOIN ( 
-				SELECT DISTINCT ON (publickey) publickey, signature FROM eth1_deposits 
-			) d ON d.publickey = v.pubkey
-			WHERE d.publickey IS NULL AND v.validatorindex < $1`, genesisValidatorsCount)
-		if err != nil {
-			logger.Errorf("error retrieving missing-eth1-deposits-count when exporting genesis-deposits")
-			time.Sleep(time.Second * 60)
-			continue
-		}
-
-		if missingEth1Deposits > 0 {
-			logger.Infof("delaying export of genesis-deposits until eth1-deposits have been exported")
+			logger.Errorf("error retrieving genesis validator data for genesis-epoch when exporting genesis-deposits: %v", err)
 			time.Sleep(time.Second * 60)
 			continue
 		}
@@ -745,7 +722,21 @@ func genesisDepositsExporter() {
 			continue
 		}
 
-		// export genesis-deposits from eth1-deposits and data already gathered from the eth2-client
+		for _, validator := range genesisValidators.Data {
+			logger.Infof("exporting deposit data for genesis validator %v", validator.Index)
+			_, err = tx.Exec(`INSERT INTO blocks_deposits (block_slot, block_root, block_index, publickey, withdrawalcredentials, amount, signature)
+			VALUES (0, '\x01', $1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`,
+				validator.Index, utils.MustParseHex(validator.Validator.Pubkey), utils.MustParseHex(validator.Validator.WithdrawalCredentials), validator.Balance, []byte{0x0},
+			)
+			if err != nil {
+				tx.Rollback()
+				logger.Errorf("error exporting genesis-deposits: %v", err)
+				time.Sleep(time.Second * 60)
+				continue
+			}
+		}
+
+		// hydrate the eth1 deposit signature for all genesis validators that have a corresponding eth1 deposit
 		_, err = tx.Exec(`
 				INSERT INTO blocks_deposits (block_slot, block_index, publickey, withdrawalcredentials, amount, signature)
 				SELECT
@@ -762,19 +753,19 @@ func genesisDepositsExporter() {
 				LEFT JOIN ( 
 					SELECT DISTINCT ON (publickey) publickey, signature FROM eth1_deposits 
 				) d ON d.publickey = v.pubkey
-				WHERE v.validatorindex < $1`, genesisValidatorsCount)
+				WHERE v.validatorindex < $1 AND d.signature IS NOT NULL
+				ON CONFLICT (block_slot, block_index) DO UPDATE SET signature = EXCLUDED.signature`, len(genesisValidators.Data))
 		if err != nil {
 			tx.Rollback()
-			logger.Errorf("error exporting genesis-deposits: %v", err)
+			logger.Errorf("error hydrating eth1 data into genesis-deposits: %v", err)
 			time.Sleep(time.Second * 60)
 			continue
 		}
-
 		// update deposits-count
-		_, err = tx.Exec("UPDATE blocks SET depositscount = $1 WHERE slot = 0", genesisValidatorsCount)
+		_, err = tx.Exec("UPDATE blocks SET depositscount = $1 WHERE slot = 0", len(genesisValidators.Data))
 		if err != nil {
 			tx.Rollback()
-			logger.Errorf("error exporting genesis-deposits: %v", err)
+			logger.Errorf("error updating deposit count for the genesis slot: %v", err)
 			time.Sleep(time.Second * 60)
 			continue
 		}
@@ -787,7 +778,7 @@ func genesisDepositsExporter() {
 			continue
 		}
 
-		logger.Infof("exported genesis-deposits for %v genesis-validators", genesisValidatorsCount)
+		logger.Infof("exported genesis-deposits for %v genesis-validators", len(genesisValidators.Data))
 		return
 	}
 }

exporter/rocketpool.go

@@ -21,6 +21,7 @@ import (
 	_ "github.com/jackc/pgx/v5/stdlib"
 	"github.com/jmoiron/sqlx"
 	"github.com/klauspost/compress/zstd"
+	"github.com/lib/pq"
 	rpDAO "github.com/rocket-pool/rocketpool-go/dao"
 	rpDAOTrustedNode "github.com/rocket-pool/rocketpool-go/dao/trustednode"
 	"github.com/rocket-pool/rocketpool-go/minipool"
@@ -1117,6 +1118,7 @@ func (rp *RocketpoolExporter) SaveDAOMembers() error {
 
 		valueStrings := make([]string, 0, batchSize)
 		valueArgs := make([]interface{}, 0, batchSize*nArgs)
+		addresses := make([][]byte, 0, batchSize)
 		for i, d := range data[start:end] {
 			for j := 0; j < nArgs; j++ {
 				valueStringsArgs[j] = i*nArgs + j + 1
@@ -1130,12 +1132,39 @@ func (rp *RocketpoolExporter) SaveDAOMembers() error {
 			valueArgs = append(valueArgs, d.LastProposalTime)
 			valueArgs = append(valueArgs, d.RPLBondAmount.String())
 			valueArgs = append(valueArgs, d.UnbondedValidatorCount)
+			addresses = append(addresses, d.Address)
 		}
-		stmt := fmt.Sprintf(`insert into rocketpool_dao_members (rocketpool_storage_address, address, id, url, joined_time, last_proposal_time, rpl_bond_amount, unbonded_validator_count) values %s on conflict (rocketpool_storage_address, address) do update set id = excluded.id, url = excluded.url, joined_time = excluded.joined_time, last_proposal_time = excluded.last_proposal_time, rpl_bond_amount = excluded.rpl_bond_amount, unbonded_validator_count = excluded.unbonded_validator_count`, strings.Join(valueStrings, ","))
+		stmt := fmt.Sprintf(`
+			INSERT INTO rocketpool_dao_members (
+				rocketpool_storage_address,
+				address,
+				id,
+				url,
+				joined_time,
+				last_proposal_time,
+				rpl_bond_amount,
+				unbonded_validator_count
+			)
+			values %s
+			on conflict (rocketpool_storage_address, address) do update set
+				id = excluded.id,
+				url = excluded.url,
+				joined_time = excluded.joined_time,
+				last_proposal_time = excluded.last_proposal_time,
+				rpl_bond_amount = excluded.rpl_bond_amount,
+				unbonded_validator_count = excluded.unbonded_validator_count
+			`, strings.Join(valueStrings, ","))
 		_, err := tx.Exec(stmt, valueArgs...)
 		if err != nil {
 			return fmt.Errorf("error inserting into rocketpool_dao_members: %w", err)
 		}
+
+		_, err = tx.Exec(`
+			DELETE FROM rocketpool_dao_members
+			WHERE NOT address = ANY($1)`, pq.ByteaArray(addresses))
+		if err != nil {
+			return fmt.Errorf("error deleting from rocketpool_dao_members: %w", err)
+		}
 	}
 
 	return tx.Commit()

handlers/api.go

@@ -16,7 +16,6 @@ import (
 	"eth2-exporter/utils"
 	"fmt"
 	"io"
-	"math"
 	"math/big"
 	"net/http"
 	"net/url"
@@ -1502,7 +1501,7 @@ func getEpoch(epoch int64) ([]interface{}, error) {
 // ApiValidator godoc
 // @Summary Get up to 100 validators
 // @Tags Validator
-// @Description Searching for too many validators based on their pubkeys will lead to an "URI too long" error
+// @Description Searching for too many validators based on their pubkeys will lead to a "URI too long" error
 // @Produce  json
 // @Param  indexOrPubkey path string true "Up to 100 validator indicesOrPubkeys, comma separated"
 // @Success 200 {object} types.ApiResponse{data=[]types.APIValidatorResponse}
@@ -1513,10 +1512,11 @@ func ApiValidatorGet(w http.ResponseWriter, r *http.Request) {
 }
 
 // ApiValidator godoc
-// @Summary Get unlimited validators
+// @Summary Get up to 100 validators
 // @Tags Validator
+// @Description This POST endpoint exists because the GET endpoint can lead to a "URI too long" error when searching for too many validators based on their pubkeys.
 // @Produce  json
-// @Param  indexOrPubkey body types.DashboardRequest true "Validator indicesOrPubkeys, comma separated"
+// @Param  indexOrPubkey body types.DashboardRequest true "Up to 100 validator indicesOrPubkeys, comma separated"
 // @Success 200 {object} types.ApiResponse{data=[]types.APIValidatorResponse}
 // @Failure 400 {object} types.ApiResponse
 // @Router /api/v1/validator [post]
@@ -1530,16 +1530,13 @@ func apiValidator(w http.ResponseWriter, r *http.Request) {
 
 	vars := mux.Vars(r)
 
-	var maxValidators int
+	maxValidators := getUserPremium(r).MaxValidators
+
 	var param string
 	if r.Method == http.MethodGet {
-		maxValidators = getUserPremium(r).MaxValidators
-
 		// Get the validators from the URL
 		param = vars["indexOrPubkey"]
 	} else {
-		maxValidators = math.MaxInt
-
 		// Get the validators from the request body
 		decoder := json.NewDecoder(r.Body)
 		req := &types.DashboardRequest{}

handlers/auth.go

@@ -156,12 +156,30 @@ func Login(w http.ResponseWriter, r *http.Request) {
 
 	w.Header().Set("Content-Type", "text/html")
 
+	q := r.URL.Query()
+
 	data := InitPageData(w, r, "login", "/login", "Login", templateFiles)
-	data.Data = types.AuthData{
+
+	authData := types.AuthData{
 		Flashes:      utils.GetFlashes(w, r, authSessionName),
 		CsrfField:    csrf.TemplateField(r),
 		RecaptchaKey: utils.Config.Frontend.RecaptchaSiteKey,
 	}
+
+	redirectData := struct {
+		Redirect_uri string
+		State        string
+	}{
+		Redirect_uri: q.Get("redirect_uri"),
+		State:        q.Get("state"),
+	}
+
+	data.Data = struct {
+		AuthData     types.AuthData
+		RedirectData interface{}
+	}{
+		AuthData:     authData,
+		RedirectData: redirectData}
 	data.Meta.NoTrack = true
 
 	if handleTemplateError(w, r, "auth.go", "Login", "", loginTemplate.ExecuteTemplate(w, "layout", data)) != nil {
@@ -171,7 +189,6 @@ func Login(w http.ResponseWriter, r *http.Request) {
 
 // LoginPost handles authenticating the user.
 func LoginPost(w http.ResponseWriter, r *http.Request) {
-
 	if err := utils.HandleRecaptcha(w, r, "/login"); err != nil {
 		return
 	}
@@ -213,29 +230,40 @@ func LoginPost(w http.ResponseWriter, r *http.Request) {
 		UserGroup string `db:"user_group"`
 	}{}
 
+	redirectParam := ""
+	redirectURI := r.FormValue("oauth_redirect_uri")
+	if redirectURI != "" {
+		redirectParam = "?redirect_uri=" + redirectURI
+
+		state := r.FormValue("state")
+		if state != "" {
+			redirectParam += "&state=" + state
+		}
+	}
+
 	err = db.FrontendWriterDB.Get(&user, "SELECT users.id, email, password, email_confirmed, COALESCE(product_id, '') as product_id, COALESCE(active, false) as active, COALESCE(user_group, '') AS user_group FROM users left join users_app_subscriptions on users_app_subscriptions.user_id = users.id WHERE email = $1", email)
 	if err != nil {
 		if err != sql.ErrNoRows {
 			logger.Errorf("error retrieving password for user %v: %v", email, err)
 		}
 		session.AddFlash("Error: Invalid email or password!")
 		session.Save(r, w)
-		http.Redirect(w, r, "/login", http.StatusSeeOther)
+		http.Redirect(w, r, "/login"+redirectParam, http.StatusSeeOther)
 		return
 	}
 
 	if !user.Confirmed {
 		session.AddFlash("Error: Email has not been confirmed, please click the link in the email we sent you or <a href='/resend'>resend link</a>!")
 		session.Save(r, w)
-		http.Redirect(w, r, "/login", http.StatusSeeOther)
+		http.Redirect(w, r, "/login"+redirectParam, http.StatusSeeOther)
 		return
 	}
 
 	err = bcrypt.CompareHashAndPassword([]byte(user.Password), []byte(pwd))
 	if err != nil {
 		session.AddFlash("Error: Invalid email or password!")
 		session.Save(r, w)
-		http.Redirect(w, r, "/login", http.StatusSeeOther)
+		http.Redirect(w, r, "/login"+redirectParam, http.StatusSeeOther)
 		return
 	}
 
@@ -282,20 +310,8 @@ func LoginPost(w http.ResponseWriter, r *http.Request) {
 		},
 	).Info("login succeeded")
 
-	redirectURI := session.GetValue("oauth_redirect_uri")
-
-	if redirectURI != nil {
-		state := session.GetValue("state")
-		var stateParam = ""
-
-		if state != nil {
-			stateParam = "&state=" + state.(string)
-		}
-
-		session.DeleteValue("oauth_redirect_uri")
-		session.DeleteValue("state")
-
-		http.Redirect(w, r, "/user/authorize?redirect_uri="+redirectURI.(string)+stateParam, http.StatusSeeOther)
+	if redirectParam != "" {
+		http.Redirect(w, r, "/user/authorize"+redirectParam, http.StatusSeeOther)
 		return
 	}
 
@@ -316,7 +332,6 @@ func Logout(w http.ResponseWriter, r *http.Request) {
 	session.SetValue("subscription", "")
 	session.SetValue("authenticated", false)
 	session.DeleteValue("user_id")
-	session.DeleteValue("oauth_redirect_uri")
 
 	err = session.SCS.Destroy(r.Context())
 	if err != nil {