(NOBIDS) fix sync committee committee_index order

exporter/sync_committees.go

@@ -75,6 +75,8 @@ func exportSyncCommitteeAtPeriod(rpcClient rpc.Client, p uint64) error {
 
 	logger.Infof("exporting sync committee assignments for period %v (epoch %v to %v)", p, firstEpoch, lastEpoch)
 
+	// Note that the order we receive the validators from the node in is crucial
+	// and determines which bit reflects them in the block sync aggregate bits
 	c, err := rpcClient.GetSyncCommittee(fmt.Sprintf("%d", stateID), epoch)
 	if err != nil {
 		return err
@@ -89,17 +91,6 @@ func exportSyncCommitteeAtPeriod(rpcClient rpc.Client, p uint64) error {
 		validatorsU64[i] = idxU64
 	}
 
-	dedupMap := make(map[uint64]bool)
-
-	for _, validator := range validatorsU64 {
-		dedupMap[validator] = true
-	}
-
-	validatorsU64 = make([]uint64, 0, len(dedupMap))
-	for validator := range dedupMap {
-		validatorsU64 = append(validatorsU64, validator)
-	}
-
 	// start := time.Now()
 	//
 	// firstSlot := firstEpoch * utils.Config.Chain.ClConfig.SlotsPerEpoch

handlers/api.go

@@ -732,6 +732,8 @@ func ApiSyncCommittee(w http.ResponseWriter, r *http.Request) {
 		period = utils.SyncPeriodOfEpoch(services.LatestEpoch()) + 1
 	}
 
+	// Beware that we do not deduplicate here since a validator can be part multiple times of the same sync committee period
+	// and the order of the committeeindex is important, deduplicating it would mess up the order
 	rows, err := db.ReaderDb.Query(`SELECT period, GREATEST(period*$2, $3) AS start_epoch, ((period+1)*$2)-1 AS end_epoch, ARRAY_AGG(validatorindex ORDER BY committeeindex) AS validators FROM sync_committees WHERE period = $1 GROUP BY period`, period, utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, utils.Config.Chain.ClConfig.AltairForkEpoch)
 	if err != nil {
 		logger.WithError(err).WithField("url", r.URL.String()).Errorf("error querying db")
@@ -1020,15 +1022,24 @@ func ApiDashboard(w http.ResponseWriter, r *http.Request) {
 }
 
 func getSyncCommitteeInfoForValidators(validators []uint64, period uint64) ([]interface{}, error) {
-	rows, err := db.ReaderDb.Query(
-		`SELECT 
-			period, 
-			GREATEST(period*$3, $4) AS start_epoch, 
-			((period+1)*$3)-1 AS end_epoch, 
-			ARRAY_AGG(validatorindex ORDER BY committeeindex) AS validators 
-		FROM sync_committees 
-		WHERE period = $1 AND validatorindex = ANY($2)
-		GROUP BY period`,
+	rows, err := db.ReaderDb.Query(`
+			WITH
+				data as (
+					SELECT 
+						period,
+						validatorindex,
+						max(committeeindex) as committeeindex
+					FROM sync_committees 
+					WHERE period = $1 AND validatorindex = ANY($2)
+					group by period, validatorindex
+				)	
+			SELECT 
+				period, 
+				GREATEST(period*$3, $4) AS start_epoch, 
+				((period+1)*$3)-1 AS end_epoch, 
+				ARRAY_AGG(validatorindex ORDER BY committeeindex) AS validators 
+			FROM data 	
+			group by period;`,
 		period, pq.Array(validators),
 		utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, utils.Config.Chain.ClConfig.AltairForkEpoch,
 	)
@@ -1218,7 +1229,7 @@ func getSyncCommitteeSlotsStatistics(validators []uint64, epoch uint64) (types.S
 			Validators pq.Int64Array `db:"validators"`
 		}
 		query, args, err := sqlx.In(`
-			SELECT period, COALESCE(ARRAY_AGG(validatorindex), '{}') AS validators
+			SELECT period, COALESCE(ARRAY_AGG(distinct validatorindex), '{}') AS validators
 			FROM sync_committees
 			WHERE period IN (?) AND validatorindex IN (?)
 			GROUP BY period

handlers/validator.go

@@ -1992,7 +1992,7 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) {
 	var syncPeriods []uint64 = []uint64{}
 
 	err = db.ReaderDb.Select(&syncPeriods, `
-		SELECT period
+		SELECT distinct period
 		FROM sync_committees 
 		WHERE validatorindex = $1
 		ORDER BY period desc`, validatorIndex)