Merge pull request #2521 from gobitfly/BIDS-2434/Attestation_inclusio…

…n_distance_wrong

(BIDS-2434) Fixed inclusion distance for missed slots

db/bigtable.go

@@ -851,16 +851,6 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Minute*5))
 	defer cancel()
 
-	slots := []uint64{}
-
-	for slot := startEpoch * utils.Config.Chain.Config.SlotsPerEpoch; slot < (endEpoch+1)*utils.Config.Chain.Config.SlotsPerEpoch; slot++ {
-		slots = append(slots, slot)
-	}
-	orphanedSlotsMap, err := GetOrphanedSlotsMap(slots)
-	if err != nil {
-		return nil, err
-	}
-
 	ranges := bigtable.getSlotRanges(startEpoch, endEpoch)
 	res := make(map[uint64][]*types.ValidatorAttestation, len(validators))
 
@@ -886,7 +876,14 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st
 	if len(columnFilters) == 0 { // special case to retrieve data for all validators
 		filter = gcp_bigtable.FamilyFilter(ATTESTATIONS_FAMILY)
 	}
-	err = bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool {
+
+	maxSlot := (endEpoch + 1) * utils.Config.Chain.Config.SlotsPerEpoch
+	// map with structure attestationsMap[validator][attesterSlot]
+	attestationsMap := make(map[uint64]map[uint64][]*types.ValidatorAttestation)
+
+	// Save info for all inclusionSlot for attestations in attestationsMap
+	// Set the maxSlot to the highest inclusionSlot
+	err := bigtable.tableBeaconchain.ReadRows(ctx, ranges, func(r gcp_bigtable.Row) bool {
 		keySplit := strings.Split(r.Key(), ":")
 
 		attesterSlot, err := strconv.ParseUint(keySplit[4], 10, 64)
@@ -902,8 +899,10 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st
 			if inclusionSlot == max_block_number {
 				inclusionSlot = 0
 				status = 0
-			} else if orphanedSlotsMap[inclusionSlot] {
-				status = 0
+			}
+
+			if inclusionSlot > maxSlot {
+				maxSlot = inclusionSlot
 			}
 
 			validator, err := strconv.ParseUint(strings.TrimPrefix(ri.Column, ATTESTATIONS_FAMILY+":"), 10, 64)
@@ -912,35 +911,93 @@ func (bigtable *Bigtable) GetValidatorAttestationHistory(validators []uint64, st
 				return false
 			}
 
-			if res[validator] == nil {
-				res[validator] = make([]*types.ValidatorAttestation, 0)
+			if attestationsMap[validator] == nil {
+				attestationsMap[validator] = make(map[uint64][]*types.ValidatorAttestation)
 			}
 
-			if len(res[validator]) > 0 && res[validator][len(res[validator])-1].AttesterSlot == attesterSlot {
-				// don't override successful attestion, that was included in a different slot
-				if status == 1 && res[validator][len(res[validator])-1].Status != 1 {
-					res[validator][len(res[validator])-1].InclusionSlot = inclusionSlot
-					res[validator][len(res[validator])-1].Status = status
-				}
-			} else {
-				res[validator] = append(res[validator], &types.ValidatorAttestation{
-					Index:          validator,
-					Epoch:          attesterSlot / utils.Config.Chain.Config.SlotsPerEpoch,
-					AttesterSlot:   attesterSlot,
-					CommitteeIndex: 0,
-					Status:         status,
-					InclusionSlot:  inclusionSlot,
-					Delay:          int64(inclusionSlot) - int64(attesterSlot) - 1,
-				})
+			if attestationsMap[validator][attesterSlot] == nil {
+				attestationsMap[validator][attesterSlot] = make([]*types.ValidatorAttestation, 0)
 			}
 
+			attestationsMap[validator][attesterSlot] = append(attestationsMap[validator][attesterSlot], &types.ValidatorAttestation{
+				InclusionSlot: inclusionSlot,
+				Status:        status,
+			})
 		}
 		return true
 	}, gcp_bigtable.RowFilter(filter))
 	if err != nil {
 		return nil, err
 	}
 
+	// Find all missed and orphaned slots
+	slots := []uint64{}
+	for slot := startEpoch * utils.Config.Chain.Config.SlotsPerEpoch; slot <= maxSlot; slot++ {
+		slots = append(slots, slot)
+	}
+
+	var missedSlotsMap map[uint64]bool
+	var orphanedSlotsMap map[uint64]bool
+
+	g := new(errgroup.Group)
+
+	g.Go(func() error {
+		missedSlotsMap, err = GetMissedSlotsMap(slots)
+		return err
+	})
+
+	g.Go(func() error {
+		orphanedSlotsMap, err = GetOrphanedSlotsMap(slots)
+		return err
+	})
+	err = g.Wait()
+	if err != nil {
+		return nil, err
+	}
+
+	// Convert the attestationsMap info to the return format
+	// Set the delay of the inclusionSlot
+	for validator, attestations := range attestationsMap {
+		if res[validator] == nil {
+			res[validator] = make([]*types.ValidatorAttestation, 0)
+		}
+		for attesterSlot, att := range attestations {
+			currentAttInfo := att[0]
+			for _, attInfo := range att {
+				if orphanedSlotsMap[attInfo.InclusionSlot] {
+					attInfo.Status = 0
+				}
+
+				if currentAttInfo.Status != 1 && attInfo.Status == 1 {
+					currentAttInfo.Status = attInfo.Status
+					currentAttInfo.InclusionSlot = attInfo.InclusionSlot
+				}
+			}
+
+			missedSlotsCount := uint64(0)
+			for slot := attesterSlot + 1; slot < currentAttInfo.InclusionSlot; slot++ {
+				if missedSlotsMap[slot] || orphanedSlotsMap[slot] {
+					missedSlotsCount++
+				}
+			}
+			currentAttInfo.Index = validator
+			currentAttInfo.Epoch = attesterSlot / utils.Config.Chain.Config.SlotsPerEpoch
+			currentAttInfo.CommitteeIndex = 0
+			currentAttInfo.AttesterSlot = attesterSlot
+			currentAttInfo.Delay = int64(currentAttInfo.InclusionSlot - attesterSlot - missedSlotsCount - 1)
+
+			res[validator] = append(res[validator], currentAttInfo)
+		}
+	}
+
+	// Sort the result by attesterSlot desc
+	for validator, att := range res {
+		sort.Slice(att, func(i, j int) bool {
+			return att[i].AttesterSlot > att[j].AttesterSlot
+		})
+		res[validator] = att
+	}
+
 	return res, nil
 }
 

db/db.go

@@ -3302,6 +3302,32 @@ func GetValidatorPropsosals(validators []uint64, proposals *[]types.ValidatorPro
 		`, validatorsPQArray)
 }
 
+func GetMissedSlots(slots []uint64) ([]uint64, error) {
+	slotsPQArray := pq.Array(slots)
+	missed := []uint64{}
+
+	err := ReaderDb.Select(&missed, `
+		SELECT
+			slot
+		FROM blocks
+		WHERE slot = ANY($1) AND status = '2'
+		`, slotsPQArray)
+
+	return missed, err
+}
+
+func GetMissedSlotsMap(slots []uint64) (map[uint64]bool, error) {
+	missedSlots, err := GetMissedSlots(slots)
+	if err != nil {
+		return nil, err
+	}
+	missedSlotsMap := make(map[uint64]bool, len(missedSlots))
+	for _, slot := range missedSlots {
+		missedSlotsMap[slot] = true
+	}
+	return missedSlotsMap, nil
+}
+
 func GetOrphanedSlots(slots []uint64) ([]uint64, error) {
 	slotsPQArray := pq.Array(slots)
 	orphaned := []uint64{}
@@ -3321,7 +3347,7 @@ func GetOrphanedSlotsMap(slots []uint64) (map[uint64]bool, error) {
 	if err != nil {
 		return nil, err
 	}
-	orphanedSlotsMap := make(map[uint64]bool)
+	orphanedSlotsMap := make(map[uint64]bool, len(orphanedSlots))
 	for _, slot := range orphanedSlots {
 		orphanedSlotsMap[slot] = true
 	}

handlers/validator.go

@@ -1986,17 +1986,12 @@ func ValidatorSync(w http.ResponseWriter, r *http.Request) {
 		}
 
 		// Search for the missed slots (status = 2), to see if it was only our validator that missed the slot or if the block was missed
-		missedSlots := []uint64{}
-		err = db.ReaderDb.Select(&missedSlots, `SELECT slot FROM blocks WHERE slot = ANY($1) AND status = '2'`, missedSyncSlots)
+		missedSlotsMap, err := db.GetMissedSlotsMap(missedSyncSlots)
 		if err != nil {
 			logger.WithError(err).Errorf("error getting missed slots data")
 			http.Error(w, "Internal server error", http.StatusInternalServerError)
 			return
 		}
-		missedSlotsMap := make(map[uint64]bool, len(missedSlots))
-		for _, slot := range missedSlots {
-			missedSlotsMap[slot] = true
-		}
 
 		// extract correct slots
 		tableData = make([][]interface{}, length)