From 6390b2525ac9b3e0dc305fceed0772c7072150fa Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Thu, 2 Apr 2020 12:25:58 -0400 Subject: [PATCH 01/12] Complete most of changes --- proto/slashing/slashing.pb.go | 6 +- proto/slashing/slashing.proto | 9 -- validator/client/validator_propose.go | 54 +------- validator/db/BUILD.bazel | 2 + validator/db/db.go | 23 +--- validator/db/iface/BUILD.bazel | 5 +- validator/db/iface/interface.go | 5 +- validator/db/proposal_history.go | 89 ++++++++----- validator/db/proposal_history_test.go | 174 +++++++++++++++----------- 9 files changed, 186 insertions(+), 181 deletions(-) diff --git a/proto/slashing/slashing.pb.go b/proto/slashing/slashing.pb.go index 260dff6ac0d2..35d7ea93195e 100755 --- a/proto/slashing/slashing.pb.go +++ b/proto/slashing/slashing.pb.go @@ -420,7 +420,7 @@ func init() { proto.RegisterType((*MinMaxEpochSpan)(nil), "ethereum.slashing.MinMaxEpochSpan") proto.RegisterType((*EpochSpanMap)(nil), "ethereum.slashing.EpochSpanMap") proto.RegisterMapType((map[uint64]*MinMaxEpochSpan)(nil), "ethereum.slashing.EpochSpanMap.EpochSpanMapEntry") - proto.RegisterType((*ProposalHistory)(nil), "ethereum.slashing.ProposalHistory") + proto.RegisterType((*ProposalHistory)(nil), "ethereum.slashing.ProposalHistoryForEpoch") proto.RegisterType((*AttestationHistory)(nil), "ethereum.slashing.AttestationHistory") proto.RegisterMapType((map[uint64]uint64)(nil), "ethereum.slashing.AttestationHistory.TargetToSourceEntry") proto.RegisterType((*SlashingStatusRequest)(nil), "ethereum.slashing.SlashingStatusRequest") @@ -1491,10 +1491,10 @@ func (m *ProposalHistory) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: ProposalHistory: wiretype end group for non-group") + return fmt.Errorf("proto: ProposalHistoryForEpoch: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: ProposalHistory: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ProposalHistoryForEpoch: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: diff --git a/proto/slashing/slashing.proto b/proto/slashing/slashing.proto index e4c2b0a664d5..0ae5c7755274 100644 --- a/proto/slashing/slashing.proto +++ b/proto/slashing/slashing.proto @@ -45,15 +45,6 @@ message EpochSpanMap { map epoch_span_map = 1; } -// ProposalHistory defines the structure for recording a validator's historical proposals. -// Using a bitlist to represent the epochs and an uint64 to mark the latest marked -// epoch of the bitlist, we can easily store which epochs a validator has proposed -// a block for while pruning the older data. -message ProposalHistory { - bytes epoch_bits = 1 [(gogoproto.casttype) = "github.com/prysmaticlabs/go-bitfield.Bitlist"]; - uint64 latest_epoch_written = 2; -} - // AttestationHistory defines the structure for recording a validator's historical attestation. // Using a map[uint64]uint64 to map its target epoch to its source epoch, in order to detect if a // vote being created is not a double vote and surrounded by, or surrounding any other votes. diff --git a/validator/client/validator_propose.go b/validator/client/validator_propose.go index 0c25c732ca60..f709ed13178b 100644 --- a/validator/client/validator_propose.go +++ b/validator/client/validator_propose.go @@ -12,7 +12,6 @@ import ( ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" "github.com/prysmaticlabs/go-ssz" "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" - slashpb "github.com/prysmaticlabs/prysm/proto/slashing" "github.com/prysmaticlabs/prysm/shared/bls" "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/featureconfig" @@ -88,7 +87,7 @@ func (v *validator) ProposeBlock(ctx context.Context, slot uint64, pubKey [48]by } if featureconfig.Get().ProtectProposer { - history, err := v.db.ProposalHistory(ctx, pubKey[:]) + slotBits, err := v.db.ProposalHistoryForEpoch(ctx, pubKey[:], epoch) if err != nil { log.WithError(err).Error("Failed to get proposal history") if v.emitAccountMetrics { @@ -97,7 +96,8 @@ func (v *validator) ProposeBlock(ctx context.Context, slot uint64, pubKey [48]by return } - if HasProposedForEpoch(history, epoch) { + // If the bit for the current slot is marked, do not propose. + if slotBits.BitAt(slot % params.BeaconConfig().SlotsPerEpoch) { log.WithField("epoch", epoch).Warn("Tried to sign a double proposal, rejected") if v.emitAccountMetrics { validatorProposeFailVec.WithLabelValues(fmtKey).Inc() @@ -131,7 +131,7 @@ func (v *validator) ProposeBlock(ctx context.Context, slot uint64, pubKey [48]by } if featureconfig.Get().ProtectProposer { - history, err := v.db.ProposalHistory(ctx, pubKey[:]) + slotBits, err := v.db.ProposalHistoryForEpoch(ctx, pubKey[:], epoch) if err != nil { log.WithError(err).Error("Failed to get proposal history") if v.emitAccountMetrics { @@ -139,8 +139,8 @@ func (v *validator) ProposeBlock(ctx context.Context, slot uint64, pubKey [48]by } return } - history = SetProposedForEpoch(history, epoch) - if err := v.db.SaveProposalHistory(ctx, pubKey[:], history); err != nil { + slotBits.SetBitAt(slot%params.BeaconConfig().SlotsPerEpoch, true) + if err := v.db.SaveProposalHistoryForEpoch(ctx, pubKey[:], epoch, slotBits); err != nil { log.WithError(err).Error("Failed to save updated proposal history") if v.emitAccountMetrics { validatorProposeFailVec.WithLabelValues(fmtKey).Inc() @@ -227,45 +227,3 @@ func (v *validator) signBlock(ctx context.Context, pubKey [48]byte, epoch uint64 } return sig.Marshal(), nil } - -// HasProposedForEpoch returns whether a validators proposal history has been marked for the entered epoch. -// If the request is more in the future than what the history contains, it will return false. -// If the request is from the past, and likely previously pruned it will return false. -func HasProposedForEpoch(history *slashpb.ProposalHistory, epoch uint64) bool { - wsPeriod := params.BeaconConfig().WeakSubjectivityPeriod - // Previously pruned, we should return false. - if int(epoch) <= int(history.LatestEpochWritten)-int(wsPeriod) { - return false - } - // Accessing future proposals that haven't been marked yet. Needs to return false. - if epoch > history.LatestEpochWritten { - return false - } - return history.EpochBits.BitAt(epoch % wsPeriod) -} - -// SetProposedForEpoch updates the proposal history to mark the indicated epoch in the bitlist -// and updates the last epoch written if needed. -// Returns the modified proposal history. -func SetProposedForEpoch(history *slashpb.ProposalHistory, epoch uint64) *slashpb.ProposalHistory { - wsPeriod := params.BeaconConfig().WeakSubjectivityPeriod - - if epoch > history.LatestEpochWritten { - // If the history is empty, just update the latest written and mark the epoch. - // This is for the first run of a validator. - if history.EpochBits.Count() < 1 { - history.LatestEpochWritten = epoch - history.EpochBits.SetBitAt(epoch%wsPeriod, true) - return history - } - // If the epoch to mark is ahead of latest written epoch, override the old votes and mark the requested epoch. - // Limit the overwriting to one weak subjectivity period as further is not needed. - maxToWrite := history.LatestEpochWritten + wsPeriod - for i := history.LatestEpochWritten + 1; i < epoch && i <= maxToWrite; i++ { - history.EpochBits.SetBitAt(i%wsPeriod, false) - } - history.LatestEpochWritten = epoch - } - history.EpochBits.SetBitAt(epoch%wsPeriod, true) - return history -} diff --git a/validator/db/BUILD.bazel b/validator/db/BUILD.bazel index 98efb1bcb8d5..2942ff58049e 100644 --- a/validator/db/BUILD.bazel +++ b/validator/db/BUILD.bazel @@ -20,6 +20,7 @@ go_library( "@com_github_pkg_errors//:go_default_library", "@com_github_prysmaticlabs_go_bitfield//:go_default_library", "@com_github_sirupsen_logrus//:go_default_library", + "@com_github_wealdtech_go_bytesutil//:go_default_library", "@io_opencensus_go//trace:go_default_library", ], ) @@ -33,6 +34,7 @@ go_test( ], embed = [":go_default_library"], deps = [ + "//beacon-chain/core/helpers:go_default_library", "//proto/slashing:go_default_library", "//shared/params:go_default_library", "@com_github_prysmaticlabs_go_bitfield//:go_default_library", diff --git a/validator/db/db.go b/validator/db/db.go index dc637eb33c16..1f9670ae77bb 100644 --- a/validator/db/db.go +++ b/validator/db/db.go @@ -8,7 +8,6 @@ import ( "github.com/boltdb/bolt" "github.com/pkg/errors" - "github.com/prysmaticlabs/go-bitfield" slashpb "github.com/prysmaticlabs/prysm/proto/slashing" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/validator/db/iface" @@ -68,7 +67,7 @@ func createBuckets(tx *bolt.Tx, buckets ...[]byte) error { // NewKVStore initializes a new boltDB key-value store at the directory // path specified, creates the kv-buckets based on the schema, and stores // an open connection db object as a property of the Store struct. -func NewKVStore(dirPath string, pubkeys [][48]byte) (*Store, error) { +func NewKVStore(dirPath string, pubKeys [][48]byte) (*Store, error) { if err := os.MkdirAll(dirPath, 0700); err != nil { return nil, err } @@ -93,21 +92,11 @@ func NewKVStore(dirPath string, pubkeys [][48]byte) (*Store, error) { return nil, err } - // Initialize the required pubkeys into the DB to ensure they're not empty. - for _, pubkey := range pubkeys { - proHistory, err := kv.ProposalHistory(context.Background(), pubkey[:]) - if err != nil { - return nil, err - } - if proHistory == nil { - cleanHistory := &slashpb.ProposalHistory{ - EpochBits: bitfield.NewBitlist(params.BeaconConfig().WeakSubjectivityPeriod), - } - if err := kv.SaveProposalHistory(context.Background(), pubkey[:], cleanHistory); err != nil { - return nil, err - } - } - + // Initialize the required pubKeys into the DB to ensure they're not empty. + if err := kv.initializeSubBuckets(pubKeys); err != nil { + return nil, err + } + for _, pubkey := range pubKeys { attHistory, err := kv.AttestationHistory(context.Background(), pubkey[:]) if err != nil { return nil, err diff --git a/validator/db/iface/BUILD.bazel b/validator/db/iface/BUILD.bazel index 987dc8362748..d999fb9accfc 100644 --- a/validator/db/iface/BUILD.bazel +++ b/validator/db/iface/BUILD.bazel @@ -6,5 +6,8 @@ go_library( importpath = "github.com/prysmaticlabs/prysm/validator/db/iface", # Other packages must use github.com/prysmaticlabs/prysm/validator/db.Database alias. visibility = ["//validator/db:__subpackages__"], - deps = ["//proto/slashing:go_default_library"], + deps = [ + "//proto/slashing:go_default_library", + "@com_github_prysmaticlabs_go_bitfield//:go_default_library", + ], ) diff --git a/validator/db/iface/interface.go b/validator/db/iface/interface.go index 6bf8ad131e6a..0b4d1b3af051 100644 --- a/validator/db/iface/interface.go +++ b/validator/db/iface/interface.go @@ -5,6 +5,7 @@ import ( "context" "io" + "github.com/prysmaticlabs/go-bitfield" slashpb "github.com/prysmaticlabs/prysm/proto/slashing" ) @@ -14,8 +15,8 @@ type ValidatorDB interface { DatabasePath() string ClearDB() error // Proposer protection related methods. - ProposalHistory(ctx context.Context, publicKey []byte) (*slashpb.ProposalHistory, error) - SaveProposalHistory(ctx context.Context, publicKey []byte, history *slashpb.ProposalHistory) error + ProposalHistoryForEpoch(ctx context.Context, publicKey []byte, epoch uint64) (bitfield.Bitlist, error) + SaveProposalHistoryForEpoch(ctx context.Context, publicKey []byte, epoch uint64, history bitfield.Bitlist) error DeleteProposalHistory(ctx context.Context, publicKey []byte) error // Attester protection related methods. AttestationHistory(ctx context.Context, publicKey []byte) (*slashpb.AttestationHistory, error) diff --git a/validator/db/proposal_history.go b/validator/db/proposal_history.go index f951fa891649..27397100f143 100644 --- a/validator/db/proposal_history.go +++ b/validator/db/proposal_history.go @@ -2,56 +2,57 @@ package db import ( "context" + "encoding/binary" + "fmt" "github.com/boltdb/bolt" - "github.com/gogo/protobuf/proto" "github.com/pkg/errors" - slashpb "github.com/prysmaticlabs/prysm/proto/slashing" + "github.com/prysmaticlabs/go-bitfield" + "github.com/prysmaticlabs/prysm/shared/params" + "github.com/wealdtech/go-bytesutil" "go.opencensus.io/trace" ) -func unmarshalProposalHistory(enc []byte) (*slashpb.ProposalHistory, error) { - history := &slashpb.ProposalHistory{} - err := proto.Unmarshal(enc, history) - if err != nil { - return nil, errors.Wrap(err, "failed to unmarshal encoding") - } - return history, nil -} - -// ProposalHistory accepts a validator public key and returns the corresponding proposal history. +// ProposalHistoryForEpoch accepts a validator public key and returns the corresponding proposal history. // Returns nil if there is no proposal history for the validator. -func (db *Store) ProposalHistory(ctx context.Context, publicKey []byte) (*slashpb.ProposalHistory, error) { - ctx, span := trace.StartSpan(ctx, "Validator.ProposalHistory") +func (db *Store) ProposalHistoryForEpoch(ctx context.Context, publicKey []byte, epoch uint64) (bitfield.Bitlist, error) { + ctx, span := trace.StartSpan(ctx, "Validator.ProposalHistoryForEpoch") defer span.End() var err error - var proposalHistory *slashpb.ProposalHistory + var slotBitlist bitfield.Bitlist err = db.view(func(tx *bolt.Tx) error { bucket := tx.Bucket(historicProposalsBucket) - enc := bucket.Get(publicKey) - if enc == nil { + valBucket := bucket.Bucket(publicKey) + slotBits := valBucket.Get(bytesutil.Bytes8(epoch)) + if slotBits == nil || len(slotBits) == 0 { + slotBitlist = bitfield.NewBitlist(params.BeaconConfig().SlotsPerEpoch) return nil } - proposalHistory, err = unmarshalProposalHistory(enc) - return err + slotBitlist = slotBits + return nil }) - return proposalHistory, err + return slotBitlist, err } -// SaveProposalHistory returns the proposal history for the requested validator public key. -func (db *Store) SaveProposalHistory(ctx context.Context, pubKey []byte, proposalHistory *slashpb.ProposalHistory) error { - ctx, span := trace.StartSpan(ctx, "Validator.SaveProposalHistory") +// SaveProposalHistoryForEpoch returns the proposal history for the requested validator public key. +func (db *Store) SaveProposalHistoryForEpoch(ctx context.Context, pubKey []byte, epoch uint64, slotBits bitfield.Bitlist) error { + ctx, span := trace.StartSpan(ctx, "Validator.SaveProposalHistoryForEpoch") defer span.End() - enc, err := proto.Marshal(proposalHistory) - if err != nil { - return errors.Wrap(err, "failed to encode proposal history") - } - - err = db.update(func(tx *bolt.Tx) error { + err := db.update(func(tx *bolt.Tx) error { bucket := tx.Bucket(historicProposalsBucket) - return bucket.Put(pubKey, enc) + valBucket := bucket.Bucket(pubKey) + if valBucket == nil { + return fmt.Errorf("validator history is empty for validator %#x", pubKey) + } + if err := valBucket.Put(bytesutil.Bytes8(epoch), slotBits); err != nil { + return err + } + //if err := pruneProposalHistory(valBucket, epoch); err != nil { + // return err + //} + return nil }) return err } @@ -69,3 +70,31 @@ func (db *Store) DeleteProposalHistory(ctx context.Context, pubkey []byte) error return nil }) } + +func pruneProposalHistory(valBucket *bolt.Bucket, newestEpoch uint64) error { + c := valBucket.Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + epoch := binary.LittleEndian.Uint64(k) + // Only delete epochs that are older than the weak subjectivity period. + if epoch+params.BeaconConfig().WeakSubjectivityPeriod < newestEpoch { + if err := valBucket.Delete(k); err != nil { + return errors.Wrapf(err, "could not prune epoch %d in proposal history", epoch) + } + } else { + return nil + } + } + return nil +} + +func (db *Store) initializeSubBuckets(pubKeys [][48]byte) error { + return db.update(func(tx *bolt.Tx) error { + bucket := tx.Bucket(historicProposalsBucket) + for _, pubKey := range pubKeys { + if _, err := bucket.CreateBucketIfNotExists(pubKey[:]); err != nil { + return errors.Wrap(err, "failed to delete the proposal history") + } + } + return nil + }) +} diff --git a/validator/db/proposal_history_test.go b/validator/db/proposal_history_test.go index f5d08c7846a7..445733bd4ccf 100644 --- a/validator/db/proposal_history_test.go +++ b/validator/db/proposal_history_test.go @@ -1,169 +1,201 @@ package db import ( + "bytes" "context" "reflect" "testing" "github.com/prysmaticlabs/go-bitfield" - slashpb "github.com/prysmaticlabs/prysm/proto/slashing" + "github.com/prysmaticlabs/prysm/beacon-chain/core/helpers" "github.com/prysmaticlabs/prysm/shared/params" ) -func TestProposalHistory_InitializesNewPubKeys(t *testing.T) { +func TestProposalHistoryForEpoch_InitializesNewPubKeys(t *testing.T) { pubkeys := [][48]byte{{30}, {25}, {20}} db := SetupDB(t, pubkeys) defer TeardownDB(t, db) for _, pub := range pubkeys { - proposalHistory, err := db.ProposalHistory(context.Background(), pub[:]) + slotBits, err := db.ProposalHistoryForEpoch(context.Background(), pub[:], 0) if err != nil { t.Fatal(err) } - clean := &slashpb.ProposalHistory{ - EpochBits: bitfield.NewBitlist(params.BeaconConfig().WeakSubjectivityPeriod), - } - if !reflect.DeepEqual(proposalHistory, clean) { - t.Fatalf("Expected proposal history epoch bits to be empty, received %v", proposalHistory) + cleanBits := bitfield.NewBitlist(params.BeaconConfig().SlotsPerEpoch) + if !bytes.Equal(slotBits.Bytes(), cleanBits.Bytes()) { + t.Fatalf("Expected proposal history slot bits to be empty, received %v", slotBits.Bytes()) } } } -func TestProposalHistory_NilDB(t *testing.T) { +func TestProposalHistoryForEpoch_NilDB(t *testing.T) { db := SetupDB(t, [][48]byte{}) defer TeardownDB(t, db) valPubkey := []byte{1, 2, 3} - proposalHistory, err := db.ProposalHistory(context.Background(), valPubkey) + slotBits, err := db.ProposalHistoryForEpoch(context.Background(), valPubkey, 0) if err != nil { t.Fatal(err) } - if proposalHistory != nil { - t.Fatalf("Expected proposal history to be nil, received: %v", proposalHistory) + if slotBits != nil { + t.Fatalf("Expected slot bits for epoch to be nil, received: %v", slotBits) } } -func TestSaveProposalHistory_OK(t *testing.T) { +func TestSaveProposalHistoryForEpoch_OK(t *testing.T) { db := SetupDB(t, [][48]byte{}) defer TeardownDB(t, db) pubkey := []byte{3} epoch := uint64(2) - history := &slashpb.ProposalHistory{ - EpochBits: bitfield.Bitlist{0x04, 0x04}, - LatestEpochWritten: 2, - } + slot := uint64(2) + slotBits := bitfield.Bitlist{0x04, 0x04} - if err := db.SaveProposalHistory(context.Background(), pubkey, history); err != nil { + if err := db.SaveProposalHistoryForEpoch(context.Background(), pubkey, epoch, slotBits); err != nil { t.Fatalf("Saving proposal history failed: %v", err) } - savedHistory, err := db.ProposalHistory(context.Background(), pubkey) + savedBits, err := db.ProposalHistoryForEpoch(context.Background(), pubkey, epoch) if err != nil { t.Fatalf("Failed to get proposal history: %v", err) } - if savedHistory == nil || !reflect.DeepEqual(history, savedHistory) { - t.Fatalf("Expected DB to keep object the same, received: %v", history) + if savedBits == nil || !bytes.Equal(slotBits.Bytes(), savedBits.Bytes()) { + t.Fatalf("Expected DB to keep object the same, received: %v", savedBits) } - if !savedHistory.EpochBits.BitAt(epoch) { - t.Fatalf("Expected epoch %d to be marked as proposed", history.EpochBits.Count()) + if !savedBits.BitAt(slot) { + t.Fatalf("Expected slot %d to be marked as proposed", slot) } - if savedHistory.EpochBits.BitAt(epoch + 1) { - t.Fatalf("Expected epoch %d to not be marked as proposed", epoch+1) + if savedBits.BitAt(slot + 1) { + t.Fatalf("Expected slot %d to not be marked as proposed", slot+1) } - if savedHistory.EpochBits.BitAt(epoch - 1) { - t.Fatalf("Expected epoch %d to not be marked as proposed", epoch-1) + if savedBits.BitAt(slot - 1) { + t.Fatalf("Expected slot %d to not be marked as proposed", slot-1) } } -func TestSaveProposalHistory_Overwrites(t *testing.T) { - db := SetupDB(t, [][48]byte{}) - defer TeardownDB(t, db) +func TestSaveProposalHistoryForEpoch_Overwrites(t *testing.T) { + pubkey := [48]byte{0} tests := []struct { - pubkey []byte - epoch uint64 - history *slashpb.ProposalHistory + slot uint64 + slotBits bitfield.Bitlist }{ { - pubkey: []byte{0}, - epoch: uint64(1), - history: &slashpb.ProposalHistory{ - EpochBits: bitfield.Bitlist{0x02, 0x02}, - LatestEpochWritten: 1, - }, + slot: uint64(1), + slotBits: bitfield.Bitlist{0x02, 0x02}, }, { - pubkey: []byte{0}, - epoch: uint64(2), - history: &slashpb.ProposalHistory{ - EpochBits: bitfield.Bitlist{0x04, 0x04}, - LatestEpochWritten: 2, - }, + slot: uint64(2), + slotBits: bitfield.Bitlist{0x04, 0x04}, }, { - pubkey: []byte{0}, - epoch: uint64(3), - history: &slashpb.ProposalHistory{ - EpochBits: bitfield.Bitlist{0x08, 0x08}, - LatestEpochWritten: 3, - }, + slot: uint64(3), + slotBits: bitfield.Bitlist{0x08, 0x08}, }, } for _, tt := range tests { - if err := db.SaveProposalHistory(context.Background(), tt.pubkey, tt.history); err != nil { + db := SetupDB(t, [][48]byte{pubkey}) + defer TeardownDB(t, db) + if err := db.SaveProposalHistoryForEpoch(context.Background(), pubkey[:], 0, tt.slotBits); err != nil { t.Fatalf("Saving proposal history failed: %v", err) } - history, err := db.ProposalHistory(context.Background(), tt.pubkey) + savedBits, err := db.ProposalHistoryForEpoch(context.Background(), pubkey[:], 0) if err != nil { t.Fatalf("Failed to get proposal history: %v", err) } - if history == nil || !reflect.DeepEqual(history, tt.history) { - t.Fatalf("Expected DB to keep object the same, received: %v", history) + if savedBits == nil || !reflect.DeepEqual(savedBits.Bytes(), tt.slotBits.Bytes()) { + t.Fatalf("Expected DB to keep object the same, received: %v, expected %v", savedBits.Bytes(), tt.slotBits.Bytes()) + } + if !savedBits.BitAt(tt.slot) { + t.Fatalf("Expected slot %d to be marked as proposed", tt.slot) + } + if savedBits.BitAt(tt.slot + 1) { + t.Fatalf("Expected slot %d to not be marked as proposed", tt.slot+1) } - if !history.EpochBits.BitAt(tt.epoch) { - t.Fatalf("Expected epoch %d to be marked as proposed", history.EpochBits.Count()) + if savedBits.BitAt(tt.slot - 1) { + t.Fatalf("Expected slot %d to not be marked as proposed", tt.slot-1) } - if history.EpochBits.BitAt(tt.epoch + 1) { - t.Fatalf("Expected epoch %d to not be marked as proposed", tt.epoch+1) + } +} + +func TestProposalHistoryForEpoch_MultipleEpochs(t *testing.T) { + pubKey := [48]byte{0} + tests := []struct { + slots []uint64 + expectedBits []bitfield.Bitlist + }{ + { + slots: []uint64{1, 2, 8}, + expectedBits: []bitfield.Bitlist{{0x02, 0x14}}, + }, + { + slots: []uint64{1, 33, 8}, + expectedBits: []bitfield.Bitlist{{0x02, 0x10}, {0x02, 0x04}}, + }, + { + slots: []uint64{2, 34, 36}, + expectedBits: []bitfield.Bitlist{{0x02, 0x04}, {0x02, 0x06}}, + }, + { + slots: []uint64{32, 33, 34}, + expectedBits: []bitfield.Bitlist{{0x02, 0x00}, {0x02, 0x05}}, + }, + } + + for _, tt := range tests { + db := SetupDB(t, [][48]byte{pubKey}) + defer TeardownDB(t, db) + for _, slot := range tt.slots { + slotBits, err := db.ProposalHistoryForEpoch(context.Background(), pubKey[:], helpers.SlotToEpoch(slot)) + if err != nil { + t.Fatalf("Failed to get proposal history: %v", err) + } + slotBits.SetBitAt(slot%params.BeaconConfig().SlotsPerEpoch, true) + if err := db.SaveProposalHistoryForEpoch(context.Background(), pubKey[:], helpers.SlotToEpoch(slot), slotBits); err != nil { + t.Fatalf("Saving proposal history failed: %v", err) + } } - if history.EpochBits.BitAt(tt.epoch - 1) { - t.Fatalf("Expected epoch %d to not be marked as proposed", tt.epoch-1) + + for i, slotBits := range tt.expectedBits { + savedBits, err := db.ProposalHistoryForEpoch(context.Background(), pubKey[:], uint64(i)) + if err != nil { + t.Fatalf("Failed to get proposal history: %v", err) + } + if !bytes.Equal(savedBits.Bytes(), slotBits.Bytes()) { + t.Fatalf("unexpected difference in bytes, expected %#x vs received %#x", savedBits.Bytes(), slotBits.Bytes()) + } } } } -func TestDeleteProposalHistory_OK(t *testing.T) { +func TestDeleteProposalHistoryForVal_OK(t *testing.T) { db := SetupDB(t, [][48]byte{}) defer TeardownDB(t, db) pubkey := []byte{2} - history := &slashpb.ProposalHistory{ - EpochBits: bitfield.Bitlist{0x01, 0x02}, - LatestEpochWritten: 1, - } + slotBits := bitfield.Bitlist{0x01, 0x02} - if err := db.SaveProposalHistory(context.Background(), pubkey, history); err != nil { + if err := db.SaveProposalHistoryForEpoch(context.Background(), pubkey, 0, slotBits); err != nil { t.Fatalf("Save proposal history failed: %v", err) } // Making sure everything is saved. - savedHistory, err := db.ProposalHistory(context.Background(), pubkey) + savedHistory, err := db.ProposalHistoryForEpoch(context.Background(), pubkey, 0) if err != nil { t.Fatalf("Failed to get proposal history: %v", err) } - if savedHistory == nil || !reflect.DeepEqual(savedHistory, history) { - t.Fatalf("Expected DB to keep object the same, received: %v, expected %v", savedHistory, history) + if savedHistory == nil || !bytes.Equal(savedHistory.Bytes(), slotBits.Bytes()) { + t.Fatalf("Expected DB to keep object the same, received: %v, expected %v", savedHistory, slotBits) } if err := db.DeleteProposalHistory(context.Background(), pubkey); err != nil { t.Fatal(err) } // Check after deleting from DB. - savedHistory, err = db.ProposalHistory(context.Background(), pubkey) + savedHistory, err = db.ProposalHistoryForEpoch(context.Background(), pubkey, 0) if err != nil { t.Fatalf("Failed to get proposal history: %v", err) } From 148a0b71f65695c3c7742f74fdf76b4284ff81f6 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Thu, 2 Apr 2020 14:44:57 -0400 Subject: [PATCH 02/12] Fix other tests --- validator/db/proposal_history.go | 5 ++- validator/db/proposal_history_test.go | 44 +++++++++++++-------------- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/validator/db/proposal_history.go b/validator/db/proposal_history.go index 27397100f143..a8ab76e6b059 100644 --- a/validator/db/proposal_history.go +++ b/validator/db/proposal_history.go @@ -24,6 +24,9 @@ func (db *Store) ProposalHistoryForEpoch(ctx context.Context, publicKey []byte, err = db.view(func(tx *bolt.Tx) error { bucket := tx.Bucket(historicProposalsBucket) valBucket := bucket.Bucket(publicKey) + if valBucket == nil { + return fmt.Errorf("validator history empty for public key %#x", publicKey) + } slotBits := valBucket.Get(bytesutil.Bytes8(epoch)) if slotBits == nil || len(slotBits) == 0 { slotBitlist = bitfield.NewBitlist(params.BeaconConfig().SlotsPerEpoch) @@ -64,7 +67,7 @@ func (db *Store) DeleteProposalHistory(ctx context.Context, pubkey []byte) error return db.update(func(tx *bolt.Tx) error { bucket := tx.Bucket(historicProposalsBucket) - if err := bucket.Delete(pubkey); err != nil { + if err := bucket.DeleteBucket(pubkey); err != nil { return errors.Wrap(err, "failed to delete the proposal history") } return nil diff --git a/validator/db/proposal_history_test.go b/validator/db/proposal_history_test.go index 445733bd4ccf..703e375ce33d 100644 --- a/validator/db/proposal_history_test.go +++ b/validator/db/proposal_history_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "reflect" + "strings" "testing" "github.com/prysmaticlabs/go-bitfield" @@ -30,34 +31,33 @@ func TestProposalHistoryForEpoch_InitializesNewPubKeys(t *testing.T) { } func TestProposalHistoryForEpoch_NilDB(t *testing.T) { + valPubkey := [48]byte{1, 2, 3} db := SetupDB(t, [][48]byte{}) defer TeardownDB(t, db) - valPubkey := []byte{1, 2, 3} - - slotBits, err := db.ProposalHistoryForEpoch(context.Background(), valPubkey, 0) - if err != nil { - t.Fatal(err) + _, err := db.ProposalHistoryForEpoch(context.Background(), valPubkey[:], 0) + if err == nil { + t.Fatal("unexpected non-error") } - if slotBits != nil { - t.Fatalf("Expected slot bits for epoch to be nil, received: %v", slotBits) + if !strings.Contains(err.Error(), "validator history empty for public key") { + t.Fatalf("Unexpected error for nil DB, received: %v", err) } } func TestSaveProposalHistoryForEpoch_OK(t *testing.T) { - db := SetupDB(t, [][48]byte{}) + pubkey := [48]byte{3} + db := SetupDB(t, [][48]byte{pubkey}) defer TeardownDB(t, db) - pubkey := []byte{3} epoch := uint64(2) slot := uint64(2) slotBits := bitfield.Bitlist{0x04, 0x04} - if err := db.SaveProposalHistoryForEpoch(context.Background(), pubkey, epoch, slotBits); err != nil { + if err := db.SaveProposalHistoryForEpoch(context.Background(), pubkey[:], epoch, slotBits); err != nil { t.Fatalf("Saving proposal history failed: %v", err) } - savedBits, err := db.ProposalHistoryForEpoch(context.Background(), pubkey, epoch) + savedBits, err := db.ProposalHistoryForEpoch(context.Background(), pubkey[:], epoch) if err != nil { t.Fatalf("Failed to get proposal history: %v", err) } @@ -172,34 +172,34 @@ func TestProposalHistoryForEpoch_MultipleEpochs(t *testing.T) { } } -func TestDeleteProposalHistoryForVal_OK(t *testing.T) { - db := SetupDB(t, [][48]byte{}) +func TestDeleteProposalHistory_OK(t *testing.T) { + pubkey := [48]byte{2} + db := SetupDB(t, [][48]byte{pubkey}) defer TeardownDB(t, db) - pubkey := []byte{2} slotBits := bitfield.Bitlist{0x01, 0x02} - if err := db.SaveProposalHistoryForEpoch(context.Background(), pubkey, 0, slotBits); err != nil { + if err := db.SaveProposalHistoryForEpoch(context.Background(), pubkey[:], 0, slotBits); err != nil { t.Fatalf("Save proposal history failed: %v", err) } // Making sure everything is saved. - savedHistory, err := db.ProposalHistoryForEpoch(context.Background(), pubkey, 0) + savedHistory, err := db.ProposalHistoryForEpoch(context.Background(), pubkey[:], 0) if err != nil { t.Fatalf("Failed to get proposal history: %v", err) } if savedHistory == nil || !bytes.Equal(savedHistory.Bytes(), slotBits.Bytes()) { t.Fatalf("Expected DB to keep object the same, received: %v, expected %v", savedHistory, slotBits) } - if err := db.DeleteProposalHistory(context.Background(), pubkey); err != nil { + if err := db.DeleteProposalHistory(context.Background(), pubkey[:]); err != nil { t.Fatal(err) } // Check after deleting from DB. - savedHistory, err = db.ProposalHistoryForEpoch(context.Background(), pubkey, 0) - if err != nil { - t.Fatalf("Failed to get proposal history: %v", err) + _, err = db.ProposalHistoryForEpoch(context.Background(), pubkey[:], 0) + if err == nil { + t.Fatalf("Unexpected success in deleting history: %v", err) } - if savedHistory != nil { - t.Fatalf("Expected proposal history to be nil, received %v", savedHistory) + if !strings.Contains(err.Error(), "validator history empty for public key ") { + t.Fatalf("Unexpected error, received %v", err) } } From a3f592964761104307738bbf2afb3d78e800fcff Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Thu, 2 Apr 2020 17:30:19 -0400 Subject: [PATCH 03/12] Test progress --- validator/db/proposal_history.go | 7 +++-- validator/db/proposal_history_test.go | 41 +++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/validator/db/proposal_history.go b/validator/db/proposal_history.go index a8ab76e6b059..723949ff1eb6 100644 --- a/validator/db/proposal_history.go +++ b/validator/db/proposal_history.go @@ -52,9 +52,9 @@ func (db *Store) SaveProposalHistoryForEpoch(ctx context.Context, pubKey []byte, if err := valBucket.Put(bytesutil.Bytes8(epoch), slotBits); err != nil { return err } - //if err := pruneProposalHistory(valBucket, epoch); err != nil { - // return err - //} + if err := pruneProposalHistory(valBucket, epoch); err != nil { + return err + } return nil }) return err @@ -84,6 +84,7 @@ func pruneProposalHistory(valBucket *bolt.Bucket, newestEpoch uint64) error { return errors.Wrapf(err, "could not prune epoch %d in proposal history", epoch) } } else { + // If starting from the oldest, we stop finding anything prunable, stop pruning. return nil } } diff --git a/validator/db/proposal_history_test.go b/validator/db/proposal_history_test.go index 703e375ce33d..904759941ed8 100644 --- a/validator/db/proposal_history_test.go +++ b/validator/db/proposal_history_test.go @@ -172,6 +172,47 @@ func TestProposalHistoryForEpoch_MultipleEpochs(t *testing.T) { } } +func TestPruneProposalHistory_OK(t *testing.T) { + slotsPerEpoch := params.BeaconConfig().SlotsPerEpoch + wsPeriod := params.BeaconConfig().WeakSubjectivityPeriod + pubKey := [48]byte{0} + tests := []struct { + slots []uint64 + removedEpochs []uint64 + }{ + { + // Go 2 epochs past pruning point, should remove epochs 0 and 1. + slots: []uint64{1, slotsPerEpoch + 5, slotsPerEpoch * 5, (wsPeriod + 3) * slotsPerEpoch}, + removedEpochs: []uint64{0, 1}, + }, + } + + for _, tt := range tests { + db := SetupDB(t, [][48]byte{pubKey}) + defer TeardownDB(t, db) + for _, slot := range tt.slots { + slotBits, err := db.ProposalHistoryForEpoch(context.Background(), pubKey[:], helpers.SlotToEpoch(slot)) + if err != nil { + t.Fatalf("Failed to get proposal history: %v", err) + } + slotBits.SetBitAt(slot%params.BeaconConfig().SlotsPerEpoch, true) + if err := db.SaveProposalHistoryForEpoch(context.Background(), pubKey[:], helpers.SlotToEpoch(slot), slotBits); err != nil { + t.Fatalf("Saving proposal history failed: %v", err) + } + } + + for _, epoch := range tt.removedEpochs { + savedBits, err := db.ProposalHistoryForEpoch(context.Background(), pubKey[:], epoch) + if err != nil { + t.Fatalf("Failed to get proposal history: %v", err) + } + if !bytes.Equal(savedBits, bitfield.NewBitlist(slotsPerEpoch)) { + t.Fatalf("unexpected difference in bytes, expected %#x vs received %#x", savedBits, bitfield.NewBitlist(slotsPerEpoch)) + } + } + } +} + func TestDeleteProposalHistory_OK(t *testing.T) { pubkey := [48]byte{2} db := SetupDB(t, [][48]byte{pubkey}) From b832a29888f2986e55d906be5f1a62cacea4ef37 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 3 Apr 2020 00:23:58 -0400 Subject: [PATCH 04/12] Tests --- validator/db/proposal_history.go | 12 ++--- validator/db/proposal_history_test.go | 63 ++++++++++++++++++++------- 2 files changed, 55 insertions(+), 20 deletions(-) diff --git a/validator/db/proposal_history.go b/validator/db/proposal_history.go index 723949ff1eb6..b25bef53a13d 100644 --- a/validator/db/proposal_history.go +++ b/validator/db/proposal_history.go @@ -20,7 +20,8 @@ func (db *Store) ProposalHistoryForEpoch(ctx context.Context, publicKey []byte, defer span.End() var err error - var slotBitlist bitfield.Bitlist + // Using 5 here since a bitfield length of 32 is always 5 bytes long. + slotBitlist := make(bitfield.Bitlist, 5) err = db.view(func(tx *bolt.Tx) error { bucket := tx.Bucket(historicProposalsBucket) valBucket := bucket.Bucket(publicKey) @@ -32,7 +33,7 @@ func (db *Store) ProposalHistoryForEpoch(ctx context.Context, publicKey []byte, slotBitlist = bitfield.NewBitlist(params.BeaconConfig().SlotsPerEpoch) return nil } - slotBitlist = slotBits + copy(slotBitlist, slotBits) return nil }) return slotBitlist, err @@ -79,13 +80,14 @@ func pruneProposalHistory(valBucket *bolt.Bucket, newestEpoch uint64) error { for k, _ := c.First(); k != nil; k, _ = c.Next() { epoch := binary.LittleEndian.Uint64(k) // Only delete epochs that are older than the weak subjectivity period. - if epoch+params.BeaconConfig().WeakSubjectivityPeriod < newestEpoch { - if err := valBucket.Delete(k); err != nil { + if epoch+params.BeaconConfig().WeakSubjectivityPeriod <= newestEpoch { + fmt.Printf("deleted %d\n", epoch) + if err := c.Delete(); err != nil { return errors.Wrapf(err, "could not prune epoch %d in proposal history", epoch) } } else { // If starting from the oldest, we stop finding anything prunable, stop pruning. - return nil + break } } return nil diff --git a/validator/db/proposal_history_test.go b/validator/db/proposal_history_test.go index 904759941ed8..8372ea25298c 100644 --- a/validator/db/proposal_history_test.go +++ b/validator/db/proposal_history_test.go @@ -3,6 +3,7 @@ package db import ( "bytes" "context" + "fmt" "reflect" "strings" "testing" @@ -129,20 +130,29 @@ func TestProposalHistoryForEpoch_MultipleEpochs(t *testing.T) { expectedBits []bitfield.Bitlist }{ { - slots: []uint64{1, 2, 8}, - expectedBits: []bitfield.Bitlist{{0x02, 0x14}}, + slots: []uint64{1, 2, 8, 31}, + expectedBits: []bitfield.Bitlist{{0b00000110, 0b00000001, 0b00000000, 0b10000000, 0b00000001}}, }, { - slots: []uint64{1, 33, 8}, - expectedBits: []bitfield.Bitlist{{0x02, 0x10}, {0x02, 0x04}}, + slots: []uint64{1, 33, 8}, + expectedBits: []bitfield.Bitlist{ + {0b00000010, 0b00000001, 0b00000000, 0b00000000, 0b00000001}, + {0b00000010, 0b00000000, 0b00000000, 0b00000000, 0b00000001}, + }, }, { - slots: []uint64{2, 34, 36}, - expectedBits: []bitfield.Bitlist{{0x02, 0x04}, {0x02, 0x06}}, + slots: []uint64{2, 34, 36}, + expectedBits: []bitfield.Bitlist{ + {0b00000100, 0b00000000, 0b00000000, 0b00000000, 0b00000001}, + {0b00010100, 0b00000000, 0b00000000, 0b00000000, 0b00000001}, + }, }, { - slots: []uint64{32, 33, 34}, - expectedBits: []bitfield.Bitlist{{0x02, 0x00}, {0x02, 0x05}}, + slots: []uint64{32, 33, 34}, + expectedBits: []bitfield.Bitlist{ + {0, 0, 0, 0, 1}, + {0b00000111, 0b00000000, 0b00000000, 0b00000000, 0b00000001}, + }, }, } @@ -165,8 +175,8 @@ func TestProposalHistoryForEpoch_MultipleEpochs(t *testing.T) { if err != nil { t.Fatalf("Failed to get proposal history: %v", err) } - if !bytes.Equal(savedBits.Bytes(), slotBits.Bytes()) { - t.Fatalf("unexpected difference in bytes, expected %#x vs received %#x", savedBits.Bytes(), slotBits.Bytes()) + if !bytes.Equal(slotBits, savedBits) { + t.Fatalf("unexpected difference in bytes for slots %v, expected %v vs received %v", tt.slots, slotBits, savedBits) } } } @@ -178,12 +188,25 @@ func TestPruneProposalHistory_OK(t *testing.T) { pubKey := [48]byte{0} tests := []struct { slots []uint64 + storedEpochs []uint64 removedEpochs []uint64 }{ { - // Go 2 epochs past pruning point, should remove epochs 0 and 1. - slots: []uint64{1, slotsPerEpoch + 5, slotsPerEpoch * 5, (wsPeriod + 3) * slotsPerEpoch}, - removedEpochs: []uint64{0, 1}, + // Go 2 epochs past pruning point. + slots: []uint64{slotsPerEpoch / 2, slotsPerEpoch*5 + 6, (wsPeriod+3)*slotsPerEpoch + 8}, + storedEpochs: []uint64{5, 54003}, + removedEpochs: []uint64{0}, + }, + { + // Go 10 epochs past pruning point. + slots: []uint64{slotsPerEpoch + 4, slotsPerEpoch * 2, slotsPerEpoch * 3, slotsPerEpoch * 4, slotsPerEpoch * 5, (wsPeriod+10)*slotsPerEpoch + 8}, + storedEpochs: []uint64{540010}, + removedEpochs: []uint64{1, 2, 3, 4}, + }, + { + // Prune none. + slots: []uint64{slotsPerEpoch + 4, slotsPerEpoch * 2, slotsPerEpoch * 3, slotsPerEpoch * 4, slotsPerEpoch * 5}, + storedEpochs: []uint64{1, 2, 3, 4, 5}, }, } @@ -191,6 +214,7 @@ func TestPruneProposalHistory_OK(t *testing.T) { db := SetupDB(t, [][48]byte{pubKey}) defer TeardownDB(t, db) for _, slot := range tt.slots { + fmt.Printf("saved epoch %d\n", helpers.SlotToEpoch(slot)) slotBits, err := db.ProposalHistoryForEpoch(context.Background(), pubKey[:], helpers.SlotToEpoch(slot)) if err != nil { t.Fatalf("Failed to get proposal history: %v", err) @@ -206,8 +230,17 @@ func TestPruneProposalHistory_OK(t *testing.T) { if err != nil { t.Fatalf("Failed to get proposal history: %v", err) } - if !bytes.Equal(savedBits, bitfield.NewBitlist(slotsPerEpoch)) { - t.Fatalf("unexpected difference in bytes, expected %#x vs received %#x", savedBits, bitfield.NewBitlist(slotsPerEpoch)) + if !bytes.Equal(bitfield.NewBitlist(slotsPerEpoch), savedBits) { + t.Fatalf("unexpected difference in bytes for epoch %d, expected %#x vs received %v", epoch, bitfield.NewBitlist(slotsPerEpoch), savedBits) + } + } + for _, epoch := range tt.storedEpochs { + savedBits, err := db.ProposalHistoryForEpoch(context.Background(), pubKey[:], epoch) + if err != nil { + t.Fatalf("Failed to get proposal history: %v", err) + } + if bytes.Equal(bitfield.NewBitlist(slotsPerEpoch), savedBits) { + t.Fatalf("unexpected difference in bytes, expected %#x vs received %v", bitfield.NewBitlist(slotsPerEpoch), savedBits) } } } From 04966eb8e83ec8e1bd1686e200714b5065f90a41 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 3 Apr 2020 01:42:49 -0400 Subject: [PATCH 05/12] Finish tests --- validator/db/proposal_history.go | 3 +-- validator/db/proposal_history_test.go | 8 +++----- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/validator/db/proposal_history.go b/validator/db/proposal_history.go index b25bef53a13d..ba56b0b362c2 100644 --- a/validator/db/proposal_history.go +++ b/validator/db/proposal_history.go @@ -77,11 +77,10 @@ func (db *Store) DeleteProposalHistory(ctx context.Context, pubkey []byte) error func pruneProposalHistory(valBucket *bolt.Bucket, newestEpoch uint64) error { c := valBucket.Cursor() - for k, _ := c.First(); k != nil; k, _ = c.Next() { + for k, _ := c.First(); k != nil; k, _ = c.First() { epoch := binary.LittleEndian.Uint64(k) // Only delete epochs that are older than the weak subjectivity period. if epoch+params.BeaconConfig().WeakSubjectivityPeriod <= newestEpoch { - fmt.Printf("deleted %d\n", epoch) if err := c.Delete(); err != nil { return errors.Wrapf(err, "could not prune epoch %d in proposal history", epoch) } diff --git a/validator/db/proposal_history_test.go b/validator/db/proposal_history_test.go index 8372ea25298c..3c522ed7f8b1 100644 --- a/validator/db/proposal_history_test.go +++ b/validator/db/proposal_history_test.go @@ -3,7 +3,6 @@ package db import ( "bytes" "context" - "fmt" "reflect" "strings" "testing" @@ -200,12 +199,12 @@ func TestPruneProposalHistory_OK(t *testing.T) { { // Go 10 epochs past pruning point. slots: []uint64{slotsPerEpoch + 4, slotsPerEpoch * 2, slotsPerEpoch * 3, slotsPerEpoch * 4, slotsPerEpoch * 5, (wsPeriod+10)*slotsPerEpoch + 8}, - storedEpochs: []uint64{540010}, + storedEpochs: []uint64{54010}, removedEpochs: []uint64{1, 2, 3, 4}, }, { // Prune none. - slots: []uint64{slotsPerEpoch + 4, slotsPerEpoch * 2, slotsPerEpoch * 3, slotsPerEpoch * 4, slotsPerEpoch * 5}, + slots: []uint64{slotsPerEpoch + 4, slotsPerEpoch*2 + 3, slotsPerEpoch*3 + 4, slotsPerEpoch*4 + 3, slotsPerEpoch*5 + 3}, storedEpochs: []uint64{1, 2, 3, 4, 5}, }, } @@ -214,7 +213,6 @@ func TestPruneProposalHistory_OK(t *testing.T) { db := SetupDB(t, [][48]byte{pubKey}) defer TeardownDB(t, db) for _, slot := range tt.slots { - fmt.Printf("saved epoch %d\n", helpers.SlotToEpoch(slot)) slotBits, err := db.ProposalHistoryForEpoch(context.Background(), pubKey[:], helpers.SlotToEpoch(slot)) if err != nil { t.Fatalf("Failed to get proposal history: %v", err) @@ -240,7 +238,7 @@ func TestPruneProposalHistory_OK(t *testing.T) { t.Fatalf("Failed to get proposal history: %v", err) } if bytes.Equal(bitfield.NewBitlist(slotsPerEpoch), savedBits) { - t.Fatalf("unexpected difference in bytes, expected %#x vs received %v", bitfield.NewBitlist(slotsPerEpoch), savedBits) + t.Fatalf("unexpected difference in bytes for epoch %d, expected %v vs received %v", epoch, bitfield.NewBitlist(slotsPerEpoch), savedBits) } } } From db4e5f71fe8ca81dd9b849090a9ce5f43b38a1ec Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 3 Apr 2020 01:47:18 -0400 Subject: [PATCH 06/12] update pbs --- proto/slashing/slashing.pb.go | 317 +++++----------------------------- 1 file changed, 46 insertions(+), 271 deletions(-) diff --git a/proto/slashing/slashing.pb.go b/proto/slashing/slashing.pb.go index 35d7ea93195e..334fc8ad3f72 100755 --- a/proto/slashing/slashing.pb.go +++ b/proto/slashing/slashing.pb.go @@ -6,16 +6,16 @@ package ethereum_slashing import ( context "context" fmt "fmt" + io "io" + math "math" + math_bits "math/bits" + _ "github.com/gogo/protobuf/gogoproto" proto "github.com/gogo/protobuf/proto" v1alpha1 "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" - github_com_prysmaticlabs_go_bitfield "github.com/prysmaticlabs/go-bitfield" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" - io "io" - math "math" - math_bits "math/bits" ) // Reference imports to suppress errors if they are not otherwise used. @@ -57,7 +57,7 @@ func (x SlashingStatusRequest_SlashingStatus) String() string { } func (SlashingStatusRequest_SlashingStatus) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_da7e95107d0081b4, []int{6, 0} + return fileDescriptor_da7e95107d0081b4, []int{5, 0} } type ProposerSlashingResponse struct { @@ -256,61 +256,6 @@ func (m *EpochSpanMap) GetEpochSpanMap() map[uint64]*MinMaxEpochSpan { return nil } -type ProposalHistory struct { - EpochBits github_com_prysmaticlabs_go_bitfield.Bitlist `protobuf:"bytes,1,opt,name=epoch_bits,json=epochBits,proto3,casttype=github.com/prysmaticlabs/go-bitfield.Bitlist" json:"epoch_bits,omitempty"` - LatestEpochWritten uint64 `protobuf:"varint,2,opt,name=latest_epoch_written,json=latestEpochWritten,proto3" json:"latest_epoch_written,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *ProposalHistory) Reset() { *m = ProposalHistory{} } -func (m *ProposalHistory) String() string { return proto.CompactTextString(m) } -func (*ProposalHistory) ProtoMessage() {} -func (*ProposalHistory) Descriptor() ([]byte, []int) { - return fileDescriptor_da7e95107d0081b4, []int{4} -} -func (m *ProposalHistory) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *ProposalHistory) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_ProposalHistory.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *ProposalHistory) XXX_Merge(src proto.Message) { - xxx_messageInfo_ProposalHistory.Merge(m, src) -} -func (m *ProposalHistory) XXX_Size() int { - return m.Size() -} -func (m *ProposalHistory) XXX_DiscardUnknown() { - xxx_messageInfo_ProposalHistory.DiscardUnknown(m) -} - -var xxx_messageInfo_ProposalHistory proto.InternalMessageInfo - -func (m *ProposalHistory) GetEpochBits() github_com_prysmaticlabs_go_bitfield.Bitlist { - if m != nil { - return m.EpochBits - } - return nil -} - -func (m *ProposalHistory) GetLatestEpochWritten() uint64 { - if m != nil { - return m.LatestEpochWritten - } - return 0 -} - type AttestationHistory struct { TargetToSource map[uint64]uint64 `protobuf:"bytes,1,rep,name=target_to_source,json=targetToSource,proto3" json:"target_to_source,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` LatestEpochWritten uint64 `protobuf:"varint,2,opt,name=latest_epoch_written,json=latestEpochWritten,proto3" json:"latest_epoch_written,omitempty"` @@ -323,7 +268,7 @@ func (m *AttestationHistory) Reset() { *m = AttestationHistory{} } func (m *AttestationHistory) String() string { return proto.CompactTextString(m) } func (*AttestationHistory) ProtoMessage() {} func (*AttestationHistory) Descriptor() ([]byte, []int) { - return fileDescriptor_da7e95107d0081b4, []int{5} + return fileDescriptor_da7e95107d0081b4, []int{4} } func (m *AttestationHistory) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -377,7 +322,7 @@ func (m *SlashingStatusRequest) Reset() { *m = SlashingStatusRequest{} } func (m *SlashingStatusRequest) String() string { return proto.CompactTextString(m) } func (*SlashingStatusRequest) ProtoMessage() {} func (*SlashingStatusRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_da7e95107d0081b4, []int{6} + return fileDescriptor_da7e95107d0081b4, []int{5} } func (m *SlashingStatusRequest) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -420,7 +365,6 @@ func init() { proto.RegisterType((*MinMaxEpochSpan)(nil), "ethereum.slashing.MinMaxEpochSpan") proto.RegisterType((*EpochSpanMap)(nil), "ethereum.slashing.EpochSpanMap") proto.RegisterMapType((map[uint64]*MinMaxEpochSpan)(nil), "ethereum.slashing.EpochSpanMap.EpochSpanMapEntry") - proto.RegisterType((*ProposalHistory)(nil), "ethereum.slashing.ProposalHistoryForEpoch") proto.RegisterType((*AttestationHistory)(nil), "ethereum.slashing.AttestationHistory") proto.RegisterMapType((map[uint64]uint64)(nil), "ethereum.slashing.AttestationHistory.TargetToSourceEntry") proto.RegisterType((*SlashingStatusRequest)(nil), "ethereum.slashing.SlashingStatusRequest") @@ -429,49 +373,45 @@ func init() { func init() { proto.RegisterFile("proto/slashing/slashing.proto", fileDescriptor_da7e95107d0081b4) } var fileDescriptor_da7e95107d0081b4 = []byte{ - // 661 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xcd, 0x6e, 0xd3, 0x40, - 0x10, 0xc6, 0x6d, 0x69, 0x61, 0x1b, 0x52, 0x77, 0x29, 0xa8, 0x8a, 0x44, 0x5b, 0x45, 0x48, 0x14, - 0x41, 0x9d, 0x36, 0x1c, 0x28, 0xdc, 0x1a, 0xa9, 0x52, 0x7b, 0xa8, 0x8a, 0x9c, 0xa2, 0x9e, 0x50, - 0xb4, 0xb6, 0xa7, 0xf6, 0xaa, 0xce, 0xee, 0xe2, 0x1d, 0xa7, 0xcd, 0x7b, 0xf0, 0x1e, 0xbc, 0x06, - 0x07, 0x0e, 0x3c, 0x01, 0x42, 0x3d, 0x23, 0x1e, 0x80, 0x13, 0xf2, 0xda, 0x29, 0x4e, 0xe2, 0x48, - 0x70, 0xdb, 0xf9, 0x66, 0xe6, 0xfb, 0xe6, 0xc7, 0x1e, 0xf2, 0x44, 0x25, 0x12, 0x65, 0x4b, 0xc7, - 0x4c, 0x47, 0x5c, 0x84, 0xb7, 0x0f, 0xc7, 0xe0, 0x74, 0x15, 0x30, 0x82, 0x04, 0xd2, 0xbe, 0x33, - 0x72, 0x34, 0x36, 0x01, 0xa3, 0xd6, 0x60, 0x8f, 0xc5, 0x2a, 0x62, 0x7b, 0x2d, 0x0f, 0x98, 0x2f, - 0x45, 0xcf, 0x8b, 0xa5, 0x7f, 0x99, 0xe7, 0x34, 0x76, 0x42, 0x8e, 0x51, 0xea, 0x39, 0xbe, 0xec, - 0xb7, 0x42, 0x19, 0xca, 0x96, 0x81, 0xbd, 0xf4, 0xc2, 0x58, 0xb9, 0x5e, 0xf6, 0xca, 0xc3, 0x9b, - 0x8a, 0xac, 0xbf, 0x4b, 0xa4, 0x92, 0x1a, 0x92, 0x6e, 0xa1, 0xe1, 0x82, 0x56, 0x52, 0x68, 0xa0, - 0x67, 0x64, 0x55, 0x15, 0xbe, 0xde, 0xa8, 0x80, 0x75, 0x6b, 0x6b, 0x7e, 0x7b, 0xb9, 0xfd, 0xcc, - 0xb9, 0x2d, 0x0d, 0x30, 0x72, 0x46, 0x05, 0x39, 0x53, 0x5c, 0xb6, 0x9a, 0x40, 0x32, 0xc5, 0x03, - 0x44, 0xd0, 0x58, 0xad, 0xc8, 0x0a, 0xdf, 0xbf, 0x2a, 0x4e, 0x71, 0xd9, 0x6c, 0x02, 0x69, 0x7e, - 0x20, 0x2b, 0x27, 0x5c, 0x9c, 0xb0, 0xeb, 0x43, 0x25, 0xfd, 0xa8, 0xab, 0x98, 0xa0, 0x4f, 0x49, - 0xbd, 0xcf, 0x45, 0x0f, 0x32, 0xa0, 0xa7, 0x15, 0x13, 0xeb, 0xd6, 0x96, 0xb5, 0xfd, 0xc0, 0xad, - 0xf5, 0xb9, 0x18, 0x8f, 0x62, 0xd7, 0xe5, 0xa8, 0xb9, 0x22, 0xaa, 0xc4, 0xd5, 0xfc, 0x6a, 0x91, - 0xda, 0xad, 0x75, 0xc2, 0x14, 0x3d, 0x27, 0xf5, 0xbf, 0x29, 0xbd, 0x3e, 0x53, 0x45, 0x0b, 0x7b, - 0xce, 0xd4, 0x3e, 0x9d, 0x72, 0xe2, 0x98, 0x71, 0x28, 0x30, 0x19, 0xba, 0x35, 0x28, 0x41, 0x0d, - 0x9f, 0xac, 0x4e, 0x85, 0x50, 0x9b, 0xcc, 0x5f, 0xc2, 0xd0, 0xd4, 0xbf, 0xe0, 0x66, 0x4f, 0xba, - 0x4f, 0xee, 0x0e, 0x58, 0x9c, 0x82, 0xa9, 0x76, 0xb9, 0xdd, 0xac, 0x90, 0x9d, 0x98, 0x87, 0x9b, - 0x27, 0xbc, 0x9d, 0xdb, 0xb7, 0x9a, 0x9f, 0x2c, 0xb2, 0x92, 0xaf, 0x91, 0xc5, 0x47, 0x5c, 0xa3, - 0x4c, 0x86, 0xf4, 0x94, 0x90, 0xbc, 0x23, 0x8f, 0xa3, 0x36, 0x52, 0xb5, 0xce, 0xee, 0xef, 0xef, - 0x9b, 0x2f, 0x4b, 0x1f, 0x9b, 0x4a, 0x86, 0xba, 0xcf, 0x90, 0xfb, 0x31, 0xf3, 0x74, 0x2b, 0x94, - 0x3b, 0x1e, 0xc7, 0x0b, 0x0e, 0x71, 0xe0, 0x74, 0x38, 0xc6, 0x5c, 0xa3, 0x7b, 0xdf, 0x70, 0x74, - 0x38, 0x6a, 0xba, 0x4b, 0xd6, 0x62, 0x96, 0xad, 0xa9, 0x18, 0xee, 0x55, 0xc2, 0x11, 0x21, 0x9f, - 0xef, 0x82, 0x4b, 0x73, 0x9f, 0x29, 0xef, 0x3c, 0xf7, 0x34, 0x7f, 0x59, 0x84, 0xe6, 0xbb, 0x66, - 0xc8, 0xa5, 0x18, 0x55, 0xe6, 0x13, 0x1b, 0x59, 0x12, 0x02, 0xf6, 0x50, 0xf6, 0xb4, 0x4c, 0x13, - 0x1f, 0x8a, 0x69, 0xbf, 0xa9, 0x68, 0x7b, 0x9a, 0xc0, 0x39, 0x33, 0xd9, 0x67, 0xb2, 0x6b, 0x72, - 0xf3, 0xa9, 0xd7, 0x71, 0x0c, 0xfc, 0xff, 0x6a, 0x1b, 0x07, 0xe4, 0x61, 0x05, 0x71, 0xc5, 0xae, - 0xd6, 0xca, 0xbb, 0x5a, 0x28, 0xef, 0xe1, 0xb3, 0x45, 0x1e, 0x8d, 0x3e, 0xe1, 0x2e, 0x32, 0x4c, - 0xb5, 0x0b, 0x1f, 0x53, 0xd0, 0x48, 0x4f, 0xc9, 0xa2, 0x36, 0x80, 0x21, 0xaa, 0xb7, 0x5f, 0x57, - 0x74, 0x5a, 0x99, 0x39, 0x89, 0x16, 0x34, 0xcd, 0x43, 0x52, 0x1f, 0xf7, 0xd0, 0x65, 0xb2, 0xf4, - 0x5e, 0x5c, 0x0a, 0x79, 0x25, 0xec, 0x3b, 0x94, 0x90, 0xc5, 0x03, 0x1f, 0xf9, 0x00, 0x6c, 0x8b, - 0xd6, 0xc8, 0xbd, 0x63, 0xe1, 0xc7, 0x69, 0x00, 0x81, 0x3d, 0x97, 0x59, 0x2e, 0x0c, 0x20, 0x41, - 0x08, 0xec, 0xf9, 0xf6, 0x4f, 0x8b, 0x2c, 0x19, 0x1e, 0x48, 0xa8, 0x22, 0x8f, 0x8f, 0xb5, 0x31, - 0x98, 0x17, 0x43, 0x69, 0xee, 0xf4, 0xf9, 0x8c, 0x1f, 0xf9, 0x58, 0x04, 0x70, 0x0d, 0x41, 0x29, - 0xb4, 0xf1, 0x62, 0xe6, 0x0a, 0x2b, 0x6e, 0x87, 0x24, 0x76, 0x49, 0xb1, 0x93, 0x9d, 0x44, 0xea, - 0xcc, 0xd0, 0xea, 0xf2, 0x50, 0x40, 0xd0, 0x31, 0xd7, 0xd3, 0x44, 0x1e, 0x01, 0x0b, 0x20, 0xa9, - 0x14, 0x9c, 0x75, 0x1e, 0x3b, 0xb5, 0x2f, 0x37, 0x1b, 0xd6, 0xb7, 0x9b, 0x0d, 0xeb, 0xc7, 0xcd, - 0x86, 0xe5, 0x2d, 0x9a, 0x7b, 0xfa, 0xea, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xf5, 0x35, 0xb6, - 0xef, 0xd3, 0x05, 0x00, 0x00, + // 593 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xc1, 0x6e, 0xd3, 0x40, + 0x10, 0x65, 0xdb, 0xd2, 0xa2, 0x69, 0x08, 0xee, 0x52, 0x50, 0x15, 0x89, 0x52, 0x59, 0x48, 0x14, + 0x21, 0x1c, 0x1a, 0x0e, 0x14, 0x6e, 0xad, 0x54, 0xa9, 0x3d, 0x54, 0x20, 0xa7, 0xa8, 0x27, 0x64, + 0xad, 0xed, 0xc1, 0xb6, 0xea, 0xec, 0x2e, 0xbb, 0xeb, 0xb4, 0xfd, 0x2a, 0x7e, 0x83, 0x03, 0x07, + 0x3e, 0x01, 0xf5, 0x8c, 0xf8, 0x06, 0xe4, 0xb5, 0x53, 0x9c, 0xc4, 0x91, 0xb8, 0xed, 0xbc, 0x99, + 0xf7, 0xde, 0xcc, 0xd8, 0xbb, 0xf0, 0x44, 0x2a, 0x61, 0x44, 0x5f, 0xe7, 0x4c, 0xa7, 0x19, 0x4f, + 0x6e, 0x0f, 0x9e, 0xc5, 0xe9, 0x06, 0x9a, 0x14, 0x15, 0x16, 0x23, 0x6f, 0x92, 0xe8, 0x3d, 0x45, + 0x93, 0xf6, 0xc7, 0x7b, 0x2c, 0x97, 0x29, 0xdb, 0xeb, 0x87, 0xc8, 0x22, 0xc1, 0x83, 0x30, 0x17, + 0xd1, 0x45, 0xc5, 0xe9, 0xbd, 0x4a, 0x32, 0x93, 0x16, 0xa1, 0x17, 0x89, 0x51, 0x3f, 0x11, 0x89, + 0xe8, 0x5b, 0x38, 0x2c, 0xbe, 0xd8, 0xa8, 0xf2, 0x2b, 0x4f, 0x55, 0xb9, 0x2b, 0x61, 0xeb, 0xa3, + 0x12, 0x52, 0x68, 0x54, 0xc3, 0xda, 0xc3, 0x47, 0x2d, 0x05, 0xd7, 0x48, 0xcf, 0x60, 0x43, 0xd6, + 0xb9, 0x60, 0xd2, 0xc0, 0x16, 0xd9, 0x59, 0xde, 0x5d, 0x1f, 0x3c, 0xf7, 0x6e, 0x5b, 0x43, 0x93, + 0x7a, 0x93, 0x86, 0xbc, 0x39, 0x2d, 0x47, 0xce, 0x20, 0xa5, 0xe3, 0x81, 0x31, 0xa8, 0x4d, 0xbb, + 0x23, 0xab, 0x73, 0xff, 0xeb, 0x38, 0xa7, 0xe5, 0xb0, 0x19, 0xc4, 0xfd, 0x0c, 0x0f, 0x4e, 0x33, + 0x7e, 0xca, 0xae, 0x8e, 0xa4, 0x88, 0xd2, 0xa1, 0x64, 0x9c, 0x3e, 0x83, 0xee, 0x28, 0xe3, 0x01, + 0x96, 0x40, 0xa0, 0x25, 0xe3, 0x5b, 0x64, 0x87, 0xec, 0xde, 0xf7, 0x3b, 0xa3, 0x8c, 0x4f, 0x57, + 0xb1, 0xab, 0x66, 0xd5, 0x52, 0x5d, 0xd5, 0xd0, 0x72, 0x7f, 0x10, 0xe8, 0xdc, 0x46, 0xa7, 0x4c, + 0xd2, 0x73, 0xe8, 0xfe, 0xa3, 0x04, 0x23, 0x26, 0xeb, 0x11, 0xf6, 0xbc, 0xb9, 0xef, 0xe9, 0x35, + 0x89, 0x53, 0xc1, 0x11, 0x37, 0xea, 0xda, 0xef, 0x60, 0x03, 0xea, 0x45, 0xb0, 0x31, 0x57, 0x42, + 0x1d, 0x58, 0xbe, 0xc0, 0x6b, 0xdb, 0xff, 0x8a, 0x5f, 0x1e, 0xe9, 0x3e, 0xdc, 0x1d, 0xb3, 0xbc, + 0x40, 0xdb, 0xed, 0xfa, 0xc0, 0x6d, 0xb1, 0x9d, 0xd9, 0x87, 0x5f, 0x11, 0xde, 0x2f, 0xed, 0x13, + 0xf7, 0x0f, 0x01, 0x5a, 0x2d, 0x95, 0x99, 0x4c, 0xf0, 0xe3, 0x4c, 0x1b, 0xa1, 0xae, 0x69, 0x04, + 0x8e, 0x61, 0x2a, 0x41, 0x13, 0x18, 0x11, 0x68, 0x51, 0xa8, 0x08, 0xeb, 0xb1, 0xde, 0xb5, 0xe8, + 0xcf, 0x0b, 0x78, 0x67, 0x96, 0x7d, 0x26, 0x86, 0x96, 0x5b, 0x8d, 0xd7, 0x35, 0x53, 0x20, 0x7d, + 0x0d, 0x9b, 0x39, 0x2b, 0x99, 0xf5, 0xce, 0x2f, 0x55, 0x66, 0x0c, 0x56, 0x6b, 0x5f, 0xf1, 0x69, + 0x95, 0xb3, 0x5d, 0x9f, 0x57, 0x99, 0xde, 0x01, 0x3c, 0x6c, 0x11, 0x6e, 0x59, 0xca, 0x66, 0x73, + 0x29, 0x2b, 0xcd, 0x81, 0xbf, 0x11, 0x78, 0x34, 0xf9, 0x57, 0x86, 0x86, 0x99, 0x42, 0xfb, 0xf8, + 0xb5, 0x40, 0x6d, 0xe8, 0x07, 0x58, 0xd5, 0x16, 0xb0, 0x42, 0xdd, 0xc1, 0xdb, 0x96, 0x49, 0x5b, + 0x99, 0xb3, 0x68, 0x2d, 0xe3, 0x1e, 0x41, 0x77, 0x3a, 0x43, 0xd7, 0x61, 0xed, 0x13, 0xbf, 0xe0, + 0xe2, 0x92, 0x3b, 0x77, 0x28, 0xc0, 0xea, 0x41, 0x64, 0xb2, 0x31, 0x3a, 0x84, 0x76, 0xe0, 0xde, + 0x09, 0x8f, 0xf2, 0x22, 0xc6, 0xd8, 0x59, 0x2a, 0x23, 0x1f, 0xc7, 0xa8, 0x0c, 0xc6, 0xce, 0xf2, + 0xe0, 0x37, 0x81, 0x35, 0xab, 0x83, 0x8a, 0x4a, 0x78, 0x7c, 0xa2, 0x6d, 0xc0, 0xc2, 0x1c, 0x1b, + 0x7b, 0xa7, 0x2f, 0x16, 0xdc, 0x98, 0x13, 0x1e, 0xe3, 0x15, 0xc6, 0x8d, 0xd2, 0xde, 0xcb, 0x85, + 0x9f, 0xb0, 0xe5, 0x92, 0x0a, 0x70, 0x1a, 0x8e, 0x87, 0xe5, 0xdb, 0x43, 0xbd, 0x05, 0x5e, 0xc3, + 0x2c, 0xe1, 0x18, 0x1f, 0xda, 0x67, 0xca, 0x56, 0x1e, 0x23, 0x8b, 0x51, 0xb5, 0x1a, 0x2e, 0x7a, + 0x87, 0x0e, 0x3b, 0xdf, 0x6f, 0xb6, 0xc9, 0xcf, 0x9b, 0x6d, 0xf2, 0xeb, 0x66, 0x9b, 0x84, 0xab, + 0xf6, 0xe1, 0x7a, 0xf3, 0x37, 0x00, 0x00, 0xff, 0xff, 0xc1, 0x45, 0x57, 0xad, 0x3c, 0x05, 0x00, + 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -760,45 +700,6 @@ func (m *EpochSpanMap) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *ProposalHistory) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *ProposalHistory) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *ProposalHistory) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if m.XXX_unrecognized != nil { - i -= len(m.XXX_unrecognized) - copy(dAtA[i:], m.XXX_unrecognized) - } - if m.LatestEpochWritten != 0 { - i = encodeVarintSlashing(dAtA, i, uint64(m.LatestEpochWritten)) - i-- - dAtA[i] = 0x10 - } - if len(m.EpochBits) > 0 { - i -= len(m.EpochBits) - copy(dAtA[i:], m.EpochBits) - i = encodeVarintSlashing(dAtA, i, uint64(len(m.EpochBits))) - i-- - dAtA[i] = 0xa - } - return len(dAtA) - i, nil -} - func (m *AttestationHistory) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -968,25 +869,6 @@ func (m *EpochSpanMap) Size() (n int) { return n } -func (m *ProposalHistory) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.EpochBits) - if l > 0 { - n += 1 + l + sovSlashing(uint64(l)) - } - if m.LatestEpochWritten != 0 { - n += 1 + sovSlashing(uint64(m.LatestEpochWritten)) - } - if m.XXX_unrecognized != nil { - n += len(m.XXX_unrecognized) - } - return n -} - func (m *AttestationHistory) Size() (n int) { if m == nil { return 0 @@ -1468,113 +1350,6 @@ func (m *EpochSpanMap) Unmarshal(dAtA []byte) error { } return nil } -func (m *ProposalHistory) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowSlashing - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ProposalHistoryForEpoch: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ProposalHistoryForEpoch: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field EpochBits", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowSlashing - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthSlashing - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthSlashing - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.EpochBits = append(m.EpochBits[:0], dAtA[iNdEx:postIndex]...) - if m.EpochBits == nil { - m.EpochBits = []byte{} - } - iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field LatestEpochWritten", wireType) - } - m.LatestEpochWritten = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowSlashing - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.LatestEpochWritten |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - default: - iNdEx = preIndex - skippy, err := skipSlashing(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthSlashing - } - if (iNdEx + skippy) < 0 { - return ErrInvalidLengthSlashing - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *AttestationHistory) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 From a708cb8b920b0fe3b1d1e2b2e6a3730efe220b3b Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 3 Apr 2020 12:52:38 -0400 Subject: [PATCH 07/12] Fix mocked tests --- validator/client/validator_propose_test.go | 155 +++++---------------- 1 file changed, 38 insertions(+), 117 deletions(-) diff --git a/validator/client/validator_propose_test.go b/validator/client/validator_propose_test.go index 046d5580a539..fd208ed3c285 100644 --- a/validator/client/validator_propose_test.go +++ b/validator/client/validator_propose_test.go @@ -7,8 +7,6 @@ import ( "github.com/golang/mock/gomock" ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" - "github.com/prysmaticlabs/go-bitfield" - slashpb "github.com/prysmaticlabs/prysm/proto/slashing" "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/testutil" @@ -223,6 +221,44 @@ func TestProposeBlock_AllowsPastProposals(t *testing.T) { testutil.AssertLogsDoNotContain(t, hook, "Tried to sign a double proposal") } +func TestProposeBlock_AllowsSameEpoch(t *testing.T) { + cfg := &featureconfig.Flags{ + ProtectProposer: true, + } + featureconfig.Init(cfg) + hook := logTest.NewGlobal() + validator, m, finish := setup(t) + defer finish() + defer db.TeardownDB(t, validator.db) + + m.validatorClient.EXPECT().DomainData( + gomock.Any(), // ctx + gomock.Any(), //epoch + ).Times(2).Return(ðpb.DomainResponse{}, nil /*err*/) + + m.validatorClient.EXPECT().GetBlock( + gomock.Any(), // ctx + gomock.Any(), + ).Times(2).Return(ðpb.BeaconBlock{Body: ðpb.BeaconBlockBody{}}, nil /*err*/) + + m.validatorClient.EXPECT().DomainData( + gomock.Any(), // ctx + gomock.Any(), //epoch + ).Times(2).Return(ðpb.DomainResponse{}, nil /*err*/) + + m.validatorClient.EXPECT().ProposeBlock( + gomock.Any(), // ctx + gomock.AssignableToTypeOf(ðpb.SignedBeaconBlock{}), + ).Times(2).Return(ðpb.ProposeResponse{}, nil /*error*/) + + farAhead := (params.BeaconConfig().WeakSubjectivityPeriod + 9) * params.BeaconConfig().SlotsPerEpoch + validator.ProposeBlock(context.Background(), farAhead, validatorPubKey) + testutil.AssertLogsDoNotContain(t, hook, "Tried to sign a double proposal") + + validator.ProposeBlock(context.Background(), farAhead-4, validatorPubKey) + testutil.AssertLogsDoNotContain(t, hook, "Tried to sign a double proposal") +} + func TestProposeBlock_BroadcastsBlock(t *testing.T) { validator, m, finish := setup(t) defer finish() @@ -287,118 +323,3 @@ func TestProposeBlock_BroadcastsBlock_WithGraffiti(t *testing.T) { t.Errorf("Block was broadcast with the wrong graffiti field, wanted \"%v\", got \"%v\"", string(validator.graffiti), string(sentBlock.Block.Body.Graffiti)) } } - -func TestSetProposedForEpoch_SetsBit(t *testing.T) { - wsPeriod := params.BeaconConfig().WeakSubjectivityPeriod - proposals := &slashpb.ProposalHistory{ - EpochBits: bitfield.NewBitlist(wsPeriod), - LatestEpochWritten: 0, - } - epoch := uint64(4) - proposals = SetProposedForEpoch(proposals, epoch) - proposed := HasProposedForEpoch(proposals, epoch) - if !proposed { - t.Fatal("Expected epoch 4 to be marked as proposed") - } - // Make sure no other bits are changed. - for i := uint64(1); i <= wsPeriod; i++ { - if i == epoch { - continue - } - if HasProposedForEpoch(proposals, i) { - t.Fatalf("Expected epoch %d to not be marked as proposed", i) - } - } -} - -func TestSetProposedForEpoch_PrunesOverWSPeriod(t *testing.T) { - wsPeriod := params.BeaconConfig().WeakSubjectivityPeriod - proposals := &slashpb.ProposalHistory{ - EpochBits: bitfield.NewBitlist(wsPeriod), - LatestEpochWritten: 0, - } - prunedEpoch := uint64(3) - proposals = SetProposedForEpoch(proposals, prunedEpoch) - - if proposals.LatestEpochWritten != prunedEpoch { - t.Fatalf("Expected latest epoch written to be %d, received %d", prunedEpoch, proposals.LatestEpochWritten) - } - - epoch := wsPeriod + 4 - proposals = SetProposedForEpoch(proposals, epoch) - if !HasProposedForEpoch(proposals, epoch) { - t.Fatalf("Expected to be marked as proposed for epoch %d", epoch) - } - if proposals.LatestEpochWritten != epoch { - t.Fatalf("Expected latest written epoch to be %d, received %d", epoch, proposals.LatestEpochWritten) - } - - if HasProposedForEpoch(proposals, epoch-wsPeriod+prunedEpoch) { - t.Fatalf("Expected the bit of pruned epoch %d to not be marked as proposed", epoch) - } - // Make sure no other bits are changed. - for i := epoch - wsPeriod + 1; i <= epoch; i++ { - if i == epoch { - continue - } - if HasProposedForEpoch(proposals, i) { - t.Fatalf("Expected epoch %d to not be marked as proposed", i) - } - } -} - -func TestSetProposedForEpoch_KeepsHistory(t *testing.T) { - wsPeriod := params.BeaconConfig().WeakSubjectivityPeriod - proposals := &slashpb.ProposalHistory{ - EpochBits: bitfield.NewBitlist(wsPeriod), - LatestEpochWritten: 0, - } - randomIndexes := []uint64{23, 423, 8900, 11347, 25033, 52225, 53999} - for i := 0; i < len(randomIndexes); i++ { - proposals = SetProposedForEpoch(proposals, randomIndexes[i]) - } - if proposals.LatestEpochWritten != 53999 { - t.Fatalf("Expected latest epoch written to be %d, received %d", 53999, proposals.LatestEpochWritten) - } - - // Make sure no other bits are changed. - for i := uint64(0); i < wsPeriod; i++ { - setIndex := false - for r := 0; r < len(randomIndexes); r++ { - if i == randomIndexes[r] { - setIndex = true - break - } - } - - if setIndex != HasProposedForEpoch(proposals, i) { - t.Fatalf("Expected epoch %d to be marked as %t", i, setIndex) - } - } - - // Set a past epoch as proposed, and make sure the recent data isn't changed. - proposals = SetProposedForEpoch(proposals, randomIndexes[1]+5) - if proposals.LatestEpochWritten != 53999 { - t.Fatalf("Expected last epoch written to not change after writing a past epoch, received %d", proposals.LatestEpochWritten) - } - // Proposal just marked should be true. - if !HasProposedForEpoch(proposals, randomIndexes[1]+5) { - t.Fatal("Expected marked past epoch to be true, received false") - } - // Previously marked proposal should stay true. - if !HasProposedForEpoch(proposals, randomIndexes[1]) { - t.Fatal("Expected marked past epoch to be true, received false") - } -} - -func TestSetProposedForEpoch_PreventsProposingFutureEpochs(t *testing.T) { - wsPeriod := params.BeaconConfig().WeakSubjectivityPeriod - proposals := &slashpb.ProposalHistory{ - EpochBits: bitfield.NewBitlist(wsPeriod), - LatestEpochWritten: 0, - } - proposals = SetProposedForEpoch(proposals, 200) - if HasProposedForEpoch(proposals, wsPeriod+200) { - t.Fatalf("Expected epoch %d to not be marked as proposed", wsPeriod+200) - } -} From 6648870a9d216a4431087bebce5cb816b9cce7e7 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 3 Apr 2020 13:13:41 -0400 Subject: [PATCH 08/12] Gazelle --- validator/db/BUILD.bazel | 1 - validator/db/proposal_history.go | 1 - 2 files changed, 2 deletions(-) diff --git a/validator/db/BUILD.bazel b/validator/db/BUILD.bazel index 309dfce5da0b..b9fc03bc3d59 100644 --- a/validator/db/BUILD.bazel +++ b/validator/db/BUILD.bazel @@ -15,7 +15,6 @@ go_library( "//proto/slashing:go_default_library", "//shared/params:go_default_library", "//validator/db/iface:go_default_library", - "@com_github_boltdb_bolt//:go_default_library", "@com_github_gogo_protobuf//proto:go_default_library", "@com_github_pkg_errors//:go_default_library", "@com_github_prysmaticlabs_go_bitfield//:go_default_library", diff --git a/validator/db/proposal_history.go b/validator/db/proposal_history.go index 7df0dc516d6b..fa0d9fb2fd7b 100644 --- a/validator/db/proposal_history.go +++ b/validator/db/proposal_history.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "fmt" - "github.com/boltdb/bolt" "github.com/gogo/protobuf/proto" "github.com/pkg/errors" "github.com/prysmaticlabs/go-bitfield" From ca8a343dac802c6f7749b366b03f9c7f96613fe3 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 3 Apr 2020 13:15:45 -0400 Subject: [PATCH 09/12] pt 2 --- validator/db/proposal_history.go | 1 - 1 file changed, 1 deletion(-) diff --git a/validator/db/proposal_history.go b/validator/db/proposal_history.go index fa0d9fb2fd7b..19b980b5c104 100644 --- a/validator/db/proposal_history.go +++ b/validator/db/proposal_history.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "fmt" - "github.com/gogo/protobuf/proto" "github.com/pkg/errors" "github.com/prysmaticlabs/go-bitfield" "github.com/prysmaticlabs/prysm/shared/params" From 86524340b22838ad102419460668fa57f5a6e356 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 3 Apr 2020 13:18:06 -0400 Subject: [PATCH 10/12] Fix --- validator/client/validator_propose_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/validator/client/validator_propose_test.go b/validator/client/validator_propose_test.go index 1633450c09d0..a20261f8f4c9 100644 --- a/validator/client/validator_propose_test.go +++ b/validator/client/validator_propose_test.go @@ -7,6 +7,7 @@ import ( "github.com/golang/mock/gomock" ethpb "github.com/prysmaticlabs/ethereumapis/eth/v1alpha1" + "github.com/prysmaticlabs/prysm/shared/bytesutil" "github.com/prysmaticlabs/prysm/shared/featureconfig" "github.com/prysmaticlabs/prysm/shared/params" "github.com/prysmaticlabs/prysm/shared/testutil" From 80213a9b0bbd2223dbdf14304df8135d1530c252 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 3 Apr 2020 13:32:36 -0400 Subject: [PATCH 11/12] Fixes --- validator/client/validator_propose_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/validator/client/validator_propose_test.go b/validator/client/validator_propose_test.go index a20261f8f4c9..918550736ceb 100644 --- a/validator/client/validator_propose_test.go +++ b/validator/client/validator_propose_test.go @@ -252,11 +252,12 @@ func TestProposeBlock_AllowsSameEpoch(t *testing.T) { gomock.AssignableToTypeOf(ðpb.SignedBeaconBlock{}), ).Times(2).Return(ðpb.ProposeResponse{}, nil /*error*/) + pubKey := bytesutil.ToBytes48(validatorPubKey.Marshal()) farAhead := (params.BeaconConfig().WeakSubjectivityPeriod + 9) * params.BeaconConfig().SlotsPerEpoch - validator.ProposeBlock(context.Background(), farAhead, validatorPubKey) + validator.ProposeBlock(context.Background(), farAhead, pubKey) testutil.AssertLogsDoNotContain(t, hook, "Tried to sign a double proposal") - validator.ProposeBlock(context.Background(), farAhead-4, validatorPubKey) + validator.ProposeBlock(context.Background(), farAhead-4, pubKey) testutil.AssertLogsDoNotContain(t, hook, "Tried to sign a double proposal") } From 8116079185f5035f22c3bde3df650502bee26172 Mon Sep 17 00:00:00 2001 From: Ivan Martinez Date: Fri, 3 Apr 2020 14:08:11 -0400 Subject: [PATCH 12/12] Fix tests wit hwrong copying --- validator/db/proposal_history_test.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/validator/db/proposal_history_test.go b/validator/db/proposal_history_test.go index 3c522ed7f8b1..2af711732e2a 100644 --- a/validator/db/proposal_history_test.go +++ b/validator/db/proposal_history_test.go @@ -52,7 +52,7 @@ func TestSaveProposalHistoryForEpoch_OK(t *testing.T) { epoch := uint64(2) slot := uint64(2) - slotBits := bitfield.Bitlist{0x04, 0x04} + slotBits := bitfield.Bitlist{0x04, 0x00, 0x00, 0x00, 0x04} if err := db.SaveProposalHistoryForEpoch(context.Background(), pubkey[:], epoch, slotBits); err != nil { t.Fatalf("Saving proposal history failed: %v", err) @@ -62,7 +62,7 @@ func TestSaveProposalHistoryForEpoch_OK(t *testing.T) { t.Fatalf("Failed to get proposal history: %v", err) } - if savedBits == nil || !bytes.Equal(slotBits.Bytes(), savedBits.Bytes()) { + if savedBits == nil || !bytes.Equal(slotBits, savedBits) { t.Fatalf("Expected DB to keep object the same, received: %v", savedBits) } if !savedBits.BitAt(slot) { @@ -84,15 +84,15 @@ func TestSaveProposalHistoryForEpoch_Overwrites(t *testing.T) { }{ { slot: uint64(1), - slotBits: bitfield.Bitlist{0x02, 0x02}, + slotBits: bitfield.Bitlist{0x02, 0x00, 0x00, 0x00, 0x02}, }, { slot: uint64(2), - slotBits: bitfield.Bitlist{0x04, 0x04}, + slotBits: bitfield.Bitlist{0x04, 0x00, 0x00, 0x00, 0x04}, }, { slot: uint64(3), - slotBits: bitfield.Bitlist{0x08, 0x08}, + slotBits: bitfield.Bitlist{0x08, 0x00, 0x00, 0x00, 0x08}, }, } @@ -107,8 +107,8 @@ func TestSaveProposalHistoryForEpoch_Overwrites(t *testing.T) { t.Fatalf("Failed to get proposal history: %v", err) } - if savedBits == nil || !reflect.DeepEqual(savedBits.Bytes(), tt.slotBits.Bytes()) { - t.Fatalf("Expected DB to keep object the same, received: %v, expected %v", savedBits.Bytes(), tt.slotBits.Bytes()) + if savedBits == nil || !reflect.DeepEqual(savedBits, tt.slotBits) { + t.Fatalf("Expected DB to keep object the same, received: %v, expected %v", savedBits, tt.slotBits) } if !savedBits.BitAt(tt.slot) { t.Fatalf("Expected slot %d to be marked as proposed", tt.slot) @@ -249,7 +249,7 @@ func TestDeleteProposalHistory_OK(t *testing.T) { db := SetupDB(t, [][48]byte{pubkey}) defer TeardownDB(t, db) - slotBits := bitfield.Bitlist{0x01, 0x02} + slotBits := bitfield.Bitlist{0x01, 0x00, 0x00, 0x00, 0x02} if err := db.SaveProposalHistoryForEpoch(context.Background(), pubkey[:], 0, slotBits); err != nil { t.Fatalf("Save proposal history failed: %v", err) @@ -259,7 +259,7 @@ func TestDeleteProposalHistory_OK(t *testing.T) { if err != nil { t.Fatalf("Failed to get proposal history: %v", err) } - if savedHistory == nil || !bytes.Equal(savedHistory.Bytes(), slotBits.Bytes()) { + if savedHistory == nil || !bytes.Equal(savedHistory, slotBits) { t.Fatalf("Expected DB to keep object the same, received: %v, expected %v", savedHistory, slotBits) } if err := db.DeleteProposalHistory(context.Background(), pubkey[:]); err != nil {