diff --git a/protocol/v2/qbft/spectest/controller_type.go b/protocol/v2/qbft/spectest/controller_type.go index 5cf150b850..c91a80ccf3 100644 --- a/protocol/v2/qbft/spectest/controller_type.go +++ b/protocol/v2/qbft/spectest/controller_type.go @@ -10,22 +10,20 @@ import ( "reflect" "testing" - "github.com/ssvlabs/ssv/exporter/convert" - specqbft "github.com/ssvlabs/ssv-spec/qbft" spectests "github.com/ssvlabs/ssv-spec/qbft/spectest/tests" spectypes "github.com/ssvlabs/ssv-spec/types" spectestingutils "github.com/ssvlabs/ssv-spec/types/testingutils" typescomparable "github.com/ssvlabs/ssv-spec/types/testingutils/comparable" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - + "github.com/ssvlabs/ssv/exporter/convert" "github.com/ssvlabs/ssv/logging" "github.com/ssvlabs/ssv/protocol/v2/qbft" "github.com/ssvlabs/ssv/protocol/v2/qbft/controller" "github.com/ssvlabs/ssv/protocol/v2/qbft/roundtimer" qbfttesting "github.com/ssvlabs/ssv/protocol/v2/qbft/testing" protocoltesting "github.com/ssvlabs/ssv/protocol/v2/testing" + "github.com/stretchr/testify/require" + "go.uber.org/zap" ) func RunControllerSpecTest(t *testing.T, test *spectests.ControllerSpecTest) { @@ -50,12 +48,7 @@ func RunControllerSpecTest(t *testing.T, test *spectests.ControllerSpecTest) { } height++ } - - if len(test.ExpectedError) != 0 { - require.EqualError(t, lastErr, test.ExpectedError) - } else { - require.NoError(t, lastErr) - } + validateError(t, lastErr, test.Name, test.ExpectedError) } func generateController(logger *zap.Logger) *controller.Controller { diff --git a/protocol/v2/qbft/spectest/create_msg_type.go b/protocol/v2/qbft/spectest/create_msg_type.go index d7a0ba7068..06cb8f8f8d 100644 --- a/protocol/v2/qbft/spectest/create_msg_type.go +++ b/protocol/v2/qbft/spectest/create_msg_type.go @@ -5,15 +5,13 @@ import ( "testing" "github.com/pkg/errors" - "github.com/stretchr/testify/require" - - "github.com/ssvlabs/ssv/protocol/v2/qbft/instance" - "github.com/ssvlabs/ssv-spec/qbft" specqbft "github.com/ssvlabs/ssv-spec/qbft" spectests "github.com/ssvlabs/ssv-spec/qbft/spectest/tests" spectypes "github.com/ssvlabs/ssv-spec/types" spectestingutils "github.com/ssvlabs/ssv-spec/types/testingutils" + "github.com/ssvlabs/ssv/protocol/v2/qbft/instance" + "github.com/stretchr/testify/require" ) type CreateMsgSpecTest struct { @@ -49,19 +47,10 @@ func (test *CreateMsgSpecTest) RunCreateMsg(t *testing.T) { default: t.Fail() } + validateError(t, err, test.Name, test.ExpectedError) - if err != nil && len(test.ExpectedError) != 0 { - require.EqualError(t, err, test.ExpectedError) - return - } + r, err := msg.GetRoot() require.NoError(t, err) - - r, err2 := msg.GetRoot() - if len(test.ExpectedError) != 0 { - require.EqualError(t, err2, test.ExpectedError) - return - } - require.NoError(t, err2) require.EqualValues(t, test.ExpectedRoot, hex.EncodeToString(r[:])) } diff --git a/protocol/v2/qbft/spectest/error_override.go b/protocol/v2/qbft/spectest/error_override.go new file mode 100644 index 0000000000..ff7400be57 --- /dev/null +++ b/protocol/v2/qbft/spectest/error_override.go @@ -0,0 +1,37 @@ +package qbft + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// expectedErrorOverride contains mapping testName -> "error substring(s)" that will be used to +// replace the expected error (ExpectedError string) in spec tests. +// This mapping allows us to manually track/approve the exact differences between what spec +// tests expect against what we actually have in our implementation so we can have details in +// our error messages (or totally different error message altogether) without accidentally +// diverging from the spec (which might happen if we don't compare our impl errors vs spec). +// For simplicity, the overriding pattern is just a bunch of substrings we expect actual error +// to contain. +var expectedErrorOverride = map[string][]string{} + +// validateError checks err against expectedErr, overriding it by a set of patterns to match +// against (defined in expectedErrorOverride) in case test testName has been mapped in this way. +func validateError(t *testing.T, err error, testName string, expectedErr string) { + if len(expectedErr) == 0 { + require.NoError(t, err) + return + } + + require.Error(t, err, expectedErr) + + wantErrors := []string{expectedErr} + if errOverride, ok := expectedErrorOverride[testName]; ok { + wantErrors = errOverride + } + for _, wantError := range wantErrors { + require.Contains(t, err.Error(), wantError, fmt.Sprintf("testName: %s", testName)) + } +} diff --git a/protocol/v2/qbft/spectest/msg_processing_type.go b/protocol/v2/qbft/spectest/msg_processing_type.go index a2945f9397..65d57a2d04 100644 --- a/protocol/v2/qbft/spectest/msg_processing_type.go +++ b/protocol/v2/qbft/spectest/msg_processing_type.go @@ -13,14 +13,13 @@ import ( spectypes "github.com/ssvlabs/ssv-spec/types" spectestingutils "github.com/ssvlabs/ssv-spec/types/testingutils" typescomparable "github.com/ssvlabs/ssv-spec/types/testingutils/comparable" - "github.com/stretchr/testify/require" - "github.com/ssvlabs/ssv/logging" "github.com/ssvlabs/ssv/protocol/v2/qbft" "github.com/ssvlabs/ssv/protocol/v2/qbft/instance" qbfttesting "github.com/ssvlabs/ssv/protocol/v2/qbft/testing" protocoltesting "github.com/ssvlabs/ssv/protocol/v2/testing" "github.com/ssvlabs/ssv/utils/casts" + "github.com/stretchr/testify/require" ) // RunMsgProcessing processes MsgProcessingSpecTest. It probably may be removed. @@ -58,12 +57,7 @@ func RunMsgProcessing(t *testing.T, test *spectests.MsgProcessingSpecTest) { lastErr = err } } - - if len(test.ExpectedError) != 0 { - require.EqualError(t, lastErr, test.ExpectedError, "expected %v, but got %v", test.ExpectedError, lastErr) - } else { - require.NoError(t, lastErr) - } + validateError(t, lastErr, test.Name, test.ExpectedError) postRoot, err := preInstance.State.GetRoot() require.NoError(t, err) diff --git a/protocol/v2/qbft/spectest/msg_type.go b/protocol/v2/qbft/spectest/msg_type.go index 4aba60f465..b15069f5a2 100644 --- a/protocol/v2/qbft/spectest/msg_type.go +++ b/protocol/v2/qbft/spectest/msg_type.go @@ -4,14 +4,12 @@ import ( "testing" specqbft "github.com/ssvlabs/ssv-spec/qbft" - "github.com/stretchr/testify/require" - spectests "github.com/ssvlabs/ssv-spec/qbft/spectest/tests" + "github.com/stretchr/testify/require" ) func RunMsg(t *testing.T, test *spectests.MsgSpecTest) { // using only spec struct so this test can be imported var lastErr error - for i, msg := range test.Messages { if err := msg.Validate(); err != nil { lastErr = err @@ -37,12 +35,5 @@ func RunMsg(t *testing.T, test *spectests.MsgSpecTest) { // using only spec stru require.EqualValues(t, test.ExpectedRoots[i], r) } } - - // check error - if len(test.ExpectedError) != 0 { - t.Log("Expected error", test.ExpectedError) - require.EqualError(t, lastErr, test.ExpectedError) - } else { - require.NoError(t, lastErr) - } + validateError(t, lastErr, test.Name, test.ExpectedError) } diff --git a/protocol/v2/qbft/spectest/qbft_mapping_test.go b/protocol/v2/qbft/spectest/qbft_mapping_test.go index 53477b59e1..e1113b5dd7 100644 --- a/protocol/v2/qbft/spectest/qbft_mapping_test.go +++ b/protocol/v2/qbft/spectest/qbft_mapping_test.go @@ -7,17 +7,15 @@ import ( "strings" "testing" - "github.com/ssvlabs/ssv/exporter/convert" - spectests "github.com/ssvlabs/ssv-spec/qbft/spectest/tests" "github.com/ssvlabs/ssv-spec/qbft/spectest/tests/timeout" "github.com/ssvlabs/ssv-spec/types/testingutils" - "github.com/stretchr/testify/require" - + "github.com/ssvlabs/ssv/exporter/convert" "github.com/ssvlabs/ssv/logging" "github.com/ssvlabs/ssv/protocol/v2/qbft/instance" testing2 "github.com/ssvlabs/ssv/protocol/v2/qbft/testing" protocoltesting "github.com/ssvlabs/ssv/protocol/v2/testing" + "github.com/stretchr/testify/require" ) func TestQBFTMapping(t *testing.T) { diff --git a/protocol/v2/qbft/spectest/timeout_type.go b/protocol/v2/qbft/spectest/timeout_type.go index 386c2bee69..3ac9408649 100644 --- a/protocol/v2/qbft/spectest/timeout_type.go +++ b/protocol/v2/qbft/spectest/timeout_type.go @@ -7,11 +7,9 @@ import ( spectypes "github.com/ssvlabs/ssv-spec/types" "github.com/ssvlabs/ssv-spec/types/testingutils" - "github.com/ssvlabs/ssv/logging" "github.com/ssvlabs/ssv/protocol/v2/qbft/instance" "github.com/ssvlabs/ssv/protocol/v2/qbft/roundtimer" - "github.com/stretchr/testify/require" ) @@ -27,12 +25,7 @@ type SpecTest struct { func RunTimeout(t *testing.T, test *SpecTest) { logger := logging.TestLogger(t) err := test.Pre.UponRoundTimeout(logger) - - if len(test.ExpectedError) != 0 { - require.EqualError(t, err, test.ExpectedError) - } else { - require.NoError(t, err) - } + validateError(t, err, test.Name, test.ExpectedError) // test calling timeout timer, ok := test.Pre.GetConfig().GetTimer().(*roundtimer.TestQBFTTimer) diff --git a/protocol/v2/ssv/spectest/committee_msg_processing_type.go b/protocol/v2/ssv/spectest/committee_msg_processing_type.go index bb65750e84..60c06c4290 100644 --- a/protocol/v2/ssv/spectest/committee_msg_processing_type.go +++ b/protocol/v2/ssv/spectest/committee_msg_processing_type.go @@ -9,6 +9,8 @@ import ( "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/pkg/errors" + "github.com/ssvlabs/ssv-spec/ssv" + "github.com/ssvlabs/ssv-spec/types" spectypes "github.com/ssvlabs/ssv-spec/types" spectestingutils "github.com/ssvlabs/ssv-spec/types/testingutils" typescomparable "github.com/ssvlabs/ssv-spec/types/testingutils/comparable" @@ -19,13 +21,11 @@ import ( protocoltesting "github.com/ssvlabs/ssv/protocol/v2/testing" "github.com/stretchr/testify/require" "go.uber.org/zap" - - "github.com/ssvlabs/ssv-spec/ssv" - "github.com/ssvlabs/ssv-spec/types" ) type CommitteeSpecTest struct { Name string + ParentName string Committee *validator.Committee Input []interface{} // Can be a types.Duty or a *types.SignedSSVMessage PostDutyCommitteeRoot string @@ -39,16 +39,15 @@ func (test *CommitteeSpecTest) TestName() string { return test.Name } +func (test *CommitteeSpecTest) FullName() string { + return strings.Replace(test.ParentName+"_"+test.Name, " ", "_", -1) +} + // RunAsPartOfMultiTest runs the test as part of a MultiCommitteeSpecTest func (test *CommitteeSpecTest) RunAsPartOfMultiTest(t *testing.T) { logger := logging.TestLogger(t) lastErr := test.runPreTesting(logger) - - if len(test.ExpectedError) != 0 { - require.EqualError(t, lastErr, test.ExpectedError) - } else { - require.NoError(t, lastErr) - } + validateError(t, lastErr, test.FullName(), test.ExpectedError) broadcastedMsgs := make([]*types.SignedSSVMessage, 0) broadcastedRoots := make([]phase0.Root, 0) @@ -140,6 +139,7 @@ func (tests *MultiCommitteeSpecTest) Run(t *testing.T) { for _, test := range tests.Tests { t.Run(test.TestName(), func(t *testing.T) { + test.ParentName = tests.Name test.RunAsPartOfMultiTest(t) }) } diff --git a/protocol/v2/ssv/spectest/error_override.go b/protocol/v2/ssv/spectest/error_override.go new file mode 100644 index 0000000000..ef5654181e --- /dev/null +++ b/protocol/v2/ssv/spectest/error_override.go @@ -0,0 +1,92 @@ +package spectest + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +// expectedErrorOverride contains mapping testName -> "error substring(s)" that will be used to +// replace the expected error (ExpectedError string) in spec tests. +// This mapping allows us to manually track/approve the exact differences between what spec +// tests expect against what we actually have in our implementation so we can have details in +// our error messages (or totally different error message altogether) without accidentally +// diverging from the spec (which might happen if we don't compare our impl errors vs spec). +// For simplicity, the overriding pattern is just a bunch of substrings we expect actual error +// to contain. +var expectedErrorOverride = map[string][]string{ + "consensus_past_message_attester": { + "no runner found for consensus message's slot", + }, + "consensus_past_message_sync_committee": { + "no runner found for consensus message's slot", + }, + "consensus_past_message_attester_and_sync_committee": { + "no runner found for consensus message's slot", + }, + "consensus_future_decided_no_running_instance_attester": { + "no runner found for consensus message's slot", + }, + "consensus_future_decided_no_running_instance_sync_committee": { + "no runner found for consensus message's slot", + }, + "consensus_future_decided_no_running_instance_attester_and_sync_committee": { + "no runner found for consensus message's slot", + }, + "post_consensus_invalid_msg_slot_attester": { + "no runner found for post consensus sig message's slot", + }, + "post_consensus_invalid_msg_slot_sync_committee": { + "no runner found for post consensus sig message's slot", + }, + "post_consensus_invalid_msg_slot_attester_and_sync_committee": { + "no runner found for post consensus sig message's slot", + }, + "consensus_future_decided_attester": { + "no runner found for consensus message's slot", + }, + "consensus_future_decided_sync_committee": { + "no runner found for consensus message's slot", + }, + "consensus_future_decided_attester_and_sync_committee": { + "no runner found for consensus message's slot", + }, + "consensus_future_message_attester": { + "no runner found for consensus message's slot", + }, + "consensus_future_message_sync_committee": { + "no runner found for consensus message's slot", + }, + "consensus_future_message_attester_sync_committee": { + "no runner found for consensus message's slot", + }, + "past_msg_duty_does_not_exist_30_attestation": { + "no runner found for consensus message's slot", + }, + "past_msg_duty_does_not_exist_30_sync_committee": { + "no runner found for consensus message's slot", + }, + "past_msg_duty_does_not_exist_30_attestation_30_sync_committee": { + "no runner found for consensus message's slot", + }, +} + +// validateError checks err against expectedErr, overriding it by a set of patterns to match +// against (defined in expectedErrorOverride) in case test testName has been mapped in this way. +func validateError(t *testing.T, err error, testName string, expectedErr string) { + if len(expectedErr) == 0 { + require.NoError(t, err) + return + } + + require.Error(t, err, expectedErr) + + wantErrors := []string{expectedErr} + if errOverride, ok := expectedErrorOverride[testName]; ok { + wantErrors = errOverride + } + for _, wantError := range wantErrors { + require.Contains(t, err.Error(), wantError, fmt.Sprintf("testName: %s", testName)) + } +} diff --git a/protocol/v2/ssv/spectest/msg_processing_type.go b/protocol/v2/ssv/spectest/msg_processing_type.go index a26a732f31..30fd849afd 100644 --- a/protocol/v2/ssv/spectest/msg_processing_type.go +++ b/protocol/v2/ssv/spectest/msg_processing_type.go @@ -15,9 +15,6 @@ import ( spectypes "github.com/ssvlabs/ssv-spec/types" spectestingutils "github.com/ssvlabs/ssv-spec/types/testingutils" typescomparable "github.com/ssvlabs/ssv-spec/types/testingutils/comparable" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - "github.com/ssvlabs/ssv/integration/qbft/tests" "github.com/ssvlabs/ssv/logging" "github.com/ssvlabs/ssv/networkconfig" @@ -27,10 +24,13 @@ import ( ssvprotocoltesting "github.com/ssvlabs/ssv/protocol/v2/ssv/testing" "github.com/ssvlabs/ssv/protocol/v2/ssv/validator" protocoltesting "github.com/ssvlabs/ssv/protocol/v2/testing" + "github.com/stretchr/testify/require" + "go.uber.org/zap" ) type MsgProcessingSpecTest struct { Name string + ParentName string Runner runner.Runner Duty spectypes.Duty Messages []*spectypes.SignedSSVMessage @@ -48,6 +48,10 @@ func (test *MsgProcessingSpecTest) TestName() string { return test.Name } +func (test *MsgProcessingSpecTest) FullName() string { + return strings.Replace(test.ParentName+"_"+test.Name, " ", "_", -1) +} + func RunMsgProcessing(t *testing.T, test *MsgProcessingSpecTest) { logger := logging.TestLogger(t) test.overrideStateComparison(t) @@ -147,12 +151,7 @@ func (test *MsgProcessingSpecTest) runPreTesting(ctx context.Context, logger *za func (test *MsgProcessingSpecTest) RunAsPartOfMultiTest(t *testing.T, logger *zap.Logger) { ctx := context.Background() v, c, lastErr := test.runPreTesting(ctx, logger) - - if len(test.ExpectedError) != 0 { - require.EqualError(t, lastErr, test.ExpectedError) - } else { - require.NoError(t, lastErr) - } + validateError(t, lastErr, test.FullName(), test.ExpectedError) network := &spectestingutils.TestingNetwork{} var beaconNetwork *tests.TestingBeaconNodeWrapped diff --git a/protocol/v2/ssv/spectest/multi_msg_processing_type.go b/protocol/v2/ssv/spectest/multi_msg_processing_type.go index d7a3e41841..40c11041ca 100644 --- a/protocol/v2/ssv/spectest/multi_msg_processing_type.go +++ b/protocol/v2/ssv/spectest/multi_msg_processing_type.go @@ -27,6 +27,7 @@ func (tests *MultiMsgProcessingSpecTest) Run(t *testing.T) { for _, test := range tests.Tests { t.Run(test.TestName(), func(t *testing.T) { + test.ParentName = tests.Name test.RunAsPartOfMultiTest(t, tests.logger) }) } diff --git a/protocol/v2/ssv/spectest/sync_committee_aggregator_proof_type.go b/protocol/v2/ssv/spectest/sync_committee_aggregator_proof_type.go index c18738c0f7..a08e4dbb4e 100644 --- a/protocol/v2/ssv/spectest/sync_committee_aggregator_proof_type.go +++ b/protocol/v2/ssv/spectest/sync_committee_aggregator_proof_type.go @@ -12,13 +12,12 @@ import ( "github.com/ssvlabs/ssv-spec/types/testingutils" typescomparable "github.com/ssvlabs/ssv-spec/types/testingutils/comparable" "github.com/ssvlabs/ssv/integration/qbft/tests" - "github.com/stretchr/testify/require" - "github.com/ssvlabs/ssv/logging" "github.com/ssvlabs/ssv/protocol/v2/ssv/queue" "github.com/ssvlabs/ssv/protocol/v2/ssv/runner" ssvtesting "github.com/ssvlabs/ssv/protocol/v2/ssv/testing" protocoltesting "github.com/ssvlabs/ssv/protocol/v2/testing" + "github.com/stretchr/testify/require" ) func RunSyncCommitteeAggProof(t *testing.T, test *synccommitteeaggregator.SyncCommitteeAggregatorProofSpecTest) { @@ -43,12 +42,7 @@ func RunSyncCommitteeAggProof(t *testing.T, test *synccommitteeaggregator.SyncCo lastErr = err } } - - if len(test.ExpectedError) != 0 { - require.EqualError(t, lastErr, test.ExpectedError) - } else { - require.NoError(t, lastErr) - } + validateError(t, lastErr, test.Name, test.ExpectedError) // post root postRoot, err := r.GetBaseRunner().State.GetRoot() diff --git a/protocol/v2/ssv/validator/committee.go b/protocol/v2/ssv/validator/committee.go index c953ed8370..b858dabf89 100644 --- a/protocol/v2/ssv/validator/committee.go +++ b/protocol/v2/ssv/validator/committee.go @@ -6,20 +6,18 @@ import ( "encoding/json" "fmt" "sync" - "time" "github.com/attestantio/go-eth2-client/spec/phase0" "github.com/pkg/errors" "github.com/ssvlabs/ssv-spec/qbft" spectypes "github.com/ssvlabs/ssv-spec/types" - "go.uber.org/zap" - "github.com/ssvlabs/ssv/ibft/storage" "github.com/ssvlabs/ssv/logging/fields" "github.com/ssvlabs/ssv/protocol/v2/message" "github.com/ssvlabs/ssv/protocol/v2/ssv/queue" "github.com/ssvlabs/ssv/protocol/v2/ssv/runner" "github.com/ssvlabs/ssv/protocol/v2/types" + "go.uber.org/zap" ) var ( @@ -34,10 +32,11 @@ type Committee struct { ctx context.Context cancel context.CancelFunc - mtx sync.RWMutex BeaconNetwork spectypes.BeaconNetwork Storage *storage.QBFTStores + // mtx syncs access to Queues, Runners, Shares. + mtx sync.RWMutex Queues map[phase0.Slot]queueContainer Runners map[phase0.Slot]*runner.CommitteeRunner Shares map[phase0.ValidatorIndex]*spectypes.Share @@ -90,80 +89,46 @@ func (c *Committee) RemoveShare(validatorIndex phase0.ValidatorIndex) { } } -func (c *Committee) StartConsumeQueue(logger *zap.Logger, duty *spectypes.CommitteeDuty) error { - c.mtx.Lock() - defer c.mtx.Unlock() - - // Setting the cancel function separately due the queue could be created in HandleMessage - q, found := c.Queues[duty.Slot] - if !found { - return errors.New(fmt.Sprintf("no queue found for slot %d", duty.Slot)) +// StartDuty starts a new duty for the given slot +func (c *Committee) StartDuty(logger *zap.Logger, duty *spectypes.CommitteeDuty) error { + r, trimmedDuty, err := c.prepareDutyRunner(logger, duty) + if err != nil { + return fmt.Errorf("could not prepare duty runner: %w", err) } - r := c.Runners[duty.Slot] - if r == nil { - return errors.New(fmt.Sprintf("no runner found for slot %d", duty.Slot)) + logger.Info("ℹ️ starting duty processing") + err = r.StartNewDuty(logger, trimmedDuty, c.CommitteeMember.GetQuorum()) + if err != nil { + return errors.Wrap(err, "runner failed to start duty") } - - // required to stop the queue consumer when timeout message is received by handler - queueCtx, cancelF := context.WithDeadline(c.ctx, time.Unix(c.BeaconNetwork.EstimatedTimeAtSlot(duty.Slot+runnerExpirySlots), 0)) - - go func() { - defer cancelF() - if err := c.ConsumeQueue(queueCtx, q, logger, duty.Slot, c.ProcessMessage, r); err != nil { - logger.Error("❗failed consuming committee queue", zap.Error(err)) - } - }() return nil } -// StartDuty starts a new duty for the given slot -func (c *Committee) StartDuty(logger *zap.Logger, duty *spectypes.CommitteeDuty) error { +func (c *Committee) prepareDutyRunner(logger *zap.Logger, duty *spectypes.CommitteeDuty) ( + r *runner.CommitteeRunner, + trimmedDuty *spectypes.CommitteeDuty, + err error, +) { c.mtx.Lock() defer c.mtx.Unlock() - if len(duty.ValidatorDuties) == 0 { - return errors.New("no beacon duties") - } if _, exists := c.Runners[duty.Slot]; exists { - return errors.New(fmt.Sprintf("CommitteeRunner for slot %d already exists", duty.Slot)) + return nil, nil, fmt.Errorf("CommitteeRunner for slot %d already exists", duty.Slot) } - // Filter out Beacon duties for which we don't have a share. - filteredDuty := &spectypes.CommitteeDuty{ - Slot: duty.Slot, - ValidatorDuties: make([]*spectypes.ValidatorDuty, 0, len(duty.ValidatorDuties)), - } - shares := make(map[phase0.ValidatorIndex]*spectypes.Share, len(duty.ValidatorDuties)) - attesters := make([]spectypes.ShareValidatorPK, 0, len(duty.ValidatorDuties)) - for _, beaconDuty := range duty.ValidatorDuties { - share, exists := c.Shares[beaconDuty.ValidatorIndex] - if !exists { - logger.Debug("no share for validator duty", - fields.BeaconRole(beaconDuty.Type), - zap.Uint64("validator_index", uint64(beaconDuty.ValidatorIndex))) - continue - } - shares[beaconDuty.ValidatorIndex] = share - filteredDuty.ValidatorDuties = append(filteredDuty.ValidatorDuties, beaconDuty) - - if beaconDuty.Type == spectypes.BNRoleAttester { - attesters = append(attesters, share.SharePubKey) - } - } - if len(shares) == 0 { - return errors.New("no shares for duty's validators") + shares, attesters, trimmedDuty, err := c.prepareDuty(logger, duty) + if err != nil { + return nil, nil, fmt.Errorf("prepare duty: %w", err) } - duty = filteredDuty - runner, err := c.CreateRunnerFn(duty.Slot, shares, attesters, c.dutyGuard) + r, err = c.CreateRunnerFn(duty.Slot, shares, attesters, c.dutyGuard) if err != nil { - return errors.Wrap(err, "could not create CommitteeRunner") + return nil, nil, errors.Wrap(err, "could not create CommitteeRunner") } // Set timeout function. - runner.GetBaseRunner().TimeoutF = c.onTimeout - c.Runners[duty.Slot] = runner + r.GetBaseRunner().TimeoutF = c.onTimeout + c.Runners[duty.Slot] = r _, queueExists := c.Queues[duty.Slot] if !queueExists { c.Queues[duty.Slot] = queueContainer{ @@ -183,25 +148,48 @@ func (c *Committee) StartDuty(logger *zap.Logger, duty *spectypes.CommitteeDuty) pruneLogger.Error("couldn't prune expired committee runners", zap.Error(err)) } - logger.Info("ℹ️ starting duty processing") - err = runner.StartNewDuty(logger, duty, c.CommitteeMember.GetQuorum()) - if err != nil { - return errors.Wrap(err, "runner failed to start duty") - } - return nil + return r, trimmedDuty, nil } -func (c *Committee) PushToQueue(slot phase0.Slot, dec *queue.SSVMessage) { - c.mtx.RLock() - queue, exists := c.Queues[slot] - c.mtx.RUnlock() - if !exists { - c.logger.Warn("cannot push to non-existing queue", zap.Uint64("slot", uint64(slot))) - return +// prepareDuty will analyse duty and prepare the data we need to further process it. +func (c *Committee) prepareDuty(logger *zap.Logger, duty *spectypes.CommitteeDuty) ( + map[phase0.ValidatorIndex]*spectypes.Share, + []spectypes.ShareValidatorPK, + *spectypes.CommitteeDuty, + error, +) { + if len(duty.ValidatorDuties) == 0 { + return nil, nil, nil, errors.New("no beacon duties") + } + + trimmedDuty := &spectypes.CommitteeDuty{ + Slot: duty.Slot, + ValidatorDuties: make([]*spectypes.ValidatorDuty, 0, len(duty.ValidatorDuties)), + } + shares := make(map[phase0.ValidatorIndex]*spectypes.Share, len(duty.ValidatorDuties)) + attesters := make([]spectypes.ShareValidatorPK, 0, len(duty.ValidatorDuties)) + for _, beaconDuty := range duty.ValidatorDuties { + share, exists := c.Shares[beaconDuty.ValidatorIndex] + if !exists { + // Filter out Beacon duties for which we don't have a share. + logger.Debug("committee has no share for validator duty", + fields.BeaconRole(beaconDuty.Type), + zap.Uint64("validator_index", uint64(beaconDuty.ValidatorIndex))) + continue + } + shares[beaconDuty.ValidatorIndex] = share + trimmedDuty.ValidatorDuties = append(trimmedDuty.ValidatorDuties, beaconDuty) + + if beaconDuty.Type == spectypes.BNRoleAttester { + attesters = append(attesters, share.SharePubKey) + } } - if pushed := queue.Q.TryPush(dec); !pushed { - c.logger.Warn("dropping ExecuteDuty message because the queue is full") + + if len(shares) == 0 { + return nil, nil, nil, errors.New("no shares for duty's validators") } + + return shares, attesters, trimmedDuty, nil } // ProcessMessage processes Network Message of all types @@ -231,13 +219,13 @@ func (c *Committee) ProcessMessage(logger *zap.Logger, msg *queue.SSVMessage) er if err := qbftMsg.Validate(); err != nil { return errors.Wrap(err, "invalid qbft Message") } - c.mtx.Lock() - runner, exists := c.Runners[phase0.Slot(qbftMsg.Height)] - c.mtx.Unlock() + c.mtx.RLock() + r, exists := c.Runners[phase0.Slot(qbftMsg.Height)] + c.mtx.RUnlock() if !exists { - return errors.New("no runner found for message's slot") + return fmt.Errorf("no runner found for consensus message's slot %d", qbftMsg.Height) } - return runner.ProcessConsensus(logger, msg.SignedSSVMessage) + return r.ProcessConsensus(logger, msg.SignedSSVMessage) case spectypes.SSVPartialSignatureMsgType: pSigMessages := &spectypes.PartialSignatureMessages{} if err := pSigMessages.Decode(msg.SignedSSVMessage.SSVMessage.GetData()); err != nil { @@ -254,13 +242,13 @@ func (c *Committee) ProcessMessage(logger *zap.Logger, msg *queue.SSVMessage) er } if pSigMessages.Type == spectypes.PostConsensusPartialSig { - c.mtx.Lock() - runner, exists := c.Runners[pSigMessages.Slot] - c.mtx.Unlock() + c.mtx.RLock() + r, exists := c.Runners[pSigMessages.Slot] + c.mtx.RUnlock() if !exists { - return errors.New("no runner found for message's slot") + return fmt.Errorf("no runner found for post consensus sig message's slot %d", pSigMessages.Slot) } - return runner.ProcessPostConsensus(logger, pSigMessages) + return r.ProcessPostConsensus(logger, pSigMessages) } case message.SSVEventMsgType: return c.handleEventMessage(logger, msg) @@ -268,8 +256,8 @@ func (c *Committee) ProcessMessage(logger *zap.Logger, msg *queue.SSVMessage) er return errors.New("unknown msg") } return nil - } + func (c *Committee) unsafePruneExpiredRunners(logger *zap.Logger, currentSlot phase0.Slot) error { if runnerExpirySlots > currentSlot { return nil diff --git a/protocol/v2/ssv/validator/committee_queue.go b/protocol/v2/ssv/validator/committee_queue.go index bfe8776df7..f1b7b6d221 100644 --- a/protocol/v2/ssv/validator/committee_queue.go +++ b/protocol/v2/ssv/validator/committee_queue.go @@ -3,17 +3,18 @@ package validator import ( "context" "errors" + "fmt" + "time" "github.com/attestantio/go-eth2-client/spec/phase0" specqbft "github.com/ssvlabs/ssv-spec/qbft" spectypes "github.com/ssvlabs/ssv-spec/types" - "go.uber.org/zap" - "github.com/ssvlabs/ssv/logging/fields" "github.com/ssvlabs/ssv/protocol/v2/message" "github.com/ssvlabs/ssv/protocol/v2/qbft/instance" "github.com/ssvlabs/ssv/protocol/v2/ssv/queue" "github.com/ssvlabs/ssv/protocol/v2/ssv/runner" + "go.uber.org/zap" ) // HandleMessage handles a spectypes.SSVMessage. @@ -30,9 +31,23 @@ func (c *Committee) HandleMessage(logger *zap.Logger, msg *queue.SSVMessage) { return } - c.mtx.RLock() // read v.Queues + q := c.prepareQueue(logger, slot) + + if pushed := q.Q.TryPush(msg); !pushed { + msgID := msg.MsgID.String() + logger.Warn("❗ dropping message because the queue is full", + zap.String("msg_type", message.MsgTypeToString(msg.MsgType)), + zap.String("msg_id", msgID)) + } else { + // logger.Debug("📬 queue: pushed message", fields.MessageID(msg.MsgID), fields.MessageType(msg.MsgType)) + } +} + +func (c *Committee) prepareQueue(logger *zap.Logger, slot phase0.Slot) queueContainer { + c.mtx.Lock() + defer c.mtx.Unlock() + q, ok := c.Queues[slot] - c.mtx.RUnlock() if !ok { q = queueContainer{ Q: queue.WithMetrics(queue.New(1000), nil), // TODO alan: get queue opts from options @@ -43,34 +58,39 @@ func (c *Committee) HandleMessage(logger *zap.Logger, msg *queue.SSVMessage) { //Quorum: options.SSVShare.Share,// TODO }, } - c.mtx.Lock() c.Queues[slot] = q - c.mtx.Unlock() logger.Debug("missing queue for slot created", fields.Slot(slot)) } - if pushed := q.Q.TryPush(msg); !pushed { - msgID := msg.MsgID.String() - logger.Warn("❗ dropping message because the queue is full", - zap.String("msg_type", message.MsgTypeToString(msg.MsgType)), - zap.String("msg_id", msgID)) - } else { - // logger.Debug("📬 queue: pushed message", fields.MessageID(msg.MsgID), fields.MessageType(msg.MsgType)) - } + return q } -//// StartQueueConsumer start ConsumeQueue with handler -//func (v *Committee) StartQueueConsumer(logger *zap.Logger, msgID spectypes.MessageID, handler MessageHandler) { -// ctx, cancel := context.WithCancel(v.ctx) -// defer cancel() -// -// for ctx.Err() == nil { -// err := v.ConsumeQueue(logger, msgID, handler) -// if err != nil { -// logger.Debug("❗ failed consuming queue", zap.Error(err)) -// } -// } -//} +func (c *Committee) StartConsumeQueue(logger *zap.Logger, duty *spectypes.CommitteeDuty) error { + c.mtx.Lock() + defer c.mtx.Unlock() + + // Setting the cancel function separately due the queue could be created in HandleMessage + q, found := c.Queues[duty.Slot] + if !found { + return fmt.Errorf("no queue found for slot %d", duty.Slot) + } + + r := c.Runners[duty.Slot] + if r == nil { + return fmt.Errorf("no runner found for slot %d", duty.Slot) + } + + // required to stop the queue consumer when timeout message is received by handler + queueCtx, cancelF := context.WithDeadline(c.ctx, time.Unix(c.BeaconNetwork.EstimatedTimeAtSlot(duty.Slot+runnerExpirySlots), 0)) + + go func() { + defer cancelF() + if err := c.ConsumeQueue(queueCtx, q, logger, c.ProcessMessage, r); err != nil { + logger.Error("❗failed consuming committee queue", zap.Error(err)) + } + }() + return nil +} // ConsumeQueue consumes messages from the queue.Queue of the controller // it checks for current state @@ -78,7 +98,6 @@ func (c *Committee) ConsumeQueue( ctx context.Context, q queueContainer, logger *zap.Logger, - slot phase0.Slot, handler MessageHandler, rnr *runner.CommitteeRunner, ) error { diff --git a/protocol/v2/ssv/validator/events.go b/protocol/v2/ssv/validator/events.go index df91517187..6fa26eac00 100644 --- a/protocol/v2/ssv/validator/events.go +++ b/protocol/v2/ssv/validator/events.go @@ -4,11 +4,10 @@ import ( "fmt" "github.com/ssvlabs/ssv/logging/fields" - "go.uber.org/zap" - "github.com/ssvlabs/ssv/protocol/v2/ssv/queue" "github.com/ssvlabs/ssv/protocol/v2/ssv/runner" "github.com/ssvlabs/ssv/protocol/v2/types" + "go.uber.org/zap" ) func (v *Validator) handleEventMessage(logger *zap.Logger, msg *queue.SSVMessage, dutyRunner runner.Runner) error { @@ -43,12 +42,12 @@ func (c *Committee) handleEventMessage(logger *zap.Logger, msg *queue.SSVMessage if err != nil { return err } - c.mtx.Lock() + c.mtx.RLock() dutyRunner, found := c.Runners[slot] - c.mtx.Unlock() + c.mtx.RUnlock() if !found { - logger.Error("no committee runner or queue found for slot", fields.Slot(slot), fields.MessageID(msg.MsgID)) + logger.Error("timeout event: no committee runner found for slot", fields.Slot(slot), fields.MessageID(msg.MsgID)) return nil } diff --git a/protocol/v2/ssv/validator/timer.go b/protocol/v2/ssv/validator/timer.go index de53a8e555..9d86aefc3c 100644 --- a/protocol/v2/ssv/validator/timer.go +++ b/protocol/v2/ssv/validator/timer.go @@ -4,17 +4,15 @@ import ( "encoding/json" "github.com/attestantio/go-eth2-client/spec/phase0" - "github.com/pkg/errors" specqbft "github.com/ssvlabs/ssv-spec/qbft" spectypes "github.com/ssvlabs/ssv-spec/types" - "go.uber.org/zap" - "github.com/ssvlabs/ssv/logging/fields" "github.com/ssvlabs/ssv/protocol/v2/message" "github.com/ssvlabs/ssv/protocol/v2/qbft/roundtimer" "github.com/ssvlabs/ssv/protocol/v2/ssv/queue" "github.com/ssvlabs/ssv/protocol/v2/types" + "go.uber.org/zap" ) func (v *Validator) onTimeout(logger *zap.Logger, identifier spectypes.MessageID, height specqbft.Height) roundtimer.OnRoundTimeoutF { @@ -82,16 +80,12 @@ func (v *Validator) createTimerMessage(identifier spectypes.MessageID, height sp }, nil } -func (v *Committee) onTimeout(logger *zap.Logger, identifier spectypes.MessageID, height specqbft.Height) roundtimer.OnRoundTimeoutF { +func (c *Committee) onTimeout(logger *zap.Logger, identifier spectypes.MessageID, height specqbft.Height) roundtimer.OnRoundTimeoutF { return func(round specqbft.Round) { - v.mtx.RLock() // read-lock for v.Queues, v.state - defer v.mtx.RUnlock() + c.mtx.RLock() // read-lock for c.Queues, c.Runners + defer c.mtx.RUnlock() - // only run if the validator is started - //if v.state != uint32(Started) { - // return - //} - dr := v.Runners[phase0.Slot(height)] + dr := c.Runners[phase0.Slot(height)] if dr == nil { logger.Warn("❗no committee runner found for slot", fields.Slot(phase0.Slot(height))) return @@ -101,7 +95,7 @@ func (v *Committee) onTimeout(logger *zap.Logger, identifier spectypes.MessageID return } - msg, err := v.createTimerMessage(identifier, height, round) + msg, err := c.createTimerMessage(identifier, height, round) if err != nil { logger.Debug("❗ failed to create timer msg", zap.Error(err)) return @@ -112,7 +106,7 @@ func (v *Committee) onTimeout(logger *zap.Logger, identifier spectypes.MessageID return } - if pushed := v.Queues[phase0.Slot(height)].Q.TryPush(dec); !pushed { + if pushed := c.Queues[phase0.Slot(height)].Q.TryPush(dec); !pushed { logger.Warn("❗️ dropping timeout message because the queue is full", fields.Role(identifier.GetRoleType())) } @@ -120,7 +114,7 @@ func (v *Committee) onTimeout(logger *zap.Logger, identifier spectypes.MessageID } } -func (v *Committee) createTimerMessage(identifier spectypes.MessageID, height specqbft.Height, round specqbft.Round) (*spectypes.SSVMessage, error) { +func (c *Committee) createTimerMessage(identifier spectypes.MessageID, height specqbft.Height, round specqbft.Round) (*spectypes.SSVMessage, error) { td := types.TimeoutData{ Height: height, Round: round, diff --git a/protocol/v2/testing/test_utils.go b/protocol/v2/testing/test_utils.go index 2ae1c05dbb..032a0df81e 100644 --- a/protocol/v2/testing/test_utils.go +++ b/protocol/v2/testing/test_utils.go @@ -18,11 +18,10 @@ import ( specqbft "github.com/ssvlabs/ssv-spec/qbft" spectypes "github.com/ssvlabs/ssv-spec/types" "github.com/ssvlabs/ssv-spec/types/testingutils" - "golang.org/x/mod/modfile" - "golang.org/x/mod/module" - qbftstorage "github.com/ssvlabs/ssv/protocol/v2/qbft/storage" "github.com/ssvlabs/ssv/utils/rsaencryption" + "golang.org/x/mod/modfile" + "golang.org/x/mod/module" ) var ( diff --git a/scripts/spec-alignment/differ.config.yaml b/scripts/spec-alignment/differ.config.yaml index c03798e393..bdfb53b82b 100644 --- a/scripts/spec-alignment/differ.config.yaml +++ b/scripts/spec-alignment/differ.config.yaml @@ -1,4 +1,4 @@ -ApprovedChanges: ["50e5bb7eda99594e", "870a3a66aeccd737","4e22a08543b079b","56ceb03cd44ff702","188adfe8914e04c1","2438f9c5b82b69a3","1a716ee3bdb3170","90b166f78390af18","68219b82a1d9d829","c4c4caa5d0938b85","dfe99ce1d27b6cb1","35f5dab1f128d193","9a3973b64d7e8932","f33f07301a770d03","3e9e0dddfad3b302","d4fef6512374c1f5","b49f54cb45787e4b","59b2375130aef5df","f094cd0460432170","8e51881e527dd603","a7d6d58d9fa06379","1d124224ca4d0fe3","39ea06bfd1477d2d","7e2550bab51f22b2","87ebd29bd49fc52f","ef39dd5223e0d080","fe14e7f0503ea188","6146023d4d5708a2","aebb8e4348b6d667","973a2e6704dbf3","fb4cac598a68c592","257c7eb81d6eb245","2a8e94fe037e13fd","5e7eb878de54eec6","960a9c64cd4ec93c","57dfd255520bd849","ec333ff8a708db69","1cc1ff39ad91ee69","5714652b88e2d44f","7a53b3b037c56325","8c02ef1964464c30","19a268910a20da3d","af6e01ed565029f3","318b5169ac4dabb6","372c6b332e8ba699","c0d8a364c0db855a","4287381be4fb1841","b1614afc1da7794f","c214975412f3fd7","8bbf7eba3fa0cf7e","8e4ec8debe331b36","7a671d8fcefc3793","e2b0e9c6454c1c08","6707ecfefa5fec21","d5a7389d730464f1","8dfae3b3223d2de0","a81c092c985de728","968df5082c727ed6","9e53c73ee60b1cc2","9d265e99dd31d4f5","a34619e078d2e42f","17e8cec4f0625d53","e913f373aa88f333","cfc1e05c372d88dc","e5de6901d78b8833","57c1885b43dd8d19","e8a49856a5edd893","22ea21d10a2f861c","954e4fce01631c4e","108b9575f7c1d4bc","1f8d076449068f64","5a7ad98296703f6","159536003eeddac8","8ca8f82e67ddd3dd","16ebe47404323cc1","48bfe5cf1e578b47","dd83182b693a7216","308d21d9830f7047","6dde03147e012b1a","730c3e5e59393b7d","5b44a4b425ecc397","df5debc50ec8babc","92a41554b2910bb8","c36c680554dde59f","447feaa5cdc1a010","fda90c61f44cb149","cdbb4930eced584c","274336ec1127e6c0","2a496f5b3ad542d2","6b395912dde33b0e","cac56ec14994216b","8850900b5d9bcc65","15e7706486c6359e","cc22f28953b787ea","3bad6ae11596a574","8f84422a240d889c","5b265432dfbbaac7","43794bf5953db193","7975821460ebe1e7","173c505e12aabb8f","47ee0d148148a56f","8cc38593ebe049b6","bda3aec7157b095a","248712911696a851","f4d9c910f1dbaef7","1a2146fcad37acb8","b0b146f9bdab64b6","edfd442b4d725fbb","122f053573538a32","d720d714a20833e1", "f9c984e71b685f9b","8c6b4fee5a4c13ce","c0a8d2019a2c30d5", "717bef26105c733f","2f70630c27062353","2f70337ba7566a69","dd607a44e1341e6b","5210501625ac3de5","f786bf475b5085aa","18a66ed6e613d9c1","e8943e7741f6843d","276a489bd5a00032","ba3bba59f10bf6b","3c50ce0c8089d871","89ee72f6c610ab84","c92b95a85da2cb11","927ea6aed3f98f20","9338904026a0ce37","9683cfa19dc544a3","4d3fa2b8dfcb5f5b", "f19e9a2b295bcfb3", "b10199b2de6f03b8", "1afc17e358f9ca79","4b58762c0b433442","d293ec1bc61bb707","3e88c3b49d093605","4890ff80c88cc41d","5227ff3a225dd20d","81a60407a3a0ba80","db2ad807eb66254a","d308bd7c553ccdcf","bdaf172971637cbe","6ade9202843071fe","2fe8e14083997744","19c9a5362d1e1d3a","5956f803d239f178","92c55a4548a8b760","9a95524213bccfff","2f51a7338b86c229","e96966a281d74505","3ee479b9cbbc3a1d","82b392ba39c6c594","b9d2404e5c570019","24f528d85fb021f2","fe9609a785305d81","b0934079dcd986cc","a9c520a19b26049","d19a9403fd732d94","74a928f5dcb2fdd9","cbbfdb5e68cdac80","10e39d2ceda91f34","f99a004cf6697875","8fa5e8ebf7d223ec","6c80c145ba705243","fbabbc90d0b4178a","ab0c7f24e551ca6","af38a11cb8682c75","b110cba51df9f8d2","c4ff2ed3d20dc419","9295a5bb10efcec7","ab56ea44a75f898a","ff51ef26ab53ba58","df3771e2589008f9","106e5689655bcfc6","f90e0fb6883bff93","667656095cec39ee","9a5597af260c748a","9168b9cfb0cb98c8"] +ApprovedChanges: ["50e5bb7eda99594e", "870a3a66aeccd737","4e22a08543b079b","56ceb03cd44ff702","188adfe8914e04c1","2438f9c5b82b69a3","1a716ee3bdb3170","90b166f78390af18","68219b82a1d9d829","c4c4caa5d0938b85","dfe99ce1d27b6cb1","35f5dab1f128d193","9a3973b64d7e8932","f33f07301a770d03","3e9e0dddfad3b302","d4fef6512374c1f5","b49f54cb45787e4b","59b2375130aef5df","f094cd0460432170","8e51881e527dd603","a7d6d58d9fa06379","1d124224ca4d0fe3","39ea06bfd1477d2d","7e2550bab51f22b2","87ebd29bd49fc52f","ef39dd5223e0d080","fe14e7f0503ea188","6146023d4d5708a2","aebb8e4348b6d667","973a2e6704dbf3","fb4cac598a68c592","257c7eb81d6eb245","2a8e94fe037e13fd","5e7eb878de54eec6","960a9c64cd4ec93c","57dfd255520bd849","ec333ff8a708db69","1cc1ff39ad91ee69","5714652b88e2d44f","7a53b3b037c56325","8c02ef1964464c30","19a268910a20da3d","af6e01ed565029f3","318b5169ac4dabb6","372c6b332e8ba699","c0d8a364c0db855a","4287381be4fb1841","b1614afc1da7794f","c214975412f3fd7","8bbf7eba3fa0cf7e","8e4ec8debe331b36","7a671d8fcefc3793","e2b0e9c6454c1c08","6707ecfefa5fec21","d5a7389d730464f1","8dfae3b3223d2de0","a81c092c985de728","968df5082c727ed6","9e53c73ee60b1cc2","9d265e99dd31d4f5","a34619e078d2e42f","17e8cec4f0625d53","e913f373aa88f333","cfc1e05c372d88dc","e5de6901d78b8833","57c1885b43dd8d19","e8a49856a5edd893","22ea21d10a2f861c","954e4fce01631c4e","108b9575f7c1d4bc","1f8d076449068f64","5a7ad98296703f6","159536003eeddac8","8ca8f82e67ddd3dd","16ebe47404323cc1","48bfe5cf1e578b47","dd83182b693a7216","308d21d9830f7047","6dde03147e012b1a","730c3e5e59393b7d","5b44a4b425ecc397","df5debc50ec8babc","92a41554b2910bb8","c36c680554dde59f","447feaa5cdc1a010","fda90c61f44cb149","cdbb4930eced584c","274336ec1127e6c0","2a496f5b3ad542d2","6b395912dde33b0e","cac56ec14994216b","8850900b5d9bcc65","15e7706486c6359e","cc22f28953b787ea","3bad6ae11596a574","8f84422a240d889c","5b265432dfbbaac7","43794bf5953db193","7975821460ebe1e7","173c505e12aabb8f","47ee0d148148a56f","8cc38593ebe049b6","bda3aec7157b095a","248712911696a851","f4d9c910f1dbaef7","1a2146fcad37acb8","b0b146f9bdab64b6","edfd442b4d725fbb","122f053573538a32","d720d714a20833e1", "f9c984e71b685f9b","8c6b4fee5a4c13ce","c0a8d2019a2c30d5", "717bef26105c733f","2f70630c27062353","2f70337ba7566a69","dd607a44e1341e6b","5210501625ac3de5","f786bf475b5085aa","18a66ed6e613d9c1","e8943e7741f6843d","276a489bd5a00032","ba3bba59f10bf6b","3c50ce0c8089d871","89ee72f6c610ab84","c92b95a85da2cb11","927ea6aed3f98f20","9338904026a0ce37","9683cfa19dc544a3","4d3fa2b8dfcb5f5b", "f19e9a2b295bcfb3", "b10199b2de6f03b8", "1afc17e358f9ca79","4b58762c0b433442","d293ec1bc61bb707","3e88c3b49d093605","4890ff80c88cc41d","5227ff3a225dd20d","81a60407a3a0ba80","db2ad807eb66254a","d308bd7c553ccdcf","bdaf172971637cbe","6ade9202843071fe","2fe8e14083997744","19c9a5362d1e1d3a","5956f803d239f178","92c55a4548a8b760","9a95524213bccfff","2f51a7338b86c229","e96966a281d74505","3ee479b9cbbc3a1d","82b392ba39c6c594","b9d2404e5c570019","24f528d85fb021f2","fe9609a785305d81","b0934079dcd986cc","a9c520a19b26049","d19a9403fd732d94","74a928f5dcb2fdd9","cbbfdb5e68cdac80","10e39d2ceda91f34","f99a004cf6697875","8fa5e8ebf7d223ec","6c80c145ba705243","fbabbc90d0b4178a","ab0c7f24e551ca6","af38a11cb8682c75","b110cba51df9f8d2","c4ff2ed3d20dc419","9295a5bb10efcec7","ab56ea44a75f898a","ff51ef26ab53ba58","df3771e2589008f9","106e5689655bcfc6","f90e0fb6883bff93","667656095cec39ee","9a5597af260c748a","9168b9cfb0cb98c8,"b40d693202be4382","9129f80a9f1b1331","b3b73f9cdf1c39fb"] IgnoredIdentifiers: - logger