From e2da5383109fb22be329fc45ad5439ac91a46cdd Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Wed, 28 Feb 2024 14:21:12 +0300 Subject: [PATCH 01/13] core: integrating go-eth2-client for for v3 endpoint --- app/eth2wrap/eth2wrap_gen.go | 31 +++++++++++++++++++++++++++++++ app/eth2wrap/genwrap/genwrap.go | 1 + core/types.go | 18 +++++++++++++++++- core/types_test.go | 3 ++- go.mod | 3 +++ go.sum | 4 ++-- testutil/beaconmock/beaconmock.go | 10 ++++++++++ 7 files changed, 66 insertions(+), 4 deletions(-) diff --git a/app/eth2wrap/eth2wrap_gen.go b/app/eth2wrap/eth2wrap_gen.go index c679fed0d..50f12f384 100644 --- a/app/eth2wrap/eth2wrap_gen.go +++ b/app/eth2wrap/eth2wrap_gen.go @@ -59,6 +59,7 @@ type Client interface { eth2client.SyncCommitteeDutiesProvider eth2client.SyncCommitteeMessagesSubmitter eth2client.SyncCommitteeSubscriptionsSubmitter + eth2client.UniversalProposalProvider eth2client.ValidatorRegistrationsSubmitter eth2client.ValidatorsProvider eth2client.VoluntaryExitSubmitter @@ -369,6 +370,26 @@ func (m multi) Proposal(ctx context.Context, opts *api.ProposalOpts) (*api.Respo return res0, err } +// Proposal fetches a universal proposal for signing. +func (m multi) UniversalProposal(ctx context.Context, opts *api.UniversalProposalOpts) (*api.Response[*api.VersionedUniversalProposal], error) { + const label = "universal_proposal" + defer latency(label)() + + res0, err := provide(ctx, m.clients, + func(ctx context.Context, cl Client) (*api.Response[*api.VersionedUniversalProposal], error) { + return cl.UniversalProposal(ctx, opts) + }, + nil, m.selector, + ) + + if err != nil { + incError(label) + err = wrapError(ctx, err, label) + } + + return res0, err +} + // BeaconBlockRoot fetches a block's root given a set of options. // Note this endpoint is cached in go-eth2-client. func (m multi) BeaconBlockRoot(ctx context.Context, opts *api.BeaconBlockRootOpts) (*api.Response[*phase0.Root], error) { @@ -908,6 +929,16 @@ func (l *lazy) Proposal(ctx context.Context, opts *api.ProposalOpts) (res0 *api. return cl.Proposal(ctx, opts) } +// Proposal fetches a universal proposal for signing. +func (l *lazy) UniversalProposal(ctx context.Context, opts *api.UniversalProposalOpts) (res0 *api.Response[*api.VersionedUniversalProposal], err error) { + cl, err := l.getOrCreateClient(ctx) + if err != nil { + return res0, err + } + + return cl.UniversalProposal(ctx, opts) +} + // BeaconBlockRoot fetches a block's root given a set of options. func (l *lazy) BeaconBlockRoot(ctx context.Context, opts *api.BeaconBlockRootOpts) (res0 *api.Response[*phase0.Root], err error) { cl, err := l.getOrCreateClient(ctx) diff --git a/app/eth2wrap/genwrap/genwrap.go b/app/eth2wrap/genwrap/genwrap.go index 45981cad3..504066f5f 100644 --- a/app/eth2wrap/genwrap/genwrap.go +++ b/app/eth2wrap/genwrap/genwrap.go @@ -125,6 +125,7 @@ type Client interface { "SyncCommitteeContributionsSubmitter": true, "SyncCommitteeMessagesSubmitter": true, "SyncCommitteeSubscriptionsSubmitter": true, + "UniversalProposalProvider": true, "ValidatorsProvider": true, "ValidatorRegistrationsSubmitter": true, "VoluntaryExitSubmitter": true, diff --git a/core/types.go b/core/types.go index 940cdc72f..c7b1719c9 100644 --- a/core/types.go +++ b/core/types.go @@ -39,9 +39,10 @@ const ( DutyPrepareSyncContribution DutyType = 11 DutySyncContribution DutyType = 12 DutyInfoSync DutyType = 13 + DutyUniversalProposer DutyType = 14 // Only ever append new types here... - dutySentinel DutyType = 14 // Must always be last + dutySentinel DutyType = 15 // Must always be last ) func (d DutyType) Valid() bool { @@ -64,6 +65,7 @@ func (d DutyType) String() string { DutyPrepareSyncContribution: "prepare_sync_contribution", DutySyncContribution: "sync_contribution", DutyInfoSync: "info_sync", + DutyUniversalProposer: "universal_proposer", }[d] } @@ -142,6 +144,20 @@ func NewProposerDuty(slot uint64) Duty { } } +// NewUniversalProposerDuty returns a new universal proposer duty. +// It is a convenience function that is slightly more readable +// and concise than the struct literal equivalent: +// +// core.Duty{Slot: slot, Type: core.DutyUniversalProposer} +// vs +// core.NewUniversalProposerDuty(slot) +func NewUniversalProposerDuty(slot uint64) Duty { + return Duty{ + Slot: slot, + Type: DutyUniversalProposer, + } +} + // NewVoluntaryExit returns a new voluntary exit duty. It is a convenience function that is // slightly more readable and concise than the struct literal equivalent: // diff --git a/core/types_test.go b/core/types_test.go index 5af985e0d..0d14e0cbe 100644 --- a/core/types_test.go +++ b/core/types_test.go @@ -28,9 +28,10 @@ func TestBackwardsCompatability(t *testing.T) { require.EqualValues(t, 11, core.DutyPrepareSyncContribution) require.EqualValues(t, 12, core.DutySyncContribution) require.EqualValues(t, 13, core.DutyInfoSync) + require.EqualValues(t, 14, core.DutyUniversalProposer) // Add more types here. - const sentinel = core.DutyType(14) + const sentinel = core.DutyType(15) for i := core.DutyUnknown; i <= sentinel; i++ { if i == core.DutyUnknown { require.False(t, i.Valid()) diff --git a/go.mod b/go.mod index f8f76c794..2ec876a71 100644 --- a/go.mod +++ b/go.mod @@ -201,3 +201,6 @@ require ( // We're replacing kryptology with our own fork, which fixes dependencies' security vulnerabilities. replace github.com/coinbase/kryptology => github.com/ObolNetwork/kryptology v0.0.0-20231016091344-eed023b6cac8 + +// We're replacing kryptology with our own fork, which adds UniversalProposel model for produceBlockV3 endpoint. +replace github.com/attestantio/go-eth2-client v0.19.10 => github.com/obolNetwork/go-eth2-client v0.0.0-20240228100742-8eb0c82f8153 diff --git a/go.sum b/go.sum index 176826d2a..aa28c7af1 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,6 @@ github.com/ObolNetwork/kryptology v0.0.0-20231016091344-eed023b6cac8/go.mod h1:q github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= -github.com/attestantio/go-eth2-client v0.19.10 h1:NLs9mcBvZpBTZ3du7Ey2NHQoj8d3UePY7pFBXX6C6qs= -github.com/attestantio/go-eth2-client v0.19.10/go.mod h1:TTz7YF6w4z6ahvxKiHuGPn6DbQn7gH6HPuWm/DEQeGE= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= @@ -372,6 +370,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= +github.com/obolNetwork/go-eth2-client v0.0.0-20240228100742-8eb0c82f8153 h1:KtEnfQIcXHxZGQNqWl1FjkAbWG0Juq0iFVbicR/tmXQ= +github.com/obolNetwork/go-eth2-client v0.0.0-20240228100742-8eb0c82f8153/go.mod h1:TTz7YF6w4z6ahvxKiHuGPn6DbQn7gH6HPuWm/DEQeGE= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= diff --git a/testutil/beaconmock/beaconmock.go b/testutil/beaconmock/beaconmock.go index f2f3d9e84..f8d9739b2 100644 --- a/testutil/beaconmock/beaconmock.go +++ b/testutil/beaconmock/beaconmock.go @@ -118,6 +118,7 @@ type Mock struct { NodePeerCountFunc func(ctx context.Context) (int, error) ProposalFunc func(ctx context.Context, opts *eth2api.ProposalOpts) (*eth2api.VersionedProposal, error) BlindedProposalFunc func(ctx context.Context, opts *eth2api.BlindedProposalOpts) (*eth2api.VersionedBlindedProposal, error) + UniversalProposalFunc func(ctx context.Context, opts *eth2api.UniversalProposalOpts) (*eth2api.VersionedUniversalProposal, error) SignedBeaconBlockFunc func(ctx context.Context, blockID string) (*eth2spec.VersionedSignedBeaconBlock, error) ProposerDutiesFunc func(context.Context, eth2p0.Epoch, []eth2p0.ValidatorIndex) ([]*eth2v1.ProposerDuty, error) SubmitAttestationsFunc func(context.Context, []*eth2p0.Attestation) error @@ -181,6 +182,15 @@ func (m Mock) Proposal(ctx context.Context, opts *eth2api.ProposalOpts) (*eth2ap return wrapResponse(block), nil } +func (m Mock) UniversalProposal(ctx context.Context, opts *eth2api.UniversalProposalOpts) (*eth2api.Response[*eth2api.VersionedUniversalProposal], error) { + block, err := m.UniversalProposalFunc(ctx, opts) + if err != nil { + return nil, err + } + + return wrapResponse(block), nil +} + func (m Mock) BlindedProposal(ctx context.Context, opts *eth2api.BlindedProposalOpts) (*eth2api.Response[*eth2api.VersionedBlindedProposal], error) { block, err := m.BlindedProposalFunc(ctx, opts) if err != nil { From d2a4baa3de0100eef385298add7dd3a982374d3b Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Wed, 28 Feb 2024 14:30:36 +0300 Subject: [PATCH 02/13] Update go.mod Co-authored-by: Dhruv Bodani --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 2ec876a71..cde677ab0 100644 --- a/go.mod +++ b/go.mod @@ -202,5 +202,5 @@ require ( // We're replacing kryptology with our own fork, which fixes dependencies' security vulnerabilities. replace github.com/coinbase/kryptology => github.com/ObolNetwork/kryptology v0.0.0-20231016091344-eed023b6cac8 -// We're replacing kryptology with our own fork, which adds UniversalProposel model for produceBlockV3 endpoint. +// We're replacing go-eth2-client with our own fork, which adds UniversalProposel model for produceBlockV3 endpoint. replace github.com/attestantio/go-eth2-client v0.19.10 => github.com/obolNetwork/go-eth2-client v0.0.0-20240228100742-8eb0c82f8153 From 83d5f2a26f30d00f4a5731214b333af296329e01 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Wed, 28 Feb 2024 18:37:52 +0300 Subject: [PATCH 03/13] Fetcher supporting universal proposal --- core/fetcher/fetcher.go | 98 +++++++++++++------ core/fetcher/fetcher_test.go | 51 ++++++++++ core/signeddata.go | 1 + ...ation_VersionedBlindedProposal.json.golden | 3 +- core/unsigneddata.go | 80 +++++++++++++++ core/unsigneddata_test.go | 43 ++++++++ testutil/beaconmock/beaconmock_fuzz.go | 7 ++ testutil/beaconmock/options.go | 13 +++ 8 files changed, 267 insertions(+), 29 deletions(-) diff --git a/core/fetcher/fetcher.go b/core/fetcher/fetcher.go index 64a97c944..517cf3008 100644 --- a/core/fetcher/fetcher.go +++ b/core/fetcher/fetcher.go @@ -51,21 +51,26 @@ func (f *Fetcher) Fetch(ctx context.Context, duty core.Duty, defSet core.DutyDef ) switch duty.Type { - case core.DutyProposer: - unsignedSet, err = f.fetchProposerData(ctx, duty.Slot, defSet) - if err != nil { - return errors.Wrap(err, "fetch proposer data") - } case core.DutyAttester: unsignedSet, err = f.fetchAttesterData(ctx, duty.Slot, defSet) if err != nil { return errors.Wrap(err, "fetch attester data") } + case core.DutyProposer: + unsignedSet, err = f.fetchProposerData(ctx, duty.Slot, defSet) + if err != nil { + return errors.Wrap(err, "fetch proposer data") + } case core.DutyBuilderProposer: unsignedSet, err = f.fetchBuilderProposerData(ctx, duty.Slot, defSet) if err != nil { return errors.Wrap(err, "fetch builder proposer data") } + case core.DutyUniversalProposer: + unsignedSet, err = f.fetchUniversalProposerData(ctx, duty.Slot, defSet) + if err != nil { + return errors.Wrap(err, "fetch universal proposer data") + } case core.DutyAggregator: unsignedSet, err = f.fetchAggregatorData(ctx, duty.Slot, defSet) if err != nil { @@ -234,23 +239,35 @@ func (f *Fetcher) fetchAggregatorData(ctx context.Context, slot uint64, defSet c return resp, nil } +func (f *Fetcher) getProposalOpts(ctx context.Context, slot uint64, pubkey core.PubKey) (eth2p0.BLSSignature, [32]byte, error) { + // Fetch previously aggregated randao reveal from AggSigDB + dutyRandao := core.Duty{ + Slot: slot, + Type: core.DutyRandao, + } + randaoData, err := f.aggSigDBFunc(ctx, dutyRandao, pubkey) + if err != nil { + return eth2p0.BLSSignature{}, [32]byte{}, err + } + + randao := randaoData.Signature().ToETH2() + + // TODO(dhruv): replace hardcoded graffiti with the one from cluster-lock.json + var graffiti [32]byte + commitSHA, _ := version.GitCommit() + copy(graffiti[:], fmt.Sprintf("charon/%v-%s", version.Version, commitSHA)) + + return randao, graffiti, nil +} + func (f *Fetcher) fetchProposerData(ctx context.Context, slot uint64, defSet core.DutyDefinitionSet) (core.UnsignedDataSet, error) { resp := make(core.UnsignedDataSet) for pubkey := range defSet { - // Fetch previously aggregated randao reveal from AggSigDB - dutyRandao := core.NewRandaoDuty(slot) - randaoData, err := f.aggSigDBFunc(ctx, dutyRandao, pubkey) + randao, graffiti, err := f.getProposalOpts(ctx, slot, pubkey) if err != nil { return nil, err } - randao := randaoData.Signature().ToETH2() - - // TODO(dhruv): replace hardcoded graffiti with the one from cluster-lock.json - var graffiti [32]byte - commitSHA, _ := version.GitCommit() - copy(graffiti[:], fmt.Sprintf("charon/%v-%s", version.Version, commitSHA)) - opts := ð2api.ProposalOpts{ Slot: eth2p0.Slot(slot), RandaoReveal: randao, @@ -279,23 +296,11 @@ func (f *Fetcher) fetchProposerData(ctx context.Context, slot uint64, defSet cor func (f *Fetcher) fetchBuilderProposerData(ctx context.Context, slot uint64, defSet core.DutyDefinitionSet) (core.UnsignedDataSet, error) { resp := make(core.UnsignedDataSet) for pubkey := range defSet { - // Fetch previously aggregated randao reveal from AggSigDB - dutyRandao := core.Duty{ - Slot: slot, - Type: core.DutyRandao, - } - randaoData, err := f.aggSigDBFunc(ctx, dutyRandao, pubkey) + randao, graffiti, err := f.getProposalOpts(ctx, slot, pubkey) if err != nil { return nil, err } - randao := randaoData.Signature().ToETH2() - - // TODO(dhruv): replace hardcoded graffiti with the one from cluster-lock.json - var graffiti [32]byte - commitSHA, _ := version.GitCommit() - copy(graffiti[:], fmt.Sprintf("charon/%v-%s", version.Version, commitSHA)) - opts := ð2api.BlindedProposalOpts{ Slot: eth2p0.Slot(slot), RandaoReveal: randao, @@ -320,6 +325,43 @@ func (f *Fetcher) fetchBuilderProposerData(ctx context.Context, slot uint64, def return resp, nil } +func (f *Fetcher) fetchUniversalProposerData(ctx context.Context, slot uint64, defSet core.DutyDefinitionSet) (core.UnsignedDataSet, error) { + resp := make(core.UnsignedDataSet) + for pubkey := range defSet { + randao, graffiti, err := f.getProposalOpts(ctx, slot, pubkey) + if err != nil { + return nil, err + } + + opts := ð2api.UniversalProposalOpts{ + Slot: eth2p0.Slot(slot), + RandaoReveal: randao, + Graffiti: graffiti, + } + eth2Resp, err := f.eth2Cl.UniversalProposal(ctx, opts) + if err != nil { + return nil, err + } + proposal := eth2Resp.Data + + // Ensure fee recipient is correctly populated in proposal. + if proposal.Full != nil { + verifyFeeRecipient(ctx, proposal.Full, f.feeRecipientFunc(pubkey)) + } else if proposal.Blinded != nil { + verifyFeeRecipientBlinded(ctx, proposal.Blinded, f.feeRecipientFunc(pubkey)) + } + + coreProposal, err := core.NewVersionedUniversalProposal(proposal) + if err != nil { + return nil, errors.Wrap(err, "new universal proposal") + } + + resp[pubkey] = coreProposal + } + + return resp, nil +} + // fetchContributionData fetches the sync committee contribution data. func (f *Fetcher) fetchContributionData(ctx context.Context, slot uint64, defSet core.DutyDefinitionSet) (core.UnsignedDataSet, error) { resp := make(core.UnsignedDataSet) diff --git a/core/fetcher/fetcher_test.go b/core/fetcher/fetcher_test.go index f3645e928..db1130541 100644 --- a/core/fetcher/fetcher_test.go +++ b/core/fetcher/fetcher_test.go @@ -345,6 +345,42 @@ func TestFetchBlocks(t *testing.T) { err = fetch.Fetch(ctx, duty, defSet) require.NoError(t, err) }) + + t.Run("fetch DutyUniversalProposer", func(t *testing.T) { + duty := core.NewUniversalProposerDuty(slot) + fetch, err := fetcher.New(bmock, func(core.PubKey) string { + return feeRecipientAddr + }) + require.NoError(t, err) + + fetch.RegisterAggSigDB(func(ctx context.Context, duty core.Duty, key core.PubKey) (core.SignedData, error) { + return randaoByPubKey[key], nil + }) + + fetch.Subscribe(func(ctx context.Context, resDuty core.Duty, resDataSet core.UnsignedDataSet) error { + require.Equal(t, duty, resDuty) + require.Len(t, resDataSet, 2) + + dutyDataA := resDataSet[pubkeysByIdx[vIdxA]].(core.VersionedUniversalProposal) + slotA, err := dutyDataA.Slot() + require.NoError(t, err) + require.EqualValues(t, slot, slotA) + require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataA.Full.Capella.Body.ExecutionPayload.FeeRecipient)) + assertRandaoUniversalProposal(t, randaoByPubKey[pubkeysByIdx[vIdxA]].Signature().ToETH2(), dutyDataA) + + dutyDataB := resDataSet[pubkeysByIdx[vIdxB]].(core.VersionedUniversalProposal) + slotB, err := dutyDataB.Slot() + require.NoError(t, err) + require.EqualValues(t, slot, slotB) + require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataB.Full.Capella.Body.ExecutionPayload.FeeRecipient)) + assertRandaoUniversalProposal(t, randaoByPubKey[pubkeysByIdx[vIdxB]].Signature().ToETH2(), dutyDataB) + + return nil + }) + + err = fetch.Fetch(ctx, duty, defSet) + require.NoError(t, err) + }) } func TestFetchSyncContribution(t *testing.T) { @@ -588,6 +624,21 @@ func assertRandaoBlindedBlock(t *testing.T, randao eth2p0.BLSSignature, block co } } +func assertRandaoUniversalProposal(t *testing.T, randao eth2p0.BLSSignature, p core.VersionedUniversalProposal) { + t.Helper() + + if p.Full != nil { + vp, err := core.NewVersionedProposal(p.Full) + require.NoError(t, err) + assertRandao(t, randao, vp) + } + if p.Blinded != nil { + vp, err := core.NewVersionedBlindedProposal(p.Blinded) + require.NoError(t, err) + assertRandaoBlindedBlock(t, randao, vp) + } +} + // blsSigFromHex returns the BLS signature from the input hex signature. func blsSigFromHex(t *testing.T, sig string) eth2p0.BLSSignature { t.Helper() diff --git a/core/signeddata.go b/core/signeddata.go index 1788f2372..9aa313bb3 100644 --- a/core/signeddata.go +++ b/core/signeddata.go @@ -519,6 +519,7 @@ func (p *VersionedSignedBlindedProposal) UnmarshalJSON(input []byte) error { type versionedRawBlockJSON struct { Version eth2util.DataVersion `json:"version"` Block json.RawMessage `json:"block"` + Blinded bool `json:"blinded,omitempty"` } // NewAttestation is a convenience function that returns a new wrapped attestation. diff --git a/core/testdata/TestJSONSerialisation_VersionedBlindedProposal.json.golden b/core/testdata/TestJSONSerialisation_VersionedBlindedProposal.json.golden index bb3f4c71d..418d4daa0 100644 --- a/core/testdata/TestJSONSerialisation_VersionedBlindedProposal.json.golden +++ b/core/testdata/TestJSONSerialisation_VersionedBlindedProposal.json.golden @@ -453,5 +453,6 @@ "0x18bd1830c7613fac2533e7634d9c3de68ba24d971c81ea87a0e1fe781890e6ae10ad72ec7f3e7b7c093e0e3c7d7f0c3f" ] } - } + }, + "blinded": true } \ No newline at end of file diff --git a/core/unsigneddata.go b/core/unsigneddata.go index 0e2d2c394..ee866cb2b 100644 --- a/core/unsigneddata.go +++ b/core/unsigneddata.go @@ -26,6 +26,7 @@ var ( _ UnsignedData = AggregatedAttestation{} _ UnsignedData = VersionedProposal{} _ UnsignedData = VersionedBlindedProposal{} + _ UnsignedData = VersionedUniversalProposal{} _ UnsignedData = SyncContribution{} // Some types also support SSZ marshalling and unmarshalling. @@ -334,6 +335,7 @@ func (p VersionedBlindedProposal) MarshalJSON() ([]byte, error) { resp, err := json.Marshal(versionedRawBlockJSON{ Version: version, Block: block, + Blinded: true, }) if err != nil { return nil, errors.Wrap(err, "marshal wrapper") @@ -377,6 +379,84 @@ func (p *VersionedBlindedProposal) UnmarshalJSON(input []byte) error { return nil } +// NewVersionedUniversalProposal validates and returns a new wrapped VersionedUniversalProposal. +func NewVersionedUniversalProposal(proposal *eth2api.VersionedUniversalProposal) (VersionedUniversalProposal, error) { + return VersionedUniversalProposal{VersionedUniversalProposal: *proposal}, nil +} + +// VersionedUniversalProposal wraps the eth2 versioned proposal and implements UnsignedData. +type VersionedUniversalProposal struct { + eth2api.VersionedUniversalProposal +} + +func (p VersionedUniversalProposal) Clone() (UnsignedData, error) { + var resp VersionedUniversalProposal + err := cloneJSONMarshaler(p, &resp) + if err != nil { + return nil, errors.Wrap(err, "clone universal proposal") + } + + return resp, nil +} + +func (p VersionedUniversalProposal) MarshalJSON() ([]byte, error) { + if p.Full != nil { + fp, err := NewVersionedProposal(p.Full) + if err != nil { + return nil, err + } + + return fp.MarshalJSON() + } + + if p.Blinded != nil { + bp, err := NewVersionedBlindedProposal(p.Blinded) + if err != nil { + return nil, err + } + return bp.MarshalJSON() + } + + return nil, errors.New("no full or blinded block") +} + +func (p *VersionedUniversalProposal) UnmarshalJSON(input []byte) error { + var raw versionedRawBlockJSON + if err := json.Unmarshal(input, &raw); err != nil { + return errors.Wrap(err, "unmarshal universal block") + } + + if raw.Blinded { + var bp VersionedBlindedProposal + if err := bp.UnmarshalJSON(input); err != nil { + return err + } + + p.Blinded = ð2api.VersionedBlindedProposal{ + Version: bp.Version, + Bellatrix: bp.Bellatrix, + Capella: bp.Capella, + Deneb: bp.Deneb, + } + } else { + var fp VersionedProposal + if err := fp.UnmarshalJSON(input); err != nil { + return err + } + + p.Full = ð2api.VersionedProposal{ + Version: fp.Version, + Phase0: fp.Phase0, + Altair: fp.Altair, + Bellatrix: fp.Bellatrix, + Capella: fp.Capella, + Deneb: fp.Deneb, + } + } + + return nil +} + // NewSyncContribution returns a new SyncContribution. func NewSyncContribution(c *altair.SyncCommitteeContribution) SyncContribution { return SyncContribution{SyncCommitteeContribution: *c} diff --git a/core/unsigneddata_test.go b/core/unsigneddata_test.go index 083a6e105..a5cb2f752 100644 --- a/core/unsigneddata_test.go +++ b/core/unsigneddata_test.go @@ -5,6 +5,7 @@ package core_test import ( "testing" + "github.com/attestantio/go-eth2-client/api" "github.com/stretchr/testify/require" "github.com/obolnetwork/charon/core" @@ -36,6 +37,14 @@ func TestUnsignedDataClone(t *testing.T) { name: "versioned blinded beacon block capella", data: testutil.RandomCapellaVersionedBlindedProposal(), }, + { + name: "versioned beacon block capella as universal proposal", + data: versionedProposalToUniversal(t, testutil.RandomCapellaCoreVersionedProposal()), + }, + { + name: "versioned blinded beacon block capella as universal proposal", + data: versionedBlindedProposalToUniversal(t, testutil.RandomCapellaVersionedBlindedProposal()), + }, { name: "aggregated attestation", data: core.NewAggregatedAttestation(testutil.RandomAttestation()), @@ -54,3 +63,37 @@ func TestUnsignedDataClone(t *testing.T) { }) } } + +func versionedProposalToUniversal(t *testing.T, p core.VersionedProposal) core.VersionedUniversalProposal { + t.Helper() + + up, err := core.NewVersionedUniversalProposal(&api.VersionedUniversalProposal{ + Full: &api.VersionedProposal{ + Version: p.Version, + Phase0: p.Phase0, + Altair: p.Altair, + Bellatrix: p.Bellatrix, + Capella: p.Capella, + Deneb: p.Deneb, + }, + }) + require.NoError(t, err) + + return up +} + +func versionedBlindedProposalToUniversal(t *testing.T, p core.VersionedBlindedProposal) core.VersionedUniversalProposal { + t.Helper() + + up, err := core.NewVersionedUniversalProposal(&api.VersionedUniversalProposal{ + Blinded: &api.VersionedBlindedProposal{ + Version: p.Version, + Bellatrix: p.Bellatrix, + Capella: p.Capella, + Deneb: p.Deneb, + }, + }) + require.NoError(t, err) + + return up +} diff --git a/testutil/beaconmock/beaconmock_fuzz.go b/testutil/beaconmock/beaconmock_fuzz.go index 5ce7396e4..263743d8b 100644 --- a/testutil/beaconmock/beaconmock_fuzz.go +++ b/testutil/beaconmock/beaconmock_fuzz.go @@ -136,6 +136,13 @@ func WithBeaconMockFuzzer() Option { return block, nil } + mock.UniversalProposalFunc = func(ctx context.Context, opts *eth2api.UniversalProposalOpts) (*eth2api.VersionedUniversalProposal, error) { + var block *eth2api.VersionedUniversalProposal + fuzz.New().Fuzz(&block) + + return block, nil + } + mock.NodePeerCountFunc = func(context.Context) (int, error) { var count int fuzz.New().Fuzz(&count) diff --git a/testutil/beaconmock/options.go b/testutil/beaconmock/options.go index 705e5d612..d697efad4 100644 --- a/testutil/beaconmock/options.go +++ b/testutil/beaconmock/options.go @@ -497,6 +497,19 @@ func defaultMock(httpMock HTTPMock, httpServer *http.Server, clock clockwork.Clo return block, nil }, + UniversalProposalFunc: func(ctx context.Context, opts *eth2api.UniversalProposalOpts) (*eth2api.VersionedUniversalProposal, error) { + block := ð2api.VersionedUniversalProposal{ + Full: ð2api.VersionedProposal{ + Version: eth2spec.DataVersionCapella, + Capella: testutil.RandomCapellaBeaconBlock(), + }, + } + block.Full.Capella.Slot = opts.Slot + block.Full.Capella.Body.RANDAOReveal = opts.RandaoReveal + block.Full.Capella.Body.Graffiti = opts.Graffiti + + return block, nil + }, SignedBeaconBlockFunc: func(_ context.Context, blockID string) (*eth2spec.VersionedSignedBeaconBlock, error) { return testutil.RandomCapellaVersionedSignedBeaconBlock(), nil // Note the slot is probably wrong. }, From 8a96bac78f1804c19abf3ca2f57122846b3b6a09 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Wed, 28 Feb 2024 18:47:01 +0300 Subject: [PATCH 04/13] Bumped go-eth2-client with better field names --- core/fetcher/fetcher.go | 8 ++++---- core/fetcher/fetcher_test.go | 12 ++++++------ core/unsigneddata.go | 12 ++++++------ core/unsigneddata_test.go | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- testutil/beaconmock/options.go | 8 ++++---- 7 files changed, 25 insertions(+), 25 deletions(-) diff --git a/core/fetcher/fetcher.go b/core/fetcher/fetcher.go index 517cf3008..c1ff950dd 100644 --- a/core/fetcher/fetcher.go +++ b/core/fetcher/fetcher.go @@ -345,10 +345,10 @@ func (f *Fetcher) fetchUniversalProposerData(ctx context.Context, slot uint64, d proposal := eth2Resp.Data // Ensure fee recipient is correctly populated in proposal. - if proposal.Full != nil { - verifyFeeRecipient(ctx, proposal.Full, f.feeRecipientFunc(pubkey)) - } else if proposal.Blinded != nil { - verifyFeeRecipientBlinded(ctx, proposal.Blinded, f.feeRecipientFunc(pubkey)) + if proposal.Proposal != nil { + verifyFeeRecipient(ctx, proposal.Proposal, f.feeRecipientFunc(pubkey)) + } else if proposal.BlindedProposal != nil { + verifyFeeRecipientBlinded(ctx, proposal.BlindedProposal, f.feeRecipientFunc(pubkey)) } coreProposal, err := core.NewVersionedUniversalProposal(proposal) diff --git a/core/fetcher/fetcher_test.go b/core/fetcher/fetcher_test.go index db1130541..1614c4e56 100644 --- a/core/fetcher/fetcher_test.go +++ b/core/fetcher/fetcher_test.go @@ -365,14 +365,14 @@ func TestFetchBlocks(t *testing.T) { slotA, err := dutyDataA.Slot() require.NoError(t, err) require.EqualValues(t, slot, slotA) - require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataA.Full.Capella.Body.ExecutionPayload.FeeRecipient)) + require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataA.Proposal.Capella.Body.ExecutionPayload.FeeRecipient)) assertRandaoUniversalProposal(t, randaoByPubKey[pubkeysByIdx[vIdxA]].Signature().ToETH2(), dutyDataA) dutyDataB := resDataSet[pubkeysByIdx[vIdxB]].(core.VersionedUniversalProposal) slotB, err := dutyDataB.Slot() require.NoError(t, err) require.EqualValues(t, slot, slotB) - require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataB.Full.Capella.Body.ExecutionPayload.FeeRecipient)) + require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataB.Proposal.Capella.Body.ExecutionPayload.FeeRecipient)) assertRandaoUniversalProposal(t, randaoByPubKey[pubkeysByIdx[vIdxB]].Signature().ToETH2(), dutyDataB) return nil @@ -627,13 +627,13 @@ func assertRandaoBlindedBlock(t *testing.T, randao eth2p0.BLSSignature, block co func assertRandaoUniversalProposal(t *testing.T, randao eth2p0.BLSSignature, p core.VersionedUniversalProposal) { t.Helper() - if p.Full != nil { - vp, err := core.NewVersionedProposal(p.Full) + if p.Proposal != nil { + vp, err := core.NewVersionedProposal(p.Proposal) require.NoError(t, err) assertRandao(t, randao, vp) } - if p.Blinded != nil { - vp, err := core.NewVersionedBlindedProposal(p.Blinded) + if p.BlindedProposal != nil { + vp, err := core.NewVersionedBlindedProposal(p.BlindedProposal) require.NoError(t, err) assertRandaoBlindedBlock(t, randao, vp) } diff --git a/core/unsigneddata.go b/core/unsigneddata.go index ee866cb2b..53dee80ff 100644 --- a/core/unsigneddata.go +++ b/core/unsigneddata.go @@ -400,8 +400,8 @@ func (p VersionedUniversalProposal) Clone() (UnsignedData, error) { } func (p VersionedUniversalProposal) MarshalJSON() ([]byte, error) { - if p.Full != nil { - fp, err := NewVersionedProposal(p.Full) + if p.Proposal != nil { + fp, err := NewVersionedProposal(p.Proposal) if err != nil { return nil, err } @@ -409,8 +409,8 @@ func (p VersionedUniversalProposal) MarshalJSON() ([]byte, error) { return fp.MarshalJSON() } - if p.Blinded != nil { - bp, err := NewVersionedBlindedProposal(p.Blinded) + if p.BlindedProposal != nil { + bp, err := NewVersionedBlindedProposal(p.BlindedProposal) if err != nil { return nil, err } @@ -432,7 +432,7 @@ func (p *VersionedUniversalProposal) UnmarshalJSON(input []byte) error { return err } - p.Blinded = ð2api.VersionedBlindedProposal{ + p.BlindedProposal = ð2api.VersionedBlindedProposal{ Version: bp.Version, Bellatrix: bp.Bellatrix, Capella: bp.Capella, @@ -444,7 +444,7 @@ func (p *VersionedUniversalProposal) UnmarshalJSON(input []byte) error { return err } - p.Full = ð2api.VersionedProposal{ + p.Proposal = ð2api.VersionedProposal{ Version: fp.Version, Phase0: fp.Phase0, Altair: fp.Altair, diff --git a/core/unsigneddata_test.go b/core/unsigneddata_test.go index a5cb2f752..8458807e5 100644 --- a/core/unsigneddata_test.go +++ b/core/unsigneddata_test.go @@ -68,7 +68,7 @@ func versionedProposalToUniversal(t *testing.T, p core.VersionedProposal) core.V t.Helper() up, err := core.NewVersionedUniversalProposal(&api.VersionedUniversalProposal{ - Full: &api.VersionedProposal{ + Proposal: &api.VersionedProposal{ Version: p.Version, Phase0: p.Phase0, Altair: p.Altair, @@ -86,7 +86,7 @@ func versionedBlindedProposalToUniversal(t *testing.T, p core.VersionedBlindedPr t.Helper() up, err := core.NewVersionedUniversalProposal(&api.VersionedUniversalProposal{ - Blinded: &api.VersionedBlindedProposal{ + BlindedProposal: &api.VersionedBlindedProposal{ Version: p.Version, Bellatrix: p.Bellatrix, Capella: p.Capella, diff --git a/go.mod b/go.mod index 9d607fd27..2f671439e 100644 --- a/go.mod +++ b/go.mod @@ -203,4 +203,4 @@ require ( replace github.com/coinbase/kryptology => github.com/ObolNetwork/kryptology v0.0.0-20231016091344-eed023b6cac8 // We're replacing go-eth2-client with our own fork, which adds UniversalProposel model for produceBlockV3 endpoint. -replace github.com/attestantio/go-eth2-client v0.19.10 => github.com/obolNetwork/go-eth2-client v0.0.0-20240228100742-8eb0c82f8153 +replace github.com/attestantio/go-eth2-client v0.19.10 => github.com/obolNetwork/go-eth2-client v0.0.0-20240228154246-66d3d0ccd2e1 diff --git a/go.sum b/go.sum index 17c72dc1a..d5d7e99a5 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/n github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOELpZAu9eioSos/OU= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20151028013722-8c68805598ab/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= -github.com/obolNetwork/go-eth2-client v0.0.0-20240228100742-8eb0c82f8153 h1:KtEnfQIcXHxZGQNqWl1FjkAbWG0Juq0iFVbicR/tmXQ= -github.com/obolNetwork/go-eth2-client v0.0.0-20240228100742-8eb0c82f8153/go.mod h1:TTz7YF6w4z6ahvxKiHuGPn6DbQn7gH6HPuWm/DEQeGE= +github.com/obolNetwork/go-eth2-client v0.0.0-20240228154246-66d3d0ccd2e1 h1:2mSMLPu6raLaYeUCX0M5Rho94G/835Jo6L66VjrA0jI= +github.com/obolNetwork/go-eth2-client v0.0.0-20240228154246-66d3d0ccd2e1/go.mod h1:TTz7YF6w4z6ahvxKiHuGPn6DbQn7gH6HPuWm/DEQeGE= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= diff --git a/testutil/beaconmock/options.go b/testutil/beaconmock/options.go index d697efad4..4d74d7d9d 100644 --- a/testutil/beaconmock/options.go +++ b/testutil/beaconmock/options.go @@ -499,14 +499,14 @@ func defaultMock(httpMock HTTPMock, httpServer *http.Server, clock clockwork.Clo }, UniversalProposalFunc: func(ctx context.Context, opts *eth2api.UniversalProposalOpts) (*eth2api.VersionedUniversalProposal, error) { block := ð2api.VersionedUniversalProposal{ - Full: ð2api.VersionedProposal{ + Proposal: ð2api.VersionedProposal{ Version: eth2spec.DataVersionCapella, Capella: testutil.RandomCapellaBeaconBlock(), }, } - block.Full.Capella.Slot = opts.Slot - block.Full.Capella.Body.RANDAOReveal = opts.RandaoReveal - block.Full.Capella.Body.Graffiti = opts.Graffiti + block.Proposal.Capella.Slot = opts.Slot + block.Proposal.Capella.Body.RANDAOReveal = opts.RandaoReveal + block.Proposal.Capella.Body.Graffiti = opts.Graffiti return block, nil }, From 80151eeffabe0ab7fe19091d559954949edec147 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Wed, 28 Feb 2024 19:12:38 +0300 Subject: [PATCH 05/13] Added UniversalProposerDefinition --- core/dutydefinition.go | 40 +++++++++++++++++++++++++++++++ core/fetcher/fetcher.go | 14 +++++++---- core/fetcher/fetcher_test.go | 46 ++++++++++++++++++++++++++++++------ 3 files changed, 89 insertions(+), 11 deletions(-) diff --git a/core/dutydefinition.go b/core/dutydefinition.go index 8c4a247fb..0d029fa66 100644 --- a/core/dutydefinition.go +++ b/core/dutydefinition.go @@ -3,6 +3,8 @@ package core import ( + "encoding/json" + eth2v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/obolnetwork/charon/app/errors" @@ -11,6 +13,7 @@ import ( var ( _ DutyDefinition = AttesterDefinition{} _ DutyDefinition = ProposerDefinition{} + _ DutyDefinition = UniversalProposerDefinition{} _ DutyDefinition = SyncCommitteeDefinition{} ) @@ -64,6 +67,43 @@ func (d ProposerDefinition) MarshalJSON() ([]byte, error) { return d.ProposerDuty.MarshalJSON() } +// NewUniversalProposerDefinition is a convenience function that returns a new universal proposer definition. +func NewUniversalProposerDefinition(duty *eth2v1.ProposerDuty, builderBoostFactor string) UniversalProposerDefinition { + return UniversalProposerDefinition{ + ProposerDuty: *duty, + BuilderBoostFactor: builderBoostFactor, + } +} + +// UniversalProposerDefinition defines a universal proposer duty. It implements DutyDefinition. +type UniversalProposerDefinition struct { + eth2v1.ProposerDuty + BuilderBoostFactor string +} + +type rawUniversalProposerDefinition struct { + Duty eth2v1.ProposerDuty `json:"duty"` + BuilderBoostFactor string `json:"builder_boost_factor"` +} + +func (d UniversalProposerDefinition) Clone() (DutyDefinition, error) { + duty := new(eth2v1.ProposerDuty) + err := cloneJSONMarshaler(&d.ProposerDuty, duty) + if err != nil { + return nil, errors.Wrap(err, "clone proposer definition") + } + + return NewUniversalProposerDefinition(duty, d.BuilderBoostFactor), nil +} + +func (d UniversalProposerDefinition) MarshalJSON() ([]byte, error) { + raw := rawUniversalProposerDefinition{ + Duty: d.ProposerDuty, + BuilderBoostFactor: d.BuilderBoostFactor, + } + return json.Marshal(raw) +} + // NewSyncCommitteeDefinition is a convenience function that returns a new SyncCommitteeDefinition. func NewSyncCommitteeDefinition(duty *eth2v1.SyncCommitteeDuty) DutyDefinition { return SyncCommitteeDefinition{SyncCommitteeDuty: *duty} diff --git a/core/fetcher/fetcher.go b/core/fetcher/fetcher.go index c1ff950dd..7353b02f4 100644 --- a/core/fetcher/fetcher.go +++ b/core/fetcher/fetcher.go @@ -327,16 +327,22 @@ func (f *Fetcher) fetchBuilderProposerData(ctx context.Context, slot uint64, def func (f *Fetcher) fetchUniversalProposerData(ctx context.Context, slot uint64, defSet core.DutyDefinitionSet) (core.UnsignedDataSet, error) { resp := make(core.UnsignedDataSet) - for pubkey := range defSet { + for pubkey, dutyDef := range defSet { + upDef, ok := dutyDef.(core.UniversalProposerDefinition) + if !ok { + return core.UnsignedDataSet{}, errors.New("invalid universal proposer definition") + } + randao, graffiti, err := f.getProposalOpts(ctx, slot, pubkey) if err != nil { return nil, err } opts := ð2api.UniversalProposalOpts{ - Slot: eth2p0.Slot(slot), - RandaoReveal: randao, - Graffiti: graffiti, + Slot: eth2p0.Slot(slot), + RandaoReveal: randao, + Graffiti: graffiti, + BuilderBoostFactor: upDef.BuilderBoostFactor, } eth2Resp, err := f.eth2Cl.UniversalProposal(ctx, opts) if err != nil { diff --git a/core/fetcher/fetcher_test.go b/core/fetcher/fetcher_test.go index 1614c4e56..42dbe5d18 100644 --- a/core/fetcher/fetcher_test.go +++ b/core/fetcher/fetcher_test.go @@ -345,6 +345,45 @@ func TestFetchBlocks(t *testing.T) { err = fetch.Fetch(ctx, duty, defSet) require.NoError(t, err) }) +} + +func TestFetchUniversalBlocks(t *testing.T) { + ctx := context.Background() + + const ( + slot = 1 + vIdxA = 2 + vIdxB = 3 + feeRecipientAddr = "0x0000000000000000000000000000000000000000" + ) + + pubkeysByIdx := map[eth2p0.ValidatorIndex]core.PubKey{ + vIdxA: testutil.RandomCorePubKey(t), + vIdxB: testutil.RandomCorePubKey(t), + } + + dutyA := eth2v1.ProposerDuty{ + Slot: slot, + ValidatorIndex: vIdxA, + } + dutyB := eth2v1.ProposerDuty{ + Slot: slot, + ValidatorIndex: vIdxB, + } + defSet := core.DutyDefinitionSet{ + pubkeysByIdx[vIdxA]: core.NewUniversalProposerDefinition(&dutyA, "100"), + pubkeysByIdx[vIdxB]: core.NewUniversalProposerDefinition(&dutyB, "200"), + } + + randaoA := testutil.RandomCoreSignature() + randaoB := testutil.RandomCoreSignature() + randaoByPubKey := map[core.PubKey]core.SignedData{ + pubkeysByIdx[vIdxA]: randaoA, + pubkeysByIdx[vIdxB]: randaoB, + } + + bmock, err := beaconmock.New() + require.NoError(t, err) t.Run("fetch DutyUniversalProposer", func(t *testing.T) { duty := core.NewUniversalProposerDuty(slot) @@ -368,13 +407,6 @@ func TestFetchBlocks(t *testing.T) { require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataA.Proposal.Capella.Body.ExecutionPayload.FeeRecipient)) assertRandaoUniversalProposal(t, randaoByPubKey[pubkeysByIdx[vIdxA]].Signature().ToETH2(), dutyDataA) - dutyDataB := resDataSet[pubkeysByIdx[vIdxB]].(core.VersionedUniversalProposal) - slotB, err := dutyDataB.Slot() - require.NoError(t, err) - require.EqualValues(t, slot, slotB) - require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataB.Proposal.Capella.Body.ExecutionPayload.FeeRecipient)) - assertRandaoUniversalProposal(t, randaoByPubKey[pubkeysByIdx[vIdxB]].Signature().ToETH2(), dutyDataB) - return nil }) From cb2a37ec737af1eb7afd8f9b7ccf83b60f5f1a4e Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Wed, 28 Feb 2024 19:19:31 +0300 Subject: [PATCH 06/13] Fixed linter issues --- core/dutydefinition.go | 1 + core/unsigneddata.go | 1 + core/unsigneddata_test.go | 10 +++++----- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/core/dutydefinition.go b/core/dutydefinition.go index 0d029fa66..11606f143 100644 --- a/core/dutydefinition.go +++ b/core/dutydefinition.go @@ -101,6 +101,7 @@ func (d UniversalProposerDefinition) MarshalJSON() ([]byte, error) { Duty: d.ProposerDuty, BuilderBoostFactor: d.BuilderBoostFactor, } + return json.Marshal(raw) } diff --git a/core/unsigneddata.go b/core/unsigneddata.go index 53dee80ff..d3b7cc999 100644 --- a/core/unsigneddata.go +++ b/core/unsigneddata.go @@ -414,6 +414,7 @@ func (p VersionedUniversalProposal) MarshalJSON() ([]byte, error) { if err != nil { return nil, err } + return bp.MarshalJSON() } diff --git a/core/unsigneddata_test.go b/core/unsigneddata_test.go index 8458807e5..70212f41b 100644 --- a/core/unsigneddata_test.go +++ b/core/unsigneddata_test.go @@ -5,7 +5,7 @@ package core_test import ( "testing" - "github.com/attestantio/go-eth2-client/api" + eth2api "github.com/attestantio/go-eth2-client/api" "github.com/stretchr/testify/require" "github.com/obolnetwork/charon/core" @@ -67,8 +67,8 @@ func TestUnsignedDataClone(t *testing.T) { func versionedProposalToUniversal(t *testing.T, p core.VersionedProposal) core.VersionedUniversalProposal { t.Helper() - up, err := core.NewVersionedUniversalProposal(&api.VersionedUniversalProposal{ - Proposal: &api.VersionedProposal{ + up, err := core.NewVersionedUniversalProposal(ð2api.VersionedUniversalProposal{ + Proposal: ð2api.VersionedProposal{ Version: p.Version, Phase0: p.Phase0, Altair: p.Altair, @@ -85,8 +85,8 @@ func versionedProposalToUniversal(t *testing.T, p core.VersionedProposal) core.V func versionedBlindedProposalToUniversal(t *testing.T, p core.VersionedBlindedProposal) core.VersionedUniversalProposal { t.Helper() - up, err := core.NewVersionedUniversalProposal(&api.VersionedUniversalProposal{ - BlindedProposal: &api.VersionedBlindedProposal{ + up, err := core.NewVersionedUniversalProposal(ð2api.VersionedUniversalProposal{ + BlindedProposal: ð2api.VersionedBlindedProposal{ Version: p.Version, Bellatrix: p.Bellatrix, Capella: p.Capella, From 19ca9a1529c236519c84e08d8e878ef450656c65 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Wed, 28 Feb 2024 19:23:03 +0300 Subject: [PATCH 07/13] Fixed linter issues --- core/dutydefinition.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/dutydefinition.go b/core/dutydefinition.go index 11606f143..6c09f8f2e 100644 --- a/core/dutydefinition.go +++ b/core/dutydefinition.go @@ -102,7 +102,12 @@ func (d UniversalProposerDefinition) MarshalJSON() ([]byte, error) { BuilderBoostFactor: d.BuilderBoostFactor, } - return json.Marshal(raw) + bytes, err := json.Marshal(raw) + if err != nil { + return nil, errors.Wrap(err, "marshal universal proposer definition") + } + + return bytes, nil } // NewSyncCommitteeDefinition is a convenience function that returns a new SyncCommitteeDefinition. From 0343f57cffced7ccbd553a737dec76d2de1911f2 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Thu, 29 Feb 2024 09:06:22 +0300 Subject: [PATCH 08/13] Some simplification --- core/unsigneddata.go | 18 ++---------------- core/unsigneddata_test.go | 30 ++++++++---------------------- 2 files changed, 10 insertions(+), 38 deletions(-) diff --git a/core/unsigneddata.go b/core/unsigneddata.go index d3b7cc999..c31f11f6c 100644 --- a/core/unsigneddata.go +++ b/core/unsigneddata.go @@ -432,27 +432,13 @@ func (p *VersionedUniversalProposal) UnmarshalJSON(input []byte) error { if err := bp.UnmarshalJSON(input); err != nil { return err } - - p.BlindedProposal = ð2api.VersionedBlindedProposal{ - Version: bp.Version, - Bellatrix: bp.Bellatrix, - Capella: bp.Capella, - Deneb: bp.Deneb, - } + p.BlindedProposal = &bp.VersionedBlindedProposal } else { var fp VersionedProposal if err := fp.UnmarshalJSON(input); err != nil { return err } - - p.Proposal = ð2api.VersionedProposal{ - Version: fp.Version, - Phase0: fp.Phase0, - Altair: fp.Altair, - Bellatrix: fp.Bellatrix, - Capella: fp.Capella, - Deneb: fp.Deneb, - } + p.Proposal = &fp.VersionedProposal } return nil diff --git a/core/unsigneddata_test.go b/core/unsigneddata_test.go index 70212f41b..b836bc1ab 100644 --- a/core/unsigneddata_test.go +++ b/core/unsigneddata_test.go @@ -67,33 +67,19 @@ func TestUnsignedDataClone(t *testing.T) { func versionedProposalToUniversal(t *testing.T, p core.VersionedProposal) core.VersionedUniversalProposal { t.Helper() - up, err := core.NewVersionedUniversalProposal(ð2api.VersionedUniversalProposal{ - Proposal: ð2api.VersionedProposal{ - Version: p.Version, - Phase0: p.Phase0, - Altair: p.Altair, - Bellatrix: p.Bellatrix, - Capella: p.Capella, - Deneb: p.Deneb, + return core.VersionedUniversalProposal{ + VersionedUniversalProposal: eth2api.VersionedUniversalProposal{ + Proposal: &p.VersionedProposal, }, - }) - require.NoError(t, err) - - return up + } } func versionedBlindedProposalToUniversal(t *testing.T, p core.VersionedBlindedProposal) core.VersionedUniversalProposal { t.Helper() - up, err := core.NewVersionedUniversalProposal(ð2api.VersionedUniversalProposal{ - BlindedProposal: ð2api.VersionedBlindedProposal{ - Version: p.Version, - Bellatrix: p.Bellatrix, - Capella: p.Capella, - Deneb: p.Deneb, + return core.VersionedUniversalProposal{ + VersionedUniversalProposal: eth2api.VersionedUniversalProposal{ + BlindedProposal: &p.VersionedBlindedProposal, }, - }) - require.NoError(t, err) - - return up + } } From a60b445f445eba1cabbb85e35a9d09eaeee6f9dc Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Thu, 29 Feb 2024 09:23:32 +0300 Subject: [PATCH 09/13] Removed odd code and added tests --- core/dutydefinition.go | 46 ------------------------ core/fetcher/fetcher.go | 9 ++--- core/fetcher/fetcher_test.go | 68 ------------------------------------ core/types_test.go | 7 ++++ 4 files changed, 9 insertions(+), 121 deletions(-) diff --git a/core/dutydefinition.go b/core/dutydefinition.go index 6c09f8f2e..8c4a247fb 100644 --- a/core/dutydefinition.go +++ b/core/dutydefinition.go @@ -3,8 +3,6 @@ package core import ( - "encoding/json" - eth2v1 "github.com/attestantio/go-eth2-client/api/v1" "github.com/obolnetwork/charon/app/errors" @@ -13,7 +11,6 @@ import ( var ( _ DutyDefinition = AttesterDefinition{} _ DutyDefinition = ProposerDefinition{} - _ DutyDefinition = UniversalProposerDefinition{} _ DutyDefinition = SyncCommitteeDefinition{} ) @@ -67,49 +64,6 @@ func (d ProposerDefinition) MarshalJSON() ([]byte, error) { return d.ProposerDuty.MarshalJSON() } -// NewUniversalProposerDefinition is a convenience function that returns a new universal proposer definition. -func NewUniversalProposerDefinition(duty *eth2v1.ProposerDuty, builderBoostFactor string) UniversalProposerDefinition { - return UniversalProposerDefinition{ - ProposerDuty: *duty, - BuilderBoostFactor: builderBoostFactor, - } -} - -// UniversalProposerDefinition defines a universal proposer duty. It implements DutyDefinition. -type UniversalProposerDefinition struct { - eth2v1.ProposerDuty - BuilderBoostFactor string -} - -type rawUniversalProposerDefinition struct { - Duty eth2v1.ProposerDuty `json:"duty"` - BuilderBoostFactor string `json:"builder_boost_factor"` -} - -func (d UniversalProposerDefinition) Clone() (DutyDefinition, error) { - duty := new(eth2v1.ProposerDuty) - err := cloneJSONMarshaler(&d.ProposerDuty, duty) - if err != nil { - return nil, errors.Wrap(err, "clone proposer definition") - } - - return NewUniversalProposerDefinition(duty, d.BuilderBoostFactor), nil -} - -func (d UniversalProposerDefinition) MarshalJSON() ([]byte, error) { - raw := rawUniversalProposerDefinition{ - Duty: d.ProposerDuty, - BuilderBoostFactor: d.BuilderBoostFactor, - } - - bytes, err := json.Marshal(raw) - if err != nil { - return nil, errors.Wrap(err, "marshal universal proposer definition") - } - - return bytes, nil -} - // NewSyncCommitteeDefinition is a convenience function that returns a new SyncCommitteeDefinition. func NewSyncCommitteeDefinition(duty *eth2v1.SyncCommitteeDuty) DutyDefinition { return SyncCommitteeDefinition{SyncCommitteeDuty: *duty} diff --git a/core/fetcher/fetcher.go b/core/fetcher/fetcher.go index 7353b02f4..3d223f966 100644 --- a/core/fetcher/fetcher.go +++ b/core/fetcher/fetcher.go @@ -327,12 +327,7 @@ func (f *Fetcher) fetchBuilderProposerData(ctx context.Context, slot uint64, def func (f *Fetcher) fetchUniversalProposerData(ctx context.Context, slot uint64, defSet core.DutyDefinitionSet) (core.UnsignedDataSet, error) { resp := make(core.UnsignedDataSet) - for pubkey, dutyDef := range defSet { - upDef, ok := dutyDef.(core.UniversalProposerDefinition) - if !ok { - return core.UnsignedDataSet{}, errors.New("invalid universal proposer definition") - } - + for pubkey := range defSet { randao, graffiti, err := f.getProposalOpts(ctx, slot, pubkey) if err != nil { return nil, err @@ -342,7 +337,7 @@ func (f *Fetcher) fetchUniversalProposerData(ctx context.Context, slot uint64, d Slot: eth2p0.Slot(slot), RandaoReveal: randao, Graffiti: graffiti, - BuilderBoostFactor: upDef.BuilderBoostFactor, + BuilderBoostFactor: "", // TODO: how to populate this? } eth2Resp, err := f.eth2Cl.UniversalProposal(ctx, opts) if err != nil { diff --git a/core/fetcher/fetcher_test.go b/core/fetcher/fetcher_test.go index 42dbe5d18..762d7499a 100644 --- a/core/fetcher/fetcher_test.go +++ b/core/fetcher/fetcher_test.go @@ -347,74 +347,6 @@ func TestFetchBlocks(t *testing.T) { }) } -func TestFetchUniversalBlocks(t *testing.T) { - ctx := context.Background() - - const ( - slot = 1 - vIdxA = 2 - vIdxB = 3 - feeRecipientAddr = "0x0000000000000000000000000000000000000000" - ) - - pubkeysByIdx := map[eth2p0.ValidatorIndex]core.PubKey{ - vIdxA: testutil.RandomCorePubKey(t), - vIdxB: testutil.RandomCorePubKey(t), - } - - dutyA := eth2v1.ProposerDuty{ - Slot: slot, - ValidatorIndex: vIdxA, - } - dutyB := eth2v1.ProposerDuty{ - Slot: slot, - ValidatorIndex: vIdxB, - } - defSet := core.DutyDefinitionSet{ - pubkeysByIdx[vIdxA]: core.NewUniversalProposerDefinition(&dutyA, "100"), - pubkeysByIdx[vIdxB]: core.NewUniversalProposerDefinition(&dutyB, "200"), - } - - randaoA := testutil.RandomCoreSignature() - randaoB := testutil.RandomCoreSignature() - randaoByPubKey := map[core.PubKey]core.SignedData{ - pubkeysByIdx[vIdxA]: randaoA, - pubkeysByIdx[vIdxB]: randaoB, - } - - bmock, err := beaconmock.New() - require.NoError(t, err) - - t.Run("fetch DutyUniversalProposer", func(t *testing.T) { - duty := core.NewUniversalProposerDuty(slot) - fetch, err := fetcher.New(bmock, func(core.PubKey) string { - return feeRecipientAddr - }) - require.NoError(t, err) - - fetch.RegisterAggSigDB(func(ctx context.Context, duty core.Duty, key core.PubKey) (core.SignedData, error) { - return randaoByPubKey[key], nil - }) - - fetch.Subscribe(func(ctx context.Context, resDuty core.Duty, resDataSet core.UnsignedDataSet) error { - require.Equal(t, duty, resDuty) - require.Len(t, resDataSet, 2) - - dutyDataA := resDataSet[pubkeysByIdx[vIdxA]].(core.VersionedUniversalProposal) - slotA, err := dutyDataA.Slot() - require.NoError(t, err) - require.EqualValues(t, slot, slotA) - require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataA.Proposal.Capella.Body.ExecutionPayload.FeeRecipient)) - assertRandaoUniversalProposal(t, randaoByPubKey[pubkeysByIdx[vIdxA]].Signature().ToETH2(), dutyDataA) - - return nil - }) - - err = fetch.Fetch(ctx, duty, defSet) - require.NoError(t, err) - }) -} - func TestFetchSyncContribution(t *testing.T) { ctx := context.Background() diff --git a/core/types_test.go b/core/types_test.go index 0d14e0cbe..ff8ab1a12 100644 --- a/core/types_test.go +++ b/core/types_test.go @@ -67,3 +67,10 @@ func TestWithDutySpanCtx(t *testing.T) { require.True(t, span2.SpanContext().IsValid()) require.True(t, span2.SpanContext().IsSampled()) } + +func TestNewUniversalProposerDuty(t *testing.T) { + duty := core.NewUniversalProposerDuty(1) + + require.Equal(t, core.DutyUniversalProposer, duty.Type) + require.Equal(t, uint64(1), duty.Slot) +} From d3db4e04479191edc98ac78852b5278b519fc186 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Thu, 29 Feb 2024 09:28:11 +0300 Subject: [PATCH 10/13] Removed odd code --- core/fetcher/fetcher_test.go | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/core/fetcher/fetcher_test.go b/core/fetcher/fetcher_test.go index 762d7499a..f3645e928 100644 --- a/core/fetcher/fetcher_test.go +++ b/core/fetcher/fetcher_test.go @@ -588,21 +588,6 @@ func assertRandaoBlindedBlock(t *testing.T, randao eth2p0.BLSSignature, block co } } -func assertRandaoUniversalProposal(t *testing.T, randao eth2p0.BLSSignature, p core.VersionedUniversalProposal) { - t.Helper() - - if p.Proposal != nil { - vp, err := core.NewVersionedProposal(p.Proposal) - require.NoError(t, err) - assertRandao(t, randao, vp) - } - if p.BlindedProposal != nil { - vp, err := core.NewVersionedBlindedProposal(p.BlindedProposal) - require.NoError(t, err) - assertRandaoBlindedBlock(t, randao, vp) - } -} - // blsSigFromHex returns the BLS signature from the input hex signature. func blsSigFromHex(t *testing.T, sig string) eth2p0.BLSSignature { t.Helper() From 056a8c7a42de9161ca1aecf8a133b82015a3e9b9 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Thu, 29 Feb 2024 09:45:17 +0300 Subject: [PATCH 11/13] DutyDB supporting UniversalProposal --- core/dutydb/memory.go | 133 ++++++++++++++++++++++++--- core/dutydb/memory_internal_test.go | 5 + core/dutydb/memory_test.go | 136 ++++++++++++++++++++++++++++ 3 files changed, 263 insertions(+), 11 deletions(-) diff --git a/core/dutydb/memory.go b/core/dutydb/memory.go index eff6d0bcb..39f22c05d 100644 --- a/core/dutydb/memory.go +++ b/core/dutydb/memory.go @@ -18,17 +18,18 @@ import ( // NewMemDB returns a new in-memory dutyDB instance. func NewMemDB(deadliner core.Deadliner) *MemDB { return &MemDB{ - attDuties: make(map[attKey]*eth2p0.AttestationData), - attPubKeys: make(map[pkKey]core.PubKey), - attKeysBySlot: make(map[uint64][]pkKey), - builderProDuties: make(map[uint64]*eth2api.VersionedBlindedProposal), - proDuties: make(map[uint64]*eth2api.VersionedProposal), - aggDuties: make(map[aggKey]core.AggregatedAttestation), - aggKeysBySlot: make(map[uint64][]aggKey), - contribDuties: make(map[contribKey]*altair.SyncCommitteeContribution), - contribKeysBySlot: make(map[uint64][]contribKey), - shutdown: make(chan struct{}), - deadliner: deadliner, + attDuties: make(map[attKey]*eth2p0.AttestationData), + attPubKeys: make(map[pkKey]core.PubKey), + attKeysBySlot: make(map[uint64][]pkKey), + builderProDuties: make(map[uint64]*eth2api.VersionedBlindedProposal), + proDuties: make(map[uint64]*eth2api.VersionedProposal), + universalProDuties: make(map[uint64]*eth2api.VersionedUniversalProposal), + aggDuties: make(map[aggKey]core.AggregatedAttestation), + aggKeysBySlot: make(map[uint64][]aggKey), + contribDuties: make(map[contribKey]*altair.SyncCommitteeContribution), + contribKeysBySlot: make(map[uint64][]contribKey), + shutdown: make(chan struct{}), + deadliner: deadliner, } } @@ -51,6 +52,10 @@ type MemDB struct { proDuties map[uint64]*eth2api.VersionedProposal proQueries []proQuery + // DutyUniversalProposer + universalProDuties map[uint64]*eth2api.VersionedUniversalProposal + universalProQueries []universalProQuery + // DutyAggregator aggDuties map[aggKey]core.AggregatedAttestation aggKeysBySlot map[uint64][]aggKey @@ -105,6 +110,18 @@ func (db *MemDB) Store(_ context.Context, duty core.Duty, unsignedSet core.Unsig } } db.resolveBuilderProQueriesUnsafe() + case core.DutyUniversalProposer: + // Sanity check max one universal proposer per slot + if len(unsignedSet) > 1 { + return errors.New("unexpected universal proposer data set length", z.Int("n", len(unsignedSet))) + } + for _, unsignedData := range unsignedSet { + err := db.storeUniversalProposalUnsafe(unsignedData) + if err != nil { + return err + } + } + db.resolveUniversalProQueriesUnsafe() case core.DutyAttester: for pubkey, unsignedData := range unsignedSet { err := db.storeAttestationUnsafe(pubkey, unsignedData) @@ -204,6 +221,31 @@ func (db *MemDB) AwaitBlindedProposal(ctx context.Context, slot uint64) (*eth2ap } } +// AwaitUniversalProposal implements core.DutyDB, see its godoc. +func (db *MemDB) AwaitUniversalProposal(ctx context.Context, slot uint64) (*eth2api.VersionedUniversalProposal, error) { + cancel := make(chan struct{}) + defer close(cancel) + response := make(chan *eth2api.VersionedUniversalProposal, 1) + + db.mu.Lock() + db.universalProQueries = append(db.universalProQueries, universalProQuery{ + Key: slot, + Response: response, + Cancel: cancel, + }) + db.resolveUniversalProQueriesUnsafe() + db.mu.Unlock() + + select { + case <-db.shutdown: + return nil, errors.New("dutydb shutdown") + case <-ctx.Done(): + return nil, ctx.Err() + case block := <-response: + return block, nil + } +} + // AwaitAttestation implements core.DutyDB, see its godoc. func (db *MemDB) AwaitAttestation(ctx context.Context, slot uint64, commIdx uint64) (*eth2p0.AttestationData, error) { cancel := make(chan struct{}) @@ -489,6 +531,45 @@ func (db *MemDB) storeProposalUnsafe(unsignedData core.UnsignedData) error { return nil } +// storeUniversalProposalUnsafe stores the unsigned UniversalProposal. +// It is unsafe since it assumes the lock is held. +func (db *MemDB) storeUniversalProposalUnsafe(unsignedData core.UnsignedData) error { + cloned, err := unsignedData.Clone() // Clone before storing. + if err != nil { + return err + } + + proposal, ok := cloned.(core.VersionedUniversalProposal) + if !ok { + return errors.New("invalid versioned universal proposal") + } + + slot, err := proposal.Slot() + if err != nil { + return err + } + + if existing, ok := db.universalProDuties[uint64(slot)]; ok { + existingRoot, err := existing.Root() + if err != nil { + return errors.Wrap(err, "universal proposal root") + } + + providedRoot, err := proposal.Root() + if err != nil { + return errors.Wrap(err, "universal proposal root") + } + + if existingRoot != providedRoot { + return errors.New("clashing blocks") + } + } else { + db.universalProDuties[uint64(slot)] = &proposal.VersionedUniversalProposal + } + + return nil +} + // storeBlindedBeaconBlockUnsafe stores the unsigned BlindedBeaconBlock. It is unsafe since it assumes the lock is held. func (db *MemDB) storeBlindedBeaconBlockUnsafe(unsignedData core.UnsignedData) error { cloned, err := unsignedData.Clone() // Clone before storing. @@ -569,6 +650,27 @@ func (db *MemDB) resolveProQueriesUnsafe() { db.proQueries = unresolved } +// resolveUniversalProQueriesUnsafe resolve any universalProQuery to a result if found. +// It is unsafe since it assume that the lock is held. +func (db *MemDB) resolveUniversalProQueriesUnsafe() { + var unresolved []universalProQuery + for _, query := range db.universalProQueries { + if cancelled(query.Cancel) { + continue // Drop cancelled queries. + } + + value, ok := db.universalProDuties[query.Key] + if !ok { + unresolved = append(unresolved, query) + continue + } + + query.Response <- value + } + + db.universalProQueries = unresolved +} + // resolveAggQueriesUnsafe resolve any aggQuery to a result if found. // It is unsafe since it assume that the lock is held. func (db *MemDB) resolveAggQueriesUnsafe() { @@ -639,6 +741,8 @@ func (db *MemDB) deleteDutyUnsafe(duty core.Duty) error { delete(db.proDuties, duty.Slot) case core.DutyBuilderProposer: delete(db.builderProDuties, duty.Slot) + case core.DutyUniversalProposer: + delete(db.universalProDuties, duty.Slot) case core.DutyAttester: for _, key := range db.attKeysBySlot[duty.Slot] { delete(db.attPubKeys, key) @@ -702,6 +806,13 @@ type proQuery struct { Cancel <-chan struct{} } +// universalProQuery is a waiting universalProQuery with a response channel. +type universalProQuery struct { + Key uint64 + Response chan<- *eth2api.VersionedUniversalProposal + Cancel <-chan struct{} +} + // aggQuery is a waiting aggQuery with a response channel. type aggQuery struct { Key aggKey diff --git a/core/dutydb/memory_internal_test.go b/core/dutydb/memory_internal_test.go index 9101cec6a..d967441a7 100644 --- a/core/dutydb/memory_internal_test.go +++ b/core/dutydb/memory_internal_test.go @@ -33,6 +33,9 @@ func TestCancelledQueries(t *testing.T) { _, err = db.AwaitBlindedProposal(ctx, slot) require.ErrorContains(t, err, "shutdown") + _, err = db.AwaitUniversalProposal(ctx, slot) + require.ErrorContains(t, err, "shutdown") + _, err = db.AwaitSyncContribution(ctx, slot, 0, eth2p0.Root{}) require.ErrorContains(t, err, "shutdown") @@ -42,6 +45,7 @@ func TestCancelledQueries(t *testing.T) { require.NotEmpty(t, db.proQueries) require.NotEmpty(t, db.aggQueries) require.NotEmpty(t, db.builderProQueries) + require.NotEmpty(t, db.universalProQueries) // Resolve queries db.resolveAggQueriesUnsafe() @@ -49,6 +53,7 @@ func TestCancelledQueries(t *testing.T) { db.resolveContribQueriesUnsafe() db.resolveProQueriesUnsafe() db.resolveBuilderProQueriesUnsafe() + db.resolveUniversalProQueriesUnsafe() // Ensure all queries are gone. require.Empty(t, db.contribQueries) diff --git a/core/dutydb/memory_test.go b/core/dutydb/memory_test.go index 04866b7cd..25aaf4d7e 100644 --- a/core/dutydb/memory_test.go +++ b/core/dutydb/memory_test.go @@ -176,6 +176,59 @@ func TestMemDBProposer(t *testing.T) { } } +func TestMemDBUniversalProposer(t *testing.T) { + ctx := context.Background() + db := dutydb.NewMemDB(new(testDeadliner)) + + const queries = 3 + slots := [queries]uint64{123, 456, 789} + + type response struct { + block *eth2api.VersionedUniversalProposal + } + var awaitResponse [queries]chan response + for i := 0; i < queries; i++ { + awaitResponse[i] = make(chan response) + go func(slot int) { + block, err := db.AwaitUniversalProposal(ctx, slots[slot]) + require.NoError(t, err) + awaitResponse[slot] <- response{block: block} + }(i) + } + + proposals := make([]*eth2api.VersionedUniversalProposal, queries) + pubkeysByIdx := make(map[eth2p0.ValidatorIndex]core.PubKey) + for i := 0; i < queries; i++ { + proposals[i] = ð2api.VersionedUniversalProposal{ + Proposal: ð2api.VersionedProposal{ + Version: eth2spec.DataVersionBellatrix, + Bellatrix: testutil.RandomBellatrixBeaconBlock(), + }, + } + proposals[i].Proposal.Bellatrix.Slot = eth2p0.Slot(slots[i]) + proposals[i].Proposal.Bellatrix.ProposerIndex = eth2p0.ValidatorIndex(i) + pubkeysByIdx[eth2p0.ValidatorIndex(i)] = testutil.RandomCorePubKey(t) + } + + // Store the Blocks + for i := 0; i < queries; i++ { + unsigned, err := core.NewVersionedUniversalProposal(proposals[i]) + require.NoError(t, err) + + duty := core.Duty{Slot: slots[i], Type: core.DutyUniversalProposer} + err = db.Store(ctx, duty, core.UnsignedDataSet{ + pubkeysByIdx[eth2p0.ValidatorIndex(i)]: unsigned, + }) + require.NoError(t, err) + } + + // Get and assert the proQuery responses + for i := 0; i < queries; i++ { + actualData := <-awaitResponse[i] + require.Equal(t, proposals[i], actualData.block) + } +} + func TestMemDBAggregator(t *testing.T) { ctx := context.Background() db := dutydb.NewMemDB(new(testDeadliner)) @@ -333,6 +386,47 @@ func TestMemDBClashingBlocks(t *testing.T) { require.ErrorContains(t, err, "clashing blocks") } +func TestMemDBClashingUniversalBlocks(t *testing.T) { + ctx := context.Background() + db := dutydb.NewMemDB(new(testDeadliner)) + + const slot = 123 + block1 := ð2api.VersionedUniversalProposal{ + Proposal: ð2api.VersionedProposal{ + Version: eth2spec.DataVersionBellatrix, + Bellatrix: testutil.RandomBellatrixBeaconBlock(), + }, + } + block1.Proposal.Bellatrix.Slot = eth2p0.Slot(slot) + block2 := ð2api.VersionedUniversalProposal{ + Proposal: ð2api.VersionedProposal{ + Version: eth2spec.DataVersionBellatrix, + Bellatrix: testutil.RandomBellatrixBeaconBlock(), + }, + } + block2.Proposal.Bellatrix.Slot = eth2p0.Slot(slot) + pubkey := testutil.RandomCorePubKey(t) + + // Encode the Blocks + unsigned1, err := core.NewVersionedUniversalProposal(block1) + require.NoError(t, err) + + unsigned2, err := core.NewVersionedUniversalProposal(block2) + require.NoError(t, err) + + // Store the Blocks + duty := core.Duty{Slot: slot, Type: core.DutyUniversalProposer} + err = db.Store(ctx, duty, core.UnsignedDataSet{ + pubkey: unsigned1, + }) + require.NoError(t, err) + + err = db.Store(ctx, duty, core.UnsignedDataSet{ + pubkey: unsigned2, + }) + require.ErrorContains(t, err, "clashing blocks") +} + func TestMemDBClashProposer(t *testing.T) { ctx := context.Background() db := dutydb.NewMemDB(new(testDeadliner)) @@ -373,6 +467,48 @@ func TestMemDBClashProposer(t *testing.T) { require.ErrorContains(t, err, "clashing blocks") } +func TestMemDBClashUniversalProposer(t *testing.T) { + ctx := context.Background() + db := dutydb.NewMemDB(new(testDeadliner)) + + const slot = 123 + + block := ð2api.VersionedUniversalProposal{ + Proposal: ð2api.VersionedProposal{ + Version: eth2spec.DataVersionBellatrix, + Bellatrix: testutil.RandomBellatrixBeaconBlock(), + }, + } + block.Proposal.Bellatrix.Slot = eth2p0.Slot(slot) + pubkey := testutil.RandomCorePubKey(t) + + // Encode the block + unsigned, err := core.NewVersionedUniversalProposal(block) + require.NoError(t, err) + + // Store the Blocks + duty := core.Duty{Slot: slot, Type: core.DutyUniversalProposer} + err = db.Store(ctx, duty, core.UnsignedDataSet{ + pubkey: unsigned, + }) + require.NoError(t, err) + + // Store same block from same validator to test idempotent inserts + err = db.Store(ctx, duty, core.UnsignedDataSet{ + pubkey: unsigned, + }) + require.NoError(t, err) + + // Store a different block for the same slot + block.Proposal.Bellatrix.ProposerIndex++ + unsignedB, err := core.NewVersionedUniversalProposal(block) + require.NoError(t, err) + err = db.Store(ctx, duty, core.UnsignedDataSet{ + pubkey: unsignedB, + }) + require.ErrorContains(t, err, "clashing blocks") +} + func TestMemDBBuilderProposer(t *testing.T) { ctx := context.Background() db := dutydb.NewMemDB(new(testDeadliner)) From 92e50a66489001d58b0308f73318d126ec4dfcf0 Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Thu, 29 Feb 2024 10:08:51 +0300 Subject: [PATCH 12/13] Optimized json unmarshaling --- core/signeddata.go | 7 ++++++- core/unsigneddata.go | 12 +++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/core/signeddata.go b/core/signeddata.go index 9aa313bb3..8132e8e7b 100644 --- a/core/signeddata.go +++ b/core/signeddata.go @@ -515,11 +515,16 @@ func (p *VersionedSignedBlindedProposal) UnmarshalJSON(input []byte) error { return nil } +// blindedJSON is a helper type for JSON marshalling of blinded blocks. +type blindedJSON struct { + Blinded bool `json:"blinded,omitempty"` +} + // versionedRawBlockJSON is a custom VersionedSignedBeaconBlock or VersionedSignedBlindedBeaconBlock serialiser. type versionedRawBlockJSON struct { Version eth2util.DataVersion `json:"version"` Block json.RawMessage `json:"block"` - Blinded bool `json:"blinded,omitempty"` + blindedJSON } // NewAttestation is a convenience function that returns a new wrapped attestation. diff --git a/core/unsigneddata.go b/core/unsigneddata.go index c31f11f6c..4522f29b4 100644 --- a/core/unsigneddata.go +++ b/core/unsigneddata.go @@ -335,7 +335,9 @@ func (p VersionedBlindedProposal) MarshalJSON() ([]byte, error) { resp, err := json.Marshal(versionedRawBlockJSON{ Version: version, Block: block, - Blinded: true, + blindedJSON: blindedJSON{ + Blinded: true, + }, }) if err != nil { return nil, errors.Wrap(err, "marshal wrapper") @@ -422,12 +424,12 @@ func (p VersionedUniversalProposal) MarshalJSON() ([]byte, error) { } func (p *VersionedUniversalProposal) UnmarshalJSON(input []byte) error { - var raw versionedRawBlockJSON - if err := json.Unmarshal(input, &raw); err != nil { - return errors.Wrap(err, "unmarshal universal block") + var bjson blindedJSON + if err := json.Unmarshal(input, &bjson); err != nil { + return errors.Wrap(err, "unmarshal blinded flag") } - if raw.Blinded { + if bjson.Blinded { var bp VersionedBlindedProposal if err := bp.UnmarshalJSON(input); err != nil { return err From f568d7243ea8a606594dfb5d2821356cf22f207d Mon Sep 17 00:00:00 2001 From: Andrei Smirnov Date: Thu, 29 Feb 2024 10:21:48 +0300 Subject: [PATCH 13/13] Added fetcher test for universal proposal --- core/fetcher/fetcher_test.go | 51 ++++++++++++++++++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/core/fetcher/fetcher_test.go b/core/fetcher/fetcher_test.go index f3645e928..b18321cbc 100644 --- a/core/fetcher/fetcher_test.go +++ b/core/fetcher/fetcher_test.go @@ -345,6 +345,42 @@ func TestFetchBlocks(t *testing.T) { err = fetch.Fetch(ctx, duty, defSet) require.NoError(t, err) }) + + t.Run("fetch DutyUniversalProposer", func(t *testing.T) { + duty := core.NewUniversalProposerDuty(slot) + fetch, err := fetcher.New(bmock, func(core.PubKey) string { + return feeRecipientAddr + }) + require.NoError(t, err) + + fetch.RegisterAggSigDB(func(ctx context.Context, duty core.Duty, key core.PubKey) (core.SignedData, error) { + return randaoByPubKey[key], nil + }) + + fetch.Subscribe(func(ctx context.Context, resDuty core.Duty, resDataSet core.UnsignedDataSet) error { + require.Equal(t, duty, resDuty) + require.Len(t, resDataSet, 2) + + dutyDataA := resDataSet[pubkeysByIdx[vIdxA]].(core.VersionedUniversalProposal) + slotA, err := dutyDataA.Slot() + require.NoError(t, err) + require.EqualValues(t, slot, slotA) + require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataA.Proposal.Capella.Body.ExecutionPayload.FeeRecipient)) + assertRandaoUniversalBlock(t, randaoByPubKey[pubkeysByIdx[vIdxA]].Signature().ToETH2(), dutyDataA) + + dutyDataB := resDataSet[pubkeysByIdx[vIdxB]].(core.VersionedUniversalProposal) + slotB, err := dutyDataB.Slot() + require.NoError(t, err) + require.EqualValues(t, slot, slotB) + require.Equal(t, feeRecipientAddr, fmt.Sprintf("%#x", dutyDataB.Proposal.Capella.Body.ExecutionPayload.FeeRecipient)) + assertRandaoUniversalBlock(t, randaoByPubKey[pubkeysByIdx[vIdxB]].Signature().ToETH2(), dutyDataB) + + return nil + }) + + err = fetch.Fetch(ctx, duty, defSet) + require.NoError(t, err) + }) } func TestFetchSyncContribution(t *testing.T) { @@ -588,6 +624,21 @@ func assertRandaoBlindedBlock(t *testing.T, randao eth2p0.BLSSignature, block co } } +func assertRandaoUniversalBlock(t *testing.T, randao eth2p0.BLSSignature, block core.VersionedUniversalProposal) { + t.Helper() + + if block.Proposal != nil { + p, err := core.NewVersionedProposal(block.Proposal) + require.NoError(t, err) + assertRandao(t, randao, p) + } + if block.BlindedProposal != nil { + p, err := core.NewVersionedBlindedProposal(block.BlindedProposal) + require.NoError(t, err) + assertRandaoBlindedBlock(t, randao, p) + } +} + // blsSigFromHex returns the BLS signature from the input hex signature. func blsSigFromHex(t *testing.T, sig string) eth2p0.BLSSignature { t.Helper()