make data source priority configurable (#136)

.github/workflows/regression-tests.yml

@@ -24,11 +24,9 @@ jobs:
           fi
 
       - name: Set up Docker
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v3
 
       - name: Run regression tests against JIT container image
         uses: 0xPolygon/[email protected]
         with:
           zkevm_node: ${{ steps.build.outputs.GITHUB_SHA_SHORT }}
-          kurtosis_cli: 0.89.3
-          kurtosis_cdk: v0.2.0

cmd/run.go

@@ -305,6 +305,7 @@ func newEtherman(c config.Config, st *state.State) (*etherman.Client, error) {
 func newDataAvailability(c config.Config, st *state.State, etherman *etherman.Client, isSequenceSender bool) (*dataavailability.DataAvailability, error) {
 	var (
 		trustedSequencerURL string
+		dataSourcePriority  []dataavailability.DataSourcePriority
 		err                 error
 	)
 	if !c.IsTrustedSequencer {
@@ -321,6 +322,11 @@ func newDataAvailability(c config.Config, st *state.State, etherman *etherman.Cl
 	}
 	zkEVMClient := client.NewClient(trustedSequencerURL)
 
+	dataSourcePriority = c.Synchronizer.L2Synchronization.DataSourcePriority
+	if len(dataSourcePriority) == 0 {
+		dataSourcePriority = dataavailability.DefaultPriority
+	}
+
 	// Backend specific config
 	daProtocolName, err := etherman.GetDAProtocolName()
 	if err != nil {
@@ -362,6 +368,7 @@ func newDataAvailability(c config.Config, st *state.State, etherman *etherman.Cl
 		daBackend,
 		st,
 		zkEVMClient,
+		dataSourcePriority,
 	)
 }
 

config/config_test.go

@@ -12,6 +12,7 @@ import (
 	"github.com/0xPolygonHermez/zkevm-node/aggregator"
 	"github.com/0xPolygonHermez/zkevm-node/config"
 	"github.com/0xPolygonHermez/zkevm-node/config/types"
+	"github.com/0xPolygonHermez/zkevm-node/dataavailability"
 	"github.com/0xPolygonHermez/zkevm-node/log"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/assert"
@@ -64,7 +65,10 @@ func Test_Defaults(t *testing.T) {
 			path:          "Synchronizer.L2Synchronization.CheckLastL2BlockHashOnCloseBatch",
 			expectedValue: true,
 		},
-
+		{
+			path:          "Synchronizer.L2Synchronization.DataSourcePriority",
+			expectedValue: []dataavailability.DataSourcePriority{"local", "trusted", "external"},
+		},
 		{
 			path:          "Sequencer.DeletePoolTxsL1BlockConfirmations",
 			expectedValue: uint64(100),

config/default.go

@@ -131,6 +131,7 @@ L1SyncCheckL2BlockNumberhModulus = 600
 		AcceptEmptyClosedBatches = false
 		ReprocessFullBatchOnClose = false
 		CheckLastL2BlockHashOnCloseBatch = true
+		DataSourcePriority = ["local","trusted","external"]
 
 [Sequencer]
 DeletePoolTxsL1BlockConfirmations = 100

dataavailability/dataavailability.go

@@ -2,6 +2,7 @@ package dataavailability
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"math/big"
 
@@ -18,15 +19,29 @@ const (
 	invalidBatchRetrievalArgs   = "invalid L2 batch data retrieval arguments, %d != %d"
 )
 
+// DataSourcePriority defines where data is retrieved from
+type DataSourcePriority string
+
+const (
+	// Local indicates data stored in this nodes DB
+	Local DataSourcePriority = "local"
+	// Trusted indicates data stored in the Trusted Sequencer
+	Trusted DataSourcePriority = "trusted"
+	// External indicates data stored in the Data Availability layer
+	External DataSourcePriority = "external"
+)
+
+// DefaultPriority is the default order in which data is retrieved
+var DefaultPriority = []DataSourcePriority{Local, Trusted, External}
+
 // DataAvailability implements an abstract data availability integration
 type DataAvailability struct {
 	isTrustedSequencer bool
-
-	state       stateInterface
-	zkEVMClient ZKEVMClientTrustedBatchesGetter
-	backend     DABackender
-
-	ctx context.Context
+	state              stateInterface
+	zkEVMClient        ZKEVMClientTrustedBatchesGetter
+	backend            DABackender
+	dataSourcePriority []DataSourcePriority
+	ctx                context.Context
 }
 
 // New creates a DataAvailability instance
@@ -35,13 +50,18 @@ func New(
 	backend DABackender,
 	state stateInterface,
 	zkEVMClient ZKEVMClientTrustedBatchesGetter,
+	priority []DataSourcePriority,
 ) (*DataAvailability, error) {
 	da := &DataAvailability{
 		isTrustedSequencer: isTrustedSequencer,
 		backend:            backend,
 		state:              state,
 		zkEVMClient:        zkEVMClient,
 		ctx:                context.Background(),
+		dataSourcePriority: priority,
+	}
+	if len(da.dataSourcePriority) == 0 {
+		da.dataSourcePriority = DefaultPriority
 	}
 	err := da.backend.Init()
 	return da, err
@@ -68,27 +88,38 @@ func (d *DataAvailability) GetBatchL2Data(batchNums []uint64, batchHashes []comm
 	if len(batchNums) != len(batchHashes) {
 		return nil, fmt.Errorf(invalidBatchRetrievalArgs, len(batchNums), len(batchHashes))
 	}
-	localData, err := d.state.GetBatchL2DataByNumbers(d.ctx, batchNums, nil)
-	if err != nil {
-		return nil, err
-	}
 
-	data, err := checkBatches(batchNums, batchHashes, localData)
-	if err != nil {
-		log.Warnf(failedDataRetrievalTemplate, batchNums, err.Error())
-	} else {
-		return data, nil
-	}
+	for _, p := range d.dataSourcePriority {
+		switch p {
+		case Local:
+			localData, err := d.state.GetBatchL2DataByNumbers(d.ctx, batchNums, nil)
+			if err != nil {
+				return nil, err
+			}
 
-	if !d.isTrustedSequencer {
-		data, err = d.rpcData(batchNums, batchHashes, d.zkEVMClient.BatchesByNumbers)
-		if err != nil {
-			log.Warnf(failedDataRetrievalTemplate, batchNums, err.Error())
-		} else {
-			return data, nil
+			data, err := checkBatches(batchNums, batchHashes, localData)
+			if err != nil {
+				log.Warnf(failedDataRetrievalTemplate, batchNums, err.Error())
+			} else {
+				return data, nil
+			}
+		case Trusted:
+			if !d.isTrustedSequencer {
+				data, err := d.rpcData(batchNums, batchHashes, d.zkEVMClient.BatchesByNumbers)
+				if err != nil {
+					log.Warnf(failedDataRetrievalTemplate, batchNums, err.Error())
+				} else {
+					return data, nil
+				}
+			}
+		case External:
+			return d.backend.GetSequence(d.ctx, batchHashes, dataAvailabilityMessage)
+		default:
+			log.Warnf("invalid data retrieval priority: %s", p)
 		}
 	}
-	return d.backend.GetSequence(d.ctx, batchHashes, dataAvailabilityMessage)
+
+	return nil, errors.New("failed to retrieve l2 batch data")
 }
 
 func checkBatches(batchNumbers []uint64, expectedHashes []common.Hash, batchData map[uint64][]byte) ([][]byte, error) {

docs/config-file/node-config-doc.html

@@ -28,7 +28,7 @@
 </pre></div> </div><div id=Synchronizer_L1ParallelSynchronization_TimeOutMainLoop_ex2 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;300ms&quot;</span>
 </pre></div> </div> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L1ParallelSynchronization.RollupInfoRetriesSpacing onclick="anchorLink('Synchronizer.L1ParallelSynchronization.RollupInfoRetriesSpacing')">Synchronizer.L1ParallelSynchronization.RollupInfoRetriesSpacing=</a> </div> <span class="badge badge-success default-value">Default: "5s"</span><span class="badge badge-dark value-type">Type: string</span><br> <span class=description><p>RollupInfoRetriesSpacing is the minimum time between retries to request rollup info (it will sleep for fulfill this time) to avoid spamming L1</p> </span> <br> <div class="badge badge-secondary">Examples:</div> <br><div id=Synchronizer_L1ParallelSynchronization_RollupInfoRetriesSpacing_ex1 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;1m&quot;</span>
 </pre></div> </div><div id=Synchronizer_L1ParallelSynchronization_RollupInfoRetriesSpacing_ex2 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;300ms&quot;</span>
-</pre></div> </div> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized onclick="anchorLink('Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized')">Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>FallbackToSequentialModeOnSynchronized if true switch to sequential mode if the system is synchronized</p> </span> <hr> </div> </div> </div> </div> <div class=accordion id=accordionSynchronizer_L2Synchronization> <div class=card> <div class=card-header id=headingSynchronizer_L2Synchronization> <h2 class=mb-0> <button class="btn btn-link property-name-button" type=button data-toggle=collapse data-target=#Synchronizer_L2Synchronization aria-expanded aria-controls=Synchronizer_L2Synchronization onclick="setAnchor('#Synchronizer_L2Synchronization')"><span class=property-name> <div class=breadcrumbs>[<a href=#Synchronizer onclick="anchorLink('Synchronizer')">Synchronizer</a> . <a href=#Synchronizer_L2Synchronization onclick="anchorLink('Synchronizer_L2Synchronization')">L2Synchronization</a>] </div></span></button> </h2> L2Synchronization Configuration for L2 synchronization </div> <div id=Synchronizer_L2Synchronization class="collapse property-definition-div" aria-labelledby=headingSynchronizer_L2Synchronization data-parent=#accordionSynchronizer_L2Synchronization> <div class="card-body pl-5"> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.AcceptEmptyClosedBatches onclick="anchorLink('Synchronizer.L2Synchronization.AcceptEmptyClosedBatches')">Synchronizer.L2Synchronization.AcceptEmptyClosedBatches=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>AcceptEmptyClosedBatches is a flag to enable or disable the acceptance of empty batches.<br> if true, the synchronizer will accept empty batches and process them.</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.ReprocessFullBatchOnClose onclick="anchorLink('Synchronizer.L2Synchronization.ReprocessFullBatchOnClose')">Synchronizer.L2Synchronization.ReprocessFullBatchOnClose=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>ReprocessFullBatchOnClose if is true when a batch is closed is force to reprocess again</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.CheckLastL2BlockHashOnCloseBatch onclick="anchorLink('Synchronizer.L2Synchronization.CheckLastL2BlockHashOnCloseBatch')">Synchronizer.L2Synchronization.CheckLastL2BlockHashOnCloseBatch=</a> </div> <span class="badge badge-success default-value">Default: true</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>CheckLastL2BlockHashOnCloseBatch if is true when a batch is closed is force to check the last L2Block hash</p> </span> <hr> </div> </div> </div> </div> </div> </div> </div> </div> <div class=accordion id=accordionSequencer> <div class=card> <div class=card-header id=headingSequencer> <h2 class=mb-0> <button class="btn btn-link property-name-button" type=button data-toggle=collapse data-target=#Sequencer aria-expanded aria-controls=Sequencer onclick="setAnchor('#Sequencer')"><span class=property-name> <div class=breadcrumbs>[<a href=#Sequencer onclick="anchorLink('Sequencer')">Sequencer</a>] </div></span></button> </h2> Configuration of the sequencer service </div> <div id=Sequencer class="collapse property-definition-div" aria-labelledby=headingSequencer data-parent=#accordionSequencer> <div class="card-body pl-5"> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.DeletePoolTxsL1BlockConfirmations onclick="anchorLink('Sequencer.DeletePoolTxsL1BlockConfirmations')">Sequencer.DeletePoolTxsL1BlockConfirmations=</a> </div> <span class="badge badge-success default-value">Default: 100</span><span class="badge badge-dark value-type">Type: integer</span><br> <span class=description><p>DeletePoolTxsL1BlockConfirmations is blocks amount after which txs will be deleted from the pool</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.DeletePoolTxsCheckInterval onclick="anchorLink('Sequencer.DeletePoolTxsCheckInterval')">Sequencer.DeletePoolTxsCheckInterval=</a> </div> <span class="badge badge-success default-value">Default: "12h0m0s"</span><span class="badge badge-dark value-type">Type: string</span><br> <span class=description><p>DeletePoolTxsCheckInterval is frequency with which txs will be checked for deleting</p> </span> <br> <div class="badge badge-secondary">Examples:</div> <br><div id=Sequencer_DeletePoolTxsCheckInterval_ex1 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;1m&quot;</span>
+</pre></div> </div> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized onclick="anchorLink('Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized')">Synchronizer.L1ParallelSynchronization.FallbackToSequentialModeOnSynchronized=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>FallbackToSequentialModeOnSynchronized if true switch to sequential mode if the system is synchronized</p> </span> <hr> </div> </div> </div> </div> <div class=accordion id=accordionSynchronizer_L2Synchronization> <div class=card> <div class=card-header id=headingSynchronizer_L2Synchronization> <h2 class=mb-0> <button class="btn btn-link property-name-button" type=button data-toggle=collapse data-target=#Synchronizer_L2Synchronization aria-expanded aria-controls=Synchronizer_L2Synchronization onclick="setAnchor('#Synchronizer_L2Synchronization')"><span class=property-name> <div class=breadcrumbs>[<a href=#Synchronizer onclick="anchorLink('Synchronizer')">Synchronizer</a> . <a href=#Synchronizer_L2Synchronization onclick="anchorLink('Synchronizer_L2Synchronization')">L2Synchronization</a>] </div></span></button> </h2> L2Synchronization Configuration for L2 synchronization </div> <div id=Synchronizer_L2Synchronization class="collapse property-definition-div" aria-labelledby=headingSynchronizer_L2Synchronization data-parent=#accordionSynchronizer_L2Synchronization> <div class="card-body pl-5"> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.AcceptEmptyClosedBatches onclick="anchorLink('Synchronizer.L2Synchronization.AcceptEmptyClosedBatches')">Synchronizer.L2Synchronization.AcceptEmptyClosedBatches=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>AcceptEmptyClosedBatches is a flag to enable or disable the acceptance of empty batches.<br> if true, the synchronizer will accept empty batches and process them.</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.ReprocessFullBatchOnClose onclick="anchorLink('Synchronizer.L2Synchronization.ReprocessFullBatchOnClose')">Synchronizer.L2Synchronization.ReprocessFullBatchOnClose=</a> </div> <span class="badge badge-success default-value">Default: false</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>ReprocessFullBatchOnClose if is true when a batch is closed is force to reprocess again</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.CheckLastL2BlockHashOnCloseBatch onclick="anchorLink('Synchronizer.L2Synchronization.CheckLastL2BlockHashOnCloseBatch')">Synchronizer.L2Synchronization.CheckLastL2BlockHashOnCloseBatch=</a> </div> <span class="badge badge-success default-value">Default: true</span><span class="badge badge-dark value-type">Type: boolean</span><br> <span class=description><p>CheckLastL2BlockHashOnCloseBatch if is true when a batch is closed is force to check the last L2Block hash</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><a href=#Synchronizer.L2Synchronization.DataSourcePriority onclick="anchorLink('Synchronizer.L2Synchronization.DataSourcePriority')">Synchronizer.L2Synchronization.DataSourcePriority=</a> </div> <span class="badge badge-success default-value">Default: ["local", "trusted", "external"]</span><span class="badge badge-dark value-type">Type: array of string</span><br> <span class=description><p>DataSourcePriority defines the order in which L2 batch should be retrieved: local, trusted, external</p> </span> <h4>Each item of this array must be:</h4> <div class=card> <div class="card-body items-definition" id=Synchronizer_L2Synchronization_DataSourcePriority_items> <div class=breadcrumbs> <!-- None --><!-- None --><!-- None --><!-- None --><a href="#Synchronizer.L2Synchronization.DataSourcePriority.DataSourcePriority items" onclick="anchorLink('Synchronizer.L2Synchronization.DataSourcePriority.DataSourcePriority items')">Synchronizer.L2Synchronization.DataSourcePriority.DataSourcePriority items=</a> </div><span class="badge badge-dark value-type">Type: string</span><br> </div> </div> <hr> </div> </div> </div> </div> </div> </div> </div> </div> <div class=accordion id=accordionSequencer> <div class=card> <div class=card-header id=headingSequencer> <h2 class=mb-0> <button class="btn btn-link property-name-button" type=button data-toggle=collapse data-target=#Sequencer aria-expanded aria-controls=Sequencer onclick="setAnchor('#Sequencer')"><span class=property-name> <div class=breadcrumbs>[<a href=#Sequencer onclick="anchorLink('Sequencer')">Sequencer</a>] </div></span></button> </h2> Configuration of the sequencer service </div> <div id=Sequencer class="collapse property-definition-div" aria-labelledby=headingSequencer data-parent=#accordionSequencer> <div class="card-body pl-5"> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.DeletePoolTxsL1BlockConfirmations onclick="anchorLink('Sequencer.DeletePoolTxsL1BlockConfirmations')">Sequencer.DeletePoolTxsL1BlockConfirmations=</a> </div> <span class="badge badge-success default-value">Default: 100</span><span class="badge badge-dark value-type">Type: integer</span><br> <span class=description><p>DeletePoolTxsL1BlockConfirmations is blocks amount after which txs will be deleted from the pool</p> </span> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.DeletePoolTxsCheckInterval onclick="anchorLink('Sequencer.DeletePoolTxsCheckInterval')">Sequencer.DeletePoolTxsCheckInterval=</a> </div> <span class="badge badge-success default-value">Default: "12h0m0s"</span><span class="badge badge-dark value-type">Type: string</span><br> <span class=description><p>DeletePoolTxsCheckInterval is frequency with which txs will be checked for deleting</p> </span> <br> <div class="badge badge-secondary">Examples:</div> <br><div id=Sequencer_DeletePoolTxsCheckInterval_ex1 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;1m&quot;</span>
 </pre></div> </div><div id=Sequencer_DeletePoolTxsCheckInterval_ex2 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;300ms&quot;</span>
 </pre></div> </div> <hr> <div class=breadcrumbs> <!-- None --><!-- None --><a href=#Sequencer.TxLifetimeCheckInterval onclick="anchorLink('Sequencer.TxLifetimeCheckInterval')">Sequencer.TxLifetimeCheckInterval=</a> </div> <span class="badge badge-success default-value">Default: "10m0s"</span><span class="badge badge-dark value-type">Type: string</span><br> <span class=description><p>TxLifetimeCheckInterval is the time the sequencer waits to check txs lifetime</p> </span> <br> <div class="badge badge-secondary">Examples:</div> <br><div id=Sequencer_TxLifetimeCheckInterval_ex1 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;1m&quot;</span>
 </pre></div> </div><div id=Sequencer_TxLifetimeCheckInterval_ex2 class="jumbotron examples"><div class=highlight><pre><span></span><span class=s2>&quot;300ms&quot;</span>