op-node: Stall block creation when block finalization lags behind

op-node/node/node.go

@@ -221,21 +221,32 @@ func (n *OpNode) initRuntimeConfig(ctx context.Context, cfg *Config) error {
 			return eth.L1BlockRef{}, err
 		}
 
-		// Apply confirmation-distance
-		blNum := l1Head.Number
-		if blNum >= confDepth {
-			blNum -= confDepth
-		}
-		fetchCtx, fetchCancel = context.WithTimeout(ctx, time.Second*10)
-		confirmed, err := n.l1Source.L1BlockRefByNumber(fetchCtx, blNum)
-		fetchCancel()
-		if err != nil {
-			n.log.Error("failed to fetch confirmed L1 block for runtime config loading", "err", err, "number", blNum)
-			return eth.L1BlockRef{}, err
+		var safeBlock eth.L1BlockRef
+		if cfg.Driver.SequencerUseFinalized {
+			fetchCtx, fetchCancel = context.WithTimeout(ctx, time.Second*10)
+			safeBlock, err = n.l1Source.L1BlockRefByLabel(fetchCtx, eth.Finalized)
+			fetchCancel()
+			if err != nil {
+				n.log.Error("failed to fetch finalized L1 block for runtime config loading", "err", err)
+				return eth.L1BlockRef{}, err
+			}
+		} else {
+			// Apply confirmation-distance
+			blNum := l1Head.Number
+			if blNum >= confDepth {
+				blNum -= confDepth
+			}
+			fetchCtx, fetchCancel = context.WithTimeout(ctx, time.Second*10)
+			safeBlock, err = n.l1Source.L1BlockRefByNumber(fetchCtx, blNum)
+			fetchCancel()
+			if err != nil {
+				n.log.Error("failed to fetch confirmed L1 block for runtime config loading", "err", err, "number", blNum)
+				return eth.L1BlockRef{}, err
+			}
 		}
 
 		fetchCtx, fetchCancel = context.WithTimeout(ctx, time.Second*10)
-		err = n.runCfg.Load(fetchCtx, confirmed)
+		err = n.runCfg.Load(fetchCtx, safeBlock)
 		fetchCancel()
 		if err != nil {
 			n.log.Error("failed to fetch runtime config data", "err", err)

op-node/rollup/driver/config.go

@@ -20,4 +20,10 @@ type Config struct {
 	// SequencerMaxSafeLag is the maximum number of L2 blocks for restricting the distance between L2 safe and unsafe.
 	// Disabled if 0.
 	SequencerMaxSafeLag uint64 `json:"sequencer_max_safe_lag"`
+
+	// SequencerUseFinalized is true when the sequencer should use only finalized L1 blocks as origin.
+	// If this is set to true, the value of `SequencerConfDepth` is ignored.
+	SequencerUseFinalized bool `json:"sequencer_use_finalized"`
+
+	SequencerMaxFinalizedLag uint64 `json:"sequencer_max_finalized_lag"`
 }

op-node/rollup/driver/driver.go

@@ -176,9 +176,16 @@ func NewDriver(
 
 	l1 = NewMeteredL1Fetcher(l1, metrics)
 	l1State := NewL1State(log, metrics)
-	sequencerConfDepth := NewConfDepth(driverCfg.SequencerConfDepth, l1State.L1Head, l1)
-	findL1Origin := NewL1OriginSelector(log, cfg, sequencerConfDepth)
-	verifConfDepth := NewConfDepth(driverCfg.VerifierConfDepth, l1State.L1Head, l1)
+	var l1Blocks L1Blocks
+	var l1Fetcher derive.L1Fetcher
+	if driverCfg.SequencerUseFinalized {
+		l1Blocks = NewFinalized(l1State.L1Finalized, l1)
+		l1Fetcher = NewFinalized(l1State.L1Finalized, l1)
+	} else {
+		l1Blocks = NewConfDepth(driverCfg.SequencerConfDepth, l1State.L1Head, l1)
+		l1Fetcher = NewConfDepth(driverCfg.VerifierConfDepth, l1State.L1Head, l1)
+	}
+	findL1Origin := NewL1OriginSelector(log, cfg, l1Blocks)
 	ec := engine.NewEngineController(l2, log, metrics, cfg, syncCfg.SyncMode, synchronousEvents)
 	engineResetDeriver := engine.NewEngineResetDeriver(driverCtx, log, cfg, l1, l2, syncCfg, synchronousEvents)
 	clSync := clsync.NewCLSync(log, cfg, metrics, synchronousEvents)
@@ -191,7 +198,7 @@ func NewDriver(
 	}
 
 	attributesHandler := attributes.NewAttributesHandler(log, cfg, driverCtx, l2, synchronousEvents)
-	derivationPipeline := derive.NewDerivationPipeline(log, cfg, verifConfDepth, l1Blobs, plasma, l2, metrics)
+	derivationPipeline := derive.NewDerivationPipeline(log, cfg, l1Fetcher, l1Blobs, plasma, l2, metrics)
 	pipelineDeriver := derive.NewPipelineDeriver(driverCtx, derivationPipeline, synchronousEvents)
 	attrBuilder := derive.NewFetchingAttributesBuilder(cfg, l1, l2)
 	meteredEngine := NewMeteredEngine(cfg, ec, metrics, log) // Only use the metered engine in the sequencer b/c it records sequencing metrics.

op-node/rollup/driver/finalized.go

@@ -0,0 +1,33 @@
+package driver
+
+import (
+	"context"
+
+	"github.com/ethereum/go-ethereum"
+
+	"github.com/ethereum-optimism/optimism/op-node/rollup/derive"
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+)
+
+type finalized struct {
+	derive.L1Fetcher
+	l1Finalized func() eth.L1BlockRef
+}
+
+func NewFinalized(l1Finalized func() eth.L1BlockRef, fetcher derive.L1Fetcher) *finalized {
+	return &finalized{L1Fetcher: fetcher, l1Finalized: l1Finalized}
+}
+
+func (f *finalized) L1BlockRefByNumber(ctx context.Context, num uint64) (eth.L1BlockRef, error) {
+	// Don't apply the finalized wrapper if l1Finalized is empty (as it is during the startup case before the l1State is initialized).
+	l1Finalized := f.l1Finalized()
+	if l1Finalized == (eth.L1BlockRef{}) {
+		return f.L1Fetcher.L1BlockRefByNumber(ctx, num)
+	}
+	if num == 0 || num <= l1Finalized.Number {
+		return f.L1Fetcher.L1BlockRefByNumber(ctx, num)
+	}
+	return eth.L1BlockRef{}, ethereum.NotFound
+}
+
+var _ derive.L1Fetcher = (*finalized)(nil)

op-node/rollup/driver/finalized_test.go

@@ -0,0 +1,56 @@
+package driver
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ethereum/go-ethereum"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum-optimism/optimism/op-service/eth"
+	"github.com/ethereum-optimism/optimism/op-service/testutils"
+)
+
+type finalizedTest struct {
+	name  string
+	final uint64
+	hash  common.Hash // hash of finalized block
+	req   uint64
+	pass  bool
+}
+
+func (ft *finalizedTest) Run(t *testing.T) {
+	l1Fetcher := &testutils.MockL1Source{}
+	l1Finalized := eth.L1BlockRef{Number: ft.final, Hash: ft.hash}
+	l1FinalizedGetter := func() eth.L1BlockRef { return l1Finalized }
+
+	f := NewFinalized(l1FinalizedGetter, l1Fetcher)
+
+	if ft.pass {
+		// no calls to the l1Fetcher are made if the block number is not finalized yet
+		l1Fetcher.ExpectL1BlockRefByNumber(ft.req, eth.L1BlockRef{Number: ft.req}, nil)
+	}
+
+	out, err := f.L1BlockRefByNumber(context.Background(), ft.req)
+	l1Fetcher.AssertExpectations(t)
+
+	if ft.pass {
+		require.NoError(t, err)
+		require.Equal(t, out, eth.L1BlockRef{Number: ft.req})
+	} else {
+		require.Equal(t, ethereum.NotFound, err)
+	}
+}
+
+func TestFinalized(t *testing.T) {
+	testCases := []finalizedTest{
+		{name: "finalized", final: 10, hash: exHash, req: 10, pass: true},
+		{name: "finalized past", final: 10, hash: exHash, req: 8, pass: true},
+		{name: "not finalized", final: 10, hash: exHash, req: 11, pass: false},
+		{name: "no L1 state", req: 10, pass: true},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, tc.Run)
+	}
+}

op-node/rollup/driver/state.go

@@ -229,6 +229,36 @@ func (s *Driver) eventLoop() {
 			s.log.Error("unexpected error from event-draining", "err", err)
 		}
 
+		if s.driverConfig.SequencerEnabled && !s.driverConfig.SequencerStopped &&
+			s.l1State.L1Head() != (eth.L1BlockRef{}) && s.l1State.L1Finalized() != (eth.L1BlockRef{}) &&
+			s.Derivation.DerivationReady() {
+
+			// If the L1 does fall behind the limit with finalizing blocks, delay creating new blocks
+			// until the finalized lag is below SequencerMaxFinalizedLag.
+			numBlocksSinceFinalization := s.l1State.L1Head().Number - s.l1State.L1Finalized().Number
+			if numBlocksSinceFinalization > s.driverConfig.SequencerMaxFinalizedLag {
+				if sequencerCh != nil {
+					s.log.Warn(
+						"Delay creating new block since finalized lag exceeds limit",
+						"finalized_l1", s.l1State.L1Finalized(),
+						"l1_head", s.l1State.L1Head(),
+						"unsafe_l2", s.Engine.UnsafeL2Head(),
+						"safe_l2", s.Engine.SafeL2Head(),
+					)
+					sequencerCh = nil
+				}
+				// respCh := make(chan hashAndError)
+				// s.stopSequencer <- respCh
+				// resp := <-respCh
+				// if resp.err != nil {
+				// 	log.Error("Failed to stop sequencer", "err", resp.err)
+				// } else {
+				// 	log.Info("Sequencer has been stopped", "hash", resp.hash)
+				// }
+				// } else {
+				// 	s.startSequencer
+			}
+		}
 		// If we are sequencing, and the L1 state is ready, update the trigger for the next sequencer action.
 		// This may adjust at any time based on fork-choice changes or previous errors.
 		// And avoid sequencing if the derivation pipeline indicates the engine is not ready.