Merge branch 'master' into waituntil-rollupcreation-isfinalized

Dockerfile

@@ -264,6 +264,8 @@ COPY --from=node-builder /workspace/target/bin/relay /usr/local/bin/
 COPY --from=node-builder /workspace/target/bin/nitro-val /usr/local/bin/
 COPY --from=node-builder /workspace/target/bin/seq-coordinator-manager /usr/local/bin/
 COPY --from=node-builder /workspace/target/bin/prover /usr/local/bin/
+COPY --from=node-builder /workspace/target/bin/dbconv /usr/local/bin/
+COPY ./scripts/convert-databases.bash /usr/local/bin/
 COPY --from=machine-versions /workspace/machines /home/user/target/machines
 COPY ./scripts/validate-wasm-module-root.sh .
 RUN ./validate-wasm-module-root.sh /home/user/target/machines /usr/local/bin/prover

Makefile

@@ -157,7 +157,7 @@ all: build build-replay-env test-gen-proofs
 	@touch .make/all
 
 .PHONY: build
-build: $(patsubst %,$(output_root)/bin/%, nitro deploy relay daserver datool seq-coordinator-invalidate nitro-val seq-coordinator-manager)
+build: $(patsubst %,$(output_root)/bin/%, nitro deploy relay daserver datool seq-coordinator-invalidate nitro-val seq-coordinator-manager dbconv)
 	@printf $(done)
 
 .PHONY: build-node-deps
@@ -310,6 +310,9 @@ $(output_root)/bin/nitro-val: $(DEP_PREDICATE) build-node-deps
 $(output_root)/bin/seq-coordinator-manager: $(DEP_PREDICATE) build-node-deps
 	go build $(GOLANG_PARAMS) -o $@ "$(CURDIR)/cmd/seq-coordinator-manager"
 
+$(output_root)/bin/dbconv: $(DEP_PREDICATE) build-node-deps
+	go build $(GOLANG_PARAMS) -o $@ "$(CURDIR)/cmd/dbconv"
+
 # recompile wasm, but don't change timestamp unless files differ
 $(replay_wasm): $(DEP_PREDICATE) $(go_source) .make/solgen
 	mkdir -p `dirname $(replay_wasm)`

README.md

@@ -17,26 +17,26 @@
 <img src="https://arbitrum.io/assets/arbitrum/logo_color.png" alt="Logo" width="80" height="80">
 
 Nitro is the latest iteration of the Arbitrum technology. It is a fully integrated, complete
-layer 2 optimistic rollup system, including fraud proofs, the sequencer, the token bridges, 
+layer 2 optimistic rollup system, including fraud proofs, the sequencer, the token bridges,
 advanced calldata compression, and more.
 
 See the live docs-site [here](https://developer.arbitrum.io/) (or [here](https://github.com/OffchainLabs/arbitrum-docs) for markdown docs source.)
 
-See [here](./audits) for security audit reports.
+See [here](https://docs.arbitrum.io/audit-reports) for security audit reports.
 
-The Nitro stack is built on several innovations. At its core is a new prover, which can do Arbitrum’s classic 
-interactive fraud proofs over WASM code. That means the L2 Arbitrum engine can be written and compiled using 
+The Nitro stack is built on several innovations. At its core is a new prover, which can do Arbitrum’s classic
+interactive fraud proofs over WASM code. That means the L2 Arbitrum engine can be written and compiled using
 standard languages and tools, replacing the custom-designed language and compiler used in previous Arbitrum
-versions. In normal execution, 
-validators and nodes run the Nitro engine compiled to native code, switching to WASM if a fraud proof is needed. 
-We compile the core of Geth, the EVM engine that practically defines the Ethereum standard, right into Arbitrum. 
+versions. In normal execution,
+validators and nodes run the Nitro engine compiled to native code, switching to WASM if a fraud proof is needed.
+We compile the core of Geth, the EVM engine that practically defines the Ethereum standard, right into Arbitrum.
 So the previous custom-built EVM emulator is replaced by Geth, the most popular and well-supported Ethereum client.
 
-The last piece of the stack is a slimmed-down version of our ArbOS component, rewritten in Go, which provides the 
-rest of what’s needed to run an L2 chain: things like cross-chain communication, and a new and improved batching 
+The last piece of the stack is a slimmed-down version of our ArbOS component, rewritten in Go, which provides the
+rest of what’s needed to run an L2 chain: things like cross-chain communication, and a new and improved batching
 and compression system to minimize L1 costs.
 
-Essentially, Nitro runs Geth at layer 2 on top of Ethereum, and can prove fraud over the core engine of Geth 
+Essentially, Nitro runs Geth at layer 2 on top of Ethereum, and can prove fraud over the core engine of Geth
 compiled to WASM.
 
 Arbitrum One successfully migrated from the Classic Arbitrum stack onto Nitro on 8/31/22. (See [state migration](https://developer.arbitrum.io/migration/state-migration) and [dapp migration](https://developer.arbitrum.io/migration/dapp_migration) for more info).
@@ -45,14 +45,12 @@ Arbitrum One successfully migrated from the Classic Arbitrum stack onto Nitro on
 
 Nitro is currently licensed under a [Business Source License](./LICENSE.md), similar to our friends at Uniswap and Aave, with an "Additional Use Grant" to ensure that everyone can have full comfort using and running nodes on all public Arbitrum chains.
 
-The Additional Use Grant also permits the deployment of the Nitro software, in a permissionless fashion and without cost, as a new blockchain provided that the chain settles to either Arbitrum One or Arbitrum Nova. 
+The Additional Use Grant also permits the deployment of the Nitro software, in a permissionless fashion and without cost, as a new blockchain provided that the chain settles to either Arbitrum One or Arbitrum Nova.
 
-For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP. 
+For those that prefer to deploy the Nitro software either directly on Ethereum (i.e. an L2) or have it settle to another Layer-2 on top of Ethereum, the [Arbitrum Expansion Program (the "AEP")](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf) was recently established. The AEP allows for the permissionless deployment in the aforementioned fashion provided that 10% of net revenue (as more fully described in the AEP) is contributed back to the Arbitrum community in accordance with the requirements of the AEP.
 
 ## Contact
 
 Discord - [Arbitrum](https://discord.com/invite/5KE54JwyTs)
 
 Twitter: [Arbitrum](https://twitter.com/arbitrum)
-
-

arbitrator/wasm-libraries/user-host-trait/src/lib.rs

@@ -534,7 +534,7 @@ pub trait UserHost<DR: DataReader>: GasMeteredMachine {
     fn return_data_size(&mut self) -> Result<u32, Self::Err> {
         self.buy_ink(HOSTIO_INK)?;
         let len = *self.evm_return_data_len();
-        trace!("return_data_size", self, be!(len), &[], len)
+        trace!("return_data_size", self, &[], be!(len), len)
     }
 
     /// Emits an EVM log with the given number of topics and data, the first bytes of which should
@@ -629,7 +629,8 @@ pub trait UserHost<DR: DataReader>: GasMeteredMachine {
         self.buy_gas(gas_cost)?;
 
         let code = code.slice();
-        trace!("account_code_size", self, address, &[], code.len() as u32)
+        let len = code.len() as u32;
+        trace!("account_code_size", self, address, be!(len), len)
     }
 
     /// Gets the code hash of the account at the given address. The semantics are equivalent
@@ -735,7 +736,7 @@ pub trait UserHost<DR: DataReader>: GasMeteredMachine {
     fn evm_gas_left(&mut self) -> Result<u64, Self::Err> {
         self.buy_ink(HOSTIO_INK)?;
         let gas = self.gas_left()?;
-        trace!("evm_gas_left", self, be!(gas), &[], gas)
+        trace!("evm_gas_left", self, &[], be!(gas), gas)
     }
 
     /// Gets the amount of ink remaining after paying for the cost of this hostio. The semantics
@@ -747,7 +748,7 @@ pub trait UserHost<DR: DataReader>: GasMeteredMachine {
     fn evm_ink_left(&mut self) -> Result<u64, Self::Err> {
         self.buy_ink(HOSTIO_INK)?;
         let ink = self.ink_ready()?;
-        trace!("evm_ink_left", self, be!(ink), &[], ink)
+        trace!("evm_ink_left", self, &[], be!(ink), ink)
     }
 
     /// Computes `value ÷ exponent` using 256-bit math, writing the result to the first.

arbnode/seq_coordinator.go

@@ -39,6 +39,7 @@ type SeqCoordinator struct {
 
 	redisutil.RedisCoordinator
 
+	sync             *SyncMonitor
 	streamer         *TransactionStreamer
 	sequencer        execution.ExecutionSequencer
 	delayedSequencer *DelayedSequencer
@@ -69,9 +70,10 @@ type SeqCoordinatorConfig struct {
 	SafeShutdownDelay     time.Duration `koanf:"safe-shutdown-delay"`
 	ReleaseRetries        int           `koanf:"release-retries"`
 	// Max message per poll.
-	MsgPerPoll arbutil.MessageIndex       `koanf:"msg-per-poll"`
-	MyUrl      string                     `koanf:"my-url"`
-	Signer     signature.SignVerifyConfig `koanf:"signer"`
+	MsgPerPoll          arbutil.MessageIndex       `koanf:"msg-per-poll"`
+	MyUrl               string                     `koanf:"my-url"`
+	DeleteFinalizedMsgs bool                       `koanf:"delete-finalized-msgs"`
+	Signer              signature.SignVerifyConfig `koanf:"signer"`
 }
 
 func (c *SeqCoordinatorConfig) Url() string {
@@ -95,6 +97,7 @@ func SeqCoordinatorConfigAddOptions(prefix string, f *flag.FlagSet) {
 	f.Int(prefix+".release-retries", DefaultSeqCoordinatorConfig.ReleaseRetries, "the number of times to retry releasing the wants lockout and chosen one status on shutdown")
 	f.Uint64(prefix+".msg-per-poll", uint64(DefaultSeqCoordinatorConfig.MsgPerPoll), "will only be marked as wanting the lockout if not too far behind")
 	f.String(prefix+".my-url", DefaultSeqCoordinatorConfig.MyUrl, "url for this sequencer if it is the chosen")
+	f.Bool(prefix+".delete-finalized-msgs", DefaultSeqCoordinatorConfig.DeleteFinalizedMsgs, "enable deleting of finalized messages from redis")
 	signature.SignVerifyConfigAddOptions(prefix+".signer", f)
 }
 
@@ -104,31 +107,33 @@ var DefaultSeqCoordinatorConfig = SeqCoordinatorConfig{
 	RedisUrl:              "",
 	LockoutDuration:       time.Minute,
 	LockoutSpare:          30 * time.Second,
-	SeqNumDuration:        24 * time.Hour,
+	SeqNumDuration:        10 * 24 * time.Hour,
 	UpdateInterval:        250 * time.Millisecond,
 	HandoffTimeout:        30 * time.Second,
 	SafeShutdownDelay:     5 * time.Second,
 	ReleaseRetries:        4,
 	RetryInterval:         50 * time.Millisecond,
 	MsgPerPoll:            2000,
 	MyUrl:                 redisutil.INVALID_URL,
+	DeleteFinalizedMsgs:   true,
 	Signer:                signature.DefaultSignVerifyConfig,
 }
 
 var TestSeqCoordinatorConfig = SeqCoordinatorConfig{
-	Enable:            false,
-	RedisUrl:          "",
-	LockoutDuration:   time.Second * 2,
-	LockoutSpare:      time.Millisecond * 10,
-	SeqNumDuration:    time.Minute * 10,
-	UpdateInterval:    time.Millisecond * 10,
-	HandoffTimeout:    time.Millisecond * 200,
-	SafeShutdownDelay: time.Millisecond * 100,
-	ReleaseRetries:    4,
-	RetryInterval:     time.Millisecond * 3,
-	MsgPerPoll:        20,
-	MyUrl:             redisutil.INVALID_URL,
-	Signer:            signature.DefaultSignVerifyConfig,
+	Enable:              false,
+	RedisUrl:            "",
+	LockoutDuration:     time.Second * 2,
+	LockoutSpare:        time.Millisecond * 10,
+	SeqNumDuration:      time.Minute * 10,
+	UpdateInterval:      time.Millisecond * 10,
+	HandoffTimeout:      time.Millisecond * 200,
+	SafeShutdownDelay:   time.Millisecond * 100,
+	ReleaseRetries:      4,
+	RetryInterval:       time.Millisecond * 3,
+	MsgPerPoll:          20,
+	MyUrl:               redisutil.INVALID_URL,
+	DeleteFinalizedMsgs: true,
+	Signer:              signature.DefaultSignVerifyConfig,
 }
 
 func NewSeqCoordinator(
@@ -149,6 +154,7 @@ func NewSeqCoordinator(
 	}
 	coordinator := &SeqCoordinator{
 		RedisCoordinator: *redisCoordinator,
+		sync:             sync,
 		streamer:         streamer,
 		sequencer:        sequencer,
 		config:           config,
@@ -338,6 +344,14 @@ func (c *SeqCoordinator) acquireLockoutAndWriteMessage(ctx context.Context, msgC
 	return nil
 }
 
+func (c *SeqCoordinator) getRemoteFinalizedMsgCount(ctx context.Context) (arbutil.MessageIndex, error) {
+	resStr, err := c.Client.Get(ctx, redisutil.FINALIZED_MSG_COUNT_KEY).Result()
+	if err != nil {
+		return 0, err
+	}
+	return c.signedBytesToMsgCount(ctx, []byte(resStr))
+}
+
 func (c *SeqCoordinator) getRemoteMsgCountImpl(ctx context.Context, r redis.Cmdable) (arbutil.MessageIndex, error) {
 	resStr, err := r.Get(ctx, redisutil.MSG_COUNT_KEY).Result()
 	if errors.Is(err, redis.Nil) {
@@ -473,6 +487,17 @@ func (c *SeqCoordinator) updateWithLockout(ctx context.Context, nextChosen strin
 		return c.noRedisError()
 	}
 	// Was, and still is, the active sequencer
+	if c.config.DeleteFinalizedMsgs {
+		// Before proceeding, first try deleting finalized messages from redis and setting the finalizedMsgCount key
+		finalized, err := c.sync.GetFinalizedMsgCount(ctx)
+		if err != nil {
+			log.Warn("Error getting finalizedMessageCount from syncMonitor: %w", err)
+		} else if finalized == 0 {
+			log.Warn("SyncMonitor returned zero finalizedMessageCount")
+		} else if err := c.deleteFinalizedMsgsFromRedis(ctx, finalized); err != nil {
+			log.Warn("Coordinator failed to delete finalized messages from redis", "err", err)
+		}
+	}
 	// We leave a margin of error of either a five times the update interval or a fifth of the lockout duration, whichever is greater.
 	marginOfError := arbmath.MaxInt(c.config.LockoutDuration/5, c.config.UpdateInterval*5)
 	if time.Now().Add(marginOfError).Before(atomicTimeRead(&c.lockoutUntil)) {
@@ -492,6 +517,62 @@ func (c *SeqCoordinator) updateWithLockout(ctx context.Context, nextChosen strin
 	return c.noRedisError()
 }
 
+func (c *SeqCoordinator) deleteFinalizedMsgsFromRedis(ctx context.Context, finalized arbutil.MessageIndex) error {
+	deleteMsgsAndUpdateFinalizedMsgCount := func(keys []string) error {
+		if len(keys) > 0 {
+			// To support cases during init we delete keys from reverse (i.e lowest seq num first), so that even if deletion fails in one of the iterations
+			// next time deleteFinalizedMsgsFromRedis is called we dont miss undeleted messages, as exists is checked from higher seqnum to lower.
+			// In non-init cases it doesn't matter how we delete as we always try to delete from prevFinalized to finalized
+			batchDeleteCount := 1000
+			for i := len(keys); i > 0; i -= batchDeleteCount {
+				if err := c.Client.Del(ctx, keys[max(0, i-batchDeleteCount):i]...).Err(); err != nil {
+					return fmt.Errorf("error deleting finalized messages and their signatures from redis: %w", err)
+				}
+			}
+		}
+		finalizedBytes, err := c.msgCountToSignedBytes(finalized)
+		if err != nil {
+			return err
+		}
+		if err = c.Client.Set(ctx, redisutil.FINALIZED_MSG_COUNT_KEY, finalizedBytes, c.config.SeqNumDuration).Err(); err != nil {
+			return fmt.Errorf("couldn't set %s key to current finalizedMsgCount in redis: %w", redisutil.FINALIZED_MSG_COUNT_KEY, err)
+		}
+		return nil
+	}
+	prevFinalized, err := c.getRemoteFinalizedMsgCount(ctx)
+	if errors.Is(err, redis.Nil) {
+		var keys []string
+		for msg := finalized - 1; msg > 0; msg-- {
+			exists, err := c.Client.Exists(ctx, redisutil.MessageKeyFor(msg), redisutil.MessageSigKeyFor(msg)).Result()
+			if err != nil {
+				// If there is an error deleting finalized messages during init, we retry later either from this sequencer or from another
+				return err
+			}
+			if exists == 0 {
+				break
+			}
+			keys = append(keys, redisutil.MessageKeyFor(msg), redisutil.MessageSigKeyFor(msg))
+		}
+		log.Info("Initializing finalizedMsgCount and deleting finalized messages from redis", "finalizedMsgCount", finalized)
+		return deleteMsgsAndUpdateFinalizedMsgCount(keys)
+	} else if err != nil {
+		return fmt.Errorf("error getting finalizedMsgCount value from redis: %w", err)
+	}
+	remoteMsgCount, err := c.getRemoteMsgCountImpl(ctx, c.Client)
+	if err != nil {
+		return fmt.Errorf("cannot get remote message count: %w", err)
+	}
+	msgToDelete := min(finalized, remoteMsgCount)
+	if prevFinalized < msgToDelete {
+		var keys []string
+		for msg := prevFinalized; msg < msgToDelete; msg++ {
+			keys = append(keys, redisutil.MessageKeyFor(msg), redisutil.MessageSigKeyFor(msg))
+		}
+		return deleteMsgsAndUpdateFinalizedMsgCount(keys)
+	}
+	return nil
+}
+
 func (c *SeqCoordinator) update(ctx context.Context) time.Duration {
 	chosenSeq, err := c.RecommendSequencerWantingLockout(ctx)
 	if err != nil {
@@ -522,19 +603,24 @@ func (c *SeqCoordinator) update(ctx context.Context) time.Duration {
 		log.Error("cannot read message count", "err", err)
 		return c.config.UpdateInterval
 	}
+	remoteFinalizedMsgCount, err := c.getRemoteFinalizedMsgCount(ctx)
+	if err != nil {
+		loglevel := log.Error
+		if errors.Is(err, redis.Nil) {
+			loglevel = log.Debug
+		}
+		loglevel("Cannot get remote finalized message count, might encounter failed to read message warnings later", "err", err)
+	}
 	remoteMsgCount, err := c.GetRemoteMsgCount()
 	if err != nil {
 		log.Warn("cannot get remote message count", "err", err)
 		return c.retryAfterRedisError()
 	}
-	readUntil := remoteMsgCount
-	if readUntil > localMsgCount+c.config.MsgPerPoll {
-		readUntil = localMsgCount + c.config.MsgPerPoll
-	}
+	readUntil := min(localMsgCount+c.config.MsgPerPoll, remoteMsgCount)
 	var messages []arbostypes.MessageWithMetadata
 	msgToRead := localMsgCount
 	var msgReadErr error
-	for msgToRead < readUntil {
+	for msgToRead < readUntil && localMsgCount >= remoteFinalizedMsgCount {
 		var resString string
 		resString, msgReadErr = c.Client.Get(ctx, redisutil.MessageKeyFor(msgToRead)).Result()
 		if msgReadErr != nil {