Merge pull request #13 from EspressoSystems/batch-validation

Add justification to batches, progress on validation pipeline

arbitrator/jit/src/machine.rs

@@ -107,6 +107,7 @@ pub fn create(opts: &Opts, env: WasmEnv) -> (Instance, FunctionEnv<WasmEnv>, Sto
             "github.com/offchainlabs/nitro/wavmio.getGlobalStateU64" => func!(wavmio::get_global_state_u64),
             "github.com/offchainlabs/nitro/wavmio.setGlobalStateU64" => func!(wavmio::set_global_state_u64),
             "github.com/offchainlabs/nitro/wavmio.readInboxMessage" => func!(wavmio::read_inbox_message),
+            "github.com/offchainlabs/nitro/wavmio.readHotShotCommitment" => func!(wavmio::read_hotshot_commitment),
             "github.com/offchainlabs/nitro/wavmio.readDelayedInboxMessage" => func!(wavmio::read_delayed_inbox_message),
             "github.com/offchainlabs/nitro/wavmio.resolvePreImage" => {
                 #[allow(deprecated)] // we're just keeping this around until we no longer need to validate old replay binaries
@@ -184,6 +185,7 @@ impl From<RuntimeError> for Escape {
 
 pub type WasmEnvMut<'a> = FunctionEnvMut<'a, WasmEnv>;
 pub type Inbox = BTreeMap<u64, Vec<u8>>;
+pub type HotShotCommitmentMap = BTreeMap<u64, [u8; 32]>;
 pub type Preimages = BTreeMap<PreimageType, BTreeMap<[u8; 32], Vec<u8>>>;
 
 #[derive(Default)]
@@ -202,6 +204,8 @@ pub struct WasmEnv {
     pub preimages: Preimages,
     /// The sequencer inbox's messages
     pub sequencer_messages: Inbox,
+    /// Mapping from batch positions to hotshot commitments
+    pub hotshot_comm_map: HotShotCommitmentMap,
     /// The delayed inbox's messages
     pub delayed_messages: Inbox,
     /// The purpose and connections of this process

arbitrator/jit/src/wavmio.rs

@@ -3,7 +3,7 @@
 
 use crate::{
     gostack::GoStack,
-    machine::{Escape, Inbox, MaybeEscape, WasmEnv, WasmEnvMut},
+    machine::{Escape, HotShotCommitmentMap, Inbox, MaybeEscape, WasmEnv, WasmEnvMut},
     socket,
 };
 
@@ -85,6 +85,14 @@ pub fn set_global_state_u64(mut env: WasmEnvMut, sp: u32) -> MaybeEscape {
     Ok(())
 }
 
+pub fn read_hotshot_commitment(mut env: WasmEnvMut, sp: u32) -> MaybeEscape {
+    let (sp, env) = GoStack::new(sp, &mut env);
+    ready_hostio(env)?;
+    let hotshot_comms = &env.hotshot_comm_map;
+
+    read_hotshot_commitment_impl(&sp, hotshot_comms, "wavmio.readHotShotCommitment")
+}
+
 pub fn read_inbox_message(mut env: WasmEnvMut, sp: u32) -> MaybeEscape {
     let (sp, env) = GoStack::new(sp, &mut env);
     ready_hostio(env)?;
@@ -101,6 +109,32 @@ pub fn read_delayed_inbox_message(mut env: WasmEnvMut, sp: u32) -> MaybeEscape {
     inbox_message_impl(&sp, inbox, "wavmio.readDelayedInboxMessage")
 }
 
+// Reads a hotshot commitment
+fn read_hotshot_commitment_impl(
+    sp: &GoStack,
+    comm_map: &HotShotCommitmentMap,
+    name: &str,
+) -> MaybeEscape {
+    let msg_num = sp.read_u64(0);
+    let out_ptr = sp.read_u64(1);
+    let out_len = sp.read_u64(2);
+    if out_len != 32 {
+        eprintln!("Go trying to read header bytes with out len {out_len} in {name}");
+        sp.write_u64(5, 0);
+        return Ok(());
+    }
+
+    let message = comm_map.get(&msg_num).unwrap_or(&[0; 32]);
+
+    if out_ptr + 32 > sp.memory_size() {
+        let text = format!("memory bounds exceeded in {}", name);
+        return Escape::hostio(&text);
+    }
+    sp.write_slice(out_ptr, message);
+    sp.write_u64(5, 32);
+    Ok(())
+}
+
 /// Reads an inbox message
 /// note: the order of the checks is very important.
 fn inbox_message_impl(sp: &GoStack, inbox: &Inbox, name: &str) -> MaybeEscape {
@@ -273,7 +307,9 @@ fn ready_hostio(env: &mut WasmEnv) -> MaybeEscape {
     while socket::read_u8(stream)? == socket::ANOTHER {
         let position = socket::read_u64(stream)?;
         let message = socket::read_bytes(stream)?;
+        let hotshot_comm = socket::read_bytes32(stream)?;
         env.sequencer_messages.insert(position, message);
+        env.hotshot_comm_map.insert(position, hotshot_comm);
     }
     while socket::read_u8(stream)? == socket::ANOTHER {
         let position = socket::read_u64(stream)?;

arbitrator/prover/src/host.rs

@@ -52,6 +52,7 @@ pub enum Hostio {
     WavmReadKeccakPreimage,
     WavmReadSha256Preimage,
     WavmReadInboxMessage,
+    WavmReadHotShotCommitment,
     WavmReadDelayedInboxMessage,
     WavmHaltAndSetFinished,
 }
@@ -76,6 +77,7 @@ impl FromStr for Hostio {
             ("env", "wavm_read_sha2_256_preimage") => WavmReadSha256Preimage,
             ("env", "wavm_read_inbox_message") => WavmReadInboxMessage,
             ("env", "wavm_read_delayed_inbox_message") => WavmReadDelayedInboxMessage,
+            ("env", "wavm_read_hotshot_header") => WavmReadHotShotCommitment,
             ("env", "wavm_halt_and_set_finished") => WavmHaltAndSetFinished,
             _ => bail!("no such hostio {} in {}", name.red(), module.red()),
         })
@@ -109,6 +111,7 @@ impl Hostio {
             WavmSetGlobalStateBytes32   => func!([I32, I32]),
             WavmGetGlobalStateU64       => func!([I32], [I64]),
             WavmSetGlobalStateU64       => func!([I32, I64]),
+            WavmReadHotShotCommitment => func!([I32, I32]),
             WavmReadKeccakPreimage      => func!([I32, I32], [I32]),
             WavmReadSha256Preimage      => func!([I32, I32], [I32]),
             WavmReadInboxMessage        => func!([I64, I32, I32], [I32]),
@@ -180,6 +183,11 @@ impl Hostio {
                 opcode!(LocalGet, 1);
                 opcode!(ReadPreImage, PreimageType::Sha2_256);
             }
+            WavmReadHotShotCommitment => {
+                // TODO implement for fault proofs
+                // https://github.com/EspressoSystems/espresso-sequencer/issues/670
+                unimplemented!()
+            }
             WavmReadInboxMessage => {
                 opcode!(LocalGet, 0);
                 opcode!(LocalGet, 1);

arbitrator/wasm-libraries/host-io/src/lib.rs

@@ -71,6 +71,15 @@ pub unsafe extern "C" fn go__github_com_offchainlabs_nitro_wavmio_setGlobalState
     wavm_set_globalstate_u64(idx, sp.read_u64(1));
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn go__github_com_offchainlabs_nitro_wavmio_readHotShotCommitment(
+    sp: GoStack,
+) {
+    // TODO implement for fault proofs
+    // https://github.com/EspressoSystems/espresso-sequencer/issues/671
+    return unimplemented!();
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn go__github_com_offchainlabs_nitro_wavmio_readInboxMessage(sp: GoStack) {
     let msg_num = sp.read_u64(0);
@@ -120,7 +129,9 @@ pub unsafe extern "C" fn go__github_com_offchainlabs_nitro_wavmio_readDelayedInb
 }
 
 #[no_mangle]
-pub unsafe extern "C" fn go__github_com_offchainlabs_nitro_wavmio_resolveTypedPreimage(sp: GoStack) {
+pub unsafe extern "C" fn go__github_com_offchainlabs_nitro_wavmio_resolveTypedPreimage(
+    sp: GoStack,
+) {
     let preimage_type = sp.read_u8(0);
     let hash_ptr = sp.read_u64(1);
     let hash_len = sp.read_u64(2);

arbos/arbostypes/incomingmessage.go

@@ -15,6 +15,7 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
 
+	"github.com/offchainlabs/nitro/arbos/espresso"
 	"github.com/offchainlabs/nitro/arbos/util"
 	"github.com/offchainlabs/nitro/util/arbmath"
 )
@@ -34,13 +35,19 @@ const (
 
 const MaxL2MessageSize = 256 * 1024
 
+type EspressoBlockJustification struct {
+	Header espresso.Header
+	Proof  espresso.NmtProof
+}
+
 type L1IncomingMessageHeader struct {
-	Kind        uint8          `json:"kind"`
-	Poster      common.Address `json:"sender"`
-	BlockNumber uint64         `json:"blockNumber"`
-	Timestamp   uint64         `json:"timestamp"`
-	RequestId   *common.Hash   `json:"requestId" rlp:"nilList"`
-	L1BaseFee   *big.Int       `json:"baseFeeL1"`
+	Kind               uint8                       `json:"kind"`
+	Poster             common.Address              `json:"sender"`
+	BlockNumber        uint64                      `json:"blockNumber"`
+	Timestamp          uint64                      `json:"timestamp"`
+	RequestId          *common.Hash                `json:"requestId" rlp:"nilList"`
+	L1BaseFee          *big.Int                    `json:"baseFeeL1"`
+	BlockJustification *EspressoBlockJustification `json:"justification,omitempty" rlp:"optional"`
 }
 
 func (h L1IncomingMessageHeader) SeqNum() (uint64, error) {
@@ -228,6 +235,7 @@ func ParseIncomingL1Message(rd io.Reader, batchFetcher FallibleBatchFetcher) (*L
 			timestamp,
 			&requestId,
 			baseFeeL1.Big(),
+			nil,
 		},
 		data,
 		nil,

arbos/block_processor.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/offchainlabs/nitro/arbos/arbosState"
 	"github.com/offchainlabs/nitro/arbos/arbostypes"
+	"github.com/offchainlabs/nitro/arbos/espresso"
 	"github.com/offchainlabs/nitro/arbos/l2pricing"
 	"github.com/offchainlabs/nitro/arbos/util"
 	"github.com/offchainlabs/nitro/solgen/go/precompilesgen"
@@ -129,6 +130,7 @@ func ProduceBlock(
 	message *arbostypes.L1IncomingMessage,
 	delayedMessagesRead uint64,
 	lastBlockHeader *types.Header,
+	lastHotShotCommitment *espresso.Commitment,
 	statedb *state.StateDB,
 	chainContext core.ChainContext,
 	chainConfig *params.ChainConfig,
@@ -158,7 +160,7 @@ func ProduceBlock(
 
 	hooks := NoopSequencingHooks()
 	return ProduceBlockAdvanced(
-		message.Header, txes, delayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, hooks,
+		message.Header, txes, delayedMessagesRead, lastBlockHeader, lastHotShotCommitment, statedb, chainContext, chainConfig, hooks,
 	)
 }
 
@@ -168,6 +170,7 @@ func ProduceBlockAdvanced(
 	txes types.Transactions,
 	delayedMessagesRead uint64,
 	lastBlockHeader *types.Header,
+	lastHotShotCommitment *espresso.Commitment,
 	statedb *state.StateDB,
 	chainContext core.ChainContext,
 	chainConfig *params.ChainConfig,
@@ -191,6 +194,29 @@ func ProduceBlockAdvanced(
 		l1Timestamp:   l1Header.Timestamp,
 	}
 
+	// Espresso-specific validation
+	// TODO test: https://github.com/EspressoSystems/espresso-sequencer/issues/772
+	if chainConfig.Espresso {
+		jst := l1Header.BlockJustification
+		if jst == nil {
+			return nil, nil, errors.New("batch missing espresso justification")
+
+		}
+		hotshotHeader := jst.Header
+		if *lastHotShotCommitment != hotshotHeader.Commit() {
+			return nil, nil, errors.New("invalid hotshot header")
+		}
+		var roots = []*espresso.NmtRoot{&hotshotHeader.TransactionsRoot}
+		var proofs = []*espresso.NmtProof{&l1Header.BlockJustification.Proof}
+		// If the validation function below were not mocked, we would need to serialize the transactions
+		// in the batch here. To avoid the unnecessary overhead, we provide an empty array instead.
+		var txs []espresso.Bytes
+		err := espresso.ValidateBatchTransactions(chainConfig.ChainID.Uint64(), roots, proofs, txs)
+		if err != nil {
+			return nil, nil, errors.New("failed to validate namespace proof)")
+		}
+	}
+
 	header := createNewHeader(lastBlockHeader, l1Info, state, chainConfig)
 	signer := types.MakeSigner(chainConfig, header.Number, header.Time)
 	// Note: blockGasLeft will diverge from the actual gas left during execution in the event of invalid txs,

cmd/replay/main.go

@@ -25,6 +25,7 @@ import (
 	"github.com/offchainlabs/nitro/arbos/arbosState"
 	"github.com/offchainlabs/nitro/arbos/arbostypes"
 	"github.com/offchainlabs/nitro/arbos/burn"
+	"github.com/offchainlabs/nitro/arbos/espresso"
 	"github.com/offchainlabs/nitro/arbstate"
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/cmd/chaininfo"
@@ -46,6 +47,12 @@ func getBlockHeaderByHash(hash common.Hash) *types.Header {
 	return header
 }
 
+func getHotShotCommitment(seqNum uint64) *espresso.Commitment {
+	headerBytes := espresso.Commitment(wavmio.ReadHotShotCommitment(seqNum))
+	log.Info("HotShot commitment", "commit", headerBytes)
+	return &headerBytes
+}
+
 type WavmChainContext struct{}
 
 func (c WavmChainContext) Engine() consensus.Engine {
@@ -238,7 +245,12 @@ func main() {
 		batchFetcher := func(batchNum uint64) ([]byte, error) {
 			return wavmio.ReadInboxMessage(batchNum), nil
 		}
-		newBlock, _, err = arbos.ProduceBlock(message.Message, message.DelayedMessagesRead, lastBlockHeader, statedb, chainContext, chainConfig, batchFetcher)
+		seqNum := wavmio.GetInboxPosition()
+		var hotShotCommitment *espresso.Commitment
+		if chainConfig.Espresso {
+			hotShotCommitment = getHotShotCommitment(seqNum)
+		}
+		newBlock, _, err = arbos.ProduceBlock(message.Message, message.DelayedMessagesRead, lastBlockHeader, hotShotCommitment, statedb, chainContext, chainConfig, batchFetcher)
 		if err != nil {
 			panic(err)
 		}

execution/gethexec/block_recorder.go

@@ -140,6 +140,7 @@ func (r *BlockRecorder) RecordBlockCreation(
 			msg.Message,
 			msg.DelayedMessagesRead,
 			prevHeader,
+			nil,
 			recordingdb,
 			chaincontext,
 			chainConfig,

execution/gethexec/espresso_sequencer.go

@@ -8,7 +8,7 @@ import (
 	"encoding/json"
 	"time"
 
-	"github.com/offchainlabs/nitro/espresso"
+	"github.com/offchainlabs/nitro/arbos/espresso"
 	"github.com/offchainlabs/nitro/util/stopwaiter"
 
 	"github.com/ethereum/go-ethereum/arbitrum_types"
@@ -108,7 +108,10 @@ func (s *EspressoSequencer) createBlock(ctx context.Context) (returnValue bool)
 		Timestamp:   header.Timestamp,
 		RequestId:   nil,
 		L1BaseFee:   nil,
-		// TODO: add justification https://github.com/EspressoSystems/espresso-sequencer/issues/733
+		BlockJustification: &arbostypes.EspressoBlockJustification{
+			Header: header,
+			Proof:  arbTxns.Proof,
+		},
 	}
 
 	hooks := s.makeSequencingHooks()

execution/gethexec/executionengine.go

@@ -17,6 +17,7 @@ import (
 	"github.com/offchainlabs/nitro/arbos"
 	"github.com/offchainlabs/nitro/arbos/arbosState"
 	"github.com/offchainlabs/nitro/arbos/arbostypes"
+	"github.com/offchainlabs/nitro/arbos/espresso"
 	"github.com/offchainlabs/nitro/arbos/l1pricing"
 	"github.com/offchainlabs/nitro/arbutil"
 	"github.com/offchainlabs/nitro/execution"
@@ -287,13 +288,19 @@ func (s *ExecutionEngine) sequenceTransactionsWithBlockMutex(header *arbostypes.
 	}
 
 	delayedMessagesRead := lastBlockHeader.Nonce.Uint64()
+	jst := header.BlockJustification
+	var hotShotHeader espresso.Commitment
+	if jst != nil {
+		hotShotHeader = jst.Header.Commit()
+	}
 
 	startTime := time.Now()
 	block, receipts, err := arbos.ProduceBlockAdvanced(
 		header,
 		txes,
 		delayedMessagesRead,
 		lastBlockHeader,
+		&hotShotHeader,
 		statedb,
 		s.bc,
 		s.bc.Config(),
@@ -441,11 +448,17 @@ func (s *ExecutionEngine) createBlockFromNextMessage(msg *arbostypes.MessageWith
 	}
 	statedb.StartPrefetcher("TransactionStreamer")
 	defer statedb.StopPrefetcher()
+	jst := msg.Message.Header.BlockJustification
+	var hotShotCommitment espresso.Commitment
+	if jst != nil {
+		hotShotCommitment = jst.Header.Commit()
+	}
 
 	block, receipts, err := arbos.ProduceBlock(
 		msg.Message,
 		msg.DelayedMessagesRead,
 		currentHeader,
+		&hotShotCommitment,
 		statedb,
 		s.bc,
 		s.bc.Config(),

scripts/espresso-ci-local

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+make lint
+
+make test-go
+make test-go-challenge
+
+container=$(docker run -d -p 6379:6379 redis)
+trap "exit" INT TERM
+trap "docker rm -f $container" EXIT
+
+make test-go-redis