random beacon cache changes (#1278)

* update the randomness commitment cache when inserting blocks into the blockchain

* added comments

* reverse iteration search for the commitment cache entry

* unsubscribe from the newblock subscription

* fixed bug and added comments

* handled randomness beacon case when smart contracts are being migrated

* fixed bug

* fixed a bug

* fixed a bug and added comments to non intuitive code

* addressed from PR comments

* addressed PR comments

Co-authored-by: Kevin Jue <[email protected]>

consensus/consensus.go

@@ -201,12 +201,16 @@ type Istanbul interface {
 	// StopProxiedValidatorEngine stops the proxied validator engine
 	StopProxiedValidatorEngine() error
 
-	// This is only implemented for Istanbul.
-	// It will update the validator set diff in the header, if the mined header is the last block of the epoch.
+	// UpdateValSetDiff will update the validator set diff in the header, if the mined header is the last block of the epoch.
 	// The changes are executed inline.
 	UpdateValSetDiff(chain ChainReader, header *types.Header, state *state.StateDB) error
 
-	// This is only implemented for Istanbul.
-	// It will check to see if the header is from the last block of an epoch
+	// IsLastBlockOfEpoch will check to see if the header is from the last block of an epoch
 	IsLastBlockOfEpoch(header *types.Header) bool
+
+	// ValidatorAddress will return the istanbul engine's validator address
+	ValidatorAddress() common.Address
+
+	// GenerateRandomness will generate the random beacon randomness
+	GenerateRandomness(parentHash common.Hash) (common.Hash, common.Hash, error)
 }

consensus/istanbul/backend/backend.go

@@ -163,13 +163,14 @@ type Backend struct {
 	config           *istanbul.Config
 	istanbulEventMux *event.TypeMux
 
-	address    common.Address       // Ethereum address of the ECDSA signing key
-	blsAddress common.Address       // Ethereum address of the BLS signing key
-	publicKey  *ecdsa.PublicKey     // The signer public key
-	decryptFn  istanbul.DecryptFn   // Decrypt function to decrypt ECIES ciphertext
-	signFn     istanbul.SignerFn    // Signer function to authorize hashes with
-	signBLSFn  istanbul.BLSSignerFn // Signer function to authorize BLS messages
-	signFnMu   sync.RWMutex         // Protects the signer fields
+	address    common.Address        // Ethereum address of the ECDSA signing key
+	blsAddress common.Address        // Ethereum address of the BLS signing key
+	publicKey  *ecdsa.PublicKey      // The signer public key
+	decryptFn  istanbul.DecryptFn    // Decrypt function to decrypt ECIES ciphertext
+	signFn     istanbul.SignerFn     // Signer function to authorize hashes with
+	signBLSFn  istanbul.BLSSignerFn  // Signer function to authorize BLS messages
+	signHashFn istanbul.HashSignerFn // Signer function to create random seed
+	signFnMu   sync.RWMutex          // Protects the signer fields
 
 	core         istanbulCore.Engine
 	logger       log.Logger
@@ -291,6 +292,10 @@ type Backend struct {
 	proxiedValidatorEngine        proxy.ProxiedValidatorEngine
 	proxiedValidatorEngineRunning bool
 	proxiedValidatorEngineMu      sync.RWMutex
+
+	// RandomSeed (and it's mutex) used to generate the random beacon randomness
+	randomSeed   []byte
+	randomSeedMu sync.Mutex
 }
 
 // IsProxy returns true if instance has proxy flag
@@ -349,7 +354,7 @@ func (sb *Backend) SendDelegateSignMsgToProxiedValidator(msg []byte) error {
 }
 
 // Authorize implements istanbul.Backend.Authorize
-func (sb *Backend) Authorize(ecdsaAddress, blsAddress common.Address, publicKey *ecdsa.PublicKey, decryptFn istanbul.DecryptFn, signFn istanbul.SignerFn, signBLSFn istanbul.BLSSignerFn) {
+func (sb *Backend) Authorize(ecdsaAddress, blsAddress common.Address, publicKey *ecdsa.PublicKey, decryptFn istanbul.DecryptFn, signFn istanbul.SignerFn, signBLSFn istanbul.BLSSignerFn, signHashFn istanbul.HashSignerFn) {
 	sb.signFnMu.Lock()
 	defer sb.signFnMu.Unlock()
 
@@ -359,6 +364,7 @@ func (sb *Backend) Authorize(ecdsaAddress, blsAddress common.Address, publicKey
 	sb.decryptFn = decryptFn
 	sb.signFn = signFn
 	sb.signBLSFn = signBLSFn
+	sb.signHashFn = signHashFn
 	sb.core.SetAddress(ecdsaAddress)
 }
 

consensus/istanbul/backend/random.go

@@ -0,0 +1,69 @@
+// Copyright 2017 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package backend
+
+import (
+	"github.com/ethereum/go-ethereum/accounts"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/contract_comm/random"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// String for creating the random seed
+var randomSeedString = []byte("Randomness seed string")
+
+// GenerateRandomness will generate the random beacon randomness
+func (sb *Backend) GenerateRandomness(parentHash common.Hash) (common.Hash, common.Hash, error) {
+	logger := sb.logger.New("func", "GenerateRandomness")
+
+	if !random.IsRunning() {
+		return common.Hash{}, common.Hash{}, nil
+	}
+
+	sb.randomSeedMu.Lock()
+	if sb.randomSeed == nil {
+		var err error
+		sb.randomSeed, err = sb.signHashFn(accounts.Account{Address: sb.address}, common.BytesToHash(randomSeedString).Bytes())
+		if err != nil {
+			logger.Error("Failed to create randomSeed", "err", err)
+			sb.randomSeedMu.Unlock()
+			return common.Hash{}, common.Hash{}, err
+		}
+	}
+	sb.randomSeedMu.Unlock()
+
+	randomness := crypto.Keccak256Hash(append(sb.randomSeed, parentHash.Bytes()...))
+
+	// Retrieve the head block's header and state.
+	// The logic to compute the commitment via the randomness is in the random smart contract.
+	// That logic is stateless, so passing in any block header and state is fine.  There is a TODO for
+	// that commitment computation logic to be removed fromthe random smart contract.
+	currentBlock := sb.currentBlock()
+	currentState, err := sb.stateAt(currentBlock.Hash())
+	if err != nil {
+		logger.Error("Failed to retrieve current state", "err", err)
+		return common.Hash{}, common.Hash{}, err
+	}
+
+	commitment, err := random.ComputeCommitment(currentBlock.Header(), currentState, randomness)
+	if err != nil {
+		logger.Error("Failed to compute commitment", "err", err)
+		return common.Hash{}, common.Hash{}, err
+	}
+
+	return randomness, commitment, nil
+}

consensus/istanbul/backend/snapshot_test.go

@@ -260,7 +260,7 @@ func TestValSetChange(t *testing.T) {
 		privateKey := accounts.accounts[tt.validators[0]]
 		address := crypto.PubkeyToAddress(privateKey.PublicKey)
 
-		engine.Authorize(address, address, &privateKey.PublicKey, DecryptFn(privateKey), SignFn(privateKey), SignBLSFn(privateKey))
+		engine.Authorize(address, address, &privateKey.PublicKey, DecryptFn(privateKey), SignFn(privateKey), SignBLSFn(privateKey), SignHashFn(privateKey))
 
 		chain.AddHeader(0, genesis.ToBlock(nil).Header())
 

consensus/istanbul/backend/test_utils.go

@@ -57,7 +57,8 @@ func newBlockChainWithKeys(isProxy bool, proxiedValAddress common.Address, isPro
 		decryptFn := DecryptFn(privateKey)
 		signerFn := SignFn(privateKey)
 		signerBLSFn := SignBLSFn(privateKey)
-		b.Authorize(address, address, &publicKey, decryptFn, signerFn, signerBLSFn)
+		signerHashFn := SignHashFn(privateKey)
+		b.Authorize(address, address, &publicKey, decryptFn, signerFn, signerBLSFn, signerHashFn)
 	} else {
 		proxyNodeKey, _ := crypto.GenerateKey()
 		publicKey = proxyNodeKey.PublicKey
@@ -301,6 +302,16 @@ func SignBLSFn(key *ecdsa.PrivateKey) istanbul.BLSSignerFn {
 	}
 }
 
+func SignHashFn(key *ecdsa.PrivateKey) istanbul.HashSignerFn {
+	if key == nil {
+		key, _ = generatePrivateKey()
+	}
+
+	return func(_ accounts.Account, data []byte) ([]byte, error) {
+		return crypto.Sign(data, key)
+	}
+}
+
 // this will return an aggregate sig by the BLS keys corresponding to the `keys` array over the
 // block's hash, on consensus round 0, without a composite hasher
 func signBlock(keys []*ecdsa.PrivateKey, block *types.Block) types.IstanbulAggregatedSeal {
@@ -346,7 +357,7 @@ func newBackend() (b *Backend) {
 
 	key, _ := generatePrivateKey()
 	address := crypto.PubkeyToAddress(key.PublicKey)
-	b.Authorize(address, address, &key.PublicKey, DecryptFn(key), SignFn(key), SignBLSFn(key))
+	b.Authorize(address, address, &key.PublicKey, DecryptFn(key), SignFn(key), SignBLSFn(key), SignHashFn(key))
 	return
 }
 

consensus/istanbul/types.go

@@ -44,6 +44,10 @@ type SignerFn func(accounts.Account, string, []byte) ([]byte, error)
 // backing account using BLS with a direct or composite hasher
 type BLSSignerFn func(accounts.Account, []byte, []byte, bool) (blscrypto.SerializedSignature, error)
 
+// HashSignerFn is a signer callback function to request a hash to be signed by a
+// backing account.
+type HashSignerFn func(accounts.Account, []byte) ([]byte, error)
+
 // UptimeEntry contains the uptime score of a validator during an epoch as well as the
 // last block they signed on
 type UptimeEntry struct {

consensus/istanbul/utils.go

@@ -216,3 +216,10 @@ func GetNodeID(enodeURL string) (*enode.ID, error) {
 	id := node.ID()
 	return &id, nil
 }
+
+// RandomnessCommitmentDBLocation will return the key for where the
+// given commitment's cached key-value entry
+func RandomnessCommitmentDBLocation(commitment common.Hash) []byte {
+	dbRandomnessPrefix := []byte("db-randomness-prefix")
+	return append(dbRandomnessPrefix, commitment.Bytes()...)
+}

contract_comm/random/random.go

@@ -7,13 +7,10 @@ import (
 	"github.com/ethereum/go-ethereum/accounts/abi"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
-	"github.com/ethereum/go-ethereum/consensus"
 	"github.com/ethereum/go-ethereum/contract_comm"
 	"github.com/ethereum/go-ethereum/contract_comm/errors"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/core/vm"
-	"github.com/ethereum/go-ethereum/crypto"
-	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/params"
 )
@@ -133,13 +130,8 @@ var (
 	randomFuncABI, _            = abi.JSON(strings.NewReader(randomAbi))
 	historicRandomFuncABI, _    = abi.JSON(strings.NewReader(historicRandomAbi))
 	zeroValue                   = common.Big0
-	dbRandomnessPrefix          = []byte("db-randomness-prefix")
 )
 
-func commitmentDbLocation(commitment common.Hash) []byte {
-	return append(dbRandomnessPrefix, commitment.Bytes()...)
-}
-
 func address() *common.Address {
 	randomAddress, err := contract_comm.GetRegisteredAddress(params.RandomRegistryId, nil, nil)
 	if err == errors.ErrSmartContractNotDeployed || err == errors.ErrRegistryContractNotDeployed {
@@ -155,11 +147,8 @@ func IsRunning() bool {
 	return randomAddress != nil && *randomAddress != common.ZeroAddress
 }
 
-// GetLastRandomness returns up the last randomness we committed to by first
-// looking up our last commitment in the smart contract, and then finding the
-// corresponding preimage in a (commitment => randomness) mapping we keep in the
-// database.
-func GetLastRandomness(coinbase common.Address, db *ethdb.Database, header *types.Header, state vm.StateDB, chain consensus.ChainReader, seed []byte) (common.Hash, error) {
+// GetLastCommitment returns up the last commitment in the smart contract
+func GetLastCommitment(coinbase common.Address, header *types.Header, state vm.StateDB) (common.Hash, error) {
 	lastCommitment := common.Hash{}
 	_, err := contract_comm.MakeStaticCall(params.RandomRegistryId, commitmentsFuncABI, "commitments", []interface{}{coinbase}, &lastCommitment, params.MaxGasForCommitments, header, state)
 	if err != nil {
@@ -169,41 +158,21 @@ func GetLastRandomness(coinbase common.Address, db *ethdb.Database, header *type
 
 	if (lastCommitment == common.Hash{}) {
 		log.Debug("Unable to find last randomness commitment in smart contract")
-		return common.Hash{}, nil
 	}
 
-	parentBlockHashBytes, err := (*db).Get(commitmentDbLocation(lastCommitment))
-	if err != nil {
-		log.Warn("Failed to get last block proposed from database", "commitment", lastCommitment.Hex(), "err", err)
-		parentBlockHash := header.ParentHash
-		for {
-			blockHeader := chain.GetHeaderByHash(parentBlockHash)
-			parentBlockHash = blockHeader.ParentHash
-			if blockHeader.Coinbase == coinbase {
-				break
-			}
-		}
-		parentBlockHashBytes = parentBlockHash.Bytes()
-	}
-	return crypto.Keccak256Hash(append(seed, parentBlockHashBytes...)), nil
+	return lastCommitment, nil
 }
 
-// GenerateNewRandomnessAndCommitment generates a new random number and a corresponding commitment.
-// The random number is stored in the database, keyed by the corresponding commitment.
-func GenerateNewRandomnessAndCommitment(header *types.Header, state vm.StateDB, db *ethdb.Database, seed []byte) (common.Hash, error) {
+// ComputeCommitment calulcates the commitment for a given randomness.
+func ComputeCommitment(header *types.Header, state vm.StateDB, randomness common.Hash) (common.Hash, error) {
 	commitment := common.Hash{}
-	randomness := crypto.Keccak256Hash(append(seed, header.ParentHash.Bytes()...))
 	// TODO(asa): Make an issue to not have to do this via StaticCall
 	_, err := contract_comm.MakeStaticCall(params.RandomRegistryId, computeCommitmentFuncABI, "computeCommitment", []interface{}{randomness}, &commitment, params.MaxGasForComputeCommitment, header, state)
 	if err != nil {
 		log.Error("Failed to call computeCommitment()", "err", err)
 		return common.Hash{}, err
 	}
 
-	err = (*db).Put(commitmentDbLocation(commitment), header.ParentHash.Bytes())
-	if err != nil {
-		log.Error("Failed to save last block parentHash to the database", "err", err)
-	}
 	return commitment, err
 }