Merge develop branch into v0.3.0-dev (ethereum#430)

* merge geth v1.10.15

* fix: Removed FastSync from cli server

* fix: TestHeadersRLPStorage

* Added t.skip(ETH2 in bor)

* fix: flow in create consensus engine

* bumped version

* Fix typo

* increase block time

* remove file

* bumped version

* merge gethv1.10.17

* bumped version

* fix failing tests

* Bump Go version to v1.18 (ethereum#368)

* Bump Go version to v1.18.1

* Build using netgo tag

This will create a static build using Go native networking stack.

Checked and it works stable for all archs and distros.

* Fix meta

* initial implementation for common ancestor approach

* extract whitelist interface

* fix types

* fix tests and format

* add unit tests for IsValidChain function

* more tests

* wip

* test ErrCheckpointMismatch

* minor fixes

* fix test

* dont panic

* fmt

* Limit state sync by gas

* Added logging for state-sync total gas usage

* Added number of event-records in log

* Minor Changes

* Minor Fix

* Adding individual gasUsed

* Minor Fix

* fix: return value for no remote block

* handle all errors

* modularise fake chain validator in downloader

* add more tests

* fix tests

* Modifying miner.recommit flag and its adjustment function. (ethereum#370)

* changed min/max/current recommit values

* Remove Hardcoded min/max

* Code Sanitization

* Skipping tests for constant recommit interval

* Adding default miner.recommit value

* Minor Change

* Increased default value of rpc.txfeecap to 5

* add debug rpc endpoints for checkpoint whitelist service

* minor fixes and enhancements

* avoid capping warnings for gas set by internal system transactions

* use typed mocks

* fix

* fix

* fix

* fix close

* fix

* Create stale.yml

* Fix bor consensus checkpoint bug

Co-authored-by: Arpit Temani <[email protected]>
Co-authored-by: Shivam Sharma <[email protected]>
Co-authored-by: Manav Darji <[email protected]>
Co-authored-by: Sandeep Sreenath <[email protected]>
Co-authored-by: Victor Castell <[email protected]>
Co-authored-by: Ferran <[email protected]>
Co-authored-by: Krishna Upadhyaya <[email protected]>
Co-authored-by: Karlo <[email protected]>
Co-authored-by: Sandeep Sreenath <[email protected]>
Co-authored-by: Jerry <[email protected]>

.github/workflows/stale.yml

@@ -0,0 +1,31 @@
+# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
+#
+# You can adjust the behavior by modifying this file.
+# For more information, see:
+# https://github.com/actions/stale
+name: Mark stale issues and pull requests
+
+on:
+  schedule:
+  - cron: '0 0 * * *'
+
+jobs:
+  stale:
+
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+
+    steps:
+    - uses: actions/stale@v5
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue is stale because it has been open 14 days with no activity. Remove stale label or comment or this will be closed in 14 days.'
+        stale-pr-message: 'This PR is stale because it has been open 21 days with no activity. Remove stale label or comment or this will be closed in 14 days.'
+        close-issue-message: 'This issue was closed because it has been stalled for 28 days with no activity.'
+        close-pr-message: 'This PR was closed because it has been stalled for 35 days with no activity.'
+        days-before-issue-stale: 14
+        days-before-pr-stale: 21
+        days-before-issue-close: 14
+        days-before-pr-close: 14

consensus/bor/bor.go

@@ -1107,6 +1107,8 @@ func (c *Bor) CommitStates(
 		lastStateID++
 	}
 
+	log.Info("StateSyncData", "Gas", totalGas, "Block-number", number, "LastStateID", lastStateID, "TotalRecords", len(eventRecords))
+
 	return stateSyncs, nil
 }
 

consensus/bor/heimdall.go

@@ -2,12 +2,14 @@ package bor
 
 import (
 	"github.com/ethereum/go-ethereum/consensus/bor/clerk"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint"
 	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
 )
 
 //go:generate mockgen -destination=../../tests/bor/mocks/IHeimdallClient.go -package=mocks . IHeimdallClient
 type IHeimdallClient interface {
 	StateSyncEvents(fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error)
 	Span(spanID uint64) (*span.HeimdallSpan, error)
+	FetchLatestCheckpoint() (*checkpoint.Checkpoint, error)
 	Close()
 }

consensus/bor/heimdall/checkpoint/checkpoint.go

@@ -0,0 +1,22 @@
+package checkpoint
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// Checkpoint defines a response object type of bor checkpoint
+type Checkpoint struct {
+	Proposer   common.Address `json:"proposer"`
+	StartBlock *big.Int       `json:"start_block"`
+	EndBlock   *big.Int       `json:"end_block"`
+	RootHash   common.Hash    `json:"root_hash"`
+	BorChainID string         `json:"bor_chain_id"`
+	Timestamp  uint64         `json:"timestamp"`
+}
+
+type CheckpointResponse struct {
+	Height string     `json:"height"`
+	Result Checkpoint `json:"result"`
+}

consensus/bor/heimdall/client.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/consensus/bor/clerk"
+	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint"
 	"github.com/ethereum/go-ethereum/consensus/bor/heimdall/span"
 	"github.com/ethereum/go-ethereum/log"
 )
@@ -53,6 +54,7 @@ func NewHeimdallClient(urlString string) *HeimdallClient {
 const (
 	fetchStateSyncEventsFormat = "from-id=%d&to-time=%d&limit=%d"
 	fetchStateSyncEventsPath   = "clerk/event-record/list"
+	fetchLatestCheckpoint      = "/checkpoints/latest"
 
 	fetchSpanFormat = "bor/span/%d"
 )
@@ -108,6 +110,21 @@ func (h *HeimdallClient) Span(spanID uint64) (*span.HeimdallSpan, error) {
 	return &response.Result, nil
 }
 
+// FetchLatestCheckpoint fetches the latest bor submitted checkpoint from heimdall
+func (h *HeimdallClient) FetchLatestCheckpoint() (*checkpoint.Checkpoint, error) {
+	url, err := latestCheckpointURL(h.urlString)
+	if err != nil {
+		return nil, err
+	}
+
+	response, err := FetchWithRetry[checkpoint.CheckpointResponse](h.client, url, h.closeCh)
+	if err != nil {
+		return nil, err
+	}
+
+	return &response.Result, nil
+}
+
 // FetchWithRetry returns data from heimdall with retry
 func FetchWithRetry[T any](client http.Client, url *url.URL, closeCh chan struct{}) (*T, error) {
 	// attempt counter
@@ -171,6 +188,10 @@ func stateSyncURL(urlString string, fromID uint64, to int64) (*url.URL, error) {
 	return makeURL(urlString, fetchStateSyncEventsPath, queryParams)
 }
 
+func latestCheckpointURL(urlString string) (*url.URL, error) {
+	return makeURL(urlString, fetchLatestCheckpoint, "")
+}
+
 func makeURL(urlString, rawPath, rawQuery string) (*url.URL, error) {
 	u, err := url.Parse(urlString)
 	if err != nil {

eth/api_backend.go

@@ -359,3 +359,11 @@ func (b *EthAPIBackend) StateAtBlock(ctx context.Context, block *types.Block, re
 func (b *EthAPIBackend) StateAtTransaction(ctx context.Context, block *types.Block, txIndex int, reexec uint64) (core.Message, vm.BlockContext, *state.StateDB, error) {
 	return b.eth.stateAtTransaction(block, txIndex, reexec)
 }
+
+func (b *EthAPIBackend) GetCheckpointWhitelist() map[uint64]common.Hash {
+	return b.eth.Downloader().ChainValidator.GetCheckpointWhitelist()
+}
+
+func (b *EthAPIBackend) PurgeCheckpointWhitelist() {
+	b.eth.Downloader().ChainValidator.PurgeCheckpointWhitelist()
+}

eth/backend.go

@@ -101,6 +101,8 @@ type Ethereum struct {
 
 	lock sync.RWMutex // Protects the variadic fields (e.g. gas price and etherbase)
 
+	closeCh chan struct{} // Channel to signal the background processes to exit
+
 	shutdownTracker *shutdowncheck.ShutdownTracker // Tracks if and when the node has shutdown ungracefully
 }
 
@@ -163,6 +165,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		bloomRequests:     make(chan chan *bloombits.Retrieval),
 		bloomIndexer:      core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms),
 		p2pServer:         stack.Server(),
+		closeCh:           make(chan struct{}),
 		shutdownTracker:   shutdowncheck.NewShutdownTracker(chainDb),
 	}
 
@@ -254,6 +257,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 		BloomCache:         uint64(cacheLimit),
 		EventMux:           eth.eventMux,
 		Checkpoint:         checkpoint,
+		EthAPI:             ethAPI,
 		PeerRequiredBlocks: config.PeerRequiredBlocks,
 	}); err != nil {
 		return nil, err
@@ -608,9 +612,77 @@ func (s *Ethereum) Start() error {
 		}
 		maxPeers -= s.config.LightPeers
 	}
+
 	// Start the networking layer and the light server if requested
 	s.handler.Start(maxPeers)
 
+	go s.startCheckpointWhitelistService()
+
+	return nil
+}
+
+// StartCheckpointWhitelistService starts the goroutine to fetch checkpoints and update the
+// checkpoint whitelist map.
+func (s *Ethereum) startCheckpointWhitelistService() {
+	// a shortcut helps with tests and early exit
+	select {
+	case <-s.closeCh:
+		return
+	default:
+	}
+
+	// first run the checkpoint whitelist
+	err := s.handleWhitelistCheckpoint()
+	if err != nil {
+		if errors.Is(err, ErrBorConsensusWithoutHeimdall) || errors.Is(err, ErrNotBorConsensus) {
+			return
+		}
+
+		log.Warn("unable to whitelist checkpoint - first run", "err", err)
+	}
+
+	ticker := time.NewTicker(100 * time.Second)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			err := s.handleWhitelistCheckpoint()
+			if err != nil {
+				log.Warn("unable to whitelist checkpoint", "err", err)
+			}
+		case <-s.closeCh:
+			return
+		}
+	}
+}
+
+var (
+	ErrNotBorConsensus             = errors.New("not bor consensus was given")
+	ErrBorConsensusWithoutHeimdall = errors.New("bor consensus without heimdall")
+)
+
+// handleWhitelistCheckpoint handles the checkpoint whitelist mechanism.
+func (s *Ethereum) handleWhitelistCheckpoint() error {
+	ethHandler := (*ethHandler)(s.handler)
+
+	bor, ok := ethHandler.chain.Engine().(*bor.Bor)
+	if !ok {
+		return ErrNotBorConsensus
+	}
+
+	if bor.HeimdallClient == nil {
+		return ErrBorConsensusWithoutHeimdall
+	}
+
+	endBlockNum, endBlockHash, err := ethHandler.fetchWhitelistCheckpoint(bor)
+	if err != nil {
+		return err
+	}
+
+	// Update the checkpoint whitelist map.
+	ethHandler.downloader.ProcessCheckpoint(endBlockNum, endBlockHash)
+
 	return nil
 }
 
@@ -626,6 +698,9 @@ func (s *Ethereum) Stop() error {
 	s.bloomIndexer.Close()
 	close(s.closeBloomHandler)
 
+	// Close all bg processes
+	close(s.closeCh)
+
 	// closing consensus engine first, as miner has deps on it
 	s.engine.Close()
 	s.txPool.Stop()