Merge branch 'master' into mining_decoupling

CHANGELOG.md

@@ -8,6 +8,7 @@
 * admin_peers json RPC response now includes the remote nodes enode URL
 * add support for keccak mining and a ecip1049_dev network [\#1882](https://github.com/hyperledger/besu/pull/1882)
 ### Bug Fixes
+* Fixed incorrect `groupId` in published maven pom files.
 
 ### Early Access Features
 
@@ -17,11 +18,45 @@
 - [Privacy users with private transactions created using v1.3.4 or earlier](KNOWN_ISSUES.md#privacy-users-with-private-transactions-created-using-v134-or-earlier)
 
 ## 21.1.0
+
+## 21.1.0 Features
+
+Features added between 20.10.0 to 21.1.0 include:
+* Berlin Network Upgrade: this release contains the activation blocks for Berlin across all supported testnets and the Ethereum mainnet. They are: 
+  * Ropsten 9_812_189 (10 Mar 2021)
+  * Goerli 4_460_644 (17 Mar 2021)
+  * Rinkeby 8_290_928 (24 Mar 2021)
+  * Ethereum 12_244_000 (14 Apr 2021)
+* Besu Launcher: Besu now has support for the [Quorum Mainnet Launcher](https://github.com/ConsenSys/quorum-mainnet-launcher) which makes it easy for users to configure and launch Besu on the Ethereum mainnet.
+* Bonsai Tries: A new database format which reduces storage requirements and improves performance for access to recent state. _Note: only full sync is currently supported._
+* Miner Data JSON-RPC: The `eth_getMinerDataByBlockHash` and `eth_getMinerDataByBlockNumber` endpoints return miner rewards and coinbase address for a given block. 
+* EIP-1898 support: [The EIP](https://eips.ethereum.org/EIPS/eip-1898) adds `blockHash` to JSON-RPC methods which accept a default block parameter.
+
+### Early Access Features
+* Bonsai Tries: A new database format which reduces storage requirements and improves performance for access to recent state. _Note: only full sync is currently supported._
+* QBFT: A new consensus algorithm to support interoperability with other Enterprise Ethereum Alliance compatible clients.
+
+### 21.1.0 Breaking Changes
+* `--skip-pow-validation-enabled` is now an error with `block import --format JSON`. This is because the JSON format doesn't include the nonce so the proof of work must be calculated.
+* `eth_call` will not return a JSON-RPC result if the call fails, but will return an error instead. If it was for a revert the revert reason will be included.
+* `eth_call` will not fail for account balance issues by default. An parameter `"strict": true` can be added to the call parameters (with `to` and `from`) to enforce balance checks.
+
+### Additions and Improvements
+* Added `besu_transaction_pool_transactions` to the reported metrics, counting the mempool size [\#1869](https://github.com/hyperledger/besu/pull/1869)
 * Added activation blocks for Berlin Network Upgrade [\#1929](https://github.com/hyperledger/besu/pull/1929)
 
 ### Bug Fixes
 * Fixed representation of access list for access list transactions in JSON-RPC results.
 
+#### Previously identified known issues
+
+- [Fast sync when running Besu on cloud providers](KNOWN_ISSUES.md#fast-sync-when-running-besu-on-cloud-providers)
+- [Privacy users with private transactions created using v1.3.4 or earlier](KNOWN_ISSUES.md#privacy-users-with-private-transactions-created-using-v134-or-earlier)
+
+### Download link
+https://hyperledger.jfrog.io/artifactory/besu-binaries/besu/21.1.0/besu-21.1.0.zip
+sha256: `e4c8fe4007e3e5f7f2528cbf1eeb5457caf06536c974a6ff4305035ff5724476`
+
 ## 21.1.0-RC2
 ### Additions and Improvements
 * Support for the Berlin Network Upgrade, although the block number must be set manually with `--override-genesis-config=berlinBlock=<blocknumber>`. This is because the block numbers haven't been determined yet. The next release will include the number in the genesis file so it will support Berlin with no intervention. [\#1898](https://github.com/hyperledger/besu/pull/1898)

build.gradle

@@ -25,7 +25,7 @@ plugins {
   id 'com.github.hierynomus.license' version '0.15.0'
   id 'com.jfrog.artifactory' version '4.20.0'
   id 'com.jfrog.bintray' version '1.8.5'
-  id 'io.spring.dependency-management' version '1.0.10.RELEASE'
+  id 'io.spring.dependency-management' version '1.0.11.RELEASE'
   id 'me.champeau.gradle.jmh' version '0.5.0' apply false
   id 'net.ltgt.errorprone' version '1.3.0'
   id 'maven-publish'
@@ -126,10 +126,11 @@ allprojects {
     mavenCentral()
     maven { url "https://hyperledger.jfrog.io/hyperledger/besu-maven" }
     maven { url "https://hyperledger-org.bintray.com/besu-repo" }
+    maven { url "https://artifacts.consensys.net/public/maven/maven/" }
     maven { url "https://consensys.bintray.com/pegasys-repo" }
-    maven { url "https://consensys.bintray.com/consensys" }
     maven { url "https://splunk.jfrog.io/splunk/ext-releases-local" }
     maven { url "https://dl.bintray.com/open-telemetry/maven" }
+    maven { url "https://dl.cloudsmith.io/public/consensys/quorum-mainnet-launcher/maven/"}
   }
 
   dependencies { errorprone "com.google.errorprone:error_prone_core" }

consensus/qbft/src/main/java/org/hyperledger/besu/consensus/qbft/messagedata/QbftV1.java

@@ -16,10 +16,10 @@
 
 /** Message codes for QBFT v1 messages */
 public class QbftV1 {
-  public static final int PROPOSAL = 0;
-  public static final int PREPARE = 1;
-  public static final int COMMIT = 2;
-  public static final int ROUND_CHANGE = 3;
+  public static final int PROPOSAL = 0x12;
+  public static final int PREPARE = 0x13;
+  public static final int COMMIT = 0x14;
+  public static final int ROUND_CHANGE = 0x15;
 
-  public static final int MESSAGE_SPACE = 4;
+  public static final int MESSAGE_SPACE = 0x16;
 }

consensus/qbft/src/test/java/org/hyperledger/besu/consensus/qbft/protocol/QbftSubProtocolTest.java

@@ -24,23 +24,23 @@ public class QbftSubProtocolTest {
   public void messageSpaceReportsCorrectly() {
     final QbftSubProtocol subProt = new QbftSubProtocol();
 
-    assertThat(subProt.messageSpace(1)).isEqualTo(4);
+    assertThat(subProt.messageSpace(1)).isEqualTo(0x16);
   }
 
   @Test
   public void allIbftMessageTypesAreRecognisedAsValidByTheSubProtocol() {
     final QbftSubProtocol subProt = new QbftSubProtocol();
 
-    assertThat(subProt.isValidMessageCode(1, 0)).isTrue();
-    assertThat(subProt.isValidMessageCode(1, 1)).isTrue();
-    assertThat(subProt.isValidMessageCode(1, 2)).isTrue();
-    assertThat(subProt.isValidMessageCode(1, 3)).isTrue();
+    assertThat(subProt.isValidMessageCode(1, 0x12)).isTrue();
+    assertThat(subProt.isValidMessageCode(1, 0x13)).isTrue();
+    assertThat(subProt.isValidMessageCode(1, 0x14)).isTrue();
+    assertThat(subProt.isValidMessageCode(1, 0x15)).isTrue();
   }
 
   @Test
   public void invalidMessageTypesAreNotAcceptedByTheSubprotocol() {
     final QbftSubProtocol subProt = new QbftSubProtocol();
 
-    assertThat(subProt.isValidMessageCode(1, 4)).isFalse();
+    assertThat(subProt.isValidMessageCode(1, 0x16)).isFalse();
   }
 }

docker/graalvm/Dockerfile

@@ -25,7 +25,9 @@ ENV BESU_PID_PATH "/tmp/pid"
 
 ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu,service.version=$VERSION"
 
-ENV PATH="/opt/besu/bin:${PATH}"
+ENV OLDPATH="${PATH}"
+ENV PATH="/opt/besu/bin:${OLDPATH}"
+
 ENTRYPOINT ["besu"]
 HEALTHCHECK --start-period=5s --interval=5s --timeout=1s --retries=10 CMD bash -c "[ -f /tmp/pid ]"
 

docker/openjdk-11/Dockerfile

@@ -25,7 +25,9 @@ ENV BESU_PID_PATH "/tmp/pid"
 
 ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu,service.version=$VERSION"
 
-ENV PATH="/opt/besu/bin:${PATH}"
+ENV OLDPATH="${PATH}"
+ENV PATH="/opt/besu/bin:${OLDPATH}"
+
 ENTRYPOINT ["besu"]
 HEALTHCHECK --start-period=5s --interval=5s --timeout=1s --retries=10 CMD bash -c "[ -f /tmp/pid ]"
 

docker/openjdk-latest/Dockerfile

@@ -25,7 +25,9 @@ ENV BESU_PID_PATH "/tmp/pid"
 
 ENV OTEL_RESOURCE_ATTRIBUTES="service.name=besu,service.version=$VERSION"
 
-ENV PATH="/opt/besu/bin:${PATH}"
+ENV OLDPATH="${PATH}"
+ENV PATH="/opt/besu/bin:${OLDPATH}"
+
 ENTRYPOINT ["besu"]
 HEALTHCHECK --start-period=5s --interval=5s --timeout=1s --retries=10 CMD bash -c "[ -f /tmp/pid ]"
 

ethereum/blockcreation/src/main/java/org/hyperledger/besu/ethereum/blockcreation/BlockTransactionSelector.java

@@ -17,6 +17,7 @@
 import org.hyperledger.besu.config.experimental.ExperimentalEIPs;
 import org.hyperledger.besu.ethereum.chain.Blockchain;
 import org.hyperledger.besu.ethereum.core.Address;
+import org.hyperledger.besu.ethereum.core.EvmAccount;
 import org.hyperledger.besu.ethereum.core.MutableWorldState;
 import org.hyperledger.besu.ethereum.core.ProcessableBlockHeader;
 import org.hyperledger.besu.ethereum.core.Transaction;
@@ -30,18 +31,24 @@
 import org.hyperledger.besu.ethereum.eth.transactions.PendingTransactions.TransactionSelectionResult;
 import org.hyperledger.besu.ethereum.mainnet.AbstractBlockProcessor;
 import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionProcessor;
+import org.hyperledger.besu.ethereum.mainnet.MainnetTransactionValidator;
 import org.hyperledger.besu.ethereum.mainnet.TransactionValidationParams;
+import org.hyperledger.besu.ethereum.mainnet.ValidationResult;
 import org.hyperledger.besu.ethereum.processing.TransactionProcessingResult;
 import org.hyperledger.besu.ethereum.transaction.TransactionInvalidReason;
 import org.hyperledger.besu.ethereum.vm.BlockHashLookup;
 import org.hyperledger.besu.plugin.data.TransactionType;
 
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.CancellationException;
 import java.util.function.Supplier;
 
 import com.google.common.collect.Lists;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.tuweni.bytes.Bytes;
 
 /**
  * Responsible for extracting transactions from PendingTransactions and determining if the
@@ -64,6 +71,7 @@
  * not cleared between executions of buildTransactionListForBlock().
  */
 public class BlockTransactionSelector {
+  private static final Logger LOG = LogManager.getLogger();
 
   private final Wei minTransactionGasPrice;
   private final Double minBlockOccupancyRatio;
@@ -213,34 +221,79 @@ private TransactionSelectionResult evaluateTransaction(
     final WorldUpdater worldStateUpdater = worldState.updater();
     final BlockHashLookup blockHashLookup = new BlockHashLookup(processableBlockHeader, blockchain);
 
-    final TransactionProcessingResult result =
-        transactionProcessor.processTransaction(
-            blockchain,
-            worldStateUpdater,
-            processableBlockHeader,
-            transaction,
-            miningBeneficiary,
-            blockHashLookup,
-            false,
-            TransactionValidationParams.mining());
-
-    if (!result.isInvalid()) {
+    TransactionProcessingResult effectiveResult;
+
+    if (transaction.isGoQuorumPrivateTransaction()) {
+      final ValidationResult<TransactionInvalidReason> validationResult =
+          validateTransaction(processableBlockHeader, transaction, worldStateUpdater);
+      if (!validationResult.isValid()) {
+        LOG.warn(
+            "Invalid transaction: {}. Block {} Transaction {}",
+            validationResult.getErrorMessage(),
+            processableBlockHeader.getParentHash().toHexString(),
+            transaction.getHash().toHexString());
+        return transactionSelectionResultForInvalidResult(validationResult);
+      } else {
+        // valid GoQuorum private tx, we need to hand craft the receipt and increment the nonce
+        effectiveResult = publicResultForWhenWeHaveAPrivateTransaction(transaction);
+        worldStateUpdater.getOrCreate(transaction.getSender()).getMutable().incrementNonce();
+      }
+    } else {
+      effectiveResult =
+          transactionProcessor.processTransaction(
+              blockchain,
+              worldStateUpdater,
+              processableBlockHeader,
+              transaction,
+              miningBeneficiary,
+              blockHashLookup,
+              false,
+              TransactionValidationParams.mining());
+    }
+
+    if (!effectiveResult.isInvalid()) {
       worldStateUpdater.commit();
-      updateTransactionResultTracking(transaction, result);
+      updateTransactionResultTracking(transaction, effectiveResult);
     } else {
-      // If the transaction has an incorrect nonce, leave it in the pool and continue
-      if (result
-          .getValidationResult()
-          .getInvalidReason()
-          .equals(TransactionInvalidReason.INCORRECT_NONCE)) {
-        return TransactionSelectionResult.CONTINUE;
-      }
-      // If the transaction was invalid for any other reason, delete it, and continue.
-      return TransactionSelectionResult.DELETE_TRANSACTION_AND_CONTINUE;
+      return transactionSelectionResultForInvalidResult(effectiveResult.getValidationResult());
     }
     return TransactionSelectionResult.CONTINUE;
   }
 
+  private TransactionSelectionResult transactionSelectionResultForInvalidResult(
+      final ValidationResult<TransactionInvalidReason> invalidReasonValidationResult) {
+    // If the transaction has an incorrect nonce, leave it in the pool and continue
+    if (invalidReasonValidationResult
+        .getInvalidReason()
+        .equals(TransactionInvalidReason.INCORRECT_NONCE)) {
+      return TransactionSelectionResult.CONTINUE;
+    }
+    // If the transaction was invalid for any other reason, delete it, and continue.
+    return TransactionSelectionResult.DELETE_TRANSACTION_AND_CONTINUE;
+  }
+
+  private ValidationResult<TransactionInvalidReason> validateTransaction(
+      final ProcessableBlockHeader blockHeader,
+      final Transaction transaction,
+      final WorldUpdater publicWorldStateUpdater) {
+    final MainnetTransactionValidator transactionValidator =
+        transactionProcessor.getTransactionValidator();
+    ValidationResult<TransactionInvalidReason> validationResult =
+        transactionValidator.validate(transaction, blockHeader.getBaseFee());
+    if (!validationResult.isValid()) {
+      return validationResult;
+    }
+
+    final Address senderAddress = transaction.getSender();
+
+    final EvmAccount sender = publicWorldStateUpdater.getOrCreate(senderAddress);
+    validationResult =
+        transactionValidator.validateForSender(
+            transaction, sender, TransactionValidationParams.processingBlock());
+
+    return validationResult;
+  }
+
   /*
   Responsible for updating the state maintained between transaction validation (i.e. receipts,
   cumulative gas, world state root hash.).
@@ -268,6 +321,16 @@ private void updateTransactionResultTracking(
         gasUsedByTransaction);
   }
 
+  private TransactionProcessingResult publicResultForWhenWeHaveAPrivateTransaction(
+      final Transaction transaction) {
+    return TransactionProcessingResult.successful(
+        Collections.emptyList(),
+        0,
+        transaction.getGasLimit(),
+        Bytes.EMPTY,
+        ValidationResult.valid());
+  }
+
   private boolean transactionTooLargeForBlock(
       final long blockNumber, final long gasLimit, final Transaction transaction) {
 

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiValue.java

@@ -68,4 +68,9 @@ void writeInnerRlp(final RLPOutput output, final BiConsumer<RLPOutput, T> writer
   boolean isUnchanged() {
     return Objects.equals(updated, original);
   }
+
+  @Override
+  public String toString() {
+    return "BonsaiValue{" + "original=" + original + ", updated=" + updated + '}';
+  }
 }

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateArchive.java

@@ -129,13 +129,16 @@ public Optional<MutableWorldState> getMutable(final Hash rootHash, final Hash bl
             (BonsaiWorldStateUpdater) persistedState.updater();
         try {
           for (final TrieLogLayer rollBack : rollBacks) {
+            LOG.debug("Attempting Rollback of {}", rollBack.getBlockHash());
             bonsaiUpdater.rollBack(rollBack);
           }
           for (int i = rollForwards.size() - 1; i >= 0; i--) {
+            LOG.debug("Attempting Rollforward of {}", rollForwards.get(i).getBlockHash());
             bonsaiUpdater.rollForward(rollForwards.get(i));
           }
           bonsaiUpdater.commit();
           persistedState.persist(blockchain.getBlockHeader(blockHash).get());
+          LOG.debug("Archive rolling finished, now at {}", blockHash);
           return Optional.of(persistedState);
         } catch (final Exception e) {
           // if we fail we must clean up the updater

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/bonsai/BonsaiWorldStateUpdater.java

@@ -154,9 +154,7 @@ public void commit() {
           accountsToUpdate.computeIfAbsent(
               deletedAddress,
               __ -> loadAccountFromParent(deletedAddress, new BonsaiValue<>(null, null)));
-      if (accountValue.getOriginal() != null) {
-        storageToClear.add(deletedAddress);
-      }
+      storageToClear.add(deletedAddress);
       final BonsaiValue<Bytes> codeValue = codeToUpdate.get(deletedAddress);
       if (codeValue != null) {
         codeValue.setUpdated(null);
@@ -271,7 +269,7 @@ public Optional<Bytes> getCode(final Address address) {
     if (localCode == null) {
       return wrappedWorldView().getCode(address);
     } else {
-      return Optional.of(localCode.getUpdated());
+      return Optional.ofNullable(localCode.getUpdated());
     }
   }
 
@@ -294,7 +292,7 @@ public Optional<UInt256> getStorageValueBySlotHash(final Address address, final
         storageToUpdate.computeIfAbsent(address, key -> new HashMap<>());
     final BonsaiValue<UInt256> value = localAccountStorage.get(slotHash);
     if (value != null) {
-      return Optional.of(value.getUpdated());
+      return Optional.ofNullable(value.getUpdated());
     } else {
       final Optional<UInt256> valueUInt =
           wrappedWorldView().getStorageValueBySlotHash(address, slotHash);
@@ -306,6 +304,9 @@ public Optional<UInt256> getStorageValueBySlotHash(final Address address, final
   @Override
   public UInt256 getOriginalStorageValue(final Address address, final UInt256 storageKey) {
     // TODO maybe log the read into the trie layer?
+    if (storageToClear.contains(address)) {
+      return UInt256.ZERO;
+    }
     final Map<Hash, BonsaiValue<UInt256>> localAccountStorage =
         storageToUpdate.computeIfAbsent(address, key -> new HashMap<>());
     final Hash slotHashBytes = Hash.hash(storageKey.toBytes());

ethereum/core/src/main/java/org/hyperledger/besu/ethereum/core/WorldView.java

@@ -19,7 +19,7 @@ public interface WorldView {
   WorldView EMPTY = address -> null;
 
   /**
-   * Get an account provided it's address.
+   * Get an account provided its address.
    *
    * @param address the address of the account to retrieve.
    * @return the {@link Account} corresponding to {@code address} or {@code null} if there is no

ethereum/eth/src/main/java/org/hyperledger/besu/ethereum/eth/sync/BlockPropagationManager.java

@@ -229,7 +229,8 @@ private void handleNewBlockHashesFromNetwork(final EthMessage message) {
 
       // Process known blocks we care about
       for (final NewBlockHash newBlock : newBlocks) {
-        processAnnouncedBlock(message.getPeer(), newBlock)
+        EthPeer bestPeer = ethContext.getEthPeers().bestPeer().orElse(message.getPeer());
+        processAnnouncedBlock(bestPeer, newBlock)
             .whenComplete((r, t) -> requestedBlocks.remove(newBlock.hash()));
       }
     } catch (final RLPException e) {