Merge branch 'main' into fix-hyperledger#7810

CHANGELOG.md

@@ -18,6 +18,7 @@
 
 ### Bug fixes
 - Fix registering new metric categories from plugins [#7825](https://github.com/hyperledger/besu/pull/7825)
+- Fix CVE-2024-47535 [7878](https://github.com/hyperledger/besu/pull/7878)
 
 ## 24.10.0
 
@@ -103,6 +104,7 @@ This release version has been deprecated release due to CI bug
 - Remove long-deprecated `perm*whitelist*` methods [#7401](https://github.com/hyperledger/besu/pull/7401)
 
 ### Additions and Improvements
+- Allow optional loading of `jemalloc` (if installed) by setting the environment variable `BESU_USING_JEMALLOC` to true/false. It that env is not set at all it will behave as if it is set to `true`
 - Expose set finalized/safe block in plugin api BlockchainService. These method can be used by plugins to set finalized/safe block for a PoA network (such as QBFT, IBFT and Clique).[#7382](https://github.com/hyperledger/besu/pull/7382)
 - In process RPC service [#7395](https://github.com/hyperledger/besu/pull/7395)
 - Added support for tracing private transactions using `priv_traceTransaction` API. [#6161](https://github.com/hyperledger/besu/pull/6161)

MAINTAINERS.md

@@ -22,6 +22,7 @@
 | Luis Pinto       | lu-pinto         | lu-pinto         |
 | Lucas Saldanha   | lucassaldanha    | lucassaldanha    |
 | Sally MacFarlane | macfarla         | macfarla         |
+| Matilda Clerke   | Matilda-Clerke   | MatildaClerke    |
 | Karim Taam       | matkt            | matkt            |
 | Matthew Whitehead| matthew1001      | matthew.whitehead      |
 | Meredith Baxter  | mbaxter          | mbaxter          |

acceptance-tests/dsl/src/main/java/org/hyperledger/besu/tests/acceptance/dsl/node/ThreadBesuNodeRunner.java

@@ -75,6 +75,7 @@
 import org.hyperledger.besu.plugin.services.TransactionSimulationService;
 import org.hyperledger.besu.plugin.services.metrics.MetricCategoryRegistry;
 import org.hyperledger.besu.plugin.services.storage.rocksdb.RocksDBPlugin;
+import org.hyperledger.besu.plugin.services.transactionpool.TransactionPoolService;
 import org.hyperledger.besu.services.BesuConfigurationImpl;
 import org.hyperledger.besu.services.BesuEventsImpl;
 import org.hyperledger.besu.services.BesuPluginContextImpl;
@@ -85,6 +86,7 @@
 import org.hyperledger.besu.services.RpcEndpointServiceImpl;
 import org.hyperledger.besu.services.SecurityModuleServiceImpl;
 import org.hyperledger.besu.services.StorageServiceImpl;
+import org.hyperledger.besu.services.TransactionPoolServiceImpl;
 import org.hyperledger.besu.services.TransactionPoolValidatorServiceImpl;
 import org.hyperledger.besu.services.TransactionSelectionServiceImpl;
 import org.hyperledger.besu.services.TransactionSimulationServiceImpl;
@@ -215,6 +217,9 @@ public void startNode(final BesuNode node) {
             besuController.getTransactionPool(),
             besuController.getSyncState(),
             besuController.getProtocolContext().getBadBlockManager()));
+    besuPluginContext.addService(
+        TransactionPoolService.class,
+        new TransactionPoolServiceImpl(besuController.getTransactionPool()));
 
     component.rpcEndpointService().init(runner.getInProcessRpcMethods());
 
@@ -327,7 +332,9 @@ RpcEndpointServiceImpl provideRpcEndpointService() {
     @Singleton
     BlockchainServiceImpl provideBlockchainService(final BesuController besuController) {
       BlockchainServiceImpl retval = new BlockchainServiceImpl();
-      retval.init(besuController.getProtocolContext(), besuController.getProtocolSchedule());
+      retval.init(
+          besuController.getProtocolContext().getBlockchain(),
+          besuController.getProtocolSchedule());
       return retval;
     }
 

besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java

@@ -1233,7 +1233,7 @@ private Runner buildRunner() {
 
   private void startPlugins(final Runner runner) {
     blockchainServiceImpl.init(
-        besuController.getProtocolContext(), besuController.getProtocolSchedule());
+        besuController.getProtocolContext().getBlockchain(), besuController.getProtocolSchedule());
     transactionSimulationServiceImpl.init(
         besuController.getProtocolContext().getBlockchain(),
         new TransactionSimulator(

besu/src/main/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilder.java

@@ -433,14 +433,18 @@ public String build() {
   private void detectJemalloc(final List<String> lines) {
     Optional.ofNullable(Objects.isNull(environment) ? null : environment.get("BESU_USING_JEMALLOC"))
         .ifPresentOrElse(
-            t -> {
+            jemallocEnabled -> {
               try {
-                final String version = PlatformDetector.getJemalloc();
-                lines.add("jemalloc: " + version);
+                if (Boolean.parseBoolean(jemallocEnabled)) {
+                  final String version = PlatformDetector.getJemalloc();
+                  lines.add("jemalloc: " + version);
+                } else {
+                  logger.warn(
+                      "besu_using_jemalloc is present but is not set to true, jemalloc library not loaded");
+                }
               } catch (final Throwable throwable) {
                 logger.warn(
-                    "BESU_USING_JEMALLOC is present but we failed to load jemalloc library to get the version",
-                    throwable);
+                    "besu_using_jemalloc is present but we failed to load jemalloc library to get the version");
               }
             },
             () -> {

besu/src/main/java/org/hyperledger/besu/controller/BesuControllerBuilder.java

@@ -26,7 +26,6 @@
 import org.hyperledger.besu.cryptoservices.NodeKey;
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.ConsensusContext;
-import org.hyperledger.besu.ethereum.ConsensusContextFactory;
 import org.hyperledger.besu.ethereum.GasLimitCalculator;
 import org.hyperledger.besu.ethereum.ProtocolContext;
 import org.hyperledger.besu.ethereum.api.jsonrpc.methods.JsonRpcMethods;
@@ -605,9 +604,11 @@ public BesuController build() {
       genesisState.writeStateTo(worldStateArchive.getMutable());
     }
 
+    final var consensusContext =
+        createConsensusContext(blockchain, worldStateArchive, protocolSchedule);
+
     final ProtocolContext protocolContext =
-        createProtocolContext(
-            blockchain, worldStateArchive, protocolSchedule, this::createConsensusContext);
+        createProtocolContext(blockchain, worldStateArchive, consensusContext);
     validateContext(protocolContext);
 
     protocolSchedule.setPublicWorldStateArchiveForPrivacyBlockProcessor(
@@ -976,7 +977,7 @@ protected SubProtocolConfiguration createSubProtocolConfiguration(
   }
 
   /**
-   * Create mining coordinator mining coordinator.
+   * Create mining coordinator.
    *
    * @param protocolSchedule the protocol schedule
    * @param protocolContext the protocol context
@@ -1017,9 +1018,9 @@ protected void validateContext(final ProtocolContext context) {}
    * @return the consensus context
    */
   protected abstract ConsensusContext createConsensusContext(
-      Blockchain blockchain,
-      WorldStateArchive worldStateArchive,
-      ProtocolSchedule protocolSchedule);
+      final Blockchain blockchain,
+      final WorldStateArchive worldStateArchive,
+      final ProtocolSchedule protocolSchedule);
 
   /**
    * Create eth protocol manager eth protocol manager.
@@ -1070,17 +1071,14 @@ protected EthProtocolManager createEthProtocolManager(
    *
    * @param blockchain the blockchain
    * @param worldStateArchive the world state archive
-   * @param protocolSchedule the protocol schedule
-   * @param consensusContextFactory the consensus context factory
+   * @param consensusContext the consensus context
    * @return the protocol context
    */
   protected ProtocolContext createProtocolContext(
       final MutableBlockchain blockchain,
       final WorldStateArchive worldStateArchive,
-      final ProtocolSchedule protocolSchedule,
-      final ConsensusContextFactory consensusContextFactory) {
-    return ProtocolContext.init(
-        blockchain, worldStateArchive, protocolSchedule, consensusContextFactory, badBlockManager);
+      final ConsensusContext consensusContext) {
+    return new ProtocolContext(blockchain, worldStateArchive, consensusContext, badBlockManager);
   }
 
   private Optional<SnapProtocolManager> createSnapProtocolManager(

besu/src/main/java/org/hyperledger/besu/controller/ConsensusScheduleBesuControllerBuilder.java

@@ -20,13 +20,12 @@
 import org.hyperledger.besu.consensus.common.CombinedProtocolScheduleFactory;
 import org.hyperledger.besu.consensus.common.ForkSpec;
 import org.hyperledger.besu.consensus.common.ForksSchedule;
-import org.hyperledger.besu.consensus.common.MigratingContext;
+import org.hyperledger.besu.consensus.common.MigratingConsensusContext;
 import org.hyperledger.besu.consensus.common.MigratingMiningCoordinator;
 import org.hyperledger.besu.consensus.common.MigratingProtocolContext;
 import org.hyperledger.besu.cryptoservices.NodeKey;
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.ConsensusContext;
-import org.hyperledger.besu.ethereum.ConsensusContextFactory;
 import org.hyperledger.besu.ethereum.GasLimitCalculator;
 import org.hyperledger.besu.ethereum.ProtocolContext;
 import org.hyperledger.besu.ethereum.api.jsonrpc.methods.JsonRpcMethods;
@@ -168,10 +167,12 @@ protected ProtocolSchedule createProtocolSchedule() {
   protected ProtocolContext createProtocolContext(
       final MutableBlockchain blockchain,
       final WorldStateArchive worldStateArchive,
-      final ProtocolSchedule protocolSchedule,
-      final ConsensusContextFactory consensusContextFactory) {
-    return MigratingProtocolContext.init(
-        blockchain, worldStateArchive, protocolSchedule, consensusContextFactory, badBlockManager);
+      final ConsensusContext consensusContext) {
+    return new MigratingProtocolContext(
+        blockchain,
+        worldStateArchive,
+        consensusContext.as(MigratingConsensusContext.class),
+        badBlockManager);
   }
 
   @Override
@@ -188,10 +189,10 @@ protected ConsensusContext createConsensusContext(
                         e.getValue()
                             .createConsensusContext(
                                 blockchain, worldStateArchive, protocolSchedule)))
-            .collect(Collectors.toList());
+            .toList();
     final ForksSchedule<ConsensusContext> consensusContextsSchedule =
         new ForksSchedule<>(consensusContextSpecs);
-    return new MigratingContext(consensusContextsSchedule);
+    return new MigratingConsensusContext(consensusContextsSchedule);
   }
 
   @Override

besu/src/main/java/org/hyperledger/besu/controller/QbftBesuControllerBuilder.java

@@ -112,10 +112,7 @@ public QbftBesuControllerBuilder() {}
 
   @Override
   protected Supplier<BftExtraDataCodec> bftExtraDataCodec() {
-    return Suppliers.memoize(
-        () -> {
-          return new QbftExtraDataCodec();
-        });
+    return Suppliers.memoize(QbftExtraDataCodec::new);
   }
 
   @Override

besu/src/main/java/org/hyperledger/besu/controller/TransitionBesuControllerBuilder.java

@@ -24,7 +24,6 @@
 import org.hyperledger.besu.cryptoservices.NodeKey;
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.ethereum.ConsensusContext;
-import org.hyperledger.besu.ethereum.ConsensusContextFactory;
 import org.hyperledger.besu.ethereum.GasLimitCalculator;
 import org.hyperledger.besu.ethereum.ProtocolContext;
 import org.hyperledger.besu.ethereum.blockcreation.MiningCoordinator;
@@ -193,11 +192,9 @@ protected ProtocolSchedule createProtocolSchedule() {
   protected ProtocolContext createProtocolContext(
       final MutableBlockchain blockchain,
       final WorldStateArchive worldStateArchive,
-      final ProtocolSchedule protocolSchedule,
-      final ConsensusContextFactory consensusContextFactory) {
+      final ConsensusContext consensusContext) {
     final ProtocolContext protocolContext =
-        super.createProtocolContext(
-            blockchain, worldStateArchive, protocolSchedule, consensusContextFactory);
+        super.createProtocolContext(blockchain, worldStateArchive, consensusContext);
     transitionProtocolSchedule.setProtocolContext(protocolContext);
     return protocolContext;
   }

besu/src/main/java/org/hyperledger/besu/services/BlockchainServiceImpl.java

@@ -16,7 +16,6 @@
 
 import org.hyperledger.besu.datatypes.Hash;
 import org.hyperledger.besu.datatypes.Wei;
-import org.hyperledger.besu.ethereum.ProtocolContext;
 import org.hyperledger.besu.ethereum.chain.MutableBlockchain;
 import org.hyperledger.besu.ethereum.core.Block;
 import org.hyperledger.besu.ethereum.mainnet.ProtocolSchedule;
@@ -40,7 +39,6 @@
 @Unstable
 public class BlockchainServiceImpl implements BlockchainService {
 
-  private ProtocolContext protocolContext;
   private ProtocolSchedule protocolSchedule;
   private MutableBlockchain blockchain;
 
@@ -50,13 +48,12 @@ public BlockchainServiceImpl() {}
   /**
    * Initialize the Blockchain service.
    *
-   * @param protocolContext the protocol context
+   * @param blockchain the blockchain
    * @param protocolSchedule the protocol schedule
    */
-  public void init(final ProtocolContext protocolContext, final ProtocolSchedule protocolSchedule) {
-    this.protocolContext = protocolContext;
+  public void init(final MutableBlockchain blockchain, final ProtocolSchedule protocolSchedule) {
     this.protocolSchedule = protocolSchedule;
-    this.blockchain = protocolContext.getBlockchain();
+    this.blockchain = blockchain;
   }
 
   /**
@@ -67,25 +64,24 @@ public void init(final ProtocolContext protocolContext, final ProtocolSchedule p
    */
   @Override
   public Optional<BlockContext> getBlockByNumber(final long number) {
-    return protocolContext
-        .getBlockchain()
+    return blockchain
         .getBlockByNumber(number)
         .map(block -> blockContext(block::getHeader, block::getBody));
   }
 
   @Override
   public Hash getChainHeadHash() {
-    return protocolContext.getBlockchain().getChainHeadHash();
+    return blockchain.getChainHeadHash();
   }
 
   @Override
   public BlockHeader getChainHeadHeader() {
-    return protocolContext.getBlockchain().getChainHeadHeader();
+    return blockchain.getChainHeadHeader();
   }
 
   @Override
   public Optional<Wei> getNextBlockBaseFee() {
-    final var chainHeadHeader = protocolContext.getBlockchain().getChainHeadHeader();
+    final var chainHeadHeader = blockchain.getChainHeadHeader();
     final var protocolSpec =
         protocolSchedule.getForNextBlockHeader(chainHeadHeader, System.currentTimeMillis());
     return Optional.of(protocolSpec.getFeeMarket())

besu/src/main/scripts/unixStartScript.txt

@@ -182,19 +182,19 @@ APP_ARGS=`save "\$@"`
 # Collect all arguments for the java command, following the shell quoting and substitution rules
 eval set -- \$DEFAULT_JVM_OPTS \$JAVA_OPTS \$${optsEnvironmentVar} <% if ( appNameSystemProperty ) { %>"\"-D${appNameSystemProperty}=\$APP_BASE_NAME\"" <% } %>-classpath "\"\$CLASSPATH\"" <% if ( mainClassName.startsWith('--module ') ) { %>--module-path "\"\$MODULE_PATH\"" <% } %>${mainClassName} "\$APP_ARGS"
 
-unset BESU_USING_JEMALLOC
 if [ "\$darwin" = "false" -a "\$msys" = "false" ]; then
-  # check if jemalloc is available
-  TEST_JEMALLOC=\$(LD_PRELOAD=libjemalloc.so sh -c true 2>&1)
-
-  # if jemalloc is available the output is empty, otherwise the output has an error line
-  if [ -z "\$TEST_JEMALLOC" ]; then
-    export LD_PRELOAD=libjemalloc.so
-    export BESU_USING_JEMALLOC=true
-  else
-    # jemalloc not available, as fallback limit malloc to 2 arenas
-    export MALLOC_ARENA_MAX=2
-  fi
+      if [ "\$BESU_USING_JEMALLOC" != "FALSE" -a "\$BESU_USING_JEMALLOC" != "false" ]; then
+      # check if jemalloc is available
+      TEST_JEMALLOC=\$(LD_PRELOAD=libjemalloc.so sh -c true 2>&1)
+
+      # if jemalloc is available the output is empty, otherwise the output has an error line
+      if [ -z "\$TEST_JEMALLOC" ]; then
+        export LD_PRELOAD=libjemalloc.so
+      else
+        # jemalloc not available, as fallback limit malloc to 2 arenas
+        export MALLOC_ARENA_MAX=2
+      fi
+    fi
 fi
 
 exec "\$JAVACMD" "\$@"

besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java

@@ -37,6 +37,7 @@
 import static org.mockito.ArgumentMatchers.contains;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.ArgumentMatchers.isNotNull;
+import static org.mockito.Mockito.argThat;
 import static org.mockito.Mockito.atLeast;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.verify;
@@ -2412,13 +2413,16 @@ public void nativeLibrariesAreEnabledByDefault() {
   @Test
   public void logsWarningWhenFailToLoadJemalloc() {
     assumeTrue(PlatformDetector.getOSType().equals("linux"));
-    setEnvironmentVariable("BESU_USING_JEMALLOC", "true");
+    setEnvironmentVariable("BESU_USING_JEMALLOC", "false");
     parseCommand();
     verify(mockLogger)
         .warn(
-            eq(
-                "BESU_USING_JEMALLOC is present but we failed to load jemalloc library to get the version"),
-            any(Throwable.class));
+            argThat(
+                arg ->
+                    arg.equals(
+                            "besu_using_jemalloc is present but is not set to true, jemalloc library not loaded")
+                        || arg.equals(
+                            "besu_using_jemalloc is present but we failed to load jemalloc library to get the version")));
   }
 
   @Test