Merge pull request #1839 from ergoplatform/v4.0.46

Candidate for 4.0.46

README.md

@@ -82,7 +82,7 @@ To run specific Ergo version `<VERSION>` as a service with custom config `/path/
         -e MAX_HEAP=3G \
         ergoplatform/ergo:<VERSION> --<networkId> -c /etc/myergo.conf
 
-Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.45`.
+Available versions can be found on [Ergo Docker image page](https://hub.docker.com/r/ergoplatform/ergo/tags), for example, `v4.0.46`.
 
 This will connect to the Ergo mainnet or testnet following your configuration passed in `myergo.conf` and network flag `--<networkId>`. Every default config value would be overwritten with corresponding value in `myergo.conf`. `MAX_HEAP` variable can be used to control how much memory can the node consume.
 

benchmarks/src/test/scala/org/ergoplatform/bench/CrawlerRunner.scala

@@ -40,7 +40,7 @@ class CrawlerRunner(args: Array[String]) extends Application {
 
   lazy val emission = new EmissionRules(ergoSettings.chainSettings.monetary)
 
-  override implicit lazy val settings: ScorexSettings = ergoSettings.scorexSettings
+  override implicit lazy val scorexSettings: ScorexSettings = ergoSettings.scorexSettings
 
   override protected lazy val additionalMessageSpecs: Seq[MessageSpec[_]] = Seq(ErgoSyncInfoMessageSpec)
   override val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider)
@@ -49,15 +49,15 @@ class CrawlerRunner(args: Array[String]) extends Application {
 
   val minerRef: ActorRef = ErgoMiner(ergoSettings, nodeViewHolderRef, readersHolderRef, timeProvider)
 
-  private val syncTracker = ErgoSyncTracker(settings.network, timeProvider)
+  private val syncTracker = ErgoSyncTracker(scorexSettings.network, timeProvider)
 
   val statsCollectorRef: ActorRef =
     ErgoStatsCollectorRef(nodeViewHolderRef, networkControllerRef, syncTracker, ergoSettings, timeProvider)
 
 
   override val apiRoutes: Seq[ApiRoute] = Seq(
     ErgoUtilsApiRoute(ergoSettings),
-    InfoApiRoute(statsCollectorRef, settings.restApi, timeProvider),
+    InfoApiRoute(statsCollectorRef, scorexSettings.restApi, timeProvider),
     BlocksApiRoute(nodeViewHolderRef, readersHolderRef, ergoSettings),
     TransactionsApiRoute(readersHolderRef, nodeViewHolderRef, ergoSettings))
 

src/main/resources/api/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: "3.0.2"
 
 info:
-  version: "4.0.45"
+  version: "4.0.46"
   title: Ergo Node API
   description: API docs for Ergo Node. Models are shared between all Ergo products
   contact:

src/main/resources/application.conf

@@ -400,7 +400,7 @@ scorex {
     nodeName = "ergo-node"
 
     # Network protocol version to be sent in handshakes
-    appVersion = 4.0.45
+    appVersion = 4.0.46
 
     # Network agent name. May contain information about client code
     # stack, starting from core code-base up to the end graphical interface.

src/main/resources/mainnet.conf

@@ -65,7 +65,7 @@ scorex {
   network {
     magicBytes = [1, 0, 2, 4]
     bindAddress = "0.0.0.0:9030"
-    nodeName = "ergo-mainnet-4.0.45"
+    nodeName = "ergo-mainnet-4.0.46"
     nodeName = ${?NODENAME}
     knownPeers = [
       "213.239.193.208:9030",

src/main/scala/org/ergoplatform/ErgoApp.scala

@@ -11,7 +11,7 @@ import org.ergoplatform.http.api._
 import org.ergoplatform.local._
 import org.ergoplatform.mining.ErgoMiner
 import org.ergoplatform.mining.ErgoMiner.StartMining
-import org.ergoplatform.network.{ErgoNodeViewSynchronizer, ErgoSyncTracker, ModePeerFeature}
+import org.ergoplatform.network.{ErgoNodeViewSynchronizer, ErgoSyncTracker}
 import org.ergoplatform.nodeView.history.ErgoSyncInfoMessageSpec
 import org.ergoplatform.nodeView.{ErgoNodeViewRef, ErgoReadersHolderRef}
 import org.slf4j.{Logger, LoggerFactory}
@@ -48,9 +48,6 @@ class ErgoApp(args: Args) extends ScorexLogging {
   implicit private val actorSystem: ActorSystem = ActorSystem(scorexSettings.network.agentName)
   implicit private val executionContext: ExecutionContext = actorSystem.dispatcher
 
-  private val features: Seq[PeerFeature] = Seq(ModePeerFeature(ergoSettings.nodeSettings))
-  private val featureSerializers: PeerFeature.Serializers = features.map(f => f.featureId -> f.serializer).toMap
-
   private val timeProvider = new NetworkTimeProvider(scorexSettings.ntp)
 
   private val upnpGateway: Option[UPnPGateway] =
@@ -66,7 +63,7 @@ class ErgoApp(args: Args) extends ScorexLogging {
   private val basicSpecs = {
     Seq(
       GetPeersSpec,
-      new PeersSpec(featureSerializers, scorexSettings.network.maxPeerSpecObjects),
+      new PeersSpec(scorexSettings.network.maxPeerSpecObjects),
       InvSpec,
       RequestModifierSpec,
       ModifiersSpec
@@ -77,7 +74,6 @@ class ErgoApp(args: Args) extends ScorexLogging {
 
   private val scorexContext = ScorexContext(
     messageSpecs = basicSpecs ++ additionalMessageSpecs,
-    features = features,
     upnpGateway = upnpGateway,
     timeProvider = timeProvider,
     externalNodeAddress = externalSocketAddress
@@ -86,7 +82,7 @@ class ErgoApp(args: Args) extends ScorexLogging {
   private val peerManagerRef = PeerManagerRef(ergoSettings, scorexContext)
 
   private val networkControllerRef: ActorRef = NetworkControllerRef(
-    "networkController", scorexSettings, peerManagerRef, scorexContext)
+    "networkController", ergoSettings, peerManagerRef, scorexContext)
 
   private val nodeViewHolderRef: ActorRef = ErgoNodeViewRef(ergoSettings, timeProvider)
 
@@ -118,7 +114,7 @@ class ErgoApp(args: Args) extends ScorexLogging {
 
   if (ergoSettings.scorexSettings.network.peerDiscovery) {
     // Launching PeerSynchronizer actor which is then registering itself at network controller
-    PeerSynchronizerRef("PeerSynchronizer", networkControllerRef, peerManagerRef, scorexSettings.network, featureSerializers)
+    PeerSynchronizerRef("PeerSynchronizer", networkControllerRef, peerManagerRef, scorexSettings.network)
   }
 
   private val apiRoutes: Seq[ApiRoute] = Seq(

src/main/scala/org/ergoplatform/network/ModePeerFeature.scala

@@ -2,7 +2,7 @@ package org.ergoplatform.network
 
 import io.circe.{Encoder, Json}
 import org.ergoplatform.nodeView.state.StateType
-import org.ergoplatform.settings.{NodeConfigurationSettings, PeerFeatureIds}
+import org.ergoplatform.settings.{NodeConfigurationSettings, PeerFeatureDescriptors}
 import scorex.core.network.PeerFeature
 import scorex.core.network.PeerFeature.Id
 import scorex.core.serialization.ScorexSerializer
@@ -24,7 +24,7 @@ case class ModePeerFeature(stateType: StateType,
                            blocksToKeep: Int) extends PeerFeature {
   override type M = ModePeerFeature
 
-  override val featureId: Id = PeerFeatureIds.ModeFeatureId
+  override val featureId: Id = PeerFeatureDescriptors.ModeFeatureId
 
   override def serializer: ScorexSerializer[ModePeerFeature] = ModeFeatureSerializer
 }

src/main/scala/org/ergoplatform/nodeView/mempool/OrderedTxPool.scala

@@ -26,6 +26,17 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT
 
   import OrderedTxPool.weighted
 
+  /**
+    * When a transaction has a parent in the mempool, we update its weight, weight of parent's parents etc.
+    * This parameter sets max update depth
+    */
+  private val MaxParentScanDepth = 500
+
+  /**
+    * See `MaxParentScanDepth`, but this parameter sets max update time
+    */
+  private val MaxParentScanTime = 500
+
   private implicit val ms: MonetarySettings = settings.chainSettings.monetary
 
   private val mempoolCapacity = settings.nodeSettings.mempoolCapacity
@@ -57,7 +68,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT
       invalidatedTxIds,
       outputs ++ tx.outputs.map(_.id -> wtx),
       inputs ++ tx.inputs.map(_.boxId -> wtx)
-    ).updateFamily(unconfirmedTx, wtx.weight)
+    ).updateFamily(unconfirmedTx, wtx.weight, System.currentTimeMillis(), 0)
     if (newPool.orderedTransactions.size > mempoolCapacity) {
       val victim = newPool.orderedTransactions.last._2
       newPool.remove(victim)
@@ -85,7 +96,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT
           invalidatedTxIds,
           outputs -- tx.outputs.map(_.id),
           inputs -- tx.inputs.map(_.boxId)
-        ).updateFamily(unconfirmedTx, -wtx.weight)
+        ).updateFamily(unconfirmedTx, -wtx.weight, System.currentTimeMillis(), 0)
       case None => this
     }
   }
@@ -100,7 +111,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT
           invalidatedTxIds.put(tx.id),
           outputs -- tx.outputs.map(_.id),
           inputs -- tx.inputs.map(_.boxId)
-        ).updateFamily(unconfirmedTx, -wtx.weight)
+        ).updateFamily(unconfirmedTx, -wtx.weight, System.currentTimeMillis(), 0)
       case None =>
         OrderedTxPool(orderedTransactions, transactionsRegistry, invalidatedTxIds.put(tx.id), outputs, inputs)
     }
@@ -138,7 +149,7 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT
 
   /**
     *
-    * Form families of transactions: take in account relations between transaction when perform ordering.
+    * Form families of transactions: take in account relations between transactions when performing ordering.
     * If transaction X is spending output of transaction Y, then X weight should be greater than of Y.
     * Y should be proceeded prior to X or swapped out of mempool after X.
     * To achieve this goal we recursively add weight of new transaction to all transactions which
@@ -148,27 +159,33 @@ case class OrderedTxPool(orderedTransactions: TreeMap[WeightedTxId, UnconfirmedT
     * @param weight
     * @return
     */
-  private def updateFamily(unconfirmedTx: UnconfirmedTransaction, weight: Long): OrderedTxPool = {
-    val tx = unconfirmedTx.transaction
-    tx.inputs.foldLeft(this)((pool, input) =>
-      pool.outputs.get(input.boxId).fold(pool)(wtx => {
-        pool.orderedTransactions.get(wtx) match {
-          case Some(ut) =>
-            val parent = ut.transaction
-            val newWtx = WeightedTxId(wtx.id, wtx.weight + weight, wtx.feePerFactor, wtx.created)
-            val newPool = OrderedTxPool(pool.orderedTransactions - wtx + (newWtx -> ut),
-              pool.transactionsRegistry.updated(parent.id, newWtx),
-              invalidatedTxIds,
-              parent.outputs.foldLeft(pool.outputs)((newOutputs, box) => newOutputs.updated(box.id, newWtx)),
-              parent.inputs.foldLeft(pool.inputs)((newInputs, inp) => newInputs.updated(inp.boxId, newWtx))
-            )
-            newPool.updateFamily(ut, weight)
-          case None =>
-            //shouldn't be the case, but better not to hide this possibility
-            log.error("Could not find transaction in pool.orderedTransactions, please report to devs")
-            pool
-        }
-      }))
+  private def updateFamily(unconfirmedTx: UnconfirmedTransaction,
+                           weight: Long,
+                           startTime: Long,
+                           depth: Int): OrderedTxPool = {
+    val now = System.currentTimeMillis()
+    val timeDiff = now - startTime
+    if (depth > MaxParentScanDepth || timeDiff > MaxParentScanTime) {
+      log.warn(s"updateFamily takes too long, depth: $depth, time diff: $timeDiff, transaction: ${unconfirmedTx.id}")
+      this
+    } else {
+      val tx = unconfirmedTx.transaction
+
+      val uniqueTxIds: Set[WeightedTxId] = tx.inputs.flatMap(input => this.outputs.get(input.boxId))(collection.breakOut)
+      val parentTxs = uniqueTxIds.flatMap(wtx => this.orderedTransactions.get(wtx).map(ut => wtx -> ut))
+
+      parentTxs.foldLeft(this) { case (pool, (wtx, ut)) =>
+        val parent = ut.transaction
+        val newWtx = WeightedTxId(wtx.id, wtx.weight + weight, wtx.feePerFactor, wtx.created)
+        val newPool = OrderedTxPool(pool.orderedTransactions - wtx + (newWtx -> ut),
+          pool.transactionsRegistry.updated(parent.id, newWtx),
+          invalidatedTxIds,
+          parent.outputs.foldLeft(pool.outputs)((newOutputs, box) => newOutputs.updated(box.id, newWtx)),
+          parent.inputs.foldLeft(pool.inputs)((newInputs, inp) => newInputs.updated(inp.boxId, newWtx))
+        )
+        newPool.updateFamily(ut, weight, startTime, depth + 1)
+      }
+    }
   }
 }
 

src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala

@@ -1,7 +1,6 @@
 package org.ergoplatform.nodeView.state
 
 import java.io.File
-import cats.Traverse
 import org.ergoplatform.ErgoBox
 import org.ergoplatform.ErgoLikeContext.Height
 import org.ergoplatform.modifiers.history.header.Header
@@ -76,7 +75,6 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32
                                        headerId: ModifierId,
                                        expectedDigest: ADDigest,
                                        currentStateContext: ErgoStateContext): Try[Unit] = {
-    import cats.implicits._
     val createdOutputs = transactions.flatMap(_.outputs).map(o => (ByteArrayWrapper(o.id), o)).toMap
 
     def checkBoxExistence(id: ErgoBox.BoxId): Try[ErgoBox] = createdOutputs
@@ -87,13 +85,23 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32
     val txProcessing = ErgoState.execTransactions(transactions, currentStateContext)(checkBoxExistence)
     if (txProcessing.isValid) {
       log.debug(s"Cost of block $headerId (${currentStateContext.currentHeight}): ${txProcessing.payload.getOrElse(0)}")
-      val resultTry =
-        ErgoState.stateChanges(transactions).map { stateChanges =>
-          val mods = stateChanges.operations
-          Traverse[List].sequence(mods.map(persistentProver.performOneOperation).toList).map(_ => ())
+      val blockOpsTry = ErgoState.stateChanges(transactions).flatMap { stateChanges =>
+        val operations = stateChanges.operations
+        var opsResult: Try[Unit] = Success(())
+        operations.foreach { op =>
+          if (opsResult.isSuccess) {
+            persistentProver.performOneOperation(op) match {
+              case Success(_) =>
+              case Failure(t) =>
+                log.error(s"Operation $op failed during $headerId transactions validation")
+                opsResult = Failure(t)
+            }
+          }
         }
+        opsResult
+      }
       ModifierValidator(stateContext.validationSettings)
-        .validateNoFailure(fbOperationFailed, resultTry, Transaction.ModifierTypeId)
+        .validateNoFailure(fbOperationFailed, blockOpsTry, Transaction.ModifierTypeId)
         .validateEquals(fbDigestIncorrect, expectedDigest, persistentProver.digest, headerId, Header.modifierTypeId)
         .result
         .toTry

src/main/scala/org/ergoplatform/settings/PeerFeatureDescriptors.scala

@@ -0,0 +1,41 @@
+package org.ergoplatform.settings
+
+import org.ergoplatform.network.ModeFeatureSerializer
+import scorex.core.network.PeerFeature
+import scorex.core.network.peer.{LocalAddressPeerFeatureSerializer, RestApiUrlPeerFeatureSerializer, SessionIdPeerFeatureSerializer}
+
+/**
+  * Repository of existing peer feature identifiers, stores their ids along with serializers
+  */
+object PeerFeatureDescriptors {
+  /**
+    * See `LocalAddressPeerFeature`
+    */
+  val LocalAddressPeerFeatureId: Byte = 2: Byte
+
+  /**
+    * See `SessionIdPeerFeature`
+    */
+  val SessionIdPeerFeatureId: Byte = 3: Byte
+
+  /**
+    * See `RestApiUrlPeerFeature`
+    */
+  val RestApiUrlFeatureId: Byte = 4: Byte
+
+  /**
+    * See `ModePeerFeature`
+    */
+  val ModeFeatureId: Byte = 16: Byte
+
+  /**
+    * All the peer feature serializers should be here
+    */
+  val FeatureSerializers: PeerFeature.Serializers = Map(
+    LocalAddressPeerFeatureId -> LocalAddressPeerFeatureSerializer,
+    SessionIdPeerFeatureId -> SessionIdPeerFeatureSerializer,
+    RestApiUrlFeatureId -> RestApiUrlPeerFeatureSerializer,
+    ModeFeatureId -> ModeFeatureSerializer
+  )
+
+}