Utxo set bootstrapping preparation, part 5

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala

@@ -95,9 +95,10 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore)
     *
     * @param dumpStorage   - non-versioned storage to dump tree to
     * @param manifestDepth - depth of manifest tree
+    * @param expectedRootHash - expected UTXO set authenticating tree root hash
     * @return - hash of root node of tree, or failure if an error (e.g. in database) happened
     */
-  def dumpSnapshot(dumpStorage: LDBKVStore, manifestDepth: Int): Try[Array[Byte]] = {
+  def dumpSnapshot(dumpStorage: LDBKVStore, manifestDepth: Int, expectedRootHash: Array[Byte]): Try[Array[Byte]] = {
     store.processSnapshot { dbReader =>
 
       def subtreeLoop(label: DigestType, builder: mutable.ArrayBuilder[Byte]): Unit = {
@@ -138,6 +139,8 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore)
       val rootNodeLabel = dbReader.get(topNodeHashKey)
       val rootNodeHeight = Ints.fromByteArray(dbReader.get(topNodeHeightKey))
 
+      require(rootNodeLabel.sameElements(expectedRootHash), "Root node hash changed")
+
       val manifestBuilder = mutable.ArrayBuilder.make[Byte]()
       manifestBuilder.sizeHint(200000)
       manifestBuilder ++= Ints.toByteArray(rootNodeHeight)

avldb/src/main/scala/scorex/db/LDBKVStore.scala

@@ -14,13 +14,20 @@ import spire.syntax.all.cfor
   */
 class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging {
 
-  def update(toInsert: Array[(K, V)], toRemove: Array[K]): Try[Unit] = {
+  /**
+    * Update this database atomically with a batch of insertion and removal operations
+    *
+    * @param toInsertKeys - keys of key-value pairs to insert into database
+    * @param toInsertValues - values of key-value pairs to insert into database
+    * @param toRemove - keys of key-value pairs to remove from the database
+    * @return - error if it happens, or success status
+    */
+  def update(toInsertKeys: Array[K], toInsertValues: Array[V], toRemove: Array[K]): Try[Unit] = {
     val batch = db.createWriteBatch()
-    val insertLen = toInsert.length
-    val removeLen = toRemove.length
     try {
-      cfor(0)(_ < insertLen, _ + 1) { i => batch.put(toInsert(i)._1, toInsert(i)._2)}
-      cfor(0)(_ < removeLen, _ + 1) { i => batch.delete(toRemove(i))}
+      require(toInsertKeys.length == toInsertValues.length)
+      cfor(0)(_ < toInsertKeys.length, _ + 1) { i => batch.put(toInsertKeys(i), toInsertValues(i))}
+      cfor(0)(_ < toRemove.length, _ + 1) { i => batch.delete(toRemove(i))}
       db.write(batch)
       Success(())
     } catch {
@@ -45,9 +52,18 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging
     }
   }
 
-  def insert(values: Array[(K, V)]): Try[Unit] = update(values, Array.empty)
+  /**
+    * `update` variant where we only insert values into this database
+    */
+  def insert(keys: Array[K], values: Array[V]): Try[Unit] = update(keys, values, Array.empty)
+
+  def insert(values: Array[(K, V)]): Try[Unit] = update(values.map(_._1), values.map(_._2), Array.empty)
+
 
-  def remove(keys: Array[K]): Try[Unit] = update(Array.empty, keys)
+  /**
+    * `update` variant where we only remove values from this database
+    */
+  def remove(keys: Array[K]): Try[Unit] = update(Array.empty, Array.empty, keys)
 
   /**
     * Get last key within some range (inclusive) by used comparator.

avldb/src/main/scala/scorex/db/LDBVersionedStore.scala

@@ -422,8 +422,8 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int)
     */
   def processSnapshot[T](logic: SnapshotReadInterface => T): Try[T] = {
     val ro = new ReadOptions()
-    ro.snapshot(db.getSnapshot)
     try {
+      ro.snapshot(db.getSnapshot)
       object readInterface extends SnapshotReadInterface {
         def get(key: Array[Byte]): Array[Byte] = db.get(key, ro)
       }

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/LDBVersionedStoreSpecification.scala

@@ -17,7 +17,6 @@ class LDBVersionedStoreSpecification extends AnyPropSpec
 
   override protected val KL = 32
   override protected val VL = 8
-  override protected val LL = 32
 
   val storeTest: LDBVersionedStore => Unit = { store =>
     var version = store.lastVersionID

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala

@@ -6,11 +6,12 @@ import org.scalatest.Assertion
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.propspec.AnyPropSpec
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
+import scorex.crypto.authds.avltree.batch.VersionedLDBAVLStorage.topNodeHashKey
 import scorex.crypto.authds.avltree.batch.helpers.TestHelper
 import scorex.crypto.authds.{ADDigest, ADKey, ADValue, SerializedAdProof}
 import scorex.util.encode.Base16
 import scorex.crypto.hash.{Blake2b256, Digest32}
-import scorex.db.LDBVersionedStore
+import scorex.db.{LDBFactory, LDBVersionedStore}
 import scorex.utils.{Random => RandomBytes}
 
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -25,7 +26,6 @@ class VersionedLDBAVLStorageSpecification extends AnyPropSpec
 
   override protected val KL = 32
   override protected val VL = 8
-  override protected val LL = 32
 
   def kvGen: Gen[(ADKey, ADValue)] = for {
     key <- Gen.listOfN(KL, Arbitrary.arbitrary[Byte]).map(_.toArray) suchThat
@@ -340,4 +340,15 @@ class VersionedLDBAVLStorageSpecification extends AnyPropSpec
     testAddInfoSaving(createVersionedStore _)
   }
 
+  property("dumping snapshot") {
+    val prover = createPersistentProver()
+    blockchainWorkflowTest(prover)
+
+    val storage = prover.storage.asInstanceOf[VersionedLDBAVLStorage]
+    val store = LDBFactory.createKvDb("/tmp/aa")
+
+    storage.dumpSnapshot(store, 4, prover.digest.dropRight(1))
+    store.get(topNodeHashKey).sameElements(prover.digest.dropRight(1)) shouldBe true
+  }
+
 }

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala

@@ -27,7 +27,6 @@ object WithLDB extends VersionedLDBAVLStorageStatefulCommands with TestHelper {
 
   override protected val KL = 32
   override protected val VL = 8
-  override protected val LL = 32
 
   override protected def createStatefulProver: PersistentBatchAVLProver[Digest32, HF] = {
     createPersistentProver(keepVersions)

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala

@@ -19,7 +19,6 @@ trait TestHelper extends FileHelper {
 
   protected val KL: Int
   protected val VL: Int
-  protected val LL: Int
 
   implicit val hf: HF = Blake2b256
 
@@ -28,7 +27,8 @@ trait TestHelper extends FileHelper {
     new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions)
   }
 
-  def createVersionedStorage(store: LDBVersionedStore): STORAGE = new VersionedLDBAVLStorage(store)
+  def createVersionedStorage(store: LDBVersionedStore): STORAGE =
+    new VersionedLDBAVLStorage(store)
 
   def createPersistentProver(storage: STORAGE): PERSISTENT_PROVER = {
     val prover = new BatchAVLProver[D, HF](KL, Some(VL))

benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala

@@ -22,7 +22,7 @@ object UtxoStateBenchmark extends HistoryTestHelpers with NVBenchmark {
     val transactionsQty = blocks.flatMap(_.transactions).size
 
     def bench(mods: Seq[BlockSection]): Long = {
-      val state = ErgoState.generateGenesisUtxoState(createTempDir, StateConstants(realNetworkSetting))._1
+      val state = ErgoState.generateGenesisUtxoState(createTempDir, realNetworkSetting)._1
       Utils.time {
         mods.foldLeft(state) { case (st, mod) =>
           st.applyModifier(mod, None)(_ => ()).get

src/main/resources/application.conf

@@ -30,6 +30,9 @@ ergo {
     # Minimal suffix size for PoPoW proof (may be pre-defined constant or settings parameter)
     minimalSuffix = 10
 
+    # how many utxo set snapshots to store, 0 means that they are not stored at all
+    storingUtxoSnapshots = 0
+
     # Is the node is doing mining
     mining = false
 

src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala

@@ -406,8 +406,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti
     val history = ErgoHistory.readOrGenerate(settings)
     log.info("History database read")
     val memPool = ErgoMemPool.empty(settings)
-    val constants = StateConstants(settings)
-    restoreConsistentState(ErgoState.readOrGenerate(settings, constants).asInstanceOf[State], history) match {
+    restoreConsistentState(ErgoState.readOrGenerate(settings).asInstanceOf[State], history) match {
       case Success(state) =>
         log.info(s"State database read, state synchronized")
         val wallet = ErgoWallet.readOrGenerate(
@@ -533,8 +532,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti
     val dir = stateDir(settings)
     deleteRecursive(dir)
 
-    val constants = StateConstants(settings)
-    ErgoState.readOrGenerate(settings, constants)
+    ErgoState.readOrGenerate(settings)
       .asInstanceOf[State]
       .ensuring(
         state => java.util.Arrays.equals(state.rootDigest, settings.chainSettings.genesisStateDigest),
@@ -580,7 +578,6 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti
     * Recovers digest state from history.
     */
   private def recoverDigestState(bestFullBlock: ErgoFullBlock, history: ErgoHistory): Try[DigestState] = {
-    val constants = StateConstants(settings)
     val votingLength = settings.chainSettings.voting.votingLength
     val bestHeight = bestFullBlock.header.height
     val newEpochHeadersQty = bestHeight % votingLength // how many blocks current epoch lasts
@@ -592,12 +589,11 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti
 
     val recoveredStateTry = firstExtensionOpt
       .fold[Try[ErgoStateContext]](Failure(new Exception("Could not find extension to recover from"))
-      )(ext => ErgoStateContext.recover(constants.genesisStateDigest, ext, lastHeaders)(settings))
+      )(ext => ErgoStateContext.recover(settings.chainSettings.genesisStateDigest, ext, lastHeaders)(settings))
       .flatMap { ctx =>
         val recoverVersion = idToVersion(lastHeaders.last.id)
         val recoverRoot = bestFullBlock.header.stateRoot
-        val parameters = ctx.currentParameters
-        DigestState.recover(recoverVersion, recoverRoot, ctx, stateDir(settings), constants, parameters)
+        DigestState.recover(recoverVersion, recoverRoot, ctx, stateDir(settings), settings)
       }
 
     recoveredStateTry match {
@@ -609,7 +605,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti
       case Failure(exception) => // recover using whole headers chain
         log.warn(s"Failed to recover state from current epoch, using whole chain: ${exception.getMessage}")
         val wholeChain = history.headerChainBack(Int.MaxValue, bestFullBlock.header, _.isGenesis).headers
-        val genesisState = DigestState.create(None, None, stateDir(settings), constants)
+        val genesisState = DigestState.create(None, None, stateDir(settings), settings)
         wholeChain.foldLeft[Try[DigestState]](Success(genesisState)) { case (acc, m) =>
           acc.flatMap(_.applyModifier(m, history.estimatedTip())(lm => self ! lm))
         }

src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala

@@ -28,16 +28,14 @@ import scala.util.{Failure, Success, Try}
 class DigestState protected(override val version: VersionTag,
                             override val rootDigest: ADDigest,
                             override val store: LDBVersionedStore,
-                            ergoSettings: ErgoSettings)
+                            override val ergoSettings: ErgoSettings)
   extends ErgoState[DigestState]
     with ScorexLogging
     with ScorexEncoding {
 
   store.lastVersionID
     .foreach(id => require(version == bytesToVersion(id), "version should always be equal to store.lastVersionID"))
 
-  override val constants: StateConstants = StateConstants(ergoSettings)
-
   private lazy val nodeSettings = ergoSettings.nodeSettings
 
   private[state] def validateTransactions(transactions: Seq[ErgoTransaction],
@@ -163,46 +161,48 @@ object DigestState extends ScorexLogging with ScorexEncoding {
               rootHash: ADDigest,
               stateContext: ErgoStateContext,
               dir: File,
-              constants: StateConstants,
-              parameters: Parameters): Try[DigestState] = {
-    val store = new LDBVersionedStore(dir, initialKeepVersions = constants.keepVersions)
+              settings: ErgoSettings): Try[DigestState] = {
+    val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions)
     val toUpdate = DigestState.metadata(version, rootHash, stateContext)
 
     store.update(scorex.core.versionToBytes(version), Seq.empty, toUpdate).map { _ =>
-      new DigestState(version, rootHash, store, constants.settings)
+      new DigestState(version, rootHash, store, settings)
     }
   }
 
+  /**
+    * Read digest state from disk, or generate it from genesis data if nothing on the disk
+    */
   def create(versionOpt: Option[VersionTag],
              rootHashOpt: Option[ADDigest],
              dir: File,
-             constants: StateConstants): DigestState = {
-    val store = new LDBVersionedStore(dir, initialKeepVersions = constants.keepVersions)
+             settings: ErgoSettings): DigestState = {
+    val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions)
     Try {
-      val context = ErgoStateReader.storageStateContext(store, constants)
+      val context = ErgoStateReader.storageStateContext(store, settings)
       (versionOpt, rootHashOpt) match {
         case (Some(version), Some(rootHash)) =>
           val state = if (store.lastVersionID.map(w => bytesToVersion(w)).contains(version)) {
-            new DigestState(version, rootHash, store, constants.settings)
+            new DigestState(version, rootHash, store, settings)
           } else {
             val inVersion = store.lastVersionID.map(w => bytesToVersion(w)).getOrElse(version)
-            new DigestState(inVersion, rootHash, store, constants.settings)
+            new DigestState(inVersion, rootHash, store, settings)
               .update(version, rootHash, context).get //sync store
           }
           state.ensuring(bytesToVersion(store.lastVersionID.get) == version)
         case (None, None) if store.lastVersionID.isEmpty =>
-          ErgoState.generateGenesisDigestState(dir, constants.settings)
+          ErgoState.generateGenesisDigestState(dir, settings)
         case _ =>
           val version = store.lastVersionID.get
           val rootHash = store.get(version).get
-          new DigestState(bytesToVersion(version), ADDigest @@ rootHash, store, constants.settings)
+          new DigestState(bytesToVersion(version), ADDigest @@ rootHash, store, settings)
       }
     } match {
       case Success(state) => state
       case Failure(e) =>
         store.close()
         log.warn(s"Failed to create state with ${versionOpt.map(encoder.encode)} and ${rootHashOpt.map(encoder.encode)}", e)
-        ErgoState.generateGenesisDigestState(dir, constants.settings)
+        ErgoState.generateGenesisDigestState(dir, settings)
     }
   }
 

src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala

@@ -255,44 +255,53 @@ object ErgoState extends ScorexLogging {
   }
 
   /**
-    * All boxes of genesis state.
-    * Emission box is always the first.
+    * Genesis state boxes generator.
+    * Genesis state is corresponding to the state before the very first block processed.
+    * For Ergo mainnet, contains emission contract box, proof-of-no--premine box, and treasury contract box
     */
   def genesisBoxes(chainSettings: ChainSettings): Seq[ErgoBox] = {
     Seq(genesisEmissionBox(chainSettings), noPremineBox(chainSettings), genesisFoundersBox(chainSettings))
   }
 
-  def generateGenesisUtxoState(stateDir: File,
-                               constants: StateConstants): (UtxoState, BoxHolder) = {
+  /**
+    * Generate genesis full (UTXO-set) state by inserting genesis boxes into empty UTXO set.
+    * Assign `genesisStateDigest` from config as its version.
+    */
+  def generateGenesisUtxoState(stateDir: File, settings: ErgoSettings): (UtxoState, BoxHolder) = {
 
     log.info("Generating genesis UTXO state")
-    val boxes = genesisBoxes(constants.settings.chainSettings)
+    val boxes = genesisBoxes(settings.chainSettings)
     val bh = BoxHolder(boxes)
 
-    UtxoState.fromBoxHolder(bh, boxes.headOption, stateDir, constants, LaunchParameters).ensuring(us => {
+    UtxoState.fromBoxHolder(bh, boxes.headOption, stateDir, settings, LaunchParameters).ensuring(us => {
       log.info(s"Genesis UTXO state generated with hex digest ${Base16.encode(us.rootDigest)}")
-      java.util.Arrays.equals(us.rootDigest, constants.settings.chainSettings.genesisStateDigest) && us.version == genesisStateVersion
+      java.util.Arrays.equals(us.rootDigest, settings.chainSettings.genesisStateDigest) && us.version == genesisStateVersion
     }) -> bh
   }
 
+  /**
+    * Generate genesis digest state similarly to `generateGenesisUtxoState`, but without really storing boxes
+    */
   def generateGenesisDigestState(stateDir: File, settings: ErgoSettings): DigestState = {
-    DigestState.create(Some(genesisStateVersion), Some(settings.chainSettings.genesisStateDigest),
-      stateDir, StateConstants(settings))
+    DigestState.create(Some(genesisStateVersion), Some(settings.chainSettings.genesisStateDigest), stateDir, settings)
   }
 
   val preGenesisStateDigest: ADDigest = ADDigest @@ Array.fill(32)(0: Byte)
 
   lazy val genesisStateVersion: VersionTag = idToVersion(Header.GenesisParentId)
 
-  def readOrGenerate(settings: ErgoSettings,
-                     constants: StateConstants): ErgoState[_] = {
+  /**
+    * Read from disk or generate genesis UTXO-set or digest based state
+    * @param settings - config used to find state database or extract genesis boxes data
+    */
+  def readOrGenerate(settings: ErgoSettings): ErgoState[_] = {
     val dir = stateDir(settings)
     dir.mkdirs()
 
     settings.nodeSettings.stateType match {
-      case StateType.Digest => DigestState.create(None, None, dir, constants)
-      case StateType.Utxo if dir.listFiles().nonEmpty => UtxoState.create(dir, constants)
-      case _ => ErgoState.generateGenesisUtxoState(dir, constants)._1
+      case StateType.Digest => DigestState.create(None, None, dir, settings)
+      case StateType.Utxo if dir.listFiles().nonEmpty => UtxoState.create(dir, settings)
+      case _ => ErgoState.generateGenesisUtxoState(dir, settings)._1
     }
   }
 

src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala

@@ -310,10 +310,6 @@ object ErgoStateContext {
     */
   val eip27Vote: Byte = 8
 
-  def empty(constants: StateConstants, parameters: Parameters): ErgoStateContext = {
-    empty(constants.settings.chainSettings.genesisStateDigest, constants.settings, parameters)
-  }
-
   def empty(settings: ErgoSettings, parameters: Parameters): ErgoStateContext = {
     empty(settings.chainSettings.genesisStateDigest, settings, parameters)
   }