Candidate for 5.0.10 release

avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/AVLTreeBatchPerformance.scala

@@ -21,7 +21,7 @@ object AVLTreeBatchPerformance extends {
     val logger = LoggerFactory.getLogger("TEST")
     var prover: Prover = _
     var store: LDBVersionedStore = _
-    var storage: VersionedLDBAVLStorage[Digest32] = _
+    var storage: VersionedLDBAVLStorage = _
     var operations: Array[Operation] = _
 
     @Setup(Level.Iteration)

avldb/benchmarks/src/main/scala/scorex/crypto/authds/benchmarks/Helper.scala

@@ -34,11 +34,11 @@ object Helper {
   }
 
   def persistentProverWithVersionedStore(initialKeepVersions: Int,
-                                         baseOperationsCount: Int = 0): (Prover, LDBVersionedStore, VersionedLDBAVLStorage[Digest32]) = {
+                                         baseOperationsCount: Int = 0): (Prover, LDBVersionedStore, VersionedLDBAVLStorage) = {
     val dir = java.nio.file.Files.createTempDirectory("bench_testing_" + scala.util.Random.alphanumeric.take(15)).toFile
     dir.deleteOnExit()
     val store = new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions)
-    val storage = new VersionedLDBAVLStorage(store, NodeParameters(kl, Some(vl), ll))
+    val storage = new VersionedLDBAVLStorage(store)
     require(storage.isEmpty)
     val prover = new BatchAVLProver[Digest32, HF](kl, Some(vl))
 

avldb/build.sbt

@@ -5,7 +5,8 @@ name := "avldb"
 libraryDependencies ++= Seq(
   "javax.xml.bind" % "jaxb-api" % "2.4.0-b180830.0359",
   "ch.qos.logback" % "logback-classic" % "1.2.3",
-  "org.scorexfoundation" %% "scrypto" % "2.2.1"
+  "com.google.guava" % "guava" % "23.0",
+  "org.scorexfoundation" %% "scrypto" % "2.3.0"
 )
 
 libraryDependencies ++= Seq(

src/main/scala/scorex/core/serialization/ScorexSerializer.scala → avldb/src/main/scala/scorex/core/serialization/ErgoSerializer.scala

@@ -1,41 +1,37 @@
 package scorex.core.serialization
 
 import java.nio.ByteBuffer
-
-import akka.util.ByteString
 import scorex.util.ByteArrayBuilder
 import scorex.util.serialization._
 
 import scala.util.Try
 
-trait ScorexSerializer[T] extends Serializer[T, T, Reader, Writer] {
-
-  def toByteString(obj: T): ByteString = {
-    val writer = new VLQByteStringWriter()
-    serialize(obj, writer)
-    writer.result()
-  }
-
-  def parseByteString(byteString: ByteString): T = {
-    val reader = new VLQByteStringReader(byteString)
-    parse(reader)
-  }
-
-  def parseByteStringTry(byteString: ByteString): Try[T] = {
-    Try(parseByteString(byteString))
-  }
+/**
+  * Basic interface for serializer with additional methods to work with bytes, not only Reader/Writer instances
+  */
+trait ErgoSerializer[T] extends Serializer[T, T, Reader, Writer] {
 
+  /**
+    * Serialize object `obj` to byte array
+    */
   def toBytes(obj: T): Array[Byte] = {
     val writer = new VLQByteBufferWriter(new ByteArrayBuilder())
     serialize(obj, writer)
     writer.result().toBytes
   }
 
+
+  /**
+    * Deserialize byte array into object of type `T` (or throw exception)
+    */
   def parseBytes(bytes: Array[Byte]): T = {
     val reader = new VLQByteBufferReader(ByteBuffer.wrap(bytes))
     parse(reader)
   }
 
+  /**
+    * Deserialize byte array into object of type `T` (or return Failure)
+    */
   def parseBytesTry(bytes: Array[Byte]): Try[T] = {
     Try(parseBytes(bytes))
   }

avldb/src/main/scala/scorex/core/serialization/ManifestSerializer.scala

@@ -0,0 +1,69 @@
+package scorex.core.serialization
+
+import com.google.common.primitives.Ints
+import scorex.crypto.authds.avltree.batch.Constants.DigestType
+import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, ProxyInternalNode}
+import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes, VersionedLDBAVLStorage}
+import scorex.util.serialization.{Reader, Writer}
+
+/**
+  * Serializer of manifest, a tree which is cut at some `manifestDepth` from root
+  */
+class ManifestSerializer(manifestDepth: Byte) extends ErgoSerializer[BatchAVLProverManifest[DigestType]] {
+  private val nodeSerializer = VersionedLDBAVLStorage.noStoreSerializer
+
+  override def serialize(manifest: BatchAVLProverManifest[DigestType], w: Writer): Unit = {
+    val height = manifest.rootHeight
+    w.putBytes(Ints.toByteArray(height))
+    w.put(manifestDepth)
+
+    def loop(node: ProverNodes[DigestType], level: Int): Unit = {
+      nodeSerializer.serialize(node, w)
+      node match {
+        case _: ProverLeaf[DigestType] =>
+        case n: ProxyInternalNode[DigestType] if n.isEmpty =>
+        case i: InternalProverNode[DigestType] if level < manifestDepth =>
+          loop(i.left, level + 1)
+          loop(i.right, level + 1)
+      }
+    }
+
+    loop(manifest.root, level = 1)
+  }
+
+  override def parse(r: Reader): BatchAVLProverManifest[DigestType] = {
+    val rootHeight = Ints.fromByteArray(r.getBytes(4))
+    val manifestDepth = r.getByte()
+
+    require(manifestDepth == this.manifestDepth, "Wrong manifest depth")
+
+    def loop(level: Int): ProverNodes[DigestType] = {
+      val node = nodeSerializer.parse(r)
+      node match {
+        case _: ProverLeaf[DigestType] => node
+        case i: InternalProverNode[DigestType] if level < manifestDepth =>
+          val left = loop(level + 1)
+          val right = loop(level + 1)
+          i.getNew(newLeft = left, newRight = right)
+        case i: InternalProverNode[DigestType] if level == manifestDepth =>
+          i
+      }
+    }
+    val tree = loop(level = 1)
+    new BatchAVLProverManifest[DigestType](tree, rootHeight)
+  }
+
+}
+
+object ManifestSerializer {
+  /**
+    * Current manifest depth in the Ergo mainnet
+    */
+  val MainnetManifestDepth: Byte = 14
+
+  /**
+    * Manifest serializer used in the Ergo mainnet
+    */
+  val defaultSerializer = new ManifestSerializer(MainnetManifestDepth)
+}
+

avldb/src/main/scala/scorex/core/serialization/SubtreeSerializer.scala

@@ -0,0 +1,46 @@
+package scorex.core.serialization
+
+import scorex.crypto.authds.avltree.batch.Constants.DigestType
+import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes, VersionedLDBAVLStorage}
+import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverSubtree, ProxyInternalNode}
+import scorex.util.ScorexLogging
+import scorex.util.serialization.{Reader, Writer}
+
+/**
+  * Serializer for subtree
+  */
+object SubtreeSerializer extends ErgoSerializer[BatchAVLProverSubtree[DigestType]] with ScorexLogging {
+  private val nodeSerializer = VersionedLDBAVLStorage.noStoreSerializer
+
+  override def serialize(subtree: BatchAVLProverSubtree[DigestType], w: Writer): Unit = {
+    def loop(node: ProverNodes[DigestType]): Unit = {
+      nodeSerializer.serialize(node, w)
+      node match {
+        case _: ProverLeaf[DigestType] =>
+        case n: ProxyInternalNode[DigestType] if n.isEmpty =>
+          log.warn("Proxy node in subtree serialization")
+        case i: InternalProverNode[DigestType] =>
+          loop(i.left)
+          loop(i.right)
+      }
+    }
+
+    loop(subtree.subtreeTop)
+  }
+
+  override def parse(r: Reader): BatchAVLProverSubtree[DigestType] = {
+    def loop(): ProverNodes[DigestType] = {
+      val node = nodeSerializer.parse(r)
+      node match {
+        case _: ProverLeaf[DigestType] => node
+        case i: InternalProverNode[DigestType] =>
+          val left = loop()
+          val right = loop()
+          i.getNew(newLeft = left, newRight = right)
+      }
+    }
+    val tree = loop()
+    new BatchAVLProverSubtree[DigestType](tree)
+  }
+
+}

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/AvlTreeParameters.scala

@@ -0,0 +1,23 @@
+package scorex.crypto.authds.avltree.batch
+
+import Constants.HashLength
+
+/**
+  * Parameters of AVL+ tree nodes (internal and leaves)
+  * @param keySize - size of a key (fixed)
+  * @param valueSize - size of a value in a leaf (fixed, if defined, arbitrary, if None)
+  * @param labelSize - size of a label (node hash), fixed
+  */
+case class AvlTreeParameters(keySize: Int, valueSize: Option[Int], labelSize: Int) {
+  /**
+    * @return whether value is fixed-size
+    */
+  def fixedSizeValue: Boolean = valueSize.isDefined
+}
+
+
+/**
+  * AVL+ tree node parameters. The tree is used to authenticate UTXO set.
+  * Keys and hashes are 256-bits long, values are boxes, so value size is dynamic.
+  */
+object StateTreeParameters extends AvlTreeParameters(keySize = HashLength, valueSize = None, labelSize = HashLength)

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/Constants.scala

@@ -0,0 +1,31 @@
+package scorex.crypto.authds.avltree.batch
+
+import scorex.crypto.hash.{Blake2b256, Digest32}
+
+
+/**
+  * Commonly used constants
+  *
+  * //todo: move to core module once it is established
+  */
+object Constants {
+  /**
+    * Type of hash function output
+    */
+  type DigestType = Digest32
+
+  /**
+    * Length of hash function output
+    */
+  val HashLength: Int = 32
+
+  /**
+    * Type of hash function used in the protocol
+    */
+  type HashFnType = Blake2b256.type
+
+  /**
+    * Thread-safe instance of hash function used in the protocol
+    */
+  val hashFn: HashFnType = Blake2b256
+}

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProverNodeSerializer.scala

@@ -0,0 +1,71 @@
+package scorex.crypto.authds.avltree.batch
+
+import com.google.common.primitives.Ints
+import scorex.core.serialization.ErgoSerializer
+import scorex.crypto.authds.{ADKey, ADValue, Balance}
+import scorex.crypto.authds.avltree.batch.Constants.{DigestType, hashFn}
+import scorex.crypto.authds.avltree.batch.serialization.ProxyInternalNode
+import scorex.crypto.hash.Digest32
+import scorex.db.LDBVersionedStore
+import scorex.util.serialization.{Reader, Writer}
+
+/**
+  * ErgoSerializer based prover nodes serializer (alternative to one provided in scrypto which is doing much more
+  * allocations)
+  *
+  * @param store - store which could be provided to fetch children of a node when a child is requested
+  */
+class ProverNodeSerializer(store: LDBVersionedStore) extends ErgoSerializer[ProverNodes[DigestType]] {
+
+  import VersionedLDBAVLStorage.{InternalNodePrefix,LeafPrefix}
+
+  override def serialize(node: ProverNodes[DigestType], w: Writer): Unit = {
+    node match {
+      case n: ProxyInternalNode[DigestType] =>
+        w.put(InternalNodePrefix)
+        w.put(n.balance)
+        w.putBytes(n.key)
+        w.putBytes(n.leftLabel)
+        w.putBytes(n.rightLabel)
+      case n: InternalProverNode[DigestType] =>
+        w.put(InternalNodePrefix)
+        w.put(n.balance)
+        w.putBytes(n.key)
+        w.putBytes(n.left.label)
+        w.putBytes(n.right.label)
+      case n: ProverLeaf[DigestType] =>
+        w.put(LeafPrefix)
+        w.putBytes(n.key)
+        w.putBytes(Ints.toByteArray(n.value.length))
+        w.putBytes(n.value)
+        w.putBytes(n.nextLeafKey)
+    }
+  }
+
+  override def parse(r: Reader): ProverNodes[DigestType] = {
+    val prefix = r.getByte()
+    prefix match {
+      case InternalNodePrefix =>
+        val balance = Balance @@ r.getByte()
+        val key = ADKey @@ r.getBytes(StateTreeParameters.keySize)
+        val leftKey = ADKey @@ r.getBytes(StateTreeParameters.labelSize)
+        val rightKey = ADKey @@ r.getBytes(StateTreeParameters.labelSize)
+
+        if (store != null) {
+          new ProxyInternalProverNode(key, leftKey, rightKey, balance)(store)
+        } else {
+          new ProxyInternalNode[DigestType](key, Digest32 @@@ leftKey, Digest32 @@@ rightKey, balance)(hashFn)
+        }
+      case LeafPrefix =>
+        val key = ADKey @@ r.getBytes(StateTreeParameters.keySize)
+        val (value, nextLeafKey) =  {
+          val valueSize = Ints.fromByteArray(r.getBytes(4))
+          val value = ADValue @@ r.getBytes(valueSize)
+          val nextLeafKey = ADKey @@ r.getBytes(StateTreeParameters.keySize)
+          value -> nextLeafKey
+        }
+        new ProverLeaf[DigestType](key, value, nextLeafKey)(hashFn)
+    }
+  }
+
+}

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/ProxyInternalProverNode.scala

@@ -1,39 +1,37 @@
 package scorex.crypto.authds.avltree.batch
 
 import scorex.crypto.authds.{ADKey, Balance}
-import scorex.crypto.hash.{CryptographicHash, Digest}
 import scorex.db.LDBVersionedStore
 import InternalNode.InternalNodePrefix
+import scorex.crypto.authds.avltree.batch.Constants.{DigestType, hashFn}
 
 /**
   * Internal node where children are not provided during node construction, only pointers to them,
   * and then children nodes are read from database and constructed only when requested (and children internal nodes are
   * of the same type). It allows for lazy loading of a tree.
   *
   */
-class ProxyInternalProverNode[D <: Digest](protected var pk: ADKey,
+class ProxyInternalProverNode(protected var pk: ADKey,
                                            val leftLabel: ADKey,
                                            val rightLabel: ADKey,
                                            protected var pb: Balance = Balance @@ 0.toByte)
-                                          (implicit val phf: CryptographicHash[D],
-                                           store: LDBVersionedStore,
-                                           nodeParameters: NodeParameters)
-  extends InternalProverNode(k = pk, l = null, r = null, b = pb)(phf) {
+                                          (store: LDBVersionedStore)
+  extends InternalProverNode(k = pk, l = null, r = null, b = pb)(hashFn) {
 
-  override protected def computeLabel: D = {
+  override protected def computeLabel: DigestType = {
     hf.hash(Array(InternalNodePrefix, b), leftLabel, rightLabel)
   }
 
-  override def left: ProverNodes[D] = {
+  override def left: ProverNodes[DigestType] = {
     if (l == null) {
-      l = VersionedLDBAVLStorage.fetch[D](leftLabel)
+      l = VersionedLDBAVLStorage.fetch(leftLabel)(store)
     }
     l
   }
 
-  override def right: ProverNodes[D] = {
+  override def right: ProverNodes[DigestType] = {
     if (r == null) {
-      r = VersionedLDBAVLStorage.fetch[D](rightLabel)
+      r = VersionedLDBAVLStorage.fetch(rightLabel)(store)
     }
     r
   }