server: calculate block basic filter the spent pubKeyScripts and the …

…outputs pubKeyScripts
wiringbits · Aug 11, 2019 · 8d8765b · 8d8765b
1 parent ec2be99
commit 8d8765b
Show file tree

Hide file tree

Showing 136 changed files with 2,896 additions and 171 deletions.
diff --git a/server/app/com/xsn/explorer/data/LedgerDataHandler.scala b/server/app/com/xsn/explorer/data/LedgerDataHandler.scala
@@ -1,6 +1,7 @@
 package com.xsn.explorer.data
 
 import com.alexitc.playsonify.core.ApplicationResult
+import com.xsn.explorer.gcs.GolombCodedSet
 import com.xsn.explorer.models.TPoSContract
 import com.xsn.explorer.models.persisted.Block
 
@@ -17,7 +18,11 @@ trait LedgerDataHandler[F[_]] {
    * - The ledger is empty and the block is the genesis one.
    * - The ledger has some blocks and the block goes just after the latest one.
    */
-  def push(block: Block.HasTransactions, tposContracts: List[TPoSContract]): F[Unit]
+  def push(
+      block: Block.HasTransactions,
+      tposContracts: List[TPoSContract],
+      filterFactory: () => GolombCodedSet
+  ): F[Unit]
 
   /**
    * Remove the latest block from the ledger, it will succeed only if the ledger is not empty.

diff --git a/server/app/com/xsn/explorer/data/TransactionDataHandler.scala b/server/app/com/xsn/explorer/data/TransactionDataHandler.scala
@@ -1,10 +1,9 @@
 package com.xsn.explorer.data
 
 import com.alexitc.playsonify.core.ApplicationResult
-import com.alexitc.playsonify.models.ordering.{FieldOrdering, OrderingCondition}
-import com.alexitc.playsonify.models.pagination.{Limit, PaginatedQuery, PaginatedResult}
+import com.alexitc.playsonify.models.ordering.OrderingCondition
+import com.alexitc.playsonify.models.pagination.Limit
 import com.xsn.explorer.models._
-import com.xsn.explorer.models.fields.TransactionField
 import com.xsn.explorer.models.persisted.Transaction
 import com.xsn.explorer.models.values.{Address, Blockhash, TransactionId}
 

diff --git a/server/app/com/xsn/explorer/data/anorm/LedgerPostgresDataHandler.scala b/server/app/com/xsn/explorer/data/anorm/LedgerPostgresDataHandler.scala
@@ -7,7 +7,7 @@ import com.alexitc.playsonify.models.ApplicationError
 import com.xsn.explorer.data.LedgerBlockingDataHandler
 import com.xsn.explorer.data.anorm.dao._
 import com.xsn.explorer.errors.{PostgresForeignKeyViolationError, PreviousBlockMissingError, RepeatedBlockHeightError}
-import com.xsn.explorer.gcs.{GolombCodedSet, GolombEncoding}
+import com.xsn.explorer.gcs.GolombCodedSet
 import com.xsn.explorer.models.TPoSContract
 import com.xsn.explorer.models.persisted.{Balance, Block}
 import com.xsn.explorer.util.Extensions.ListOptionExt
@@ -33,10 +33,14 @@ class LedgerPostgresDataHandler @Inject()(
    * Push a block into the database chain, note that even if the block is supposed
    * to have a next block, we remove the link because that block is not stored yet.
    */
-  override def push(block: Block.HasTransactions, tposContracts: List[TPoSContract]): ApplicationResult[Unit] = {
+  override def push(
+      block: Block.HasTransactions,
+      tposContracts: List[TPoSContract],
+      filterFactory: () => GolombCodedSet
+  ): ApplicationResult[Unit] = {
 
     // the filter is computed outside the transaction to avoid unnecessary locking
-    val filter = GolombEncoding.encode(block)
+    val filter = filterFactory()
     val result = withTransaction { implicit conn =>
       val result = for {
         _ <- upsertBlockCascade(block.asTip, filter, tposContracts)
@@ -72,17 +76,15 @@ class LedgerPostgresDataHandler @Inject()(
 
   private def upsertBlockCascade(
       block: Block.HasTransactions,
-      filter: Option[GolombCodedSet],
+      filter: GolombCodedSet,
       tposContracts: List[TPoSContract]
   )(implicit conn: Connection): Option[Unit] = {
 
     val result = for {
       // block
       _ <- deleteBlockCascade(block.block).orElse(Some(()))
       _ <- blockPostgresDAO.insert(block.block)
-      _ = filter.foreach { f =>
-        blockFilterPostgresDAO.insert(block.hash, f)
-      }
+      _ = blockFilterPostgresDAO.insert(block.hash, filter)
 
       // batch insert
       _ <- transactionPostgresDAO.insert(block.transactions, tposContracts)

diff --git a/server/app/com/xsn/explorer/data/anorm/TransactionPostgresDataHandler.scala b/server/app/com/xsn/explorer/data/anorm/TransactionPostgresDataHandler.scala
@@ -1,13 +1,12 @@
 package com.xsn.explorer.data.anorm
 
 import com.alexitc.playsonify.core.ApplicationResult
-import com.alexitc.playsonify.models.ordering.{FieldOrdering, OrderingCondition}
-import com.alexitc.playsonify.models.pagination.{Limit, PaginatedQuery, PaginatedResult}
+import com.alexitc.playsonify.models.ordering.OrderingCondition
+import com.alexitc.playsonify.models.pagination.Limit
 import com.xsn.explorer.data.TransactionBlockingDataHandler
 import com.xsn.explorer.data.anorm.dao.{TransactionOutputPostgresDAO, TransactionPostgresDAO}
 import com.xsn.explorer.errors.TransactionError
 import com.xsn.explorer.models._
-import com.xsn.explorer.models.fields.TransactionField
 import com.xsn.explorer.models.persisted.Transaction
 import com.xsn.explorer.models.values.{Address, Blockhash, TransactionId}
 import javax.inject.Inject

diff --git a/server/app/com/xsn/explorer/data/anorm/dao/BlockFilterPostgresDAO.scala b/server/app/com/xsn/explorer/data/anorm/dao/BlockFilterPostgresDAO.scala
@@ -25,7 +25,7 @@ class BlockFilterPostgresDAO {
         'm -> filter.m,
         'n -> filter.n,
         'p -> filter.p,
-        'hex -> filter.hex.string
+        'hex -> filter.getHexString.string
       )
       .as(parseFilter.single)
   }
@@ -48,7 +48,7 @@ class BlockFilterPostgresDAO {
         'm -> filter.m,
         'n -> filter.n,
         'p -> filter.p,
-        'hex -> filter.hex.string
+        'hex -> filter.getHexString.string
       )
       .execute()
   }

diff --git a/server/app/com/xsn/explorer/data/anorm/parsers/BlockFilterParsers.scala b/server/app/com/xsn/explorer/data/anorm/parsers/BlockFilterParsers.scala
@@ -3,6 +3,7 @@ package com.xsn.explorer.data.anorm.parsers
 import anorm.SqlParser._
 import anorm._
 import com.xsn.explorer.gcs.GolombCodedSet
+import com.xsn.explorer.models.values.{CompactSizeInt, HexString}
 
 object BlockFilterParsers {
 
@@ -15,11 +16,12 @@ object BlockFilterParsers {
 
   val parseFilter = (parseN ~ parseM ~ parseP ~ parseHex).map {
     case n ~ m ~ p ~ hex =>
+      val compactN = CompactSizeInt(n)
       new GolombCodedSet(
         n = n,
         m = m,
         p = p,
-        hex = hex
+        hex = hex.drop(compactN.hex.length)
       )
   }
 }
diff --git a/server/app/com/xsn/explorer/data/anorm/parsers/TransactionParsers.scala b/server/app/com/xsn/explorer/data/anorm/parsers/TransactionParsers.scala
@@ -32,11 +32,12 @@ object TransactionParsers {
       TransactionWithValues(txid, blockhash, time, size, sent, received)
   }
 
-  val parseTransactionInput = (parseFromTxid ~ parseFromOutputIndex ~ parseIndex ~ parseValue ~ parseAddresses)
-    .map {
-      case fromTxid ~ fromOutputIndex ~ index ~ value ~ addresses =>
-        Transaction.Input(fromTxid, fromOutputIndex, index, value, addresses)
-    }
+  val parseTransactionInput =
+    (parseFromTxid ~ parseFromOutputIndex ~ parseIndex ~ parseValue ~ parseAddresses)
+      .map {
+        case fromTxid ~ fromOutputIndex ~ index ~ value ~ addresses =>
+          Transaction.Input(fromTxid, fromOutputIndex, index, value, addresses)
+      }
 
   val parseTransactionOutput = (parseTransactionId() ~
     parseIndex ~

diff --git a/server/app/com/xsn/explorer/data/async/LedgerFutureDataHandler.scala b/server/app/com/xsn/explorer/data/async/LedgerFutureDataHandler.scala
@@ -3,6 +3,7 @@ package com.xsn.explorer.data.async
 import com.alexitc.playsonify.core.FutureApplicationResult
 import com.xsn.explorer.data.{LedgerBlockingDataHandler, LedgerDataHandler}
 import com.xsn.explorer.executors.DatabaseExecutionContext
+import com.xsn.explorer.gcs.GolombCodedSet
 import com.xsn.explorer.models.TPoSContract
 import com.xsn.explorer.models.persisted.Block
 import javax.inject.Inject
@@ -13,10 +14,15 @@ class LedgerFutureDataHandler @Inject()(blockingDataHandler: LedgerBlockingDataH
     implicit ec: DatabaseExecutionContext
 ) extends LedgerDataHandler[FutureApplicationResult] {
 
-  override def push(block: Block.HasTransactions, tposContracts: List[TPoSContract]): FutureApplicationResult[Unit] =
+  override def push(
+      block: Block.HasTransactions,
+      tposContracts: List[TPoSContract],
+      filterFactory: () => GolombCodedSet
+  ): FutureApplicationResult[Unit] = {
     Future {
-      blockingDataHandler.push(block, tposContracts)
+      blockingDataHandler.push(block, tposContracts, filterFactory)
     }
+  }
 
   override def pop(): FutureApplicationResult[Block] = Future {
     blockingDataHandler.pop()

diff --git a/server/app/com/xsn/explorer/data/async/TransactionFutureDataHandler.scala b/server/app/com/xsn/explorer/data/async/TransactionFutureDataHandler.scala
@@ -1,12 +1,11 @@
 package com.xsn.explorer.data.async
 
-import com.alexitc.playsonify.core.{FutureApplicationResult, FuturePaginatedResult}
-import com.alexitc.playsonify.models.ordering.{FieldOrdering, OrderingCondition}
-import com.alexitc.playsonify.models.pagination.{Limit, PaginatedQuery}
+import com.alexitc.playsonify.core.FutureApplicationResult
+import com.alexitc.playsonify.models.ordering.OrderingCondition
+import com.alexitc.playsonify.models.pagination.Limit
 import com.xsn.explorer.data.{TransactionBlockingDataHandler, TransactionDataHandler}
 import com.xsn.explorer.executors.DatabaseExecutionContext
 import com.xsn.explorer.models._
-import com.xsn.explorer.models.fields.TransactionField
 import com.xsn.explorer.models.persisted.Transaction
 import com.xsn.explorer.models.values.{Address, Blockhash, TransactionId}
 import javax.inject.Inject

diff --git a/server/app/com/xsn/explorer/gcs/GolombCodedSet.scala b/server/app/com/xsn/explorer/gcs/GolombCodedSet.scala
@@ -1,8 +1,14 @@
 package com.xsn.explorer.gcs
 
-import com.xsn.explorer.models.values.HexString
+import com.xsn.explorer.models.values.{CompactSizeInt, HexString}
 
-class GolombCodedSet(val p: Int, val m: Int, val n: Int, val hex: HexString)
+class GolombCodedSet(val p: Int, val m: Int, val n: Int, val hex: HexString) {
+
+  def getHexString: HexString = {
+    val compactSizeInt = CompactSizeInt(n)
+    compactSizeInt.hex.concat(hex)
+  }
+}
 
 object GolombCodedSet {
 

diff --git a/server/app/com/xsn/explorer/gcs/GolombEncoding.scala b/server/app/com/xsn/explorer/gcs/GolombEncoding.scala
@@ -1,7 +1,7 @@
 package com.xsn.explorer.gcs
 
 import com.google.common.hash.Hashing
-import com.xsn.explorer.models.persisted.Block
+import com.xsn.explorer.models._
 
 import scala.collection.SortedSet
 
@@ -20,17 +20,20 @@ class GolombEncoding(p: Int, m: Int, key: SipHashKey) {
   /**
    * Encodes the given word set.
    */
-  def encode(words: Set[String]): Option[GolombCodedSet] = {
-    if (words.isEmpty) {
-      Option.empty
+  def encode(words: Set[String]): GolombCodedSet = {
+    encodeData(words.map(_.getBytes))
+  }
+
+  def encodeData(data: Set[Array[Byte]]): GolombCodedSet = {
+    if (data.isEmpty) {
+      GolombCodedSet(p = p, m = m, n = data.size, data = List.empty)
     } else {
-      val gcs = encodeNonEmptySet(words)
-      Option(gcs)
+      encodeNonEmptySet(data)
     }
   }
 
-  private def encodeNonEmptySet(words: Set[String]): GolombCodedSet = {
-    val sortedHashes = hashes(words)
+  private def encodeNonEmptySet(data: Set[Array[Byte]]): GolombCodedSet = {
+    val sortedHashes = hashes(data)
     val diffList = differences(sortedHashes)
     val encodedBits = diffList.flatMap(golombEncode)
     val encodedBytes = encodedBits
@@ -40,7 +43,7 @@ class GolombEncoding(p: Int, m: Int, key: SipHashKey) {
       }
       .toList
 
-    GolombCodedSet.apply(p = p, m = m, n = words.size, data = encodedBytes)
+    GolombCodedSet.apply(p = p, m = m, n = data.size, data = encodedBytes)
   }
 
   /**
@@ -70,10 +73,10 @@ class GolombEncoding(p: Int, m: Int, key: SipHashKey) {
   /**
    * Maps the word set to a sorted set of hashes.
    */
-  private[gcs] def hashes(words: Set[String]): SortedSet[BigInt] = {
-    val modulus = BigInt(m) * words.size
+  private[gcs] def hashes(data: Set[Array[Byte]]): SortedSet[BigInt] = {
+    val modulus = BigInt(m) * data.size
     val f = fastReduction(_: BigInt, modulus)
-    words
+    data
       .map(hash)
       .map(f)
       .to[SortedSet]
@@ -107,8 +110,8 @@ class GolombEncoding(p: Int, m: Int, key: SipHashKey) {
       .toList
   }
 
-  private def hash(string: String): BigInt = {
-    val x = hasher.hashBytes(string.getBytes)
+  private def hash(bytes: Array[Byte]): BigInt = {
+    val x = hasher.hashBytes(bytes)
     BigInt(java.lang.Long.toUnsignedString(x.asLong()))
   }
 
@@ -182,10 +185,20 @@ object GolombEncoding {
     new GolombEncoding(p = DefaultP, m = DefaultM, key = key)
   }
 
-  def encode(block: Block.HasTransactions): Option[GolombCodedSet] = {
+  def encode(block: rpc.Block.HasTransactions[rpc.TransactionVIN.HasValues]): GolombCodedSet = {
     val key = SipHashKey.fromBtcutil(block.hash)
     val encoder = default(key)
-    val addresses = block.collectAddresses
-    encoder.encode(addresses.map(_.string))
+
+    val pubKeyScripts = block.transactions.flatMap { transaction =>
+      val inputScripts = transaction.vin.map(_.pubKeyScript)
+      val outputScripts = transaction.vout.flatMap(_.scriptPubKey).collect {
+        case script if !script.asm.contains("OP_RETURN") => script.hex
+      }
+
+      inputScripts ::: outputScripts
+    }
+
+    val data = pubKeyScripts.distinct.map(_.toBytes).filterNot(_.isEmpty).toSet
+    encoder.encodeData(data)
   }
 }
diff --git a/server/app/com/xsn/explorer/models/TransactionDetails.scala b/server/app/com/xsn/explorer/models/TransactionDetails.scala
@@ -25,7 +25,7 @@ object TransactionDetails {
 
   def from(tx: rpc.Transaction[rpc.TransactionVIN.HasValues]): TransactionDetails = {
     val input = tx.vin.map { vin =>
-      TransactionValue(vin.addresses, vin.value)
+      TransactionValue(vin.addresses, vin.value, vin.pubKeyScript)
     }
 
     val output = tx.vout.flatMap(TransactionValue.from)

diff --git a/server/app/com/xsn/explorer/models/TransactionValue.scala b/server/app/com/xsn/explorer/models/TransactionValue.scala
@@ -1,22 +1,24 @@
 package com.xsn.explorer.models
 
 import com.xsn.explorer.models.rpc.TransactionVOUT
-import com.xsn.explorer.models.values.Address
+import com.xsn.explorer.models.values.{Address, HexString}
 import play.api.libs.json.{JsNull, JsString, Json, Writes}
 
-case class TransactionValue(addresses: List[Address], value: BigDecimal) {
+case class TransactionValue(addresses: List[Address], value: BigDecimal, pubKeyScript: HexString) {
   def address: Option[Address] = addresses.headOption
 }
 
 object TransactionValue {
 
-  def apply(address: Address, value: BigDecimal): TransactionValue = {
-    TransactionValue(List(address), value)
+  def apply(address: Address, value: BigDecimal, pubKeyScript: HexString): TransactionValue = {
+    TransactionValue(List(address), value, pubKeyScript)
   }
 
   def from(vout: TransactionVOUT): Option[TransactionValue] = {
-    for (addresses <- vout.addresses)
-      yield TransactionValue(addresses, vout.value)
+    for {
+      addresses <- vout.addresses
+      pubKeyScript <- vout.scriptPubKey
+    } yield TransactionValue(addresses, vout.value, pubKeyScript.hex)
   }
 
   implicit val writes: Writes[TransactionValue] = (obj: TransactionValue) => {

diff --git a/server/app/com/xsn/explorer/models/persisted/Transaction.scala b/server/app/com/xsn/explorer/models/persisted/Transaction.scala
@@ -24,7 +24,13 @@ object Transaction {
 
   object Input {
 
-    def apply(fromTxid: TransactionId, fromOutputIndex: Int, index: Int, value: BigDecimal, address: Address): Input = {
+    def apply(
+        fromTxid: TransactionId,
+        fromOutputIndex: Int,
+        index: Int,
+        value: BigDecimal,
+        address: Address
+    ): Input = {
 
       Input(fromTxid, fromOutputIndex, index, value, List(address))
     }

diff --git a/server/app/com/xsn/explorer/models/rpc/TransactionVIN.scala b/server/app/com/xsn/explorer/models/rpc/TransactionVIN.scala
@@ -1,19 +1,19 @@
 package com.xsn.explorer.models.rpc
 
-import com.xsn.explorer.models.values.{Address, TransactionId}
+import com.xsn.explorer.models.values.{Address, HexString, TransactionId}
 import play.api.libs.functional.syntax._
 import play.api.libs.json.{Reads, __}
 
 sealed trait TransactionVIN {
   def txid: TransactionId
   def voutIndex: Int
 
-  def withValues(value: BigDecimal, addresses: List[Address]): TransactionVIN.HasValues = {
-    TransactionVIN.HasValues(txid, voutIndex, value, addresses)
+  def withValues(value: BigDecimal, addresses: List[Address], pubKeyScript: HexString): TransactionVIN.HasValues = {
+    TransactionVIN.HasValues(txid, voutIndex, value, addresses, pubKeyScript)
   }
 
-  def withValues(value: BigDecimal, address: Address): TransactionVIN.HasValues = {
-    TransactionVIN.HasValues(txid, voutIndex, value, List(address))
+  def withValues(value: BigDecimal, address: Address, pubKeyScript: HexString): TransactionVIN.HasValues = {
+    TransactionVIN.HasValues(txid, voutIndex, value, List(address), pubKeyScript)
   }
 }
 
@@ -24,7 +24,8 @@ object TransactionVIN {
       override val txid: TransactionId,
       override val voutIndex: Int,
       value: BigDecimal,
-      addresses: List[Address]
+      addresses: List[Address],
+      pubKeyScript: HexString
   ) extends TransactionVIN
 
   implicit val reads: Reads[TransactionVIN] = {