Skip to content

Commit

Permalink
server: Update the GolombCodedSet to hold the hex encoded filter
Browse files Browse the repository at this point in the history
  • Loading branch information
AlexITC committed Mar 19, 2019
1 parent 2f3d2b0 commit aeeb7f5
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 5 deletions.
8 changes: 5 additions & 3 deletions server/app/com/xsn/explorer/gcs/GolombCodedSet.scala
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,14 @@ class GolombCodedSet(
val p: Int,
val m: Int,
val n: Int,
val data: List[UnsignedByte]) {
val hex: HexString)

def hex: HexString = {
object GolombCodedSet {

def apply(p: Int, m: Int, n: Int, data: List[UnsignedByte]): GolombCodedSet = {
val string = data.map(_.byte).map("%02x".format(_)).mkString("")
HexString.from(string) match {
case Some(value) => value
case Some(value) => new GolombCodedSet(p = p, m = m, n = n, hex = value)
case None => throw new RuntimeException("Unexpected error, unable to create hex value")
}
}
Expand Down
24 changes: 23 additions & 1 deletion server/app/com/xsn/explorer/gcs/GolombEncoding.scala
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ class GolombEncoding(p: Int, m: Int, key: SipHashKey) {
.map { bits => UnsignedByte.parse(bits.padTo(8, Bit.Zero)) }
.toList

new GolombCodedSet(
GolombCodedSet.apply(
p = p,
m = m,
n = words.size,
Expand Down Expand Up @@ -115,6 +115,28 @@ class GolombEncoding(p: Int, m: Int, key: SipHashKey) {
List.fill(size - bits.size)(Bit.Zero) ++ bits
}

/**
* NOTE: This is a copy from https://github.com/btcsuite/btcutil/blob/master/gcs/gcs.go
* that is used for compatibility reasons, here we don't care about such optimizations
* because a filter is built once per block and never queried.
*
* Original docs:
* fastReduction calculates a mapping that's more ore less equivalent to: x mod N.
*
* However, instead of using a mod operation, which using a non-power of two
* will lead to slowness on many processors due to unnecessary division, we
* instead use a "multiply-and-shift" trick which eliminates all divisions,
* described in:
* https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/
*
* * v * N >> log_2(N)
*
* In our case, using 64-bit integers, log_2 is 64. As most processors don't
* support 128-bit arithmetic natively, we'll be super portable and unfold the
* operation into several operations with 64-bit arithmetic. As inputs, we the
* number to reduce, and our modulus N divided into its high 32-bits and lower
* 32-bits.
*/
private def fastReduction(v: BigInt, modulus: BigInt): BigInt = {
val nHi = modulus >> 32
val nLo = modulus & 0xFFFFFFFFL
Expand Down
9 changes: 8 additions & 1 deletion server/test/com/xsn/explorer/gcs/GolombEncodingSpec.scala
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
package com.xsn.explorer.gcs

import com.google.common.io.BaseEncoding
import org.scalatest.{MustMatchers, WordSpec}

class GolombEncodingSpec extends WordSpec with MustMatchers {
Expand Down Expand Up @@ -35,7 +36,13 @@ class GolombEncodingSpec extends WordSpec with MustMatchers {

"decode the same hashes" in {
val hashes = golomb.hashes(words)
val decoded = golomb.decode(encoded.data, words.size)
val bytes = BaseEncoding
.base16()
.decode(encoded.hex.string.toUpperCase)
.toList
.map(new UnsignedByte(_))

val decoded = golomb.decode(bytes, words.size)

decoded mustEqual hashes
}
Expand Down

0 comments on commit aeeb7f5

Please sign in to comment.