Added a more performant Hyperloglog.

Gopkg.toml

@@ -1,4 +1,3 @@
-
 # Gopkg.toml example
 #
 # Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md

samplers/samplers.go

@@ -4,12 +4,11 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
-	"hash/fnv"
 	"math"
 	"strings"
 	"time"
 
-	"github.com/clarkduvall/hyperloglog"
+	"github.com/axiomhq/hyperloglog"
 	"github.com/stripe/veneur/tdigest"
 )
 
@@ -213,22 +212,20 @@ func NewGauge(Name string, Tags []string) *Gauge {
 type Set struct {
 	Name string
 	Tags []string
-	Hll  *hyperloglog.HyperLogLogPlus
+	Hll  *hyperloglog.Sketch
 }
 
 // Sample checks if the supplied value has is already in the filter. If not, it increments
 // the counter!
 func (s *Set) Sample(sample string, sampleRate float32) {
-	hasher := fnv.New64a()
-	hasher.Write([]byte(sample))
-	s.Hll.Add(hasher)
+	s.Hll.Insert([]byte(sample))
 }
 
 // NewSet generates a new Set and returns it
 func NewSet(Name string, Tags []string) *Set {
 	// error is only returned if precision is outside the 4-18 range
 	// TODO: this is the maximum precision, should it be configurable?
-	Hll, _ := hyperloglog.NewPlus(18)
+	Hll := hyperloglog.New()
 	return &Set{
 		Name: Name,
 		Tags: Tags,
@@ -242,15 +239,15 @@ func (s *Set) Flush() []DDMetric {
 	copy(tags, s.Tags)
 	return []DDMetric{{
 		Name:       s.Name,
-		Value:      [1][2]float64{{float64(time.Now().Unix()), float64(s.Hll.Count())}},
+		Value:      [1][2]float64{{float64(time.Now().Unix()), float64(s.Hll.Estimate())}},
 		Tags:       tags,
 		MetricType: "gauge",
 	}}
 }
 
 // Export converts a Set into a JSONMetric which reports the Tags in the set.
 func (s *Set) Export() (JSONMetric, error) {
-	val, err := s.Hll.GobEncode()
+	val, err := s.Hll.MarshalBinary()
 	if err != nil {
 		return JSONMetric{}, err
 	}
@@ -267,8 +264,8 @@ func (s *Set) Export() (JSONMetric, error) {
 
 // Combine merges the values seen with another set (marshalled as a byte slice)
 func (s *Set) Combine(other []byte) error {
-	otherHLL, _ := hyperloglog.NewPlus(18)
-	if err := otherHLL.GobDecode(other); err != nil {
+	otherHLL := hyperloglog.New()
+	if err := otherHLL.UnmarshalBinary(other); err != nil {
 		return err
 	}
 	if err := s.Hll.Merge(otherHLL); err != nil {

samplers/samplers_test.go

@@ -135,7 +135,7 @@ func TestSetMerge(t *testing.T) {
 	for i := 0; i < 100; i++ {
 		s.Sample(strconv.Itoa(rand.Int()), 1.0)
 	}
-	assert.Equal(t, uint64(100), s.Hll.Count(), "counts did not match")
+	assert.Equal(t, uint64(100), s.Hll.Estimate(), "counts did not match")
 
 	jm, err := s.Export()
 	assert.NoError(t, err, "should have exported successfully")
@@ -144,8 +144,8 @@ func TestSetMerge(t *testing.T) {
 	assert.NoError(t, s2.Combine(jm.Value), "should have combined successfully")
 	// HLLs are approximate, and we've seen error of +-1 here in the past, so
 	// we're giving the test some room for error to reduce flakes
-	count1 := int(s.Hll.Count())
-	count2 := int(s2.Hll.Count())
+	count1 := int(s.Hll.Estimate())
+	count2 := int(s2.Hll.Estimate())
 	countDifference := count1 - count2
 	assert.True(t, -1 <= countDifference && countDifference <= 1, "counts did not match after merging (%d and %d)", count1, count2)
 }