Merge pull request #140 from elon0823/hotfix/replace-gorocksdb

Replace the `gorocksdb` dependency with the `grocksdb`

README.md

@@ -43,7 +43,7 @@ $ docker-compose -f {file name} up
   ```
   **MacOS**
     ```
-  $ brew install libtool autoconf coreutils cmake
+  $ brew install libtool autoconf coreutils cmake automake snappy zstd
   $ xcode-select --install // if command line tools are not installed,
   ```
 

broker/storage/qrocksdb.go

@@ -3,7 +3,7 @@ package storage
 import (
 	"encoding/binary"
 	"errors"
-	"github.com/tecbot/gorocksdb"
+	"github.com/linxGnu/grocksdb"
 	"path/filepath"
 	"unsafe"
 )
@@ -19,41 +19,41 @@ const (
 // QRocksDB is helper for gorocksdb
 type QRocksDB struct {
 	dbPath              string
-	db                  *gorocksdb.DB
-	ro                  *gorocksdb.ReadOptions
-	wo                  *gorocksdb.WriteOptions
-	columnFamilyHandles gorocksdb.ColumnFamilyHandles
+	db                  *grocksdb.DB
+	ro                  *grocksdb.ReadOptions
+	wo                  *grocksdb.WriteOptions
+	columnFamilyHandles grocksdb.ColumnFamilyHandles
 }
 
 func NewQRocksDB(name, dir string) (*QRocksDB, error) {
 
 	dbPath := filepath.Join(dir, name+".dbstorage")
 	columnFamilyNames := []string{"default", "topic", "record"}
 
-	bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
-	bbto.SetBlockCache(gorocksdb.NewLRUCache(1 << 30))
-	defaultOpts := gorocksdb.NewDefaultOptions()
+	bbto := grocksdb.NewDefaultBlockBasedTableOptions()
+	bbto.SetBlockCache(grocksdb.NewLRUCache(1 << 30))
+	defaultOpts := grocksdb.NewDefaultOptions()
 	defaultOpts.SetBlockBasedTableFactory(bbto)
 	defaultOpts.SetCreateIfMissing(true)
 	defaultOpts.SetCreateIfMissingColumnFamilies(true)
-	defaultOpts.SetCompression(gorocksdb.SnappyCompression)
-	opts := gorocksdb.NewDefaultOptions()
-	db, columnFamilyHandles, err := gorocksdb.OpenDbColumnFamilies(defaultOpts, dbPath, columnFamilyNames, []*gorocksdb.Options{opts, opts, opts})
+	defaultOpts.SetCompression(grocksdb.SnappyCompression)
+	opts := grocksdb.NewDefaultOptions()
+	db, columnFamilyHandles, err := grocksdb.OpenDbColumnFamilies(defaultOpts, dbPath, columnFamilyNames, []*grocksdb.Options{opts, opts, opts})
 	if err != nil {
 		return nil, err
 	}
 
-	ro := gorocksdb.NewDefaultReadOptions()
+	ro := grocksdb.NewDefaultReadOptions()
 	ro.SetTailing(true)
-	wo := gorocksdb.NewDefaultWriteOptions()
+	wo := grocksdb.NewDefaultWriteOptions()
 	return &QRocksDB{dbPath: dbPath, db: db, ro: ro, wo: wo, columnFamilyHandles: columnFamilyHandles}, nil
 }
 
 func (db QRocksDB) Flush() error {
-	return db.db.Flush(&gorocksdb.FlushOptions{})
+	return db.db.Flush(&grocksdb.FlushOptions{})
 }
 
-func (db QRocksDB) GetRecord(topic string, offset uint64) (*gorocksdb.Slice, error) {
+func (db QRocksDB) GetRecord(topic string, offset uint64) (*grocksdb.Slice, error) {
 	key := NewRecordKeyFromData(topic, offset)
 	return db.db.GetCF(db.ro, db.ColumnFamilyHandles()[RecordCF], key.Data())
 }
@@ -80,19 +80,19 @@ func (db *QRocksDB) Close() {
 }
 
 func (db *QRocksDB) Destroy() error {
-	return gorocksdb.DestroyDb(db.dbPath, gorocksdb.NewDefaultOptions())
+	return grocksdb.DestroyDb(db.dbPath, grocksdb.NewDefaultOptions())
 }
 
-func (db QRocksDB) ColumnFamilyHandles() gorocksdb.ColumnFamilyHandles {
+func (db QRocksDB) ColumnFamilyHandles() grocksdb.ColumnFamilyHandles {
 	return db.columnFamilyHandles
 }
 
-func (db QRocksDB) Scan(cfIndex CFIndex) *gorocksdb.Iterator {
+func (db QRocksDB) Scan(cfIndex CFIndex) *grocksdb.Iterator {
 	return db.db.NewIteratorCF(db.ro, db.ColumnFamilyHandles()[cfIndex])
 }
 
 type RecordKey struct {
-	*gorocksdb.Slice
+	*grocksdb.Slice
 	data    []byte
 	isSlice bool
 }
@@ -104,7 +104,7 @@ func NewRecordKeyFromData(topic string, offset uint64) *RecordKey {
 	return &RecordKey{data: data, isSlice: false}
 }
 
-func NewRecordKey(slice *gorocksdb.Slice) *RecordKey {
+func NewRecordKey(slice *grocksdb.Slice) *RecordKey {
 	return &RecordKey{Slice: slice, isSlice: true}
 }
 
@@ -135,7 +135,7 @@ func (k RecordKey) Offset() uint64 {
 }
 
 type RecordValue struct {
-	*gorocksdb.Slice
+	*grocksdb.Slice
 	data    []byte
 	isSlice bool
 }
@@ -149,7 +149,7 @@ func NewRecordValueFromData(nodeId string, seqNum uint64, publishedData []byte)
 	return &RecordValue{data: data, isSlice: false}
 }
 
-func NewRecordValue(slice *gorocksdb.Slice) *RecordValue {
+func NewRecordValue(slice *grocksdb.Slice) *RecordValue {
 	return &RecordValue{Slice: slice, isSlice: true}
 }
 

go.mod

@@ -5,32 +5,33 @@ go 1.17
 require (
 	github.com/golang/protobuf v1.4.0
 	github.com/google/uuid v1.3.0
+	github.com/linxGnu/grocksdb v1.6.45
 	github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.7.0
-	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
 	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
+	google.golang.org/protobuf v1.21.0
 	gopkg.in/confluentinc/confluent-kafka-go.v1 v1.4.2
 )
 
 require (
 	github.com/confluentinc/confluent-kafka-go v1.4.2 // indirect
-	github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c // indirect
-	github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
-	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mitchellh/mapstructure v1.3.2 // indirect
 	github.com/pelletier/go-toml v1.8.0 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/spf13/afero v1.3.2 // indirect
 	github.com/spf13/cast v1.3.1 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/stretchr/testify v1.7.0 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
 	golang.org/x/text v0.3.3 // indirect
-	google.golang.org/protobuf v1.21.0 // indirect
 	gopkg.in/ini.v1 v1.57.0 // indirect
 	gopkg.in/yaml.v2 v2.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

go.sum

@@ -42,12 +42,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0=
-github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64=
-github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A=
-github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg=
-github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk=
-github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
@@ -135,6 +129,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/linxGnu/grocksdb v1.6.45 h1:GbOFk71UKGl+5fjaYC5ZNZaIMDoNzXMKx5N56o6JnRo=
+github.com/linxGnu/grocksdb v1.6.45/go.mod h1:JcMMDBFaDNhRXFYcYXmgQwb/RarSld1PulTI7UzE+w0=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -211,12 +207,11 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
-github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok=
-github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
@@ -362,6 +357,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=