benchmark all dbs

Makefile

@@ -47,18 +47,18 @@ lint:
 bench:
 	cd benchmarks && \
 		go test $(LDFLAGS) -bench=RandomBytes . && \
-		go test $(LDFLAGS) -bench=Small . && \
-		go test $(LDFLAGS) -bench=Medium . && \
+		go test $(LDFLAGS) -tags cleveldb,rocksdb,boltdb,badgerdb -bench=Small . && \
+		go test $(LDFLAGS) -tags cleveldb,rocksdb,boltdb,badgerdb -bench=Medium . && \
 		go test $(LDFLAGS) -bench=BenchmarkMemKeySizes .
 .PHONY: bench
 
 # fullbench is extra tests needing lots of memory and to run locally
 fullbench:
 	cd benchmarks && \
 		go test $(LDFLAGS) -bench=RandomBytes . && \
-		go test $(LDFLAGS) -bench=Small . && \
-		go test $(LDFLAGS) -bench=Medium . && \
-		go test $(LDFLAGS) -timeout=30m -bench=Large . && \
+		go test $(LDFLAGS) -tags cleveldb,rocksdb,boltdb,badgerdb -bench=Small . && \
+		go test $(LDFLAGS) -tags cleveldb,rocksdb,boltdb,badgerdb -bench=Medium . && \
+		go test $(LDFLAGS) -tags cleveldb,rocksdb,boltdb,badgerdb -timeout=30m -bench=Large . && \
 		go test $(LDFLAGS) -bench=Mem . && \
 		go test $(LDFLAGS) -timeout=60m -bench=LevelDB .
 .PHONY: fullbench

benchmarks/README.md

@@ -37,3 +37,19 @@ Copy them back from your local machine:
 scp user@host:go/src/github.com/cosmos/iavl/results.txt results.txt
 git add results
 ```
+
+## Running benchmarks with docker
+
+Run the command below to install leveldb and rocksdb from source then run the benchmarks all the dbs (memdb, goleveldb, rocksdb, badgerdb) except boltdb.
+
+replace:
+- `baabeetaa` with your repo username and 
+- `fix-bencharks` with your branch.
+
+```
+docker run --rm -it ubuntu:16.04 /bin/bash -c \
+"apt-get update && apt-get install -y curl && \
+sh <(curl -s https://raw.githubusercontent.com/baabeetaa/iavl/fix-bencharks/benchmarks/setup/INSTALL_ROOT.sh) && \
+sh <(curl -s https://raw.githubusercontent.com/baabeetaa/iavl/fix-bencharks/benchmarks/setup/RUN_BENCHMARKS.sh) fix-bencharks baabeetaa && \
+cat ~/iavl/results.txt"
+```

benchmarks/bench_test.go

@@ -175,8 +175,11 @@ func BenchmarkMedium(b *testing.B) {
 	benchmarks := []benchmark{
 		{"memdb", 100000, 100, 16, 40},
 		{"goleveldb", 100000, 100, 16, 40},
-		// FIXME: this crashes on init! Either remove support, or make it work.
-		// {"cleveldb", 100000, 100, 16, 40},
+		{"cleveldb", 100000, 100, 16, 40},
+		// FIXME: idk why boltdb is too slow !?
+		//{"boltdb", 100000, 100, 16, 40},
+		{"rocksdb", 100000, 100, 16, 40},
+		{"badgerdb", 100000, 100, 16, 40},
 	}
 	runBenchmarks(b, benchmarks)
 }
@@ -185,8 +188,10 @@ func BenchmarkSmall(b *testing.B) {
 	benchmarks := []benchmark{
 		{"memdb", 1000, 100, 4, 10},
 		{"goleveldb", 1000, 100, 4, 10},
-		// FIXME: this crashes on init! Either remove support, or make it work.
-		// {"cleveldb", 100000, 100, 16, 40},
+		{"cleveldb", 1000, 100, 4, 10},
+		{"boltdb", 1000, 100, 4, 10},
+		{"rocksdb", 1000, 100, 4, 10},
+		{"badgerdb", 1000, 100, 4, 10},
 	}
 	runBenchmarks(b, benchmarks)
 }
@@ -195,8 +200,10 @@ func BenchmarkLarge(b *testing.B) {
 	benchmarks := []benchmark{
 		{"memdb", 1000000, 100, 16, 40},
 		{"goleveldb", 1000000, 100, 16, 40},
-		// FIXME: this crashes on init! Either remove support, or make it work.
-		// {"cleveldb", 100000, 100, 16, 40},
+		// FIXME: idk why boltdb is too slow !?
+		//{"boltdb", 1000000, 100, 16, 40},
+		{"rocksdb", 1000000, 100, 16, 40},
+		{"badgerdb", 1000000, 100, 16, 40},
 	}
 	runBenchmarks(b, benchmarks)
 }
@@ -233,6 +240,10 @@ func runBenchmarks(b *testing.B, benchmarks []benchmark) {
 
 		// prepare a dir for the db and cleanup afterwards
 		dirName := fmt.Sprintf("./%s-db", prefix)
+		if (bb.dbType == db.RocksDBBackend) || (bb.dbType == db.CLevelDBBackend) || (bb.dbType == db.BoltDBBackend) {
+			_ = os.Mkdir(dirName, 0755)
+		}
+
 		defer func() {
 			err := os.RemoveAll(dirName)
 			if err != nil {

benchmarks/setup/INSTALL_ROOT.sh

@@ -1,10 +1,32 @@
 #!/bin/sh
 
+export DEBIAN_FRONTEND=noninteractive
+
 apt-get update
 apt-get -y upgrade
-apt-get -y install make screen
+apt-get -y install screen wget git build-essential libsnappy-dev libgflags-dev zlib1g-dev libbz2-dev liblz4-dev libzstd-dev
+
+# Installing leveldb from source
+cd ~/
+git clone https://github.com/google/leveldb
+cd leveldb
+git checkout v1.7
+make -j2
+cp --preserve=links libleveldb.* /usr/local/lib
+cp -r include/leveldb /usr/local/include/
+ldconfig
+
+# installing rocksdb from source
+cd ~/
+git clone https://github.com/facebook/rocksdb
+cd rocksdb
+git checkout v6.15.5
+make -j4 install-shared
+ldconfig
 
-GOFILE=go1.12.7.linux-amd64.tar.gz
+# install go
+cd ~/
+mkdir go
+wget https://go.dev/dl/go1.17.6.linux-amd64.tar.gz
+tar -C /usr/local -xzf go1.17.6.linux-amd64.tar.gz
 
-wget https://storage.googleapis.com/golang/${GOFILE}
-tar -C /usr/local -xzf ${GOFILE}

benchmarks/setup/RUN_BENCHMARKS.sh

@@ -12,11 +12,16 @@ BRANCH=${1:-master}
 REPOUSER=${2:-tendermint}
 
 export PATH=$PATH:/usr/local/go/bin
-export PATH=$PATH:$HOME/go/bin
+export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin
+export GOROOT=/usr/local/go
 export GOPATH=$HOME/go
 
-go get -u github.com/${REPOUSER}/iavl
-cd ~/go/src/github.com/${REPOUSER}/iavl
+export CGO_CFLAGS="-I/usr/local/include"
+export CGO_LDFLAGS="-L/usr/local/lib -lleveldb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
+
+cd ~/
+git clone https://github.com/${REPOUSER}/iavl
+cd iavl
 git checkout ${BRANCH}
 
 make bench > results.txt