Fuzz storage proof decoder

.changelog/2637.internal.md

@@ -0,0 +1 @@
+go/storage/mkvs: Fuzz storage proof decoder

.changelog/2703.bugfix.1.md

@@ -0,0 +1 @@
+storage/mkvs: Fix node unmarshallers

.changelog/2703.bugfix.2.md

@@ -0,0 +1 @@
+storage/mkvs: Fix proof verifier

go/.codecov.yml

@@ -5,3 +5,6 @@ ignore:
   - "oasis-test-runner" # E2E test runner.
   - "oasis-net-runner" # Test local network runner.
   - "staking/gen_vectors" # Staking test vector generator.
+  - "storage/fuzz" # Fuzz tests.
+  - "storage/mkvs/urkel/fuzz" # Fuzz tests.
+  - "consensus/tendermint/fuzz" # Fuzz tests.

go/Makefile

@@ -69,37 +69,55 @@ integrationrunner:
 	@$(ECHO) "$(CYAN)*** Testing oasis-node with coverate...$(OFF)"
 	@$(GO) test $(GOFLAGS) -c -covermode=atomic -coverpkg=./... -o oasis-node/$@/[email protected] ./oasis-node/$@
 
-# Fuzzing binaries.
-fuzz-targets := consensus/tendermint/fuzz/fuzz-fuzz.zip \
-	storage/fuzz/fuzz-fuzz.zip \
-	storage/mkvs/urkel/fuzz/fuzz-fuzz.zip
+# Fuzzing.
+fuzz-targets := fuzz-consensus \
+	fuzz-storage \
+	fuzz-mkvs/Tree \
+	fuzz-mkvs/Proof \
+	fuzz-mkvs/Node
 
-build-fuzz: $(fuzz-targets)
-%/fuzz-fuzz.zip: .FORCE
-	@echo "Building $@"
-	@cd "$$(dirname "$@")"; go-fuzz-build
-	@cd "$$(dirname "$@")/gencorpus"; env -u GOPATH $(OASIS_GO) build -tags gofuzz
-
-# Run fuzzing.
 define canned-fuzz-run
-set -x; cd "$<"; \
-if ! [ -d corpus ]; then \
-	mkdir corpus; \
-	pushd corpus; \
-	../gencorpus/gencorpus; \
-	popd; \
+@TARGETDIR=$(shell pwd)/$<; \
+WORKDIR=/tmp/oasis-node-$@; \
+if [ "$(FUZZ_NO_BUILD)" != "1" ]; then \
+	mkdir -p "$$WORKDIR"; \
+	pushd $$TARGETDIR >/dev/null; \
+		$(ECHO) "$(CYAN)*** Building fuzzer for $@...$(OFF)"; \
+		go-fuzz-build -o $$WORKDIR/fuzz.zip; \
+	popd >/dev/null; \
 fi; \
-go-fuzz -bin=./fuzz-fuzz.zip
+if [ "$(FUZZ_BUILD_ONLY)" != "1" ]; then \
+	mkdir -p "$$WORKDIR"; \
+	cd "$$WORKDIR"; \
+	$(ECHO) "$(CYAN)*** Running fuzzer for $@...$(OFF)"; \
+	if [ "$(@D)" == "." ]; then \
+		go-fuzz -bin=$$WORKDIR/fuzz.zip -workdir=$$WORKDIR; \
+	else \
+		go-fuzz -bin=$$WORKDIR/fuzz.zip -workdir=$$WORKDIR -func Fuzz$(@F); \
+	fi; \
+fi;
 endef
+
+# Fuzz consensus transactions.
 fuzz-consensus: consensus/tendermint/fuzz/
 	$(canned-fuzz-run)
+# Fuzz general storage interface.
 fuzz-storage: storage/fuzz/ oasis-node/oasis-node
 	@mkdir -p /tmp/oasis-node-fuzz-storage/identity
 	@chmod 0700 /tmp/oasis-node-fuzz-storage/identity
 	@oasis-node/oasis-node identity init --datadir /tmp/oasis-node-fuzz-storage/identity
 	$(canned-fuzz-run)
-fuzz-storage-mkvs: storage/mkvs/urkel/fuzz/
+# Fuzz MKVS data structures.
+fuzz-mkvs/Tree: storage/mkvs/urkel/fuzz
+	$(canned-fuzz-run)
+fuzz-mkvs/Proof: storage/mkvs/urkel/fuzz
 	$(canned-fuzz-run)
+fuzz-mkvs/Node: storage/mkvs/urkel/fuzz
+	$(canned-fuzz-run)
+
+# Target that only builds all fuzzing infrastructure.
+build-fuzz: FUZZ_BUILD_ONLY=1
+build-fuzz: $(fuzz-targets)
 
 # Clean.
 clean:
@@ -109,7 +127,7 @@ clean:
 # List of targets that are not actual files.
 .PHONY: \
 	generate $(go-binaries) build \
-	$(test-helpers) build-helpers $(test-vectors) gen-test-vectors \
+	$(test-helpers) build-helpers $(test-vectors) gen-test-vectors $(fuzz-targets) \
 	fmt lint test integrationrunner clean all
 
 .FORCE:

go/common/fuzz/fuzz.go

@@ -5,105 +5,23 @@ package fuzz
 
 import (
 	"context"
-	"encoding/binary"
 	"fmt"
-	"math/rand"
 	"reflect"
 
-	gofuzz "github.com/google/gofuzz"
+	"github.com/thepudds/fzgo/randparam"
 )
 
-var (
-	_ rand.Source64 = (*Source)(nil)
-)
-
-// Source is a randomness source for the standard random generator.
-type Source struct {
-	Backing   []byte
-	Exhausted int
-
-	pos   int
-	track bool
-
-	traceback []byte
-}
-
-func (s *Source) Int63() int64 {
-	return int64(s.Uint64()&((1<<63)-1))
-}
-
-func (s *Source) Seed(_ int64) {
-	// Nothing to do here.
-}
-
-func (s *Source) Uint64() uint64 {
-	if s.pos+8 > len(s.Backing) {
-		s.Exhausted += 8
-		r := rand.Uint64()
-		if s.track {
-			chunk := make([]byte, 8)
-			binary.BigEndian.PutUint64(chunk[0:], r)
-			s.traceback = append(s.traceback, chunk...)
-		}
-		return r
-	}
-
-	s.pos += 8
-	return binary.BigEndian.Uint64(s.Backing[s.pos-8 : s.pos])
-}
-
-// GetTraceback returns the array of bytes returned from the random generator so far.
-func (s *Source) GetTraceback() []byte {
-	return s.traceback
-}
-
-// NewRandSource returns a new random source with the given backing array.
-func NewRandSource(backing []byte) *Source {
-	return &Source{
-		Backing: backing,
-	}
-}
-
-// NewTrackingRandSource returns a new random source that keeps track of the bytes returned.
-func NewTrackingRandSource() *Source {
-	return &Source{
-		Backing: []byte{},
-		track:   true,
-	}
-}
-
 // NewFilledInstance fills the given object with random values from the given blob.
-func NewFilledInstance(data []byte, typ interface{}) (interface{}, bool) {
+func NewFilledInstance(data []byte, typ interface{}) interface{} {
 	if typ == nil {
-		return nil, true
+		return nil
 	}
 
-	source := NewRandSource(data)
-	fuzzer := gofuzz.New()
-	fuzzer = fuzzer.RandSource(source)
-
+	fuzzer := randparam.NewFuzzer(data)
 	obj := reflect.New(reflect.TypeOf(typ)).Interface()
-
 	fuzzer.Fuzz(obj)
 
-	return obj, source.Exhausted == 0
-}
-
-// MakeSampleBlob creates and returns a sample blob of bytes for filling the given object.
-func MakeSampleBlob(typ interface{}) []byte {
-	if typ == nil {
-		return []byte{}
-	}
-
-	source := NewTrackingRandSource()
-	fuzzer := gofuzz.New()
-	fuzzer = fuzzer.RandSource(source)
-
-	obj := reflect.New(reflect.TypeOf(typ)).Interface()
-
-	fuzzer.Fuzz(obj)
-
-	return source.GetTraceback()
+	return obj
 }
 
 // InterfaceFuzzer is a helper class for fuzzing methods in structs or interfaces.
@@ -140,9 +58,7 @@ func (i *InterfaceFuzzer) DispatchBlob(blob []byte) ([]reflect.Value, bool) {
 	methType := i.typeObject.Method(meth).Type
 	method := i.valObject.Method(meth)
 
-	source := NewRandSource(blob[1:])
-	fuzzer := gofuzz.New()
-	fuzzer = fuzzer.RandSource(source).NilChance(0)
+	fuzzer := randparam.NewFuzzer(blob[1:])
 
 	in := []reflect.Value{}
 
@@ -167,49 +83,27 @@ func (i *InterfaceFuzzer) DispatchBlob(blob []byte) ([]reflect.Value, bool) {
 	return method.Call(in), true
 }
 
-// MakeSampleBlobs returns an array of sample blobs for all methods in the interface.
-func (i *InterfaceFuzzer) MakeSampleBlobs() [][]byte {
-	blobList := [][]byte{}
-	for seq, meth := range i.methodList {
-		source := NewTrackingRandSource()
-		fuzzer := gofuzz.New()
-		fuzzer = fuzzer.RandSource(source).NilChance(0)
-
-		method := i.typeObject.Method(meth)
-		blob := []byte{byte(seq)}
-		for arg := 1; arg < method.Type.NumIn(); arg++ {
-			inType := method.Type.In(arg)
-			inTypeName := fmt.Sprintf("%s.%s", inType.PkgPath(), inType.Name())
-			if _, ok := i.typeOverrides[inTypeName]; !ok {
-				newValue := reflect.New(inType)
-				if newValue.Interface() != nil {
-					fuzzer.Fuzz(newValue.Interface())
-				}
-			}
-		}
-
-		blob = append(blob, source.GetTraceback()...)
-		blobList = append(blobList, blob)
-	}
-
-	return blobList
-}
-
 // Method returns the method object associated with the fuzzer's index-th method for this instance.
 func (i *InterfaceFuzzer) Method(method int) reflect.Method {
 	return i.typeObject.Method(i.methodList[method])
 }
 
 // IgnoreMethodNames makes the interface fuzzer skip the named methods.
 func (i *InterfaceFuzzer) IgnoreMethodNames(names []string) {
-	for _, name := range names {
-		for listIndex, methIndex := range i.methodList {
-			if i.typeObject.Method(methIndex).Name == name {
-				i.methodList = append(i.methodList[:listIndex], i.methodList[listIndex+1:]...)
-				break
+	var newMethodList []int
+
+FilterLoop:
+	for _, index := range i.methodList {
+		name := i.typeObject.Method(index).Name
+		for _, ignoreName := range names {
+			if name == ignoreName {
+				continue FilterLoop
 			}
 		}
+		newMethodList = append(newMethodList, index)
 	}
+
+	i.methodList = newMethodList
 }
 
 // NewInterfaceFuzzer creates a new InterfaceFuzzer for the given instance.

go/consensus/tendermint/fuzz/fuzz.go

@@ -71,7 +71,7 @@ func Fuzz(data []byte) int {
 
 	methodName := FuzzableMethods[meth]
 
-	blob, _ := fuzz.NewFilledInstance(data, methodName.BodyType())
+	blob := fuzz.NewFilledInstance(data, methodName.BodyType())
 
 	tx := &transaction.Transaction{
 		Method: methodName,

go/go.mod

@@ -64,6 +64,7 @@ require (
 	github.com/tendermint/iavl v0.12.2
 	github.com/tendermint/tendermint v0.32.8
 	github.com/tendermint/tm-db v0.2.0
+	github.com/thepudds/fzgo v0.2.2
 	github.com/uber-go/atomic v1.4.0 // indirect
 	github.com/uber/jaeger-client-go v2.16.0+incompatible
 	github.com/uber/jaeger-lib v2.0.0+incompatible // indirect

go/go.sum

@@ -463,6 +463,8 @@ github.com/tendermint/go-amino v0.15.0 h1:TC4e66P59W7ML9+bxio17CPKnxW3nKIRAYsknt
 github.com/tendermint/go-amino v0.15.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME=
 github.com/tendermint/tm-db v0.2.0 h1:rJxgdqn6fIiVJZy4zLpY1qVlyD0TU6vhkT4kEf71TQQ=
 github.com/tendermint/tm-db v0.2.0/go.mod h1:0cPKWu2Mou3IlxecH+MEUSYc1Ch537alLe6CpFrKzgw=
+github.com/thepudds/fzgo v0.2.2 h1:bGofmgAGfTLpVgETkL9jvhg6azylvCF/kW6JPy5fkzQ=
+github.com/thepudds/fzgo v0.2.2/go.mod h1:ZgigL1toyKrar3rWdXz7Fuv7bUpKZ4BAYN49TpEFMCI=
 github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=