diff --git a/combinators.go b/combinators.go index 991063d..cacaecd 100644 --- a/combinators.go +++ b/combinators.go @@ -98,8 +98,10 @@ func (g *filteredGen) value(t *T) value { func (g *filteredGen) maybeValue(t *T) value { v := g.g.value(t) if g.fn(v) { + Event(t, g.String(), "satisfied") return v } else { + Event(t, g.String(), "failed") return nil } } diff --git a/engine.go b/engine.go index 62fb1b8..9bdb779 100644 --- a/engine.go +++ b/engine.go @@ -263,6 +263,9 @@ func findBug(tb tb, checks int, seed uint64, prop func(*T)) (uint64, int, int, * return seed, valid, invalid, err } } + // Finally, print the stats + // TODO; Does not print the stats, but seems to count correctly? + printStats(t) return 0, valid, invalid, nil } @@ -275,7 +278,6 @@ func checkOnce(t *T, prop func(*T)) (err *testError) { prop(t) t.failOnError() - return nil } @@ -409,6 +411,9 @@ type T struct { refDraws []value mu sync.RWMutex failed stopTest + allstats stats + numStats nStats + statMux sync.Mutex } func newT(tb tb, s bitStream, tbLog bool, rawLog *log.Logger, refDraws ...value) *T { diff --git a/event.go b/event.go new file mode 100644 index 0000000..d84bd34 --- /dev/null +++ b/event.go @@ -0,0 +1,294 @@ +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +package rapid + +import ( + "log" + "sort" +) + +// counter maps a (stringified) event to a frequency counter +type counter map[string]int +type typeEnum byte + +const ( + UNKNOWN typeEnum = iota + UINT + INT + FLOAT +) + +type numCounter struct { + ints []int64 + uints []uint64 + floats []float64 + t typeEnum +} + +// stats maps labels to counters +type stats map[string]counter +type nStats map[string]*numCounter +type counterPair struct { + frequency int + event string +} + +// Event records an event for test `t` and +// stores the event for calculating statistics. +// +// Recording events and printing a their statistic is a tool for +// analysing test data generation. It helps to understand if +// your (custom) generators produce values in the expected range. +// +// Each event has a label and an event value. To see the statistics, +// run the tests with `go test -v`. +// +func Event(t *T, label string, value string) { + if t.tb != nil { + t.tb.Helper() + } + t.statMux.Lock() + defer t.statMux.Unlock() + if t.allstats == nil { + t.allstats = make(stats) + } + c, found := t.allstats[label] + if !found { + c = make(counter) + t.allstats[label] = c + } + c[value]++ +} + +// NumericEvent records a numeric event for given label +// for statistic calculation of minimum, median, arithmetic mean +// and maximum values for that label. Allowed numeric values are +// all signed and unsigned integers and floats. Howver, for each +// label, all numeric values must be consistent, ie. have the the +// same type. Otherwise, a run-time panic will occur. +// +// Recording events and printing a their statistic is a tool for +// analysing test data generation. It helps to understand if +// your (custom) generators produce values in the expected range. +// +// Each event has a label and an event value. To see the statistics, +// run the tests with `go test -v`. +func NumericEvent(t *T, label string, numValue value) { + valType := UNKNOWN + var uintVal uint64 + var intVal int64 + var floatVal float64 + if t.tb != nil { + t.tb.Helper() + } + switch x := numValue.(type) { + case uint8: + valType = UINT + uintVal = uint64(x) + case uint16: + valType = UINT + uintVal = uint64(x) + case uint32: + valType = UINT + uintVal = uint64(x) + case uint64: + valType = UINT + uintVal = uint64(x) + case uint: + valType = UINT + uintVal = uint64(x) + case int8: + valType = INT + intVal = int64(x) + case int16: + valType = INT + intVal = int64(x) + case int32: + valType = INT + intVal = int64(x) + case int64: + valType = INT + intVal = int64(x) + case int: + valType = INT + intVal = int64(x) + case float32: + valType = FLOAT + floatVal = float64(x) + case float64: + valType = FLOAT + floatVal = float64(x) + default: + t.Fatalf("numeric event is not a numeric value") + return + } + t.statMux.Lock() + defer t.statMux.Unlock() + if t.numStats == nil { + t.numStats = make(nStats) + } + c, found := t.numStats[label] + if !found { + newCounter := &numCounter{ + ints: []int64{}, + uints: []uint64{}, + floats: []float64{}, + t: valType, + } + t.numStats[label] = newCounter + c = newCounter + } + if valType != c.t { + t.Fatalf("Type of numeric event does not match. Expected: %#v, was %#v", c.t, valType) + } + switch valType { + case UINT: + c.uints = append(c.uints, uintVal) + case INT: + c.ints = append(c.ints, intVal) + case FLOAT: + c.floats = append(c.floats, floatVal) + } +} + +func minMaxMeanInt(values []int64) (min, max, median int64, mean float64) { + min, max, mean = values[0], values[0], float64(values[0]) + sum := mean + for i := 1; i < len(values); i++ { + if values[i] < min { + min = values[i] + } + if values[i] > max { + max = values[i] + } + sum += float64(values[i]) + } + mean = sum / float64(len(values)) + sort.Sort(int64Slice(values)) + if len(values)%2 == 1 { + median = values[(len(values)-1)/2] + } else { + n := len(values) / 2 + median = (values[n] + values[n-1]) / 2 + } + return +} + +func minMaxMeanUint(values []uint64) (min, max, median uint64, mean float64) { + min, max, mean = values[0], values[0], float64(values[0]) + sum := mean + // log.Printf("rapid mean: values[%d] = %v", 0, values[0]) + for i := 1; i < len(values); i++ { + if values[i] < min { + min = values[i] + } + if values[i] > max { + max = values[i] + } + // log.Printf("rapid mean: values[%d] = %v", i, values[i]) + sum += float64(values[i]) + } + mean = sum / float64(len(values)) + sort.Sort(uint64Slice(values)) + if len(values)%2 == 1 { + median = values[(len(values)-1)/2] + } else { + n := len(values) / 2 + median = (values[n] + values[n-1]) / 2 + } + return +} + +func minMaxMeanFloat(values []float64) (min, max, median, mean float64) { + min, max, mean = values[0], values[0], values[0] + sum := mean + for i := 1; i < len(values); i++ { + if values[i] < min { + min = values[i] + } + if values[i] > max { + max = values[i] + } + sum += values[i] + } + mean = sum / float64(len(values)) + sort.Float64s(values) + if len(values)%2 == 1 { + median = values[(len(values)-1)/2] + } else { + n := len(values) / 2 + median = (values[n] + values[n-1]) / 2 + } + return +} + +// printStats logs a table of events and their relative frequency. +func printStats(t *T) { + t.statMux.Lock() + defer t.statMux.Unlock() + if t.tbLog && t.tb != nil { + t.tb.Helper() + } + if len(t.allstats) > 0 { + log.Printf("[rapid] Statistics for %s\n", t.Name()) + for label := range t.allstats { + log.Printf("[rapid] Events with label %s", label) + s := t.allstats[label] + events := make([]counterPair, 0) + sum := 0 + count := 0 + for ev := range s { + sum += s[ev] + count++ + events = append(events, counterPair{event: ev, frequency: s[ev]}) + } + log.Printf("[rapid] Total of %d different events\n", count) + // we sort twice to sort same frequency alphabetically + sort.Slice(events, func(i, j int) bool { return events[i].event < events[j].event }) + sort.SliceStable(events, func(i, j int) bool { return events[i].frequency > events[j].frequency }) + for _, ev := range events { + log.Printf("[rapid] %s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) + } + } + } + if len(t.numStats) > 0 { + log.Printf("[rapid] Numerical Statistics for %s\n", t.Name()) + for label := range t.numStats { + log.Printf("[rapid] Numerical events with label %s", label) + c := t.numStats[label] + switch c.t { + case UINT: + min, max, median, mean := minMaxMeanUint(c.uints) + log.Printf("[rapid] Min: %d\n", min) + log.Printf("[rapid] Median: %d\n", median) + log.Printf("[rapid] Mean: %f\n", mean) + log.Printf("[rapid] Max: %d\n", max) + case INT: + min, max, median, mean := minMaxMeanInt(c.ints) + log.Printf("[rapid] Min: %d\n", min) + log.Printf("[rapid] Median: %d\n", median) + log.Printf("[rapid] Mean: %f\n", mean) + log.Printf("[rapid] Max: %d\n", max) + case FLOAT: + min, max, median, mean := minMaxMeanFloat(c.floats) + log.Printf("[rapid] Min: %f\n", min) + log.Printf("[rapid] Median: %f\n", median) + log.Printf("[rapid] Mean: %f\n", mean) + log.Printf("[rapid] Max: %f\n", max) + } + } + } +} + +// implementing the sorting interfaces for int64 and uint64 slices +type uint64Slice []uint64 +type int64Slice []int64 + +func (x uint64Slice) Len() int { return len(x) } +func (x uint64Slice) Less(i, j int) bool { return x[i] < x[j] } +func (x uint64Slice) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x int64Slice) Len() int { return len(x) } +func (x int64Slice) Less(i, j int) bool { return x[i] < x[j] } +func (x int64Slice) Swap(i, j int) { x[i], x[j] = x[j], x[i] } diff --git a/event_test.go b/event_test.go new file mode 100644 index 0000000..b52d8cb --- /dev/null +++ b/event_test.go @@ -0,0 +1,70 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package rapid + +import ( + "fmt" + "regexp" + "testing" +) + +func TestEventEmitter(t *testing.T) { + t.Parallel() + Check(t, func(te *T) { + // te.rawLog.SetOutput(te.rawLog.Output()) + Event(te, "var", "x") + Event(te, "var", "y") + + // checkMatch(te, fmt.Sprintf("Statistics.*%s", te.Name()), te.output[0]) + // checkMatch(te, "of 2 ", te.output[1]) + // checkMatch(te, "x: 1 \\(50.0+ %", te.output[3]) + // checkMatch(te, "y: 1 \\(50.0+ %", te.output[4]) + + }) +} + +func checkMatch(t *T, pattern, str string) { + matched, err := regexp.MatchString(pattern, str) + if err != nil { + t.Fatalf("Regex compile failed") + } + if !matched { + t.Fatalf("Pattern <%s> does not match in <%s>", pattern, str) + } +} + +func TestTrivialPropertyWithEvents(t *testing.T) { + t.Parallel() + Check(t, func(te *T) { + x := Uint8().Draw(te, "x").(uint8) + Event(te, "x", fmt.Sprintf("%d", x)) + if x > 255 { + t.Fatalf("x should fit into a byte") + } + }) +} + +func TestTrivialPropertyWithNumEvents(t *testing.T) { + t.Parallel() + Check(t, func(te *T) { + x := Uint8().Draw(te, "x").(uint8) + Event(te, "x", fmt.Sprintf("%d", x)) + NumericEvent(te, "x", x) + if x > 255 { + t.Fatalf("x should fit into a byte") + } + }) +} + +func TestFilteredGenWithAutoEvents(t *testing.T) { + t.Parallel() + Check(t, func(te *T) { + x := Uint8().Filter(func(n uint8) bool { return n%2 == 0 }).Draw(te, "even x").(uint8) + Event(te, "x", fmt.Sprintf("%d", x)) + if x > 255 { + t.Fatalf("x should fit into a byte") + } + }) +} diff --git a/example_event_test.go b/example_event_test.go new file mode 100644 index 0000000..f4c0369 --- /dev/null +++ b/example_event_test.go @@ -0,0 +1,30 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package rapid_test + +import ( + "fmt" + "testing" + + "pgregory.net/rapid" +) + +func ExampleEvent(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + // For any integers x, y ... + x := rapid.Int().Draw(t, "x").(int) + y := rapid.Int().Draw(t, "y").(int) + // ... report them ... + rapid.Event(t, "x", fmt.Sprintf("%d", x)) + rapid.Event(t, "y", fmt.Sprintf("%d", y)) + + // ... the property holds + if x+y != y+x { + t.Fatalf("associativty of + does not hold") + } + // statistics are printed after the property (if called with go test -v) + }) + // Output: +}