From e7e4781dbaca302f2777b679650713110a630491 Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Tue, 9 Nov 2021 23:21:02 +0100 Subject: [PATCH 01/12] Event and PrintStats --- event.go | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++ event_test.go | 54 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 120 insertions(+) create mode 100644 event.go create mode 100644 event_test.go diff --git a/event.go b/event.go new file mode 100644 index 0000000..95a4c2c --- /dev/null +++ b/event.go @@ -0,0 +1,66 @@ +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. +package rapid + +import ( + "sort" +) + +type counter map[string]int +type stats map[string]counter + +type counterPair struct { + frequency int + event string +} + +// TEvent is a minimal interface for recording events and +// printing statistics. +type TEvent interface { + Helper() + Name() string + Logf(fmt string, data ...interface{}) +} + +// global variable for statistics +var all_stats stats = make(stats) + +// Event records an event for test `t` and +// stores the event for calculating statistics. +func Event(t TEvent, event string) { + t.Helper() + c, found := all_stats[t.Name()] + if !found { + c = make(counter) + all_stats[t.Name()] = c + } + c[event]++ +} + +// PrintStats logs a table of events and their relative frequency. +// To see these statistics, run the tests with `go test -v` +func PrintStats(t TEvent) { + t.Helper() + s, found := all_stats[t.Name()] + if !found { + t.Logf("No events stored for test %s", t) + return + } + events := make([]counterPair, 0) + sum := 0 + count := 0 + for ev := range s { + sum += s[ev] + count++ + events = append(events, counterPair{event: ev, frequency: s[ev]}) + } + t.Logf("Statistics for %s\n", t.Name()) + t.Logf("Total of %d different events\n", count) + t.Logf(" =====> Sorted by frequency:") + sort.Slice(events, func(i, j int) bool { return events[i].frequency > events[j].frequency }) + for _, ev := range events { + t.Logf("%s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) + } +} diff --git a/event_test.go b/event_test.go new file mode 100644 index 0000000..da9497b --- /dev/null +++ b/event_test.go @@ -0,0 +1,54 @@ +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package rapid_test + +import ( + "fmt" + "regexp" + "testing" + + . "pgregory.net/rapid" +) + +type tEvent struct { + t *testing.T + output []string +} + +func NewTEvent(t *testing.T) *tEvent { + return &tEvent{ + t: t, + output: make([]string, 0), + } +} +func (t *tEvent) Helper() { t.t.Helper() } +func (t *tEvent) Name() string { return t.t.Name() } +func (t *tEvent) Logf(format string, data ...interface{}) { + t.t.Logf(format, data...) + t.output = append(t.output, fmt.Sprintf(format, data...)) +} + +func TestEventEmitter(t *testing.T) { + te := NewTEvent(t) + Event(te, "x") + Event(te, "y") + + PrintStats(te) + checkMatch(t, fmt.Sprintf("Statistics.*%s", t.Name()), te.output[0]) + checkMatch(t, "of 2 ", te.output[1]) + checkMatch(t, "x: 1 \\(50.0+ %", te.output[3]) + checkMatch(t, "y: 1 \\(50.0+ %", te.output[4]) +} + +func checkMatch(t *testing.T, pattern, str string) { + matched, err := regexp.MatchString(pattern, str) + if err != nil { + t.Fatalf("Regex compile failed") + } + if !matched { + t.Fatalf("Pattern <%s> does not match in <%s>", pattern, str) + } +} From 7bb727e0a93bc97f949377aca3a24a1e2f756262 Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Wed, 10 Nov 2021 21:33:23 +0100 Subject: [PATCH 02/12] improved documentation --- event.go | 4 ++++ event_example_test.go | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100644 event_example_test.go diff --git a/event.go b/event.go index 95a4c2c..a3fad35 100644 --- a/event.go +++ b/event.go @@ -29,6 +29,10 @@ var all_stats stats = make(stats) // Event records an event for test `t` and // stores the event for calculating statistics. +// +// Recording events and printing a their statistic is a tool for +// analysing test data generations. It helps to understand if +// your customer generators produce value in the expected range. func Event(t TEvent, event string) { t.Helper() c, found := all_stats[t.Name()] diff --git a/event_example_test.go b/event_example_test.go new file mode 100644 index 0000000..9f0f1a5 --- /dev/null +++ b/event_example_test.go @@ -0,0 +1,30 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +package rapid_test + +import ( + "fmt" + "testing" + + "pgregory.net/rapid" +) + +func ExampleEvent(t *testing.T) { + rapid.Check(t, func(t *rapid.T) { + // For any integers x, y ... + x := rapid.Int().Draw(t, "x").(int) + y := rapid.Int().Draw(t, "y").(int) + // ... report them ... + rapid.Event(t, fmt.Sprintf("x = %d", x)) + rapid.Event(t, fmt.Sprintf("y = %d", y)) + + // ... the property holds + if x+y != y+x { + t.Fatalf("associativty of + does not hold") + } + }) + // print statistics after the property (if called with go test -v) + rapid.PrintStats(t) +} From 89a45f231f5dc7fd98a7d8232525a62c009fffcd Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Wed, 10 Nov 2021 21:57:59 +0100 Subject: [PATCH 03/12] using rapid.TB as interface for Event & PrintStats arguments --- event.go | 12 ++---------- event_test.go | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/event.go b/event.go index a3fad35..24c70c0 100644 --- a/event.go +++ b/event.go @@ -16,14 +16,6 @@ type counterPair struct { event string } -// TEvent is a minimal interface for recording events and -// printing statistics. -type TEvent interface { - Helper() - Name() string - Logf(fmt string, data ...interface{}) -} - // global variable for statistics var all_stats stats = make(stats) @@ -33,7 +25,7 @@ var all_stats stats = make(stats) // Recording events and printing a their statistic is a tool for // analysing test data generations. It helps to understand if // your customer generators produce value in the expected range. -func Event(t TEvent, event string) { +func Event(t TB, event string) { t.Helper() c, found := all_stats[t.Name()] if !found { @@ -45,7 +37,7 @@ func Event(t TEvent, event string) { // PrintStats logs a table of events and their relative frequency. // To see these statistics, run the tests with `go test -v` -func PrintStats(t TEvent) { +func PrintStats(t TB) { t.Helper() s, found := all_stats[t.Name()] if !found { diff --git a/event_test.go b/event_test.go index da9497b..e9f6b8b 100644 --- a/event_test.go +++ b/event_test.go @@ -1,4 +1,3 @@ -// // This Source Code Form is subject to the terms of the Mozilla Public // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. @@ -24,8 +23,20 @@ func NewTEvent(t *testing.T) *tEvent { output: make([]string, 0), } } -func (t *tEvent) Helper() { t.t.Helper() } -func (t *tEvent) Name() string { return t.t.Name() } + +// implement rapid.TB trivially except for Logf +func (t *tEvent) Helper() { t.t.Helper() } +func (t *tEvent) Name() string { return t.t.Name() } +func (t *tEvent) Error(args ...interface{}) { t.t.Error(args...) } +func (t *tEvent) Errorf(format string, args ...interface{}) { t.t.Errorf(format, args...) } +func (t *tEvent) Fail() { t.t.Fail() } +func (t *tEvent) FailNow() { t.t.FailNow() } +func (t *tEvent) Failed() bool { return t.t.Failed() } +func (t *tEvent) Fatal(args ...interface{}) { t.t.Fatal(args...) } +func (t *tEvent) Fatalf(format string, args ...interface{}) { t.t.Fatalf(format, args...) } +func (t *tEvent) SkipNow() { t.t.SkipNow() } +func (t *tEvent) Skipped() bool { return t.t.Skipped() } +func (t *tEvent) Log(args ...interface{}) { t.t.Log(args...) } func (t *tEvent) Logf(format string, data ...interface{}) { t.t.Logf(format, data...) t.output = append(t.output, fmt.Sprintf(format, data...)) From 4b7466dae1cafb3d3a484a4d3c835c4b5750b232 Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Wed, 17 Nov 2021 12:48:37 +0100 Subject: [PATCH 04/12] Implement events and printing with go-routines --- engine.go | 6 ++- event.go | 108 +++++++++++++++++++++++++++++++++++++++--------------- 2 files changed, 83 insertions(+), 31 deletions(-) diff --git a/engine.go b/engine.go index 62fb1b8..3b1cff9 100644 --- a/engine.go +++ b/engine.go @@ -263,6 +263,9 @@ func findBug(tb tb, checks int, seed uint64, prop func(*T)) (uint64, int, int, * return seed, valid, invalid, err } } + // Finally, print the stats + // TODO; Does not print the stats, but seems to count correctly? + printStats(t) return 0, valid, invalid, nil } @@ -275,7 +278,6 @@ func checkOnce(t *T, prop func(*T)) (err *testError) { prop(t) t.failOnError() - return nil } @@ -409,6 +411,8 @@ type T struct { refDraws []value mu sync.RWMutex failed stopTest + evChan chan event + evDone chan done } func newT(tb tb, s bitStream, tbLog bool, rawLog *log.Logger, refDraws ...value) *T { diff --git a/event.go b/event.go index 24c70c0..bda2640 100644 --- a/event.go +++ b/event.go @@ -5,58 +5,106 @@ package rapid import ( + "log" "sort" ) +// counter maps a (stringified) event to a frequency counter type counter map[string]int + +// stats maps labels to counters type stats map[string]counter +type event struct { + label string + value string +} +type done struct { + result chan stats +} + type counterPair struct { frequency int event string } -// global variable for statistics -var all_stats stats = make(stats) - // Event records an event for test `t` and // stores the event for calculating statistics. // // Recording events and printing a their statistic is a tool for // analysing test data generations. It helps to understand if // your customer generators produce value in the expected range. -func Event(t TB, event string) { +// +// Each event has a label and an event value. To see the statistics, +// run the tests with `go test -v`. +// +func Event(t *T, label string, value string) { t.Helper() - c, found := all_stats[t.Name()] - if !found { - c = make(counter) - all_stats[t.Name()] = c + if t.evChan == nil { + log.Printf("Creating the channels for test %s", t.Name()) + t.evChan = make(chan event) + t.evDone = make(chan done) + go eventRecorder(t.evChan, t.evDone) } - c[event]++ + ev := event{value: value, label: label} + // log.Printf("Send the event %+v", ev) + t.evChan <- ev } -// PrintStats logs a table of events and their relative frequency. -// To see these statistics, run the tests with `go test -v` -func PrintStats(t TB) { - t.Helper() - s, found := all_stats[t.Name()] - if !found { - t.Logf("No events stored for test %s", t) - return +// eventRecorder is a goroutine that stores event for a test execution. +func eventRecorder(incomingEvent <-chan event, done <-chan done) { + all_stats := make(stats) + for { + select { + case ev := <-incomingEvent: + c, found := all_stats[ev.label] + if !found { + c = make(counter) + all_stats[ev.label] = c + } + c[ev.value]++ + case d := <-done: + log.Printf("event Recorder: Done. Send the stats\n") + d.result <- all_stats + log.Printf("event Recorder: Done. Will return now\n") + return + } } - events := make([]counterPair, 0) - sum := 0 - count := 0 - for ev := range s { - sum += s[ev] - count++ - events = append(events, counterPair{event: ev, frequency: s[ev]}) + // log.Printf("event Recorder: This shall never happen\n") + +} + +// printStats logs a table of events and their relative frequency. +func printStats(t *T) { + // log.Printf("What about printing the stats for t = %+v", t) + if t.evChan == nil || t.evDone == nil { + return } - t.Logf("Statistics for %s\n", t.Name()) - t.Logf("Total of %d different events\n", count) - t.Logf(" =====> Sorted by frequency:") - sort.Slice(events, func(i, j int) bool { return events[i].frequency > events[j].frequency }) - for _, ev := range events { - t.Logf("%s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) + log.Printf("Now we can print the stats") + d := done{result: make(chan stats)} + t.evDone <- d + stats := <-d.result + log.Printf("stats received") + log.Printf("Statistics for %s\n", t.Name()) + for label := range stats { + log.Printf("Events with label %s", label) + s := stats[label] + events := make([]counterPair, 0) + sum := 0 + count := 0 + for ev := range s { + sum += s[ev] + count++ + events = append(events, counterPair{event: ev, frequency: s[ev]}) + } + log.Printf("Total of %d different events\n", count) + // we sort twice to sort same frequency alphabetically + sort.Slice(events, func(i, j int) bool { return events[i].event < events[j].event }) + sort.SliceStable(events, func(i, j int) bool { return events[i].frequency > events[j].frequency }) + for _, ev := range events { + log.Printf("%s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) + } } + close(t.evChan) + close(t.evDone) } From 8c95998fc4d653f4f6d791783902652361665978 Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Wed, 17 Nov 2021 12:51:56 +0100 Subject: [PATCH 05/12] File rename and adoption to new API --- event_example_test.go => example_event_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) rename event_example_test.go => example_event_test.go (76%) diff --git a/event_example_test.go b/example_event_test.go similarity index 76% rename from event_example_test.go rename to example_event_test.go index 9f0f1a5..f4c0369 100644 --- a/event_example_test.go +++ b/example_event_test.go @@ -17,14 +17,14 @@ func ExampleEvent(t *testing.T) { x := rapid.Int().Draw(t, "x").(int) y := rapid.Int().Draw(t, "y").(int) // ... report them ... - rapid.Event(t, fmt.Sprintf("x = %d", x)) - rapid.Event(t, fmt.Sprintf("y = %d", y)) + rapid.Event(t, "x", fmt.Sprintf("%d", x)) + rapid.Event(t, "y", fmt.Sprintf("%d", y)) // ... the property holds if x+y != y+x { t.Fatalf("associativty of + does not hold") } + // statistics are printed after the property (if called with go test -v) }) - // print statistics after the property (if called with go test -v) - rapid.PrintStats(t) + // Output: } From 671efaec423a536063a53f60029e3526d94e5747 Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Wed, 17 Nov 2021 12:52:26 +0100 Subject: [PATCH 06/12] more tests but only for visual inspection --- event_test.go | 68 +++++++++++++++++++-------------------------------- 1 file changed, 25 insertions(+), 43 deletions(-) diff --git a/event_test.go b/event_test.go index e9f6b8b..61f3272 100644 --- a/event_test.go +++ b/event_test.go @@ -2,59 +2,30 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -package rapid_test +package rapid import ( "fmt" "regexp" "testing" - - . "pgregory.net/rapid" ) -type tEvent struct { - t *testing.T - output []string -} - -func NewTEvent(t *testing.T) *tEvent { - return &tEvent{ - t: t, - output: make([]string, 0), - } -} - -// implement rapid.TB trivially except for Logf -func (t *tEvent) Helper() { t.t.Helper() } -func (t *tEvent) Name() string { return t.t.Name() } -func (t *tEvent) Error(args ...interface{}) { t.t.Error(args...) } -func (t *tEvent) Errorf(format string, args ...interface{}) { t.t.Errorf(format, args...) } -func (t *tEvent) Fail() { t.t.Fail() } -func (t *tEvent) FailNow() { t.t.FailNow() } -func (t *tEvent) Failed() bool { return t.t.Failed() } -func (t *tEvent) Fatal(args ...interface{}) { t.t.Fatal(args...) } -func (t *tEvent) Fatalf(format string, args ...interface{}) { t.t.Fatalf(format, args...) } -func (t *tEvent) SkipNow() { t.t.SkipNow() } -func (t *tEvent) Skipped() bool { return t.t.Skipped() } -func (t *tEvent) Log(args ...interface{}) { t.t.Log(args...) } -func (t *tEvent) Logf(format string, data ...interface{}) { - t.t.Logf(format, data...) - t.output = append(t.output, fmt.Sprintf(format, data...)) -} - func TestEventEmitter(t *testing.T) { - te := NewTEvent(t) - Event(te, "x") - Event(te, "y") - - PrintStats(te) - checkMatch(t, fmt.Sprintf("Statistics.*%s", t.Name()), te.output[0]) - checkMatch(t, "of 2 ", te.output[1]) - checkMatch(t, "x: 1 \\(50.0+ %", te.output[3]) - checkMatch(t, "y: 1 \\(50.0+ %", te.output[4]) + t.Parallel() + Check(t, func(te *T) { + // te.rawLog.SetOutput(te.rawLog.Output()) + Event(te, "var", "x") + Event(te, "var", "y") + + // checkMatch(te, fmt.Sprintf("Statistics.*%s", te.Name()), te.output[0]) + // checkMatch(te, "of 2 ", te.output[1]) + // checkMatch(te, "x: 1 \\(50.0+ %", te.output[3]) + // checkMatch(te, "y: 1 \\(50.0+ %", te.output[4]) + + }) } -func checkMatch(t *testing.T, pattern, str string) { +func checkMatch(t *T, pattern, str string) { matched, err := regexp.MatchString(pattern, str) if err != nil { t.Fatalf("Regex compile failed") @@ -63,3 +34,14 @@ func checkMatch(t *testing.T, pattern, str string) { t.Fatalf("Pattern <%s> does not match in <%s>", pattern, str) } } + +func TestTrivialPropertyWithEvents(t *testing.T) { + t.Parallel() + Check(t, func(te *T) { + x := Uint8().Draw(te, "x").(uint8) + Event(te, "x", fmt.Sprintf("%d", x)) + if x > 255 { + t.Fatalf("x should fit into a byte") + } + }) +} From b40be41977584972c62000a6c6c24680181dc773 Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Wed, 17 Nov 2021 22:26:32 +0100 Subject: [PATCH 07/12] Simplified, shortened, and (hopyfully) thread-safe It uses now a mutex to protect both, writing to the stats and printing the stats. The channels are gone. Since printing is done after finishing the property, there should be most likely no still running go-routines exposing event - but if so, then we will miss them. This is ok. --- engine.go | 4 ++-- event.go | 63 ++++++++++++------------------------------------------- 2 files changed, 15 insertions(+), 52 deletions(-) diff --git a/engine.go b/engine.go index 3b1cff9..93158fe 100644 --- a/engine.go +++ b/engine.go @@ -411,8 +411,8 @@ type T struct { refDraws []value mu sync.RWMutex failed stopTest - evChan chan event - evDone chan done + allstats stats + statMux sync.Mutex } func newT(tb tb, s bitStream, tbLog bool, rawLog *log.Logger, refDraws ...value) *T { diff --git a/event.go b/event.go index bda2640..9b6e0f7 100644 --- a/event.go +++ b/event.go @@ -14,15 +14,6 @@ type counter map[string]int // stats maps labels to counters type stats map[string]counter - -type event struct { - label string - value string -} -type done struct { - result chan stats -} - type counterPair struct { frequency int event string @@ -40,55 +31,29 @@ type counterPair struct { // func Event(t *T, label string, value string) { t.Helper() - if t.evChan == nil { - log.Printf("Creating the channels for test %s", t.Name()) - t.evChan = make(chan event) - t.evDone = make(chan done) - go eventRecorder(t.evChan, t.evDone) + t.statMux.Lock() + defer t.statMux.Unlock() + if t.allstats == nil { + t.allstats = make(stats) } - ev := event{value: value, label: label} - // log.Printf("Send the event %+v", ev) - t.evChan <- ev -} - -// eventRecorder is a goroutine that stores event for a test execution. -func eventRecorder(incomingEvent <-chan event, done <-chan done) { - all_stats := make(stats) - for { - select { - case ev := <-incomingEvent: - c, found := all_stats[ev.label] - if !found { - c = make(counter) - all_stats[ev.label] = c - } - c[ev.value]++ - case d := <-done: - log.Printf("event Recorder: Done. Send the stats\n") - d.result <- all_stats - log.Printf("event Recorder: Done. Will return now\n") - return - } + c, found := t.allstats[label] + if !found { + c = make(counter) + t.allstats[label] = c } - // log.Printf("event Recorder: This shall never happen\n") - + c[value]++ } // printStats logs a table of events and their relative frequency. func printStats(t *T) { - // log.Printf("What about printing the stats for t = %+v", t) - if t.evChan == nil || t.evDone == nil { - return - } + t.statMux.Lock() + defer t.statMux.Unlock() log.Printf("Now we can print the stats") - d := done{result: make(chan stats)} - t.evDone <- d - stats := <-d.result log.Printf("stats received") log.Printf("Statistics for %s\n", t.Name()) - for label := range stats { + for label := range t.allstats { log.Printf("Events with label %s", label) - s := stats[label] + s := t.allstats[label] events := make([]counterPair, 0) sum := 0 count := 0 @@ -105,6 +70,4 @@ func printStats(t *T) { log.Printf("%s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) } } - close(t.evChan) - close(t.evDone) } From f4685c61619405b4267356811c0f1764d3e062bc Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Mon, 3 Jan 2022 14:25:44 +0100 Subject: [PATCH 08/12] logging of statistics is done only if there are any events recorded. --- event.go | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/event.go b/event.go index 9b6e0f7..0e629cb 100644 --- a/event.go +++ b/event.go @@ -48,26 +48,29 @@ func Event(t *T, label string, value string) { func printStats(t *T) { t.statMux.Lock() defer t.statMux.Unlock() - log.Printf("Now we can print the stats") - log.Printf("stats received") - log.Printf("Statistics for %s\n", t.Name()) - for label := range t.allstats { - log.Printf("Events with label %s", label) - s := t.allstats[label] - events := make([]counterPair, 0) - sum := 0 - count := 0 - for ev := range s { - sum += s[ev] - count++ - events = append(events, counterPair{event: ev, frequency: s[ev]}) - } - log.Printf("Total of %d different events\n", count) - // we sort twice to sort same frequency alphabetically - sort.Slice(events, func(i, j int) bool { return events[i].event < events[j].event }) - sort.SliceStable(events, func(i, j int) bool { return events[i].frequency > events[j].frequency }) - for _, ev := range events { - log.Printf("%s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) + if t.tbLog && t.tb != nil { + t.tb.Helper() + } + if len(t.allstats) > 0 { + log.Printf("Statistics for %s\n", t.Name()) + for label := range t.allstats { + log.Printf("Events with label %s", label) + s := t.allstats[label] + events := make([]counterPair, 0) + sum := 0 + count := 0 + for ev := range s { + sum += s[ev] + count++ + events = append(events, counterPair{event: ev, frequency: s[ev]}) + } + log.Printf("Total of %d different events\n", count) + // we sort twice to sort same frequency alphabetically + sort.Slice(events, func(i, j int) bool { return events[i].event < events[j].event }) + sort.SliceStable(events, func(i, j int) bool { return events[i].frequency > events[j].frequency }) + for _, ev := range events { + log.Printf("%s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) + } } } } From ae7fb6268b683f68213f43846b9aee785aaaf408 Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Mon, 3 Jan 2022 16:37:11 +0100 Subject: [PATCH 09/12] indented reports and more robust Event function --- event.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/event.go b/event.go index 0e629cb..3927063 100644 --- a/event.go +++ b/event.go @@ -30,7 +30,9 @@ type counterPair struct { // run the tests with `go test -v`. // func Event(t *T, label string, value string) { - t.Helper() + if t.tb != nil { + t.Helper() + } t.statMux.Lock() defer t.statMux.Unlock() if t.allstats == nil { @@ -69,7 +71,7 @@ func printStats(t *T) { sort.Slice(events, func(i, j int) bool { return events[i].event < events[j].event }) sort.SliceStable(events, func(i, j int) bool { return events[i].frequency > events[j].frequency }) for _, ev := range events { - log.Printf("%s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) + log.Printf(" %s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) } } } From 3a507a81c8bbcbd7951499564bc0e4c302224e92 Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Mon, 3 Jan 2022 16:37:36 +0100 Subject: [PATCH 10/12] combinator "filter" creates autoamtically events --- combinators.go | 2 ++ event_test.go | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/combinators.go b/combinators.go index 991063d..cacaecd 100644 --- a/combinators.go +++ b/combinators.go @@ -98,8 +98,10 @@ func (g *filteredGen) value(t *T) value { func (g *filteredGen) maybeValue(t *T) value { v := g.g.value(t) if g.fn(v) { + Event(t, g.String(), "satisfied") return v } else { + Event(t, g.String(), "failed") return nil } } diff --git a/event_test.go b/event_test.go index 61f3272..fb2de9f 100644 --- a/event_test.go +++ b/event_test.go @@ -45,3 +45,14 @@ func TestTrivialPropertyWithEvents(t *testing.T) { } }) } + +func TestFilteredGenWithAutoEvents(t *testing.T) { + t.Parallel() + Check(t, func(te *T) { + x := Uint8().Filter(func(n uint8) bool { return n%2 == 0 }).Draw(te, "even x").(uint8) + Event(te, "x", fmt.Sprintf("%d", x)) + if x > 255 { + t.Fatalf("x should fit into a byte") + } + }) +} From f0e64f3715bcbcc8834737520d110a9cac103829 Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Tue, 4 Jan 2022 12:59:44 +0100 Subject: [PATCH 11/12] call Helper() on tb, not on t --- event.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/event.go b/event.go index 3927063..9db8055 100644 --- a/event.go +++ b/event.go @@ -31,7 +31,7 @@ type counterPair struct { // func Event(t *T, label string, value string) { if t.tb != nil { - t.Helper() + t.tb.Helper() } t.statMux.Lock() defer t.statMux.Unlock() From fc8b2e0b81ba65e94a6795a76b31e44362c7ecba Mon Sep 17 00:00:00 2001 From: Klaus Alfert Date: Tue, 4 Jan 2022 22:02:01 +0100 Subject: [PATCH 12/12] Numerical events produce numeric statistics --- engine.go | 1 + event.go | 228 ++++++++++++++++++++++++++++++++++++++++++++++++-- event_test.go | 12 +++ 3 files changed, 235 insertions(+), 6 deletions(-) diff --git a/engine.go b/engine.go index 93158fe..9bdb779 100644 --- a/engine.go +++ b/engine.go @@ -412,6 +412,7 @@ type T struct { mu sync.RWMutex failed stopTest allstats stats + numStats nStats statMux sync.Mutex } diff --git a/event.go b/event.go index 9db8055..d84bd34 100644 --- a/event.go +++ b/event.go @@ -11,9 +11,25 @@ import ( // counter maps a (stringified) event to a frequency counter type counter map[string]int +type typeEnum byte + +const ( + UNKNOWN typeEnum = iota + UINT + INT + FLOAT +) + +type numCounter struct { + ints []int64 + uints []uint64 + floats []float64 + t typeEnum +} // stats maps labels to counters type stats map[string]counter +type nStats map[string]*numCounter type counterPair struct { frequency int event string @@ -23,8 +39,8 @@ type counterPair struct { // stores the event for calculating statistics. // // Recording events and printing a their statistic is a tool for -// analysing test data generations. It helps to understand if -// your customer generators produce value in the expected range. +// analysing test data generation. It helps to understand if +// your (custom) generators produce values in the expected range. // // Each event has a label and an event value. To see the statistics, // run the tests with `go test -v`. @@ -46,6 +62,168 @@ func Event(t *T, label string, value string) { c[value]++ } +// NumericEvent records a numeric event for given label +// for statistic calculation of minimum, median, arithmetic mean +// and maximum values for that label. Allowed numeric values are +// all signed and unsigned integers and floats. Howver, for each +// label, all numeric values must be consistent, ie. have the the +// same type. Otherwise, a run-time panic will occur. +// +// Recording events and printing a their statistic is a tool for +// analysing test data generation. It helps to understand if +// your (custom) generators produce values in the expected range. +// +// Each event has a label and an event value. To see the statistics, +// run the tests with `go test -v`. +func NumericEvent(t *T, label string, numValue value) { + valType := UNKNOWN + var uintVal uint64 + var intVal int64 + var floatVal float64 + if t.tb != nil { + t.tb.Helper() + } + switch x := numValue.(type) { + case uint8: + valType = UINT + uintVal = uint64(x) + case uint16: + valType = UINT + uintVal = uint64(x) + case uint32: + valType = UINT + uintVal = uint64(x) + case uint64: + valType = UINT + uintVal = uint64(x) + case uint: + valType = UINT + uintVal = uint64(x) + case int8: + valType = INT + intVal = int64(x) + case int16: + valType = INT + intVal = int64(x) + case int32: + valType = INT + intVal = int64(x) + case int64: + valType = INT + intVal = int64(x) + case int: + valType = INT + intVal = int64(x) + case float32: + valType = FLOAT + floatVal = float64(x) + case float64: + valType = FLOAT + floatVal = float64(x) + default: + t.Fatalf("numeric event is not a numeric value") + return + } + t.statMux.Lock() + defer t.statMux.Unlock() + if t.numStats == nil { + t.numStats = make(nStats) + } + c, found := t.numStats[label] + if !found { + newCounter := &numCounter{ + ints: []int64{}, + uints: []uint64{}, + floats: []float64{}, + t: valType, + } + t.numStats[label] = newCounter + c = newCounter + } + if valType != c.t { + t.Fatalf("Type of numeric event does not match. Expected: %#v, was %#v", c.t, valType) + } + switch valType { + case UINT: + c.uints = append(c.uints, uintVal) + case INT: + c.ints = append(c.ints, intVal) + case FLOAT: + c.floats = append(c.floats, floatVal) + } +} + +func minMaxMeanInt(values []int64) (min, max, median int64, mean float64) { + min, max, mean = values[0], values[0], float64(values[0]) + sum := mean + for i := 1; i < len(values); i++ { + if values[i] < min { + min = values[i] + } + if values[i] > max { + max = values[i] + } + sum += float64(values[i]) + } + mean = sum / float64(len(values)) + sort.Sort(int64Slice(values)) + if len(values)%2 == 1 { + median = values[(len(values)-1)/2] + } else { + n := len(values) / 2 + median = (values[n] + values[n-1]) / 2 + } + return +} + +func minMaxMeanUint(values []uint64) (min, max, median uint64, mean float64) { + min, max, mean = values[0], values[0], float64(values[0]) + sum := mean + // log.Printf("rapid mean: values[%d] = %v", 0, values[0]) + for i := 1; i < len(values); i++ { + if values[i] < min { + min = values[i] + } + if values[i] > max { + max = values[i] + } + // log.Printf("rapid mean: values[%d] = %v", i, values[i]) + sum += float64(values[i]) + } + mean = sum / float64(len(values)) + sort.Sort(uint64Slice(values)) + if len(values)%2 == 1 { + median = values[(len(values)-1)/2] + } else { + n := len(values) / 2 + median = (values[n] + values[n-1]) / 2 + } + return +} + +func minMaxMeanFloat(values []float64) (min, max, median, mean float64) { + min, max, mean = values[0], values[0], values[0] + sum := mean + for i := 1; i < len(values); i++ { + if values[i] < min { + min = values[i] + } + if values[i] > max { + max = values[i] + } + sum += values[i] + } + mean = sum / float64(len(values)) + sort.Float64s(values) + if len(values)%2 == 1 { + median = values[(len(values)-1)/2] + } else { + n := len(values) / 2 + median = (values[n] + values[n-1]) / 2 + } + return +} + // printStats logs a table of events and their relative frequency. func printStats(t *T) { t.statMux.Lock() @@ -54,9 +232,9 @@ func printStats(t *T) { t.tb.Helper() } if len(t.allstats) > 0 { - log.Printf("Statistics for %s\n", t.Name()) + log.Printf("[rapid] Statistics for %s\n", t.Name()) for label := range t.allstats { - log.Printf("Events with label %s", label) + log.Printf("[rapid] Events with label %s", label) s := t.allstats[label] events := make([]counterPair, 0) sum := 0 @@ -66,13 +244,51 @@ func printStats(t *T) { count++ events = append(events, counterPair{event: ev, frequency: s[ev]}) } - log.Printf("Total of %d different events\n", count) + log.Printf("[rapid] Total of %d different events\n", count) // we sort twice to sort same frequency alphabetically sort.Slice(events, func(i, j int) bool { return events[i].event < events[j].event }) sort.SliceStable(events, func(i, j int) bool { return events[i].frequency > events[j].frequency }) for _, ev := range events { - log.Printf(" %s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) + log.Printf("[rapid] %s: %d (%f %%)\n", ev.event, ev.frequency, float32(ev.frequency)/float32(sum)*100.0) + } + } + } + if len(t.numStats) > 0 { + log.Printf("[rapid] Numerical Statistics for %s\n", t.Name()) + for label := range t.numStats { + log.Printf("[rapid] Numerical events with label %s", label) + c := t.numStats[label] + switch c.t { + case UINT: + min, max, median, mean := minMaxMeanUint(c.uints) + log.Printf("[rapid] Min: %d\n", min) + log.Printf("[rapid] Median: %d\n", median) + log.Printf("[rapid] Mean: %f\n", mean) + log.Printf("[rapid] Max: %d\n", max) + case INT: + min, max, median, mean := minMaxMeanInt(c.ints) + log.Printf("[rapid] Min: %d\n", min) + log.Printf("[rapid] Median: %d\n", median) + log.Printf("[rapid] Mean: %f\n", mean) + log.Printf("[rapid] Max: %d\n", max) + case FLOAT: + min, max, median, mean := minMaxMeanFloat(c.floats) + log.Printf("[rapid] Min: %f\n", min) + log.Printf("[rapid] Median: %f\n", median) + log.Printf("[rapid] Mean: %f\n", mean) + log.Printf("[rapid] Max: %f\n", max) } } } } + +// implementing the sorting interfaces for int64 and uint64 slices +type uint64Slice []uint64 +type int64Slice []int64 + +func (x uint64Slice) Len() int { return len(x) } +func (x uint64Slice) Less(i, j int) bool { return x[i] < x[j] } +func (x uint64Slice) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x int64Slice) Len() int { return len(x) } +func (x int64Slice) Less(i, j int) bool { return x[i] < x[j] } +func (x int64Slice) Swap(i, j int) { x[i], x[j] = x[j], x[i] } diff --git a/event_test.go b/event_test.go index fb2de9f..b52d8cb 100644 --- a/event_test.go +++ b/event_test.go @@ -46,6 +46,18 @@ func TestTrivialPropertyWithEvents(t *testing.T) { }) } +func TestTrivialPropertyWithNumEvents(t *testing.T) { + t.Parallel() + Check(t, func(te *T) { + x := Uint8().Draw(te, "x").(uint8) + Event(te, "x", fmt.Sprintf("%d", x)) + NumericEvent(te, "x", x) + if x > 255 { + t.Fatalf("x should fit into a byte") + } + }) +} + func TestFilteredGenWithAutoEvents(t *testing.T) { t.Parallel() Check(t, func(te *T) {