Documented Statistics.collect

TODO.md

@@ -63,8 +63,6 @@
 - Shrinking
   - Time limit (100ms default?) for shrinking
     - Make it configurable
-  - Statistics:
-    - collected statistics (similar to ScalaCheck's collect-feature)
   - Make it configurable (default on/off)
 
 - Evaluate properties in parallel (max tries worker thread per property)

build.gradle

@@ -21,11 +21,11 @@ apply plugin: 'org.junit.platform.gradle.plugin'
 apply plugin: 'maven'
 
 group = 'net.jqwik'
-version = '0.7.2'
+version = '0.7.3'
 
 jar {
 	baseName = 'jqwik'
-	version = '0.7.2'
+	version = '0.7.3'
 	manifest {
 		attributes('Automatic-Module-Name': moduleName)
 	}

docs/user-guide.md

@@ -33,6 +33,7 @@ Volunteers for polishing and extending it are more than welcome._
 - [Result Shrinking](#result-shrinking)
   - [Integrated Shrinking](#integrated-shrinking)
   - [Switch Shrinking Off](#switch-shrinking-off)
+- [Collecting and Reporting Statistics](#collecting-and-reporting-statistics)
 - [Running and Configuration](#running-and-configuration)
   - [jqwik Configuration](#jqwik-configuration)
 - [Program your own Generators and Arbitraries](#program-your-own-generators-and-arbitraries)
@@ -77,7 +78,7 @@ repositories {
 
 ext.junitPlatformVersion = '1.0.2'
 ext.junitJupiterVersion = '5.0.2'
-ext.jqwikVersion = '0.7.2'
+ext.jqwikVersion = '0.7.3'
 
 junitPlatform {
 	filters {
@@ -129,7 +130,7 @@ Add the following repository and dependency to your `pom.xml` file:
     <dependency>
         <groupId>com.github.jlink</groupId>
         <artifactId>jqwik</artifactId>
-        <version>0.7.2</version>
+        <version>0.7.3</version>
         <scope>test</scope>
     </dependency>
 </dependencies>
@@ -145,7 +146,7 @@ for details on how to configure the surefire plugin and other dependencies.
 I never tried it but using jqwik without gradle or some other tool to manage dependencies should also work.
 You will have to add _at least_ the following jars:
 
-- `jqwik-0.7.2.jar`
+- `jqwik-0.7.3.jar`
 - `junit-platform-engine-1.0.2.jar`
 - `junit-platform-commons-1.0.2.jar`
 - `opentest4j-1.0.0.jar`
@@ -920,6 +921,99 @@ void aPropertyWithLongShrinkingTimes(
 ) {	... }
 ```
 
+## Collecting and Reporting Statistics
+
+In many situations you'd like to know if _jqwik_ will really generate
+the kind of values you expect and if the frequency and distribution of
+certain value classes meets your testing needs. 
+`Statistics.collect()` is made for this exact purpose.
+
+In the most simple case you'd like to know how often a certain value
+is being generated:
+
+```java
+@Property
+void simpleStats(@ForAll RoundingMode mode) {
+    Statistics.collect(mode);
+}
+```
+
+will create an output similar to that:
+
+```
+collected statistics = 
+     UNNECESSARY : 15 %
+     DOWN        : 14 %
+     FLOOR       : 13 %
+     UP          : 13 %
+     HALF_DOWN   : 13 %
+     HALF_EVEN   : 12 %
+     CEILING     : 11 %
+     HALF_UP     : 11 %
+```
+
+More typical is the case in which you'll classify generated values
+into two or more groups:
+
+```java
+@Property
+void integerStats(@ForAll int anInt) {
+    Statistics.collect(anInt > 0 ? "positive" : "negative");
+}
+```
+
+```
+collected statistics = 
+     negative : 52 %
+     positive : 48 %
+```
+
+You can also collect the distribution in more than one category
+and combine those categories:
+
+```java
+@Property
+void combinedIntegerStats(@ForAll int anInt) {
+    String posOrNeg = anInt > 0 ? "positive" : "negative";
+    String evenOrOdd = anInt % 2 == 0 ? "even" : "odd";
+    String bigOrSmall = Math.abs(anInt) > 50 ? "big" : "small";
+    Statistics.collect(posOrNeg, evenOrOdd, bigOrSmall);
+}
+```
+
+```
+collected statistics = 
+     positive odd big    : 23 %
+     negative even big   : 22 %
+     positive even big   : 22 %
+     negative odd big    : 21 %
+     positive odd small  : 4 %
+     negative odd small  : 3 %
+     negative even small : 3 %
+     positive even small : 2 %
+```
+
+And, of course, you can combine different generated parameters into
+one statistical group:
+
+```java
+@Property
+void twoParameterStats(
+    @ForAll @Size(min = 1, max = 10) List<Integer> aList, //
+    @ForAll @IntRange(min = 0, max = 10) int index //
+) {
+    Statistics.collect(aList.size() > index ? "index within size" : null);
+}
+```
+
+```
+collected statistics = 
+     index within size : 48 %
+```
+
+As you can see, collected `null` values are not being reported.
+
+
 ## Running and Configuration
 
 When running _jqwik_ tests (through your IDE or your build tool) you might notice 

src/main/java/net/jqwik/properties/StatisticsCollector.java

@@ -1,14 +1,14 @@
 package net.jqwik.properties;
 
+import org.junit.platform.engine.reporting.*;
+
 import java.util.*;
 import java.util.function.*;
 import java.util.stream.*;
 
-import org.junit.platform.engine.reporting.*;
-
 public class StatisticsCollector {
 
-	public static final String KEY_STATISTICS = "statistics";
+	public static final String KEY_STATISTICS = "collected statistics";
 	private static ThreadLocal<StatisticsCollector> collector = ThreadLocal.withInitial(StatisticsCollector::new);
 
 	public static void clearAll() {
@@ -44,6 +44,7 @@ public ReportEntry createReportEntry() {
 				.sorted((e1, e2) -> e2.getValue().compareTo(e1.getValue())) //
 				.forEach(entry -> {
 					double percentage = entry.getValue() * 100.0 / sum;
+					if (entry.getKey().equals(Collections.emptyList())) return;
 					statistics.append(String.format("%n     %1$-" + maxKeyLength + "s : %2$s %%", //
 							displayKey(entry.getKey()), //
 							displayPercentage(percentage)));
@@ -62,7 +63,13 @@ private String displayKey(List<Object> key) {
 	}
 
 	public void collect(Object... values) {
-		List<Object> key = Arrays.asList(values);
+		List<Object> key = Collections.emptyList();
+		if (values != null) {
+			key = Arrays.stream(values) //
+				.filter(Objects::nonNull) //
+				.collect(Collectors.toList());
+		}
+
 		int count = counts.computeIfAbsent(key, any -> 0);
 		counts.put(key, ++count);
 	}

src/test/java/examples/docs/StatisticsExamples.java

@@ -0,0 +1,36 @@
+package examples.docs;
+
+import net.jqwik.api.*;
+import net.jqwik.api.constraints.*;
+
+import java.math.*;
+import java.util.*;
+
+class StatisticsExamples {
+
+	@Property
+	void simpleStats(@ForAll RoundingMode mode) {
+		Statistics.collect(mode);
+	}
+
+	@Property
+	void integerStats(@ForAll int anInt) {
+		Statistics.collect(anInt > 0 ? "positive" : "negative");
+	}
+
+	@Property
+	void combinedIntegerStats(@ForAll int anInt) {
+		String posOrNeg = anInt > 0 ? "positive" : "negative";
+		String evenOrOdd = anInt % 2 == 0 ? "even" : "odd";
+		String bigOrSmall = Math.abs(anInt) > 50 ? "big" : "small";
+		Statistics.collect(posOrNeg, evenOrOdd, bigOrSmall);
+	}
+
+	@Property
+	void twoParameterStats(
+		@ForAll @Size(min = 1, max = 10) List<Integer> aList, //
+		@ForAll @IntRange(min = 0, max = 10) int index //
+	) {
+		Statistics.collect(aList.size() > index ? "index within size" : null);
+	}
+}

src/test/java/net/jqwik/properties/GenericPropertyTests.java

@@ -1,19 +1,18 @@
 package net.jqwik.properties;
 
-import static org.assertj.core.api.Assertions.*;
-import static org.mockito.Mockito.*;
+import net.jqwik.api.*;
+import net.jqwik.descriptor.*;
+import org.assertj.core.api.*;
+import org.junit.platform.engine.reporting.*;
+import org.mockito.*;
 
 import java.util.*;
 import java.util.concurrent.atomic.*;
 import java.util.function.*;
 import java.util.stream.*;
 
-import org.assertj.core.api.*;
-import org.junit.platform.engine.reporting.*;
-import org.mockito.*;
-
-import net.jqwik.api.*;
-import net.jqwik.descriptor.*;
+import static org.assertj.core.api.Assertions.*;
+import static org.mockito.Mockito.*;
 
 @Group
 class GenericPropertyTests {
@@ -41,10 +40,10 @@ void collectStatistics() {
 		verify(mockPublisher, atLeast(2)).accept(reportEntryCaptor.capture());
 
 		Set<String> keys = reportEntryCaptor.getAllValues().stream() //
-				.flatMap(entry -> entry.getKeyValuePairs().keySet().stream()) //
-				.collect(Collectors.toSet());
+			.flatMap(entry -> entry.getKeyValuePairs().keySet().stream()) //
+			.collect(Collectors.toSet());
 
-		Assertions.assertThat(keys).contains("statistics");
+		Assertions.assertThat(keys).contains("collected statistics");
 
 		// Remove statistics from this test from ThreadLocal<Collector>:
 		StatisticsCollector.clearAll();
@@ -214,8 +213,7 @@ void exhaustedWithMaxDiscardRatioExceeded() {
 			List<Arbitrary> arbitraries = arbitraries(arbitrary);
 
 			GenericProperty property = new GenericProperty("exhausted property", arbitraries, forAllFunction);
-			PropertyConfiguration configuration = new PropertyConfiguration(42L, 20, maxDiscardRatio, ShrinkingMode.ON,
-					ReportingMode.MINIMAL);
+			PropertyConfiguration configuration = new PropertyConfiguration(42L, 20, maxDiscardRatio, ShrinkingMode.ON, ReportingMode.MINIMAL);
 			PropertyCheckResult result = property.check(configuration, NULL_PUBLISHER);
 
 			assertThat(result.status()).isEqualTo(PropertyCheckResult.Status.EXHAUSTED);
@@ -229,8 +227,7 @@ void exceptionInForAllFunctionMakesPropertyErroneous() {
 			RuntimeException thrownException = new RuntimeException("thrown in test");
 
 			ForAllSpy forAllFunction = new ForAllSpy(aTry -> {
-				if (aTry == erroneousTry)
-					throw thrownException;
+				if (aTry == erroneousTry) throw thrownException;
 				return true;
 			}, exactlyOneInteger);
 

src/test/java/net/jqwik/properties/StatisticsCollectionTests.java

@@ -1,14 +1,13 @@
 package net.jqwik.properties;
 
-import java.util.*;
-import java.util.stream.*;
-
+import net.jqwik.api.*;
 import org.assertj.core.api.*;
 import org.junit.platform.engine.reporting.*;
 
-import net.jqwik.api.*;
+import java.util.*;
+import java.util.stream.*;
 
-import static java.util.Arrays.asList;
+import static java.util.Arrays.*;
 
 class StatisticsCollectionTests {
 
@@ -65,12 +64,7 @@ void reportCollectedPercentagesInDecreasingOrder() {
 
 		ReportEntry entry = collector.createReportEntry();
 
-		List<String> stats = Arrays
-				.stream(entry.getKeyValuePairs().get(StatisticsCollector.KEY_STATISTICS).split(System.getProperty("line.separator"))) //
-				.map(String::trim) //
-				.filter(s -> !s.isEmpty()) //
-				.collect(Collectors.toList());
-
+		List<String> stats = parseStatistics(entry);
 		Assertions.assertThat(stats).containsExactly( //
 				"four  : 40 %", //
 				"three : 30 %", //
@@ -79,6 +73,30 @@ void reportCollectedPercentagesInDecreasingOrder() {
 		);
 	}
 
+	@Example
+	void nullKeysAreNotReported() {
+		StatisticsCollector collector = new StatisticsCollector();
+
+		collector.collect("aKey");
+		collector.collect(null);
+		collector.collect("aKey");
+		collector.collect(null);
+
+		ReportEntry entry = collector.createReportEntry();
+
+		List<String> stats = parseStatistics(entry);
+		Assertions.assertThat(stats).containsExactly( //
+			"aKey : 50 %" //
+		);
+	}
+
+	private List<String> parseStatistics(ReportEntry entry) {
+		return Arrays.stream(entry.getKeyValuePairs().get(StatisticsCollector.KEY_STATISTICS).split(System.getProperty("line.separator"))) //
+			.map(String::trim) //
+			.filter(s -> !s.isEmpty()) //
+			.collect(Collectors.toList());
+	}
+
 	@Example
 	void reportDoubleValues() {
 		StatisticsCollector collector = new StatisticsCollector();
@@ -90,12 +108,7 @@ void reportDoubleValues() {
 
 		ReportEntry entry = collector.createReportEntry();
 
-		List<String> stats = Arrays
-				.stream(entry.getKeyValuePairs().get(StatisticsCollector.KEY_STATISTICS).split(System.getProperty("line.separator"))) //
-				.map(String::trim) //
-				.filter(s -> !s.isEmpty()) //
-				.collect(Collectors.toList());
-
+		List<String> stats = parseStatistics(entry);
 		Assertions.assertThat(stats).containsExactlyInAnyOrder( //
 				"two 2   : 25 %", //
 				"three 2 : 25 %", //