From e7b9fd5e4dace47fd6b35497ca557101d3d6e0fd Mon Sep 17 00:00:00 2001 From: Fabian Meumertzheim Date: Mon, 16 Oct 2023 10:55:25 +0200 Subject: [PATCH] tests: Do not retain values in StressTest StressTest required a lot of memory since it collected all `init` and `mutate` return values in a list. Instead, cross values off of a short list for "contains" type checks and use `hashCode()` to stand in for the actual value in statistical tests. Verified locally that the test now passes with `--jvmopt=-Xmx512M`. --- .../jazzer/mutation/mutator/StressTest.java | 176 +++++++++++++----- 1 file changed, 129 insertions(+), 47 deletions(-) diff --git a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/StressTest.java b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/StressTest.java index 8b4f7a125..523d1cf57 100644 --- a/src/test/java/com/code_intelligence/jazzer/mutation/mutator/StressTest.java +++ b/src/test/java/com/code_intelligence/jazzer/mutation/mutator/StressTest.java @@ -20,14 +20,19 @@ import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.extendWithZeros; import static com.code_intelligence.jazzer.mutation.support.Preconditions.require; import static com.code_intelligence.jazzer.mutation.support.TestSupport.anyPseudoRandom; +import static com.code_intelligence.jazzer.mutation.support.TestSupport.asMap; import static com.code_intelligence.jazzer.mutation.support.TypeSupport.asAnnotatedType; import static com.google.common.truth.Truth.assertThat; import static com.google.common.truth.Truth.assertWithMessage; import static java.lang.Math.floor; import static java.lang.Math.pow; import static java.lang.Math.sqrt; +import static java.util.Arrays.asList; +import static java.util.Arrays.stream; import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; +import static java.util.stream.Collectors.toList; +import static java.util.stream.Collectors.toMap; import static java.util.stream.IntStream.rangeClosed; import static org.junit.jupiter.params.provider.Arguments.arguments; @@ -77,6 +82,7 @@ import java.io.IOException; import java.lang.reflect.AnnotatedType; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -179,10 +185,22 @@ null, emptyList(), singletonList(null), singletonList(false), singletonList(true new TypeHolder<@NotNull Map<@NotNull Boolean, @NotNull Boolean>>() {}.annotatedType(), "Map", false, - // 1 0-element map, 4 1-element maps - distinctElements(1 + 4), - // 1 0-element map, 4 1-element maps, 4 2-element maps - distinctElements(1 + 4 + 4)), + exactly( + asMap(), + asMap(false, false), + asMap(false, true), + asMap(true, false), + asMap(true, true)), + exactly( + asMap(), + asMap(false, false), + asMap(false, true), + asMap(true, false), + asMap(true, true), + asMap(false, false, true, false), + asMap(false, false, true, true), + asMap(false, true, true, false), + asMap(false, true, true, true))), arguments( asAnnotatedType(byte.class), "Byte", @@ -567,20 +585,25 @@ public static Stream protoStressTestCases() { SingleOptionOneOfField3.newBuilder().setBoolField(true).build()))); } - @SafeVarargs - private static Consumer> all(Consumer>... checks) { - return list -> { - for (Consumer> check : checks) { - check.accept(list); + private static CloseableConsumer all(CloseableConsumer... checks) { + return new CloseableConsumer() { + @Override + public void close() throws Exception { + for (CloseableConsumer check : checks) { + check.close(); + } } - }; - } - private static Consumer> distinctElements(int num) { - return list -> assertThat(new HashSet<>(list).size()).isAtLeast(num); + @Override + public void accept(Object value) { + for (CloseableConsumer check : checks) { + check.accept(value); + } + } + }; } - private static Consumer> manyDistinctElements() { + private static CloseableConsumer manyDistinctElements() { return distinctElementsRatio(MANY_DISTINCT_ELEMENTS_RATIO); } @@ -603,8 +626,7 @@ private static int boundHits(long domainSize, double probability) { * Asserts that a given list contains at least as many distinct elements as can be expected when * picking {@code picks} out of {@code domainSize} elements uniformly at random. */ - private static Consumer> expectedNumberOfDistinctElements( - long domainSize, int picks) { + private static CloseableConsumer expectedNumberOfDistinctElements(long domainSize, int picks) { // https://www.randomservices.org/random/urn/Birthday.html#mom2 double expectedValue = domainSize * (1 - pow(1 - 1.0 / domainSize, picks)); double variance = @@ -615,56 +637,118 @@ private static Consumer> expectedNumberOfDistinctElements( // Allow missing the expected value by two standard deviations. For a normal distribution, // this would correspond to 95% of all cases. int almostCertainLowerBound = (int) floor(expectedValue - 2 * standardDeviation); - return list -> + HashSet hashes = new HashSet<>(); + return new CloseableConsumer() { + @Override + public void accept(Object value) { + hashes.add(Objects.hashCode(value)); + } + + @Override + public void close() { assertWithMessage( "V=distinct elements among %s picked out of %s\nE[V]=%s\nσ[V]=%s", picks, domainSize, expectedValue, standardDeviation) - .that(new HashSet<>(list).size()) + .that(hashes.size()) .isAtLeast(almostCertainLowerBound); + } + }; } - private static Consumer> distinctElementsRatio(double ratio) { + private static CloseableConsumer distinctElementsRatio(double ratio) { require(ratio > 0); require(ratio <= 1); - return list -> assertThat(new HashSet<>(list).size() / (double) list.size()).isAtLeast(ratio); + List hashes = new ArrayList<>(); + return new CloseableConsumer() { + @Override + public void accept(Object value) { + hashes.add(Objects.hashCode(value)); + } + + @Override + public void close() { + assertThat(new HashSet<>(hashes).size() / (double) hashes.size()).isAtLeast(ratio); + } + }; } - private static Consumer> exactly(Object... expected) { - return list -> assertThat(new HashSet<>(list)).containsExactly(expected); + private static CloseableConsumer exactly(Object... expected) { + return containsInternal(true, expected); } - private static Consumer> contains(Object... expected) { - return list -> assertThat(new HashSet<>(list)).containsAtLeastElementsIn(expected); + private static CloseableConsumer contains(Object... expected) { + return containsInternal(false, expected); } - private static Consumer> doesNotContain(Object... expected) { - return list -> assertThat(new HashSet<>(list)).containsNoneIn(expected); + private static CloseableConsumer containsInternal(boolean exactly, Object... expected) { + Map sawValue = + stream(expected) + .collect( + toMap( + value -> value, + value -> false, + (a, b) -> { + throw new IllegalStateException("Duplicate value " + a); + }, + HashMap::new)); + return new CloseableConsumer() { + @Override + public void accept(Object value) { + if (exactly) { + assertThat(value).isIn(sawValue.keySet()); + } + sawValue.put(value, true); + } + + @Override + public void close() { + assertThat(sawValue.entrySet().stream().filter(e -> !e.getValue()).collect(toList())) + .isEmpty(); + } + }; } - private static Consumer> mapSizeInClosedRange(int min, int max) { - return list -> { - list.forEach( - map -> { - if (map instanceof Map) { - assertThat(((Map) map).size()).isAtLeast(min); - assertThat(((Map) map).size()).isAtMost(max); - } else { - throw new IllegalArgumentException( - "Expected a list of maps, got list of" + map.getClass().getName()); - } - }); + private static CloseableConsumer doesNotContain(Object... expected) { + return new CloseableConsumer() { + @Override + public void accept(Object value) { + assertThat(value).isNotIn(asList(expected)); + } + + @Override + public void close() {} }; } + private static CloseableConsumer mapSizeInClosedRange(int min, int max) { + return new CloseableConsumer() { + @Override + public void accept(Object map) { + if (map instanceof Map) { + assertThat(((Map) map).size()).isAtLeast(min); + assertThat(((Map) map).size()).isAtMost(max); + } else { + throw new IllegalArgumentException( + "Expected a list of maps, got list of" + map.getClass().getName()); + } + } + + @Override + public void close() {} + }; + } + + interface CloseableConsumer extends AutoCloseable, Consumer {} + @ParameterizedTest(name = "{index} {0}, {1}") @MethodSource({"stressTestCases", "protoStressTestCases"}) void genericMutatorStressTest( AnnotatedType type, String mutatorTree, boolean hasFixedSize, - Consumer> expectedInitValues, - Consumer> expectedMutatedValues) - throws IOException { + CloseableConsumer checkInitValues, + CloseableConsumer checkMutatedValues) + throws Exception { validateAnnotationUsage(type); SerializingMutator mutator = Mutators.newFactory().createOrThrow(type); assertThat(mutator.toString()).isEqualTo(mutatorTree); @@ -678,8 +762,6 @@ void genericMutatorStressTest( PseudoRandom rng = anyPseudoRandom(); - List initValues = new ArrayList<>(); - List mutatedValues = new ArrayList<>(); for (int i = 0; i < NUM_INITS; i++) { Object value = mutator.init(rng); @@ -689,7 +771,7 @@ void genericMutatorStressTest( testReadWriteRoundtrip(mutator, fixedValue); testReadWriteExclusiveRoundtrip(mutator, fixedValue); - initValues.add(mutator.detach(value)); + checkInitValues.accept(value); value = fixFloatingPointsForProtos(value); for (int mutation = 0; mutation < NUM_MUTATE_PER_INIT; mutation++) { @@ -705,7 +787,7 @@ void genericMutatorStressTest( } } - mutatedValues.add(mutator.detach(value)); + checkMutatedValues.accept(value); // For proto messages, each float field with value -0.0f, and double field with value -0.0 // will be converted to 0.0f and 0.0, respectively. This is because the values -0f and 0f @@ -720,8 +802,8 @@ void genericMutatorStressTest( } } - expectedInitValues.accept(initValues); - expectedMutatedValues.accept(mutatedValues); + checkInitValues.close(); + checkMutatedValues.close(); } private static void testReadWriteExclusiveRoundtrip(Serializer serializer, T value)