Skip to content

Commit

Permalink
tests: Do not retain values in StressTest
Browse files Browse the repository at this point in the history 
StressTest required a lot of memory since it collected all `init` and
`mutate` return values in a list. Instead, cross values off of a short
list for "contains" type checks and use `hashCode()` to stand in for the
actual value in statistical tests.

Verified locally that the test now passes with `--jvmopt=-Xmx512M`.
  • Loading branch information
@fmeum
fmeum committed Oct 16, 2023
1 parent f90c39d commit e7b9fd5
Showing 1 changed file with 129 additions and 47 deletions.
176 changes: 129 additions & 47 deletions src/test/java/com/code_intelligence/jazzer/mutation/mutator/StressTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,19 @@
import static com.code_intelligence.jazzer.mutation.support.InputStreamSupport.extendWithZeros;
import static com.code_intelligence.jazzer.mutation.support.Preconditions.require;
import static com.code_intelligence.jazzer.mutation.support.TestSupport.anyPseudoRandom;
import static com.code_intelligence.jazzer.mutation.support.TestSupport.asMap;
import static com.code_intelligence.jazzer.mutation.support.TypeSupport.asAnnotatedType;
import static com.google.common.truth.Truth.assertThat;
import static com.google.common.truth.Truth.assertWithMessage;
import static java.lang.Math.floor;
import static java.lang.Math.pow;
import static java.lang.Math.sqrt;
import static java.util.Arrays.asList;
import static java.util.Arrays.stream;
import static java.util.Collections.emptyList;
import static java.util.Collections.singletonList;
import static java.util.stream.Collectors.toList;
import static java.util.stream.Collectors.toMap;
import static java.util.stream.IntStream.rangeClosed;
import static org.junit.jupiter.params.provider.Arguments.arguments;

Expand Down Expand Up @@ -77,6 +82,7 @@
import java.io.IOException;
import java.lang.reflect.AnnotatedType;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -179,10 +185,22 @@ null, emptyList(), singletonList(null), singletonList(false), singletonList(true
new TypeHolder<@NotNull Map<@NotNull Boolean, @NotNull Boolean>>() {}.annotatedType(),
"Map<Boolean,Boolean>",
false,
// 1 0-element map, 4 1-element maps
distinctElements(1 + 4),
// 1 0-element map, 4 1-element maps, 4 2-element maps
distinctElements(1 + 4 + 4)),
exactly(
asMap(),
asMap(false, false),
asMap(false, true),
asMap(true, false),
asMap(true, true)),
exactly(
asMap(),
asMap(false, false),
asMap(false, true),
asMap(true, false),
asMap(true, true),
asMap(false, false, true, false),
asMap(false, false, true, true),
asMap(false, true, true, false),
asMap(false, true, true, true))),
arguments(
asAnnotatedType(byte.class),
"Byte",
Expand Down Expand Up @@ -567,20 +585,25 @@ public static Stream<Arguments> protoStressTestCases() {
SingleOptionOneOfField3.newBuilder().setBoolField(true).build())));
}

@SafeVarargs
private static Consumer<List<Object>> all(Consumer<List<Object>>... checks) {
return list -> {
for (Consumer<List<Object>> check : checks) {
check.accept(list);
private static CloseableConsumer all(CloseableConsumer... checks) {
return new CloseableConsumer() {
@Override
public void close() throws Exception {
for (CloseableConsumer check : checks) {
check.close();
}
}
};
}

private static Consumer<List<Object>> distinctElements(int num) {
return list -> assertThat(new HashSet<>(list).size()).isAtLeast(num);
@Override
public void accept(Object value) {
for (CloseableConsumer check : checks) {
check.accept(value);
}
}
};
}

private static Consumer<List<Object>> manyDistinctElements() {
private static CloseableConsumer manyDistinctElements() {
return distinctElementsRatio(MANY_DISTINCT_ELEMENTS_RATIO);
}

Expand All @@ -603,8 +626,7 @@ private static int boundHits(long domainSize, double probability) {
* Asserts that a given list contains at least as many distinct elements as can be expected when
* picking {@code picks} out of {@code domainSize} elements uniformly at random.
*/
private static Consumer<List<Object>> expectedNumberOfDistinctElements(
long domainSize, int picks) {
private static CloseableConsumer expectedNumberOfDistinctElements(long domainSize, int picks) {
// https://www.randomservices.org/random/urn/Birthday.html#mom2
double expectedValue = domainSize * (1 - pow(1 - 1.0 / domainSize, picks));
double variance =
Expand All @@ -615,56 +637,118 @@ private static Consumer<List<Object>> expectedNumberOfDistinctElements(
// Allow missing the expected value by two standard deviations. For a normal distribution,
// this would correspond to 95% of all cases.
int almostCertainLowerBound = (int) floor(expectedValue - 2 * standardDeviation);
return list ->
HashSet<Integer> hashes = new HashSet<>();
return new CloseableConsumer() {
@Override
public void accept(Object value) {
hashes.add(Objects.hashCode(value));
}

@Override
public void close() {
assertWithMessage(
"V=distinct elements among %s picked out of %s\nE[V]=%s\nσ[V]=%s",
picks, domainSize, expectedValue, standardDeviation)
.that(new HashSet<>(list).size())
.that(hashes.size())
.isAtLeast(almostCertainLowerBound);
}
};
}

private static Consumer<List<Object>> distinctElementsRatio(double ratio) {
private static CloseableConsumer distinctElementsRatio(double ratio) {
require(ratio > 0);
require(ratio <= 1);
return list -> assertThat(new HashSet<>(list).size() / (double) list.size()).isAtLeast(ratio);
List<Integer> hashes = new ArrayList<>();
return new CloseableConsumer() {
@Override
public void accept(Object value) {
hashes.add(Objects.hashCode(value));
}

@Override
public void close() {
assertThat(new HashSet<>(hashes).size() / (double) hashes.size()).isAtLeast(ratio);
}
};
}

private static Consumer<List<Object>> exactly(Object... expected) {
return list -> assertThat(new HashSet<>(list)).containsExactly(expected);
private static CloseableConsumer exactly(Object... expected) {
return containsInternal(true, expected);
}

private static Consumer<List<Object>> contains(Object... expected) {
return list -> assertThat(new HashSet<>(list)).containsAtLeastElementsIn(expected);
private static CloseableConsumer contains(Object... expected) {
return containsInternal(false, expected);
}

private static Consumer<List<Object>> doesNotContain(Object... expected) {
return list -> assertThat(new HashSet<>(list)).containsNoneIn(expected);
private static CloseableConsumer containsInternal(boolean exactly, Object... expected) {
Map<Object, Boolean> sawValue =
stream(expected)
.collect(
toMap(
value -> value,
value -> false,
(a, b) -> {
throw new IllegalStateException("Duplicate value " + a);
},
HashMap::new));
return new CloseableConsumer() {
@Override
public void accept(Object value) {
if (exactly) {
assertThat(value).isIn(sawValue.keySet());
}
sawValue.put(value, true);
}

@Override
public void close() {
assertThat(sawValue.entrySet().stream().filter(e -> !e.getValue()).collect(toList()))
.isEmpty();
}
};
}

private static Consumer<List<Object>> mapSizeInClosedRange(int min, int max) {
return list -> {
list.forEach(
map -> {
if (map instanceof Map) {
assertThat(((Map) map).size()).isAtLeast(min);
assertThat(((Map) map).size()).isAtMost(max);
} else {
throw new IllegalArgumentException(
"Expected a list of maps, got list of" + map.getClass().getName());
}
});
private static CloseableConsumer doesNotContain(Object... expected) {
return new CloseableConsumer() {
@Override
public void accept(Object value) {
assertThat(value).isNotIn(asList(expected));
}

@Override
public void close() {}
};
}

private static CloseableConsumer mapSizeInClosedRange(int min, int max) {
return new CloseableConsumer() {
@Override
public void accept(Object map) {
if (map instanceof Map) {
assertThat(((Map) map).size()).isAtLeast(min);
assertThat(((Map) map).size()).isAtMost(max);
} else {
throw new IllegalArgumentException(
"Expected a list of maps, got list of" + map.getClass().getName());
}
}

@Override
public void close() {}
};
}

interface CloseableConsumer extends AutoCloseable, Consumer<Object> {}

@ParameterizedTest(name = "{index} {0}, {1}")
@MethodSource({"stressTestCases", "protoStressTestCases"})
void genericMutatorStressTest(
AnnotatedType type,
String mutatorTree,
boolean hasFixedSize,
Consumer<List<Object>> expectedInitValues,
Consumer<List<Object>> expectedMutatedValues)
throws IOException {
CloseableConsumer checkInitValues,
CloseableConsumer checkMutatedValues)
throws Exception {
validateAnnotationUsage(type);
SerializingMutator mutator = Mutators.newFactory().createOrThrow(type);
assertThat(mutator.toString()).isEqualTo(mutatorTree);
Expand All @@ -678,8 +762,6 @@ void genericMutatorStressTest(

PseudoRandom rng = anyPseudoRandom();

List<Object> initValues = new ArrayList<>();
List<Object> mutatedValues = new ArrayList<>();
for (int i = 0; i < NUM_INITS; i++) {
Object value = mutator.init(rng);

Expand All @@ -689,7 +771,7 @@ void genericMutatorStressTest(
testReadWriteRoundtrip(mutator, fixedValue);
testReadWriteExclusiveRoundtrip(mutator, fixedValue);

initValues.add(mutator.detach(value));
checkInitValues.accept(value);
value = fixFloatingPointsForProtos(value);

for (int mutation = 0; mutation < NUM_MUTATE_PER_INIT; mutation++) {
Expand All @@ -705,7 +787,7 @@ void genericMutatorStressTest(
}
}

mutatedValues.add(mutator.detach(value));
checkMutatedValues.accept(value);

// For proto messages, each float field with value -0.0f, and double field with value -0.0
// will be converted to 0.0f and 0.0, respectively. This is because the values -0f and 0f
Expand All @@ -720,8 +802,8 @@ void genericMutatorStressTest(
}
}

expectedInitValues.accept(initValues);
expectedMutatedValues.accept(mutatedValues);
checkInitValues.close();
checkMutatedValues.close();
}

private static <T> void testReadWriteExclusiveRoundtrip(Serializer<T> serializer, T value)
Expand Down

0 comments on commit e7b9fd5

Please sign in to comment.