diff --git a/.bazelversion b/.bazelversion index 643d68899..c579054ee 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -132d2497f4337473fb084677c7f2b98c85e67c92 +b29649fbdc983cd62a58b9b09ef699867e7c5b69 diff --git a/README.md b/README.md index be0a953bf..210dcc0b9 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ After a few seconds, Jazzer should trigger an `AssertionError`, reproducing a bu ### JUnit 5 -The following steps assume that JUnit 5 is set up for your project, for example based on the official [junit5-samples](https://github.com/junit-team/junit5-samples). +The following steps assume that JUnit 5.9.0 or higher is set up for your project, for example based on the official [junit5-samples](https://github.com/junit-team/junit5-samples). 1. Add a dependency on `com.code-intelligence:jazzer-junit:`. All Jazzer Maven artifacts are signed with [this key](deploy/maven.pub). diff --git a/WORKSPACE.bazel b/WORKSPACE.bazel index 5f36ed627..f13f30dd8 100644 --- a/WORKSPACE.bazel +++ b/WORKSPACE.bazel @@ -35,10 +35,10 @@ rules_java_toolchains() http_archive( name = "com_google_protobuf", patches = ["//third_party:protobuf-disable-layering_check.patch"], - sha256 = "ddf8c9c1ffccb7e80afd183b3bd32b3b62f7cc54b106be190bf49f2bc09daab5", - strip_prefix = "protobuf-23.2", + sha256 = "0930b1a6eb840a2295dfcb13bb5736d1292c3e0d61a90391181399327be7d8f1", + strip_prefix = "protobuf-24.1", # Keep in sync with com_google_protobuf_protobuf_java in repositories.bzl. - urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v23.2/protobuf-23.2.tar.gz"], + urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v24.1/protobuf-24.1.tar.gz"], ) http_archive( diff --git a/bazel/fuzz_target.bzl b/bazel/fuzz_target.bzl index 62c7a7b39..ab7ea5006 100644 --- a/bazel/fuzz_target.bzl +++ b/bazel/fuzz_target.bzl @@ -128,3 +128,81 @@ def java_fuzz_target_test( use_testrunner = False, tags = tags, ) + +_BASE_SEED = 2735196724 + +def fuzzer_benchmark( + name, + *, + num_seeds, + max_runs, + env = {}, + fuzzer_args = [], + tags = [], + **kwargs): + """Creates multiple instances of a Java fuzz target test with different seeds for benchmarking. + + The target `` is a `test_suite` tagged with `"manual"`that can be used to run all + individual instances of the fuzz target test at once. The individual tests are tagged with + `"benchmark"` and `"manual"`. This is meant to run in CI and ensure that the maximum number of + runs does not regress. + + The target `.stats` can be run with `bazel run` to execute the benchmark and derive some + statistics about the number of runs. + + This macro is set up specifically to make efficient use of Bazel's scheduling and caching + capabilities: By having one target per run instead of a single target that runs the fuzz test + multiple times, Bazel can schedule the runs concurrently and avoid timeouts on slow runners. + When increasing the number of seeds, existing results can be reused from the cache. + + Args: + num_seeds: The number of different seeds to try; corresponds to the number of individual tests + generated. + max_runs: The maximum number of runs that each individual test is allowed to run for. Keep + this as low as possible with a small margin to catch regressions. + """ + seed = _BASE_SEED + tests = [] + for i in range(num_seeds): + test_name = "{}_{}".format(name, i + 1) + tests.append(test_name) + java_fuzz_target_test( + name = test_name, + fuzzer_args = fuzzer_args + [ + "-print_final_stats=1", + "-seed={}".format(seed), + "-runs={}".format(max_runs), + ], + env = env | {"JAZZER_NO_EXPLICIT_SEED": "1"}, + tags = tags + ["manual", "benchmark"], + verify_crash_input = False, + verify_crash_reproducer = False, + **kwargs + ) + seed = (31 * seed) % 4294967295 + + native.test_suite( + name = name, + tests = tests, + tags = ["manual"], + ) + + native.sh_binary( + name = name + ".stats", + srcs = [Label("//bazel/tools:compute_benchmark_stats.sh")], + env = { + "TEST_SUITE_LABEL": str(native.package_relative_label(name)), + }, + args = [ + native.package_name() + "/" + test + for test in tests + ], + ) + +def all_tests_above(): + """Returns the labels of all test targets in the current package defined before this call.""" + return [ + ":" + r["name"] + for r in native.existing_rules().values() + if r["kind"].endswith("_test") + ] diff --git a/bazel/tools/BUILD.bazel b/bazel/tools/BUILD.bazel new file mode 100644 index 000000000..8b4b0be4c --- /dev/null +++ b/bazel/tools/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["compute_benchmark_stats.sh"]) diff --git a/bazel/tools/compute_benchmark_stats.sh b/bazel/tools/compute_benchmark_stats.sh new file mode 100755 index 000000000..7cc26a0f2 --- /dev/null +++ b/bazel/tools/compute_benchmark_stats.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright 2023 Code Intelligence GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# Run the benchmark given by $TEST_SUITE_LABEL, then get all "stat::number_of_executed_units: 12345" +# lines printed by libFuzzer from the logs for the tests passed in on the command line in the form +# "path/to/pkg/name" and compute statistics. +# +# Requires jq to be installed locally. + +cd "$BUILD_WORKSPACE_DIRECTORY" || exit 1 +# Remove the -runs limit to collect statistics even if the current run limit is too low. +bazel test "$TEST_SUITE_LABEL" --test_arg=-runs=999999999 +echo "$@" \ + | xargs -L1 printf "bazel-testlogs/%s/test.log " \ + | xargs -L1 cat \ + | grep '^stat::number_of_executed_units' \ + | cut -d' ' -f2 \ + | jq -s '{values:sort,minimum:min,maximum:max,average:(add/length),median:(sort|if length%2==1 then.[length/2|floor]else[.[length/2-1,length/2]]|add/2 end)}' diff --git a/deploy/BUILD.bazel b/deploy/BUILD.bazel index c6d3bbc08..a598fd7eb 100644 --- a/deploy/BUILD.bazel +++ b/deploy/BUILD.bazel @@ -20,6 +20,10 @@ sh_binary( name = "deploy", srcs = ["deploy.sh"], args = [JAZZER_COORDINATES], + data = ["@bazel_tools//tools/jdk:current_host_java_runtime"], + env = {"JAVA_EXECPATH": "$(JAVA)"}, + toolchains = ["@bazel_tools//tools/jdk:current_host_java_runtime"], + deps = ["@bazel_tools//tools/bash/runfiles"], ) java_export( @@ -90,6 +94,21 @@ java_export( ], ) +sh_test( + name = "jazzer_version_test", + srcs = ["jazzer_version_test.sh"], + data = [ + ":jazzer", + "@bazel_tools//tools/jdk:current_java_runtime", + ], + env = { + "JAVA_EXECPATH": "$(JAVA)", + "JAZZER_RLOCATIONPATH": "$(rlocationpath :jazzer)", + }, + toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"], + deps = ["@bazel_tools//tools/bash/runfiles"], +) + [ sh_test( name = artifact + "_artifact_test", diff --git a/deploy/deploy.sh b/deploy/deploy.sh index edf8b4071..ffd45455f 100755 --- a/deploy/deploy.sh +++ b/deploy/deploy.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env sh +#!/usr/bin/env bash # Copyright 2022 Code Intelligence GmbH # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -33,6 +33,24 @@ JAZZER_COORDINATES=$1 [ ! -f "${JAZZER_JAR_PATH}" ] && \ fail "JAZZER_JAR_PATH does not exist at '$JAZZER_JAR_PATH'" +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- + +# JAZZER_EXECPATH is a path of the form "external/remotejdk_17/bin/java". We need to strip of the +# leading external to get a path we can pass to rlocation. +java_rlocationpath=$(echo "$JAVA_EXECPATH" | cut -d/ -f2-) +java=$(rlocation "$java_rlocationpath") +"$java" -jar "${JAZZER_JAR_PATH}" --version 2>&1 | grep '^Jazzer v' || \ + fail "JAZZER_JAR_PATH is not a valid jazzer.jar" + MAVEN_REPO=https://oss.sonatype.org/service/local/staging/deploy/maven2 # The Jazzer jar itself bundles native libraries for multiple architectures and thus can't be built diff --git a/deploy/jazzer_version_test.sh b/deploy/jazzer_version_test.sh new file mode 100755 index 000000000..8cb9af356 --- /dev/null +++ b/deploy/jazzer_version_test.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copyright 2023 Code Intelligence GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- + +# JAZZER_EXECPATH is a path of the form "external/remotejdk_17/bin/java". We need to strip of the +# leading external to get a path we can pass to rlocation. +java_rlocationpath=$(echo "$JAVA_EXECPATH" | cut -d/ -f2-) +java=$(rlocation "$java_rlocationpath") +jazzer=$(rlocation "$JAZZER_RLOCATIONPATH") +[ -f "$jazzer" ] || exit 1 +jazzer_version_output=$("$java" -jar "$jazzer" --version 2>&1) +echo "$jazzer_version_output" +echo "$jazzer_version_output" | tr -d '\n' | grep -q '^Jazzer v[0-9.]*$' || exit 1 diff --git a/examples/junit/src/test/java/com/example/BUILD.bazel b/examples/junit/src/test/java/com/example/BUILD.bazel index 8bbdcecd2..e903d7dcd 100644 --- a/examples/junit/src/test/java/com/example/BUILD.bazel +++ b/examples/junit/src/test/java/com/example/BUILD.bazel @@ -1,5 +1,11 @@ load("//bazel:fuzz_target.bzl", "java_fuzz_target_test") +java_library( + name = "test_successful_exception", + srcs = ["TestSuccessfulException.java"], + visibility = ["//src/test/java/com/code_intelligence/jazzer/junit:__subpackages__"], +) + java_binary( name = "ExampleFuzzTests", testonly = True, @@ -9,11 +15,13 @@ java_binary( "//src/test/java/com/code_intelligence/jazzer/junit:__pkg__", ], deps = [ + ":test_successful_exception", "//deploy:jazzer", "//deploy:jazzer-api", "//deploy:jazzer-junit", "//examples/junit/src/main/java/com/example:parser", "//examples/junit/src/test/resources:example_seed_corpora", + "@maven//:com_google_truth_truth", "@maven//:org_junit_jupiter_junit_jupiter_api", "@maven//:org_junit_jupiter_junit_jupiter_params", "@maven//:org_mockito_mockito_core", @@ -63,9 +71,9 @@ java_fuzz_target_test( java_fuzz_target_test( name = "LifecycleFuzzTest", srcs = ["LifecycleFuzzTest.java"], - allowed_findings = ["java.io.IOException"], + allowed_findings = ["com.example.TestSuccessfulException"], fuzzer_args = [ - "-runs=0", + "-runs=3", ], target_class = "com.example.LifecycleFuzzTest", verify_crash_reproducer = False, @@ -73,8 +81,10 @@ java_fuzz_target_test( ":junit_runtime", ], deps = [ + ":test_successful_exception", "//examples/junit/src/main/java/com/example:parser", "//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test", + "@maven//:com_google_truth_truth", "@maven//:org_junit_jupiter_junit_jupiter_api", ], ) diff --git a/examples/junit/src/test/java/com/example/LifecycleFuzzTest.java b/examples/junit/src/test/java/com/example/LifecycleFuzzTest.java index 0d5dc2c71..6afe78f37 100644 --- a/examples/junit/src/test/java/com/example/LifecycleFuzzTest.java +++ b/examples/junit/src/test/java/com/example/LifecycleFuzzTest.java @@ -16,73 +16,125 @@ package com.example; +import static com.google.common.truth.Truth.assertThat; +import static java.util.Arrays.asList; +import static java.util.Collections.unmodifiableList; + import com.code_intelligence.jazzer.junit.FuzzTest; -import java.io.IOException; +import com.example.LifecycleFuzzTest.LifecycleCallbacks1; +import com.example.LifecycleFuzzTest.LifecycleCallbacks2; +import com.example.LifecycleFuzzTest.LifecycleCallbacks3; +import java.util.ArrayList; +import java.util.List; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.MethodOrderer; import org.junit.jupiter.api.TestMethodOrder; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.api.extension.ExtensionContext; import org.junit.jupiter.api.extension.TestInstancePostProcessor; @TestMethodOrder(MethodOrderer.MethodName.class) @ExtendWith(LifecycleFuzzTest.LifecycleInstancePostProcessor.class) +@ExtendWith(LifecycleCallbacks1.class) +@ExtendWith(LifecycleCallbacks2.class) +@ExtendWith(LifecycleCallbacks3.class) class LifecycleFuzzTest { - // In fuzzing mode, the test is invoked once on the empty input and once with Jazzer. - private static final int EXPECTED_EACH_COUNT = - System.getenv().getOrDefault("JAZZER_FUZZ", "").isEmpty() ? 1 : 2; - - private static int beforeAllCount = 0; - private static int beforeEachGlobalCount = 0; - private static int afterEachGlobalCount = 0; - private static int afterAllCount = 0; + private static final ArrayList events = new ArrayList<>(); + private static final long RUNS = 3; private boolean beforeEachCalledOnInstance = false; private boolean testInstancePostProcessorCalledOnInstance = false; @BeforeAll static void beforeAll() { - beforeAllCount++; + events.add("beforeAll"); } @BeforeEach - void beforeEach() { - beforeEachGlobalCount++; + void beforeEach1() { + events.add("beforeEach1"); beforeEachCalledOnInstance = true; } + @BeforeEach + void beforeEach2() { + events.add("beforeEach2"); + } + + @BeforeEach + void beforeEach3() { + events.add("beforeEach3"); + } + @Disabled @FuzzTest void disabledFuzz(byte[] data) { + events.add("disabledFuzz"); throw new AssertionError("This test should not be executed"); } - @FuzzTest(maxDuration = "1s") + @FuzzTest(maxExecutions = RUNS) void lifecycleFuzz(byte[] data) { - Assertions.assertEquals(1, beforeAllCount); - Assertions.assertEquals(beforeEachGlobalCount, afterEachGlobalCount + 1); - Assertions.assertTrue(beforeEachCalledOnInstance); - Assertions.assertTrue(testInstancePostProcessorCalledOnInstance); + events.add("lifecycleFuzz"); + assertThat(beforeEachCalledOnInstance).isTrue(); + assertThat(testInstancePostProcessorCalledOnInstance).isTrue(); + } + + @AfterEach + void afterEach1() { + events.add("afterEach1"); } @AfterEach - void afterEach() { - afterEachGlobalCount++; + void afterEach2() { + events.add("afterEach2"); + } + + @AfterEach + void afterEach3() { + events.add("afterEach3"); } @AfterAll - static void afterAll() throws IOException { - afterAllCount++; - Assertions.assertEquals(1, beforeAllCount); - Assertions.assertEquals(EXPECTED_EACH_COUNT, beforeEachGlobalCount); - Assertions.assertEquals(EXPECTED_EACH_COUNT, afterEachGlobalCount); - Assertions.assertEquals(1, afterAllCount); - throw new IOException(); + static void afterAll() throws TestSuccessfulException { + events.add("afterAll"); + + boolean isRegressionTest = "".equals(System.getenv("JAZZER_FUZZ")); + boolean isFuzzingFromCommandLine = System.getenv("JAZZER_FUZZ") == null; + boolean isFuzzingFromJUnit = !isFuzzingFromCommandLine && !isRegressionTest; + + final List expectedBeforeEachEvents = unmodifiableList(asList("beforeEachCallback1", + "beforeEachCallback2", "beforeEachCallback3", "beforeEach1", "beforeEach2", "beforeEach3")); + final List expectedAfterEachEvents = unmodifiableList(asList("afterEach1", "afterEach2", + "afterEach3", "afterEachCallback3", "afterEachCallback2", "afterEachCallback1")); + + ArrayList expectedEvents = new ArrayList<>(); + expectedEvents.add("beforeAll"); + + // When run from the command-line, the fuzz test is not separately executed on the empty seed. + if (isRegressionTest || isFuzzingFromJUnit) { + expectedEvents.addAll(expectedBeforeEachEvents); + expectedEvents.add("lifecycleFuzz"); + expectedEvents.addAll(expectedAfterEachEvents); + } + if (isFuzzingFromJUnit || isFuzzingFromCommandLine) { + for (int i = 0; i < RUNS; i++) { + expectedEvents.addAll(expectedBeforeEachEvents); + expectedEvents.add("lifecycleFuzz"); + expectedEvents.addAll(expectedAfterEachEvents); + } + } + + expectedEvents.add("afterAll"); + + assertThat(events).containsExactlyElementsIn(expectedEvents).inOrder(); + throw new TestSuccessfulException("Lifecycle methods invoked as expected"); } static class LifecycleInstancePostProcessor implements TestInstancePostProcessor { @@ -91,4 +143,40 @@ public void postProcessTestInstance(Object o, ExtensionContext extensionContext) ((LifecycleFuzzTest) o).testInstancePostProcessorCalledOnInstance = true; } } + + static class LifecycleCallbacks1 implements BeforeEachCallback, AfterEachCallback { + @Override + public void beforeEach(ExtensionContext extensionContext) { + events.add("beforeEachCallback1"); + } + + @Override + public void afterEach(ExtensionContext extensionContext) { + events.add("afterEachCallback1"); + } + } + + static class LifecycleCallbacks2 implements BeforeEachCallback, AfterEachCallback { + @Override + public void beforeEach(ExtensionContext extensionContext) { + events.add("beforeEachCallback2"); + } + + @Override + public void afterEach(ExtensionContext extensionContext) { + events.add("afterEachCallback2"); + } + } + + static class LifecycleCallbacks3 implements BeforeEachCallback, AfterEachCallback { + @Override + public void beforeEach(ExtensionContext extensionContext) { + events.add("beforeEachCallback3"); + } + + @Override + public void afterEach(ExtensionContext extensionContext) { + events.add("afterEachCallback3"); + } + } } diff --git a/examples/junit/src/test/java/com/example/TestSuccessfulException.java b/examples/junit/src/test/java/com/example/TestSuccessfulException.java new file mode 100644 index 000000000..04608c181 --- /dev/null +++ b/examples/junit/src/test/java/com/example/TestSuccessfulException.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example; + +/** + * An exception thrown by a java_fuzz_target_test if the test run is considered successful. + * + *

Use this instead of a generic exception to ensure that tests do not pass if such a generic + * exception is thrown unexpectedly. + *

Use this instead of {@link com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow} and other + * Jazzer-specific exceptions as using them in tests leads to classloader issues: The exception + * classes may be loaded both in the bootstrap and the system classloader depending on when exactly + * the agent (and with it the bootstrap jar) is installed, which can cause in `instanceof` checks + * failing unexpectedly. + */ +public class TestSuccessfulException extends Exception { + public TestSuccessfulException(String message) { + super(message); + } +} diff --git a/maven.bzl b/maven.bzl index 903581570..7c1c1641c 100644 --- a/maven.bzl +++ b/maven.bzl @@ -14,19 +14,19 @@ load("@rules_jvm_external//:specs.bzl", "maven") -JAZZER_VERSION = "0.19.0" +JAZZER_VERSION = "0.20.1" JAZZER_COORDINATES = "com.code-intelligence:jazzer:%s" % JAZZER_VERSION JAZZER_API_COORDINATES = "com.code-intelligence:jazzer-api:%s" % JAZZER_VERSION JAZZER_JUNIT_COORDINATES = "com.code-intelligence:jazzer-junit:%s" % JAZZER_VERSION # keep sorted MAVEN_ARTIFACTS = [ - "org.junit.jupiter:junit-jupiter-api:5.8.2", - "org.junit.jupiter:junit-jupiter-engine:5.8.2", - "org.junit.jupiter:junit-jupiter-params:5.8.2", - "org.junit.platform:junit-platform-commons:jar:1.8.2", - "org.junit.platform:junit-platform-engine:jar:1.8.2", - "org.junit.platform:junit-platform-launcher:jar:1.8.2", + "org.junit.jupiter:junit-jupiter-api:5.9.0", + "org.junit.jupiter:junit-jupiter-engine:5.9.0", + "org.junit.jupiter:junit-jupiter-params:5.9.0", + "org.junit.platform:junit-platform-commons:jar:1.9.0", + "org.junit.platform:junit-platform-engine:jar:1.9.0", + "org.junit.platform:junit-platform-launcher:jar:1.9.0", "org.opentest4j:opentest4j:1.2.0", ] diff --git a/maven_install.json b/maven_install.json index 23b1845ea..0e626fb36 100644 --- a/maven_install.json +++ b/maven_install.json @@ -1,7 +1,7 @@ { "__AUTOGENERATED_FILE_DO_NOT_MODIFY_THIS_FILE_MANUALLY": "THERE_IS_NO_DATA_ONLY_ZUUL", - "__INPUT_ARTIFACTS_HASH": 1342424233, - "__RESOLVED_ARTIFACTS_HASH": 1070162798, + "__INPUT_ARTIFACTS_HASH": 68876267, + "__RESOLVED_ARTIFACTS_HASH": 1123237997, "conflict_resolution": { "junit:junit:4.12": "junit:junit:4.13.2" }, @@ -428,39 +428,39 @@ }, "org.junit.jupiter:junit-jupiter-api": { "shasums": { - "jar": "1808ee87e0f718cd6e25f3b75afc17956ac8a3edc48c7e9bab9f19f9a79e3801" + "jar": "3e370bcbb1e857fda5f0b203724116d02b05e788faa1eb2518814accf9cfb5b1" }, - "version": "5.8.2" + "version": "5.9.0" }, "org.junit.jupiter:junit-jupiter-engine": { "shasums": { - "jar": "753b7726cdd158bb34cedb94c161e2291896f47832a1e9eda53d970020a8184e" + "jar": "db86cbb3352719fa0a97800edfd09c20463c7f2ab4a04699244430bd8954583b" }, - "version": "5.8.2" + "version": "5.9.0" }, "org.junit.jupiter:junit-jupiter-params": { "shasums": { - "jar": "d1c22d6fe5483568c08c8913f34abd2303490c3480ce6c18a2ea31c65e44102a" + "jar": "b8cef7982dd53df84c957a6e9ac89ede967cf2e6d9340ef4c51786e20548c41b" }, - "version": "5.8.2" + "version": "5.9.0" }, "org.junit.platform:junit-platform-commons": { "shasums": { - "jar": "d2e015fca7130e79af2f4608dc54415e4b10b592d77333decb4b1a274c185050" + "jar": "e5894b710094b4caafc6280b8829a439fb764901ea0ae18d06ed80388b309b7a" }, - "version": "1.8.2" + "version": "1.9.0" }, "org.junit.platform:junit-platform-engine": { "shasums": { - "jar": "0b7d000f8c3e8e5f7d6b819649936e7b9938314e87c8f983805218ea57567e59" + "jar": "aaec735f7444a9fc055e206598de3d829c24e9c7a8eea6efdeeb1962087fe811" }, - "version": "1.8.2" + "version": "1.9.0" }, "org.junit.platform:junit-platform-launcher": { "shasums": { - "jar": "822156409fd83e682e4c5199b3460054299b538a058c2c6d0f5c9b6a5bdb7594" + "jar": "13000d464938249dce876d2c783c5319ad8863da2b214bcb9001f8c0ee491214" }, - "version": "1.8.2" + "version": "1.9.0" }, "org.junit.platform:junit-platform-reporting": { "shasums": { diff --git a/repositories.bzl b/repositories.bzl index 8399c7f9a..48cd98697 100644 --- a/repositories.bzl +++ b/repositories.bzl @@ -22,10 +22,10 @@ def jazzer_dependencies(android = False): maybe( http_archive, name = "platforms", - sha256 = "5308fc1d8865406a49427ba24a9ab53087f17f5266a7aabbfc28823f3916e1ca", + sha256 = "3a561c99e7bdbe9173aa653fd579fe849f1d8d67395780ab4770b1f381431d51", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.6/platforms-0.0.6.tar.gz", - "https://github.com/bazelbuild/platforms/releases/download/0.0.6/platforms-0.0.6.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz", + "https://github.com/bazelbuild/platforms/releases/download/0.0.7/platforms-0.0.7.tar.gz", ], ) @@ -47,6 +47,9 @@ def jazzer_dependencies(android = False): # https://github.com/bazelbuild/rules_kotlin/pull/1000 # Remove unnecessary dependency on a Java runtime for the target platform. "//third_party:rules_kotlin-remove-java-runtime-dep.patch", + # https://github.com/bazelbuild/rules_kotlin/pull/1005 + # Required for compatibility with recent Bazel 7 pre-releases. + "//third_party:rules_kotlin-remove-java-info-transitive-deps.patch", ], sha256 = "01293740a16e474669aba5b5a1fe3d368de5832442f164e4fbfc566815a8bc3a", url = "https://github.com/bazelbuild/rules_kotlin/releases/download/v1.8/rules_kotlin_release.tgz", @@ -61,6 +64,9 @@ def jazzer_dependencies(android = False): # Fixes an incompatibility with latest Bazel introduced in # https://github.com/bazelbuild/bazel/commit/d0e29582a2e788e8acdaf53fe30ab7f7dc592df3 "//third_party:rules_jvm_external-add-toolchain-to-maven-project-jar.patch", + # https://github.com/bazelbuild/rules_jvm_external/pull/952 + # Fixes javadoc generation when using neverlink dependencies. + "//third_party:rules_jvm_external-add-neverlink-deps-to-javadoc-classpath.patch", ], sha256 = "d31e369b854322ca5098ea12c69d7175ded971435e55c18dd9dd5f29cc5249ac", strip_prefix = "rules_jvm_external-5.3", @@ -168,9 +174,9 @@ def jazzer_dependencies(android = False): maybe( http_jar, name = "com_google_protobuf_protobuf_java", - sha256 = "18a057f5e0f828daa92b71c19df91f6bcc2aad067ca2cdd6b5698055ca7bcece", + sha256 = "b7eb9203fd2dd6e55b929debf2d079c949e0f9a85f15ec3a298b7534bc7ebd41", # Keep in sync with com_google_protobuf in WORKSPACE. - url = "https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.23.2/protobuf-java-3.23.2.jar", + url = "https://repo1.maven.org/maven2/com/google/protobuf/protobuf-java/3.24.1/protobuf-java-3.24.1.jar", ) maybe( diff --git a/selffuzz/cifuzz.yaml b/selffuzz/cifuzz.yaml index 95f3af76f..8d19f1073 100644 --- a/selffuzz/cifuzz.yaml +++ b/selffuzz/cifuzz.yaml @@ -40,7 +40,7 @@ engine-args: - --experimental_mutator ## Maximum time to run fuzz tests. The default is to run indefinitely. -#timeout: 30m +timeout: 5m ## By default, fuzz tests are executed in a sandbox to prevent accidental ## damage to the system. Set to false to run fuzz tests unsandboxed. diff --git a/src/main/java/com/code_intelligence/jazzer/BUILD.bazel b/src/main/java/com/code_intelligence/jazzer/BUILD.bazel index f000fa309..a5b73bf45 100644 --- a/src/main/java/com/code_intelligence/jazzer/BUILD.bazel +++ b/src/main/java/com/code_intelligence/jazzer/BUILD.bazel @@ -63,7 +63,9 @@ java_binary( "//src/main/java/com/code_intelligence/jazzer/api:api_deploy_env", "//src/main/java/com/code_intelligence/jazzer/runtime:jazzer_bootstrap_env", ], - main_class = "com.code_intelligence.jazzer.Jazzer", + deploy_manifest_lines = [ + "Main-Class: com.code_intelligence.jazzer.Jazzer", + ], runtime_deps = [":jazzer_lib"], ) diff --git a/src/main/java/com/code_intelligence/jazzer/driver/BUILD.bazel b/src/main/java/com/code_intelligence/jazzer/driver/BUILD.bazel index 6d560cc25..6b5f3870a 100644 --- a/src/main/java/com/code_intelligence/jazzer/driver/BUILD.bazel +++ b/src/main/java/com/code_intelligence/jazzer/driver/BUILD.bazel @@ -60,7 +60,9 @@ java_library( visibility = ["//src/test/java/com/code_intelligence/jazzer/driver:__pkg__"], deps = [ ":fuzz_target_holder", + ":libfuzzer_lifecycle_methods_invoker", ":opt", + ":reflection_utils", "//src/main/java/com/code_intelligence/jazzer/api", "//src/main/java/com/code_intelligence/jazzer/runtime:constants", "//src/main/java/com/code_intelligence/jazzer/utils:log", @@ -76,6 +78,7 @@ java_library( "//src/test/java/com/code_intelligence/jazzer/driver:__pkg__", ], deps = [ + ":lifecycle_methods_invoker", ":opt", "//src/main/java/com/code_intelligence/jazzer/api", "//src/main/java/com/code_intelligence/jazzer/autofuzz", @@ -102,6 +105,7 @@ java_jni_library( ":exception_utils", ":fuzz_target_holder", ":fuzzed_data_provider_impl", + ":lifecycle_methods_invoker", ":opt", ":recording_fuzzed_data_provider", ":reproducer_template", @@ -134,6 +138,23 @@ java_jni_library( ], ) +java_library( + name = "libfuzzer_lifecycle_methods_invoker", + srcs = ["LibFuzzerLifecycleMethodsInvoker.java"], + deps = [ + ":lifecycle_methods_invoker", + ":opt", + ":reflection_utils", + "//src/main/java/com/code_intelligence/jazzer/utils:log", + ], +) + +java_library( + name = "lifecycle_methods_invoker", + srcs = ["LifecycleMethodsInvoker.java"], + visibility = ["//src/main/java/com/code_intelligence/jazzer/junit:__pkg__"], +) + java_library( name = "reproducer_template", srcs = ["ReproducerTemplate.java"], @@ -182,6 +203,11 @@ java_library( deps = ["//src/main/java/com/code_intelligence/jazzer/api"], ) +java_library( + name = "reflection_utils", + srcs = ["ReflectionUtils.java"], +) + java_jni_library( name = "signal_handler", srcs = ["SignalHandler.java"], diff --git a/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetFinder.java b/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetFinder.java index a550484ff..f4db00879 100644 --- a/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetFinder.java +++ b/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetFinder.java @@ -16,6 +16,7 @@ package com.code_intelligence.jazzer.driver; +import static com.code_intelligence.jazzer.driver.ReflectionUtils.targetPublicStaticMethod; import static com.code_intelligence.jazzer.runtime.Constants.IS_ANDROID; import static java.lang.System.exit; @@ -28,14 +29,10 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; -import java.util.concurrent.Callable; import java.util.stream.Collectors; -import java.util.stream.Stream; class FuzzTargetFinder { private static final String FUZZER_TEST_ONE_INPUT = "fuzzerTestOneInput"; - private static final String FUZZER_INITIALIZE = "fuzzerInitialize"; - private static final String FUZZER_TEAR_DOWN = "fuzzerTearDown"; static String findFuzzTargetClassName() { if (!Opt.targetClass.get().isEmpty()) { @@ -102,37 +99,6 @@ private static FuzzTarget findFuzzTargetByMethodName(Class clazz) { fuzzTargetMethod = dataFuzzTarget.orElseGet(bytesFuzzTarget::get); } - Callable initialize = - Stream - .of(targetPublicStaticMethod(clazz, FUZZER_INITIALIZE, String[].class) - .map(init -> (Callable) () -> { - init.invoke(null, (Object) Opt.targetArgs.get().toArray(new String[] {})); - return null; - }), - targetPublicStaticMethod(clazz, FUZZER_INITIALIZE) - .map(init -> (Callable) () -> { - init.invoke(null); - return null; - })) - .filter(Optional::isPresent) - .map(Optional::get) - .findFirst() - .orElse(() -> null); - - return new FuzzTarget( - fuzzTargetMethod, initialize, targetPublicStaticMethod(clazz, FUZZER_TEAR_DOWN)); - } - - private static Optional targetPublicStaticMethod( - Class clazz, String name, Class... parameterTypes) { - try { - Method method = clazz.getMethod(name, parameterTypes); - if (!Modifier.isStatic(method.getModifiers()) || !Modifier.isPublic(method.getModifiers())) { - return Optional.empty(); - } - return Optional.of(method); - } catch (NoSuchMethodException e) { - return Optional.empty(); - } + return new FuzzTarget(fuzzTargetMethod, () -> null, LibFuzzerLifecycleMethodsInvoker.of(clazz)); } } diff --git a/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetHolder.java b/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetHolder.java index 67a48d343..1c833c61a 100644 --- a/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetHolder.java +++ b/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetHolder.java @@ -26,7 +26,8 @@ public static FuzzTarget autofuzzFuzzTarget(Callable newInstance) { try { Method fuzzerTestOneInput = com.code_intelligence.jazzer.autofuzz.FuzzTarget.class.getMethod( "fuzzerTestOneInput", FuzzedDataProvider.class); - return new FuzzTargetHolder.FuzzTarget(fuzzerTestOneInput, newInstance, Optional.empty()); + return new FuzzTargetHolder.FuzzTarget( + fuzzerTestOneInput, newInstance, LifecycleMethodsInvoker.NOOP); } catch (NoSuchMethodException e) { throw new IllegalStateException(e); } @@ -46,12 +47,13 @@ public static FuzzTarget autofuzzFuzzTarget(Callable newInstance) { public static class FuzzTarget { public final Method method; public final Callable newInstance; - public final Optional tearDown; + public final LifecycleMethodsInvoker lifecycleMethodsInvoker; - public FuzzTarget(Method method, Callable newInstance, Optional tearDown) { + public FuzzTarget(Method method, Callable newInstance, + LifecycleMethodsInvoker lifecycleMethodsInvoker) { this.method = method; this.newInstance = newInstance; - this.tearDown = tearDown; + this.lifecycleMethodsInvoker = lifecycleMethodsInvoker; } public boolean usesFuzzedDataProvider() { diff --git a/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetRunner.java b/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetRunner.java index 0c1ee46f6..0dc2e6101 100644 --- a/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetRunner.java +++ b/src/main/java/com/code_intelligence/jazzer/driver/FuzzTargetRunner.java @@ -38,8 +38,6 @@ import java.io.IOException; import java.lang.invoke.MethodHandle; import java.lang.invoke.MethodHandles; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; import java.lang.reflect.Modifier; import java.math.BigInteger; import java.nio.charset.StandardCharsets; @@ -118,17 +116,17 @@ public final class FuzzTargetRunner { private static final FuzzedDataProviderImpl fuzzedDataProvider = FuzzedDataProviderImpl.withNativeData(); private static final MethodHandle fuzzTargetMethod; + private static final LifecycleMethodsInvoker lifecycleMethodsInvoker; private static final boolean useFuzzedDataProvider; // Reused in every iteration analogous to JUnit's PER_CLASS lifecycle. private static final Object fuzzTargetInstance; - private static final Method fuzzerTearDown; private static final ArgumentsMutator mutator; private static final ReproducerTemplate reproducerTemplate; private static Predicate findingHandler; static { FuzzTargetHolder.FuzzTarget fuzzTarget = FuzzTargetHolder.fuzzTarget; - Class fuzzTargetClass = fuzzTarget.method.getDeclaringClass(); + lifecycleMethodsInvoker = fuzzTarget.lifecycleMethodsInvoker; // The method may not be accessible - JUnit test classes and methods are usually declared // without access modifiers and thus package-private. @@ -145,13 +143,14 @@ public final class FuzzTargetRunner { throw new IllegalStateException("Not reached"); } - fuzzerTearDown = fuzzTarget.tearDown.orElse(null); + Class fuzzTargetClass = fuzzTarget.method.getDeclaringClass(); reproducerTemplate = new ReproducerTemplate(fuzzTargetClass.getName(), useFuzzedDataProvider); JazzerInternal.onFuzzTargetReady(fuzzTargetClass.getName()); try { fuzzTargetInstance = fuzzTarget.newInstance.call(); + lifecycleMethodsInvoker.beforeFirstExecution(); } catch (Throwable t) { Log.finding(t); exit(1); @@ -176,7 +175,11 @@ public final class FuzzTargetRunner { CoverageRecorder.updateCoveredIdsWithCoverageMap(); } - Runtime.getRuntime().addShutdownHook(new Thread(FuzzTargetRunner::shutdown)); + // When running with a custom finding handler, such as from within JUnit, we can't reason about + // when the JVM shuts down and thus don't use shutdown handlers. + if (findingHandler == null) { + Runtime.getRuntime().addShutdownHook(new Thread(FuzzTargetRunner::shutdown)); + } } /** @@ -198,8 +201,8 @@ static int runOne(byte[] data) { * @param dataPtr a native pointer to beginning of the input provided by the fuzzer for this * execution * @param dataLength length of the fuzzer input - * @return the value that the native LLVMFuzzerTestOneInput function should return. Currently, - * this is always 0. The function may exit the process instead of returning. + * @return the value that the native LLVMFuzzerTestOneInput function should return. The function + * may exit the process instead of returning. */ private static int runOne(long dataPtr, int dataLength) { Throwable finding = null; @@ -233,6 +236,8 @@ private static int runOne(long dataPtr, int dataLength) { argument = data; } try { + lifecycleMethodsInvoker.beforeEachExecution(); + if (useExperimentalMutator) { // No need to detach as we are currently reading in the mutator state from bytes in every // iteration. @@ -282,6 +287,15 @@ private static int runOne(long dataPtr, int dataLength) { if (findingHandler.test(finding)) { return LIBFUZZER_CONTINUE; } else { + try { + // We have to call afterLastExecution here as we do not register the shutdown hook that + // would otherwise call it when findingHandler != null. + lifecycleMethodsInvoker.afterLastExecution(); + } catch (Throwable t) { + // We already have a finding and do not know whether the fuzz target is in an expected + // state, so report this as a warning rather than an error or finding. + Log.warn("Failed to run @AfterAll or fuzzerTearDown methods", t); + } return LIBFUZZER_RETURN_FROM_DRIVER; } } @@ -311,8 +325,7 @@ private static int runOne(long dataPtr, int dataLength) { } if (!emitDedupToken || Long.compareUnsigned(ignoredTokens.size(), keepGoing) >= 0) { - // Reached the maximum amount of findings to keep going for, crash after shutdown. We use - // _Exit rather than System.exit to not trigger libFuzzer's exit handlers. + // Reached the maximum amount of findings to keep going for, crash after shutdown. if (!Opt.autofuzz.get().isEmpty() && Opt.dedup.get()) { Log.println(""); Log.info(String.format( @@ -446,18 +459,11 @@ private static void shutdown() { } } - if (fuzzerTearDown == null) { - return; - } - Log.info("calling fuzzerTearDown function"); try { - fuzzerTearDown.invoke(null); - } catch (InvocationTargetException e) { - Log.finding(e.getCause()); - System.exit(JAZZER_FINDING_EXIT_CODE); + lifecycleMethodsInvoker.afterLastExecution(); } catch (Throwable t) { - Log.error(t); - System.exit(1); + Log.finding(t); + System.exit(JAZZER_FINDING_EXIT_CODE); } } diff --git a/src/main/java/com/code_intelligence/jazzer/driver/LibFuzzerLifecycleMethodsInvoker.java b/src/main/java/com/code_intelligence/jazzer/driver/LibFuzzerLifecycleMethodsInvoker.java new file mode 100644 index 000000000..862c1700d --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/driver/LibFuzzerLifecycleMethodsInvoker.java @@ -0,0 +1,84 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.driver; + +import static com.code_intelligence.jazzer.driver.ReflectionUtils.targetPublicStaticMethod; + +import com.code_intelligence.jazzer.utils.Log; +import java.lang.reflect.InvocationTargetException; +import java.util.Optional; +import java.util.stream.Stream; + +class LibFuzzerLifecycleMethodsInvoker implements LifecycleMethodsInvoker { + private static final String FUZZER_INITIALIZE = "fuzzerInitialize"; + private static final String FUZZER_TEAR_DOWN = "fuzzerTearDown"; + + private final Optional fuzzerInitialize; + private final Optional fuzzerTearDown; + + private LibFuzzerLifecycleMethodsInvoker( + Optional fuzzerInitialize, Optional fuzzerTearDown) { + this.fuzzerInitialize = fuzzerInitialize; + this.fuzzerTearDown = fuzzerTearDown; + } + + static LifecycleMethodsInvoker of(Class clazz) { + Optional fuzzerInitialize = + Stream + .of(targetPublicStaticMethod(clazz, FUZZER_INITIALIZE, String[].class) + .map(init + -> (ThrowingRunnable) () + -> init.invoke( + null, (Object) Opt.targetArgs.get().toArray(new String[] {}))), + targetPublicStaticMethod(clazz, FUZZER_INITIALIZE) + .map(init -> (ThrowingRunnable) () -> init.invoke(null))) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst(); + Optional fuzzerTearDown = targetPublicStaticMethod(clazz, FUZZER_TEAR_DOWN) + .map(tearDown -> () -> tearDown.invoke(null)); + + return new LibFuzzerLifecycleMethodsInvoker(fuzzerInitialize, fuzzerTearDown); + } + + @Override + public void beforeFirstExecution() throws Throwable { + if (fuzzerInitialize.isPresent()) { + try { + fuzzerInitialize.get().run(); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + } + + @Override + public void beforeEachExecution() {} + + @Override + public void afterLastExecution() throws Throwable { + if (fuzzerTearDown.isPresent()) { + // Only preserved for backwards compatibility. + Log.info("calling fuzzerTearDown function"); + try { + fuzzerTearDown.get().run(); + } catch (InvocationTargetException e) { + throw e.getCause(); + } + } + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/driver/LifecycleMethodsInvoker.java b/src/main/java/com/code_intelligence/jazzer/driver/LifecycleMethodsInvoker.java new file mode 100644 index 000000000..a791ab3ed --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/driver/LifecycleMethodsInvoker.java @@ -0,0 +1,59 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.driver; + +/** + * Can provide callbacks to be invoked by {@link FuzzTargetRunner} at certain times during the + * execution of a fuzz target. + */ +public interface LifecycleMethodsInvoker { + /** + * An implementation of {@link LifecycleMethodsInvoker} with empty implementations. + */ + LifecycleMethodsInvoker NOOP = new LifecycleMethodsInvoker() { + @Override + public void beforeFirstExecution() {} + + @Override + public void beforeEachExecution() {} + + @Override + public void afterLastExecution() {} + }; + + /** + * Invoked before the first execution of the fuzz target. + */ + void beforeFirstExecution() throws Throwable; + + /** + * Invoked before each execution of the fuzz target. + * + *

This is invoked after {@link #beforeFirstExecution()} for the first execution. + */ + void beforeEachExecution() throws Throwable; + + /** + * Invoked after the last execution of the fuzz target, regardless of whether there was a + * finding. + */ + void afterLastExecution() throws Throwable; + + interface ThrowingRunnable { + void run() throws Throwable; + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/driver/ReflectionUtils.java b/src/main/java/com/code_intelligence/jazzer/driver/ReflectionUtils.java new file mode 100644 index 000000000..776120754 --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/driver/ReflectionUtils.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.driver; + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Optional; + +class ReflectionUtils { + static Optional targetPublicStaticMethod( + Class clazz, String name, Class... parameterTypes) { + try { + Method method = clazz.getMethod(name, parameterTypes); + if (!Modifier.isStatic(method.getModifiers()) || !Modifier.isPublic(method.getModifiers())) { + return Optional.empty(); + } + return Optional.of(method); + } catch (NoSuchMethodException e) { + return Optional.empty(); + } + } +} diff --git a/src/main/java/com/code_intelligence/jazzer/instrumentor/HookMethodVisitor.kt b/src/main/java/com/code_intelligence/jazzer/instrumentor/HookMethodVisitor.kt index f5118fd6f..1312c18d6 100644 --- a/src/main/java/com/code_intelligence/jazzer/instrumentor/HookMethodVisitor.kt +++ b/src/main/java/com/code_intelligence/jazzer/instrumentor/HookMethodVisitor.kt @@ -15,6 +15,7 @@ package com.code_intelligence.jazzer.instrumentor import com.code_intelligence.jazzer.api.HookType +import com.code_intelligence.jazzer.utils.Log import org.objectweb.asm.Handle import org.objectweb.asm.Label import org.objectweb.asm.MethodVisitor @@ -407,10 +408,10 @@ private class HookMethodVisitor( private fun isReplaceHookInJava6mode(hook: Hook): Boolean { if (java6Mode && hook.hookType == HookType.REPLACE) { if (showUnsupportedHookWarning.getAndSet(false)) { - println( - """WARN: Some hooks could not be applied to class files built for Java 7 or lower. - |WARN: Ensure that the fuzz target and its dependencies are compiled with - |WARN: -target 8 or higher to identify as many bugs as possible. + Log.warn( + """Some hooks could not be applied to class files built for Java 7 or lower. + Ensure that the fuzz target and its dependencies are compiled with + -target 8 or higher to identify as many bugs as possible. """.trimMargin(), ) } diff --git a/src/main/java/com/code_intelligence/jazzer/junit/AgentConfiguringArgumentsProvider.java b/src/main/java/com/code_intelligence/jazzer/junit/AgentConfiguringArgumentsProvider.java index e65f028b4..807c98d87 100644 --- a/src/main/java/com/code_intelligence/jazzer/junit/AgentConfiguringArgumentsProvider.java +++ b/src/main/java/com/code_intelligence/jazzer/junit/AgentConfiguringArgumentsProvider.java @@ -37,7 +37,11 @@ public Stream provideArguments(ExtensionContext extensionCo // FIXME(fmeum): Calling this here feels like a hack. There should be a lifecycle hook that runs // before the argument discovery for a ParameterizedTest is kicked off, but I haven't found // one. - FuzzTestExecutor.configureAndInstallAgent(extensionContext, fuzzTest.maxDuration()); + // We need to call this method here in addition to the call in FuzzTestExtensions as our + // ArgumentProviders need the bootstrap jar on the classpath and there may be no user-provided + // ArgumentProviders to trigger the call in FuzzTestExtensions. + FuzzTestExecutor.configureAndInstallAgent( + extensionContext, fuzzTest.maxDuration(), fuzzTest.maxExecutions()); return Stream.empty(); } } diff --git a/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel index 5d5897c35..a47cc4424 100644 --- a/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel +++ b/src/main/java/com/code_intelligence/jazzer/junit/BUILD.bazel @@ -66,6 +66,7 @@ java_jni_library( ], deps = [ ":agent_configurator", + ":junit_lifecycle_methods_invoker", ":seed_serializer", ":utils", "//src/main/java/com/code_intelligence/jazzer/agent:agent_installer", @@ -83,6 +84,31 @@ java_jni_library( ], ) +java_library( + name = "junit_internals_compile_only", + neverlink = True, + # Do not add a dependency on junit-jupiter-engine to the POM file. The user + # has to add and control this dependency themselves if they want to use + # JUnit 5. Even if we added it, Maven resolution mechanics mean that we do + # not control the version anyway. + # https://github.com/junit-team/junit5/discussions/3441#discussioncomment-6884855 + tags = ["maven:compile-only"], + exports = [ + "@maven//:org_junit_jupiter_junit_jupiter_engine", + ], +) + +java_library( + name = "junit_lifecycle_methods_invoker", + srcs = ["JUnitLifecycleMethodsInvoker.java"], + deps = [ + ":junit_internals_compile_only", + "//src/main/java/com/code_intelligence/jazzer/driver:lifecycle_methods_invoker", + "//src/main/java/com/code_intelligence/jazzer/utils:unsafe_provider", + "@maven//:org_junit_jupiter_junit_jupiter_api", + ], +) + java_library( name = "seed_serializer", srcs = ["SeedSerializer.java"], diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTest.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTest.java index 041db96dc..2ebdbb9f8 100644 --- a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTest.java +++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTest.java @@ -103,9 +103,22 @@ * A duration string such as "1h 2m 30s" indicating for how long the fuzz test should be executed * during fuzzing. * + *

To remove the default limit of 5 minutes, set this element to {@code ""}. + * *

This option has no effect during regression testing. */ String maxDuration() default "5m"; + + /** + * If set to a positive number, the fuzz test function will be executed at most this many times + * during fuzzing. Otherwise (default), there is no bound on the number of executions. + * + *

Prefer this element over {@link #maxDuration()} if you want to ensure comparable levels of + * fuzzing across machine's with different performance characteristics. + * + *

This option has no effect during regression testing. + */ + long maxExecutions() default 0; } // Internal use only. diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java index 225240dec..989cbc4e8 100644 --- a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java +++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExecutor.java @@ -70,7 +70,7 @@ private FuzzTestExecutor( this.isRunFromCommandLine = isRunFromCommandLine; } - public static FuzzTestExecutor prepare(ExtensionContext context, String maxDuration) + public static FuzzTestExecutor prepare(ExtensionContext context, String maxDuration, long maxRuns) throws IOException { if (!hasBeenPrepared.compareAndSet(false, true)) { throw new IllegalStateException( @@ -113,6 +113,9 @@ public static FuzzTestExecutor prepare(ExtensionContext context, String maxDurat } libFuzzerArgs.add("-max_total_time=" + durationStringToSeconds(maxDuration)); + if (maxRuns > 0) { + libFuzzerArgs.add("-runs=" + maxRuns); + } // Disable libFuzzer's out of memory detection: It is only useful for native library fuzzing, // which we don't support without our native driver, and leads to false positives where it picks // up IntelliJ's memory usage. @@ -256,13 +259,13 @@ private static List getLibFuzzerArgs(ExtensionContext extensionContext) return args; } - static void configureAndInstallAgent(ExtensionContext extensionContext, String maxDuration) - throws IOException { + static void configureAndInstallAgent(ExtensionContext extensionContext, String maxDuration, + long maxExecutions) throws IOException { if (!agentInstalled.compareAndSet(false, true)) { return; } if (Utils.isFuzzing(extensionContext)) { - FuzzTestExecutor executor = prepare(extensionContext, maxDuration); + FuzzTestExecutor executor = prepare(extensionContext, maxDuration, maxExecutions); extensionContext.getRoot().getStore(Namespace.GLOBAL).put(FuzzTestExecutor.class, executor); AgentConfigurator.forFuzzing(extensionContext); } else { @@ -300,8 +303,8 @@ public void addSeed(byte[] bytes) throws IOException { } @SuppressWarnings("OptionalGetWithoutIsPresent") - public Optional execute( - ReflectiveInvocationContext invocationContext, SeedSerializer seedSerializer) { + public Optional execute(ReflectiveInvocationContext invocationContext, + ExtensionContext extensionContext, SeedSerializer seedSerializer) { if (seedSerializer instanceof AutofuzzSeedSerializer) { FuzzTargetHolder.fuzzTarget = FuzzTargetHolder.autofuzzFuzzTarget(() -> { // Provide an empty throws declaration to prevent autofuzz from @@ -319,7 +322,9 @@ public Optional execute( } else { FuzzTargetHolder.fuzzTarget = new FuzzTargetHolder.FuzzTarget(invocationContext.getExecutable(), - () -> invocationContext.getTarget().get(), Optional.empty()); + () + -> invocationContext.getTarget().get(), + JUnitLifecycleMethodsInvoker.of(extensionContext)); } // Only register a finding handler in case the fuzz test is executed by JUnit. diff --git a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java index 176608224..2c0497acc 100644 --- a/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java +++ b/src/main/java/com/code_intelligence/jazzer/junit/FuzzTestExtensions.java @@ -44,7 +44,11 @@ public void interceptTestTemplateMethod(Invocation invocation, throws Throwable { FuzzTest fuzzTest = AnnotationSupport.findAnnotation(invocationContext.getExecutable(), FuzzTest.class).get(); - FuzzTestExecutor.configureAndInstallAgent(extensionContext, fuzzTest.maxDuration()); + // We need to call this method here in addition to the call in AgentConfiguringArgumentsProvider + // as that provider isn't invoked before fuzz test executions for the arguments provided by + // user-provided ArgumentsProviders ("Java seeds"). + FuzzTestExecutor.configureAndInstallAgent( + extensionContext, fuzzTest.maxDuration(), fuzzTest.maxExecutions()); // Skip the invocation of the test method with the special arguments provided by // FuzzTestArgumentsProvider and start fuzzing instead. if (Utils.isMarkedInvocation(invocationContext)) { @@ -105,9 +109,9 @@ private static void startFuzzing(Invocation invocation, ReflectiveInvocationContext invocationContext, ExtensionContext extensionContext) throws Throwable { invocation.skip(); - Optional throwable = - FuzzTestExecutor.fromContext(extensionContext) - .execute(invocationContext, getOrCreateSeedSerializer(extensionContext)); + Optional throwable = FuzzTestExecutor.fromContext(extensionContext) + .execute(invocationContext, extensionContext, + getOrCreateSeedSerializer(extensionContext)); if (throwable.isPresent()) { throw throwable.get(); } diff --git a/src/main/java/com/code_intelligence/jazzer/junit/JUnitLifecycleMethodsInvoker.java b/src/main/java/com/code_intelligence/jazzer/junit/JUnitLifecycleMethodsInvoker.java new file mode 100644 index 000000000..2a5d552c9 --- /dev/null +++ b/src/main/java/com/code_intelligence/jazzer/junit/JUnitLifecycleMethodsInvoker.java @@ -0,0 +1,137 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.code_intelligence.jazzer.junit; + +import static java.util.stream.Collectors.toCollection; + +import com.code_intelligence.jazzer.driver.LifecycleMethodsInvoker; +import com.code_intelligence.jazzer.utils.UnsafeProvider; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Optional; +import java.util.stream.Stream; +import org.junit.jupiter.api.extension.AfterEachCallback; +import org.junit.jupiter.api.extension.BeforeEachCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.junit.jupiter.engine.execution.AfterEachMethodAdapter; +import org.junit.jupiter.engine.execution.BeforeEachMethodAdapter; +import org.junit.jupiter.engine.execution.DefaultExecutableInvoker; +import org.junit.jupiter.engine.extension.ExtensionRegistry; + +/** + * Adapts JUnit BeforeEach and AfterEach callbacks to + * {@link com.code_intelligence.jazzer.driver.FuzzTargetRunner} lifecycle hooks. + */ +public class JUnitLifecycleMethodsInvoker implements LifecycleMethodsInvoker { + private final ThrowingRunnable[] beforeEachExecutionRunnables; + + private long timesCalledBetweenExecutions = 0; + + private JUnitLifecycleMethodsInvoker(ThrowingRunnable[] beforeEachExecutionRunnables) { + this.beforeEachExecutionRunnables = beforeEachExecutionRunnables; + } + + static LifecycleMethodsInvoker of(ExtensionContext extensionContext) { + // ExtensionRegistry is private JUnit API that is the source of truth for all lifecycle + // callbacks, both annotation- and extension-based. + Optional maybeExtensionRegistry = + getExtensionRegistryViaHack(extensionContext); + if (!maybeExtensionRegistry.isPresent()) { + extensionContext.publishReportEntry( + "Jazzer does not support BeforeEach and AfterEach callbacks with this version of JUnit."); + return LifecycleMethodsInvoker.NOOP; + } + ExtensionRegistry extensionRegistry = maybeExtensionRegistry.get(); + + // BeforeEachCallback implementations take precedence over @BeforeEach methods. The annotations + // are turned into extensions using an internal adapter class, BeforeEachMethodAdapter. + // https://junit.org/junit5/docs/current/user-guide/#extensions-execution-order-wrapping-behavior + ArrayList beforeEachMethods = + Stream + .concat( + extensionRegistry.stream(BeforeEachCallback.class) + .map(callback -> () -> callback.beforeEach(extensionContext)), + extensionRegistry.stream(BeforeEachMethodAdapter.class) + .map(callback + -> () + -> callback.invokeBeforeEachMethod( + extensionContext, extensionRegistry))) + .collect(toCollection(ArrayList::new)); + + ArrayList afterEachMethods = + Stream + .concat( + extensionRegistry.stream(AfterEachCallback.class) + .map(callback -> () -> callback.afterEach(extensionContext)), + extensionRegistry.stream(AfterEachMethodAdapter.class) + .map(callback + -> () + -> callback.invokeAfterEachMethod(extensionContext, extensionRegistry))) + .collect(toCollection(ArrayList::new)); + // JUnit calls AfterEach methods in reverse order of registration so that the methods registered + // first run last. + Collections.reverse(afterEachMethods); + + return new JUnitLifecycleMethodsInvoker( + Stream.concat(afterEachMethods.stream(), beforeEachMethods.stream()) + .toArray(ThrowingRunnable[] ::new)); + } + + private static Optional getExtensionRegistryViaHack( + ExtensionContext extensionContext) { + // Do not fail on JUnit versions < 5.9.0 that do not have DefaultExecutableInvoker. + try { + Class.forName("org.junit.jupiter.engine.execution.DefaultExecutableInvoker"); + } catch (ClassNotFoundException e) { + return Optional.empty(); + } + // Get the private DefaultExecutableInvoker#extensionRegistry field, using the type rather than + // the name for slightly better forwards compatibility. + return Arrays.stream(DefaultExecutableInvoker.class.getDeclaredFields()) + .filter(field -> field.getType() == ExtensionRegistry.class) + .findFirst() + .flatMap(extensionRegistryField -> { + DefaultExecutableInvoker invoker = + (DefaultExecutableInvoker) extensionContext.getExecutableInvoker(); + long extensionRegistryFieldOffset = + UnsafeProvider.getUnsafe().objectFieldOffset(extensionRegistryField); + return Optional.ofNullable((ExtensionRegistry) UnsafeProvider.getUnsafe().getObject( + invoker, extensionRegistryFieldOffset)); + }); + } + + @Override + public void beforeFirstExecution() {} + + @Override + public void beforeEachExecution() throws Throwable { + if (timesCalledBetweenExecutions++ == 0) { + // BeforeEach callbacks are run by JUnit right before the fuzz test starts executing and thus + // shouldn't be run again before the first fuzz test execution. + // AfterEach callbacks should be run between two executions and thus also not before the first + // fuzz test execution. + return; + } + for (ThrowingRunnable runnable : beforeEachExecutionRunnables) { + runnable.run(); + } + } + + @Override + public void afterLastExecution() {} +} diff --git a/src/main/java/com/code_intelligence/jazzer/junit/Utils.java b/src/main/java/com/code_intelligence/jazzer/junit/Utils.java index 5f59f12dd..64fb79a0d 100644 --- a/src/main/java/com/code_intelligence/jazzer/junit/Utils.java +++ b/src/main/java/com/code_intelligence/jazzer/junit/Utils.java @@ -243,6 +243,9 @@ static boolean permissivelyParseBoolean(String value) { * allow for duration units longer than hours, so we can always prepend PT. */ static long durationStringToSeconds(String duration) { + if (duration.isEmpty()) { + return 0; + } String isoDuration = "PT" + duration.replace("sec", "s").replace("min", "m").replace("hr", "h").replace(" ", ""); return Duration.parse(isoDuration).getSeconds(); diff --git a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel index a11de3646..d724a3e8b 100644 --- a/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel +++ b/src/test/java/com/code_intelligence/jazzer/junit/BUILD.bazel @@ -245,6 +245,7 @@ java_test( "@maven//:org_junit_jupiter_junit_jupiter_engine", ], deps = [ + "//examples/junit/src/test/java/com/example:test_successful_exception", "//src/main/java/com/code_intelligence/jazzer/api:hooks", "@maven//:junit_junit", "@maven//:org_junit_platform_junit_platform_engine", diff --git a/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java b/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java index 29dfc6649..8a682ad31 100644 --- a/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java +++ b/src/test/java/com/code_intelligence/jazzer/junit/LifecycleTest.java @@ -32,6 +32,8 @@ import static org.junit.platform.testkit.engine.EventType.STARTED; import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; +import com.code_intelligence.jazzer.api.FuzzerSecurityIssueLow; +import com.example.TestSuccessfulException; import java.io.IOException; import java.nio.file.Path; import org.junit.Before; @@ -80,7 +82,7 @@ public void fuzzingEnabled() { event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ)), finishedSuccessfully()), event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), - finishedWithFailure(instanceOf(IOException.class))), + finishedWithFailure(instanceOf(TestSuccessfulException.class))), event(type(FINISHED), container(ENGINE), finishedSuccessfully())); results.testEvents().assertEventsMatchExactly( @@ -116,7 +118,7 @@ public void fuzzingDisabled() { container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ, LIFECYCLE_FUZZ))), event(type(FINISHED), container(uniqueIdSubstrings(ENGINE, CLAZZ)), - finishedWithFailure(instanceOf(IOException.class))), + finishedWithFailure(instanceOf(TestSuccessfulException.class))), event(type(FINISHED), container(ENGINE), finishedSuccessfully())); results.testEvents().assertEventsMatchExactly( diff --git a/src/test/java/com/code_intelligence/jazzer/junit/UtilsTest.java b/src/test/java/com/code_intelligence/jazzer/junit/UtilsTest.java index 0b94acc78..d45741794 100644 --- a/src/test/java/com/code_intelligence/jazzer/junit/UtilsTest.java +++ b/src/test/java/com/code_intelligence/jazzer/junit/UtilsTest.java @@ -59,6 +59,8 @@ public class UtilsTest implements InvocationInterceptor { @Test void testDurationStringToSeconds() { + assertThat(durationStringToSeconds("")).isEqualTo(0); + assertThat(durationStringToSeconds("0s")).isEqualTo(0); assertThat(durationStringToSeconds("1m")).isEqualTo(60); assertThat(durationStringToSeconds("1min")).isEqualTo(60); assertThat(durationStringToSeconds("1h")).isEqualTo(60 * 60); diff --git a/tests/benchmarks/BUILD.bazel b/tests/benchmarks/BUILD.bazel new file mode 100644 index 000000000..bcc48d8ab --- /dev/null +++ b/tests/benchmarks/BUILD.bazel @@ -0,0 +1,53 @@ +# Run all benchmarks (not run as part of `bazel test //...`) via: +# bazel test //tests/benchmark +# Run a particular benchmark and show stats via (requires jq to be installed locally): +# bazel run //tests/benchmark:.stats + +load("//bazel:fuzz_target.bzl", "all_tests_above", "fuzzer_benchmark") + +fuzzer_benchmark( + name = "UnstructuredPackedMazeFuzzerBenchmark", + srcs = [ + "src/test/java/com/example/UnstructuredPackedMazeFuzzer.java", + ], + allowed_findings = ["com.example.UnstructuredPackedMazeFuzzer$$TreasureFoundException"], + fuzzer_args = [ + "-use_value_profile=1", + ], + max_runs = 90000, + num_seeds = 15, + target_class = "com.example.UnstructuredPackedMazeFuzzer", + deps = [ + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation", + ], +) + +fuzzer_benchmark( + name = "StructuredMutatorMazeFuzzerBenchmark", + srcs = [ + "src/test/java/com/example/StructuredMutatorMazeFuzzer.java", + ], + allowed_findings = ["com.example.StructuredMutatorMazeFuzzer$$TreasureFoundException"], + fuzzer_args = [ + "--experimental_mutator", + "-use_value_profile=1", + ], + max_runs = 55000, + num_seeds = 15, + target_class = "com.example.StructuredMutatorMazeFuzzer", + deps = [ + "//src/main/java/com/code_intelligence/jazzer/mutation/annotation", + ], +) + +# Keep at the bottom for existing_rules() to capture everything else in this package. +test_suite( + name = "benchmarks", + tags = [ + # Only run tests with this tag. + "benchmark", + # Do not run this test_suite with bazel test //... + "manual", + ], + tests = all_tests_above(), +) diff --git a/tests/benchmarks/src/test/java/com/example/StructuredMutatorMazeFuzzer.java b/tests/benchmarks/src/test/java/com/example/StructuredMutatorMazeFuzzer.java new file mode 100644 index 000000000..709dcd15f --- /dev/null +++ b/tests/benchmarks/src/test/java/com/example/StructuredMutatorMazeFuzzer.java @@ -0,0 +1,140 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.example; + +import com.code_intelligence.jazzer.api.Consumer3; +import com.code_intelligence.jazzer.api.Jazzer; +import com.code_intelligence.jazzer.mutation.annotation.NotNull; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +// A variant of //examples:MazeFuzzer that uses the structured mutator. +public final class StructuredMutatorMazeFuzzer { + private static final String[] MAZE_STRING = new String[] { + " ███████████████████", + " █ █ █ █ █ █", + "█ █ █ █ ███ █ █ █ ███", + "█ █ █ █ █ █", + "█ █████ ███ ███ █ ███", + "█ █ █ █ █ █", + "█ ███ ███████ █ ███ █", + "█ █ █ █ █ █", + "███████ █ █ █████ ███", + "█ █ █ █ █", + "█ ███████ █ ███ ███ █", + "█ █ █ █ █ █ █", + "███ ███ █ ███ █ ███ █", + "█ █ █ █ █ █", + "█ ███████ █ █ █ █ █ █", + "█ █ █ █ █ █ █", + "█ █ █████████ ███ ███", + "█ █ █ █ █ █ █", + "█ █ █ ███ █████ ███ █", + "█ █ █ ", + "███████████████████ #", + }; + + private static final char[][] MAZE = parseMaze(); + private static final char[][] REACHED_FIELDS = parseMaze(); + + enum Command { LEFT, RIGHT, UP, DOWN } + + public static void fuzzerTestOneInput(@NotNull List<@NotNull Command> commands) { + executeCommands(commands, (x, y, won) -> { + if (won) { + throw new TreasureFoundException(commands); + } + // This is the key line that makes this fuzz target work: It instructs the fuzzer to track + // every new combination of x and y as a new feature. Without it, the fuzzer would be + // completely lost in the maze as guessing an escaping path by chance is close to impossible. + Jazzer.exploreState((byte) Objects.hash(x, y), 0); + if (REACHED_FIELDS[y][x] == ' ') { + // Fuzzer reached a new field in the maze, print its progress. + REACHED_FIELDS[y][x] = '.'; + // The following line is commented out to reduce test log sizes. + // System.out.println(renderMaze(REACHED_FIELDS)); + } + }); + } + + private static class TreasureFoundException extends RuntimeException { + TreasureFoundException(List commands) { + super(renderPath(commands)); + } + } + + private static void executeCommands( + List commands, Consumer3 callback) { + byte x = 0; + byte y = 0; + callback.accept(x, y, false); + + for (Command command : commands) { + byte nextX = x; + byte nextY = y; + switch (command) { + case LEFT: + nextX--; + break; + case RIGHT: + nextX++; + break; + case UP: + nextY--; + break; + case DOWN: + nextY++; + break; + default: + return; + } + char nextFieldType; + try { + nextFieldType = MAZE[nextY][nextX]; + } catch (IndexOutOfBoundsException e) { + // Fuzzer tried to walk through the exterior walls of the maze. + continue; + } + if (nextFieldType != ' ' && nextFieldType != '#') { + // Fuzzer tried to walk through the interior walls of the maze. + continue; + } + // Fuzzer performed a valid move. + x = nextX; + y = nextY; + callback.accept(x, y, nextFieldType == '#'); + } + } + + private static char[][] parseMaze() { + return Arrays.stream(MAZE_STRING).map(String::toCharArray).toArray(char[][] ::new); + } + + private static String renderMaze(char[][] maze) { + return Arrays.stream(maze).map(String::new).collect(Collectors.joining("\n", "\n", "\n")); + } + + private static String renderPath(List commands) { + char[][] mutableMaze = parseMaze(); + executeCommands(commands, (x, y, won) -> { + if (!won) { + mutableMaze[y][x] = '.'; + } + }); + return renderMaze(mutableMaze); + } +} diff --git a/tests/benchmarks/src/test/java/com/example/UnstructuredPackedMazeFuzzer.java b/tests/benchmarks/src/test/java/com/example/UnstructuredPackedMazeFuzzer.java new file mode 100644 index 000000000..fe85eaa71 --- /dev/null +++ b/tests/benchmarks/src/test/java/com/example/UnstructuredPackedMazeFuzzer.java @@ -0,0 +1,144 @@ +// Copyright 2022 Code Intelligence GmbH +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.example; + +import com.code_intelligence.jazzer.api.Consumer3; +import com.code_intelligence.jazzer.api.Jazzer; +import java.util.Arrays; +import java.util.Objects; +import java.util.stream.Collectors; + +// A variant of //examples:MazeFuzzer with a more efficient scheme of translating fuzzer input into +// commands: Every change of a bit in the input results in a different command sequence. Since +// libFuzzer is biased towards generating null bytes, the value 0 is also interpreted as going right +// in the maze, which is more useful than going left. This makes the comparison with the structured +// mutator version more fair. +public final class UnstructuredPackedMazeFuzzer { + private static final String[] MAZE_STRING = new String[] { + " ███████████████████", + " █ █ █ █ █ █", + "█ █ █ █ ███ █ █ █ ███", + "█ █ █ █ █ █", + "█ █████ ███ ███ █ ███", + "█ █ █ █ █ █", + "█ ███ ███████ █ ███ █", + "█ █ █ █ █ █", + "███████ █ █ █████ ███", + "█ █ █ █ █", + "█ ███████ █ ███ ███ █", + "█ █ █ █ █ █ █", + "███ ███ █ ███ █ ███ █", + "█ █ █ █ █ █", + "█ ███████ █ █ █ █ █ █", + "█ █ █ █ █ █ █", + "█ █ █████████ ███ ███", + "█ █ █ █ █ █ █", + "█ █ █ ███ █████ ███ █", + "█ █ █ ", + "███████████████████ #", + }; + + private static final char[][] MAZE = parseMaze(); + private static final char[][] REACHED_FIELDS = parseMaze(); + + public static void fuzzerTestOneInput(byte[] commands) { + executeCommands(commands, (x, y, won) -> { + if (won) { + throw new TreasureFoundException(commands); + } + // This is the key line that makes this fuzz target work: It instructs the fuzzer to track + // every new combination of x and y as a new feature. Without it, the fuzzer would be + // completely lost in the maze as guessing an escaping path by chance is close to impossible. + Jazzer.exploreState((byte) Objects.hash(x, y), 0); + if (REACHED_FIELDS[y][x] == ' ') { + // Fuzzer reached a new field in the maze, print its progress. + REACHED_FIELDS[y][x] = '.'; + // The following line is commented out to reduce test log sizes. + // System.out.println(renderMaze(REACHED_FIELDS)); + } + }); + } + + private static class TreasureFoundException extends RuntimeException { + TreasureFoundException(byte[] commands) { + super(renderPath(commands)); + } + } + + private static void executeCommands(byte[] commands, Consumer3 callback) { + byte x = 0; + byte y = 0; + callback.accept(x, y, false); + + for (byte b : commands) { + for (int i = 0; i < 4; i++) { + byte command = (byte) (b & 0b11); + b >>>= 2; + + byte nextX = x; + byte nextY = y; + switch (command) { + case 0: + nextX++; + break; + case 1: + nextX--; + break; + case 2: + nextY++; + break; + case 3: + nextY--; + break; + default: + return; + } + char nextFieldType; + try { + nextFieldType = MAZE[nextY][nextX]; + } catch (IndexOutOfBoundsException e) { + // Fuzzer tried to walk through the exterior walls of the maze. + continue; + } + if (nextFieldType != ' ' && nextFieldType != '#') { + // Fuzzer tried to walk through the interior walls of the maze. + continue; + } + // Fuzzer performed a valid move. + x = nextX; + y = nextY; + callback.accept(x, y, nextFieldType == '#'); + } + } + } + + private static char[][] parseMaze() { + return Arrays.stream(MAZE_STRING).map(String::toCharArray).toArray(char[][] ::new); + } + + private static String renderMaze(char[][] maze) { + return Arrays.stream(maze).map(String::new).collect(Collectors.joining("\n", "\n", "\n")); + } + + private static String renderPath(byte[] commands) { + char[][] mutableMaze = parseMaze(); + executeCommands(commands, (x, y, won) -> { + if (!won) { + mutableMaze[y][x] = '.'; + } + }); + return renderMaze(mutableMaze); + } +} diff --git a/third_party/protobuf-disable-layering_check.patch b/third_party/protobuf-disable-layering_check.patch index 69d3449a5..76c7ca8e9 100644 --- a/third_party/protobuf-disable-layering_check.patch +++ b/third_party/protobuf-disable-layering_check.patch @@ -40,138 +40,18 @@ index 77ed2309f..8c38fb872 100644 proto_library( diff --git src/google/protobuf/compiler/BUILD.bazel src/google/protobuf/compiler/BUILD.bazel -index a2171c806..8dcd34667 100644 +index 9b4c243d1..e258c7298 100644 --- src/google/protobuf/compiler/BUILD.bazel +++ src/google/protobuf/compiler/BUILD.bazel -@@ -13,6 +13,8 @@ load("@rules_proto//proto:defs.bzl", "proto_library") - load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test") - load("//build_defs:cpp_opts.bzl", "COPTS", "LINK_OPTS") - +@@ -14,6 +14,8 @@ load("//build_defs:arch_tests.bzl", "aarch64_test", "x86_64_test") + load("//build_defs:cpp_opts.bzl", "COPTS") + load("test_plugin_injection.bzl", "inject_plugin_paths") + +package(features = ["-layering_check"]) + proto_library( name = "plugin_proto", srcs = ["plugin.proto"], -diff --git src/google/protobuf/compiler/allowlists/BUILD.bazel src/google/protobuf/compiler/allowlists/BUILD.bazel -index 569a142fc..0a90b312f 100644 ---- src/google/protobuf/compiler/allowlists/BUILD.bazel -+++ src/google/protobuf/compiler/allowlists/BUILD.bazel -@@ -1,7 +1,10 @@ - load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") - load("//build_defs:cpp_opts.bzl", "COPTS") - --package(default_visibility = ["//visibility:private"]) -+package( -+ default_visibility = ["//visibility:private"], -+ features = ["-layering_check"], -+) - - cc_library( - name = "allowlist", -diff --git src/google/protobuf/compiler/cpp/BUILD.bazel src/google/protobuf/compiler/cpp/BUILD.bazel -index ac1184d32..deacbf582 100644 ---- src/google/protobuf/compiler/cpp/BUILD.bazel -+++ src/google/protobuf/compiler/cpp/BUILD.bazel -@@ -7,6 +7,8 @@ load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") - load("@rules_proto//proto:defs.bzl", "proto_library") - load("//build_defs:cpp_opts.bzl", "COPTS") - -+package(features = ["-layering_check"]) -+ - cc_library( - name = "names", - hdrs = ["names.h"], -diff --git src/google/protobuf/compiler/csharp/BUILD.bazel src/google/protobuf/compiler/csharp/BUILD.bazel -index 96b8dcbc0..a2d549f26 100644 ---- src/google/protobuf/compiler/csharp/BUILD.bazel -+++ src/google/protobuf/compiler/csharp/BUILD.bazel -@@ -6,6 +6,8 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") - load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") - load("//build_defs:cpp_opts.bzl", "COPTS") - -+package(features = ["-layering_check"]) -+ - cc_library( - name = "names", - hdrs = ["names.h"], -diff --git src/google/protobuf/compiler/java/BUILD.bazel src/google/protobuf/compiler/java/BUILD.bazel -index 94573892c..c94f472d6 100644 ---- src/google/protobuf/compiler/java/BUILD.bazel -+++ src/google/protobuf/compiler/java/BUILD.bazel -@@ -6,6 +6,8 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") - load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") - load("//build_defs:cpp_opts.bzl", "COPTS") - -+package(features = ["-layering_check"]) -+ - cc_library( - name = "names", - hdrs = ["names.h"], -diff --git src/google/protobuf/compiler/objectivec/BUILD.bazel src/google/protobuf/compiler/objectivec/BUILD.bazel -index f78990394..6c534219a 100644 ---- src/google/protobuf/compiler/objectivec/BUILD.bazel -+++ src/google/protobuf/compiler/objectivec/BUILD.bazel -@@ -6,6 +6,8 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") - load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") - load("//build_defs:cpp_opts.bzl", "COPTS") - -+package(features = ["-layering_check"]) -+ - cc_library( - name = "names", - hdrs = ["names.h"], -diff --git src/google/protobuf/compiler/php/BUILD.bazel src/google/protobuf/compiler/php/BUILD.bazel -index fe9e75c2c..a569a1c9d 100644 ---- src/google/protobuf/compiler/php/BUILD.bazel -+++ src/google/protobuf/compiler/php/BUILD.bazel -@@ -6,6 +6,8 @@ load("@rules_cc//cc:defs.bzl", "cc_library") - load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") - load("//build_defs:cpp_opts.bzl", "COPTS") - -+package(features = ["-layering_check"]) -+ - cc_library( - name = "names", - hdrs = ["names.h"], -diff --git src/google/protobuf/compiler/python/BUILD.bazel src/google/protobuf/compiler/python/BUILD.bazel -index 5d26e0ce9..ce017acf1 100644 ---- src/google/protobuf/compiler/python/BUILD.bazel -+++ src/google/protobuf/compiler/python/BUILD.bazel -@@ -6,6 +6,8 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") - load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") - load("//build_defs:cpp_opts.bzl", "COPTS") - -+package(features = ["-layering_check"]) -+ - cc_library( - name = "python", - srcs = [ -diff --git src/google/protobuf/compiler/ruby/BUILD.bazel src/google/protobuf/compiler/ruby/BUILD.bazel -index 520b69194..1e437e7bc 100644 ---- src/google/protobuf/compiler/ruby/BUILD.bazel -+++ src/google/protobuf/compiler/ruby/BUILD.bazel -@@ -6,6 +6,8 @@ load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") - load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix") - load("//build_defs:cpp_opts.bzl", "COPTS") - -+package(features = ["-layering_check"]) -+ - cc_library( - name = "ruby", - srcs = ["ruby_generator.cc"], -diff --git src/google/protobuf/compiler/rust/BUILD.bazel src/google/protobuf/compiler/rust/BUILD.bazel -index 7c1f5b856..4a10038d1 100644 ---- src/google/protobuf/compiler/rust/BUILD.bazel -+++ src/google/protobuf/compiler/rust/BUILD.bazel -@@ -5,6 +5,8 @@ - load("@rules_cc//cc:defs.bzl", "cc_library") - load("//build_defs:cpp_opts.bzl", "COPTS") - -+package(features = ["-layering_check"]) -+ - cc_library( - name = "rust", - srcs = ["generator.cc"], diff --git src/google/protobuf/io/BUILD.bazel src/google/protobuf/io/BUILD.bazel index 8f39625c2..fc2f8e002 100644 --- src/google/protobuf/io/BUILD.bazel diff --git a/third_party/rules_jvm_external-add-neverlink-deps-to-javadoc-classpath.patch b/third_party/rules_jvm_external-add-neverlink-deps-to-javadoc-classpath.patch new file mode 100644 index 000000000..506bb189a --- /dev/null +++ b/third_party/rules_jvm_external-add-neverlink-deps-to-javadoc-classpath.patch @@ -0,0 +1,33 @@ +From 920048a2b213e2c7cb6ce679ffa5a414054339f6 Mon Sep 17 00:00:00 2001 +From: Fabian Meumertzheim +Date: Fri, 1 Sep 2023 22:12:42 +0200 +Subject: [PATCH] Add compile-only deps to javadocs classpath + +javadoc may have to inspect compile-only dependencies. + +Also removes a line that only added elements to a depset that are +already contained in this depset. +--- + private/rules/javadoc.bzl | 9 ++++++--- + 1 file changed, 6 insertions(+), 3 deletions(-) + +diff --git a/private/rules/javadoc.bzl b/private/rules/javadoc.bzl +index 325aced1..3261248a 100644 +--- a/private/rules/javadoc.bzl ++++ b/private/rules/javadoc.bzl +@@ -21,9 +21,12 @@ def _javadoc_impl(ctx): + + jar_file = ctx.actions.declare_file("%s.jar" % ctx.attr.name) + +- # Gather additional files to add to the classpath +- additional_deps = depset(transitive = [dep[JavaInfo].transitive_runtime_jars for dep in ctx.attr.deps]) +- classpath = depset(transitive = [dep[JavaInfo].transitive_runtime_jars for dep in ctx.attr.deps] + [additional_deps]) ++ # javadoc may need to inspect compile-time dependencies (neverlink) ++ # of the runtime classpath. ++ classpath = depset( ++ transitive = [dep[JavaInfo].transitive_runtime_jars for dep in ctx.attr.deps] + ++ [dep[JavaInfo].transitive_compile_time_jars for dep in ctx.attr.deps], ++ ) + + # javadoc options and javac options overlap, but we cannot + # necessarily rely on those to derive the javadoc options we need diff --git a/third_party/rules_kotlin-remove-java-info-transitive-deps.patch b/third_party/rules_kotlin-remove-java-info-transitive-deps.patch new file mode 100644 index 000000000..1f79b78d1 --- /dev/null +++ b/third_party/rules_kotlin-remove-java-info-transitive-deps.patch @@ -0,0 +1,27 @@ +From 5633d284a6c77882487ef58885dbcbfd24c07f9c Mon Sep 17 00:00:00 2001 +From: hvadehra +Date: Fri, 11 Aug 2023 09:10:16 +0200 +Subject: [PATCH] Migrate usages deprecated `JavaInfo` fields + +transitive_deps was an alias for transitive_compile_time_jars transitive_runtime_deps was an alias for transitive_runtime_jars + +The fields were deprecated in 2021, and are dropped in Bazel@HEAD + +Fixes bazelbuild#1003 +--- + kotlin/internal/jvm/compile.bzl | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/kotlin/internal/jvm/compile.bzl b/kotlin/internal/jvm/compile.bzl +index 80cbf7a8..327a0bdc 100644 +--- a/kotlin/internal/jvm/compile.bzl ++++ b/kotlin/internal/jvm/compile.bzl +@@ -261,7 +261,7 @@ def _run_merge_jdeps_action(ctx, toolchains, jdeps, outputs, deps): + ) + + # For sandboxing to work, and for this action to be deterministic, the compile jars need to be passed as inputs +- inputs = depset(jdeps, transitive = [depset([], transitive = [dep.transitive_deps for dep in deps])]) ++ inputs = depset(jdeps, transitive = [depset([], transitive = [dep.transitive_compile_time_jars for dep in deps])]) + + ctx.actions.run( + mnemonic = mnemonic,