Merge branch 'main' into FUZZ-767-fuzz-fuzzeddataprovider

.bazelversion

@@ -1 +1 @@
-132d2497f4337473fb084677c7f2b98c85e67c92
+b29649fbdc983cd62a58b9b09ef699867e7c5b69

README.md

@@ -56,7 +56,7 @@ After a few seconds, Jazzer should trigger an `AssertionError`, reproducing a bu
 
 ### JUnit 5
 
-The following steps assume that JUnit 5 is set up for your project, for example based on the official [junit5-samples](https://github.com/junit-team/junit5-samples).
+The following steps assume that JUnit 5.9.0 or higher is set up for your project, for example based on the official [junit5-samples](https://github.com/junit-team/junit5-samples).
 
 1. Add a dependency on `com.code-intelligence:jazzer-junit:<latest version>`.
    All Jazzer Maven artifacts are signed with [this key](deploy/maven.pub).

WORKSPACE.bazel

@@ -35,10 +35,10 @@ rules_java_toolchains()
 http_archive(
     name = "com_google_protobuf",
     patches = ["//third_party:protobuf-disable-layering_check.patch"],
-    sha256 = "ddf8c9c1ffccb7e80afd183b3bd32b3b62f7cc54b106be190bf49f2bc09daab5",
-    strip_prefix = "protobuf-23.2",
+    sha256 = "0930b1a6eb840a2295dfcb13bb5736d1292c3e0d61a90391181399327be7d8f1",
+    strip_prefix = "protobuf-24.1",
     # Keep in sync with com_google_protobuf_protobuf_java in repositories.bzl.
-    urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v23.2/protobuf-23.2.tar.gz"],
+    urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v24.1/protobuf-24.1.tar.gz"],
 )
 
 http_archive(

bazel/fuzz_target.bzl

@@ -128,3 +128,81 @@ def java_fuzz_target_test(
         use_testrunner = False,
         tags = tags,
     )
+
+_BASE_SEED = 2735196724
+
+def fuzzer_benchmark(
+        name,
+        *,
+        num_seeds,
+        max_runs,
+        env = {},
+        fuzzer_args = [],
+        tags = [],
+        **kwargs):
+    """Creates multiple instances of a Java fuzz target test with different seeds for benchmarking.
+
+    The target `<name>` is a `test_suite` tagged with `"manual"`that can be used to run all
+    individual instances of the fuzz target test at once. The individual tests are tagged with
+    `"benchmark"` and `"manual"`. This is meant to run in CI and ensure that the maximum number of
+    runs does not regress.
+
+    The target `<name>.stats` can be run with `bazel run` to execute the benchmark and derive some
+    statistics about the number of runs.
+
+    This macro is set up specifically to make efficient use of Bazel's scheduling and caching
+    capabilities: By having one target per run instead of a single target that runs the fuzz test
+    multiple times, Bazel can schedule the runs concurrently and avoid timeouts on slow runners.
+    When increasing the number of seeds, existing results can be reused from the cache.
+
+    Args:
+      num_seeds: The number of different seeds to try; corresponds to the number of individual tests
+        generated.
+      max_runs: The maximum number of runs that each individual test is allowed to run for. Keep
+        this as low as possible with a small margin to catch regressions.
+    """
+    seed = _BASE_SEED
+    tests = []
+    for i in range(num_seeds):
+        test_name = "{}_{}".format(name, i + 1)
+        tests.append(test_name)
+        java_fuzz_target_test(
+            name = test_name,
+            fuzzer_args = fuzzer_args + [
+                "-print_final_stats=1",
+                "-seed={}".format(seed),
+                "-runs={}".format(max_runs),
+            ],
+            env = env | {"JAZZER_NO_EXPLICIT_SEED": "1"},
+            tags = tags + ["manual", "benchmark"],
+            verify_crash_input = False,
+            verify_crash_reproducer = False,
+            **kwargs
+        )
+        seed = (31 * seed) % 4294967295
+
+    native.test_suite(
+        name = name,
+        tests = tests,
+        tags = ["manual"],
+    )
+
+    native.sh_binary(
+        name = name + ".stats",
+        srcs = [Label("//bazel/tools:compute_benchmark_stats.sh")],
+        env = {
+            "TEST_SUITE_LABEL": str(native.package_relative_label(name)),
+        },
+        args = [
+            native.package_name() + "/" + test
+            for test in tests
+        ],
+    )
+
+def all_tests_above():
+    """Returns the labels of all test targets in the current package defined before this call."""
+    return [
+        ":" + r["name"]
+        for r in native.existing_rules().values()
+        if r["kind"].endswith("_test")
+    ]

bazel/tools/BUILD.bazel

@@ -0,0 +1 @@
+exports_files(["compute_benchmark_stats.sh"])

bazel/tools/compute_benchmark_stats.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+# Copyright 2023 Code Intelligence GmbH
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# Run the benchmark given by $TEST_SUITE_LABEL, then get all "stat::number_of_executed_units: 12345"
+# lines printed by libFuzzer from the logs for the tests passed in on the command line in the form
+# "path/to/pkg/name" and compute statistics.
+#
+# Requires jq to be installed locally.
+
+cd "$BUILD_WORKSPACE_DIRECTORY" || exit 1
+# Remove the -runs limit to collect statistics even if the current run limit is too low.
+bazel test "$TEST_SUITE_LABEL" --test_arg=-runs=999999999
+echo "$@" \
+    | xargs -L1 printf "bazel-testlogs/%s/test.log " \
+    | xargs -L1 cat \
+    | grep '^stat::number_of_executed_units' \
+    | cut -d' ' -f2 \
+    | jq -s '{values:sort,minimum:min,maximum:max,average:(add/length),median:(sort|if length%2==1 then.[length/2|floor]else[.[length/2-1,length/2]]|add/2 end)}'

deploy/BUILD.bazel

@@ -20,6 +20,10 @@ sh_binary(
     name = "deploy",
     srcs = ["deploy.sh"],
     args = [JAZZER_COORDINATES],
+    data = ["@bazel_tools//tools/jdk:current_host_java_runtime"],
+    env = {"JAVA_EXECPATH": "$(JAVA)"},
+    toolchains = ["@bazel_tools//tools/jdk:current_host_java_runtime"],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
 )
 
 java_export(
@@ -90,6 +94,21 @@ java_export(
     ],
 )
 
+sh_test(
+    name = "jazzer_version_test",
+    srcs = ["jazzer_version_test.sh"],
+    data = [
+        ":jazzer",
+        "@bazel_tools//tools/jdk:current_java_runtime",
+    ],
+    env = {
+        "JAVA_EXECPATH": "$(JAVA)",
+        "JAZZER_RLOCATIONPATH": "$(rlocationpath :jazzer)",
+    },
+    toolchains = ["@bazel_tools//tools/jdk:current_java_runtime"],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)
+
 [
     sh_test(
         name = artifact + "_artifact_test",

deploy/deploy.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 # Copyright 2022 Code Intelligence GmbH
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,6 +33,24 @@ JAZZER_COORDINATES=$1
 [ ! -f "${JAZZER_JAR_PATH}" ] && \
   fail "JAZZER_JAR_PATH does not exist at '$JAZZER_JAR_PATH'"
 
+# --- begin runfiles.bash initialization v3 ---
+# Copy-pasted from the Bazel Bash runfiles library v3.
+set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v3 ---
+
+# JAZZER_EXECPATH is a path of the form "external/remotejdk_17/bin/java". We need to strip of the
+# leading external to get a path we can pass to rlocation.
+java_rlocationpath=$(echo "$JAVA_EXECPATH" | cut -d/ -f2-)
+java=$(rlocation "$java_rlocationpath")
+"$java" -jar "${JAZZER_JAR_PATH}" --version 2>&1 | grep '^Jazzer v' || \
+  fail "JAZZER_JAR_PATH is not a valid jazzer.jar"
+
 MAVEN_REPO=https://oss.sonatype.org/service/local/staging/deploy/maven2
 
 # The Jazzer jar itself bundles native libraries for multiple architectures and thus can't be built

deploy/jazzer_version_test.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+# Copyright 2023 Code Intelligence GmbH
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# --- begin runfiles.bash initialization v3 ---
+# Copy-pasted from the Bazel Bash runfiles library v3.
+set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash
+source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$0.runfiles/$f" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \
+  { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e
+# --- end runfiles.bash initialization v3 ---
+
+# JAZZER_EXECPATH is a path of the form "external/remotejdk_17/bin/java". We need to strip of the
+# leading external to get a path we can pass to rlocation.
+java_rlocationpath=$(echo "$JAVA_EXECPATH" | cut -d/ -f2-)
+java=$(rlocation "$java_rlocationpath")
+jazzer=$(rlocation "$JAZZER_RLOCATIONPATH")
+[ -f "$jazzer" ] || exit 1
+jazzer_version_output=$("$java" -jar "$jazzer" --version 2>&1)
+echo "$jazzer_version_output"
+echo "$jazzer_version_output" | tr -d '\n' | grep -q '^Jazzer v[0-9.]*$' || exit 1

examples/junit/src/test/java/com/example/BUILD.bazel

@@ -1,5 +1,11 @@
 load("//bazel:fuzz_target.bzl", "java_fuzz_target_test")
 
+java_library(
+    name = "test_successful_exception",
+    srcs = ["TestSuccessfulException.java"],
+    visibility = ["//src/test/java/com/code_intelligence/jazzer/junit:__subpackages__"],
+)
+
 java_binary(
     name = "ExampleFuzzTests",
     testonly = True,
@@ -9,11 +15,13 @@ java_binary(
         "//src/test/java/com/code_intelligence/jazzer/junit:__pkg__",
     ],
     deps = [
+        ":test_successful_exception",
         "//deploy:jazzer",
         "//deploy:jazzer-api",
         "//deploy:jazzer-junit",
         "//examples/junit/src/main/java/com/example:parser",
         "//examples/junit/src/test/resources:example_seed_corpora",
+        "@maven//:com_google_truth_truth",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
         "@maven//:org_junit_jupiter_junit_jupiter_params",
         "@maven//:org_mockito_mockito_core",
@@ -63,18 +71,20 @@ java_fuzz_target_test(
 java_fuzz_target_test(
     name = "LifecycleFuzzTest",
     srcs = ["LifecycleFuzzTest.java"],
-    allowed_findings = ["java.io.IOException"],
+    allowed_findings = ["com.example.TestSuccessfulException"],
     fuzzer_args = [
-        "-runs=0",
+        "-runs=3",
     ],
     target_class = "com.example.LifecycleFuzzTest",
     verify_crash_reproducer = False,
     runtime_deps = [
         ":junit_runtime",
     ],
     deps = [
+        ":test_successful_exception",
         "//examples/junit/src/main/java/com/example:parser",
         "//src/main/java/com/code_intelligence/jazzer/junit:fuzz_test",
+        "@maven//:com_google_truth_truth",
         "@maven//:org_junit_jupiter_junit_jupiter_api",
     ],
 )