diff --git a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java index fa43246933dc37..b42afd9c102529 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java +++ b/src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java @@ -274,7 +274,7 @@ public final class RemoteOptions extends CommonRemoteOptions { @Option( name = "remote_build_event_upload", oldName = "experimental_remote_build_event_upload", - defaultValue = "all", + defaultValue = "minimal", documentationCategory = OptionDocumentationCategory.REMOTE, effectTags = {OptionEffectTag.UNKNOWN}, converter = RemoteBuildEventUploadModeConverter.class, @@ -284,7 +284,7 @@ public final class RemoteOptions extends CommonRemoteOptions { + " remote cache, except for files that are important to the consumers of BEP (e.g." + " test logs and timing profile). bytestream:// scheme is always used for the uri of" + " files even if they are missing from remote cache.\n" - + "Default to 'all'.") + + "Default to 'minimal'.") public RemoteBuildEventUploadMode remoteBuildEventUploadMode; /** Build event upload mode flag parser */ diff --git a/src/test/shell/bazel/remote/remote_build_event_uploader_test.sh b/src/test/shell/bazel/remote/remote_build_event_uploader_test.sh index 895b4bb14ef088..8a1c72ab00cff5 100755 --- a/src/test/shell/bazel/remote/remote_build_event_uploader_test.sh +++ b/src/test/shell/bazel/remote/remote_build_event_uploader_test.sh @@ -117,6 +117,26 @@ EOF expect_bes_file_uploaded command.profile.gz } +function test_upload_all_convert_paths_for_existed_blobs() { + mkdir -p a + cat > a/BUILD < \$@", +) +EOF + + bazel build \ + --remote_executor=grpc://localhost:${worker_port} \ + --remote_build_event_upload=all \ + --build_event_json_file=$BEP_JSON \ + //a:foo >& $TEST_log || fail "Failed to build" + + expect_bes_file_uploaded foo.txt + expect_bes_file_uploaded command.profile.gz +} + function test_upload_minimal_doesnt_upload_missing_blobs() { mkdir -p a cat > a/BUILD < a/BUILD < \$@", + tags = ["no-remote"], +) +EOF + + bazel build \ + --remote_executor=grpc://localhost:${worker_port} \ + --remote_build_event_upload=all \ + --build_event_json_file=$BEP_JSON \ + //a:foo >& $TEST_log || fail "Failed to build" + + expect_bes_file_uploaded foo.txt + expect_bes_file_uploaded command.profile.gz +} + function test_upload_minimal_respect_no_upload_results() { mkdir -p a cat > a/BUILD < a/BUILD < \$@", +) +EOF + + bazel build \ + --remote_cache=grpc://localhost:${worker_port} \ + --remote_upload_local_results=false \ + --remote_build_event_upload=all \ + --build_event_json_file=$BEP_JSON \ + //a:foo >& $TEST_log || fail "Failed to build" + + expect_bes_file_uploaded foo.txt + expect_bes_file_uploaded command.profile.gz +} + function test_upload_minimal_respect_no_upload_results_combined_cache() { local cache_dir="${TEST_TMPDIR}/disk_cache" mkdir -p a @@ -189,6 +251,35 @@ EOF [[ "$disk_cas_files" == 3 ]] || fail "Expected 3 disk cas entries, not $disk_cas_files" } +function test_upload_all_combined_cache() { + local cache_dir="${TEST_TMPDIR}/disk_cache" + mkdir -p a + cat > a/BUILD < \$@", +) +EOF + + rm -rf $cache_dir + bazel build \ + --remote_cache=grpc://localhost:${worker_port} \ + --disk_cache=$cache_dir \ + --incompatible_remote_results_ignore_disk \ + --remote_upload_local_results=false \ + --remote_build_event_upload=all \ + --build_event_json_file=$BEP_JSON \ + //a:foo >& $TEST_log || fail "Failed to build" + + expect_bes_file_not_uploaded foo.txt + expect_bes_file_uploaded command.profile.gz + remote_cas_files="$(count_remote_cas_files)" + [[ "$remote_cas_files" == 1 ]] || fail "Expected 1 remote cas entries, not $remote_cas_files" + disk_cas_files="$(count_disk_cas_files $cache_dir)" + # foo.txt, stdout and stderr for action 'foo' + [[ "$disk_cas_files" == 3 ]] || fail "Expected 3 disk cas entries, not $disk_cas_files" +} function test_upload_minimal_alias_action_doesnt_upload_missing_blobs() { mkdir -p a @@ -216,6 +307,32 @@ EOF expect_bes_file_uploaded command.profile.gz } +function test_upload_all_alias_action() { + mkdir -p a + cat > a/BUILD < \$@", + tags = ["no-remote"], +) + +alias( + name = 'foo-alias', + actual = '//a:foo', +) +EOF + + bazel build \ + --remote_executor=grpc://localhost:${worker_port} \ + --remote_build_event_upload=all \ + --build_event_json_file=$BEP_JSON \ + //a:foo-alias >& $TEST_log || fail "Failed to build" + + expect_bes_file_uploaded foo.txt + expect_bes_file_uploaded command.profile.gz +} + function test_upload_minimal_trees_doesnt_upload_missing_blobs() { mkdir -p a cat > a/output_dir.bzl <<'EOF' @@ -264,6 +381,54 @@ EOF expect_bes_file_uploaded command.profile.gz } +function test_upload_all_trees() { + mkdir -p a + cat > a/output_dir.bzl <<'EOF' +def _gen_output_dir_impl(ctx): + output_dir = ctx.actions.declare_directory(ctx.attr.outdir) + ctx.actions.run_shell( + outputs = [output_dir], + inputs = [], + command = """ + echo 0 > $1/0.txt + echo 1 > $1/1.txt + mkdir -p $1/sub + echo "Shuffle, duffle, muzzle, muff" > $1/sub/bar + """, + arguments = [output_dir.path], + execution_requirements = {"no-remote": ""}, + ) + return [ + DefaultInfo(files = depset(direct = [output_dir])), + ] +gen_output_dir = rule( + implementation = _gen_output_dir_impl, + attrs = { + "outdir": attr.string(mandatory = True), + }, +) +EOF + + cat > a/BUILD <& $TEST_log || fail "Failed to build" + + expect_bes_file_uploaded dir/0.txt + expect_bes_file_uploaded dir/1.txt + expect_bes_file_uploaded dir/sub/bar + expect_bes_file_uploaded command.profile.gz +} + function test_upload_minimal_upload_testlogs() { mkdir -p a cat > a/BUILD < a/BUILD < a/test.sh <& $TEST_log || fail "Failed to build" + + expect_bes_file_uploaded test.sh + expect_bes_file_uploaded test.log + expect_bes_file_uploaded test.xml + expect_bes_file_uploaded command.profile.gz +} + function test_upload_minimal_upload_buildlogs() { mkdir -p a cat > a/BUILD < a/BUILD <& $TEST_log || true + + expect_bes_file_uploaded stdout + expect_bes_file_uploaded stderr + expect_bes_file_uploaded command.profile.gz +} + function test_upload_minimal_upload_profile() { mkdir -p a cat > a/BUILD < a/BUILD < \$@", +) +EOF + + bazel build \ + --remote_executor=grpc://localhost:${worker_port} \ + --remote_build_event_upload=all \ + --profile=mycommand.profile.gz \ + --build_event_json_file=$BEP_JSON \ + //a:foo >& $TEST_log || fail "Failed to build" + + expect_bes_file_uploaded "mycommand.profile.gz" +} + run_suite "Remote build event uploader tests" diff --git a/src/test/shell/bazel/remote/remote_execution_test.sh b/src/test/shell/bazel/remote/remote_execution_test.sh index c17e594d853e7f..ef98dfaf9059f4 100755 --- a/src/test/shell/bazel/remote/remote_execution_test.sh +++ b/src/test/shell/bazel/remote/remote_execution_test.sh @@ -2509,6 +2509,7 @@ EOF bazel build \ --remote_cache=grpc://localhost:${worker_port} \ --incompatible_remote_build_event_upload_respect_no_cache \ + --remote_build_event_upload=all \ --build_event_json_file=bep.json \ //a:foo >& $TEST_log || fail "Failed to build" @@ -2532,6 +2533,7 @@ EOF --remote_cache=grpc://localhost:${worker_port} \ --remote_download_minimal \ --incompatible_remote_build_event_upload_respect_no_cache \ + --remote_build_event_upload=all \ --build_event_json_file=bep.json \ //a:foo >& $TEST_log || fail "Failed to build" @@ -2559,6 +2561,7 @@ EOF bazel build \ --remote_cache=grpc://localhost:${worker_port} \ --incompatible_remote_build_event_upload_respect_no_cache \ + --remote_build_event_upload=all \ --build_event_json_file=bep.json \ //a:foo-alias >& $TEST_log || fail "Failed to build" @@ -2605,6 +2608,7 @@ EOF bazel build \ --remote_cache=grpc://localhost:${worker_port} \ --incompatible_remote_build_event_upload_respect_no_cache \ + --remote_build_event_upload=all \ --build_event_json_file=bep.json \ //a:foo >& $TEST_log || fail "Failed to build" @@ -2628,6 +2632,7 @@ EOF --remote_cache=grpc://localhost:${worker_port} \ --remote_upload_local_results=false \ --incompatible_remote_build_event_upload_respect_no_cache \ + --remote_build_event_upload=all \ --build_event_json_file=bep.json \ //a:foo >& $TEST_log || fail "Failed to build" @@ -2653,6 +2658,7 @@ EOF --disk_cache=$cache_dir \ --remote_upload_local_results=false \ --incompatible_remote_build_event_upload_respect_no_cache \ + --remote_build_event_upload=all \ --build_event_json_file=bep.json \ //a:foo >& $TEST_log || fail "Failed to build" @@ -2684,6 +2690,7 @@ EOF --build_event_json_file=bep.json \ --noremote_upload_local_results \ --incompatible_remote_build_event_upload_respect_no_cache \ + --remote_build_event_upload=all \ //a:test >& $TEST_log || fail "Failed to test //a:test" cat bep.json > $TEST_log @@ -2709,6 +2716,7 @@ EOF --remote_cache=grpc://localhost:${worker_port} \ --disk_cache=$cache_dir \ --incompatible_remote_build_event_upload_respect_no_cache \ + --remote_build_event_upload=all \ --build_event_json_file=bep.json \ //a:foo >& $TEST_log || fail "Failed to build" @@ -2737,6 +2745,7 @@ EOF --remote_cache=grpc://localhost:${worker_port} \ --disk_cache=$cache_dir \ --incompatible_remote_build_event_upload_respect_no_cache \ + --remote_build_event_upload=all \ --incompatible_remote_results_ignore_disk \ --build_event_json_file=bep.json \ //a:foo >& $TEST_log || fail "Failed to build"