From 795d5b500d1cd637d32162ab32537139f5f9976e Mon Sep 17 00:00:00 2001 From: Googler Date: Fri, 2 Feb 2024 07:40:04 -0800 Subject: [PATCH] [7.1.0] Implement a new execution log conversion tool. For now, it knows how to convert between the binary and JSON formats. Ability to convert from the new compact format will be added in a followup. Tested by running manually. I will add end-to-end test coverage once the converter has been fully implemented. PiperOrigin-RevId: 603684017 Change-Id: Idf0d851feb9ea22e7021a1b62f003f0978bcd378 --- src/tools/execlog/BUILD | 7 ++ src/tools/execlog/README.md | 15 ++- .../com/google/devtools/build/execlog/BUILD | 21 +++- .../build/execlog/ConverterOptions.java | 101 ++++++++++++++++++ .../build/execlog/ExecLogConverter.java | 87 +++++++++++++++ 5 files changed, 229 insertions(+), 2 deletions(-) create mode 100644 src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ConverterOptions.java create mode 100644 src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ExecLogConverter.java diff --git a/src/tools/execlog/BUILD b/src/tools/execlog/BUILD index 17eea8626d4de2..8983f552c6faa5 100644 --- a/src/tools/execlog/BUILD +++ b/src/tools/execlog/BUILD @@ -20,3 +20,10 @@ java_binary( visibility = ["//visibility:public"], runtime_deps = ["//src/tools/execlog/src/main/java/com/google/devtools/build/execlog:parser"], ) + +java_binary( + name = "converter", + main_class = "com.google.devtools.build.execlog.ExecLogConverter", + visibility = ["//visibility:public"], + runtime_deps = ["//src/tools/execlog/src/main/java/com/google/devtools/build/execlog:converter"], +) diff --git a/src/tools/execlog/README.md b/src/tools/execlog/README.md index 0830029e6c9605..b532b2132f08a2 100644 --- a/src/tools/execlog/README.md +++ b/src/tools/execlog/README.md @@ -8,7 +8,7 @@ To generate the execution log, run e.g.: Then build the parser and run it. - bazel build src/tools/execlog:all + bazel build src/tools/execlog:parser bazel-bin/src/tools/execlog/parser --log_path=/tmp/exec.log This will simply print the log contents to stdout in text form. @@ -44,3 +44,16 @@ are put at the end of `/tmp/exec2.log.txt`. Note that this reordering makes it easier to see differences using text-based diffing tools, but may break the logical sequence of actions in `/tmp/exec2.log.txt`. + +# Execution Log Converter + +This tool is used to convert between Bazel execution log formats. + +For example, to convert from the binary format to the JSON format: + + bazel build src/tools/execlog:converter + bazel-bin/src/tools/execlog/converter \ + --input binary:/tmp/binary.log --output json:/tmp/json.log + +By default, the output will be in the same order as the input. To sort in a +deterministic order, use --sort. diff --git a/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/BUILD b/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/BUILD index f1018b399e0910..e87e67acb563ee 100644 --- a/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/BUILD +++ b/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/BUILD @@ -13,10 +13,29 @@ filegroup( java_library( name = "parser", - srcs = glob(["*.java"]), + srcs = [ + "ExecLogParser.java", + "ParserOptions.java", + ], + deps = [ + "//src/main/java/com/google/devtools/common/options", + "//src/main/protobuf:spawn_java_proto", + "//third_party:guava", + ], +) + +java_library( + name = "converter", + srcs = [ + "ConverterOptions.java", + "ExecLogConverter.java", + ], deps = [ + "//src/main/java/com/google/devtools/build/lib/exec:spawn_log_context_utils", + "//src/main/java/com/google/devtools/build/lib/util/io:io-proto", "//src/main/java/com/google/devtools/common/options", "//src/main/protobuf:spawn_java_proto", + "//third_party:auto_value", "//third_party:guava", ], ) diff --git a/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ConverterOptions.java b/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ConverterOptions.java new file mode 100644 index 00000000000000..61b515efb11203 --- /dev/null +++ b/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ConverterOptions.java @@ -0,0 +1,101 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.execlog; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; + +import com.google.auto.value.AutoValue; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableMap; +import com.google.devtools.common.options.Converter; +import com.google.devtools.common.options.Option; +import com.google.devtools.common.options.OptionDocumentationCategory; +import com.google.devtools.common.options.OptionEffectTag; +import com.google.devtools.common.options.OptionsBase; +import com.google.devtools.common.options.OptionsParsingException; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.List; +import java.util.Locale; + +/** Options for execution log converter. */ +public class ConverterOptions extends OptionsBase { + private static final Splitter COLON_SPLITTER = Splitter.on(':').limit(2); + + @Option( + name = "input", + defaultValue = "null", + converter = FormatAndPathConverter.class, + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Input log format and path.") + public FormatAndPath input; + + @Option( + name = "output", + defaultValue = "null", + converter = FormatAndPathConverter.class, + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Output log format and path.") + public FormatAndPath output; + + @Option( + name = "sort", + defaultValue = "false", + documentationCategory = OptionDocumentationCategory.UNCATEGORIZED, + effectTags = {OptionEffectTag.UNKNOWN}, + help = "Whether to sort the output in a deterministic order.") + public boolean sort; + + enum Format { + BINARY, + JSON + } + + private static final ImmutableMap FORMAT_BY_NAME = + Arrays.stream(Format.values()) + .collect(toImmutableMap(f -> f.name().toLowerCase(Locale.US), f -> f)); + + @AutoValue + abstract static class FormatAndPath { + public abstract Format format(); + + public abstract Path path(); + + public static FormatAndPath of(Format format, Path path) { + return new AutoValue_ConverterOptions_FormatAndPath(format, path); + } + } + + private static class FormatAndPathConverter extends Converter.Contextless { + + @Override + public FormatAndPath convert(String input) throws OptionsParsingException { + List parts = COLON_SPLITTER.splitToList(input); + if (parts.size() != 2 + || !FORMAT_BY_NAME.containsKey(parts.get(0)) + || parts.get(1).isEmpty()) { + throw new OptionsParsingException("'" + input + "' is not a valid log format and path."); + } + return FormatAndPath.of(FORMAT_BY_NAME.get(parts.get(0)), Path.of(parts.get(1))); + } + + @Override + public String getTypeDescription() { + return "type:path, where type is one of " + String.join(" ", FORMAT_BY_NAME.keySet()); + } + } +} diff --git a/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ExecLogConverter.java b/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ExecLogConverter.java new file mode 100644 index 00000000000000..4ab556d7c2b3af --- /dev/null +++ b/src/tools/execlog/src/main/java/com/google/devtools/build/execlog/ExecLogConverter.java @@ -0,0 +1,87 @@ +// Copyright 2024 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package com.google.devtools.build.execlog; + +import com.google.devtools.build.execlog.ConverterOptions.FormatAndPath; +import com.google.devtools.build.lib.exec.Protos.SpawnExec; +import com.google.devtools.build.lib.exec.StableSort; +import com.google.devtools.build.lib.util.io.MessageInputStream; +import com.google.devtools.build.lib.util.io.MessageInputStreamWrapper.BinaryInputStreamWrapper; +import com.google.devtools.build.lib.util.io.MessageInputStreamWrapper.JsonInputStreamWrapper; +import com.google.devtools.build.lib.util.io.MessageOutputStream; +import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.BinaryOutputStreamWrapper; +import com.google.devtools.build.lib.util.io.MessageOutputStreamWrapper.JsonOutputStreamWrapper; +import com.google.devtools.common.options.OptionsParser; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; + +/** A tool to convert between Bazel execution log formats. */ +final class ExecLogConverter { + private ExecLogConverter() {} + + private static MessageInputStream getMessageInputStream(FormatAndPath log) + throws IOException { + InputStream in = Files.newInputStream(log.path()); + switch (log.format()) { + case BINARY: + return new BinaryInputStreamWrapper<>(in, SpawnExec.getDefaultInstance()); + case JSON: + return new JsonInputStreamWrapper<>(in, SpawnExec.getDefaultInstance()); + } + throw new AssertionError("unsupported input format"); + } + + private static MessageOutputStream getMessageOutputStream(FormatAndPath log) + throws IOException { + OutputStream out = Files.newOutputStream(log.path()); + switch (log.format()) { + case BINARY: + return new BinaryOutputStreamWrapper<>(out); + case JSON: + return new JsonOutputStreamWrapper<>(out); + } + throw new AssertionError("unsupported output format"); + } + + public static void main(String[] args) throws Exception { + OptionsParser op = OptionsParser.builder().optionsClasses(ConverterOptions.class).build(); + op.parseAndExitUponError(args); + + ConverterOptions options = op.getOptions(ConverterOptions.class); + + if (options.input == null) { + System.err.println("--input must be specified."); + System.exit(1); + } + + if (options.output == null) { + System.err.println("--output must be specified."); + System.exit(1); + } + + try (MessageInputStream in = getMessageInputStream(options.input); + MessageOutputStream out = getMessageOutputStream(options.output)) { + if (options.sort) { + StableSort.stableSort(in, out); + } else { + SpawnExec ex; + while ((ex = in.read()) != null) { + out.write(ex); + } + } + } + } +}