diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD index db22b5ce5ba1ef..d9f59a4928d74f 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD @@ -49,6 +49,7 @@ java_library( srcs = glob(["*.java"]), deps = [ "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", "//third_party/java/jcommander", @@ -117,6 +118,21 @@ java_library( ], ) +java_library( + name = "GcovJsonParser", + srcs = [ + "GcovJsonParser.java", + "Parser.java", + ], + deps = [ + ":BranchCoverage", + ":Constants", + ":LineCoverage", + ":SourceFileCoverage", + "//third_party:gson", + ], +) + java_library( name = "LcovParser", srcs = [ @@ -158,6 +174,7 @@ java_library( deps = [ ":Constants", ":Coverage", + ":GcovJsonParser", ":GcovParser", ":LcovMergerFlags", ":LcovParser", diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java index a4b1b5398dd5da..808e257d8f521f 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java @@ -39,6 +39,7 @@ class Constants { static final String TAKEN = "-"; static final String TRACEFILE_EXTENSION = ".dat"; static final String GCOV_EXTENSION = ".gcov"; + static final String GCOV_JSON_EXTENSION = ".gcov.json.gz"; static final String PROFDATA_EXTENSION = ".profdata"; static final String DELIMITER = ","; static final String GCOV_VERSION_MARKER = "version:"; diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java index aa3d87fbaea683..afd38992785066 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Coverage.java @@ -41,13 +41,12 @@ void add(SourceFileCoverage input) { } } - static Coverage merge(Coverage c1, Coverage c2) { + static Coverage merge(Coverage... coverages) { Coverage merged = new Coverage(); - for (SourceFileCoverage sourceFile : c1.getAllSourceFiles()) { - merged.add(sourceFile); - } - for (SourceFileCoverage sourceFile : c2.getAllSourceFiles()) { - merged.add(sourceFile); + for (Coverage c : coverages) { + for (SourceFileCoverage sourceFile : c.getAllSourceFiles()) { + merged.add(sourceFile); + } } return merged; } diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java new file mode 100644 index 00000000000000..72ac862a3cf897 --- /dev/null +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java @@ -0,0 +1,128 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.coverageoutputgenerator; + +import com.google.gson.Gson; +import com.google.gson.annotations.SerializedName; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; + +/** + * A {@link Parser} for gcov intermediate json format introduced in GCC 9.1. See the flag {@code + * --intermediate-format} in gcov documentation. + */ +public class GcovJsonParser { + private static final Logger logger = Logger.getLogger(GcovJsonParser.class.getName()); + private final InputStream inputStream; + + private GcovJsonParser(InputStream inputStream) { + this.inputStream = inputStream; + } + + public static List parse(InputStream inputStream) throws IOException { + return new GcovJsonParser(inputStream).parse(); + } + + private List parse() throws IOException { + ArrayList allSourceFiles = new ArrayList<>(); + try (InputStream gzipStream = new GZIPInputStream(inputStream)) { + ByteArrayOutputStream contents = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = gzipStream.read(buffer)) != -1) { + contents.write(buffer, 0, length); + } + Gson gson = new Gson(); + GcovJsonFormat document = gson.fromJson(contents.toString(), GcovJsonFormat.class); + if (!document.format_version.equals("1")) { + logger.log( + Level.WARNING, + "Expect GCov JSON format version 1, got format version " + document.format_version); + } + for (GcovJsonFile file : document.files) { + SourceFileCoverage currentFileCoverage = new SourceFileCoverage(file.file); + for (GcovJsonFunction function : file.functions) { + currentFileCoverage.addLineNumber(function.name, function.start_line); + currentFileCoverage.addFunctionExecution(function.name, function.execution_count); + } + for (GcovJsonLine line : file.lines) { + currentFileCoverage.addLine( + line.line_number, LineCoverage.create(line.line_number, line.count, null)); + for (GcovJsonBranch branch : line.branches) { + currentFileCoverage.addBranch( + line.line_number, BranchCoverage.create(line.line_number, branch.count)); + } + } + allSourceFiles.add(currentFileCoverage); + } + } + + return allSourceFiles; + } + + // Classes for the Gson data mapper representing the structure of the GCov JSON format + // These do not follow the Java naming styleguide as they need to match the JSON field names + // Documentation can be found in GCov's manpage, of which the source is available at + // https://gcc.gnu.org/git/?p=gcc.git;a=blob;f=gcc/doc/gcov.texi;h=dcdd7831ff063483d43e5347af0b67083c85ecc4;hb=4212a6a3e44f870412d9025eeb323fd4f50a61da#l184 + + static class GcovJsonFormat { + String gcc_version; + GcovJsonFile[] files; + String format_version; + String current_working_directory; + String data_file; + } + + static class GcovJsonFile { + String file; + GcovJsonFunction[] functions; + GcovJsonLine[] lines; + } + + static class GcovJsonFunction { + int blocks; + int end_column; + int start_line; + String name; + int blocks_executed; + int execution_count; + String demangled_name; + int start_column; + int end_line; + } + + static class GcovJsonLine { + GcovJsonBranch[] branches; + int count; + int line_number; + boolean unexecuted_block; + String function_name; + } + + static class GcovJsonBranch { + boolean fallthrough; + int count; + + @SerializedName("throw") + boolean _throw; + } +} diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java index 21f3565359f9c2..7f24d3245b1bbe 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java @@ -15,6 +15,7 @@ package com.google.devtools.coverageoutputgenerator; import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_EXTENSION; +import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_JSON_EXTENSION; import static com.google.devtools.coverageoutputgenerator.Constants.PROFDATA_EXTENSION; import static com.google.devtools.coverageoutputgenerator.Constants.TRACEFILE_EXTENSION; import static java.nio.charset.StandardCharsets.UTF_8; @@ -79,7 +80,11 @@ static int runWithArgs(String... args) throws ExecutionException, InterruptedExc LcovParser::parse, flags.parseParallelism()), parseFiles( - getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism())); + getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()), + parseFiles( + getGcovJsonInfoFiles(filesInCoverageDir), + GcovJsonParser::parse, + flags.parseParallelism())); if (flags.sourcesToReplaceFile() != null) { coverage.maybeReplaceSourceFileNames(getMapFromFile(flags.sourcesToReplaceFile())); @@ -221,6 +226,16 @@ private static List getGcovInfoFiles(List filesInCoverageDir) { return gcovFiles; } + private static List getGcovJsonInfoFiles(List filesInCoverageDir) { + List gcovJsonFiles = getFilesWithExtension(filesInCoverageDir, GCOV_JSON_EXTENSION); + if (gcovJsonFiles.isEmpty()) { + logger.log(Level.INFO, "No gcov json file found."); + } else { + logger.log(Level.INFO, "Found " + gcovJsonFiles.size() + " gcov json files."); + } + return gcovJsonFiles; + } + /** * Returns a .profdata file from the given files or null if none or more profdata files were * found. @@ -352,6 +367,7 @@ static List getCoverageFilesInDir(String dir) { p -> p.toString().endsWith(TRACEFILE_EXTENSION) || p.toString().endsWith(GCOV_EXTENSION) + || p.toString().endsWith(GCOV_JSON_EXTENSION) || p.toString().endsWith(PROFDATA_EXTENSION)) .map(path -> path.toFile()) .collect(Collectors.toList()); diff --git a/tools/test/collect_cc_coverage.sh b/tools/test/collect_cc_coverage.sh index 181433bf33ff1f..1d50238a169eb5 100755 --- a/tools/test/collect_cc_coverage.sh +++ b/tools/test/collect_cc_coverage.sh @@ -125,10 +125,23 @@ function gcov_coverage() { # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84879). "${GCOV}" -i $COVERAGE_GCOV_OPTIONS -o "$(dirname ${gcda})" "${gcda}" - # Append all .gcov files in the current directory to the output file. - cat *.gcov >> "$output_file" - # Delete the .gcov files. - rm *.gcov + # Extract gcov's version: the output of `gcov --version` contains the + # version as a set of major-minor-patch numbers, of which we extract + # the major version. + gcov_major_version=$("${GCOV}" --version | sed -n -E -e 's/^.*\s([0-9]+)\.[0-9]+\.[0-9]+\s?.*$/\1/p') + + # Check the gcov version so we can process the data correctly + if [[ $gcov_major_version -ge 9 ]]; then + # gcov 9 or higher use a JSON based format for their coverage reports. + # The output is generated into multiple files: "$(basename ${gcda}).gcov.json.gz" + # Concatenating JSON documents does not yield a valid document, so they are moved individually + mv -- *.gcov.json.gz "$(dirname "$output_file")" + else + # Append all .gcov files in the current directory to the output file. + cat -- *.gcov >> "$output_file" + # Delete the .gcov files. + rm -- *.gcov + fi fi fi done < "${COVERAGE_MANIFEST}"