From 848e91c7d6ed29668b98669370ee64f5d823f97e Mon Sep 17 00:00:00 2001 From: Henk van der Laan Date: Tue, 2 Jun 2020 19:44:06 +0200 Subject: [PATCH] Fix GCC 9 Code coverage by adding its JSON format GCC 9 changed the intermediate format to a JSON based format and with it changed the meaning of the `-i` flag. Because of these changes it was not possible to generate code coverage with GCC 9. This patch addresses that by adding its format in parallel to the existing GCov parser Addresses: #9406 --- .../devtools/coverageoutputgenerator/BUILD | 17 +++ .../coverageoutputgenerator/Constants.java | 1 + .../GcovJsonParser.java | 121 ++++++++++++++++++ .../coverageoutputgenerator/Main.java | 22 +++- tools/test/collect_cc_coverage.sh | 18 ++- 5 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD index db22b5ce5ba1ef..84031601852e82 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/BUILD @@ -49,6 +49,7 @@ java_library( srcs = glob(["*.java"]), deps = [ "//third_party:auto_value", + "//third_party:gson", "//third_party:guava", "//third_party:jsr305", "//third_party/java/jcommander", @@ -117,6 +118,21 @@ java_library( ], ) +java_library( + name = "GcovJsonParser", + srcs = [ + "GcovJsonParser.java", + "Parser.java", + ], + deps = [ + ":BranchCoverage", + ":Constants", + ":LineCoverage", + ":SourceFileCoverage", + "//third_party:gson", + ], +) + java_library( name = "LcovParser", srcs = [ @@ -159,6 +175,7 @@ java_library( ":Constants", ":Coverage", ":GcovParser", + ":GcovJsonParser", ":LcovMergerFlags", ":LcovParser", ":LcovPrinter", diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java index a4b1b5398dd5da..808e257d8f521f 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Constants.java @@ -39,6 +39,7 @@ class Constants { static final String TAKEN = "-"; static final String TRACEFILE_EXTENSION = ".dat"; static final String GCOV_EXTENSION = ".gcov"; + static final String GCOV_JSON_EXTENSION = ".gcov.json.gz"; static final String PROFDATA_EXTENSION = ".profdata"; static final String DELIMITER = ","; static final String GCOV_VERSION_MARKER = "version:"; diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java new file mode 100644 index 00000000000000..4c9842c1cf56e8 --- /dev/null +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/GcovJsonParser.java @@ -0,0 +1,121 @@ +// Copyright 2020 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.coverageoutputgenerator; + +import java.io.BufferedReader; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.zip.GZIPInputStream; +import com.google.gson.Gson; +import com.google.gson.annotations.SerializedName; + +/** + * A {@link Parser} for gcov intermediate json format introduced in GCC 9.1. See the flag {@code --intermediate-format} in gcov documentation. + */ +public class GcovJsonParser { + private static final Logger logger = Logger.getLogger(GcovParser.class.getName()); + private final InputStream inputStream; + + private GcovJsonParser(InputStream inputStream) { + this.inputStream = inputStream; + } + + public static List parse(InputStream inputStream) throws IOException { + return new GcovJsonParser(inputStream).parse(); + } + + private List parse() throws IOException { + ArrayList allSourceFiles = new ArrayList<>(); + try (InputStream gzipStream = new GZIPInputStream(inputStream)) { + ByteArrayOutputStream contents = new ByteArrayOutputStream(); + byte[] buffer = new byte[1024]; + int length; + while ((length = gzipStream.read(buffer)) != -1) { + contents.write(buffer, 0, length); + } + Gson gson = new Gson(); + //logger.log(Level.INFO, contents.toString()); + GcovJsonFormat document = gson.fromJson(contents.toString(), GcovJsonFormat.class); + if (!document.format_version.equals("1")) { + logger.log(Level.WARNING, "Expect GCov JSON format version 1, got format version " + document.format_version); + } + for (GcovJsonFile file : document.files) { + SourceFileCoverage currentFileCoverage = new SourceFileCoverage(file.file); + for (GcovJsonFunction function : file.functions) { + currentFileCoverage.addLineNumber(function.name, function.start_line); + currentFileCoverage.addFunctionExecution(function.name, function.execution_count); + } + for (GcovJsonLine line : file.lines) { + currentFileCoverage.addLine(line.line_number, LineCoverage.create(line.line_number, line.count, null)); + for (GcovJsonBranch branch : line.branches) { + currentFileCoverage.addBranch(line.line_number, BranchCoverage.create(line.line_number, branch.count)); + } + } + allSourceFiles.add(currentFileCoverage); + } + } + + return allSourceFiles; + } +} + +// Classes for the Gson data mapper representing the structure of the GCov JSON format + +class GcovJsonFormat { + String gcc_version; + GcovJsonFile[] files; + String format_version; + String current_working_directory; + String data_file; +} + +class GcovJsonFile { + String file; + GcovJsonFunction[] functions; + GcovJsonLine[] lines; +} + +class GcovJsonFunction { + int blocks; + int end_column; + int start_line; + String name; + int blocks_executed; + int execution_count; + String demangled_name; + int start_column; + int end_line; +} + +class GcovJsonLine { + GcovJsonBranch[] branches; + int count; + int line_number; + boolean unexecuted_block; + String function_name; +} + +class GcovJsonBranch { + boolean fallthrough; + int count; + @SerializedName("throw") + boolean _throw; +} diff --git a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java index 21f3565359f9c2..649f659d41b8e4 100644 --- a/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java +++ b/tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator/Main.java @@ -15,6 +15,7 @@ package com.google.devtools.coverageoutputgenerator; import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_EXTENSION; +import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_JSON_EXTENSION; import static com.google.devtools.coverageoutputgenerator.Constants.PROFDATA_EXTENSION; import static com.google.devtools.coverageoutputgenerator.Constants.TRACEFILE_EXTENSION; import static java.nio.charset.StandardCharsets.UTF_8; @@ -52,6 +53,7 @@ public static void main(String... args) { int exitCode = runWithArgs(args); System.exit(exitCode); } catch (Exception e) { + e.printStackTrace(); logger.log(Level.SEVERE, "Unhandled exception on lcov tool: " + e.getMessage()); System.exit(1); } @@ -78,8 +80,13 @@ static int runWithArgs(String... args) throws ExecutionException, InterruptedExc getTracefiles(flags, filesInCoverageDir), LcovParser::parse, flags.parseParallelism()), - parseFiles( - getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism())); + Coverage.merge( + parseFiles( + getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()), + parseFiles( + getGcovJsonInfoFiles(filesInCoverageDir), + GcovJsonParser::parse, + flags.parseParallelism()))); if (flags.sourcesToReplaceFile() != null) { coverage.maybeReplaceSourceFileNames(getMapFromFile(flags.sourcesToReplaceFile())); @@ -221,6 +228,16 @@ private static List getGcovInfoFiles(List filesInCoverageDir) { return gcovFiles; } + private static List getGcovJsonInfoFiles(List filesInCoverageDir) { + List gcovJsonFiles = getFilesWithExtension(filesInCoverageDir, GCOV_JSON_EXTENSION); + if (gcovJsonFiles.isEmpty()) { + logger.log(Level.INFO, "No gcov json file found."); + } else { + logger.log(Level.INFO, "Found " + gcovJsonFiles.size() + " gcov json files."); + } + return gcovJsonFiles; + } + /** * Returns a .profdata file from the given files or null if none or more profdata files were * found. @@ -352,6 +369,7 @@ static List getCoverageFilesInDir(String dir) { p -> p.toString().endsWith(TRACEFILE_EXTENSION) || p.toString().endsWith(GCOV_EXTENSION) + || p.toString().endsWith(GCOV_JSON_EXTENSION) || p.toString().endsWith(PROFDATA_EXTENSION)) .map(path -> path.toFile()) .collect(Collectors.toList()); diff --git a/tools/test/collect_cc_coverage.sh b/tools/test/collect_cc_coverage.sh index 181433bf33ff1f..cdbccd77487e77 100755 --- a/tools/test/collect_cc_coverage.sh +++ b/tools/test/collect_cc_coverage.sh @@ -125,10 +125,20 @@ function gcov_coverage() { # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84879). "${GCOV}" -i $COVERAGE_GCOV_OPTIONS -o "$(dirname ${gcda})" "${gcda}" - # Append all .gcov files in the current directory to the output file. - cat *.gcov >> "$output_file" - # Delete the .gcov files. - rm *.gcov + gcov_version=$("${GCOV}" --version | sed -n -r -e 's/^.*\s([0-9]\.[0-9]\.[0-9])\s?.*$/\1/p') + + # Check the gcov version so we can process the data correctly + if [ "$(printf '%s\n%s' "9.1.0" "$gcov_version" | sort -V | head -n 1)" != "$gcov_version" ]; then + # GCOV 9.1.0 or higher generate a JSON based coverage format + # The output is generated into multiple files: "$(basename ${gcda}).gcov.json.gz" + # Concatenating JSON documents does not yield a valid document, so they are moved individually + mv -- *.gcov.json.gz "$(dirname "$output_file")" + else + # Append all .gcov files in the current directory to the output file. + cat -- *.gcov >> "$output_file" + # Delete the .gcov files. + rm -- *.gcov + fi fi fi done < "${COVERAGE_MANIFEST}"