Skip to content

Commit

Permalink
Fix GCC 9 Code coverage by adding its JSON format
Browse files Browse the repository at this point in the history
GCC 9 changed the intermediate format to a JSON based format and with it
changed the meaning of the `-i` flag. Because of these changes it was not
possible to generate code coverage with GCC 9. This patch addresses that
by adding its format next to the existing GCov parser.

Addresses: #9406

Closes #11538.

PiperOrigin-RevId: 316641962
  • Loading branch information
helaan authored and copybara-github committed Jun 16, 2020
1 parent 5691bde commit 3b75bc7
Show file tree
Hide file tree
Showing 6 changed files with 185 additions and 11 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ java_library(
srcs = glob(["*.java"]),
deps = [
"//third_party:auto_value",
"//third_party:gson",
"//third_party:guava",
"//third_party:jsr305",
"//third_party/java/jcommander",
Expand Down Expand Up @@ -117,6 +118,21 @@ java_library(
],
)

java_library(
name = "GcovJsonParser",
srcs = [
"GcovJsonParser.java",
"Parser.java",
],
deps = [
":BranchCoverage",
":Constants",
":LineCoverage",
":SourceFileCoverage",
"//third_party:gson",
],
)

java_library(
name = "LcovParser",
srcs = [
Expand Down Expand Up @@ -158,6 +174,7 @@ java_library(
deps = [
":Constants",
":Coverage",
":GcovJsonParser",
":GcovParser",
":LcovMergerFlags",
":LcovParser",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ class Constants {
static final String TAKEN = "-";
static final String TRACEFILE_EXTENSION = ".dat";
static final String GCOV_EXTENSION = ".gcov";
static final String GCOV_JSON_EXTENSION = ".gcov.json.gz";
static final String PROFDATA_EXTENSION = ".profdata";
static final String DELIMITER = ",";
static final String GCOV_VERSION_MARKER = "version:";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,12 @@ void add(SourceFileCoverage input) {
}
}

static Coverage merge(Coverage c1, Coverage c2) {
static Coverage merge(Coverage... coverages) {
Coverage merged = new Coverage();
for (SourceFileCoverage sourceFile : c1.getAllSourceFiles()) {
merged.add(sourceFile);
}
for (SourceFileCoverage sourceFile : c2.getAllSourceFiles()) {
merged.add(sourceFile);
for (Coverage c : coverages) {
for (SourceFileCoverage sourceFile : c.getAllSourceFiles()) {
merged.add(sourceFile);
}
}
return merged;
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// Copyright 2020 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.devtools.coverageoutputgenerator;

import com.google.gson.Gson;
import com.google.gson.annotations.SerializedName;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.GZIPInputStream;

/**
* A {@link Parser} for gcov intermediate json format introduced in GCC 9.1. See the flag {@code
* --intermediate-format} in <a
* href="https://gcc.gnu.org/onlinedocs/gcc-9.3.0/gcc/Invoking-Gcov.html">gcov documentation</a>.
*/
public class GcovJsonParser {
private static final Logger logger = Logger.getLogger(GcovJsonParser.class.getName());
private final InputStream inputStream;

private GcovJsonParser(InputStream inputStream) {
this.inputStream = inputStream;
}

public static List<SourceFileCoverage> parse(InputStream inputStream) throws IOException {
return new GcovJsonParser(inputStream).parse();
}

private List<SourceFileCoverage> parse() throws IOException {
ArrayList<SourceFileCoverage> allSourceFiles = new ArrayList<>();
try (InputStream gzipStream = new GZIPInputStream(inputStream)) {
ByteArrayOutputStream contents = new ByteArrayOutputStream();
byte[] buffer = new byte[1024];
int length;
while ((length = gzipStream.read(buffer)) != -1) {
contents.write(buffer, 0, length);
}
Gson gson = new Gson();
GcovJsonFormat document = gson.fromJson(contents.toString(), GcovJsonFormat.class);
if (!document.format_version.equals("1")) {
logger.log(
Level.WARNING,
"Expect GCov JSON format version 1, got format version " + document.format_version);
}
for (GcovJsonFile file : document.files) {
SourceFileCoverage currentFileCoverage = new SourceFileCoverage(file.file);
for (GcovJsonFunction function : file.functions) {
currentFileCoverage.addLineNumber(function.name, function.start_line);
currentFileCoverage.addFunctionExecution(function.name, function.execution_count);
}
for (GcovJsonLine line : file.lines) {
currentFileCoverage.addLine(
line.line_number, LineCoverage.create(line.line_number, line.count, null));
for (GcovJsonBranch branch : line.branches) {
currentFileCoverage.addBranch(
line.line_number, BranchCoverage.create(line.line_number, branch.count));
}
}
allSourceFiles.add(currentFileCoverage);
}
}

return allSourceFiles;
}

// Classes for the Gson data mapper representing the structure of the GCov JSON format
// These do not follow the Java naming styleguide as they need to match the JSON field names
// Documentation can be found in GCov's manpage, of which the source is available at
// https://gcc.gnu.org/git/?p=gcc.git;a=blob;f=gcc/doc/gcov.texi;h=dcdd7831ff063483d43e5347af0b67083c85ecc4;hb=4212a6a3e44f870412d9025eeb323fd4f50a61da#l184

static class GcovJsonFormat {
String gcc_version;
GcovJsonFile[] files;
String format_version;
String current_working_directory;
String data_file;
}

static class GcovJsonFile {
String file;
GcovJsonFunction[] functions;
GcovJsonLine[] lines;
}

static class GcovJsonFunction {
int blocks;
int end_column;
int start_line;
String name;
int blocks_executed;
int execution_count;
String demangled_name;
int start_column;
int end_line;
}

static class GcovJsonLine {
GcovJsonBranch[] branches;
int count;
int line_number;
boolean unexecuted_block;
String function_name;
}

static class GcovJsonBranch {
boolean fallthrough;
int count;

@SerializedName("throw")
boolean _throw;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package com.google.devtools.coverageoutputgenerator;

import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_EXTENSION;
import static com.google.devtools.coverageoutputgenerator.Constants.GCOV_JSON_EXTENSION;
import static com.google.devtools.coverageoutputgenerator.Constants.PROFDATA_EXTENSION;
import static com.google.devtools.coverageoutputgenerator.Constants.TRACEFILE_EXTENSION;
import static java.nio.charset.StandardCharsets.UTF_8;
Expand Down Expand Up @@ -79,7 +80,11 @@ static int runWithArgs(String... args) throws ExecutionException, InterruptedExc
LcovParser::parse,
flags.parseParallelism()),
parseFiles(
getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()));
getGcovInfoFiles(filesInCoverageDir), GcovParser::parse, flags.parseParallelism()),
parseFiles(
getGcovJsonInfoFiles(filesInCoverageDir),
GcovJsonParser::parse,
flags.parseParallelism()));

if (flags.sourcesToReplaceFile() != null) {
coverage.maybeReplaceSourceFileNames(getMapFromFile(flags.sourcesToReplaceFile()));
Expand Down Expand Up @@ -221,6 +226,16 @@ private static List<File> getGcovInfoFiles(List<File> filesInCoverageDir) {
return gcovFiles;
}

private static List<File> getGcovJsonInfoFiles(List<File> filesInCoverageDir) {
List<File> gcovJsonFiles = getFilesWithExtension(filesInCoverageDir, GCOV_JSON_EXTENSION);
if (gcovJsonFiles.isEmpty()) {
logger.log(Level.INFO, "No gcov json file found.");
} else {
logger.log(Level.INFO, "Found " + gcovJsonFiles.size() + " gcov json files.");
}
return gcovJsonFiles;
}

/**
* Returns a .profdata file from the given files or null if none or more profdata files were
* found.
Expand Down Expand Up @@ -352,6 +367,7 @@ static List<File> getCoverageFilesInDir(String dir) {
p ->
p.toString().endsWith(TRACEFILE_EXTENSION)
|| p.toString().endsWith(GCOV_EXTENSION)
|| p.toString().endsWith(GCOV_JSON_EXTENSION)
|| p.toString().endsWith(PROFDATA_EXTENSION))
.map(path -> path.toFile())
.collect(Collectors.toList());
Expand Down
21 changes: 17 additions & 4 deletions tools/test/collect_cc_coverage.sh
Original file line number Diff line number Diff line change
Expand Up @@ -125,10 +125,23 @@ function gcov_coverage() {
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=84879).
"${GCOV}" -i $COVERAGE_GCOV_OPTIONS -o "$(dirname ${gcda})" "${gcda}"

# Append all .gcov files in the current directory to the output file.
cat *.gcov >> "$output_file"
# Delete the .gcov files.
rm *.gcov
# Extract gcov's version: the output of `gcov --version` contains the
# version as a set of major-minor-patch numbers, of which we extract
# the major version.
gcov_major_version=$("${GCOV}" --version | sed -n -E -e 's/^.*\s([0-9]+)\.[0-9]+\.[0-9]+\s?.*$/\1/p')

# Check the gcov version so we can process the data correctly
if [[ $gcov_major_version -ge 9 ]]; then
# gcov 9 or higher use a JSON based format for their coverage reports.
# The output is generated into multiple files: "$(basename ${gcda}).gcov.json.gz"
# Concatenating JSON documents does not yield a valid document, so they are moved individually
mv -- *.gcov.json.gz "$(dirname "$output_file")"
else
# Append all .gcov files in the current directory to the output file.
cat -- *.gcov >> "$output_file"
# Delete the .gcov files.
rm -- *.gcov
fi
fi
fi
done < "${COVERAGE_MANIFEST}"
Expand Down

0 comments on commit 3b75bc7

Please sign in to comment.