Reduce memory footprint of fastcov

Description: - Fix #43 - Bump version to 1.6 - Refactor GCDA processing to do incremental report merging - Refactor GCDA processing to use multiprocess.Process instead of threads - When branch lists mismatch, overlay smaller list on bigger list
RPGillespie6 · May 19, 2020 · d616eb6 · d616eb6
1 parent 7793d3e
commit d616eb6
Show file tree

Hide file tree

Showing 5 changed files with 81 additions and 64 deletions.
diff --git a/fastcov.py b/fastcov.py
@@ -31,7 +31,7 @@
 import subprocess
 import multiprocessing
 
-FASTCOV_VERSION = (1,5)
+FASTCOV_VERSION = (1,6)
 MINIMUM_PYTHON  = (3,5)
 MINIMUM_GCOV    = (9,0,0)
 
@@ -100,36 +100,52 @@ def findCoverageFiles(cwd, coverage_files, use_gcno):
     logging.debug("Coverage files found:\n    %s", "\n    ".join(coverage_files))
     return coverage_files
 
-def gcovWorker(cwd, gcov, files, chunk, gcov_filter_options, branch_coverage):
+def gcovWorker(q, args, chunk, gcov_filter_options):
+    base_report = {
+        "sources": {}
+    }
+
     gcov_args = "-it"
-    if branch_coverage:
+    if args.branchcoverage or args.xbranchcoverage:
         gcov_args += "b"
 
-    p = subprocess.Popen([gcov, gcov_args] + chunk, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
+    p = subprocess.Popen([args.gcov, gcov_args] + chunk, cwd=args.cdirectory, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL)
     for line in iter(p.stdout.readline, b''):
         intermediate_json = json.loads(line.decode(sys.stdout.encoding))
-        intermediate_json_files = processGcovs(cwd, intermediate_json["files"], gcov_filter_options)
+        intermediate_json_files = processGcovs(args.cdirectory, intermediate_json["files"], gcov_filter_options)
         for f in intermediate_json_files:
-            files.append(f) #thread safe, there might be a better way to do this though
+            distillSource(f, base_report["sources"], args.test_name, args.xbranchcoverage)
         GCOVS_TOTAL.append(len(intermediate_json["files"]))
         GCOVS_SKIPPED.append(len(intermediate_json["files"])-len(intermediate_json_files))
+
     p.wait()
+    q.put(base_report)
 
-def processGcdas(cwd, gcov, jobs, coverage_files, gcov_filter_options, branch_coverage, min_chunk_size):
-    chunk_size = max(min_chunk_size, int(len(coverage_files) / jobs) + 1)
+def processGcdas(args, coverage_files, gcov_filter_options):
+    chunk_size = max(args.minimum_chunk, int(len(coverage_files) / args.jobs) + 1)
 
-    threads = []
-    intermediate_json_files = []
+    processes = []
+    q = multiprocessing.Queue()
     for chunk in chunks(coverage_files, chunk_size):
-        t = threading.Thread(target=gcovWorker, args=(cwd, gcov, intermediate_json_files, chunk, gcov_filter_options, branch_coverage))
-        threads.append(t)
-        t.start()
+        p = multiprocessing.Process(target=gcovWorker, args=(q, args, chunk, gcov_filter_options))
+        processes.append(p)
+        p.start()
 
-    logging.info("Spawned {} gcov threads, each processing at most {} coverage files".format(len(threads), chunk_size))
-    for t in threads:
-        t.join()
+    logging.info("Spawned {} gcov processes, each processing at most {} coverage files".format(len(processes), chunk_size))
+
+    fastcov_jsons = []
+    for p in processes:
+        fj = q.get()
+        fastcov_jsons.append(fj)
+
+    for p in processes:
+        p.join()
 
-    return intermediate_json_files
+    base_fastcov = fastcov_jsons.pop()
+    for fj in fastcov_jsons:
+        combineReports(base_fastcov, fj)
+
+    return base_fastcov
 
 def processGcov(cwd, gcov, files, gcov_filter_options):
     # Add absolute path
@@ -377,16 +393,6 @@ def distillSource(source_raw, sources, test_name, include_exceptional_branches):
     for line in source_raw["lines"]:
         distillLine(line, sources[source_name][test_name]["lines"], sources[source_name][test_name]["branches"], include_exceptional_branches)
 
-def distillReport(report_raw, args):
-    report_json = {
-        "sources": {}
-    }
-
-    for source in report_raw:
-        distillSource(source, report_json["sources"], args.test_name, args.xbranchcoverage)
-
-    return report_json
-
 def dumpToJson(intermediate, output):
     with open(output, "w") as f:
         json.dump(intermediate, f)
@@ -411,12 +417,16 @@ def addDicts(dict1, dict2):
 
 def addLists(list1, list2):
     """Add lists together by value. i.e. addLists([1,1], [2,2]) == [3,3]"""
-    if len(list1) == len(list2):
-        return [b1 + b2 for b1, b2 in zip(list1, list2)]
-    else:
-        # One report had different number branch measurements than the other, print a warning
-        logging.warning("Possible loss of correctness. Different number of branches for same line when combining reports ({} vs {})\n".format(list1, list2))
-        return list1 if len(list1) > len(list2) else list2
+    # Find big list and small list
+    blist, slist = list(list2), list(list1)
+    if len(list1) > len(list2):
+        blist, slist = slist, blist
+
+    # Overlay small list onto big list
+    for i, b in enumerate(slist):
+        blist[i] += b
+
+    return blist
 
 def combineReports(base, overlay):
     for source, scov in overlay["sources"].items():
@@ -645,16 +655,12 @@ def main():
 
     # Fire up one gcov per cpu and start processing gcdas
     gcov_filter_options = getGcovFilterOptions(args)
-    intermediate_json_files = processGcdas(args.cdirectory, args.gcov, args.jobs, coverage_files, gcov_filter_options, args.branchcoverage or args.xbranchcoverage, args.minimum_chunk)
+    fastcov_json = processGcdas(args, coverage_files, gcov_filter_options)
 
     # Summarize processing results
     gcov_total = sum(GCOVS_TOTAL)
     gcov_skipped = sum(GCOVS_SKIPPED)
     logging.info("Processed {} .gcov files ({} total, {} skipped)".format(gcov_total - gcov_skipped, gcov_total, gcov_skipped))
-
-    # Distill all the extraneous info gcov gives us down to the core report
-    fastcov_json = distillReport(intermediate_json_files, args)
-    logging.info("Aggregated raw gcov JSON into fastcov JSON report")
     logging.debug("Final report will contain coverage for the following %d source files:\n    %s", len(fastcov_json["sources"]), "\n    ".join(fastcov_json["sources"]))
 
     # Scan for exclusion markers

diff --git a/setup.cfg b/setup.cfg
@@ -4,7 +4,7 @@
 
 [metadata]
 name = fastcov
-version = 1.5
+version = 1.6
 description = A massively parallel gcov wrapper for generating intermediate coverage formats fast
 author = Bryan Gillespie
 author-email = [email protected]

diff --git a/test/functional/cmake_project/.coveragerc b/test/functional/cmake_project/.coveragerc
@@ -0,0 +1,2 @@
+[run]
+concurrency = multiprocessing
diff --git a/test/functional/run_all.sh b/test/functional/run_all.sh
@@ -21,84 +21,87 @@ ninja
 # Erase coverage
 coverage erase
 
+# Get coverage config for multiprocessing
+cp ../.coveragerc .
+
 # Generate GCDA
 ctest
 
 # Test zerocounters
 test `find . -name *.gcda | wc -l` -ne 0
-coverage run --append ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters
+coverage run -a ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters
 test `find . -name *.gcda | wc -l` -eq 0
 
 # Run ctest_1
 ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters  # Clear previous test coverage
 ctest -R ctest_1
 
 # Test (basic report generation - no branches)
-coverage run --append ${TEST_DIR}/fastcov.py --gcov gcov-9 --verbose --exclude cmake_project/test/ --lcov -o test1.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --gcov gcov-9 --verbose --exclude cmake_project/test/ --lcov -o test1.actual.info
 cmp test1.actual.info ${TEST_DIR}/expected_results/test1.expected.info
 
-coverage run --append ${TEST_DIR}/fastcov.py --gcov gcov-9 --exclude cmake_project/test/ -o test1.actual.fastcov.json
+coverage run -a ${TEST_DIR}/fastcov.py --gcov gcov-9 --exclude cmake_project/test/ -o test1.actual.fastcov.json
 ${TEST_DIR}/json_cmp.py test1.actual.fastcov.json ${TEST_DIR}/expected_results/test1.expected.fastcov.json
 
 # Test the setting of the "testname" field
-coverage run --append ${TEST_DIR}/fastcov.py -t FunctionalTest1 --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test1.tn.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py -t FunctionalTest1 --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test1.tn.actual.info
 cmp test1.tn.actual.info ${TEST_DIR}/expected_results/test1.tn.expected.info
 
 # Test (basic report info - with branches)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test2.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test2.actual.info
 cmp test2.actual.info ${TEST_DIR}/expected_results/test2.expected.info
 
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ -o test2.actual.fastcov.json
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ -o test2.actual.fastcov.json
 ${TEST_DIR}/json_cmp.py test2.actual.fastcov.json ${TEST_DIR}/expected_results/test2.expected.fastcov.json
 
 # Test (basic lcov info - with branches; equivalent --include)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --include src/ --lcov -o test3.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --include src/ --lcov -o test3.actual.info
 cmp test3.actual.info ${TEST_DIR}/expected_results/test2.expected.info
 
 # Test (basic lcov info - with branches; equivalent --exclude-gcda)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --exclude-gcda test1.cpp.gcda --lcov -o test4.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --exclude-gcda test1.cpp.gcda --lcov -o test4.actual.info
 cmp test4.actual.info ${TEST_DIR}/expected_results/test2.expected.info
 
 # Test (basic lcov info - with branches; equivalent --source-files)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --source-files ../src/source1.cpp ../src/source2.cpp --lcov -o test5.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --source-files ../src/source1.cpp ../src/source2.cpp --lcov -o test5.actual.info
 cmp test5.actual.info ${TEST_DIR}/expected_results/test2.expected.info
 
 # Test (basic lcov info - with branches; gcno untested file coverage)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --source-files ../src/source1.cpp ../src/source2.cpp ../src/untested.cpp --lcov -o untested.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --source-files ../src/source1.cpp ../src/source2.cpp ../src/untested.cpp --lcov -o untested.actual.info
 cmp untested.actual.info ${TEST_DIR}/expected_results/untested.expected.info
 
 # Test (basic lcov info - with branches; gcno untested file coverage; equivalent --gcda-files)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --gcda-files ./test/CMakeFiles/test1.dir/__/src/source1.cpp.gcno ./test/CMakeFiles/test1.dir/__/src/source2.cpp.gcno ./src/CMakeFiles/untested.dir/untested.cpp.gcno --lcov -o untested2.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --gcda-files ./test/CMakeFiles/test1.dir/__/src/source1.cpp.gcno ./test/CMakeFiles/test1.dir/__/src/source2.cpp.gcno ./src/CMakeFiles/untested.dir/untested.cpp.gcno --lcov -o untested2.actual.info
 cmp untested2.actual.info ${TEST_DIR}/expected_results/untested.expected.info
 
 # Test (gcov version fail)
-if coverage run --append ${TEST_DIR}/fastcov.py --gcov ${TEST_DIR}/fake-gcov.sh ; then
+if coverage run -a ${TEST_DIR}/fastcov.py --gcov ${TEST_DIR}/fake-gcov.sh ; then
     echo "Expected gcov version check to fail"
     exit 1
 fi
 
 # Combine operation
-coverage run --append ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/test2.expected.info ${TEST_DIR}/expected_results/test1.tn.expected.info --lcov -o combine1.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/test2.expected.info ${TEST_DIR}/expected_results/test1.tn.expected.info --lcov -o combine1.actual.info
 cmp combine1.actual.info ${TEST_DIR}/expected_results/combine1.expected.info
 
 # Combine operation - Mix and Match json/info
-coverage run --append ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/test2.expected.fastcov.json ${TEST_DIR}/expected_results/test1.tn.expected.info --lcov -o combine1.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/test2.expected.fastcov.json ${TEST_DIR}/expected_results/test1.tn.expected.info --lcov -o combine1.actual.info
 cmp combine1.actual.info ${TEST_DIR}/expected_results/combine1.expected.info
 
 # Combine operation- source files across coverage reports
 ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --source-files ../src/source1.cpp --lcov -o combine.source1.actual.info
 ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --source-files ../src/source2.cpp --lcov -o combine.source2.actual.info
-coverage run --append ${TEST_DIR}/fastcov.py -C combine.source1.actual.info combine.source2.actual.info --lcov -o combine2.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py -C combine.source1.actual.info combine.source2.actual.info --lcov -o combine2.actual.info
 cmp combine2.actual.info ${TEST_DIR}/expected_results/combine2.expected.info
 
 # Combine files with different function inclusion
-coverage run --append ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/combine3b.info ${TEST_DIR}/expected_results/combine3a.info --lcov -o combine3.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/combine3b.info ${TEST_DIR}/expected_results/combine3a.info --lcov -o combine3.actual.info
 cmp combine3.actual.info ${TEST_DIR}/expected_results/combine3.expected.info
 
 # Combine files with different test names
 # Expected result generated with:
 # lcov -a combine4a.info -a combine4b.info -a combine4c.info --rc lcov_branch_coverage=1 -o combine4.expected.info
-coverage run --append ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/combine4a.info ${TEST_DIR}/expected_results/combine4b.info ${TEST_DIR}/expected_results/combine4c.info --lcov -o combine4.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/combine4a.info ${TEST_DIR}/expected_results/combine4b.info ${TEST_DIR}/expected_results/combine4c.info --lcov -o combine4.actual.info
 cmp combine4.actual.info ${TEST_DIR}/expected_results/combine4.expected.info
 
 # Run ctest_2
@@ -113,37 +116,38 @@ if [ "${ENCODING}" != "../src/latin1_enc.cpp: iso-8859-1" ]; then
 fi
 
 # Test (lcov info - with non-utf8 encoding and fallback)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --fallback-encodings latin1 --lcov -o test8.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --fallback-encodings latin1 --lcov -o test8.actual.info
 cmp test8.actual.info ${TEST_DIR}/expected_results/latin1_test.expected.info
 
 # Test (lcov info - with non-utf8 encoding and no fallback)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test9.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test9.actual.info
 cmp test9.actual.info ${TEST_DIR}/expected_results/latin1_test.expected.info
 
 # Run ctest_3
 ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters # Clear previous test coverage
 ctest -R ctest_3
 
 # Test (lcov info - with inclusive branch filtering)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --include-br-lines-starting-with if else --lcov -o include_branches_sw.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --include-br-lines-starting-with if else --lcov -o include_branches_sw.actual.info
 cmp include_branches_sw.actual.info ${TEST_DIR}/expected_results/include_branches_sw.expected.info
 
 # Test (lcov info - with exclusive branch filtering)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --exclude-br-lines-starting-with for if else --lcov -o exclude_branches_sw.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --exclude-br-lines-starting-with for if else --lcov -o exclude_branches_sw.actual.info
 cmp exclude_branches_sw.actual.info ${TEST_DIR}/expected_results/exclude_branches_sw.expected.info
 
 # Test (lcov info - with smart branch filtering)
-coverage run --append ${TEST_DIR}/fastcov.py --branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --lcov -o filter_branches.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --lcov -o filter_branches.actual.info
 cmp filter_branches.actual.info ${TEST_DIR}/expected_results/filter_branches.expected.info
 
 # Run ctest_all
 ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters # Clear previous test coverage
 ctest # Run all
 
 # Test (multiple tests touching same source)
-coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --source-files ../src/source1.cpp --lcov -o multitest.actual.info
+coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --source-files ../src/source1.cpp --lcov -o multitest.actual.info
 cmp multitest.actual.info ${TEST_DIR}/expected_results/multitest.expected.info
 
 # Write out coverage as xml
+coverage combine
 coverage xml -o coverage.xml
 coverage html # Generate HTML report
diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py
@@ -48,11 +48,16 @@ def test_addLists():
     list1 = [1,2,3,4]
     list2 = [1,0,0,4]
     list3 = [1,0]
+    list4 = [9,9,9]
 
     # Should return a new list
     result = fastcov.addLists(list1, list2)
     assert(result == [2,2,3,8])
 
-    # if lens are mismatched, make sure it chooses the bigger one
+    # if lens are mismatched, make sure it overlays onto bigger one
     result = fastcov.addLists(list1, list3)
-    assert(result == list1)
+    assert(result == [2,2,3,4])
+
+    # if lens are mismatched, make sure it overlays onto bigger one
+    result = fastcov.addLists(list4, list1)
+    assert(result == [10,11,12,4])