From d616eb6bd55b8443423e62ae5e5e050bfa764420 Mon Sep 17 00:00:00 2001 From: Bryan Gillespie Date: Mon, 18 May 2020 17:14:16 -0400 Subject: [PATCH] Reduce memory footprint of fastcov Description: - Fix #43 - Bump version to 1.6 - Refactor GCDA processing to do incremental report merging - Refactor GCDA processing to use multiprocess.Process instead of threads - When branch lists mismatch, overlay smaller list on bigger list --- fastcov.py | 82 ++++++++++++----------- setup.cfg | 2 +- test/functional/cmake_project/.coveragerc | 2 + test/functional/run_all.sh | 50 +++++++------- test/unit/test_utils.py | 9 ++- 5 files changed, 81 insertions(+), 64 deletions(-) create mode 100644 test/functional/cmake_project/.coveragerc diff --git a/fastcov.py b/fastcov.py index 03ad38e..d96fe1a 100755 --- a/fastcov.py +++ b/fastcov.py @@ -31,7 +31,7 @@ import subprocess import multiprocessing -FASTCOV_VERSION = (1,5) +FASTCOV_VERSION = (1,6) MINIMUM_PYTHON = (3,5) MINIMUM_GCOV = (9,0,0) @@ -100,36 +100,52 @@ def findCoverageFiles(cwd, coverage_files, use_gcno): logging.debug("Coverage files found:\n %s", "\n ".join(coverage_files)) return coverage_files -def gcovWorker(cwd, gcov, files, chunk, gcov_filter_options, branch_coverage): +def gcovWorker(q, args, chunk, gcov_filter_options): + base_report = { + "sources": {} + } + gcov_args = "-it" - if branch_coverage: + if args.branchcoverage or args.xbranchcoverage: gcov_args += "b" - p = subprocess.Popen([gcov, gcov_args] + chunk, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) + p = subprocess.Popen([args.gcov, gcov_args] + chunk, cwd=args.cdirectory, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL) for line in iter(p.stdout.readline, b''): intermediate_json = json.loads(line.decode(sys.stdout.encoding)) - intermediate_json_files = processGcovs(cwd, intermediate_json["files"], gcov_filter_options) + intermediate_json_files = processGcovs(args.cdirectory, intermediate_json["files"], gcov_filter_options) for f in intermediate_json_files: - files.append(f) #thread safe, there might be a better way to do this though + distillSource(f, base_report["sources"], args.test_name, args.xbranchcoverage) GCOVS_TOTAL.append(len(intermediate_json["files"])) GCOVS_SKIPPED.append(len(intermediate_json["files"])-len(intermediate_json_files)) + p.wait() + q.put(base_report) -def processGcdas(cwd, gcov, jobs, coverage_files, gcov_filter_options, branch_coverage, min_chunk_size): - chunk_size = max(min_chunk_size, int(len(coverage_files) / jobs) + 1) +def processGcdas(args, coverage_files, gcov_filter_options): + chunk_size = max(args.minimum_chunk, int(len(coverage_files) / args.jobs) + 1) - threads = [] - intermediate_json_files = [] + processes = [] + q = multiprocessing.Queue() for chunk in chunks(coverage_files, chunk_size): - t = threading.Thread(target=gcovWorker, args=(cwd, gcov, intermediate_json_files, chunk, gcov_filter_options, branch_coverage)) - threads.append(t) - t.start() + p = multiprocessing.Process(target=gcovWorker, args=(q, args, chunk, gcov_filter_options)) + processes.append(p) + p.start() - logging.info("Spawned {} gcov threads, each processing at most {} coverage files".format(len(threads), chunk_size)) - for t in threads: - t.join() + logging.info("Spawned {} gcov processes, each processing at most {} coverage files".format(len(processes), chunk_size)) + + fastcov_jsons = [] + for p in processes: + fj = q.get() + fastcov_jsons.append(fj) + + for p in processes: + p.join() - return intermediate_json_files + base_fastcov = fastcov_jsons.pop() + for fj in fastcov_jsons: + combineReports(base_fastcov, fj) + + return base_fastcov def processGcov(cwd, gcov, files, gcov_filter_options): # Add absolute path @@ -377,16 +393,6 @@ def distillSource(source_raw, sources, test_name, include_exceptional_branches): for line in source_raw["lines"]: distillLine(line, sources[source_name][test_name]["lines"], sources[source_name][test_name]["branches"], include_exceptional_branches) -def distillReport(report_raw, args): - report_json = { - "sources": {} - } - - for source in report_raw: - distillSource(source, report_json["sources"], args.test_name, args.xbranchcoverage) - - return report_json - def dumpToJson(intermediate, output): with open(output, "w") as f: json.dump(intermediate, f) @@ -411,12 +417,16 @@ def addDicts(dict1, dict2): def addLists(list1, list2): """Add lists together by value. i.e. addLists([1,1], [2,2]) == [3,3]""" - if len(list1) == len(list2): - return [b1 + b2 for b1, b2 in zip(list1, list2)] - else: - # One report had different number branch measurements than the other, print a warning - logging.warning("Possible loss of correctness. Different number of branches for same line when combining reports ({} vs {})\n".format(list1, list2)) - return list1 if len(list1) > len(list2) else list2 + # Find big list and small list + blist, slist = list(list2), list(list1) + if len(list1) > len(list2): + blist, slist = slist, blist + + # Overlay small list onto big list + for i, b in enumerate(slist): + blist[i] += b + + return blist def combineReports(base, overlay): for source, scov in overlay["sources"].items(): @@ -645,16 +655,12 @@ def main(): # Fire up one gcov per cpu and start processing gcdas gcov_filter_options = getGcovFilterOptions(args) - intermediate_json_files = processGcdas(args.cdirectory, args.gcov, args.jobs, coverage_files, gcov_filter_options, args.branchcoverage or args.xbranchcoverage, args.minimum_chunk) + fastcov_json = processGcdas(args, coverage_files, gcov_filter_options) # Summarize processing results gcov_total = sum(GCOVS_TOTAL) gcov_skipped = sum(GCOVS_SKIPPED) logging.info("Processed {} .gcov files ({} total, {} skipped)".format(gcov_total - gcov_skipped, gcov_total, gcov_skipped)) - - # Distill all the extraneous info gcov gives us down to the core report - fastcov_json = distillReport(intermediate_json_files, args) - logging.info("Aggregated raw gcov JSON into fastcov JSON report") logging.debug("Final report will contain coverage for the following %d source files:\n %s", len(fastcov_json["sources"]), "\n ".join(fastcov_json["sources"])) # Scan for exclusion markers diff --git a/setup.cfg b/setup.cfg index 2a2d150..cb81a9f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,7 @@ [metadata] name = fastcov -version = 1.5 +version = 1.6 description = A massively parallel gcov wrapper for generating intermediate coverage formats fast author = Bryan Gillespie author-email = rpgillespie6@gmail.com diff --git a/test/functional/cmake_project/.coveragerc b/test/functional/cmake_project/.coveragerc new file mode 100644 index 0000000..7781a1d --- /dev/null +++ b/test/functional/cmake_project/.coveragerc @@ -0,0 +1,2 @@ +[run] +concurrency = multiprocessing \ No newline at end of file diff --git a/test/functional/run_all.sh b/test/functional/run_all.sh index 089e0ad..43f7e9f 100755 --- a/test/functional/run_all.sh +++ b/test/functional/run_all.sh @@ -21,12 +21,15 @@ ninja # Erase coverage coverage erase +# Get coverage config for multiprocessing +cp ../.coveragerc . + # Generate GCDA ctest # Test zerocounters test `find . -name *.gcda | wc -l` -ne 0 -coverage run --append ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters +coverage run -a ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters test `find . -name *.gcda | wc -l` -eq 0 # Run ctest_1 @@ -34,71 +37,71 @@ ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters # Clear previous test cover ctest -R ctest_1 # Test (basic report generation - no branches) -coverage run --append ${TEST_DIR}/fastcov.py --gcov gcov-9 --verbose --exclude cmake_project/test/ --lcov -o test1.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --gcov gcov-9 --verbose --exclude cmake_project/test/ --lcov -o test1.actual.info cmp test1.actual.info ${TEST_DIR}/expected_results/test1.expected.info -coverage run --append ${TEST_DIR}/fastcov.py --gcov gcov-9 --exclude cmake_project/test/ -o test1.actual.fastcov.json +coverage run -a ${TEST_DIR}/fastcov.py --gcov gcov-9 --exclude cmake_project/test/ -o test1.actual.fastcov.json ${TEST_DIR}/json_cmp.py test1.actual.fastcov.json ${TEST_DIR}/expected_results/test1.expected.fastcov.json # Test the setting of the "testname" field -coverage run --append ${TEST_DIR}/fastcov.py -t FunctionalTest1 --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test1.tn.actual.info +coverage run -a ${TEST_DIR}/fastcov.py -t FunctionalTest1 --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test1.tn.actual.info cmp test1.tn.actual.info ${TEST_DIR}/expected_results/test1.tn.expected.info # Test (basic report info - with branches) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test2.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test2.actual.info cmp test2.actual.info ${TEST_DIR}/expected_results/test2.expected.info -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ -o test2.actual.fastcov.json +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ -o test2.actual.fastcov.json ${TEST_DIR}/json_cmp.py test2.actual.fastcov.json ${TEST_DIR}/expected_results/test2.expected.fastcov.json # Test (basic lcov info - with branches; equivalent --include) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --include src/ --lcov -o test3.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --include src/ --lcov -o test3.actual.info cmp test3.actual.info ${TEST_DIR}/expected_results/test2.expected.info # Test (basic lcov info - with branches; equivalent --exclude-gcda) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --exclude-gcda test1.cpp.gcda --lcov -o test4.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --exclude-gcda test1.cpp.gcda --lcov -o test4.actual.info cmp test4.actual.info ${TEST_DIR}/expected_results/test2.expected.info # Test (basic lcov info - with branches; equivalent --source-files) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --source-files ../src/source1.cpp ../src/source2.cpp --lcov -o test5.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --verbose --source-files ../src/source1.cpp ../src/source2.cpp --lcov -o test5.actual.info cmp test5.actual.info ${TEST_DIR}/expected_results/test2.expected.info # Test (basic lcov info - with branches; gcno untested file coverage) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --source-files ../src/source1.cpp ../src/source2.cpp ../src/untested.cpp --lcov -o untested.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --source-files ../src/source1.cpp ../src/source2.cpp ../src/untested.cpp --lcov -o untested.actual.info cmp untested.actual.info ${TEST_DIR}/expected_results/untested.expected.info # Test (basic lcov info - with branches; gcno untested file coverage; equivalent --gcda-files) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --gcda-files ./test/CMakeFiles/test1.dir/__/src/source1.cpp.gcno ./test/CMakeFiles/test1.dir/__/src/source2.cpp.gcno ./src/CMakeFiles/untested.dir/untested.cpp.gcno --lcov -o untested2.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --gcda-files ./test/CMakeFiles/test1.dir/__/src/source1.cpp.gcno ./test/CMakeFiles/test1.dir/__/src/source2.cpp.gcno ./src/CMakeFiles/untested.dir/untested.cpp.gcno --lcov -o untested2.actual.info cmp untested2.actual.info ${TEST_DIR}/expected_results/untested.expected.info # Test (gcov version fail) -if coverage run --append ${TEST_DIR}/fastcov.py --gcov ${TEST_DIR}/fake-gcov.sh ; then +if coverage run -a ${TEST_DIR}/fastcov.py --gcov ${TEST_DIR}/fake-gcov.sh ; then echo "Expected gcov version check to fail" exit 1 fi # Combine operation -coverage run --append ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/test2.expected.info ${TEST_DIR}/expected_results/test1.tn.expected.info --lcov -o combine1.actual.info +coverage run -a ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/test2.expected.info ${TEST_DIR}/expected_results/test1.tn.expected.info --lcov -o combine1.actual.info cmp combine1.actual.info ${TEST_DIR}/expected_results/combine1.expected.info # Combine operation - Mix and Match json/info -coverage run --append ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/test2.expected.fastcov.json ${TEST_DIR}/expected_results/test1.tn.expected.info --lcov -o combine1.actual.info +coverage run -a ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/test2.expected.fastcov.json ${TEST_DIR}/expected_results/test1.tn.expected.info --lcov -o combine1.actual.info cmp combine1.actual.info ${TEST_DIR}/expected_results/combine1.expected.info # Combine operation- source files across coverage reports ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --source-files ../src/source1.cpp --lcov -o combine.source1.actual.info ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --process-gcno --source-files ../src/source2.cpp --lcov -o combine.source2.actual.info -coverage run --append ${TEST_DIR}/fastcov.py -C combine.source1.actual.info combine.source2.actual.info --lcov -o combine2.actual.info +coverage run -a ${TEST_DIR}/fastcov.py -C combine.source1.actual.info combine.source2.actual.info --lcov -o combine2.actual.info cmp combine2.actual.info ${TEST_DIR}/expected_results/combine2.expected.info # Combine files with different function inclusion -coverage run --append ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/combine3b.info ${TEST_DIR}/expected_results/combine3a.info --lcov -o combine3.actual.info +coverage run -a ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/combine3b.info ${TEST_DIR}/expected_results/combine3a.info --lcov -o combine3.actual.info cmp combine3.actual.info ${TEST_DIR}/expected_results/combine3.expected.info # Combine files with different test names # Expected result generated with: # lcov -a combine4a.info -a combine4b.info -a combine4c.info --rc lcov_branch_coverage=1 -o combine4.expected.info -coverage run --append ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/combine4a.info ${TEST_DIR}/expected_results/combine4b.info ${TEST_DIR}/expected_results/combine4c.info --lcov -o combine4.actual.info +coverage run -a ${TEST_DIR}/fastcov.py -C ${TEST_DIR}/expected_results/combine4a.info ${TEST_DIR}/expected_results/combine4b.info ${TEST_DIR}/expected_results/combine4c.info --lcov -o combine4.actual.info cmp combine4.actual.info ${TEST_DIR}/expected_results/combine4.expected.info # Run ctest_2 @@ -113,11 +116,11 @@ if [ "${ENCODING}" != "../src/latin1_enc.cpp: iso-8859-1" ]; then fi # Test (lcov info - with non-utf8 encoding and fallback) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --fallback-encodings latin1 --lcov -o test8.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --fallback-encodings latin1 --lcov -o test8.actual.info cmp test8.actual.info ${TEST_DIR}/expected_results/latin1_test.expected.info # Test (lcov info - with non-utf8 encoding and no fallback) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test9.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude cmake_project/test/ --lcov -o test9.actual.info cmp test9.actual.info ${TEST_DIR}/expected_results/latin1_test.expected.info # Run ctest_3 @@ -125,15 +128,15 @@ ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters # Clear previous test covera ctest -R ctest_3 # Test (lcov info - with inclusive branch filtering) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --include-br-lines-starting-with if else --lcov -o include_branches_sw.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --include-br-lines-starting-with if else --lcov -o include_branches_sw.actual.info cmp include_branches_sw.actual.info ${TEST_DIR}/expected_results/include_branches_sw.expected.info # Test (lcov info - with exclusive branch filtering) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --exclude-br-lines-starting-with for if else --lcov -o exclude_branches_sw.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --exclude-br-lines-starting-with for if else --lcov -o exclude_branches_sw.actual.info cmp exclude_branches_sw.actual.info ${TEST_DIR}/expected_results/exclude_branches_sw.expected.info # Test (lcov info - with smart branch filtering) -coverage run --append ${TEST_DIR}/fastcov.py --branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --lcov -o filter_branches.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --branch-coverage --gcov gcov-9 --exclude /usr/include cmake_project/test/ --lcov -o filter_branches.actual.info cmp filter_branches.actual.info ${TEST_DIR}/expected_results/filter_branches.expected.info # Run ctest_all @@ -141,9 +144,10 @@ ${TEST_DIR}/fastcov.py --gcov gcov-9 --zerocounters # Clear previous test covera ctest # Run all # Test (multiple tests touching same source) -coverage run --append ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --source-files ../src/source1.cpp --lcov -o multitest.actual.info +coverage run -a ${TEST_DIR}/fastcov.py --exceptional-branch-coverage --gcov gcov-9 --source-files ../src/source1.cpp --lcov -o multitest.actual.info cmp multitest.actual.info ${TEST_DIR}/expected_results/multitest.expected.info # Write out coverage as xml +coverage combine coverage xml -o coverage.xml coverage html # Generate HTML report \ No newline at end of file diff --git a/test/unit/test_utils.py b/test/unit/test_utils.py index 5de9afa..6966ea6 100644 --- a/test/unit/test_utils.py +++ b/test/unit/test_utils.py @@ -48,11 +48,16 @@ def test_addLists(): list1 = [1,2,3,4] list2 = [1,0,0,4] list3 = [1,0] + list4 = [9,9,9] # Should return a new list result = fastcov.addLists(list1, list2) assert(result == [2,2,3,8]) - # if lens are mismatched, make sure it chooses the bigger one + # if lens are mismatched, make sure it overlays onto bigger one result = fastcov.addLists(list1, list3) - assert(result == list1) \ No newline at end of file + assert(result == [2,2,3,4]) + + # if lens are mismatched, make sure it overlays onto bigger one + result = fastcov.addLists(list4, list1) + assert(result == [10,11,12,4]) \ No newline at end of file