Added coverage stats and report generation.

Sample Usage:
  build/android/coverage.py -v --out <output file path> --emma-dir <EMMA file directory> --lines-for-coverage-file <path to file containing lines for coverage>

This CL adds the ability to generate code coverage reports for Java code. To use this script, a JSON file should exist that maps file paths to lists of integers, representing the lines that have changed for each file (i.e. incremental changes).

The generated report contains overall coverage, coverage for the lines specified in the input file, and line by line coverage for each file included in the input file.

BUG=501536

Review URL: https://codereview.chromium.org/1216033009

Cr-Original-Commit-Position: refs/heads/master@{#341437}
Cr-Mirrored-From: https://chromium.googlesource.com/chromium/src
Cr-Mirrored-Commit: 67aa871b52c94f30b52ef19fa71388a54fc3410f
This commit is contained in:
estevenson 2015-07-31 16:34:38 -07:00 коммит произвёл Commit bot
Родитель 2fc1d7542b
Коммит e866860f65
2 изменённых файлов: 557 добавлений и 7 удалений

Просмотреть файл

@ -3,12 +3,29 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""Generates incremental code coverage reports for Java code in Chromium."""
"""Generates incremental code coverage reports for Java code in Chromium.
Usage:
build/android/coverage.py -v --out <output file path> --emma-dir
<EMMA file directory> --lines-for-coverage-file
<path to file containing lines for coverage>
Creates a JSON representation of the overall and file coverage stats and saves
this information to the specified output file.
"""
import argparse
import collections
import json
import logging
import os
import re
import sys
from xml.etree import ElementTree
from pylib.utils import run_tests_helper
NOT_EXECUTABLE = -1
NOT_COVERED = 0
COVERED = 1
@ -67,7 +84,7 @@ class _EmmaHtmlParser(object):
_XPATH_SELECT_PACKAGE_ELEMENTS = './/BODY/TABLE[4]/TR/TD/A'
# Selector to match all <a> elements within the rows that are in the table
# that displays all of the different packages within a class.
# that displays all of the different classes within a package.
_XPATH_SELECT_CLASS_ELEMENTS = './/BODY/TABLE[3]/TR/TD/A'
# Selector to match all <tr> elements within the table containing Java source
@ -158,7 +175,7 @@ class _EmmaHtmlParser(object):
package_link_elements = self._FindElements(
self._index_path, self._XPATH_SELECT_PACKAGE_ELEMENTS)
# Maps file path of package directory (EMMA generated) to package name.
# Ex. emma_dir/f.html: org.chromium.chrome.
# Example: emma_dir/f.html: org.chromium.chrome.
package_links = {
os.path.join(self._base_dir, link.attrib['HREF']): link.text
for link in package_link_elements if 'HREF' in link.attrib
@ -171,10 +188,10 @@ class _EmmaHtmlParser(object):
coverage_file_link_elements = self._FindElements(
package_emma_file_path, self._XPATH_SELECT_CLASS_ELEMENTS)
for coverage_file_element in coverage_file_link_elements:
for class_name_element in coverage_file_link_elements:
emma_coverage_file_path = os.path.join(
self._emma_files_path, coverage_file_element.attrib['HREF'])
full_package_name = '%s.%s' % (package_name, coverage_file_element.text)
self._emma_files_path, class_name_element.attrib['HREF'])
full_package_name = '%s.%s' % (package_name, class_name_element.text)
package_to_emma[full_package_name] = emma_coverage_file_path
return package_to_emma
@ -194,3 +211,274 @@ class _EmmaHtmlParser(object):
file_contents = f.read().decode('ISO-8859-1').encode('UTF-8')
root = ElementTree.fromstring(file_contents)
return root.findall(xpath_selector)
class _EmmaCoverageStats(object):
"""Computes code coverage stats for Java code using the coverage tool EMMA.
This class provides an API that allows users to capture absolute code coverage
and code coverage on a subset of lines for each Java source file. Coverage
reports are generated in JSON format.
"""
# Regular expression to get package name from Java package statement.
RE_PACKAGE_MATCH_GROUP = 'package'
RE_PACKAGE = re.compile(r'package (?P<%s>[\w.]*);' % RE_PACKAGE_MATCH_GROUP)
def __init__(self, emma_file_base_dir, files_for_coverage):
"""Initialize _EmmaCoverageStats.
Args:
emma_file_base_dir: String representing the path to the base directory
where EMMA HTML coverage files are stored, i.e. parent of index.html.
files_for_coverage: A list of Java source code file paths to get EMMA
coverage for.
"""
self._emma_parser = _EmmaHtmlParser(emma_file_base_dir)
self._source_to_emma = self._GetSourceFileToEmmaFileDict(files_for_coverage)
def GetCoverageDict(self, lines_for_coverage):
"""Returns a dict containing detailed coverage information.
Gets detailed coverage stats for each file specified in the
|lines_for_coverage| dict and the total incremental number of lines covered
and executable for all files in |lines_for_coverage|.
Args:
lines_for_coverage: A dict mapping Java source file paths to lists of line
numbers.
Returns:
A dict containing coverage stats for the given dict of files and lines.
Contains absolute coverage stats for each file, coverage stats for each
file's lines specified in |lines_for_coverage|, line by line coverage
for each file, and overall coverage stats for the lines specified in
|lines_for_coverage|.
"""
file_coverage = {}
for file_path, line_numbers in lines_for_coverage.iteritems():
file_coverage[file_path] = self.GetCoverageDictForFile(
file_path, line_numbers)
covered_statuses = [s['incremental'] for s in file_coverage.itervalues()]
num_covered_lines = sum(s['covered'] for s in covered_statuses)
num_total_lines = sum(s['total'] for s in covered_statuses)
return {
'files': file_coverage,
'patch': {
'incremental': {
'covered': num_covered_lines,
'total': num_total_lines
}
}
}
def GetCoverageDictForFile(self, file_path, line_numbers):
"""Returns a dict containing detailed coverage info for the given file.
Args:
file_path: The path to the Java source file that we want to create the
coverage dict for.
line_numbers: A list of integer line numbers to retrieve additional stats
for.
Returns:
A dict containing absolute, incremental, and line by line coverage for
a file.
"""
total_line_coverage = self._GetLineCoverageForFile(file_path)
incremental_line_coverage = [line for line in total_line_coverage
if line.lineno in line_numbers]
line_by_line_coverage = [
{
'line': line.source,
'coverage': line.covered_status,
'changed': line.lineno in line_numbers,
}
for line in total_line_coverage
]
total_covered_lines, total_lines = (
self.GetSummaryStatsForLines(total_line_coverage))
incremental_covered_lines, incremental_total_lines = (
self.GetSummaryStatsForLines(incremental_line_coverage))
file_coverage_stats = {
'absolute': {
'covered': total_covered_lines,
'total': total_lines
},
'incremental': {
'covered': incremental_covered_lines,
'total': incremental_total_lines
},
'source': line_by_line_coverage,
}
return file_coverage_stats
def GetSummaryStatsForLines(self, line_coverage):
"""Gets summary stats for a given list of LineCoverage objects.
Args:
line_coverage: A list of LineCoverage objects.
Returns:
A tuple containing the number of lines that are covered and the total
number of lines that are executable, respectively
"""
partially_covered_sum = 0
covered_status_totals = {COVERED: 0, NOT_COVERED: 0, PARTIALLY_COVERED: 0}
for line in line_coverage:
status = line.covered_status
if status == NOT_EXECUTABLE:
continue
covered_status_totals[status] += 1
if status == PARTIALLY_COVERED:
partially_covered_sum += line.fractional_line_coverage
total_covered = covered_status_totals[COVERED] + partially_covered_sum
total_lines = sum(covered_status_totals.values())
return total_covered, total_lines
def _GetLineCoverageForFile(self, file_path):
"""Gets a list of LineCoverage objects corresponding to the given file path.
Args:
file_path: String representing the path to the Java source file.
Returns:
A list of LineCoverage objects, or None if there is no EMMA file
for the given Java source file.
"""
if file_path in self._source_to_emma:
emma_file = self._source_to_emma[file_path]
return self._emma_parser.GetLineCoverage(emma_file)
else:
logging.warning(
'No code coverage data for %s, skipping.', file_path)
return None
def _GetSourceFileToEmmaFileDict(self, files):
"""Gets a dict used to correlate Java source files with EMMA HTML files.
This method gathers the information needed to correlate EMMA HTML
files with Java source files. EMMA XML and plain text reports do not provide
line by line coverage data, so HTML reports must be used instead.
Unfortunately, the HTML files that are created are given garbage names
(i.e 1.html) so we need to manually correlate EMMA HTML files
with the original Java source files.
Args:
files: A list of file names for which coverage information is desired.
Returns:
A dict mapping Java source file paths to EMMA HTML file paths.
"""
# Maps Java source file paths to package names.
# Example: /usr/code/file.java -> org.chromium.file.java.
source_to_package = {}
for file_path in files:
package = self.GetPackageNameFromFile(file_path)
if package:
source_to_package[file_path] = package
else:
logging.warning("Skipping %s because it doesn\'t have a package "
"statement.", file_path)
# Maps package names to EMMA report HTML files.
# Example: org.chromium.file.java -> out/coverage/1a.html.
package_to_emma = self._emma_parser.GetPackageNameToEmmaFileDict()
# Finally, we have a dict mapping Java file paths to EMMA report files.
# Example: /usr/code/file.java -> out/coverage/1a.html.
source_to_emma = {source: package_to_emma.get(package)
for source, package in source_to_package.iteritems()}
return source_to_emma
@staticmethod
def NeedsCoverage(file_path):
"""Checks to see if the file needs to be analyzed for code coverage.
Args:
file_path: A string representing path to the file.
Returns:
True for Java files that exist, False for all others.
"""
if os.path.splitext(file_path)[1] == '.java' and os.path.exists(file_path):
return True
else:
logging.debug(
'Skipping file %s, cannot compute code coverage.', file_path)
return False
@staticmethod
def GetPackageNameFromFile(file_path):
"""Gets the full package name including the file name for a given file path.
Args:
file_path: String representing the path to the Java source file.
Returns:
A string representing the full package name with file name appended or
None if there is no package statement in the file.
"""
with open(file_path) as f:
file_content = f.read()
package_match = re.search(_EmmaCoverageStats.RE_PACKAGE, file_content)
if package_match:
package = package_match.group(_EmmaCoverageStats.RE_PACKAGE_MATCH_GROUP)
file_name = os.path.basename(file_path)
return '%s.%s' % (package, file_name)
else:
return None
def GenerateCoverageReport(line_coverage_file, out_file_path, coverage_dir):
"""Generates a coverage report for a given set of lines.
Writes the results of the coverage analysis to the file specified by
|out_file_path|.
Args:
line_coverage_file: The path to a file which contains a dict mapping file
names to lists of line numbers. Example: {file1: [1, 2, 3], ...} means
that we should compute coverage information on lines 1 - 3 for file1.
out_file_path: A string representing the location to write the JSON report.
coverage_dir: A string representing the file path where the EMMA
HTML coverage files are located (i.e. folder where index.html is located).
"""
with open(line_coverage_file) as f:
potential_files_for_coverage = json.load(f)
files_for_coverage = {f: lines
for f, lines in potential_files_for_coverage.iteritems()
if _EmmaCoverageStats.NeedsCoverage(f)}
if not files_for_coverage:
logging.info('No Java files requiring coverage were included in %s.',
line_coverage_file)
return
code_coverage = _EmmaCoverageStats(coverage_dir, files_for_coverage.keys())
coverage_results = code_coverage.GetCoverageDict(
files_for_coverage)
with open(out_file_path, 'w+') as out_status_file:
json.dump(coverage_results, out_status_file)
def main():
argparser = argparse.ArgumentParser()
argparser.add_argument('--out', required=True, type=str,
help='Report output file path.')
argparser.add_argument('--emma-dir', required=True, type=str,
help='EMMA HTML report directory.')
argparser.add_argument('--lines-for-coverage-file', required=True, type=str,
help='File containing a JSON object. Should contain a '
'dict mapping file names to lists of line numbers of '
'code for which coverage information is desired.')
argparser.add_argument('-v', '--verbose', action='count',
help='Print verbose log information.')
args = argparser.parse_args()
run_tests_helper.SetLogLevel(args.verbose)
GenerateCoverageReport(args.lines_for_coverage_file, args.out, args.emma_dir)
if __name__ == '__main__':
sys.exit(main())

Просмотреть файл

@ -3,6 +3,7 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import json
import os
import sys
import unittest
@ -15,6 +16,15 @@ sys.path.append(os.path.join(
constants.DIR_SOURCE_ROOT, 'third_party', 'pymock'))
import mock # pylint: disable=F0401
EMPTY_COVERAGE_STATS_DICT = {
'files': {},
'patch': {
'incremental': {
'covered': 0, 'total': 0
}
}
}
class _EmmaHtmlParserTest(unittest.TestCase):
"""Tests for _EmmaHtmlParser.
@ -223,6 +233,11 @@ class _EmmaHtmlParserTest(unittest.TestCase):
return_dict = self.parser.GetPackageNameToEmmaFileDict()
self.assertDictEqual({}, return_dict)
def testGetPackageNameToEmmaFileDict_badFilePath(self):
self.parser._FindElements = mock.Mock(return_value=[])
return_dict = self.parser.GetPackageNameToEmmaFileDict()
self.assertEqual(return_dict, {})
def testGetLineCoverage_status_basic(self):
line_coverage = self.GetLineCoverageWithFakeElements([self.covered_tr_html])
self.assertEqual(line_coverage[0].covered_status,
@ -284,9 +299,255 @@ class _EmmaHtmlParserTest(unittest.TestCase):
return self.parser.GetLineCoverage('fake_path')
class _EmmaCoverageStatsTest(unittest.TestCase):
"""Tests for _EmmaCoverageStats."""
def setUp(self):
self.good_source_to_emma = {
'/path/to/1/File1.java': '/emma/1.html',
'/path/2/File2.java': '/emma/2.html',
'/path/2/File3.java': '/emma/3.html'
}
self.line_coverage = [
emma_coverage_stats.LineCoverage(
1, '', emma_coverage_stats.COVERED, 1.0),
emma_coverage_stats.LineCoverage(
2, '', emma_coverage_stats.COVERED, 1.0),
emma_coverage_stats.LineCoverage(
3, '', emma_coverage_stats.NOT_EXECUTABLE, 1.0),
emma_coverage_stats.LineCoverage(
4, '', emma_coverage_stats.NOT_COVERED, 1.0),
emma_coverage_stats.LineCoverage(
5, '', emma_coverage_stats.PARTIALLY_COVERED, 0.85),
emma_coverage_stats.LineCoverage(
6, '', emma_coverage_stats.PARTIALLY_COVERED, 0.20)
]
self.lines_for_coverage = [1, 3, 5, 6]
with mock.patch('emma_coverage_stats._EmmaHtmlParser._FindElements',
return_value=[]):
self.simple_coverage = emma_coverage_stats._EmmaCoverageStats(
'fake_dir', {})
def testInit(self):
coverage_stats = self.simple_coverage
self.assertIsInstance(coverage_stats._emma_parser,
emma_coverage_stats._EmmaHtmlParser)
self.assertIsInstance(coverage_stats._source_to_emma, dict)
def testNeedsCoverage_withExistingJavaFile(self):
test_file = '/path/to/file/File.java'
with mock.patch('os.path.exists', return_value=True):
self.assertTrue(
emma_coverage_stats._EmmaCoverageStats.NeedsCoverage(test_file))
def testNeedsCoverage_withNonJavaFile(self):
test_file = '/path/to/file/File.c'
with mock.patch('os.path.exists', return_value=True):
self.assertFalse(
emma_coverage_stats._EmmaCoverageStats.NeedsCoverage(test_file))
def testNeedsCoverage_fileDoesNotExist(self):
test_file = '/path/to/file/File.java'
with mock.patch('os.path.exists', return_value=False):
self.assertFalse(
emma_coverage_stats._EmmaCoverageStats.NeedsCoverage(test_file))
def testGetPackageNameFromFile_basic(self):
test_file_text = """// Test Copyright
package org.chromium.chrome.browser;
import android.graphics.RectF;"""
result_package, _ = MockOpenForFunction(
emma_coverage_stats._EmmaCoverageStats.GetPackageNameFromFile,
[test_file_text], file_path='/path/to/file/File.java')
self.assertEqual(result_package, 'org.chromium.chrome.browser.File.java')
def testGetPackageNameFromFile_noPackageStatement(self):
result_package, _ = MockOpenForFunction(
emma_coverage_stats._EmmaCoverageStats.GetPackageNameFromFile,
['not a package statement'], file_path='/path/to/file/File.java')
self.assertIsNone(result_package)
def testGetSummaryStatsForLines_basic(self):
covered, total = self.simple_coverage.GetSummaryStatsForLines(
self.line_coverage)
self.assertEqual(covered, 3.05)
self.assertEqual(total, 5)
def testGetSourceFileToEmmaFileDict(self):
package_names = {
'/path/to/1/File1.java': 'org.fake.one.File1.java',
'/path/2/File2.java': 'org.fake.File2.java',
'/path/2/File3.java': 'org.fake.File3.java'
}
package_to_emma = {
'org.fake.one.File1.java': '/emma/1.html',
'org.fake.File2.java': '/emma/2.html',
'org.fake.File3.java': '/emma/3.html'
}
with mock.patch('os.path.exists', return_value=True):
coverage_stats = self.simple_coverage
coverage_stats._emma_parser.GetPackageNameToEmmaFileDict = mock.MagicMock(
return_value=package_to_emma)
coverage_stats.GetPackageNameFromFile = lambda x: package_names[x]
result_dict = coverage_stats._GetSourceFileToEmmaFileDict(
package_names.keys())
self.assertDictEqual(result_dict, self.good_source_to_emma)
def testGetLineCoverageForFile_basic(self):
java_file_path = '/path/to/1/File1.java'
line_coverage = emma_coverage_stats.LineCoverage(
1, '', emma_coverage_stats.COVERED, 1.0)
expected_line_coverage = list(line_coverage)
coverage_stats = self.simple_coverage
coverage_stats._source_to_emma = self.good_source_to_emma
coverage_stats._emma_parser.GetLineCoverage = mock.MagicMock(
return_value=expected_line_coverage)
coverage_info = coverage_stats._GetLineCoverageForFile(java_file_path)
self.assertListEqual(coverage_info, expected_line_coverage)
def testGetLineCoverageForFile_noInfo(self):
with mock.patch('os.path.exists', return_value=False):
coverage_info = self.simple_coverage._GetLineCoverageForFile('fake_path')
self.assertIsNone(coverage_info)
def testGetCoverageDictForFile(self):
line_coverage = self.line_coverage
self.simple_coverage._GetLineCoverageForFile = mock.Mock(
return_value=line_coverage)
lines = self.lines_for_coverage
expected_dict = {
'absolute': {
'covered': 3.05,
'total': 5
},
'incremental': {
'covered': 2.05,
'total': 3
},
'source': [
{
'line': line_coverage[0].source,
'coverage': line_coverage[0].covered_status,
'changed': True
},
{
'line': line_coverage[1].source,
'coverage': line_coverage[1].covered_status,
'changed': False
},
{
'line': line_coverage[2].source,
'coverage': line_coverage[2].covered_status,
'changed': True
},
{
'line': line_coverage[3].source,
'coverage': line_coverage[3].covered_status,
'changed': False
},
{
'line': line_coverage[4].source,
'coverage': line_coverage[4].covered_status,
'changed': True
},
{
'line': line_coverage[5].source,
'coverage': line_coverage[5].covered_status,
'changed': True
}
]
}
result_dict = self.simple_coverage.GetCoverageDictForFile(
line_coverage, lines)
self.assertDictEqual(result_dict, expected_dict)
def testGetCoverageDictForFile_emptyCoverage(self):
expected_dict = {
'absolute': {'covered': 0, 'total': 0},
'incremental': {'covered': 0, 'total': 0},
'source': []
}
self.simple_coverage._GetLineCoverageForFile = mock.Mock(return_value=[])
result_dict = self.simple_coverage.GetCoverageDictForFile('fake_dir', {})
self.assertDictEqual(result_dict, expected_dict)
def testGetCoverageDictFor_basic(self):
files_for_coverage = {
'/path/to/1/File1.java': [1, 3, 4],
'/path/2/File2.java': [1, 2]
}
coverage_info = {
'/path/to/1/File1.java': [
emma_coverage_stats.LineCoverage(
1, '', emma_coverage_stats.COVERED, 1.0),
emma_coverage_stats.LineCoverage(
2, '', emma_coverage_stats.PARTIALLY_COVERED, 0.5),
emma_coverage_stats.LineCoverage(
3, '', emma_coverage_stats.NOT_EXECUTABLE, 1.0),
emma_coverage_stats.LineCoverage(
4, '', emma_coverage_stats.COVERED, 1.0)
],
'/path/2/File2.java': [
emma_coverage_stats.LineCoverage(
1, '', emma_coverage_stats.NOT_COVERED, 1.0),
emma_coverage_stats.LineCoverage(
2, '', emma_coverage_stats.COVERED, 1.0)
]
}
expected_dict = {
'files': {
'/path/2/File2.java': {
'absolute': {'covered': 1, 'total': 2},
'incremental': {'covered': 1, 'total': 2},
'source': [{'changed': True, 'coverage': 0, 'line': ''},
{'changed': True, 'coverage': 1, 'line': ''}]
},
'/path/to/1/File1.java': {
'absolute': {'covered': 2.5, 'total': 3},
'incremental': {'covered': 2, 'total': 2},
'source': [{'changed': True, 'coverage': 1, 'line': ''},
{'changed': False, 'coverage': 2, 'line': ''},
{'changed': True, 'coverage': -1, 'line': ''},
{'changed': True, 'coverage': 1, 'line': ''}]
}
},
'patch': {'incremental': {'covered': 3, 'total': 4}}
}
# Return the relevant coverage info for each file. We aren't testing
# _GetCoverageStatusForFile here.
self.simple_coverage._GetLineCoverageForFile = lambda x: coverage_info[x]
result_dict = self.simple_coverage.GetCoverageDict(
files_for_coverage)
self.assertDictEqual(result_dict, expected_dict)
def testGetCoverageDict_noCoverage(self):
result_dict = self.simple_coverage.GetCoverageDict({})
self.assertDictEqual(result_dict, EMPTY_COVERAGE_STATS_DICT)
class EmmaCoverageStatsGenerateCoverageReport(unittest.TestCase):
"""Tests for GenerateCoverageReport."""
def testGenerateCoverageReport_missingJsonFile(self):
with self.assertRaises(IOError):
with mock.patch('os.path.exists', return_value=False):
emma_coverage_stats.GenerateCoverageReport('', '', '')
def testGenerateCoverageReport_invalidJsonFile(self):
with self.assertRaises(ValueError):
with mock.patch('os.path.exists', return_value=True):
MockOpenForFunction(emma_coverage_stats.GenerateCoverageReport, [''],
line_coverage_file='', out_file_path='',
coverage_dir='')
def MockOpenForFunction(func, side_effects, **kwargs):
"""Allows easy mock open and read for callables that open multiple files.
Will mock the python open function in a way such that each time read() is
called on an open file, the next element in |side_effects| is returned. This
makes it easier to test functions that call open() multiple times.
Args:
func: The callable to invoke once mock files are setup.
side_effects: A list of return values for each file to return once read.
@ -305,4 +566,5 @@ def MockOpenForFunction(func, side_effects, **kwargs):
if __name__ == '__main__':
unittest.main()
# Suppress logging messages.
unittest.main(buffer=True)