From 6a2a7a239275d1064479e57e252af3f740167ded Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Mon, 15 May 2017 17:42:11 +0800 Subject: [PATCH] Enable codecov.io and add coverage grouping. --- .coveragerc | 2 +- .gitignore | 4 +- test/runner/lib/cover.py | 100 +++++++++++++++++++++++++----- test/runner/lib/executor.py | 2 +- test/runner/test.py | 19 ++++++ test/utils/shippable/shippable.sh | 26 +++++++- 6 files changed, 131 insertions(+), 22 deletions(-) diff --git a/.coveragerc b/.coveragerc index 698302a7a1..f205885c9c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -20,5 +20,5 @@ data_file = test/results/coverage/coverage omit = */python*/dist-packages/* */python*/site-packages/* - */python*/distutils + */python*/distutils/* */pytest diff --git a/.gitignore b/.gitignore index ece590c4d5..3f19acdfc4 100644 --- a/.gitignore +++ b/.gitignore @@ -68,8 +68,8 @@ packaging/release/ansible_release /.cache/ /test/results/coverage/*=coverage.* /test/results/coverage/coverage* -/test/results/reports/coverage.xml -/test/results/reports/coverage/ +/test/results/reports/coverage*.xml +/test/results/reports/coverage*/ /test/results/bot/*.json /test/results/junit/*.xml /test/results/logs/*.log diff --git a/test/runner/lib/cover.py b/test/runner/lib/cover.py index a3747f706b..741ecc5eb3 100644 --- a/test/runner/lib/cover.py +++ b/test/runner/lib/cover.py @@ -14,6 +14,7 @@ from lib.util import ( ApplicationError, EnvironmentConfig, run_command, + common_environment, ) from lib.executor import ( @@ -23,11 +24,13 @@ from lib.executor import ( COVERAGE_DIR = 'test/results/coverage' COVERAGE_FILE = os.path.join(COVERAGE_DIR, 'coverage') +COVERAGE_GROUPS = ('command', 'target', 'environment', 'version') def command_coverage_combine(args): """Patch paths in coverage files and merge into a single file. :type args: CoverageConfig + :rtype: list[str] """ coverage = initialize_coverage(args) @@ -35,12 +38,11 @@ def command_coverage_combine(args): coverage_files = [os.path.join(COVERAGE_DIR, f) for f in os.listdir(COVERAGE_DIR) if '=coverage.' in f] - arc_data = {} - ansible_path = os.path.abspath('lib/ansible/') + '/' root_path = os.getcwd() + '/' counter = 0 + groups = {} for coverage_file in coverage_files: counter += 1 @@ -48,6 +50,12 @@ def command_coverage_combine(args): original = coverage.CoverageData() + group = get_coverage_group(args, coverage_file) + + if group is None: + display.warning('Unexpected name for coverage file: %s' % coverage_file) + continue + if os.path.getsize(coverage_file) == 0: display.warning('Empty coverage file: %s' % coverage_file) continue @@ -83,46 +91,77 @@ def command_coverage_combine(args): display.info('%s -> %s' % (filename, new_name), verbosity=3) filename = new_name + if group not in groups: + groups[group] = {} + + arc_data = groups[group] + if filename not in arc_data: arc_data[filename] = set() arc_data[filename].update(arcs) - updated = coverage.CoverageData() + output_files = [] - for filename in arc_data: - if not os.path.isfile(filename): - display.warning('Invalid coverage path: %s' % filename) - continue + for group in sorted(groups): + arc_data = groups[group] - updated.add_arcs({filename: list(arc_data[filename])}) + updated = coverage.CoverageData() - if not args.explain: - updated.write_file(COVERAGE_FILE) + for filename in arc_data: + if not os.path.isfile(filename): + display.warning('Invalid coverage path: %s' % filename) + continue + + updated.add_arcs({filename: list(arc_data[filename])}) + + if not args.explain: + output_file = COVERAGE_FILE + group + updated.write_file(output_file) + output_files.append(output_file) + + return sorted(output_files) def command_coverage_report(args): """ :type args: CoverageConfig """ - command_coverage_combine(args) - run_command(args, ['coverage', 'report']) + output_files = command_coverage_combine(args) + + for output_file in output_files: + if args.group_by: + display.info('>>> Coverage Group: %s' % ' '.join(os.path.basename(output_file).split('=')[1:])) + + env = common_environment() + env.update(dict(COVERAGE_FILE=output_file)) + run_command(args, env=env, cmd=['coverage', 'report']) def command_coverage_html(args): """ :type args: CoverageConfig """ - command_coverage_combine(args) - run_command(args, ['coverage', 'html', '-d', 'test/results/reports/coverage']) + output_files = command_coverage_combine(args) + + for output_file in output_files: + dir_name = 'test/results/reports/%s' % os.path.basename(output_file) + env = common_environment() + env.update(dict(COVERAGE_FILE=output_file)) + run_command(args, env=env, cmd=['coverage', 'html', '-d', dir_name]) def command_coverage_xml(args): """ :type args: CoverageConfig """ - command_coverage_combine(args) - run_command(args, ['coverage', 'xml', '-o', 'test/results/reports/coverage.xml']) + output_files = command_coverage_combine(args) + + for output_file in output_files: + xml_name = 'test/results/reports/%s.xml' % os.path.basename(output_file) + env = common_environment() + env.update(dict(COVERAGE_FILE=output_file)) + run_command(args, env=env, cmd=['coverage', 'xml', '-o', xml_name]) def command_coverage_erase(args): @@ -163,6 +202,33 @@ def initialize_coverage(args): return coverage +def get_coverage_group(args, coverage_file): + """ + :type args: CoverageConfig + :type coverage_file: str + :rtype: str + """ + parts = os.path.basename(coverage_file).split('=', 4) + + if len(parts) != 5 or not parts[4].startswith('coverage.'): + return None + + names = dict( + command=parts[0], + target=parts[1], + environment=parts[2], + version=parts[3], + ) + + group = '' + + for part in COVERAGE_GROUPS: + if part in args.group_by: + group += '=%s' % names[part] + + return group + + class CoverageConfig(EnvironmentConfig): """Configuration for the coverage command.""" def __init__(self, args): @@ -170,3 +236,5 @@ class CoverageConfig(EnvironmentConfig): :type args: any """ super(CoverageConfig, self).__init__(args, 'coverage') + + self.group_by = frozenset(args.group_by) if 'group_by' in args and args.group_by else set() # type: frozenset[str] diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 4a488a6f6d..41b7e3e7f0 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -858,7 +858,7 @@ def intercept_command(args, cmd, target_name, capture=False, env=None, data=None version = python_version or args.python_version interpreter = find_executable('python%s' % version) coverage_file = os.path.abspath(os.path.join(inject_path, '..', 'output', '%s=%s=%s=%s=coverage' % ( - args.command, target_name, args.coverage_label or 'local-%s' % version, version))) + args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version))) env['PATH'] = inject_path + os.pathsep + env['PATH'] env['ANSIBLE_TEST_PYTHON_VERSION'] = version diff --git a/test/runner/test.py b/test/runner/test.py index 8371976801..71e7b1f7ac 100755 --- a/test/runner/test.py +++ b/test/runner/test.py @@ -337,6 +337,8 @@ def parse_args(): coverage_combine.set_defaults(func=lib.cover.command_coverage_combine, config=lib.cover.CoverageConfig) + add_extra_coverage_options(coverage_combine) + coverage_erase = coverage_subparsers.add_parser('erase', parents=[coverage_common], help='erase coverage data files') @@ -351,6 +353,8 @@ def parse_args(): coverage_report.set_defaults(func=lib.cover.command_coverage_report, config=lib.cover.CoverageConfig) + add_extra_coverage_options(coverage_report) + coverage_html = coverage_subparsers.add_parser('html', parents=[coverage_common], help='generate html coverage report') @@ -358,6 +362,8 @@ def parse_args(): coverage_html.set_defaults(func=lib.cover.command_coverage_html, config=lib.cover.CoverageConfig) + add_extra_coverage_options(coverage_html) + coverage_xml = coverage_subparsers.add_parser('xml', parents=[coverage_common], help='generate xml coverage report') @@ -365,6 +371,8 @@ def parse_args(): coverage_xml.set_defaults(func=lib.cover.command_coverage_xml, config=lib.cover.CoverageConfig) + add_extra_coverage_options(coverage_xml) + if argcomplete: argcomplete.autocomplete(parser, always_complete_options=False, validator=lambda i, k: True) @@ -498,6 +506,17 @@ def add_environments(parser, tox_version=False, tox_only=False): default='never') +def add_extra_coverage_options(parser): + """ + :type parser: argparse.ArgumentParser + """ + parser.add_argument('--group-by', + metavar='GROUP', + action='append', + choices=lib.cover.COVERAGE_GROUPS, + help='group output by: %s' % ', '.join(lib.cover.COVERAGE_GROUPS)) + + def add_extra_docker_options(parser, integration=True): """ :type parser: argparse.ArgumentParser diff --git a/test/utils/shippable/shippable.sh b/test/utils/shippable/shippable.sh index e330154aa2..553135ad3c 100755 --- a/test/utils/shippable/shippable.sh +++ b/test/utils/shippable/shippable.sh @@ -52,8 +52,30 @@ find lib/ansible/modules -type d -empty -print -delete function cleanup { if find test/results/coverage/ -mindepth 1 -name '.*' -prune -o -print -quit | grep -q .; then - ansible-test coverage xml --color -v --requirements - cp -a test/results/reports/coverage.xml shippable/codecoverage/coverage.xml + ansible-test coverage xml --color -v --requirements --group-by command --group-by version + cp -a test/results/reports/coverage=*.xml shippable/codecoverage/ + + # upload coverage report to codecov.io only when using complete on-demand coverage + if [ "${COVERAGE}" ] && [ "${CHANGED}" == "" ]; then + for file in test/results/reports/coverage=*.xml; do + flags="${file##*/coverage=}" + flags="${flags%.xml}" + flags="${flags//=/,}" + flags="${flags//[^a-zA-Z0-9_,]/_}" + + bash <(curl -s https://codecov.io/bash) \ + -f "${file}" \ + -F "${flags}" \ + -n "${TEST}" \ + -t 83cd8957-dc76-488c-9ada-210dcea51633 \ + -X coveragepy \ + -X gcov \ + -X fix \ + -X search \ + -X xcode \ + || echo "Failed to upload code coverage report to codecov.io: ${file}" + done + fi fi rmdir shippable/testresults/