From 86ad4c99bad2abe1d638b61c07eba0995620d899 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Wed, 10 Jul 2019 22:00:34 -0700 Subject: [PATCH] Reorganize util code in ansible-test. Code in util.py that depends on CommonConfig is now in util_common.py. --- test/runner/lib/ansible_util.py | 5 +- test/runner/lib/changes.py | 5 +- test/runner/lib/cloud/tower.py | 5 +- test/runner/lib/config.py | 5 +- test/runner/lib/core_ci.py | 5 +- test/runner/lib/cover.py | 5 +- test/runner/lib/delegation.py | 5 +- test/runner/lib/docker_util.py | 10 +- test/runner/lib/executor.py | 11 +- test/runner/lib/http.py | 7 +- test/runner/lib/integration/__init__.py | 5 +- test/runner/lib/manage_ci.py | 7 +- test/runner/lib/sanity/__init__.py | 5 +- test/runner/lib/sanity/ansible_doc.py | 5 +- test/runner/lib/sanity/compile.py | 5 +- test/runner/lib/sanity/import.py | 7 +- test/runner/lib/sanity/pep8.py | 5 +- test/runner/lib/sanity/pslint.py | 5 +- test/runner/lib/sanity/pylint.py | 6 +- test/runner/lib/sanity/rstcheck.py | 5 +- test/runner/lib/sanity/shellcheck.py | 5 +- test/runner/lib/sanity/validate_modules.py | 5 +- test/runner/lib/sanity/yamllint.py | 5 +- test/runner/lib/util.py | 231 ------------------- test/runner/lib/util_common.py | 251 +++++++++++++++++++++ 25 files changed, 351 insertions(+), 264 deletions(-) create mode 100644 test/runner/lib/util_common.py diff --git a/test/runner/lib/ansible_util.py b/test/runner/lib/ansible_util.py index 86cb7927ad..b089ea4108 100644 --- a/test/runner/lib/ansible_util.py +++ b/test/runner/lib/ansible_util.py @@ -13,11 +13,14 @@ from lib.util import ( common_environment, display, find_python, - run_command, ApplicationError, INSTALL_ROOT, ) +from lib.util_common import ( + run_command, +) + from lib.config import ( IntegrationConfig, EnvironmentConfig, diff --git a/test/runner/lib/changes.py b/test/runner/lib/changes.py index ca31219b9c..9a7f6b38dc 100644 --- a/test/runner/lib/changes.py +++ b/test/runner/lib/changes.py @@ -9,10 +9,13 @@ from lib.util import ( ApplicationError, SubprocessError, MissingEnvironmentVariable, - CommonConfig, display, ) +from lib.util_common import ( + CommonConfig, +) + from lib.http import ( HttpClient, urlencode, diff --git a/test/runner/lib/cloud/tower.py b/test/runner/lib/cloud/tower.py index 3d50bf23e8..3e7cd7dc25 100644 --- a/test/runner/lib/cloud/tower.py +++ b/test/runner/lib/cloud/tower.py @@ -8,11 +8,14 @@ from lib.util import ( display, ApplicationError, is_shippable, - run_command, SubprocessError, ConfigParser, ) +from lib.util_common import ( + run_command, +) + from lib.cloud import ( CloudProvider, CloudEnvironment, diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py index 7292211179..348cda0c65 100644 --- a/test/runner/lib/config.py +++ b/test/runner/lib/config.py @@ -8,7 +8,6 @@ import sys import lib.types as t from lib.util import ( - CommonConfig, is_shippable, docker_qualify_image, find_python, @@ -17,6 +16,10 @@ from lib.util import ( ApplicationError, ) +from lib.util_common import ( + CommonConfig, +) + from lib.metadata import ( Metadata, ) diff --git a/test/runner/lib/core_ci.py b/test/runner/lib/core_ci.py index 7013199d76..27b38b7af7 100644 --- a/test/runner/lib/core_ci.py +++ b/test/runner/lib/core_ci.py @@ -18,12 +18,15 @@ from lib.http import ( from lib.util import ( ApplicationError, - run_command, make_dirs, display, is_shippable, ) +from lib.util_common import ( + run_command, +) + from lib.config import ( EnvironmentConfig, ) diff --git a/test/runner/lib/cover.py b/test/runner/lib/cover.py index 3e61de6a95..e9948ca2ca 100644 --- a/test/runner/lib/cover.py +++ b/test/runner/lib/cover.py @@ -13,10 +13,13 @@ from lib.target import ( from lib.util import ( display, ApplicationError, - run_command, common_environment, ) +from lib.util_common import ( + run_command, +) + from lib.config import ( CoverageConfig, CoverageReportConfig, diff --git a/test/runner/lib/delegation.py b/test/runner/lib/delegation.py index 6c4517c918..7c1e5cb85a 100644 --- a/test/runner/lib/delegation.py +++ b/test/runner/lib/delegation.py @@ -42,12 +42,15 @@ from lib.manage_ci import ( from lib.util import ( ApplicationError, - run_command, common_environment, pass_vars, display, ) +from lib.util_common import ( + run_command, +) + from lib.docker_util import ( docker_exec, docker_get, diff --git a/test/runner/lib/docker_util.py b/test/runner/lib/docker_util.py index 118a1929d2..52e3af46b0 100644 --- a/test/runner/lib/docker_util.py +++ b/test/runner/lib/docker_util.py @@ -6,16 +6,16 @@ import json import os import time -from lib.executor import ( - SubprocessError, -) - from lib.util import ( ApplicationError, - run_command, common_environment, display, find_executable, + SubprocessError, +) + +from lib.util_common import ( + run_command, ) from lib.config import ( diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index f9a891f76a..9189df0595 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -44,26 +44,29 @@ from lib.util import ( ApplicationError, SubprocessError, display, - run_command, - intercept_command, remove_tree, make_dirs, is_shippable, is_binary_file, find_executable, raw_command, - get_python_path, get_available_port, generate_pip_command, find_python, get_docker_completion, get_remote_completion, - named_temporary_file, COVERAGE_OUTPUT_PATH, cmd_quote, INSTALL_ROOT, ) +from lib.util_common import ( + get_python_path, + intercept_command, + named_temporary_file, + run_command, +) + from lib.docker_util import ( docker_pull, docker_run, diff --git a/test/runner/lib/http.py b/test/runner/lib/http.py index ac2c6a35e3..93d0550152 100644 --- a/test/runner/lib/http.py +++ b/test/runner/lib/http.py @@ -22,13 +22,16 @@ except ImportError: from urllib.parse import urlparse, urlunparse, parse_qs # pylint: disable=locally-disabled, ungrouped-imports from lib.util import ( - CommonConfig, ApplicationError, - run_command, SubprocessError, display, ) +from lib.util_common import ( + CommonConfig, + run_command, +) + class HttpClient(object): """Make HTTP requests via curl.""" diff --git a/test/runner/lib/integration/__init__.py b/test/runner/lib/integration/__init__.py index c6f82b1118..c5a54bba63 100644 --- a/test/runner/lib/integration/__init__.py +++ b/test/runner/lib/integration/__init__.py @@ -23,7 +23,6 @@ from lib.util import ( ApplicationError, display, make_dirs, - named_temporary_file, COVERAGE_CONFIG_PATH, COVERAGE_OUTPUT_PATH, MODE_DIRECTORY, @@ -32,6 +31,10 @@ from lib.util import ( INSTALL_ROOT, ) +from lib.util_common import ( + named_temporary_file, +) + from lib.cache import ( CommonCache, ) diff --git a/test/runner/lib/manage_ci.py b/test/runner/lib/manage_ci.py index 0e1df84a89..8dce216aa7 100644 --- a/test/runner/lib/manage_ci.py +++ b/test/runner/lib/manage_ci.py @@ -11,11 +11,14 @@ import lib.pytar from lib.util import ( SubprocessError, ApplicationError, - run_command, - intercept_command, cmd_quote, ) +from lib.util_common import ( + intercept_command, + run_command, +) + from lib.core_ci import ( AnsibleCoreCI, ) diff --git a/test/runner/lib/sanity/__init__.py b/test/runner/lib/sanity/__init__.py index 9ca36e4b10..9b4a644160 100644 --- a/test/runner/lib/sanity/__init__.py +++ b/test/runner/lib/sanity/__init__.py @@ -12,7 +12,6 @@ from lib.util import ( ApplicationError, SubprocessError, display, - run_command, import_plugins, load_plugins, parse_to_list_of_dict, @@ -22,6 +21,10 @@ from lib.util import ( read_lines_without_comments, ) +from lib.util_common import ( + run_command, +) + from lib.ansible_util import ( ansible_environment, check_pyyaml, diff --git a/test/runner/lib/sanity/ansible_doc.py b/test/runner/lib/sanity/ansible_doc.py index cfbe2e5549..bc012fd94f 100644 --- a/test/runner/lib/sanity/ansible_doc.py +++ b/test/runner/lib/sanity/ansible_doc.py @@ -16,10 +16,13 @@ from lib.sanity import ( from lib.util import ( SubprocessError, display, - intercept_command, read_lines_without_comments, ) +from lib.util_common import ( + intercept_command, +) + from lib.ansible_util import ( ansible_environment, ) diff --git a/test/runner/lib/sanity/compile.py b/test/runner/lib/sanity/compile.py index 0969796d3b..5f06fd84a9 100644 --- a/test/runner/lib/sanity/compile.py +++ b/test/runner/lib/sanity/compile.py @@ -13,7 +13,6 @@ from lib.sanity import ( from lib.util import ( SubprocessError, - run_command, display, find_python, read_lines_without_comments, @@ -21,6 +20,10 @@ from lib.util import ( INSTALL_ROOT, ) +from lib.util_common import ( + run_command, +) + from lib.config import ( SanityConfig, ) diff --git a/test/runner/lib/sanity/import.py b/test/runner/lib/sanity/import.py index 5c2c0a477a..a1a8527c04 100644 --- a/test/runner/lib/sanity/import.py +++ b/test/runner/lib/sanity/import.py @@ -13,8 +13,6 @@ from lib.sanity import ( from lib.util import ( SubprocessError, - run_command, - intercept_command, remove_tree, display, find_python, @@ -23,6 +21,11 @@ from lib.util import ( make_dirs, ) +from lib.util_common import ( + intercept_command, + run_command, +) + from lib.ansible_util import ( ansible_environment, ) diff --git a/test/runner/lib/sanity/pep8.py b/test/runner/lib/sanity/pep8.py index bba262bf20..d120499e75 100644 --- a/test/runner/lib/sanity/pep8.py +++ b/test/runner/lib/sanity/pep8.py @@ -14,12 +14,15 @@ from lib.sanity import ( from lib.util import ( SubprocessError, display, - run_command, read_lines_without_comments, parse_to_list_of_dict, INSTALL_ROOT, ) +from lib.util_common import ( + run_command, +) + from lib.config import ( SanityConfig, ) diff --git a/test/runner/lib/sanity/pslint.py b/test/runner/lib/sanity/pslint.py index 6e629dd2d8..0620360d7d 100644 --- a/test/runner/lib/sanity/pslint.py +++ b/test/runner/lib/sanity/pslint.py @@ -16,11 +16,14 @@ from lib.sanity import ( from lib.util import ( SubprocessError, - run_command, find_executable, read_lines_without_comments, ) +from lib.util_common import ( + run_command, +) + from lib.config import ( SanityConfig, ) diff --git a/test/runner/lib/sanity/pylint.py b/test/runner/lib/sanity/pylint.py index de3bd7c1e4..642f3406c1 100644 --- a/test/runner/lib/sanity/pylint.py +++ b/test/runner/lib/sanity/pylint.py @@ -19,13 +19,17 @@ from lib.sanity import ( from lib.util import ( SubprocessError, - run_command, display, read_lines_without_comments, ConfigParser, INSTALL_ROOT, ) +from lib.util_common import ( + intercept_command, + run_command, +) + from lib.executor import ( SUPPORTED_PYTHON_VERSIONS, ) diff --git a/test/runner/lib/sanity/rstcheck.py b/test/runner/lib/sanity/rstcheck.py index 8dd6a5d828..3004718478 100644 --- a/test/runner/lib/sanity/rstcheck.py +++ b/test/runner/lib/sanity/rstcheck.py @@ -13,13 +13,16 @@ from lib.sanity import ( from lib.util import ( SubprocessError, - run_command, parse_to_list_of_dict, display, read_lines_without_comments, INSTALL_ROOT, ) +from lib.util_common import ( + run_command, +) + from lib.config import ( SanityConfig, ) diff --git a/test/runner/lib/sanity/shellcheck.py b/test/runner/lib/sanity/shellcheck.py index 0e9f7834d6..8aeaf25a25 100644 --- a/test/runner/lib/sanity/shellcheck.py +++ b/test/runner/lib/sanity/shellcheck.py @@ -18,10 +18,13 @@ from lib.sanity import ( from lib.util import ( SubprocessError, - run_command, read_lines_without_comments, ) +from lib.util_common import ( + run_command, +) + from lib.config import ( SanityConfig, ) diff --git a/test/runner/lib/sanity/validate_modules.py b/test/runner/lib/sanity/validate_modules.py index 4846a62adc..5f2c6761d6 100644 --- a/test/runner/lib/sanity/validate_modules.py +++ b/test/runner/lib/sanity/validate_modules.py @@ -16,11 +16,14 @@ from lib.sanity import ( from lib.util import ( SubprocessError, display, - run_command, read_lines_without_comments, INSTALL_ROOT, ) +from lib.util_common import ( + run_command, +) + from lib.ansible_util import ( ansible_environment, ) diff --git a/test/runner/lib/sanity/yamllint.py b/test/runner/lib/sanity/yamllint.py index ff495c26df..b7ee0a5d83 100644 --- a/test/runner/lib/sanity/yamllint.py +++ b/test/runner/lib/sanity/yamllint.py @@ -14,11 +14,14 @@ from lib.sanity import ( from lib.util import ( SubprocessError, - run_command, display, INSTALL_ROOT, ) +from lib.util_common import ( + run_command, +) + from lib.config import ( SanityConfig, ) diff --git a/test/runner/lib/util.py b/test/runner/lib/util.py index 4dba50aef2..df59f5c232 100644 --- a/test/runner/lib/util.py +++ b/test/runner/lib/util.py @@ -2,7 +2,6 @@ from __future__ import absolute_import, print_function -import atexit import contextlib import errno import fcntl @@ -17,8 +16,6 @@ import stat import string import subprocess import sys -import tempfile -import textwrap import time from struct import unpack, pack @@ -158,127 +155,6 @@ def read_lines_without_comments(path, remove_blank_lines=False, optional=False): return lines -def get_python_path(args, interpreter): - """ - :type args: TestConfig - :type interpreter: str - :rtype: str - """ - # When the python interpreter is already named "python" its directory can simply be added to the path. - # Using another level of indirection is only required when the interpreter has a different name. - if os.path.basename(interpreter) == 'python': - return os.path.dirname(interpreter) - - python_path = PYTHON_PATHS.get(interpreter) - - if python_path: - return python_path - - prefix = 'python-' - suffix = '-ansible' - - root_temp_dir = '/tmp' - - if args.explain: - return os.path.join(root_temp_dir, ''.join((prefix, 'temp', suffix))) - - python_path = tempfile.mkdtemp(prefix=prefix, suffix=suffix, dir=root_temp_dir) - injected_interpreter = os.path.join(python_path, 'python') - - # A symlink is faster than the execv wrapper, but isn't compatible with virtual environments. - # Attempt to detect when it is safe to use a symlink by checking the real path of the interpreter. - use_symlink = os.path.dirname(os.path.realpath(interpreter)) == os.path.dirname(interpreter) - - if use_symlink: - display.info('Injecting "%s" as a symlink to the "%s" interpreter.' % (injected_interpreter, interpreter), verbosity=1) - - os.symlink(interpreter, injected_interpreter) - else: - display.info('Injecting "%s" as a execv wrapper for the "%s" interpreter.' % (injected_interpreter, interpreter), verbosity=1) - - code = textwrap.dedent(''' - #!%s - - from __future__ import absolute_import - - from os import execv - from sys import argv - - python = '%s' - - execv(python, [python] + argv[1:]) - ''' % (interpreter, interpreter)).lstrip() - - with open(injected_interpreter, 'w') as python_fd: - python_fd.write(code) - - os.chmod(injected_interpreter, MODE_FILE_EXECUTE) - - os.chmod(python_path, MODE_DIRECTORY) - - if not PYTHON_PATHS: - atexit.register(cleanup_python_paths) - - PYTHON_PATHS[interpreter] = python_path - - return python_path - - -def cleanup_python_paths(): - """Clean up all temporary python directories.""" - for path in sorted(PYTHON_PATHS.values()): - display.info('Cleaning up temporary python directory: %s' % path, verbosity=2) - shutil.rmtree(path) - - -def get_coverage_environment(args, target_name, version, temp_path, module_coverage): - """ - :type args: TestConfig - :type target_name: str - :type version: str - :type temp_path: str - :type module_coverage: bool - :rtype: dict[str, str] - """ - if temp_path: - # integration tests (both localhost and the optional testhost) - # config and results are in a temporary directory - coverage_config_base_path = temp_path - coverage_output_base_path = temp_path - else: - # unit tests, sanity tests and other special cases (localhost only) - # config and results are in the source tree - coverage_config_base_path = args.coverage_config_base_path or INSTALL_ROOT - coverage_output_base_path = os.path.abspath(os.path.join('test/results')) - - config_file = os.path.join(coverage_config_base_path, COVERAGE_CONFIG_PATH) - coverage_file = os.path.join(coverage_output_base_path, COVERAGE_OUTPUT_PATH, '%s=%s=%s=%s=coverage' % ( - args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version)) - - if args.coverage_check: - # cause the 'coverage' module to be found, but not imported or enabled - coverage_file = '' - - # Enable code coverage collection on local Python programs (this does not include Ansible modules). - # Used by the injectors in test/runner/injector/ to support code coverage. - # Used by unit tests in test/units/conftest.py to support code coverage. - # The COVERAGE_FILE variable is also used directly by the 'coverage' module. - env = dict( - COVERAGE_CONF=config_file, - COVERAGE_FILE=coverage_file, - ) - - if module_coverage: - # Enable code coverage collection on Ansible modules (both local and remote). - # Used by the AnsiballZ wrapper generator in lib/ansible/executor/module_common.py to support code coverage. - env.update(dict( - _ANSIBLE_COVERAGE_CONFIG=config_file, - _ANSIBLE_COVERAGE_OUTPUT=coverage_file, - )) - - return env - - def find_executable(executable, cwd=None, path=None, required=True): """ :type executable: str @@ -355,68 +231,6 @@ def generate_pip_command(python): return [python, '-m', 'pip.__main__'] -def intercept_command(args, cmd, target_name, env, capture=False, data=None, cwd=None, python_version=None, temp_path=None, module_coverage=True, - virtualenv=None): - """ - :type args: TestConfig - :type cmd: collections.Iterable[str] - :type target_name: str - :type env: dict[str, str] - :type capture: bool - :type data: str | None - :type cwd: str | None - :type python_version: str | None - :type temp_path: str | None - :type module_coverage: bool - :type virtualenv: str | None - :rtype: str | None, str | None - """ - if not env: - env = common_environment() - - cmd = list(cmd) - version = python_version or args.python_version - interpreter = virtualenv or find_python(version) - inject_path = os.path.join(INSTALL_ROOT, 'test/runner/injector') - - if not virtualenv: - # injection of python into the path is required when not activating a virtualenv - # otherwise scripts may find the wrong interpreter or possibly no interpreter - python_path = get_python_path(args, interpreter) - inject_path = python_path + os.path.pathsep + inject_path - - env['PATH'] = inject_path + os.path.pathsep + env['PATH'] - env['ANSIBLE_TEST_PYTHON_VERSION'] = version - env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = interpreter - - if args.coverage: - # add the necessary environment variables to enable code coverage collection - env.update(get_coverage_environment(args, target_name, version, temp_path, module_coverage)) - - return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd) - - -def run_command(args, cmd, capture=False, env=None, data=None, cwd=None, always=False, stdin=None, stdout=None, - cmd_verbosity=1, str_errors='strict'): - """ - :type args: CommonConfig - :type cmd: collections.Iterable[str] - :type capture: bool - :type env: dict[str, str] | None - :type data: str | None - :type cwd: str | None - :type always: bool - :type stdin: file | None - :type stdout: file | None - :type cmd_verbosity: int - :type str_errors: str - :rtype: str | None, str | None - """ - explain = args.explain and not always - return raw_command(cmd, capture=capture, env=env, data=data, cwd=cwd, explain=explain, stdin=stdin, stdout=stdout, - cmd_verbosity=cmd_verbosity, str_errors=str_errors) - - def raw_command(cmd, capture=False, env=None, data=None, cwd=None, explain=False, stdin=None, stdout=None, cmd_verbosity=1, str_errors='strict'): """ @@ -859,28 +673,6 @@ class MissingEnvironmentVariable(ApplicationError): self.name = name -class CommonConfig(object): - """Configuration common to all commands.""" - def __init__(self, args, command): - """ - :type args: any - :type command: str - """ - self.command = command - - self.color = args.color # type: bool - self.explain = args.explain # type: bool - self.verbosity = args.verbosity # type: int - self.debug = args.debug # type: bool - self.truncate = args.truncate # type: int - self.redact = args.redact # type: bool - - if is_shippable(): - self.redact = True - - self.cache = {} - - def docker_qualify_image(name): """ :type name: str @@ -891,29 +683,6 @@ def docker_qualify_image(name): return config.get('name', name) -@contextlib.contextmanager -def named_temporary_file(args, prefix, suffix, directory, content): - """ - :param args: CommonConfig - :param prefix: str - :param suffix: str - :param directory: str - :param content: str | bytes | unicode - :rtype: str - """ - if not isinstance(content, bytes): - content = content.encode('utf-8') - - if args.explain: - yield os.path.join(directory, '%stemp%s' % (prefix, suffix)) - else: - with tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix, dir=directory) as tempfile_fd: - tempfile_fd.write(content) - tempfile_fd.flush() - - yield tempfile_fd.name - - def parse_to_list_of_dict(pattern, value): """ :type pattern: str diff --git a/test/runner/lib/util_common.py b/test/runner/lib/util_common.py new file mode 100644 index 0000000000..1f5548fb05 --- /dev/null +++ b/test/runner/lib/util_common.py @@ -0,0 +1,251 @@ +"""Common utility code that depends on CommonConfig.""" +from __future__ import absolute_import, print_function + +import atexit +import contextlib +import os +import shutil +import tempfile +import textwrap + +from lib.util import ( + common_environment, + COVERAGE_CONFIG_PATH, + COVERAGE_OUTPUT_PATH, + display, + find_python, + INSTALL_ROOT, + is_shippable, + MODE_DIRECTORY, + MODE_FILE_EXECUTE, + PYTHON_PATHS, + raw_command, +) + + +class CommonConfig(object): + """Configuration common to all commands.""" + def __init__(self, args, command): + """ + :type args: any + :type command: str + """ + self.command = command + + self.color = args.color # type: bool + self.explain = args.explain # type: bool + self.verbosity = args.verbosity # type: int + self.debug = args.debug # type: bool + self.truncate = args.truncate # type: int + self.redact = args.redact # type: bool + + if is_shippable(): + self.redact = True + + self.cache = {} + + +@contextlib.contextmanager +def named_temporary_file(args, prefix, suffix, directory, content): + """ + :param args: CommonConfig + :param prefix: str + :param suffix: str + :param directory: str + :param content: str | bytes | unicode + :rtype: str + """ + if not isinstance(content, bytes): + content = content.encode('utf-8') + + if args.explain: + yield os.path.join(directory, '%stemp%s' % (prefix, suffix)) + else: + with tempfile.NamedTemporaryFile(prefix=prefix, suffix=suffix, dir=directory) as tempfile_fd: + tempfile_fd.write(content) + tempfile_fd.flush() + + yield tempfile_fd.name + + +def get_python_path(args, interpreter): + """ + :type args: TestConfig + :type interpreter: str + :rtype: str + """ + # When the python interpreter is already named "python" its directory can simply be added to the path. + # Using another level of indirection is only required when the interpreter has a different name. + if os.path.basename(interpreter) == 'python': + return os.path.dirname(interpreter) + + python_path = PYTHON_PATHS.get(interpreter) + + if python_path: + return python_path + + prefix = 'python-' + suffix = '-ansible' + + root_temp_dir = '/tmp' + + if args.explain: + return os.path.join(root_temp_dir, ''.join((prefix, 'temp', suffix))) + + python_path = tempfile.mkdtemp(prefix=prefix, suffix=suffix, dir=root_temp_dir) + injected_interpreter = os.path.join(python_path, 'python') + + # A symlink is faster than the execv wrapper, but isn't compatible with virtual environments. + # Attempt to detect when it is safe to use a symlink by checking the real path of the interpreter. + use_symlink = os.path.dirname(os.path.realpath(interpreter)) == os.path.dirname(interpreter) + + if use_symlink: + display.info('Injecting "%s" as a symlink to the "%s" interpreter.' % (injected_interpreter, interpreter), verbosity=1) + + os.symlink(interpreter, injected_interpreter) + else: + display.info('Injecting "%s" as a execv wrapper for the "%s" interpreter.' % (injected_interpreter, interpreter), verbosity=1) + + code = textwrap.dedent(''' + #!%s + + from __future__ import absolute_import + + from os import execv + from sys import argv + + python = '%s' + + execv(python, [python] + argv[1:]) + ''' % (interpreter, interpreter)).lstrip() + + with open(injected_interpreter, 'w') as python_fd: + python_fd.write(code) + + os.chmod(injected_interpreter, MODE_FILE_EXECUTE) + + os.chmod(python_path, MODE_DIRECTORY) + + if not PYTHON_PATHS: + atexit.register(cleanup_python_paths) + + PYTHON_PATHS[interpreter] = python_path + + return python_path + + +def cleanup_python_paths(): + """Clean up all temporary python directories.""" + for path in sorted(PYTHON_PATHS.values()): + display.info('Cleaning up temporary python directory: %s' % path, verbosity=2) + shutil.rmtree(path) + + +def get_coverage_environment(args, target_name, version, temp_path, module_coverage): + """ + :type args: TestConfig + :type target_name: str + :type version: str + :type temp_path: str + :type module_coverage: bool + :rtype: dict[str, str] + """ + if temp_path: + # integration tests (both localhost and the optional testhost) + # config and results are in a temporary directory + coverage_config_base_path = temp_path + coverage_output_base_path = temp_path + else: + # unit tests, sanity tests and other special cases (localhost only) + # config and results are in the source tree + coverage_config_base_path = args.coverage_config_base_path or INSTALL_ROOT + coverage_output_base_path = os.path.abspath(os.path.join('test/results')) + + config_file = os.path.join(coverage_config_base_path, COVERAGE_CONFIG_PATH) + coverage_file = os.path.join(coverage_output_base_path, COVERAGE_OUTPUT_PATH, '%s=%s=%s=%s=coverage' % ( + args.command, target_name, args.coverage_label or 'local-%s' % version, 'python-%s' % version)) + + if args.coverage_check: + # cause the 'coverage' module to be found, but not imported or enabled + coverage_file = '' + + # Enable code coverage collection on local Python programs (this does not include Ansible modules). + # Used by the injectors in test/runner/injector/ to support code coverage. + # Used by unit tests in test/units/conftest.py to support code coverage. + # The COVERAGE_FILE variable is also used directly by the 'coverage' module. + env = dict( + COVERAGE_CONF=config_file, + COVERAGE_FILE=coverage_file, + ) + + if module_coverage: + # Enable code coverage collection on Ansible modules (both local and remote). + # Used by the AnsiballZ wrapper generator in lib/ansible/executor/module_common.py to support code coverage. + env.update(dict( + _ANSIBLE_COVERAGE_CONFIG=config_file, + _ANSIBLE_COVERAGE_OUTPUT=coverage_file, + )) + + return env + + +def intercept_command(args, cmd, target_name, env, capture=False, data=None, cwd=None, python_version=None, temp_path=None, module_coverage=True, + virtualenv=None): + """ + :type args: TestConfig + :type cmd: collections.Iterable[str] + :type target_name: str + :type env: dict[str, str] + :type capture: bool + :type data: str | None + :type cwd: str | None + :type python_version: str | None + :type temp_path: str | None + :type module_coverage: bool + :type virtualenv: str | None + :rtype: str | None, str | None + """ + if not env: + env = common_environment() + + cmd = list(cmd) + version = python_version or args.python_version + interpreter = virtualenv or find_python(version) + inject_path = os.path.join(INSTALL_ROOT, 'test/runner/injector') + + if not virtualenv: + # injection of python into the path is required when not activating a virtualenv + # otherwise scripts may find the wrong interpreter or possibly no interpreter + python_path = get_python_path(args, interpreter) + inject_path = python_path + os.path.pathsep + inject_path + + env['PATH'] = inject_path + os.path.pathsep + env['PATH'] + env['ANSIBLE_TEST_PYTHON_VERSION'] = version + env['ANSIBLE_TEST_PYTHON_INTERPRETER'] = interpreter + + if args.coverage: + # add the necessary environment variables to enable code coverage collection + env.update(get_coverage_environment(args, target_name, version, temp_path, module_coverage)) + + return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd) + + +def run_command(args, cmd, capture=False, env=None, data=None, cwd=None, always=False, stdin=None, stdout=None, + cmd_verbosity=1, str_errors='strict'): + """ + :type args: CommonConfig + :type cmd: collections.Iterable[str] + :type capture: bool + :type env: dict[str, str] | None + :type data: str | None + :type cwd: str | None + :type always: bool + :type stdin: file | None + :type stdout: file | None + :type cmd_verbosity: int + :type str_errors: str + :rtype: str | None, str | None + """ + explain = args.explain and not always + return raw_command(cmd, capture=capture, env=env, data=data, cwd=cwd, explain=explain, stdin=stdin, stdout=stdout, + cmd_verbosity=cmd_verbosity, str_errors=str_errors)