From 4978fc633c03ad5d19a97f1d80ff18f822910db6 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Fri, 20 Oct 2017 08:48:01 -0700 Subject: [PATCH] Fix ansible-test default image. (#31966) * Add openssh-client to default docker container. * Include Azure requirements in default container. To do so, handling of pip requirements was updated to install each set of requirements separately and then run a verification pass to make sure there are no conflicts between requirements. * Add missing --docker-no-pull option. * Add documentation for the azure-requirements test. (cherry picked from commit 36b13e3e3d2d7d976adb3f976527fc916285777c) --- .../testing/sanity/azure-requirements.rst | 10 +++ test/integration/targets/setup_azure/aliases | 0 .../targets/setup_azure/tasks/main.yml | 8 -- test/runner/Dockerfile | 1 + test/runner/docker/requirements.sh | 51 +++++++++++- test/runner/lib/executor.py | 83 +++++++++++++++---- .../requirements/integration.cloud.azure.txt | 16 ++++ test/runner/test.py | 4 + test/sanity/code-smell/azure-requirements.py | 12 +++ test/sanity/code-smell/test-constraints.sh | 1 + 10 files changed, 159 insertions(+), 27 deletions(-) create mode 100644 docs/docsite/rst/dev_guide/testing/sanity/azure-requirements.rst create mode 100644 test/integration/targets/setup_azure/aliases delete mode 100644 test/integration/targets/setup_azure/tasks/main.yml create mode 100644 test/runner/requirements/integration.cloud.azure.txt create mode 100755 test/sanity/code-smell/azure-requirements.py diff --git a/docs/docsite/rst/dev_guide/testing/sanity/azure-requirements.rst b/docs/docsite/rst/dev_guide/testing/sanity/azure-requirements.rst new file mode 100644 index 0000000000..a11534e3e1 --- /dev/null +++ b/docs/docsite/rst/dev_guide/testing/sanity/azure-requirements.rst @@ -0,0 +1,10 @@ +Sanity Tests ยป azure-requirements +================================= + +Update the Azure integration test requirements file when changes are made to the Azure packaging requirements file: + +.. code-block:: bash + + cp packaging/requirements/requirements-azure.txt test/runner/requirements/integration.cloud.azure.txt + +This copy of the requirements file is used when building the ``ansible/ansible:default`` Docker container from ``test/runner/Dockerfile``. diff --git a/test/integration/targets/setup_azure/aliases b/test/integration/targets/setup_azure/aliases new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/integration/targets/setup_azure/tasks/main.yml b/test/integration/targets/setup_azure/tasks/main.yml deleted file mode 100644 index 8aee658c1d..0000000000 --- a/test/integration/targets/setup_azure/tasks/main.yml +++ /dev/null @@ -1,8 +0,0 @@ -- name: install cryptography requirement - # preempt the installation of cryptography from requirements-azure.txt to limit the version installed - # requests[security] requires cryptography >= 1.3.4 - # cryptography 2.1 requires pip 8.1.2+ - command: pip install cryptography>=1.3.4,<2.1 - -- pip: - requirements: '{{ role_path }}/../../../../packaging/requirements/requirements-azure.txt' diff --git a/test/runner/Dockerfile b/test/runner/Dockerfile index d08e7f8616..7795b28b6b 100644 --- a/test/runner/Dockerfile +++ b/test/runner/Dockerfile @@ -13,6 +13,7 @@ RUN apt-get update -y && \ libxslt1-dev \ locales \ make \ + openssh-client \ python2.6-dev \ python2.7-dev \ python3.5-dev \ diff --git a/test/runner/docker/requirements.sh b/test/runner/docker/requirements.sh index f16e1b7e98..3c1492ef93 100755 --- a/test/runner/docker/requirements.sh +++ b/test/runner/docker/requirements.sh @@ -11,13 +11,60 @@ requirements=() for requirement in *.txt; do if [ "${requirement}" != "constraints.txt" ]; then - requirements+=("-r" "${requirement}") + requirements+=("${requirement}") fi done for python_version in "${python_versions[@]}"; do + version_requirements=() + + for requirement in "${requirements[@]}"; do + case "${python_version}" in + "2.6") + case "${requirement}" in + "integration.cloud.azure.txt") continue ;; + esac + esac + + version_requirements+=("${requirement}") + done + + echo "==> Installing pip for python ${python_version} ..." + set -x "python${python_version}" /tmp/get-pip.py -c constraints.txt - "pip${python_version}" install --disable-pip-version-check -c constraints.txt "${requirements[@]}" set +x + + echo "==> Installing requirements for python ${python_version} ..." + + for requirement in "${version_requirements[@]}"; do + set -x + "pip${python_version}" install --disable-pip-version-check -c constraints.txt -r "${requirement}" + set +x + done + + echo "==> Checking for requirements conflicts for ${python_version} ..." + + after=$("pip${python_version}" list) + + for requirement in "${version_requirements[@]}"; do + before="${after}" + + set -x + "pip${python_version}" install --disable-pip-version-check -c constraints.txt -r "${requirement}" + set +x + + after=$("pip${python_version}" list) + + if [ "${before}" != "${after}" ]; then + echo "==> Conflicts detected in requirements for python ${python_version}: ${requirement}" + echo ">>> Before" + echo "${before}" + echo ">>> After" + echo "${after}" + exit 1 + fi + done + + echo "==> Finished with requirements for python ${python_version}." done diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 8c77edba60..9a8ade3b53 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -13,6 +13,7 @@ import textwrap import functools import shutil import stat +import pipes import random import string import atexit @@ -162,30 +163,78 @@ def install_command_requirements(args): if args.junit: packages.append('junit-xml') - extras = [] + commands = [generate_pip_install(args.command, packages=packages)] if isinstance(args, IntegrationConfig): - extras += ['cloud.%s' % cp for cp in get_cloud_platforms(args)] + for cloud_platform in get_cloud_platforms(args): + commands.append(generate_pip_install('%s.cloud.%s' % (args.command, cloud_platform))) - cmd = generate_pip_install(args.command, packages, extras) + # only look for changes when more than one requirements file is needed + detect_pip_changes = len(commands) > 1 - if not cmd: - return + # first pass to install requirements, changes expected unless environment is already set up + changes = run_pip_commands(args, commands, detect_pip_changes) - try: - run_command(args, cmd) - except SubprocessError as ex: - if ex.status != 2: - raise + if not changes: + return # no changes means we can stop early - # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it. + # second pass to check for conflicts in requirements, changes are not expected here + changes = run_pip_commands(args, commands, detect_pip_changes) - # Installing "coverage" on ubuntu 16.04 fails with the error: - # AttributeError: 'Requirement' object has no attribute 'project_name' - # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258 - # Upgrading pip works around the issue. - run_command(args, ['pip', 'install', '--upgrade', 'pip']) - run_command(args, cmd) + if not changes: + return # no changes means no conflicts + + raise ApplicationError('Conflicts detected in requirements. The following commands reported changes during verification:\n%s' % + '\n'.join((' '.join(pipes.quote(c) for c in cmd) for cmd in changes))) + + +def run_pip_commands(args, commands, detect_pip_changes=False): + """ + :type args: EnvironmentConfig + :type commands: list[list[str]] + :type detect_pip_changes: bool + :rtype: list[list[str]] + """ + changes = [] + + after_list = pip_list(args) if detect_pip_changes else None + + for cmd in commands: + if not cmd: + continue + + before_list = after_list + + try: + run_command(args, cmd) + except SubprocessError as ex: + if ex.status != 2: + raise + + # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it. + + # Installing "coverage" on ubuntu 16.04 fails with the error: + # AttributeError: 'Requirement' object has no attribute 'project_name' + # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258 + # Upgrading pip works around the issue. + run_command(args, ['pip', 'install', '--upgrade', 'pip']) + run_command(args, cmd) + + after_list = pip_list(args) if detect_pip_changes else None + + if before_list != after_list: + changes.append(cmd) + + return changes + + +def pip_list(args): + """ + :type args: EnvironmentConfig + :rtype: str + """ + stdout, _ = run_command(args, ['pip', 'list'], capture=True, always=True) + return stdout def generate_egg_info(args): diff --git a/test/runner/requirements/integration.cloud.azure.txt b/test/runner/requirements/integration.cloud.azure.txt new file mode 100644 index 0000000000..cc2ce7e006 --- /dev/null +++ b/test/runner/requirements/integration.cloud.azure.txt @@ -0,0 +1,16 @@ +packaging +requests[security] +azure-mgmt-compute>=2.0.0,<3 +azure-mgmt-network>=1.3.0,<2 +azure-mgmt-storage>=1.2.0,<2 +azure-mgmt-resource>=1.1.0,<2 +azure-storage>=0.35.1,<0.36 +azure-cli-core>=2.0.12,<3 +msrest!=0.4.15 +msrestazure>=0.4.11,<0.5 +azure-mgmt-dns>=1.0.1,<2 +azure-mgmt-keyvault>=0.40.0,<0.41 +azure-mgmt-batch>=4.1.0,<5 +azure-mgmt-sql>=0.7.1,<0.8 +azure-mgmt-web>=0.32.0,<0.33 +azure-mgmt-containerservice>=1.0.0 \ No newline at end of file diff --git a/test/runner/test.py b/test/runner/test.py index a25c265cd8..48de07a556 100755 --- a/test/runner/test.py +++ b/test/runner/test.py @@ -255,6 +255,8 @@ def parse_args(): targets=walk_network_integration_targets, config=NetworkIntegrationConfig) + add_extra_docker_options(network_integration, integration=False) + network_integration.add_argument('--platform', metavar='PLATFORM', action='append', @@ -272,6 +274,8 @@ def parse_args(): targets=walk_windows_integration_targets, config=WindowsIntegrationConfig) + add_extra_docker_options(windows_integration, integration=False) + windows_integration.add_argument('--windows', metavar='VERSION', action='append', diff --git a/test/sanity/code-smell/azure-requirements.py b/test/sanity/code-smell/azure-requirements.py new file mode 100755 index 0000000000..aeaf480b88 --- /dev/null +++ b/test/sanity/code-smell/azure-requirements.py @@ -0,0 +1,12 @@ +#!/usr/bin/env python +"""Make sure the Azure requirements files match.""" + +import filecmp + +src = 'packaging/requirements/requirements-azure.txt' +dst = 'test/runner/requirements/integration.cloud.azure.txt' + +if not filecmp.cmp(src, dst): + print('Update the Azure integration test requirements with the packaging test requirements:') + print('cp %s %s' % (src, dst)) + exit(1) diff --git a/test/sanity/code-smell/test-constraints.sh b/test/sanity/code-smell/test-constraints.sh index 773306c4d0..34020e68a4 100755 --- a/test/sanity/code-smell/test-constraints.sh +++ b/test/sanity/code-smell/test-constraints.sh @@ -5,6 +5,7 @@ constraints=$( | grep -v '(sanity_ok)$' \ | sed 's/ *;.*$//; s/ #.*$//' \ | grep -v '/constraints.txt:' \ + | grep -v '/integration.cloud.azure.txt:' \ | grep '[<>=]' )