From 1059dee51fc29b3400dd0e62cc5e1a9c9f198d97 Mon Sep 17 00:00:00 2001 From: Matt Clay Date: Tue, 4 Dec 2018 13:35:00 -0800 Subject: [PATCH] [stable-2.5] Backport test infra updates and test fixes. (#49204) * Switch tests from RHEL 7.5 to 7.6. (cherry picked from commit 6745ee7cc86c50f79b24fb701d1e4680320a576b) * Remove CI platform: freebsd/10.4 (cherry picked from commit e6ffc4f89a27853e2ce983bc51a982b5a138d1bd) * Add `--raw` option to ansible-test shell command. It is currently supported only with the `--remote` option. This makes it easier to troubleshoot new instances which are not yet supported by the setup scripts used by ansible-test. (cherry picked from commit 0826a008039c45f4fcf2d428c9267decdaeb7de2) * Support skip of platforms by version in tests. (#48826) * Support skip of platforms by version in tests. Previously a remote platform could be skipped completely using the alias: `skip/{platform}` such as `skip/rhel` Now a specific platform version can be skipped using the alias: `skip/{platform}{version}` such as `skip/rhel7.6` This feature is available for platforms specified with the `--remote` option. * Add skip by version to the docs. (cherry picked from commit 8066acc90c13595039812bd8f9eb1fcaaca1a890) * Fix ansible-test skip warning message. (cherry picked from commit 3b705efc93fdf6553c05d139dfb49fe6a5aa6483) * Fix lookup_passwordstore test skipping. (#49178) * Fix lookup_passwordstore test skipping. Skip all of RHEL instead of specific versions. Skip all of CentOS < 7 instead of specific versions. This makes the test more robust when testing newer versions. Tests could be executed on RHEL if EPEL was installed during the test. (cherry picked from commit 704dae2cda5ef3a7303b37de7bb0004f0a2ba581) --- .../testing/sanity/integration-aliases.rst | 5 +++++ shippable.yml | 9 +++------ .../targets/lookup_passwordstore/aliases | 1 + .../targets/lookup_passwordstore/tasks/main.yml | 10 ++++++---- .../targets/lookup_passwordstore/vars/main.yml | 4 ---- test/runner/completion/remote.txt | 3 +-- test/runner/lib/cli.py | 4 ++++ test/runner/lib/config.py | 5 +++++ test/runner/lib/delegation.py | 15 +++++++++++++++ test/runner/lib/executor.py | 11 +++++++++++ test/runner/lib/manage_ci.py | 9 +++++++++ 11 files changed, 60 insertions(+), 16 deletions(-) diff --git a/docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst b/docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst index 7da748460f..71e5104191 100644 --- a/docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst +++ b/docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst @@ -44,6 +44,11 @@ Aliases can be used to skip platforms using one of the following: - ``skip/rhel`` - Skip tests on RHEL. - ``skip/docker`` - Skip tests when running on a Docker container +Platform versions, as specified using the ``--remote`` option with ``/`` removed, can also be skipped: + +- ``skip/freebsd11.1`` - Skip tests on FreeBSD 11.1. +- ``skip/rhel7.6`` - Skip tests on RHEL 7.6. + Aliases can be used to skip Python major versions using one of the following: - ``skip/python2`` - Skip tests on Python 2.x. diff --git a/shippable.yml b/shippable.yml index 7d2bb04735..f7209f8671 100644 --- a/shippable.yml +++ b/shippable.yml @@ -46,8 +46,7 @@ matrix: - env: T=network - env: T=osx/10.11/1 - - env: T=rhel/7.5/1 - - env: T=freebsd/10.4/1 + - env: T=rhel/7.6/1 - env: T=freebsd/11.1/1 - env: T=linux/centos6/1 - env: T=linux/centos7/1 @@ -59,8 +58,7 @@ matrix: - env: T=linux/ubuntu1604py3/1 - env: T=osx/10.11/2 - - env: T=rhel/7.5/2 - - env: T=freebsd/10.4/2 + - env: T=rhel/7.6/2 - env: T=freebsd/11.1/2 - env: T=linux/centos6/2 - env: T=linux/centos7/2 @@ -72,8 +70,7 @@ matrix: - env: T=linux/ubuntu1604py3/2 - env: T=osx/10.11/3 - - env: T=rhel/7.5/3 - - env: T=freebsd/10.4/3 + - env: T=rhel/7.6/3 - env: T=freebsd/11.1/3 - env: T=linux/centos6/3 - env: T=linux/centos7/3 diff --git a/test/integration/targets/lookup_passwordstore/aliases b/test/integration/targets/lookup_passwordstore/aliases index ca7c912851..d1597d2d97 100644 --- a/test/integration/targets/lookup_passwordstore/aliases +++ b/test/integration/targets/lookup_passwordstore/aliases @@ -1,2 +1,3 @@ shippable/posix/group2 destructive +skip/rhel diff --git a/test/integration/targets/lookup_passwordstore/tasks/main.yml b/test/integration/targets/lookup_passwordstore/tasks/main.yml index 2f39bb637d..1177714a04 100644 --- a/test/integration/targets/lookup_passwordstore/tasks/main.yml +++ b/test/integration/targets/lookup_passwordstore/tasks/main.yml @@ -1,4 +1,6 @@ -- include: "package.yml" - when: "ansible_distribution_version not in passwordstore_skip_os.get(ansible_distribution, [])" -- include: "tests.yml" - when: "ansible_distribution_version not in passwordstore_skip_os.get(ansible_distribution, [])" +- block: + - include: "package.yml" + - include: "tests.yml" + when: + - not (ansible_distribution == 'RedHat') # requires EPEL + - not (ansible_distribution == 'CentOS' and ansible_distribution_version is version_compare('7', '<')) diff --git a/test/integration/targets/lookup_passwordstore/vars/main.yml b/test/integration/targets/lookup_passwordstore/vars/main.yml index 69f347ead7..aa464879ef 100644 --- a/test/integration/targets/lookup_passwordstore/vars/main.yml +++ b/test/integration/targets/lookup_passwordstore/vars/main.yml @@ -56,7 +56,3 @@ passwordstore_privkey: | SxHTvI2pKk+gx0FB8wWhd/CocAHJpx9oNUs/7A== =ZF3O -----END PGP PRIVATE KEY BLOCK----- -passwordstore_skip_os: - Ubuntu: ['12.04'] - RedHat: ['7.4'] - CentOS: ['6.9', '6.10'] diff --git a/test/runner/completion/remote.txt b/test/runner/completion/remote.txt index 506267da72..955271c99b 100644 --- a/test/runner/completion/remote.txt +++ b/test/runner/completion/remote.txt @@ -1,4 +1,3 @@ -freebsd/10.4 freebsd/11.1 osx/10.11 -rhel/7.5 +rhel/7.6 diff --git a/test/runner/lib/cli.py b/test/runner/lib/cli.py index 67a2888760..fa960600f0 100644 --- a/test/runner/lib/cli.py +++ b/test/runner/lib/cli.py @@ -409,6 +409,10 @@ def parse_args(): shell.set_defaults(func=command_shell, config=ShellConfig) + shell.add_argument('--raw', + action='store_true', + help='direct to shell with no setup') + add_environments(shell, tox_version=True) add_extra_docker_options(shell) add_httptester_options(shell, argparse) diff --git a/test/runner/lib/config.py b/test/runner/lib/config.py index fce8326a33..13cf9b2113 100644 --- a/test/runner/lib/config.py +++ b/test/runner/lib/config.py @@ -134,6 +134,11 @@ class ShellConfig(EnvironmentConfig): """ super(ShellConfig, self).__init__(args, 'shell') + self.raw = args.raw # type: bool + + if self.raw: + self.httptester = False + class SanityConfig(TestConfig): """Configuration for the sanity command.""" diff --git a/test/runner/lib/delegation.py b/test/runner/lib/delegation.py index 53407c6dc5..c0098cb4a4 100644 --- a/test/runner/lib/delegation.py +++ b/test/runner/lib/delegation.py @@ -334,9 +334,11 @@ def delegate_remote(args, exclude, require, integration_targets): core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider) success = False + raw = False if isinstance(args, ShellConfig): use_httptester = args.httptester + raw = args.raw else: use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets) @@ -359,6 +361,9 @@ def delegate_remote(args, exclude, require, integration_targets): # Windows doesn't need the ansible-test fluff, just run the SSH command manage = ManageWindowsCI(core_ci) cmd = ['powershell.exe'] + elif raw: + manage = ManagePosixCI(core_ci) + cmd = create_shell_command(['bash']) else: options = { '--remote': 1, @@ -384,6 +389,7 @@ def delegate_remote(args, exclude, require, integration_targets): manage = ManagePosixCI(core_ci) manage.setup() + if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) @@ -394,7 +400,16 @@ def delegate_remote(args, exclude, require, integration_targets): manage.ssh(cmd, ssh_options) success = True finally: + download = False + if platform != 'windows': + download = True + + if isinstance(args, ShellConfig): + if args.raw: + download = False + + if download: manage.ssh('rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results') manage.download('/tmp/results', 'test') finally: diff --git a/test/runner/lib/executor.py b/test/runner/lib/executor.py index 7045d478c9..0add66e292 100644 --- a/test/runner/lib/executor.py +++ b/test/runner/lib/executor.py @@ -161,6 +161,10 @@ def install_command_requirements(args, python_version=None): :type args: EnvironmentConfig :type python_version: str | None """ + if isinstance(args, ShellConfig): + if args.raw: + return + generate_egg_info(args) if not args.requirements: @@ -1628,6 +1632,13 @@ def get_integration_remote_filter(args, targets): display.warning('Excluding tests marked "%s" which are not supported on %s: %s' % (skip.rstrip('/'), platform, ', '.join(skipped))) + skip = 'skip/%s/' % args.remote.replace('/', '') + skipped = [target.name for target in targets if skip in target.aliases] + if skipped: + exclude.append(skip) + display.warning('Excluding tests marked "%s" which are not supported on %s: %s' + % (skip.rstrip('/'), args.remote.replace('/', ' '), ', '.join(skipped))) + python_version = 2 # remotes are expected to default to python 2 skip = 'skip/python%d/' % python_version diff --git a/test/runner/lib/manage_ci.py b/test/runner/lib/manage_ci.py index a8b64098e0..ed6aa25f60 100644 --- a/test/runner/lib/manage_ci.py +++ b/test/runner/lib/manage_ci.py @@ -24,6 +24,10 @@ from lib.ansible_util import ( ansible_environment, ) +from lib.config import ( + ShellConfig, +) + class ManageWindowsCI(object): """Manage access to a Windows instance provided by Ansible Core CI.""" @@ -203,6 +207,11 @@ class ManagePosixCI(object): def setup(self): """Start instance and wait for it to become ready and respond to an ansible ping.""" self.wait() + + if isinstance(self.core_ci.args, ShellConfig): + if self.core_ci.args.raw: + return + self.configure() self.upload_source()