[stable-2.5] Backport test infra updates and test fixes. (#49204)

* Switch tests from RHEL 7.5 to 7.6.

(cherry picked from commit 6745ee7cc8)

* Remove CI platform: freebsd/10.4

(cherry picked from commit e6ffc4f89a)

* Add `--raw` option to ansible-test shell command.

It is currently supported only with the `--remote` option.

This makes it easier to troubleshoot new instances which are not
yet supported by the setup scripts used by ansible-test.

(cherry picked from commit 0826a00803)

* Support skip of platforms by version in tests. (#48826)

* Support skip of platforms by version in tests.

Previously a remote platform could be skipped completely using the alias:

`skip/{platform}` such as `skip/rhel`

Now a specific platform version can be skipped using the alias:

`skip/{platform}{version}` such as `skip/rhel7.6`

This feature is available for platforms specified with the `--remote` option.

* Add skip by version to the docs.

(cherry picked from commit 8066acc90c)

* Fix ansible-test skip warning message.

(cherry picked from commit 3b705efc93)

* Fix lookup_passwordstore test skipping. (#49178)

* Fix lookup_passwordstore test skipping.

Skip all of RHEL instead of specific versions.
Skip all of CentOS < 7 instead of specific versions.

This makes the test more robust when testing newer versions.

Tests could be executed on RHEL if EPEL was installed during the test.
(cherry picked from commit 704dae2cda)

docs/docsite/rst/dev_guide/testing/sanity/integration-aliases.rst

@@ -44,6 +44,11 @@ Aliases can be used to skip platforms using one of the following:
 - ``skip/rhel`` - Skip tests on RHEL.
 - ``skip/docker`` - Skip tests when running on a Docker container
 
+Platform versions, as specified using the ``--remote`` option with ``/`` removed, can also be skipped:
+
+- ``skip/freebsd11.1`` - Skip tests on FreeBSD 11.1.
+- ``skip/rhel7.6`` - Skip tests on RHEL 7.6.
+
 Aliases can be used to skip Python major versions using one of the following:
 
 - ``skip/python2`` - Skip tests on Python 2.x.

shippable.yml

@@ -46,8 +46,7 @@ matrix:
     - env: T=network
 
     - env: T=osx/10.11/1
-    - env: T=rhel/7.5/1
+    - env: T=rhel/7.6/1
-    - env: T=freebsd/10.4/1
     - env: T=freebsd/11.1/1
     - env: T=linux/centos6/1
     - env: T=linux/centos7/1
@@ -59,8 +58,7 @@ matrix:
     - env: T=linux/ubuntu1604py3/1
 
     - env: T=osx/10.11/2
-    - env: T=rhel/7.5/2
+    - env: T=rhel/7.6/2
-    - env: T=freebsd/10.4/2
     - env: T=freebsd/11.1/2
     - env: T=linux/centos6/2
     - env: T=linux/centos7/2
@@ -72,8 +70,7 @@ matrix:
     - env: T=linux/ubuntu1604py3/2
 
     - env: T=osx/10.11/3
-    - env: T=rhel/7.5/3
+    - env: T=rhel/7.6/3
-    - env: T=freebsd/10.4/3
     - env: T=freebsd/11.1/3
     - env: T=linux/centos6/3
     - env: T=linux/centos7/3

test/integration/targets/lookup_passwordstore/aliases

@@ -1,2 +1,3 @@
 shippable/posix/group2
 destructive
+skip/rhel

test/integration/targets/lookup_passwordstore/tasks/main.yml

@@ -1,4 +1,6 @@
-- include: "package.yml"
+- block:
-  when: "ansible_distribution_version not in passwordstore_skip_os.get(ansible_distribution, [])"
+    - include: "package.yml"
-- include: "tests.yml"
+    - include: "tests.yml"
-  when: "ansible_distribution_version not in passwordstore_skip_os.get(ansible_distribution, [])"
+  when:
+    - not (ansible_distribution == 'RedHat')  # requires EPEL
+    - not (ansible_distribution == 'CentOS' and ansible_distribution_version is version_compare('7', '<'))

test/integration/targets/lookup_passwordstore/vars/main.yml

@@ -56,7 +56,3 @@ passwordstore_privkey: |
   SxHTvI2pKk+gx0FB8wWhd/CocAHJpx9oNUs/7A==
   =ZF3O
   -----END PGP PRIVATE KEY BLOCK-----
-passwordstore_skip_os:
-  Ubuntu: ['12.04']
-  RedHat: ['7.4']
-  CentOS: ['6.9', '6.10']

test/runner/completion/remote.txt

@@ -1,4 +1,3 @@
-freebsd/10.4
 freebsd/11.1
 osx/10.11
-rhel/7.5
+rhel/7.6

test/runner/lib/cli.py

@@ -409,6 +409,10 @@ def parse_args():
     shell.set_defaults(func=command_shell,
                        config=ShellConfig)
 
+    shell.add_argument('--raw',
+                       action='store_true',
+                       help='direct to shell with no setup')
+
     add_environments(shell, tox_version=True)
     add_extra_docker_options(shell)
     add_httptester_options(shell, argparse)

test/runner/lib/config.py

@@ -134,6 +134,11 @@ class ShellConfig(EnvironmentConfig):
         """
         super(ShellConfig, self).__init__(args, 'shell')
 
+        self.raw = args.raw  # type: bool
+
+        if self.raw:
+            self.httptester = False
+
 
 class SanityConfig(TestConfig):
     """Configuration for the sanity command."""

test/runner/lib/delegation.py

@@ -334,9 +334,11 @@ def delegate_remote(args, exclude, require, integration_targets):
 
     core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider)
     success = False
+    raw = False
 
     if isinstance(args, ShellConfig):
         use_httptester = args.httptester
+        raw = args.raw
     else:
         use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets)
 
@@ -359,6 +361,9 @@ def delegate_remote(args, exclude, require, integration_targets):
             # Windows doesn't need the ansible-test fluff, just run the SSH command
             manage = ManageWindowsCI(core_ci)
             cmd = ['powershell.exe']
+        elif raw:
+            manage = ManagePosixCI(core_ci)
+            cmd = create_shell_command(['bash'])
         else:
             options = {
                 '--remote': 1,
@@ -384,6 +389,7 @@ def delegate_remote(args, exclude, require, integration_targets):
             manage = ManagePosixCI(core_ci)
 
         manage.setup()
+
         if isinstance(args, IntegrationConfig):
             cloud_platforms = get_cloud_providers(args)
 
@@ -394,7 +400,16 @@ def delegate_remote(args, exclude, require, integration_targets):
             manage.ssh(cmd, ssh_options)
             success = True
         finally:
+            download = False
+
             if platform != 'windows':
+                download = True
+
+            if isinstance(args, ShellConfig):
+                if args.raw:
+                    download = False
+
+            if download:
                 manage.ssh('rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results')
                 manage.download('/tmp/results', 'test')
     finally:

test/runner/lib/executor.py

@@ -161,6 +161,10 @@ def install_command_requirements(args, python_version=None):
     :type args: EnvironmentConfig
     :type python_version: str | None
     """
+    if isinstance(args, ShellConfig):
+        if args.raw:
+            return
+
     generate_egg_info(args)
 
     if not args.requirements:
@@ -1628,6 +1632,13 @@ def get_integration_remote_filter(args, targets):
         display.warning('Excluding tests marked "%s" which are not supported on %s: %s'
                         % (skip.rstrip('/'), platform, ', '.join(skipped)))
 
+    skip = 'skip/%s/' % args.remote.replace('/', '')
+    skipped = [target.name for target in targets if skip in target.aliases]
+    if skipped:
+        exclude.append(skip)
+        display.warning('Excluding tests marked "%s" which are not supported on %s: %s'
+                        % (skip.rstrip('/'), args.remote.replace('/', ' '), ', '.join(skipped)))
+
     python_version = 2  # remotes are expected to default to python 2
 
     skip = 'skip/python%d/' % python_version

test/runner/lib/manage_ci.py

@@ -24,6 +24,10 @@ from lib.ansible_util import (
     ansible_environment,
 )
 
+from lib.config import (
+    ShellConfig,
+)
+
 
 class ManageWindowsCI(object):
     """Manage access to a Windows instance provided by Ansible Core CI."""
@@ -203,6 +207,11 @@ class ManagePosixCI(object):
     def setup(self):
         """Start instance and wait for it to become ready and respond to an ansible ping."""
         self.wait()
+
+        if isinstance(self.core_ci.args, ShellConfig):
+            if self.core_ci.args.raw:
+                return
+
         self.configure()
         self.upload_source()