Add support for setup targets to ansible-test. (#28544)

* Add support for setup targets to ansible-test.
* Code cleanup.

test/runner/lib/classification.py

@@ -247,14 +247,14 @@ class PathMapper(object):
             return minimal
 
         if path.startswith('lib/ansible/modules/'):
-            module = self.module_names_by_path.get(path)
+            module_name = self.module_names_by_path.get(path)
 
-            if module:
+            if module_name:
                 return {
-                    'units': module if module in self.units_modules else None,
+                    'units': module_name if module_name in self.units_modules else None,
-                    'integration': self.posix_integration_by_module.get(module) if ext == '.py' else None,
+                    'integration': self.posix_integration_by_module.get(module_name) if ext == '.py' else None,
-                    'windows-integration': self.windows_integration_by_module.get(module) if ext == '.ps1' else None,
+                    'windows-integration': self.windows_integration_by_module.get(module_name) if ext == '.ps1' else None,
-                    'network-integration': self.network_integration_by_module.get(module),
+                    'network-integration': self.network_integration_by_module.get(module_name),
                 }
 
             return minimal

test/runner/lib/cloud/__init__.py

@@ -343,7 +343,7 @@ class CloudEnvironment(CloudBase):
 
     def on_failure(self, target, tries):
         """
-        :type target: TestTarget
+        :type target: IntegrationTarget
         :type tries: int
         """
         pass

test/runner/lib/cover.py

@@ -92,11 +92,11 @@ def command_coverage_combine(args):
                 display.info('%s -> %s' % (filename, new_name), verbosity=3)
                 filename = new_name
             elif '/ansible_module_' in filename:
-                module = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
+                module_name = re.sub('^.*/ansible_module_(?P<module>.*).py$', '\\g<module>', filename)
-                if module not in modules:
+                if module_name not in modules:
-                    display.warning('Skipping coverage of unknown module: %s' % module)
+                    display.warning('Skipping coverage of unknown module: %s' % module_name)
                     continue
-                new_name = os.path.abspath(modules[module])
+                new_name = os.path.abspath(modules[module_name])
                 display.info('%s -> %s' % (filename, new_name), verbosity=3)
                 filename = new_name
             elif re.search('^(/.*?)?/root/ansible/', filename):

test/runner/lib/executor.py

@@ -4,6 +4,7 @@ from __future__ import absolute_import, print_function
 
 import json
 import os
+import collections
 import re
 import tempfile
 import time
@@ -244,8 +245,9 @@ def command_posix_integration(args):
     """
     :type args: PosixIntegrationConfig
     """
-    internal_targets = command_integration_filter(args, walk_posix_integration_targets())
+    all_targets = tuple(walk_posix_integration_targets(include_hidden=True))
-    command_integration_filtered(args, internal_targets)
+    internal_targets = command_integration_filter(args, all_targets)
+    command_integration_filtered(args, internal_targets, all_targets)
 
 
 def command_network_integration(args):
@@ -270,7 +272,8 @@ def command_network_integration(args):
             'See also inventory template: %s.template' % (filename, default_filename)
         )
 
-    internal_targets = command_integration_filter(args, walk_network_integration_targets())
+    all_targets = tuple(walk_network_integration_targets(include_hidden=True))
+    internal_targets = command_integration_filter(args, all_targets)
     platform_targets = set(a for t in internal_targets for a in t.aliases if a.startswith('network/'))
 
     if args.platform:
@@ -309,7 +312,7 @@ def command_network_integration(args):
     else:
         install_command_requirements(args)
 
-    command_integration_filtered(args, internal_targets)
+    command_integration_filtered(args, internal_targets, all_targets)
 
 
 def network_run(args, platform, version):
@@ -377,7 +380,8 @@ def command_windows_integration(args):
     if not args.explain and not args.windows and not os.path.isfile(filename):
         raise ApplicationError('Use the --windows option or provide an inventory file (see %s.template).' % filename)
 
-    internal_targets = command_integration_filter(args, walk_windows_integration_targets())
+    all_targets = tuple(walk_windows_integration_targets(include_hidden=True))
+    internal_targets = command_integration_filter(args, all_targets)
 
     if args.windows:
         instances = []  # type: list [lib.thread.WrappedThread]
@@ -405,7 +409,7 @@ def command_windows_integration(args):
         install_command_requirements(args)
 
     try:
-        command_integration_filtered(args, internal_targets)
+        command_integration_filtered(args, internal_targets, all_targets)
     finally:
         pass
 
@@ -477,7 +481,7 @@ def command_integration_filter(args, targets):
     :type targets: collections.Iterable[IntegrationTarget]
     :rtype: tuple[IntegrationTarget]
     """
-    targets = tuple(targets)
+    targets = tuple(target for target in targets if 'hidden/' not in target.aliases)
     changes = get_changes_filter(args)
     require = (args.require or []) + changes
     exclude = (args.exclude or [])
@@ -507,16 +511,29 @@ def command_integration_filter(args, targets):
     return internal_targets
 
 
-def command_integration_filtered(args, targets):
+def command_integration_filtered(args, targets, all_targets):
     """
     :type args: IntegrationConfig
     :type targets: tuple[IntegrationTarget]
+    :type all_targets: tuple[IntegrationTarget]
     """
     found = False
     passed = []
     failed = []
 
     targets_iter = iter(targets)
+    all_targets_dict = dict((target.name, target) for target in all_targets)
+
+    setup_errors = []
+    setup_targets_executed = set()
+
+    for target in all_targets:
+        for setup_target in target.setup_once + target.setup_always:
+            if setup_target not in all_targets_dict:
+                setup_errors.append('Target "%s" contains invalid setup target: %s' % (target.name, setup_target))
+
+    if setup_errors:
+        raise ApplicationError('Found %d invalid setup aliases:\n%s' % (len(setup_errors), '\n'.join(setup_errors)))
 
     test_dir = os.path.expanduser('~/ansible_testing')
 
@@ -561,12 +578,15 @@ def command_integration_filtered(args, targets):
             while tries:
                 tries -= 1
 
-                if not args.explain:
-                    # create a fresh test directory for each test target
-                    remove_tree(test_dir)
-                    make_dirs(test_dir)
-
                 try:
+                    run_setup_targets(args, test_dir, target.setup_once, all_targets_dict, setup_targets_executed, False)
+                    run_setup_targets(args, test_dir, target.setup_always, all_targets_dict, setup_targets_executed, True)
+
+                    if not args.explain:
+                        # create a fresh test directory for each test target
+                        remove_tree(test_dir)
+                        make_dirs(test_dir)
+
                     if target.script_path:
                         command_integration_script(args, target)
                     else:
@@ -611,6 +631,34 @@ def command_integration_filtered(args, targets):
             len(failed), len(passed) + len(failed), '\n'.join(target.name for target in failed)))
 
 
+def run_setup_targets(args, test_dir, target_names, targets_dict, targets_executed, always):
+    """
+    :param args: IntegrationConfig
+    :param test_dir: str
+    :param target_names: list[str]
+    :param targets_dict: dict[str, IntegrationTarget]
+    :param targets_executed: set[str]
+    :param always: bool
+    """
+    for target_name in target_names:
+        if not always and target_name in targets_executed:
+            continue
+
+        target = targets_dict[target_name]
+
+        if not args.explain:
+            # create a fresh test directory for each test target
+            remove_tree(test_dir)
+            make_dirs(test_dir)
+
+        if target.script_path:
+            command_integration_script(args, target)
+        else:
+            command_integration_role(args, target, None)
+
+        targets_executed.add(target_name)
+
+
 def integration_environment(args, target, cmd):
     """
     :type args: IntegrationConfig
@@ -667,7 +715,7 @@ def command_integration_role(args, target, start_at_task):
     """
     :type args: IntegrationConfig
     :type target: IntegrationTarget
-    :type start_at_task: str
+    :type start_at_task: str | None
     """
     display.info('Running %s integration test role' % target.name)
 

test/runner/lib/metadata.py

@@ -10,6 +10,7 @@ from lib.util import (
 
 from lib.diff import (
     parse_diff,
+    FileDiff,
 )
 
 

test/runner/lib/target.py

@@ -2,6 +2,7 @@
 
 from __future__ import absolute_import, print_function
 
+import collections
 import os
 import re
 import errno
@@ -11,7 +12,6 @@ import sys
 
 from lib.util import (
     ApplicationError,
-    ABC,
 )
 
 MODULE_EXTENSIONS = '.py', '.ps1'
@@ -222,30 +222,33 @@ def walk_sanity_targets():
     return walk_test_targets(module_path='lib/ansible/modules/')
 
 
-def walk_posix_integration_targets():
+def walk_posix_integration_targets(include_hidden=False):
     """
+    :type include_hidden: bool
     :rtype: collections.Iterable[IntegrationTarget]
     """
     for target in walk_integration_targets():
-        if 'posix/' in target.aliases:
+        if 'posix/' in target.aliases or (include_hidden and 'hidden/posix/' in target.aliases):
             yield target
 
 
-def walk_network_integration_targets():
+def walk_network_integration_targets(include_hidden=False):
     """
+    :type include_hidden: bool
     :rtype: collections.Iterable[IntegrationTarget]
     """
     for target in walk_integration_targets():
-        if 'network/' in target.aliases:
+        if 'network/' in target.aliases or (include_hidden and 'hidden/network/' in target.aliases):
             yield target
 
 
-def walk_windows_integration_targets():
+def walk_windows_integration_targets(include_hidden=False):
     """
+    :type include_hidden: bool
     :rtype: collections.Iterable[IntegrationTarget]
     """
     for target in walk_integration_targets():
-        if 'windows/' in target.aliases:
+        if 'windows/' in target.aliases or (include_hidden and 'hidden/windows/' in target.aliases):
             yield target
 
 
@@ -338,7 +341,12 @@ def analyze_integration_target_dependencies(integration_targets):
     """
     hidden_role_target_names = set(t.name for t in integration_targets if t.type == 'role' and 'hidden/' in t.aliases)
     normal_role_targets = [t for t in integration_targets if t.type == 'role' and 'hidden/' not in t.aliases]
-    dependencies = dict((target_name, set()) for target_name in hidden_role_target_names)
+    dependencies = collections.defaultdict(set)
+
+    # handle setup dependencies
+    for target in integration_targets:
+        for setup_target_name in target.setup_always + target.setup_once:
+            dependencies[setup_target_name].add(target.name)
 
     # intentionally primitive analysis of role meta to avoid a dependency on pyyaml
     for role_target in normal_role_targets:
@@ -511,13 +519,13 @@ class IntegrationTarget(CompletionTarget):
         # modules
 
         if self.name in modules:
-            module = self.name
+            module_name = self.name
         elif self.name.startswith('win_') and self.name[4:] in modules:
-            module = self.name[4:]
+            module_name = self.name[4:]
         else:
-            module = None
+            module_name = None
 
-        self.modules = tuple(sorted(a for a in static_aliases + tuple([module]) if a in modules))
+        self.modules = tuple(sorted(a for a in static_aliases + tuple([module_name]) if a in modules))
 
         # groups
 
@@ -576,6 +584,11 @@ class IntegrationTarget(CompletionTarget):
 
         self.aliases = tuple(sorted(set(aliases)))
 
+        # configuration
+
+        self.setup_once = tuple(sorted(set(g.split('/')[2] for g in groups if g.startswith('setup/once/'))))
+        self.setup_always = tuple(sorted(set(g.split('/')[2] for g in groups if g.startswith('setup/always/'))))
+
 
 class TargetPatternsNotMatched(ApplicationError):
     """One or more targets were not matched when a match was required."""