Look for globally installed collection plugins

* devel: (513 commits)
  Fix systemd service is already masked issue (#44730)
  fix issue with no_log in py3
  modules/terraform: Quote the variable values in the command line (#43493)
  YUM4/DNF compatibility via yum action plugin (#44322)
  BOTMETA.yml: remove superfluous labels (#44628)
  Share the implementation of hashing for both vars_prompt and password_hash (#21215)
  one_host environment variables, Fixes #44163 (#44568)
  ec2: add "IAM Role" to instance_profile_name
  ios_vrf speed fix (#43765)
  fix typo (#44712)
  junos cli_config idempotence fix (#44706)
  Switch to LiteralPath instead of Path. Closes #44508 (#44509)
  Module win_domain_computer fix delete computer with child (#44500)
  ACME: improve documentation (#44691)
  doc: fixed typo (#44685)
  IPA: Add option to specify timeout (#44572)
  Added nios_txt_record module (#39264)
  adds the bigip_cli_script module (#44674)
  Clean up BOTMETA.yml (#44574)
  Change validate-modules for removed modules
  ...

lib/ansible/cli/__init__.py

@@ -623,7 +623,10 @@ class CLI(with_metaclass(ABCMeta, object)):
             cpath = "Default w/o overrides"
         else:
             cpath = C.DEFAULT_MODULE_PATH
+        conpath = C.DEFAULT_CONTENT_PATH or "Default w/o overrides"
+
         result = result + "\n  configured module search path = %s" % cpath
+        result = result + "\n  configured galaxy content search path = %s" % conpath
         result = result + "\n  ansible python module location = %s" % ':'.join(ansible.__path__)
         result = result + "\n  executable location = %s" % sys.argv[0]
         result = result + "\n  python version = %s" % ''.join(sys.version.splitlines())

lib/ansible/config/base.yml

@@ -343,7 +343,7 @@ LOCALHOST_WARNING:
   version_added: "2.6"
 DEFAULT_ACTION_PLUGIN_PATH:
   name: Action plugins path
-  default: ~/.ansible/plugins/action:/usr/share/ansible/plugins/action
+  default: ~/.ansible/content/*/*/plugins/actions:~/.ansible/plugins/action:/usr/share/ansible/content/*/*/plugins/action:/usr/share/ansible/plugins/action
   description: Colon separated paths in which Ansible will search for Action Plugins.
   env: [{name: ANSIBLE_ACTION_PLUGINS}]
   ini:
@@ -475,7 +475,7 @@ DEFAULT_CALLABLE_WHITELIST:
   type: list
 DEFAULT_CALLBACK_PLUGIN_PATH:
   name: Callback Plugins Path
-  default: ~/.ansible/plugins/callback:/usr/share/ansible/plugins/callback
+  default: ~/.ansible/content/*/*/plugins/callback:~/.ansible/plugins/callback:/usr/share/ansible/content/*/*/plugins/callback:/usr/share/ansible/plugins/callback
   description: Colon separated paths in which Ansible will search for Callback Plugins.
   env: [{name: ANSIBLE_CALLBACK_PLUGINS}]
   ini:
@@ -495,7 +495,7 @@ DEFAULT_CALLBACK_WHITELIST:
   yaml: {key: plugins.callback.whitelist}
 DEFAULT_CLICONF_PLUGIN_PATH:
   name: Cliconf Plugins Path
-  default: ~/.ansible/plugins/cliconf:/usr/share/ansible/plugins/cliconf
+  default: ~/.ansible/content/*/*/plugins/cliconf:~/.ansible/plugins/cliconf:/usr/share/ansible/plugins/cliconf
   description: Colon separated paths in which Ansible will search for Cliconf Plugins.
   env: [{name: ANSIBLE_CLICONF_PLUGINS}]
   ini:
@@ -503,13 +503,21 @@ DEFAULT_CLICONF_PLUGIN_PATH:
   type: pathspec
 DEFAULT_CONNECTION_PLUGIN_PATH:
   name: Connection Plugins Path
-  default: ~/.ansible/plugins/connection:/usr/share/ansible/plugins/connection
+  default: ~/.ansible/content/*/*/plugins/connection:~/.ansible/plugins/connection:/usr/share/ansible/content/*/*/plugins/connection:/usr/share/ansible/plugins/connection
   description: Colon separated paths in which Ansible will search for Connection Plugins.
   env: [{name: ANSIBLE_CONNECTION_PLUGINS}]
   ini:
   - {key: connection_plugins, section: defaults}
   type: pathspec
   yaml: {key: plugins.connection.path}
+DEFAULT_CONTENT_PATH:
+  name: Ansible Galaxy Content Path
+  description: Colon separated paths in which Ansible will search for Roles, Modules and Plugins installed as Galaxy Content.
+  default: ~/.ansible/content
+  env: [{name: ANSIBLE_CONTENT_PATH}]
+  ini:
+  - {key: content_path, section: defaults}
+  type: pathspec
 DEFAULT_DEBUG:
   name: Debug mode
   default: False
@@ -545,7 +553,7 @@ DEFAULT_FACT_PATH:
   yaml: {key: facts.gathering.fact_path}
 DEFAULT_FILTER_PLUGIN_PATH:
   name: Jinja2 Filter Plugins Path
-  default: ~/.ansible/plugins/filter:/usr/share/ansible/plugins/filter
+  default: ~/.ansible/content/*/*/plugins/filter:~/.ansible/plugins/filter:/usr/share/ansible/content/*/*/plugins/action:/usr/share/ansible/plugins/filter
   description: Colon separated paths in which Ansible will search for Jinja2 Filter Plugins.
   env: [{name: ANSIBLE_FILTER_PLUGINS}]
   ini:
@@ -665,7 +673,7 @@ DEFAULT_HOST_LIST:
   yaml: {key: defaults.inventory}
 DEFAULT_HTTPAPI_PLUGIN_PATH:
   name: HttpApi Plugins Path
-  default: ~/.ansible/plugins/httpapi:/usr/share/ansible/plugins/httpapi
+  default: ~/.ansible/content/*/*/plugins/httpapi:~/.ansible/plugins/httpapi:/usr/share/ansible/plugins/httpapi
   description: Colon separated paths in which Ansible will search for HttpApi Plugins.
   env: [{name: ANSIBLE_HTTPAPI_PLUGINS}]
   ini:
@@ -687,7 +695,7 @@ DEFAULT_INTERNAL_POLL_INTERVAL:
     - "The default corresponds to the value hardcoded in Ansible <= 2.1"
 DEFAULT_INVENTORY_PLUGIN_PATH:
   name: Inventory Plugins Path
-  default: ~/.ansible/plugins/inventory:/usr/share/ansible/plugins/inventory
+  default: ~/.ansible/content/*/*/plugins/inventory:~/.ansible/plugins/inventory:/usr/share/ansible/content/*/*/plugins/inventory:/usr/share/ansible/plugins/inventory
   description: Colon separated paths in which Ansible will search for Inventory Plugins.
   env: [{name: ANSIBLE_INVENTORY_PLUGINS}]
   ini:
@@ -771,7 +779,7 @@ DEFAULT_LOG_FILTER:
 DEFAULT_LOOKUP_PLUGIN_PATH:
   name: Lookup Plugins Path
   description: Colon separated paths in which Ansible will search for Lookup Plugins.
-  default: ~/.ansible/plugins/lookup:/usr/share/ansible/plugins/lookup
+  default: ~/.ansible/content/*/*/plugins/lookup:~/.ansible/plugins/lookup:/usr/share/ansible/content/*/*/plugins/lookup:/usr/share/ansible/plugins/lookup
   env: [{name: ANSIBLE_LOOKUP_PLUGINS}]
   ini:
   - {key: lookup_plugins, section: defaults}
@@ -825,7 +833,7 @@ DEFAULT_MODULE_NAME:
 DEFAULT_MODULE_PATH:
   name: Modules Path
   description: Colon separated paths in which Ansible will search for Modules.
-  default: ~/.ansible/plugins/modules:/usr/share/ansible/plugins/modules
+  default: ~/.ansible/content/*/*/modules:~/.ansible/plugins/modules:/usr/share/ansible/content/*/*/modules:/usr/share/ansible/plugins/modules
   env: [{name: ANSIBLE_LIBRARY}]
   ini:
   - {key: library, section: defaults}
@@ -845,14 +853,14 @@ DEFAULT_MODULE_SET_LOCALE:
 DEFAULT_MODULE_UTILS_PATH:
   name: Module Utils Path
   description: Colon separated paths in which Ansible will search for Module utils files, which are shared by modules.
-  default: ~/.ansible/plugins/module_utils:/usr/share/ansible/plugins/module_utils
+  default: ~/.ansible/content/*/*/module_utils:~/.ansible/plugins/module_utils:/usr/share/ansible/content/*/*/module_utils:/usr/share/ansible/plugins/module_utils
   env: [{name: ANSIBLE_MODULE_UTILS}]
   ini:
   - {key: module_utils, section: defaults}
   type: pathspec
 DEFAULT_NETCONF_PLUGIN_PATH:
   name: Netconf Plugins Path
-  default: ~/.ansible/plugins/netconf:/usr/share/ansible/plugins/netconf
+  default: ~/.ansible/content/*/*/netconf_plugins:~/.ansible/plugins/netconf:/usr/share/ansible/content/*/*/plugins/netconf:/usr/share/ansible/plugins/netconf
   description: Colon separated paths in which Ansible will search for Netconf Plugins.
   env: [{name: ANSIBLE_NETCONF_PLUGINS}]
   ini:
@@ -937,7 +945,7 @@ DEFAULT_REMOTE_USER:
   - {key: remote_user, section: defaults}
 DEFAULT_ROLES_PATH:
   name: Roles path
-  default: ~/.ansible/roles:/usr/share/ansible/roles:/etc/ansible/roles
+  default: ~/ansible/content/*/*/roles:~/.ansible/roles:/usr/share/ansible/content/*/*/roles:/usr/share/ansible/roles:/etc/ansible/roles
   description: Colon separated paths in which Ansible will search for Roles.
   env: [{name: ANSIBLE_ROLES_PATH}]
   expand_relative_paths: True
@@ -1046,7 +1054,7 @@ DEFAULT_STRATEGY:
 DEFAULT_STRATEGY_PLUGIN_PATH:
   name: Strategy Plugins Path
   description: Colon separated paths in which Ansible will search for Strategy Plugins.
-  default: ~/.ansible/plugins/strategy:/usr/share/ansible/plugins/strategy
+  default: ~/.ansible/content/*/*/plugins/strategy:~/.ansible/plugins/strategy:/usr/share/ansible/content/*/*/plugins/strategy:/usr/share/ansible/plugins/strategy
   env: [{name: ANSIBLE_STRATEGY_PLUGINS}]
   ini:
   - {key: strategy_plugins, section: defaults}
@@ -1159,7 +1167,7 @@ DEFAULT_TASK_INCLUDES_STATIC:
     alternatives: None, as its already built into the decision between include_tasks and import_tasks
 DEFAULT_TERMINAL_PLUGIN_PATH:
   name: Terminal Plugins Path
-  default: ~/.ansible/plugins/terminal:/usr/share/ansible/plugins/terminal
+  default: ~/.ansible/content/*/*/plugins/terminal:~/.ansible/plugins/terminal:/usr/share/ansible/content/*/*/plugins/terminal:/usr/share/ansible/plugins/terminal
   description: Colon separated paths in which Ansible will search for Terminal Plugins.
   env: [{name: ANSIBLE_TERMINAL_PLUGINS}]
   ini:
@@ -1168,7 +1176,7 @@ DEFAULT_TERMINAL_PLUGIN_PATH:
 DEFAULT_TEST_PLUGIN_PATH:
   name: Jinja2 Test Plugins Path
   description: Colon separated paths in which Ansible will search for Jinja2 Test Plugins.
-  default: ~/.ansible/plugins/test:/usr/share/ansible/plugins/test
+  default: ~/.ansible/content/*/*/plugins/test:~/.ansible/plugins/test:/usr/share/ansible/content/*/*/plugins/test:/usr/share/ansible/plugins/test
   env: [{name: ANSIBLE_TEST_PLUGINS}]
   ini:
   - {key: test_plugins, section: defaults}
@@ -1201,7 +1209,7 @@ DEFAULT_UNDEFINED_VAR_BEHAVIOR:
   type: boolean
 DEFAULT_VARS_PLUGIN_PATH:
   name: Vars Plugins Path
-  default: ~/.ansible/plugins/vars:/usr/share/ansible/plugins/vars
+  default: ~/.ansible/content/*/*/plugins/vars:~/.ansible/plugins/vars:/usr/share/ansible/content/*/*/plugins/vars:/usr/share/ansible/plugins/vars
   description: Colon separated paths in which Ansible will search for Vars Plugins.
   env: [{name: ANSIBLE_VARS_PLUGINS}]
   ini:

lib/ansible/playbook/role/definition.py

@@ -20,12 +20,14 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import os
+import pprint
 
-from ansible import constants as C
+from ansible import constants as C  # noqa
 from ansible.errors import AnsibleError, AnsibleAssertionError
 from ansible.module_utils.six import iteritems, string_types
 from ansible.parsing.yaml.objects import AnsibleBaseYAMLObject, AnsibleMapping
-from ansible.playbook.attribute import Attribute, FieldAttribute
+# from ansible.playbook.attribute import Attribute
+from ansible.playbook.attribute import FieldAttribute
 from ansible.playbook.base import Base
 from ansible.playbook.become import Become
 from ansible.playbook.conditional import Conditional
@@ -33,6 +35,9 @@ from ansible.playbook.taggable import Taggable
 from ansible.template import Templar
 from ansible.utils.path import unfrackpath
 
+import logging
+log = logging.getLogger(__name__)
+
 try:
     from __main__ import display
 except ImportError:
@@ -43,6 +48,112 @@ except ImportError:
 __all__ = ['RoleDefinition']
 
 
+def role_name_to_relative_content_path(role_name):
+    '''Translate a namespace.reponame.rolename to relative content path.
+
+    ie, testing.some_repo.some_role -> testing/some_repo/roles/some_role
+    '''
+
+    content_rel_role_path = None
+
+    # TODO: decide if something like 'geerlingguy.nginx' is sufficient
+    #       ie, if there will be a short names/aliases/default resolvers
+    try:
+        # namespace, repo, name = role_name.split('.', 2)
+        name_parts = role_name.split('.', 2)
+    except (ValueError, AttributeError):
+        log.debug('Could not "." split role_name "%s"',
+                  role_name)
+        return None
+
+    # name_parts namespace, repo, rolename or
+    #  namespace.reponame (and assume there is a also a role named 'reponame'
+    #  at namespace/repo/roles/reponame. ie, old style roles with one role
+    # in a repo.)
+    # geerlingguy.apache -> geerlingguy/apache/roles/apache
+    # geerlingguy.apache.apache -> geerlingguy/apache/roles/apache
+    # testing.multi.apache -> testing/multi/roles/apache
+    # testing.multi ->  testing/multi/roles/multi (if it exists)
+    # WARNING: if a repo name for a multicontent repo matches a role name
+    #          coincedently, this introduces an ambiquity
+    # mynamespace.install.add_user -> mynamespace/install/roles/add_user
+    # mynamespace.install.install -> mynamespace/install/roles/install
+    # mynamespace.install -> mynamespace/install/roles/install
+
+    if len(name_parts) < 2:
+        return None
+
+    log.debug(name_parts)
+
+    # catches 'namespace.' (trailing dot) as well as rel path cases
+    # like '../some_dir.foo.bar'
+    if not all(name_parts):
+        return None
+
+    log.debug('name_parts: %s', name_parts)
+
+    namespace = name_parts.pop(0)
+    repo = name_parts.pop(0)
+
+    name = None
+    try:
+        name = name_parts.pop(0)
+    except IndexError:
+        log.debug('role name "%s" only had two name parts (namespace="%s", repo="%s") and no role name',
+                  role_name, namespace, repo)
+
+    if name:
+        content_rel_role_path = os.path.join(namespace, repo, 'roles', name)
+    else:
+        content_rel_role_path = os.path.join(namespace, repo, 'roles', repo)
+
+    return content_rel_role_path
+
+
+def find_role_in_content_path(role_name, loader, content_search_paths):
+    '''search for role in ~/.ansible/content and return first match.
+
+    return None if no matches'''
+
+    # try the galaxy content paths
+    # TODO: this is where a 'role spec resolver' could be plugged into.
+    #       The resolver would be responsible parsing/understanding the role spec
+    #       (a formatted string or a dict), and figuring out the approriate galaxy
+    #       namespace, repo name, and role name.
+    #
+    #       The next step would be finding that role on the fs.
+    #       If there are conflicts or ambiquity, the resolver would apply
+    #       any rules or convention or precedence to choose the correct role.
+    #       For ex, if namespace isnt provided, and 2 or more namespaces have a
+    #       role that matches, the resolver would choose.
+    # FIXME: mv to method, deindent, return early, etc
+
+    log.debug('content_search_paths: %s', content_search_paths)
+
+    content_rel_role_path = role_name_to_relative_content_path(role_name)
+
+    log.debug('content_rel_role_path: %s', content_rel_role_path)
+
+    # didn't parse the role_name, return None
+    if not content_rel_role_path:
+        return None
+
+    # TODO: the for loop isnt needed if we really really only
+    #       support one content path
+    for content_search_path in content_search_paths:
+
+        fq_role_path = os.path.join(content_search_path, content_rel_role_path)
+        fq_role_path = unfrackpath(fq_role_path)
+
+        log.debug('fq_role_path: %s', fq_role_path)
+
+        if loader.path_exists(fq_role_path):
+            log.info('FOUND:     %s at content path "%s"', role_name, fq_role_path)
+            return (role_name, fq_role_path)
+
+    return None
+
+
 class RoleDefinition(Base, Become, Conditional, Taggable):
 
     _role = FieldAttribute(isa='string')
@@ -119,6 +230,7 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
         '''
 
         if isinstance(ds, string_types):
+            log.debug('role_name: %s (role ds was a string)', ds)
             return ds
 
         role_name = ds.get('role', ds.get('name'))
@@ -133,6 +245,8 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
             if templar._contains_vars(role_name):
                 role_name = templar.template(role_name)
 
+        log.info('using role_name: %s', role_name)
+        log.info('role_name: %s ds:\n%s', role_name, pprint.pformat(ds))
         return role_name
 
     def _load_role_path(self, role_name):
@@ -143,6 +257,9 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
         append it to the default role path
         '''
 
+        log.info('Look for:  %s', role_name)
+        # log.debug('installed_role_spec: %s', installed_role_spec)
+
         # we always start the search for roles in the base directory of the playbook
         role_search_paths = [
             os.path.join(self._loader.get_basedir(), u'roles'),
@@ -161,6 +278,7 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
         # in the loader (which should be the playbook dir itself) but without
         # the roles/ dir appended
         role_search_paths.append(self._loader.get_basedir())
+        log.debug('role_search_paths: %s', role_search_paths)
 
         # create a templar class to template the dependency names, in
         # case they contain variables
@@ -172,19 +290,46 @@ class RoleDefinition(Base, Become, Conditional, Taggable):
         templar = Templar(loader=self._loader, variables=all_vars)
         role_name = templar.template(role_name)
 
+        # Look for roles in the content search path (~/.ansible/content) based on dotted role
+        # names.
+        content_results = find_role_in_content_path(role_name, self._loader,
+                                                    content_search_paths=C.DEFAULT_CONTENT_PATH)
+        log.debug('content_results: %s', content_results)
+
+        if content_results:
+            log.debug('returning %s (found a role in content path for role_name=%s)',
+                      repr(content_results), role_name)
+
+            return content_results
+
         # now iterate through the possible paths and return the first one we find
         for path in role_search_paths:
             path = templar.template(path)
+
+            # fq_role_name = resolve_role_name(role_name)
             role_path = unfrackpath(os.path.join(path, role_name))
+
+            log.debug('search for role=%s in path: %s (role_path=%s)', role_name, path, role_path)
             if self._loader.path_exists(role_path):
+                log.info('FOUND:     %s at role path: "%s"', role_name, role_path)
+
                 return (role_name, role_path)
 
         # if not found elsewhere try to extract path from name
         role_path = unfrackpath(role_name)
+
+        log.debug('trying role_name=%s as a path: %s ', role_name, role_path)
+
         if self._loader.path_exists(role_path):
+            log.debug('FOUND:     %s (via relative path) at path: "%s"', role_name, role_path)
+
             role_name = os.path.basename(role_name)
             return (role_name, role_path)
 
+        log.info('Failed to find the role "%s" in content paths %s', role_name, C.DEFAULT_CONTENT_PATH)
+        log.info('Failed to find the role "%s" in any of the roles paths: %s',
+                 role_name, ":".join(role_search_paths))
+
         raise AnsibleError("the role '%s' was not found in %s" % (role_name, ":".join(role_search_paths)), obj=self._ds)
 
     def _split_role_params(self, ds):

lib/ansible/playbook/role/requirement.py

@@ -112,7 +112,7 @@ class RoleRequirement(RoleDefinition):
         else:
             role = role.copy()
 
-            if 'src'in role:
+            if 'src' in role:
                 # New style: { src: 'galaxy.role,version,name', other_vars: "here" }
                 if 'github.com' in role["src"] and 'http' in role["src"] and '+' not in role["src"] and not role["src"].endswith('.tar.gz'):
                     role["src"] = "git+" + role["src"]

lib/ansible/plugins/loader.py

@@ -58,7 +58,7 @@ class PluginLoader:
         elif not config:
             config = []
 
-        self.config = config
+        self.config = self.expand_path_globs(config)
 
         if class_name not in MODULE_CACHE:
             MODULE_CACHE[class_name] = {}
@@ -111,6 +111,19 @@ class PluginLoader:
             PLUGIN_PATH_CACHE=PLUGIN_PATH_CACHE[self.class_name],
         )
 
+    def expand_path_globs(self, paths):
+        generated = []
+        for path in paths:
+            if 'content/*' not in path:
+                generated.append(path)
+                continue
+
+            dirs = glob.glob(path)
+            if dirs:
+                generated += dirs
+
+        return generated
+
     def format_paths(self, paths):
         ''' Returns a string suitable for printing of the search path '''
 
@@ -267,6 +280,7 @@ class PluginLoader:
         #       looks like _get_paths() never forces a cache refresh so if we expect
         #       additional directories to be added later, it is buggy.
         for path in (p for p in self._get_paths() if p not in self._searched_paths and os.path.isdir(p)):
+
             try:
                 full_paths = (os.path.join(path, f) for f in os.listdir(path))
             except OSError as e:
@@ -301,6 +315,28 @@ class PluginLoader:
                 if full_name not in self._plugin_path_cache[extension]:
                     self._plugin_path_cache[extension][full_name] = full_path
 
+                # MAZER
+                if '/content/' in path:
+                    path_elements = path.split(os.path.sep)
+                    content_index = path_elements.index('content')
+                    user_namespace = path_elements[content_index + 1]
+                    repository_name = path_elements[content_index + 2]
+
+                    pyfqn = (user_namespace + '.' + repository_name + '.' + base_name).replace('-', '_')
+                    fqn = user_namespace + '.' + repository_name + '.' + base_name
+                    pyrqn = (repository_name + '.' + base_name).replace('-', '_')
+                    rqn = repository_name + '.' + base_name
+
+                    for ext in ['', extension]:
+                        for qn in [pyfqn, fqn, pyrqn, rqn]:
+                            self._plugin_path_cache[ext][qn] = full_path
+
+                            # not sure why this is necessary for lookups
+                            pull_cache[qn] = full_path
+                            if full_path.endswith('.py'):
+                                self._plugin_path_cache[ext][qn + '.py'] = full_path
+                                pull_cache[qn + '.py'] = full_path
+
             self._searched_paths.add(path)
             try:
                 return pull_cache[name]
@@ -523,6 +559,29 @@ class PluginLoader:
                 self._load_config_defs(basename, path)
 
             self._update_object(obj, basename, path)
+
+            # MAZER
+            if hasattr(obj, 'filters') and '/content/' in path:
+                path_elements = path.split(os.path.sep)
+                content_index = path_elements.index('content')
+                namespace = path_elements[content_index + 1]
+                repository = path_elements[content_index + 2]
+
+                filter_dict = obj.filters()
+                for key in filter_dict.keys():
+                    newkeys = [
+                        namespace + '.' + repository + '.' + key,
+                        repository + '.' + key
+                    ]
+                    for nk in newkeys:
+                        nk = nk.replace('-', '_')
+                        filter_dict[nk] = filter_dict[key]
+
+                def patch_filters():
+                    return  filter_dict
+
+                obj.filters = patch_filters
+
             yield obj
 
 

test/units/playbook/role/test_definition.py

@@ -0,0 +1,153 @@
+import logging
+import os.path
+
+import pytest
+
+from ansible import errors
+from ansible.playbook.role import definition
+
+log = logging.getLogger(__name__)
+
+
+def test_role_definition(mocker):
+    res = definition.RoleDefinition(play=None, role_basedir=None, variable_manager=None, loader=None)
+
+    assert isinstance(res, definition.RoleDefinition)
+
+
+def test_role_definition_load_role_path_from_content_path(mocker):
+    mock_loader = mocker.Mock(name='MockDataLoader')
+    mock_loader.get_basedir = mocker.Mock(return_value='/dev/null/roles')
+    mock_loader.path_exists = mocker.Mock(return_value=True)
+
+    content_path = '/dev/null/content/'
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_CONTENT_PATH',
+                 [content_path])
+    rd = definition.RoleDefinition(play=None, role_basedir=None, variable_manager=None, loader=mock_loader)
+
+    role_name = 'namespace.repo.role'
+    res = rd._load_role_path(role_name)
+
+    log.debug('res: %s', res)
+    assert isinstance(res, tuple)
+    assert res[0] == role_name
+    assert res[1] == os.path.join(content_path, 'namespace/repo/roles/role')
+
+
+def test_role_definition_load_role_path_from_role_path(mocker):
+    mock_loader = mocker.Mock(name='MockDataLoader')
+    mock_loader.get_basedir = mocker.Mock(return_value='/dev/null/playbook_rel_roles_dir')
+    mock_loader.path_exists = mocker.Mock(return_value=True)
+
+    content_path = '/dev/null/content/'
+    role_path = '/dev/null/the_default_roles_path'
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_CONTENT_PATH',
+                 [content_path])
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_ROLES_PATH',
+                 [role_path])
+    rd = definition.RoleDefinition(play=None, role_basedir='/dev/null/role_basedir', variable_manager=None, loader=mock_loader)
+
+    role_name = 'some_role'
+    rd.preprocess_data({'role': role_name})
+    res = rd._load_role_path(role_name)
+
+    log.debug('res: %s', res)
+    assert isinstance(res, tuple)
+    # The playbook rel roles/ dir is the first path checked
+    assert res[1] == '/dev/null/playbook_rel_roles_dir/roles/some_role'
+
+
+def test_role_definition_load_role_path_from_role_path_not_found(mocker):
+    mock_loader = mocker.Mock(name='MockDataLoader')
+    mock_loader.get_basedir = mocker.Mock(return_value='/dev/null/playbook_rel_roles_dir')
+    mock_loader.path_exists = mocker.Mock(return_value=False)
+
+    content_path = '/dev/null/content/'
+    role_path = '/dev/null/the_default_roles_path'
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_CONTENT_PATH',
+                 [content_path])
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_ROLES_PATH',
+                 [role_path])
+    rd = definition.RoleDefinition(play=None, role_basedir='/dev/null/role_basedir', variable_manager=None, loader=mock_loader)
+
+    role_name = 'some_role'
+    with pytest.raises(errors.AnsibleError, match='.'):
+        # pres = rd.preprocess_data({'role': role_name})
+        pres = rd.preprocess_data(role_name)
+        log.debug('pres: %s', pres)
+        rd._load_role_path(role_name)
+
+
+def test_role_definition_load_role_no_name_in_role_ds(mocker):
+    mock_loader = mocker.Mock(name='MockDataLoader')
+    mock_loader.get_basedir = mocker.Mock(return_value='/dev/null/playbook_rel_roles_dir')
+    mock_loader.path_exists = mocker.Mock(return_value=True)
+
+    content_path = '/dev/null/content/'
+    role_path = '/dev/null/the_default_roles_path'
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_CONTENT_PATH',
+                 [content_path])
+    mocker.patch('ansible.playbook.role.definition.C.DEFAULT_ROLES_PATH',
+                 [role_path])
+    rd = definition.RoleDefinition(play=None, role_basedir='/dev/null/role_basedir', variable_manager=None, loader=mock_loader)
+
+    role_name = 'some_role'
+    with pytest.raises(errors.AnsibleError, match='role definitions must contain a role name'):
+        rd.preprocess_data({'stuff': role_name,
+                            'things_i_like': ['cheese', 'naps']})
+
+
+@pytest.mark.parametrize("role_name,expected",
+                         [('namespace.repo.role', 'namespace/repo/roles/role'),
+                          ('a.a.a', 'a/a/roles/a'),
+                          ('too.many.dots.lets.assume.extra.dots.are.role.name',
+                           'too/many/roles/dots.lets.assume.extra.dots.are.role.name'),
+                          # valid if role names can include sub paths? should raise an error?
+                          ('ns.repo.role/name/roles/role', 'ns/repo/roles/role/name/roles/role'),
+                          # DWIM
+                          ('geerlingguy.apache', 'geerlingguy/apache/roles/apache'),
+
+                          # these are actually legit 'role_name' inside ansible
+                          # they get used as relative paths
+                          ('../../some_dir/foo', None),
+                          ('../some_dir.foo.bar', None),
+                          ('.', None),
+                          ('/', None),
+                          ('./foo', None),
+                          ('.', None),
+
+                          ('justaname', None),
+                          ('namespace_dot.', None),
+                          ('somenamespace/somerepo/roles/somerole', None),
+
+                          (None, None),
+                          ])
+def test_role_name_to_relative_content_path(role_name, expected):
+    res = definition.role_name_to_relative_content_path(role_name)
+    log.debug('res: %s', res)
+    assert res == expected
+
+
+def test_find_role_in_content_path_invalid_role_name(mocker):
+    mock_loader = mocker.Mock(return_value=False)
+    res = definition.find_role_in_content_path('justaname', mock_loader, '/dev/null/foo')
+
+    assert res is None
+
+
+def test_find_role_in_content_path_loader_cant_find(mocker):
+    mock_loader = mocker.Mock(path_exists=mocker.Mock(return_value=False))
+    res = definition.find_role_in_content_path('namespace.repo.rolename', mock_loader, ['/dev/null/foo'])
+
+    assert res is None
+
+
+def test_find_role_in_content_path_loader(mocker):
+    content_search_path = '/dev/null/foo'
+    role_name = 'namespace.repo.rolename'
+    mock_loader = mocker.Mock(path_exists=mocker.Mock(return_value=True))
+    res = definition.find_role_in_content_path(role_name, mock_loader, [content_search_path])
+
+    assert isinstance(res, tuple)
+    assert res[0] == role_name
+    assert res[1] == os.path.join(content_search_path, 'namespace/repo/roles/rolename')

test/units/playbook/role/test_metadata.py

@@ -0,0 +1,13 @@
+import logging
+
+from ansible.playbook.role import metadata
+
+log = logging.getLogger(__name__)
+
+
+def test_role_metadata():
+    rmd = metadata.RoleMetadata()
+
+    log.debug('rmd: %s', rmd)
+
+    assert isinstance(rmd, metadata.RoleMetadata)

test/units/playbook/role/test_requirement.py

@@ -0,0 +1,87 @@
+import logging
+
+from ansible.playbook.role import definition
+from ansible.playbook.role import requirement
+
+log = logging.getLogger(__name__)
+
+
+def test_role_requirement():
+    rr = requirement.RoleRequirement()
+
+    assert isinstance(rr, requirement.RoleRequirement)
+    assert isinstance(rr, definition.RoleDefinition)
+
+
+def test_repo_url_to_repo_name():
+    repo_url = 'http://git.example.com/repos/repo.git'
+    res = requirement.RoleRequirement.repo_url_to_role_name(repo_url)
+
+    log.debug('res: %s', res)
+
+    assert res == 'repo'
+
+
+def test_role_spec_parse():
+
+    res = requirement.RoleRequirement.role_spec_parse('foo.bar')
+
+    log.debug('res: %s', res)
+
+    assert isinstance(res, dict)
+    assert res['name'] == 'foo.bar'
+    assert res['src'] == 'foo.bar'
+
+
+def test_role_spec_parse_example():
+    role_spec = 'git+http://git.example.com/repos/repo.git,v1.0'
+    res = requirement.RoleRequirement.role_spec_parse(role_spec)
+
+    # {'scm': 'git', 'src': 'http://git.example.com/repos/repo.git',
+    #  'version': 'v1.0', 'name': 'repo'}
+    log.debug('res: %s', res)
+
+    assert isinstance(res, dict)
+    assert res['name'] == 'repo'
+    assert res['scm'] == 'git'
+    assert res['src'] == 'http://git.example.com/repos/repo.git'
+    assert res['version'] == 'v1.0'
+
+
+def test_role_yaml_parse_dict_old_style():
+    role_yaml = {'role': "galaxy.role,v1.2.3,role_name", 'other_vars': "here"}
+    res = requirement.RoleRequirement.role_yaml_parse(role_yaml)
+
+    log.debug('res: %s', res)
+    # {'scm': None, 'src': 'galaxy.role', 'version': 'v1.2.3', 'name': 'role_name'}
+    assert isinstance(res, dict)
+    assert res['name'] == 'role_name'
+    assert res['version'] == 'v1.2.3'
+    assert res['src'] == 'galaxy.role'
+
+#
+# def test_role_yaml_parse_dict_new_style():
+#     # NOTE: the example in the comments for role_yaml_parse fails
+#     role_yaml = {'src': 'git+http://github.com/some_galaxy/some_role,v1.2.3,role_name', 'other_vars': "here"}
+#
+#     res = requirement.RoleRequirement.role_yaml_parse(role_yaml)
+#
+#     log.debug('res: %s', res)
+#     # {'scm': None, 'src': 'galaxy.role', 'version': 'v1.2.3', 'name': 'role_name'}
+#     assert isinstance(res, dict)
+#     assert res['name'] == 'role_name'
+#     assert res['version'] == 'v1.2.3'
+#     assert res['src'] == 'galaxy.role'
+#
+
+
+def test_role_yaml_parse_string():
+    role_yaml = 'galaxy.role,v1.2.3,role_name'
+    res = requirement.RoleRequirement.role_yaml_parse(role_yaml)
+
+    log.debug('res: %s', res)
+
+    assert isinstance(res, dict)
+    assert res['name'] == 'role_name'
+    assert res['version'] == 'v1.2.3'
+    assert res['src'] == 'galaxy.role'

test/units/playbook/role/test_role.py

@@ -20,6 +20,7 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 import collections
+import logging
 
 from ansible.compat.tests import unittest
 from ansible.compat.tests.mock import patch, MagicMock
@@ -35,6 +36,8 @@ from ansible.playbook.role import Role
 from ansible.playbook.role.include import RoleInclude
 from ansible.playbook.role import hash_params
 
+log = logging.getLogger(__name__)
+
 
 class TestHashParams(unittest.TestCase):
     def test(self):
@@ -378,3 +381,32 @@ class TestRole(unittest.TestCase):
         r = Role.load(i, play=mock_play)
 
         self.assertEqual(r.get_name(), "foo_complex")
+
+    @patch('ansible.playbook.role.definition.unfrackpath', mock_unfrackpath_noop)
+    def test_serialize(self):
+
+        fake_loader = DictDataLoader({
+            "/etc/ansible/roles/foo_vars/defaults/main/foo/bar.yml": """
+            foo: bar
+            """,
+            "/etc/ansible/roles/foo_vars/vars/main/bar/foo.yml": """
+            foo: bam
+            """,
+        })
+
+        mock_play = MagicMock()
+        mock_play.ROLE_CACHE = {}
+
+        i = RoleInclude.load('foo_vars', play=mock_play, loader=fake_loader)
+        r = Role.load(i, play=mock_play)
+
+        res = r.serialize()
+        log.debug('res: %s', res)
+
+        self.assertEqual(r._default_vars, dict(foo='bar'))
+        self.assertEqual(r._role_vars, dict(foo='bam'))
+
+        r2 = Role.load(i, play=mock_play)
+        r2.deserialize(res)
+
+        log.debug('r2: %s', r2)