allow external collections under the ansible NS (#61908)

* fixes #59988

changelogs/fragments/allow_ansible_ns.yml

@@ -0,0 +1,2 @@
+bugfixes:
+- allow external collections to be created in the 'ansible' collection namespace (https://github.com/ansible/ansible/issues/59988)

lib/ansible/utils/collection_loader.py

@@ -23,7 +23,10 @@ except ImportError:
     import_module = __import__
 
 _SYNTHETIC_PACKAGES = {
-    'ansible_collections.ansible': dict(type='pkg_only'),
+    # these provide fallback package definitions when there are no on-disk paths
+    'ansible_collections': dict(type='pkg_only', allow_external_subpackages=True),
+    'ansible_collections.ansible': dict(type='pkg_only', allow_external_subpackages=True),
+    # these implement the ansible.builtin synthetic collection mapped to the packages inside the ansible distribution
     'ansible_collections.ansible.builtin': dict(type='pkg_only'),
     'ansible_collections.ansible.builtin.plugins': dict(type='map', map='ansible.plugins'),
     'ansible_collections.ansible.builtin.plugins.module_utils': dict(type='map', map='ansible.module_utils', graft=True),
@@ -101,7 +104,7 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
 
     def find_module(self, fullname, path=None):
         # this loader is only concerned with items under the Ansible Collections namespace hierarchy, ignore others
-        if fullname.startswith('ansible_collections.') or fullname == 'ansible_collections':
+        if fullname and fullname.split('.', 1)[0] == 'ansible_collections':
             return self
 
         return None
@@ -110,6 +113,8 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
         if sys.modules.get(fullname):
             return sys.modules[fullname]
 
+        newmod = None
+
         # this loader implements key functionality for Ansible collections
         # * implicit distributed namespace packages for the root Ansible namespace (no pkgutil.extend_path hackery reqd)
         # * implicit package support for Python 2.7 (no need for __init__.py in collections, except to use standard Py2.7 tooling)
@@ -132,10 +137,13 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
         synpkg_remainder = ''
 
         if not synpkg_def:
-            synpkg_def = _SYNTHETIC_PACKAGES.get(parent_pkg_name)
-            synpkg_remainder = '.' + fullname.rpartition('.')[2]
+            # if the parent is a grafted package, we have some special work to do, otherwise just look for stuff on disk
+            parent_synpkg_def = _SYNTHETIC_PACKAGES.get(parent_pkg_name)
+            if parent_synpkg_def and parent_synpkg_def.get('graft'):
+                synpkg_def = parent_synpkg_def
+                synpkg_remainder = '.' + fullname.rpartition('.')[2]
 
-        # FIXME: collapse as much of this back to on-demand as possible (maybe stub packages that get replaced when actually loaded?)
+        # FUTURE: collapse as much of this back to on-demand as possible (maybe stub packages that get replaced when actually loaded?)
         if synpkg_def:
             pkg_type = synpkg_def.get('type')
             if not pkg_type:
@@ -159,9 +167,13 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
                 newmod.__loader__ = self
                 newmod.__path__ = []
 
-                sys.modules[fullname] = newmod
+                if not synpkg_def.get('allow_external_subpackages'):
+                    # if external subpackages are NOT allowed, we're done
+                    sys.modules[fullname] = newmod
+                    return newmod
 
-                return newmod
+                # if external subpackages ARE allowed, check for on-disk implementations and return a normal
+                # package if we find one, otherwise return the one we created here
 
         if not parent_pkg:  # top-level package, look for NS subpackages on all collection paths
             package_paths = [self._extend_path_with_ns(p, fullname) for p in self.n_collection_paths]
@@ -217,6 +229,11 @@ class AnsibleCollectionLoader(with_metaclass(Singleton, object)):
 
             return newmod
 
+        # even if we didn't find one on disk, fall back to a synthetic package if we have one...
+        if newmod:
+            sys.modules[fullname] = newmod
+            return newmod
+
         # FIXME: need to handle the "no dirs present" case for at least the root and synthetic internal collections like ansible.builtin
 
         raise ImportError('module {0} not found'.format(fullname))

test/integration/targets/collections/collection_root_user/ansible_collections/ansible/builtin/plugins/modules/ping.py

@@ -0,0 +1,13 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+
+def main():
+    print(json.dumps(dict(changed=False, source='overridden ansible.builtin (should not be possible)')))
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/collections/collection_root_user/ansible_collections/ansible/bullcoll/plugins/modules/bullmodule.py

@@ -0,0 +1,13 @@
+#!/usr/bin/python
+from __future__ import (absolute_import, division, print_function)
+__metaclass__ = type
+
+import json
+
+
+def main():
+    print(json.dumps(dict(changed=False, source='user_ansible_bullcoll')))
+
+
+if __name__ == '__main__':
+    main()

test/integration/targets/collections/posix.yml

@@ -22,6 +22,16 @@
       name: testns.testcoll.maskedmodule
     register: maskedmodule_out
 
+  # ensure the ansible ns can have real collections added to it
+  - name: call an external module in the ansible namespace
+    ansible.bullcoll.bullmodule:
+    register: bullmodule_out
+
+  # ensure the ansible ns cannot override ansible.builtin externally
+  - name: call an external module in the ansible.builtin collection (should use the built in module)
+    ansible.builtin.ping:
+    register: builtin_ping_out
+
   # action in a collection subdir
   - name: test subdir action FQ
     testns.testcoll.action_subdir.subdir_ping_action:
@@ -59,6 +69,9 @@
       - systestmodule_out.source == 'sys'
       - contentadjmodule_out.source == 'content_adj'
       - not maskedmodule_out.plugin_path
+      - bullmodule_out.source == 'user_ansible_bullcoll'
+      - builtin_ping_out.source is not defined
+      - builtin_ping_out.ping == 'pong'
       - subdir_ping_action_out is not changed
       - subdir_ping_module_out is not changed
       - granular_out.mu_result == 'thingtocall in leaf'