Fixes to the documentation build (#15356)

* Could only have one alias before.  Subsequent aliases overrode the
  previous ones.  Now multiple aliases work.
* Fix BLACKLISTED_MODULES.   Previously, modules were listed in the
  generated documentation despite being blacklisted
* Deprecated modules form extras were showing the (E) tag and not the
  (D) tag. Reversed that now (Probably not necessary to also show the
  E tag).
* Sort the deprecated modules alphabetically in the Category docs as
  well as the list of all modules
* Optimization: Previously rendered the modules to rst twice once in all
  group and once in individual categories.  Fixed to only render them
  once.
* Add fireball to blacklist and remove async_status (as people need to
  use that).

hacking/module_formatter.py

@@ -19,6 +19,7 @@
 #
 
 from __future__ import print_function
+
 import os
 import glob
 import sys
@@ -28,6 +29,8 @@ import optparse
 import datetime
 import cgi
 import warnings
+from collections import defaultdict
+
 from jinja2 import Environment, FileSystemLoader
 from six import iteritems
 
@@ -126,55 +129,45 @@ def write_data(text, options, outputname, module):
 def list_modules(module_dir, depth=0):
     ''' returns a hash of categories, each category being a hash of module names to file paths '''
 
-    categories = dict(all=dict(),_aliases=dict())
+    categories = dict()
-    if depth <= 3: # limit # of subdirs
+    module_info = dict()
+    aliases = defaultdict(set)
 
-        files = glob.glob("%s/*" % module_dir)
+    # * windows powershell modules have documentation stubs in python docstring
-        for d in files:
+    #   format (they are not executed) so skip the ps1 format files
+    # * One glob level for every module level that we're going to traverse
+    files = glob.glob("%s/*.py" % module_dir) + glob.glob("%s/*/*.py" % module_dir) + glob.glob("%s/*/*/*.py" % module_dir) + glob.glob("%s/*/*/*/*.py" % module_dir)
 
-            category = os.path.splitext(os.path.basename(d))[0]
+    for module_path in files:
-            if os.path.isdir(d):
+        if module_path.endswith('__init__.py'):
+            continue
+        category = categories
+        mod_path_only = os.path.dirname(module_path[len(module_dir) + 1:])
+        # Start at the second directory because we don't want the "vendor"
+        # directories (core, extras)
+        for new_cat in mod_path_only.split('/')[1:]:
+            if new_cat not in category:
+                category[new_cat] = dict()
+            category = category[new_cat]
 
-                res = list_modules(d, depth + 1)
+        module = os.path.splitext(os.path.basename(module_path))[0]
-                for key in list(res.keys()):
+        if module in module_docs.BLACKLIST_MODULES:
-                    if key in categories:
+            # Do not list blacklisted modules
-                        categories[key] = merge_hash(categories[key], res[key])
+            continue
-                        res.pop(key, None)
+        if module.startswith("_") and os.path.islink(module_path):
+            source = os.path.splitext(os.path.basename(os.path.realpath(module_path)))[0]
+            module = module.replace("_","",1)
+            aliases[source].add(module)
+            continue
 
-                if depth < 2:
+        category[module] = module_path
-                    categories.update(res)
+        module_info[module] = module_path
-                else:
-                    category = module_dir.split("/")[-1]
-                    if not category in categories:
-                        categories[category] = res
-                    else:
-                        categories[category].update(res)
-            else:
-                module = category
-                category = os.path.basename(module_dir)
-                if not d.endswith(".py") or d.endswith('__init__.py'):
-                    # windows powershell modules have documentation stubs in python docstring
-                    # format (they are not executed) so skip the ps1 format files
-                    continue
-                elif module.startswith("_") and os.path.islink(d):
-                    source = os.path.splitext(os.path.basename(os.path.realpath(d)))[0]
-                    module = module.replace("_","",1)
-                    if not d in categories['_aliases']:
-                        categories['_aliases'][source] = [module]
-                    else:
-                        categories['_aliases'][source].update(module)
-                    continue
 
-                if not category in categories:
+    # keep module tests out of becoming module docs
-                    categories[category] = {}
-                categories[category][module] = d
-                categories['all'][module] = d
-
-    # keep module tests out of becomeing module docs
     if 'test' in categories:
         del categories['test']
 
-    return categories
+    return module_info, categories, aliases
 
 #####################################################################################
 
@@ -258,11 +251,8 @@ def process_module(module, options, env, template, outputname, module_map, alias
 
     # crash if module is missing documentation and not explicitly hidden from docs index
     if doc is None:
-        if module in module_docs.BLACKLIST_MODULES:
+        sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
-            return "SKIPPED"
+        sys.exit(1)
-        else:
-            sys.stderr.write("*** ERROR: MODULE MISSING DOCUMENTATION: %s, %s ***\n" % (fname, module))
-            sys.exit(1)
 
     if deprecated and 'deprecated' not in doc:
         sys.stderr.write("*** ERROR: DEPRECATED MODULE MISSING 'deprecated' DOCUMENTATION: %s, %s ***\n" % (fname, module))
@@ -334,21 +324,32 @@ def process_module(module, options, env, template, outputname, module_map, alias
 
 def print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases):
     modstring = module
-    modname = module
+    if modstring.startswith('_'):
+        modstring = module[1:]
+    modname = modstring
     if module in deprecated:
         modstring = modstring + DEPRECATED
-        modname = "_" + module
     elif module not in core:
         modstring = modstring + NOTCORE
 
-    result = process_module(modname, options, env, template, outputname, module_map, aliases)
+    category_file.write("  %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(module_map[module][1])), to_bytes(modname)))
-
-    if result != "SKIPPED":
-        category_file.write("  %s - %s <%s_module>\n" % (to_bytes(modstring), to_bytes(rst_ify(result)), to_bytes(module)))
 
 def process_category(category, categories, options, env, template, outputname):
 
+    ### FIXME:
+    # We no longer conceptually deal with a mapping of category names to
+    # modules to file paths.  Instead we want several different records:
+    # (1) Mapping of module names to file paths (what's presently used
+    #     as categories['all']
+    # (2) Mapping of category names to lists of module names (what you'd
+    #     presently get from categories[category_name][subcategory_name].keys()
+    # (3) aliases (what's presently in categories['_aliases']
+    #
+    # list_modules() now returns those.  Need to refactor this function and
+    # main to work with them.
+
     module_map = categories[category]
+    module_info = categories['all']
 
     aliases = {}
     if '_aliases' in categories:
@@ -369,21 +370,21 @@ def process_category(category, categories, options, env, template, outputname):
     for module in module_map.keys():
 
         if isinstance(module_map[module], dict):
-            for mod in module_map[module].keys():
+            for mod in (m for m in module_map[module].keys() if m in module_info):
                 if mod.startswith("_"):
-                    mod = mod.replace("_","",1)
                     deprecated.append(mod)
-                elif '/core/' in module_map[module][mod]:
+                elif '/core/' in module_info[mod][0]:
                     core.append(mod)
         else:
+            if module not in module_info:
+                continue
             if module.startswith("_"):
-                module = module.replace("_","",1)
                 deprecated.append(module)
-            elif '/core/' in module_map[module]:
+            elif '/core/' in module_info[module][0]:
                 core.append(module)
         modules.append(module)
 
-    modules.sort()
+    modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
 
     category_header = "%s Modules" % (category.title())
     underscores = "`" * len(category_header)
@@ -401,7 +402,7 @@ def process_category(category, categories, options, env, template, outputname):
             sections.append(module)
             continue
         else:
-            print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map, aliases)
+            print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases)
 
     sections.sort()
     for section in sections:
@@ -409,10 +410,10 @@ def process_category(category, categories, options, env, template, outputname):
         category_file.write(".. toctree:: :maxdepth: 1\n\n")
 
         section_modules = module_map[section].keys()
-        section_modules.sort()
+        section_modules.sort(key=lambda k: k[1:] if k.startswith('_') else k)
         #for module in module_map[section]:
-        for module in section_modules:
+        for module in (m for m in section_modules if m in module_info):
-            print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_map[section], aliases)
+            print_modules(module, category_file, deprecated, core, options, env, template, outputname, module_info, aliases)
 
     category_file.write("""\n\n
 .. note::
@@ -450,25 +451,42 @@ def main():
 
     env, template, outputname = jinja2_environment(options.template_dir, options.type)
 
-    categories = list_modules(options.module_dir)
+    mod_info, categories, aliases = list_modules(options.module_dir)
-    category_names = list(categories.keys())
+    categories['all'] = mod_info
+    categories['_aliases'] = aliases
+    category_names = [c for c in categories.keys() if not c.startswith('_')]
     category_names.sort()
 
+    # Write master category list
     category_list_path = os.path.join(options.output_dir, "modules_by_category.rst")
-    category_list_file = open(category_list_path, "w")
+    with open(category_list_path, "w") as category_list_file:
-    category_list_file.write("Module Index\n")
+        category_list_file.write("Module Index\n")
-    category_list_file.write("============\n")
+        category_list_file.write("============\n")
-    category_list_file.write("\n\n")
+        category_list_file.write("\n\n")
-    category_list_file.write(".. toctree::\n")
+        category_list_file.write(".. toctree::\n")
-    category_list_file.write("   :maxdepth: 1\n\n")
+        category_list_file.write("   :maxdepth: 1\n\n")
 
+        for category in category_names:
+            category_list_file.write("   list_of_%s_modules\n" % category)
+
+    #
+    # Import all the docs into memory
+    #
+    module_map = mod_info.copy()
+    skipped_modules = set()
+
+    for modname in module_map:
+        result = process_module(modname, options, env, template, outputname, module_map, aliases)
+        if result == 'SKIPPED':
+            del categories['all'][modname]
+        else:
+            categories['all'][modname] = (categories['all'][modname], result)
+
+    #
+    # Render all the docs to rst via category pages
+    #
     for category in category_names:
-        if category.startswith("_"):
-            continue
-        category_list_file.write("   list_of_%s_modules\n" % category)
         process_category(category, categories, options, env, template, outputname)
 
-    category_list_file.close()
-
 if __name__ == '__main__':
     main()

lib/ansible/utils/module_docs.py

@@ -37,9 +37,11 @@ except ImportError:
     display = Display()
 
 # modules that are ok that they do not have documentation strings
-BLACKLIST_MODULES = [
+BLACKLIST_MODULES = frozenset((
-   'async_wrapper', 'accelerate', 'async_status'
+   'async_wrapper',
-]
+   'accelerate',
+   'fireball',
+))
 
 def get_docstring(filename, verbose=False):
     """