Add role skeleton support (#17079)

* Add role skeleton support

The default role skeleton used by ansible-galaxy is good, but it doesn't
allow organizations to customize it to suit their needs.  This patch
addresses that by adding the option to point ansible-galaxy at a
role skeleton directory.  The contents of this directory are then
copied (or rendered) into the output role directory.  Appropriate
command line options and configuration entries are added to allow for
further customization from the role skeleton.

* Remove dependency on platforms list from test

Platforms are no longer provided to the template by galaxy init.
Removing the code in our test meta/main.yml template that relied on it.

* Fix whitespace issue

docs/docsite/rst/galaxy.rst

@@ -211,7 +211,31 @@ If you are creating a Container Enabled role, use the *--container-enabled* opti
 with default files appropriate for a Container Enabled role. For instance, the README.md has a slightly different structure, the *.travis.yml* file tests
 the role using `Ansible Container <https://github.com/ansible/ansible-container>`_, and the meta directory includes a *container.yml* file.
 
-Search for roles
+Using a Custom Role Skeleton
+============================
+
+A custom role skeleton directory can be supplied as follows:
+
+::
+
+   $ ansible-galaxy init --role-skeleton=/path/to/skeleton role_name
+
+When a skeleton is provided, init will:
+
+- copy all files and directories from the skeleton to the new role
+- any .j2 files found outside of a templates folder will be rendered as templates. The only useful variable at the moment is role_name
+- The .git folder and any .git_keep files will not be copied
+
+Alternatively, the role_skeleton and ignoring of files can be configured via ansible.cfg
+
+::
+
+  [galaxy]
+  role_skeleton = /path/to/skeleton
+  role_skeleton_ignore = ^.git$,^.*/.git_keep$
+
+
+Search for Roles
 ----------------
 
 Search the Galaxy database by tags, platforms, author and multiple keywords. For example:

lib/ansible/cli/galaxy.py

@@ -26,6 +26,7 @@ import os.path
 import sys
 import yaml
 import time
+import re
 import shutil
 
 from jinja2 import Environment, FileSystemLoader
@@ -88,6 +89,7 @@ class GalaxyCLI(CLI):
             self.parser.add_option('-p', '--init-path', dest='init_path', default="./", help='The path in which the skeleton role will be created. The default is the current working directory.')
             self.parser.add_option('--container-enabled', dest='container_enabled', action='store_true', default=False,
                                    help='Initialize the skeleton role with default contents for a Container Enabled role.')
+            self.parser.add_option('--role-skeleton', dest='role_skeleton', default=None, help='The path to a role skeleton that the new role should be based upon.')
         elif self.action == "install":
             self.parser.set_usage("usage: %prog install [options] [-r FILE | role_name(s)[,version] | scm+role_repo_url[,version] | tar_file(s)]")
             self.parser.add_option('-i', '--ignore-errors', dest='ignore_errors', action='store_true', default=False, help='Ignore errors and continue with the next specified role.')
@@ -176,6 +178,7 @@ class GalaxyCLI(CLI):
 
         init_path  = self.get_opt('init_path', './')
         force      = self.get_opt('force', False)
+        role_skeleton = self.get_opt('role_skeleton', C.GALAXY_ROLE_SKELETON)
 
         role_name = self.args.pop(0).strip() if self.args else None
         if not role_name:
@@ -205,16 +208,27 @@ class GalaxyCLI(CLI):
         if not os.path.exists(role_path):
             os.makedirs(role_path)
 
-        role_skeleton = self.galaxy.default_role_skeleton_path
+        if role_skeleton is not None:
+            skeleton_ignore_expressions = C.GALAXY_ROLE_SKELETON_IGNORE
+        else:
+            role_skeleton = self.galaxy.default_role_skeleton_path
+            skeleton_ignore_expressions = ['^.*/.git_keep$']
+
         role_skeleton = os.path.expanduser(role_skeleton)
+        skeleton_ignore_re = list(map(lambda x: re.compile(x), skeleton_ignore_expressions))
+
         template_env = Environment(loader=FileSystemLoader(role_skeleton))
 
         for root, dirs, files in os.walk(role_skeleton, topdown=True):
             rel_root = os.path.relpath(root, role_skeleton)
             in_templates_dir = rel_root.split(os.sep, 1)[0] == 'templates'
+            dirs[:] = filter(lambda d: not any(map(lambda r: r.match(os.path.join(rel_root, d)), skeleton_ignore_re)), dirs)
+
             for f in files:
                 filename, ext = os.path.splitext(f)
-                if ext == ".j2" and not in_templates_dir:
+                if any(map(lambda r: r.match(os.path.join(rel_root, f)), skeleton_ignore_re)):
+                    continue
+                elif ext == ".j2" and not in_templates_dir:
                     src_template = os.path.join(rel_root, f)
                     dest_file = os.path.join(role_path, rel_root, filename)
                     template_env.get_template(src_template).stream(inject_data).dump(dest_file)

lib/ansible/constants.py

@@ -364,6 +364,8 @@ GALAXY_SERVER                  = get_config(p, 'galaxy', 'server', 'ANSIBLE_GALA
 GALAXY_IGNORE_CERTS            = get_config(p, 'galaxy', 'ignore_certs', 'ANSIBLE_GALAXY_IGNORE', False, value_type='boolean')
 # this can be configured to blacklist SCMS but cannot add new ones unless the code is also updated
 GALAXY_SCMS                    = get_config(p, 'galaxy', 'scms', 'ANSIBLE_GALAXY_SCMS', 'git, hg', value_type='list')
+GALAXY_ROLE_SKELETON = get_config(p, 'galaxy', 'role_skeleton', 'ANSIBLE_GALAXY_ROLE_SKELETON', None, value_type='path')
+GALAXY_ROLE_SKELETON_IGNORE = get_config(p, 'galaxy', 'role_skeleton_ignore', 'ANSIBLE_GALAXY_ROLE_SKELETON_IGNORE', ['^.git$', '^.*/.git_keep$'], value_type='list')
 
 STRING_TYPE_FILTERS = get_config(p, 'jinja2', 'dont_type_filters', 'ANSIBLE_STRING_TYPE_FILTERS', ['string', 'to_json', 'to_nice_json', 'to_yaml', 'ppretty', 'json'], value_type='list' )
 

test/units/cli/test_data/role_skeleton/.travis.yml

@@ -0,0 +1,29 @@
+---
+language: python
+python: "2.7"
+
+# Use the new container infrastructure
+sudo: false
+
+# Install ansible
+addons:
+  apt:
+    packages:
+    - python-pip
+
+install:
+  # Install ansible
+  - pip install ansible
+
+  # Check ansible version
+  - ansible --version
+
+  # Create ansible.cfg with correct roles_path
+  - printf '[defaults]\nroles_path=../' >ansible.cfg
+
+script:
+  # Basic role syntax check
+  - ansible-playbook tests/test.yml -i tests/inventory --syntax-check
+
+notifications:
+  webhooks: https://galaxy.ansible.com/api/v1/notifications/

test/units/cli/test_data/role_skeleton/README.md

@@ -0,0 +1,38 @@
+Role Name
+=========
+
+A brief description of the role goes here.
+
+Requirements
+------------
+
+Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
+
+Role Variables
+--------------
+
+A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
+
+Dependencies
+------------
+
+A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
+
+Example Playbook
+----------------
+
+Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
+
+    - hosts: servers
+      roles:
+         - { role: username.rolename, x: 42 }
+
+License
+-------
+
+BSD
+
+Author Information
+------------------
+
+An optional section for the role authors to include contact information, or a website (HTML is not allowed).

test/units/cli/test_data/role_skeleton/defaults/main.yml.j2

@@ -0,0 +1,2 @@
+---
+# defaults file for {{ role_name }}

test/units/cli/test_data/role_skeleton/handlers/main.yml.j2

@@ -0,0 +1,2 @@
+---
+# handlers file for {{ role_name }}

test/units/cli/test_data/role_skeleton/inventory

@@ -0,0 +1 @@
+localhost

test/units/cli/test_data/role_skeleton/meta/main.yml.j2

@@ -0,0 +1,59 @@
+galaxy_info:
+  author: {{ author }}
+  description: {{ description }}
+  company: {{ company }}
+
+  # If the issue tracker for your role is not on github, uncomment the
+  # next line and provide a value
+  # issue_tracker_url: {{ issue_tracker_url }}
+
+  # Some suggested licenses:
+  # - BSD (default)
+  # - MIT
+  # - GPLv2
+  # - GPLv3
+  # - Apache
+  # - CC-BY
+  license: {{ license }}
+
+  min_ansible_version: {{ min_ansible_version }}
+
+  # Optionally specify the branch Galaxy will use when accessing the GitHub
+  # repo for this role. During role install, if no tags are available,
+  # Galaxy will use this branch. During import Galaxy will access files on
+  # this branch. If travis integration is cofigured, only notification for this
+  # branch will be accepted. Otherwise, in all cases, the repo's default branch
+  # (usually master) will be used.
+  #github_branch:
+
+  #
+  # platforms is a list of platforms, and each platform has a name and a list of versions.
+  #
+  # platforms:
+  # - name: Fedora
+  #   versions:
+  #   - all
+  #   - 25
+  # - name: SomePlatform
+  #   versions:
+  #   - all
+  #   - 1.0
+  #   - 7
+  #   - 99.99
+
+  galaxy_tags: []
+    # List tags for your role here, one per line. A tag is
+    # a keyword that describes and categorizes the role.
+    # Users find roles by searching for tags. Be sure to
+    # remove the '[]' above if you add tags to this list.
+    #
+    # NOTE: A tag is limited to a single word comprised of
+    # alphanumeric characters. Maximum 20 tags per role.
+
+dependencies: []
+  # List your role dependencies here, one per line.
+  # Be sure to remove the '[]' above if you add dependencies
+  # to this list.
+  {%- for dependency in dependencies %}
+  #- {{ dependency }}
+  {%- endfor %}

test/units/cli/test_data/role_skeleton/tasks/main.yml.j2

@@ -0,0 +1,2 @@
+---
+# tasks file for {{ role_name }}

test/units/cli/test_data/role_skeleton/templates/subfolder/test.conf.j2

@@ -0,0 +1,2 @@
+[defaults]
+test_key = {{ test_variable }}

test/units/cli/test_data/role_skeleton/templates/test.conf.j2

@@ -0,0 +1,2 @@
+[defaults]
+test_key = {{ test_variable }}

test/units/cli/test_data/role_skeleton/templates_extra/templates.txt.j2

@@ -0,0 +1 @@
+{{ role_name }}

test/units/cli/test_data/role_skeleton/tests/test.yml.j2

@@ -0,0 +1,5 @@
+---
+- hosts: localhost
+  remote_user: root
+  roles:
+    - {{ role_name }}

test/units/cli/test_data/role_skeleton/vars/main.yml.j2

@@ -0,0 +1,2 @@
+---
+# vars file for {{ role_name }}

test/units/cli/test_galaxy.py

@@ -23,6 +23,7 @@ import os
 import shutil
 import tarfile
 import tempfile
+import yaml
 
 from ansible.compat.six import PY3
 from ansible.compat.tests import unittest
@@ -264,6 +265,170 @@ class TestGalaxy(unittest.TestCase):
         ''' testing the options parser when the action 'setup' is given '''
         gc = GalaxyCLI(args=["setup"])
         self.run_parse_common(gc, "setup")
+
         self.assertEqual(gc.options.verbosity, 0)
         self.assertEqual(gc.options.remove_id, None)
         self.assertEqual(gc.options.setup_list, False)
+
+
+class ValidRoleTests(object):
+
+    expected_role_dirs = ('defaults', 'files', 'handlers', 'meta', 'tasks', 'templates', 'vars', 'tests')
+
+    @classmethod
+    def setUpRole(cls, role_name, galaxy_args=None, skeleton_path=None):
+        if galaxy_args is None:
+            galaxy_args = []
+
+        if skeleton_path is not None:
+            cls.role_skeleton_path = skeleton_path
+            galaxy_args += ['--role-skeleton', skeleton_path]
+
+        # Make temp directory for testing
+        cls.test_dir = tempfile.mkdtemp()
+        if not os.path.isdir(cls.test_dir):
+            os.makedirs(cls.test_dir)
+
+        cls.role_dir = os.path.join(cls.test_dir, role_name)
+        cls.role_name = role_name
+
+        # create role using default skeleton
+        gc = GalaxyCLI(args=['init', '-c', '--offline'] + galaxy_args + ['-p', cls.test_dir, cls.role_name])
+        gc.parse()
+        gc.run()
+        cls.gc = gc
+
+        if skeleton_path is None:
+            cls.role_skeleton_path = gc.galaxy.default_role_skeleton_path
+
+    @classmethod
+    def tearDownClass(cls):
+        if os.path.isdir(cls.test_dir):
+            shutil.rmtree(cls.test_dir)
+
+    def test_metadata(self):
+        with open(os.path.join(self.role_dir, 'meta', 'main.yml'), 'r') as mf:
+            metadata = yaml.safe_load(mf)
+        self.assertIn('galaxy_info', metadata, msg='unable to find galaxy_info in metadata')
+        self.assertIn('dependencies', metadata, msg='unable to find dependencies in metadata')
+
+    def test_readme(self):
+        readme_path = os.path.join(self.role_dir, 'README.md')
+        self.assertTrue(os.path.exists(readme_path), msg='Readme doesn\'t exist')
+
+    def test_main_ymls(self):
+        need_main_ymls = set(self.expected_role_dirs) - set(['meta', 'tests', 'files', 'templates'])
+        for d in need_main_ymls:
+            main_yml = os.path.join(self.role_dir, d, 'main.yml')
+            self.assertTrue(os.path.exists(main_yml))
+            expected_string = "---\n# {0} file for {1}".format(d, self.role_name)
+            with open(main_yml, 'r') as f:
+                self.assertEqual(expected_string, f.read().strip())
+
+    def test_role_dirs(self):
+        for d in self.expected_role_dirs:
+            self.assertTrue(os.path.isdir(os.path.join(self.role_dir, d)), msg="Expected role subdirectory {0} doesn't exist".format(d))
+
+    def test_travis_yml(self):
+        with open(os.path.join(self.role_dir,'.travis.yml'), 'r') as f:
+            contents = f.read()
+
+        with open(os.path.join(self.role_skeleton_path, '.travis.yml'), 'r') as f:
+            expected_contents = f.read()
+
+        self.assertEqual(expected_contents, contents, msg='.travis.yml does not match expected')
+
+    def test_readme_contents(self):
+        with open(os.path.join(self.role_dir, 'README.md'), 'r') as readme:
+            contents = readme.read()
+
+        with open(os.path.join(self.role_skeleton_path, 'README.md'), 'r') as f:
+            expected_contents = f.read()
+
+        self.assertEqual(expected_contents, contents, msg='README.md does not match expected')
+
+    def test_test_yml(self):
+        with open(os.path.join(self.role_dir, 'tests', 'test.yml'), 'r') as f:
+            test_playbook = yaml.safe_load(f)
+        print(test_playbook)
+        self.assertEqual(len(test_playbook), 1)
+        self.assertEqual(test_playbook[0]['hosts'], 'localhost')
+        self.assertEqual(test_playbook[0]['remote_user'], 'root')
+        self.assertListEqual(test_playbook[0]['roles'], [self.role_name], msg='The list of roles included in the test play doesn\'t match')
+
+
+class TestGalaxyInitDefault(unittest.TestCase, ValidRoleTests):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.setUpRole(role_name='delete_me')
+
+    def test_metadata_contents(self):
+        with open(os.path.join(self.role_dir, 'meta', 'main.yml'), 'r') as mf:
+            metadata = yaml.safe_load(mf)
+        self.assertEqual(metadata.get('galaxy_info', dict()).get('author'), 'your name', msg='author was not set properly in metadata')
+
+
+class TestGalaxyInitContainerEnabled(unittest.TestCase, ValidRoleTests):
+
+    @classmethod
+    def setUpClass(cls):
+        cls.setUpRole('delete_me_container', galaxy_args=['--container-enabled'])
+
+    def test_metadata_container_tag(self):
+        with open(os.path.join(self.role_dir, 'meta', 'main.yml'), 'r') as mf:
+            metadata = yaml.safe_load(mf)
+        self.assertIn('container', metadata.get('galaxy_info', dict()).get('galaxy_tags',[]), msg='container tag not set in role metadata')
+
+    def test_metadata_contents(self):
+        with open(os.path.join(self.role_dir, 'meta', 'main.yml'), 'r') as mf:
+            metadata = yaml.safe_load(mf)
+        self.assertEqual(metadata.get('galaxy_info', dict()).get('author'), 'your name', msg='author was not set properly in metadata')
+
+    def test_meta_container_yml(self):
+        self.assertTrue(os.path.exists(os.path.join(self.role_dir, 'meta', 'container.yml')), msg='container.yml was not created')
+
+    def test_test_yml(self):
+        with open(os.path.join(self.role_dir, 'tests', 'test.yml'), 'r') as f:
+            test_playbook = yaml.safe_load(f)
+        print(test_playbook)
+        self.assertEqual(len(test_playbook), 1)
+        self.assertEqual(test_playbook[0]['hosts'], 'localhost')
+        self.assertFalse(test_playbook[0]['gather_facts'])
+        self.assertEqual(test_playbook[0]['connection'], 'local')
+        self.assertIsNone(test_playbook[0]['tasks'], msg='We\'re expecting an unset list of tasks in test.yml')
+
+
+class TestGalaxyInitSkeleton(unittest.TestCase, ValidRoleTests):
+
+    @classmethod
+    def setUpClass(cls):
+        role_skeleton_path = os.path.join(os.path.split(__file__)[0], 'test_data', 'role_skeleton')
+        cls.setUpRole('delete_me_skeleton', skeleton_path=role_skeleton_path)
+
+    def test_empty_files_dir(self):
+        files_dir = os.path.join(self.role_dir, 'files')
+        self.assertTrue(os.path.isdir(files_dir))
+        self.assertListEqual(os.listdir(files_dir), [], msg='we expect the files directory to be empty, is ignore working?')
+
+    def test_template_ignore_jinja(self):
+        test_conf_j2 = os.path.join(self.role_dir, 'templates', 'test.conf.j2')
+        self.assertTrue(os.path.exists(test_conf_j2), msg="The test.conf.j2 template doesn't seem to exist, is it being rendered as test.conf?")
+        with open(test_conf_j2, 'r') as f:
+            contents = f.read()
+        expected_contents = '[defaults]\ntest_key = {{ test_variable }}'
+        self.assertEqual(expected_contents, contents.strip(), msg="test.conf.j2 doesn't contain what it should, is it being rendered?")
+
+    def test_template_ignore_jinja_subfolder(self):
+        test_conf_j2 = os.path.join(self.role_dir, 'templates', 'subfolder', 'test.conf.j2')
+        self.assertTrue(os.path.exists(test_conf_j2), msg="The test.conf.j2 template doesn't seem to exist, is it being rendered as test.conf?")
+        with open(test_conf_j2, 'r') as f:
+            contents = f.read()
+        expected_contents = '[defaults]\ntest_key = {{ test_variable }}'
+        self.assertEqual(expected_contents, contents.strip(), msg="test.conf.j2 doesn't contain what it should, is it being rendered?")
+
+    def test_template_ignore_similar_folder(self):
+        self.assertTrue(os.path.exists(os.path.join(self.role_dir, 'templates_extra', 'templates.txt')))
+
+    def test_skeleton_option(self):
+        self.assertEquals(self.role_skeleton_path, self.gc.get_opt('role_skeleton'), msg='Skeleton path was not parsed properly from the command line')