Add the ability to ignore files and collection build (#64688)

changelogs/fragments/ansible-galaxy-ignore.yaml

@@ -0,0 +1,3 @@
+minor_changes:
+- ansible-galaxy - Always ignore the ``tests/output`` directory when building a collection as it is used by ``ansible-test`` for test output (https://github.com/ansible/ansible/issues/59228).
+- ansible-galaxy - Added the ability to ignore further files and folders using a pattern with the ``build_ignore`` key in a collection's ``galaxy.yml`` (https://github.com/ansible/ansible/issues/59228).

docs/docsite/rst/dev_guide/developing_collections.rst

@@ -234,15 +234,50 @@ a tarball of the built collection in the current directory which can be uploaded
 
 
 .. note::
-    * Certain files and folders are excluded when building the collection artifact. This is not currently configurable and is a work in progress so the collection artifact may contain files you would not wish to distribute.
+    * Certain files and folders are excluded when building the collection artifact. See :ref:`ignoring_files_and_folders_collections`  to exclude other files you would not wish to distribute.
     * If you used the now-deprecated ``Mazer`` tool for any of your collections, delete any and all files it added to your :file:`releases/` directory before you build your collection with ``ansible-galaxy``.
-    * You must also delete the :file:`tests/output` directory if you have been testing with ``ansible-test``.
     * The current Galaxy maximum tarball size is 2 MB.
 
 
 This tarball is mainly intended to upload to Galaxy
 as a distribution method, but you can use it directly to install the collection on target systems.
 
+.. _ignoring_files_and_folders_collections:
+
+Ignoring files and folders
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+By default the build step will include all the files in the collection directory in the final build artifact except for the following:
+
+* ``galaxy.yml``
+* ``*.pyc``
+* ``*.retry``
+* ``tests/output``
+* previously built artifacts in the root directory
+* Various version control directories like ``.git/``
+
+To exclude other files and folders when building the collection, you can set a list of file glob-like patterns in the
+``build_ignore`` key in the collection's ``galaxy.yml`` file. These patterns use the following special characters for
+wildcard matching:
+
+* ``*``: Matches everything
+* ``?``: Matches any single character
+* ``[seq]``: Matches and character in seq
+* ``[!seq]``:Matches any character not in seq
+
+For example, if you wanted to exclude the :file:`sensitive` folder within the ``playbooks`` folder as well any ``.tar.gz`` archives you
+can set the following in your ``galaxy.yml`` file:
+
+.. code-block:: yaml
+
+     build_ignore:
+     - playbooks/sensitive
+     - '*.tar.gz'
+
+.. note::
+     This feature is only supported when running ``ansible-galaxy collection build`` with Ansible 2.10 or newer.
+
+
 .. _trying_collection_locally:
 
 Trying collection locally

docs/templates/collections_galaxy_meta.rst.j2

@@ -43,6 +43,15 @@ The ``galaxy.yml`` file must contain the following keys in valid YAML:
         required
         {%- endif %}
 
+        {% if 'version_added' in entry -%}
+
+        .. rst-class:: value-added-in
+
+        |br| version_added: @{ entry.version_added }@
+
+        |_|
+
+        {%- endif %}
 
       - {% for desc in entry.description -%}
         @{ desc | trim | rst_ify }@

lib/ansible/cli/galaxy.py

@@ -651,6 +651,7 @@ class GalaxyCLI(CLI):
                 documentation='http://docs.example.com',
                 homepage='http://example.com',
                 issues='http://example.com/issue/tracker',
+                build_ignore=[],
             ))
 
             obj_path = os.path.join(init_path, namespace, collection_name)

lib/ansible/galaxy/collection.py

@@ -353,7 +353,8 @@ def build_collection(collection_path, output_path, force):
         raise AnsibleError("The collection galaxy.yml path '%s' does not exist." % to_native(b_galaxy_path))
 
     collection_meta = _get_galaxy_yml(b_galaxy_path)
-    file_manifest = _build_files_manifest(b_collection_path, collection_meta['namespace'], collection_meta['name'])
+    file_manifest = _build_files_manifest(b_collection_path, collection_meta['namespace'], collection_meta['name'],
+                                          collection_meta['build_ignore'])
     collection_manifest = _build_manifest(**collection_meta)
 
     collection_output = os.path.join(output_path, "%s-%s-%s.tar.gz" % (collection_meta['namespace'],
@@ -598,12 +599,18 @@ def _get_galaxy_yml(b_galaxy_yml_path):
     return galaxy_yml
 
 
-def _build_files_manifest(b_collection_path, namespace, name):
-    # Contains tuple of (b_filename, only root) where 'only root' means to only ignore the file in the root dir
-    b_ignore_files = frozenset([(b'*.pyc', False), (b'*.retry', False),
-                                (to_bytes('{0}-{1}-*.tar.gz'.format(namespace, name)), True)])
-    b_ignore_dirs = frozenset([(b'CVS', False), (b'.bzr', False), (b'.hg', False), (b'.git', False), (b'.svn', False),
-                               (b'__pycache__', False), (b'.tox', False)])
+def _build_files_manifest(b_collection_path, namespace, name, ignore_patterns):
+    # We always ignore .pyc and .retry files as well as some well known version control directories. The ignore
+    # patterns can be extended by the build_ignore key in galaxy.yml
+    b_ignore_patterns = [
+        b'galaxy.yml',
+        b'*.pyc',
+        b'*.retry',
+        b'tests/output',  # Ignore ansible-test result output directory.
+        to_bytes('{0}-{1}-*.tar.gz'.format(namespace, name)),  # Ignores previously built artifacts in the root dir.
+    ]
+    b_ignore_patterns += [to_bytes(p) for p in ignore_patterns]
+    b_ignore_dirs = frozenset([b'CVS', b'.bzr', b'.hg', b'.git', b'.svn', b'__pycache__', b'.tox'])
 
     entry_template = {
         'name': None,
@@ -626,16 +633,15 @@ def _build_files_manifest(b_collection_path, namespace, name):
     }
 
     def _walk(b_path, b_top_level_dir):
-        is_root = b_path == b_top_level_dir
-
         for b_item in os.listdir(b_path):
             b_abs_path = os.path.join(b_path, b_item)
             b_rel_base_dir = b'' if b_path == b_top_level_dir else b_path[len(b_top_level_dir) + 1:]
-            rel_path = to_text(os.path.join(b_rel_base_dir, b_item), errors='surrogate_or_strict')
+            b_rel_path = os.path.join(b_rel_base_dir, b_item)
+            rel_path = to_text(b_rel_path, errors='surrogate_or_strict')
 
             if os.path.isdir(b_abs_path):
-                if any(b_item == b_path for b_path, root_only in b_ignore_dirs
-                       if not root_only or root_only == is_root):
+                if any(b_item == b_path for b_path in b_ignore_dirs) or \
+                        any(fnmatch.fnmatch(b_rel_path, b_pattern) for b_pattern in b_ignore_patterns):
                     display.vvv("Skipping '%s' for collection build" % to_text(b_abs_path))
                     continue
 
@@ -655,10 +661,7 @@ def _build_files_manifest(b_collection_path, namespace, name):
 
                 _walk(b_abs_path, b_top_level_dir)
             else:
-                if b_item == b'galaxy.yml':
-                    continue
-                elif any(fnmatch.fnmatch(b_item, b_pattern) for b_pattern, root_only in b_ignore_files
-                         if not root_only or root_only == is_root):
+                if any(fnmatch.fnmatch(b_rel_path, b_pattern) for b_pattern in b_ignore_patterns):
                     display.vvv("Skipping '%s' for collection build" % to_text(b_abs_path))
                     continue
 

lib/ansible/galaxy/data/collections_galaxy_meta.yml

@@ -96,3 +96,15 @@
   description:
   - The URL to the collection issue tracker.
   type: str
+
+- key: build_ignore
+  description:
+  - A list of file glob-like patterns used to filter any files or directories
+    that should not be included in the build artifact.
+  - A pattern is matched from the relative path of the file or directory of the
+    collection directory.
+  - This uses C(fnmatch) to match the files or directories.
+  - Some directories and files like C(galaxy.yml), C(*.pyc), C(*.retry), and
+    C(.git) are always filtered.
+  type: list
+  version_added: '2.10'

test/units/galaxy/test_collection.py

@@ -248,24 +248,37 @@ def test_build_ignore_files_and_folders(collection_input, monkeypatch):
     git_folder = os.path.join(input_dir, '.git')
     retry_file = os.path.join(input_dir, 'ansible.retry')
 
+    tests_folder = os.path.join(input_dir, 'tests', 'output')
+    tests_output_file = os.path.join(tests_folder, 'result.txt')
+
     os.makedirs(git_folder)
+    os.makedirs(tests_folder)
+
     with open(retry_file, 'w+') as ignore_file:
         ignore_file.write('random')
         ignore_file.flush()
 
-    actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection')
+    with open(tests_output_file, 'w+') as tests_file:
+        tests_file.write('random')
+        tests_file.flush()
+
+    actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection', [])
 
     assert actual['format'] == 1
     for manifest_entry in actual['files']:
-        assert manifest_entry['name'] not in ['.git', 'ansible.retry', 'galaxy.yml']
+        assert manifest_entry['name'] not in ['.git', 'ansible.retry', 'galaxy.yml', 'tests/output', 'tests/output/result.txt']
 
     expected_msgs = [
+        "Skipping '%s/galaxy.yml' for collection build" % to_text(input_dir),
         "Skipping '%s' for collection build" % to_text(retry_file),
         "Skipping '%s' for collection build" % to_text(git_folder),
+        "Skipping '%s' for collection build" % to_text(tests_folder),
     ]
-    assert mock_display.call_count == 2
+    assert mock_display.call_count == 4
     assert mock_display.mock_calls[0][1][0] in expected_msgs
     assert mock_display.mock_calls[1][1][0] in expected_msgs
+    assert mock_display.mock_calls[2][1][0] in expected_msgs
+    assert mock_display.mock_calls[3][1][0] in expected_msgs
 
 
 def test_build_ignore_older_release_in_root(collection_input, monkeypatch):
@@ -285,7 +298,7 @@ def test_build_ignore_older_release_in_root(collection_input, monkeypatch):
             file_obj.write('random')
             file_obj.flush()
 
-    actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection')
+    actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection', [])
     assert actual['format'] == 1
 
     plugin_release_found = False
@@ -296,8 +309,62 @@ def test_build_ignore_older_release_in_root(collection_input, monkeypatch):
 
     assert plugin_release_found
 
-    assert mock_display.call_count == 1
-    assert mock_display.mock_calls[0][1][0] == "Skipping '%s' for collection build" % to_text(release_file)
+    expected_msgs = [
+        "Skipping '%s/galaxy.yml' for collection build" % to_text(input_dir),
+        "Skipping '%s' for collection build" % to_text(release_file)
+    ]
+    assert mock_display.call_count == 2
+    assert mock_display.mock_calls[0][1][0] in expected_msgs
+    assert mock_display.mock_calls[1][1][0] in expected_msgs
+
+
+def test_build_ignore_patterns(collection_input, monkeypatch):
+    input_dir = collection_input[0]
+
+    mock_display = MagicMock()
+    monkeypatch.setattr(Display, 'vvv', mock_display)
+
+    actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection',
+                                              ['*.md', 'plugins/action', 'playbooks/*.j2'])
+    assert actual['format'] == 1
+
+    expected_missing = [
+        'README.md',
+        'docs/My Collection.md',
+        'plugins/action',
+        'playbooks/templates/test.conf.j2',
+        'playbooks/templates/subfolder/test.conf.j2',
+    ]
+
+    # Files or dirs that are close to a match but are not, make sure they are present
+    expected_present = [
+        'docs',
+        'roles/common/templates/test.conf.j2',
+        'roles/common/templates/subfolder/test.conf.j2',
+    ]
+
+    actual_files = [e['name'] for e in actual['files']]
+    for m in expected_missing:
+        assert m not in actual_files
+
+    for p in expected_present:
+        assert p in actual_files
+
+    expected_msgs = [
+        "Skipping '%s/galaxy.yml' for collection build" % to_text(input_dir),
+        "Skipping '%s/README.md' for collection build" % to_text(input_dir),
+        "Skipping '%s/docs/My Collection.md' for collection build" % to_text(input_dir),
+        "Skipping '%s/plugins/action' for collection build" % to_text(input_dir),
+        "Skipping '%s/playbooks/templates/test.conf.j2' for collection build" % to_text(input_dir),
+        "Skipping '%s/playbooks/templates/subfolder/test.conf.j2' for collection build" % to_text(input_dir),
+    ]
+    assert mock_display.call_count == len(expected_msgs)
+    assert mock_display.mock_calls[0][1][0] in expected_msgs
+    assert mock_display.mock_calls[1][1][0] in expected_msgs
+    assert mock_display.mock_calls[2][1][0] in expected_msgs
+    assert mock_display.mock_calls[3][1][0] in expected_msgs
+    assert mock_display.mock_calls[4][1][0] in expected_msgs
+    assert mock_display.mock_calls[5][1][0] in expected_msgs
 
 
 def test_build_ignore_symlink_target_outside_collection(collection_input, monkeypatch):
@@ -309,7 +376,7 @@ def test_build_ignore_symlink_target_outside_collection(collection_input, monkey
     link_path = os.path.join(input_dir, 'plugins', 'connection')
     os.symlink(outside_dir, link_path)
 
-    actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection')
+    actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection', [])
     for manifest_entry in actual['files']:
         assert manifest_entry['name'] != 'plugins/connection'
 
@@ -333,7 +400,7 @@ def test_build_copy_symlink_target_inside_collection(collection_input):
 
     os.symlink(roles_target, roles_link)
 
-    actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection')
+    actual = collection._build_files_manifest(to_bytes(input_dir), 'namespace', 'collection', [])
 
     linked_entries = [e for e in actual['files'] if e['name'].startswith('playbooks/roles/linked')]
     assert len(linked_entries) == 3