Make ini parsing slightly more robust

Prior to this commit, the ini parser would fail if the inventory was
not 100% utf-8.  This commit makes this slightly more robust by
omitting full line comments from that requirement.

Fixes #17593

(cherry picked from commit 23305540b4)

lib/ansible/inventory/dir.py

@@ -45,10 +45,9 @@ def get_file_parser(hostsfile, groups, loader):
     parser = None
 
     try:
-        inv_file = open(hostsfile)
+        with open(hostsfile, 'rb') as inv_file:
-        first_line = inv_file.readlines()[0]
+            initial_chars = inv_file.read(2)
-        inv_file.close()
+            if initial_chars.startswith(b'#!'):
-        if first_line.startswith('#!'):
                 shebang_present = True
     except:
         pass

lib/ansible/inventory/ini.py

@@ -40,7 +40,6 @@ class InventoryParser(object):
     """
 
     def __init__(self, loader, groups, filename=C.DEFAULT_HOST_LIST):
-        self._loader = loader
         self.filename = filename
 
         # Start with an empty host list and whatever groups we're passed in
@@ -53,12 +52,17 @@ class InventoryParser(object):
         # Read in the hosts, groups, and variables defined in the
         # inventory file.
 
-        if loader:
+        with open(filename, 'rb') as fh:
-            (data, private) = loader._get_file_contents(filename)
+            data = fh.read()
-        else:
+            try:
-            with open(filename) as fh:
+                # Faster to do to_text once on a long string than many
-                data = to_text(fh.read())
+                # times on smaller strings
-        data = data.split('\n')
+                data = to_text(data, errors='surrogate_or_strict')
+                data = [line for line in data.splitlines() if not (line.startswith(u';') or line.startswith(u'#'))]
+            except UnicodeError:
+                # Skip comment lines here to avoid potential undecodable
+                # errors in comments: https://github.com/ansible/ansible/issues/17593
+                data = [to_text(line, errors='surrogate_or_strict') for line in data.splitlines() if not (line.startswith(b';') or line.startswith(b'#'))]
 
         self._parse(data)
 
@@ -88,8 +92,8 @@ class InventoryParser(object):
 
             line = line.strip()
 
-            # Skip empty lines and comments
+            # Skip empty lines
-            if line == '' or line.startswith(";") or line.startswith("#"):
+            if not line:
                 continue
 
             # Is this a [section] header? That tells us what group we're parsing

test/units/vars/test_variable_manager.py

@@ -20,17 +20,20 @@ from __future__ import (absolute_import, division, print_function)
 __metaclass__ = type
 
 from collections import defaultdict
-from six import iteritems
 
+from ansible.compat.six import iteritems
+from ansible.compat.six.moves import builtins
 from ansible.compat.tests import unittest
-from ansible.compat.tests.mock import patch, MagicMock
+from ansible.compat.tests.mock import MagicMock, mock_open, patch
 from ansible.inventory import Inventory
 from ansible.playbook.play import Play
-from ansible.vars import VariableManager
 
 from units.mock.loader import DictDataLoader
 from units.mock.path import mock_unfrackpath_noop
 
+from ansible.vars import VariableManager
+
+
 class TestVariableManager(unittest.TestCase):
 
     def setUp(self):
@@ -192,9 +195,7 @@ class TestVariableManager(unittest.TestCase):
         v = VariableManager()
         v._fact_cache = defaultdict(dict)
 
-        fake_loader = DictDataLoader({
+        inventory1_filedata = """
-            # inventory1
-            '/etc/ansible/inventory1': """
             [group2:children]
             group1
 
@@ -206,8 +207,11 @@ class TestVariableManager(unittest.TestCase):
 
             [group2:vars]
             group_var = group_var_from_inventory_group2
-            """,
+            """
 
+        fake_loader = DictDataLoader({
+            # inventory1
+            '/etc/ansible/inventory1': inventory1_filedata,
             # role defaults_only1
             '/etc/ansible/roles/defaults_only1/defaults/main.yml': """
             default_var: "default_var_from_defaults_only1"
@@ -231,6 +235,7 @@ class TestVariableManager(unittest.TestCase):
         })
 
         mock_basedir.return_value = './'
+        with patch.object(builtins, 'open', mock_open(read_data=inventory1_filedata)):
             inv1 = Inventory(loader=fake_loader, variable_manager=v, host_list='/etc/ansible/inventory1')
         inv1.set_playbook_basedir('./')