diff --git a/lib/ansible/module_utils/_text.py b/lib/ansible/module_utils/_text.py index 81943bbea8..34d8f52024 100644 --- a/lib/ansible/module_utils/_text.py +++ b/lib/ansible/module_utils/_text.py @@ -44,7 +44,7 @@ except LookupError: HAS_SURROGATEESCAPE = False -_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_escape', +_COMPOSED_ERROR_HANDLERS = frozenset((None, 'surrogate_or_replace', 'surrogate_or_strict', 'surrogate_then_replace')) @@ -133,6 +133,9 @@ def to_bytes(obj, encoding='utf-8', errors=None, nonstring='simplerepr'): return obj.encode(encoding, errors) except UnicodeEncodeError: if original_errors in (None, 'surrogate_then_replace'): + # We should only reach this if encoding was non-utf8 original_errors was + # surrogate_then_escape and errors was surrogateescape + # Slow but works return_string = obj.encode('utf-8', 'surrogateescape') return_string = return_string.decode('utf-8', 'replace') diff --git a/lib/ansible/module_utils/basic.py b/lib/ansible/module_utils/basic.py index d353ba4555..39c118da30 100644 --- a/lib/ansible/module_utils/basic.py +++ b/lib/ansible/module_utils/basic.py @@ -1011,7 +1011,7 @@ class AnsibleModule(object): return context def user_and_group(self, path, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) st = os.lstat(b_path) @@ -1105,10 +1105,10 @@ class AnsibleModule(object): return changed def set_owner_if_different(self, path, owner, changed, diff=None, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') if owner is None: return changed orig_uid, orig_gid = self.user_and_group(path, expand) @@ -1139,10 +1139,10 @@ class AnsibleModule(object): return changed def set_group_if_different(self, path, group, changed, diff=None, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') if group is None: return changed orig_uid, orig_gid = self.user_and_group(b_path, expand) @@ -1173,10 +1173,10 @@ class AnsibleModule(object): return changed def set_mode_if_different(self, path, mode, changed, diff=None, expand=True): - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') path_stat = os.lstat(b_path) if mode is None: @@ -1253,10 +1253,10 @@ class AnsibleModule(object): if attributes is None: return changed - b_path = to_bytes(path, errors='surrogate_then_strict') + b_path = to_bytes(path, errors='surrogate_or_strict') if expand: b_path = os.path.expanduser(os.path.expandvars(b_path)) - path = to_text(b_path, errors='surrogate_then_strict') + path = to_text(b_path, errors='surrogate_or_strict') existing = self.get_file_attributes(b_path) diff --git a/lib/ansible/plugins/filter/core.py b/lib/ansible/plugins/filter/core.py index 995688989e..4d7b2b25db 100644 --- a/lib/ansible/plugins/filter/core.py +++ b/lib/ansible/plugins/filter/core.py @@ -262,7 +262,7 @@ def get_hash(data, hashtype='sha1'): except: return None - h.update(to_bytes(data, errors='surrogate_then_strict')) + h.update(to_bytes(data, errors='surrogate_or_strict')) return h.hexdigest() @@ -453,11 +453,11 @@ def do_groupby(environment, value, attribute): def b64encode(string): - return to_text(base64.b64encode(to_bytes(string, errors='surrogate_then_strict'))) + return to_text(base64.b64encode(to_bytes(string, errors='surrogate_or_strict'))) def b64decode(string): - return to_text(base64.b64decode(to_bytes(string, errors='surrogate_then_strict'))) + return to_text(base64.b64decode(to_bytes(string, errors='surrogate_or_strict'))) class FilterModule(object):