wbrawner/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

More fixes for unicode handling in the connection plugins.

Browse source 
Tested that ssh, docker, local, lxc-libvirt, chroot all work with the
updated unicode integration test.
This commit is contained in:
Toshio Kuratomi 2016-01-06 15:18:22 -08:00
parent 669b311dbe
commit 46903c80fa
7 changed files with 62 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
lib/ansible/inventory/script.py
View file

@ -31,6 +31,7 @@ from ansible.errors import AnsibleError
from ansible.inventory.host import Host
from ansible.inventory.group import Group
from ansible.module_utils.basic import json_dict_bytes_to_unicode
from ansible.utils.unicode import to_str
class InventoryScript:
@ -72,11 +73,11 @@ class InventoryScript:
self.raw = self._loader.load(self.data)
except Exception as e:
sys.stderr.write(err + "\n")
raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(self.filename, str(e)))
raise AnsibleError("failed to parse executable inventory script results from {0}: {1}".format(to_str(self.filename_, to_str(e)))
if not isinstance(self.raw, Mapping):
sys.stderr.write(err + "\n")
raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(self.filename))
raise AnsibleError("failed to parse executable inventory script results from {0}: data needs to be formatted as a json dict".format(to_str(self.filename)))
self.raw = json_dict_bytes_to_unicode(self.raw)

4
lib/ansible/plugins/connection/docker.py
View file

@ -113,7 +113,7 @@ class Connection(ConnectionBase):
""" Connect to the container. Nothing to do """
super(Connection, self)._connect()
if not self._connected:
display.vvv("ESTABLISH DOCKER CONNECTION FOR USER: {0}".format(
display.vvv(u"ESTABLISH DOCKER CONNECTION FOR USER: {0}".format(
self._play_context.remote_user, host=self._play_context.remote_addr)
)
self._connected = True
@ -172,7 +172,7 @@ class Connection(ConnectionBase):
# running containers, so we use docker exec to implement this
executable = C.DEFAULT_EXECUTABLE.split()[0] if C.DEFAULT_EXECUTABLE else '/bin/sh'
args = [self.docker_cmd, "exec", "-i", self._play_context.remote_addr, executable, "-c",
"dd of={0} bs={1}".format(out_path, BUFSIZE)]
"dd of=%s bs=%s" % (out_path, BUFSIZE)]
args = map(to_bytes, args)
with open(in_path, 'rb') as in_file:
try:

14
lib/ansible/plugins/connection/local.py
View file

@ -31,7 +31,7 @@ import ansible.constants as C
from ansible.errors import AnsibleError, AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.unicode import to_bytes
from ansible.utils.unicode import to_bytes, to_str
try:
from __main__ import display
@ -57,7 +57,7 @@ class Connection(ConnectionBase):
self._play_context.remote_user = getpass.getuser()
if not self._connected:
display.vvv("ESTABLISH LOCAL CONNECTION FOR USER: {0}".format(self._play_context.remote_user, host=self._play_context.remote_addr))
display.vvv(u"ESTABLISH LOCAL CONNECTION FOR USER: {0}".format(self._play_context.remote_user, host=self._play_context.remote_addr))
self._connected = True
return self
@ -126,22 +126,22 @@ class Connection(ConnectionBase):
super(Connection, self).put_file(in_path, out_path)
display.vvv("{0} PUT {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path))
display.vvv(u"{0} PUT {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path))
if not os.path.exists(in_path):
raise AnsibleFileNotFound("file or module does not exist: {0}".format(in_path))
raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
try:
shutil.copyfile(in_path, out_path)
except shutil.Error:
raise AnsibleError("failed to copy: {0} and {1} are the same".format(in_path, out_path))
raise AnsibleError("failed to copy: {0} and {1} are the same".format(to_str(in_path), to_str(out_path)))
except IOError as e:
raise AnsibleError("failed to transfer file to {0}: {1}".format(out_path, e))
raise AnsibleError("failed to transfer file to {0}: {1}".format(to_str(out_path), to_str(e)))
def fetch_file(self, in_path, out_path):
''' fetch a file from local to local -- for copatibility '''
super(Connection, self).fetch_file(in_path, out_path)
display.vvv("{0} FETCH {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path))
display.vvv(u"{0} FETCH {1} TO {2}".format(self._play_context.remote_addr, in_path, out_path))
self.put_file(in_path, out_path)
def close(self):

32
lib/ansible/plugins/connection/ssh.py
View file

@ -32,7 +32,7 @@ from ansible import constants as C
from ansible.errors import AnsibleError, AnsibleConnectionFailure, AnsibleFileNotFound
from ansible.plugins.connection import ConnectionBase
from ansible.utils.path import unfrackpath, makedirs_safe
from ansible.utils.unicode import to_bytes, to_unicode
from ansible.utils.unicode import to_bytes, to_unicode, to_str
from ansible.compat.six import text_type, binary_type
try:
@ -197,7 +197,7 @@ class Connection(ConnectionBase):
if user:
self._add_args(
"ANSIBLE_REMOTE_USER/remote_user/ansible_user/user/-u set",
("-o", "User={0}".format(self._play_context.remote_user))
("-o", "User={0}".format(to_bytes(self._play_context.remote_user)))
)
self._add_args(
@ -231,7 +231,7 @@ class Connection(ConnectionBase):
raise AnsibleError("Cannot write to ControlPath %s" % cpdir)
args = ("-o", "ControlPath={0}".format(
C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir))
to_bytes(C.ANSIBLE_SSH_CONTROL_PATH % dict(directory=cpdir)))
)
self._add_args("found only ControlPersist; added ControlPath", args)
@ -320,7 +320,7 @@ class Connection(ConnectionBase):
Starts the command and communicates with it until it ends.
'''
display_cmd = map(pipes.quote, cmd)
display_cmd = map(to_unicode, map(pipes.quote, cmd))
display.vvv(u'SSH: EXEC {0}'.format(u' '.join(display_cmd)), host=self.host)
# Start the given command. If we don't need to pipeline data, we can try
@ -354,7 +354,7 @@ class Connection(ConnectionBase):
if self._play_context.password:
os.close(self.sshpass_pipe[0])
os.write(self.sshpass_pipe[1], "{0}\n".format(self._play_context.password))
os.write(self.sshpass_pipe[1], "{0}\n".format(to_bytes(self._play_context.password)))
os.close(self.sshpass_pipe[1])
## SSH state machine
@ -562,7 +562,7 @@ class Connection(ConnectionBase):
super(Connection, self).exec_command(cmd, in_data=in_data, sudoable=sudoable)
display.vvv("ESTABLISH SSH CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr)
display.vvv(u"ESTABLISH SSH CONNECTION FOR USER: {0}".format(self._play_context.remote_user), host=self._play_context.remote_addr)
# we can only use tty when we are not pipelining the modules. piping
# data into /usr/bin/python inside a tty automatically invokes the
@ -630,44 +630,46 @@ class Connection(ConnectionBase):
super(Connection, self).put_file(in_path, out_path)
display.vvv("PUT {0} TO {1}".format(in_path, out_path), host=self.host)
display.vvv(u"PUT {0} TO {1}".format(in_path, out_path), host=self.host)
if not os.path.exists(in_path):
raise AnsibleFileNotFound("file or module does not exist: {0}".format(in_path))
raise AnsibleFileNotFound("file or module does not exist: {0}".format(to_str(in_path)))
# scp and sftp require square brackets for IPv6 addresses, but
# accept them for hostnames and IPv4 addresses too.
host = '[%s]' % self.host
if C.DEFAULT_SCP_IF_SSH:
cmd = self._build_command('scp', in_path, '{0}:{1}'.format(host, pipes.quote(out_path)))
cmd = self._build_command('scp', in_path, u'{0}:{1}'.format(host, pipes.quote(out_path)))
in_data = None
else:
cmd = self._build_command('sftp', host)
in_data = "put {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))
cmd = self._build_command('sftp', to_bytes(host))
in_data = u"put {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))
in_data = to_bytes(in_data, nonstring='passthru')
(returncode, stdout, stderr) = self._run(cmd, in_data)
if returncode != 0:
raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(out_path, stdout, stderr))
raise AnsibleError("failed to transfer file to {0}:\n{1}\n{2}".format(to_str(out_path), to_str(stdout), to_str(stderr)))
def fetch_file(self, in_path, out_path):
''' fetch a file from remote to local '''
super(Connection, self).fetch_file(in_path, out_path)
display.vvv("FETCH {0} TO {1}".format(in_path, out_path), host=self.host)
display.vvv(u"FETCH {0} TO {1}".format(in_path, out_path), host=self.host)
# scp and sftp require square brackets for IPv6 addresses, but
# accept them for hostnames and IPv4 addresses too.
host = '[%s]' % self.host
if C.DEFAULT_SCP_IF_SSH:
cmd = self._build_command('scp', '{0}:{1}'.format(host, pipes.quote(in_path)), out_path)
cmd = self._build_command('scp', u'{0}:{1}'.format(host, pipes.quote(in_path)), out_path)
in_data = None
else:
cmd = self._build_command('sftp', host)
in_data = "get {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))
in_data = u"get {0} {1}\n".format(pipes.quote(in_path), pipes.quote(out_path))
in_data = to_bytes(in_data, nonstring='passthru')
(returncode, stdout, stderr) = self._run(cmd, in_data)
if returncode != 0:

2
lib/ansible/plugins/connection/winrm.py
View file

@ -318,7 +318,7 @@ class Connection(ConnectionBase):
local_sha1 = secure_hash(in_path)
if not remote_sha1 == local_sha1:
raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(remote_sha1, local_sha1))
raise AnsibleError("Remote sha1 hash {0} does not match local hash {1}".format(to_str(remote_sha1), to_str(local_sha1)))
def fetch_file(self, in_path, out_path):

4
lib/ansible/plugins/shell/sh.py
View file

@ -136,8 +136,8 @@ class ShellModule(object):
shell_escaped_path = pipes.quote(path)
test = "rc=flag; [ -r %(p)s ] %(shell_or)s rc=2; [ -f %(p)s ] %(shell_or)s rc=1; [ -d %(p)s ] %(shell_and)s rc=3; %(i)s -V 2>/dev/null %(shell_or)s rc=4; [ x\"$rc\" != \"xflag\" ] %(shell_and)s echo \"${rc} \"%(p)s %(shell_and)s exit 0" % dict(p=shell_escaped_path, i=python_interp, shell_and=self._SHELL_AND, shell_or=self._SHELL_OR)
csums = [
"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python > 2.4 (including python3)
"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python == 2.4
u"({0} -c 'import hashlib; BLOCKSIZE = 65536; hasher = hashlib.sha1();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python > 2.4 (including python3)
u"({0} -c 'import sha; BLOCKSIZE = 65536; hasher = sha.sha();{2}afile = open(\"'{1}'\", \"rb\"){2}buf = afile.read(BLOCKSIZE){2}while len(buf) > 0:{2}\thasher.update(buf){2}\tbuf = afile.read(BLOCKSIZE){2}afile.close(){2}print(hasher.hexdigest())' 2>/dev/null)".format(python_interp, shell_escaped_path, self._SHELL_EMBEDDED_PY_EOL), # Python == 2.4
]
cmd = (" %s " % self._SHELL_OR).join(csums)

30
test/integration/unicode.yml
View file

@ -93,6 +93,36 @@
that:
- "'Zażółć' in results.stdout_lines"
- name: Clean a temp directory
file:
path: /var/tmp/ansible_test_unicode_get_put
state: absent
- name: Create a temp directory
file:
path: /var/tmp/ansible_test_unicode_get_put
state: directory
- name: Create a file with a non-ascii filename
file:
path: /var/tmp/ansible_test_unicode_get_put/Zażółć
state: touch
delegate_to: localhost
- name: Put with unicode filename
copy:
src: /var/tmp/ansible_test_unicode_get_put/Zażółć
dest: /var/tmp/ansible_test_unicode_get_put/Zażółć2
- name: Fetch with unicode filename
fetch:
src: /var/tmp/ansible_test_unicode_get_put/Zażółć2
dest: /var/tmp/ansible_test_unicode_get_put/
- name: Clean a temp directory
file:
path: /var/tmp/ansible_test_unicode_get_put
state: absent
- name: 'A play for hosts in group: ĪīĬĭ'
hosts: 'ĪīĬĭ'