wbrawner/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Ziploader

Browse source 
* Ziploader proof of concept (jimi-c)

* Cleanups to proof of concept ziploader branch:

* python3 compatible base64 encoding
* zipfile compression (still need to enable toggling this off for
  systems without zlib support in python)
* Allow non-wildcard imports (still need to make this recusrsive so that
  we can have module_utils code that imports other module_utils code.)
* Better tracebacks: module filename is kept and module_utils directory
  is kept so that tracebacks show the real filenames that the errors
  appear in.

* Make sure we import modules that are used into the module_utils files that they are used in.

* Set ansible version in a more pythonic way for ziploader than we were doing in module replacer

* Make it possible to set the module compression as an inventory var

This may be necessary on systems where python has been compiled without
zlib compression.

* Refactoring of module_common code:

* module replacer only replaces values that make sense for that type of
  file (example: don't attempt to replace python imports if we're in
  a powershell module).
* Implement configurable shebang support for ziploader wrapper
* Implement client-side constants (for SELINUX_SPECIAL_FS and SYSLOG)
  via environment variable.
* Remove strip_comments param as we're never going to use it (ruins line
  numbering)

* Don't repeat ourselves about detecting REPLACER

* Add an easy way to debug

* Port test-module to the ziploader-aware modify_module()

* strip comments and blank lines from the wrapper so we send less over the wire.

* Comments cleanup

* Remember to output write the module line itself in powershell modules

* for line in lines strips the newlines so we have to add them back in
This commit is contained in:
Toshio Kuratomi 2016-04-05 11:06:17 -07:00
parent 6a3670b1f0
commit 4b0aa1214c
32 changed files with 438 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
examples/ansible.cfg
View file

@ -236,6 +236,16 @@
# is used. This value must be an integer from 0 to 9.
#var_compression_level = 9
# controls what compression method is used for new-style ansible modules when
# they are sent to the remote system. The compression types depend on having
# support compiled into both the controller's python and the client's python.
# The names should match with the python Zipfile compression types:
# * ZIP_STORED (no compression. available everywhere)
# * ZIP_DEFLATED (uses zlib, the default)
# These values may be set per host via the ansible_module_compression inventory
# variable
#module_compression = 'ZIP_DEFLATED'
# This controls the cutoff point (in bytes) on --diff for files
# set to 0 for unlimited (RAM may suffer!).
#max_diff_size = 1048576

3
hacking/test-module
View file

@ -131,7 +131,10 @@ def boilerplate_module(modfile, args, interpreter, check, destfile):
if check:
complex_args['_ansible_check_mode'] = True
modname = os.path.basename(modfile)
modname = os.path.splitext(modname)[0]
(module_data, module_style, shebang) = module_common.modify_module(
modname,
modfile,
complex_args,
task_vars=task_vars

1
lib/ansible/constants.py
View file

@ -140,6 +140,7 @@ DEFAULT_MODULE_NAME = get_config(p, DEFAULTS, 'module_name', None,
DEFAULT_FORKS = get_config(p, DEFAULTS, 'forks', 'ANSIBLE_FORKS', 5, integer=True)
DEFAULT_MODULE_ARGS = get_config(p, DEFAULTS, 'module_args', 'ANSIBLE_MODULE_ARGS', '')
DEFAULT_MODULE_LANG = get_config(p, DEFAULTS, 'module_lang', 'ANSIBLE_MODULE_LANG', os.getenv('LANG', 'en_US.UTF-8'))
DEFAULT_MODULE_COMPRESSION= get_config(p, DEFAULTS, 'module_compression', None, 'ZIP_DEFLATED')
DEFAULT_TIMEOUT = get_config(p, DEFAULTS, 'timeout', 'ANSIBLE_TIMEOUT', 10, integer=True)
DEFAULT_POLL_INTERVAL = get_config(p, DEFAULTS, 'poll_interval', 'ANSIBLE_POLL_INTERVAL', 15, integer=True)
DEFAULT_REMOTE_USER = get_config(p, DEFAULTS, 'remote_user', 'ANSIBLE_REMOTE_USER', None)

328
lib/ansible/executor/module_common.py
View file

@ -20,11 +20,12 @@
from __future__ import (absolute_import, division, print_function)
__metaclass__ = type
# from python and deps
from io import BytesIO
import base64
import json
import os
import shlex
import zipfile
from io import BytesIO
# from Ansible
from ansible import __version__
@ -32,13 +33,17 @@ from ansible import constants as C
from ansible.errors import AnsibleError
from ansible.utils.unicode import to_bytes, to_unicode
try:
from __main__ import display
except ImportError:
from ansible.utils.display import Display
display = Display()
REPLACER = b"#<<INCLUDE_ANSIBLE_MODULE_COMMON>>"
REPLACER_ARGS = b"\"<<INCLUDE_ANSIBLE_MODULE_ARGS>>\""
REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\""
REPLACER_COMPLEX = b"\"<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>\""
REPLACER_WINDOWS = b"# POWERSHELL_COMMON"
REPLACER_WINARGS = b"<<INCLUDE_ANSIBLE_MODULE_WINDOWS_ARGS>>"
REPLACER_JSONARGS = b"<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
REPLACER_VERSION = b"\"<<ANSIBLE_VERSION>>\""
REPLACER_SELINUX = b"<<SELINUX_SPECIAL_FILESYSTEMS>>"
# We could end up writing out parameters with unicode characters so we need to
@ -50,6 +55,82 @@ _SNIPPET_PATH = os.path.join(os.path.dirname(__file__), '..', 'module_utils')
# ******************************************************************************
ZIPLOADER_TEMPLATE = u'''%(shebang)s
# -*- coding: utf-8 -*-'
import os
import sys
import base64
import tempfile
ZIPDATA = """%(zipdata)s"""
def debug(command, zipped_mod):
# The code here normally doesn't run. It's only used for debugging on the
# remote machine. Run with ANSIBLE_KEEP_REMOTE_FILES=1 envvar and -vvv
# to save the module file remotely. Login to the remote machine and use
# /path/to/module explode to extract the ZIPDATA payload into source
# files. Edit the source files to instrument the code or experiment with
# different values. Then use /path/to/module execute to run the extracted
# files you've edited instead of the actual zipped module.
#
# Okay to use __file__ here because we're running from a kept file
basedir = os.path.dirname(__file__)
if command == 'explode':
import zipfile
z = zipfile.ZipFile(zipped_mod)
for filename in z.namelist():
if filename.startswith('/'):
raise Exception('Something wrong with this module zip file: should not contain absolute paths')
dest_filename = os.path.join(basedir, filename)
if dest_filename.endswith(os.path.sep) and not os.path.exists(dest_filename):
os.makedirs(dest_filename)
else:
directory = os.path.dirname(dest_filename)
if not os.path.exists(directory):
os.makedirs(directory)
f = open(dest_filename, 'w')
f.write(z.read(filename))
f.close()
print('Module expanded into: %%s' %% os.path.join(basedir, 'ansible'))
elif command == 'execute':
sys.path.insert(0, basedir)
from ansible.module_exec.%(ansible_module)s.__main__ import main
main()
os.environ['ANSIBLE_MODULE_ARGS'] = %(args)s
os.environ['ANSIBLE_MODULE_CONSTANTS'] = %(constants)s
try:
temp_fd, temp_path = tempfile.mkstemp(prefix='ansible_')
os.write(temp_fd, base64.b64decode(ZIPDATA))
if len(sys.argv) == 2:
debug(sys.argv[1], temp_path)
else:
sys.path.insert(0, temp_path)
from ansible.module_exec.%(ansible_module)s.__main__ import main
main()
finally:
try:
os.close(temp_fd)
os.remove(temp_path)
except NameError:
# mkstemp failed
pass
'''
def _strip_comments(source):
# Strip comments and blank lines from the wrapper
buf = []
for line in source.splitlines():
l = line.strip()
if not l or l.startswith(u'#'):
continue
buf.append(line)
return u'\n'.join(buf)
# ZIPLOADER_TEMPLATE stripped of comments for smaller over the wire size
STRIPPED_ZIPLOADER_TEMPLATE = _strip_comments(ZIPLOADER_TEMPLATE)
def _slurp(path):
if not os.path.exists(path):
raise AnsibleError("imported module support code does not exist at %s" % path)
@ -58,69 +139,171 @@ def _slurp(path):
fd.close()
return data
def _find_snippet_imports(module_data, module_path, strip_comments):
def _get_shebang(interpreter, task_vars, args=tuple()):
"""
Note not stellar API:
Returns None instead of always returning a shebang line. Doing it this
way allows the caller to decide to use the shebang it read from the
file rather than trust that we reformatted what they already have
correctly.
"""
interpreter_config = u'ansible_%s_interpreter' % os.path.basename(interpreter)
if interpreter_config not in task_vars:
return None
interpreter = task_vars[interpreter_config]
shebang = u'#!' + interpreter
if args:
shebang = shebang + u' ' + u' '.join(args)
return shebang
def _get_facility(task_vars):
facility = C.DEFAULT_SYSLOG_FACILITY
if 'ansible_syslog_facility' in task_vars:
facility = task_vars['ansible_syslog_facility']
return facility
def _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression):
"""
Given the source of the module, convert it to a Jinja2 template to insert
module code and return whether it's a new or old style module.
"""
module_style = 'old'
module_substyle = module_style = 'old'
# module_style is something important to calling code (ActionBase). It
# determines how arguments are formatted (json vs k=v) and whether
# a separate arguments file needs to be sent over the wire.
# module_substyle is extra information that's useful internally. It tells
# us what we have to look to substitute in the module files and whether
# we're using module replacer or ziploader to format the module itself.
if REPLACER in module_data:
# Do REPLACER before from ansible.module_utils because we need make sure
# we substitute "from ansible.module_utils basic" for REPLACER
module_style = 'new'
elif REPLACER_WINDOWS in module_data:
module_style = 'new'
elif REPLACER_JSONARGS in module_data:
module_style = 'new'
module_substyle = 'python'
module_data = module_data.replace(REPLACER, b'from ansible.module_utils.basic import *')
elif b'from ansible.module_utils.' in module_data:
module_style = 'new'
module_substyle = 'python'
elif REPLACER_WINDOWS in module_data:
module_style = 'new'
module_substyle = 'powershell'
elif REPLACER_JSONARGS in module_data:
module_style = 'new'
module_substyle = 'jsonargs'
elif b'WANT_JSON' in module_data:
module_style = 'non_native_want_json'
module_substyle = module_style = 'non_native_want_json'
shebang = None
# Neither old-style nor non_native_want_json modules should be modified
# except for the shebang line (Done by modify_module)
if module_style in ('old', 'non_native_want_json'):
return module_data, module_style, shebang
module_args_json = to_bytes(json.dumps(module_args))
output = BytesIO()
lines = module_data.split(b'\n')
snippet_names = []
for line in lines:
snippet_names = set()
if REPLACER in line:
output.write(_slurp(os.path.join(_SNIPPET_PATH, "basic.py")))
snippet_names.append(b'basic')
if REPLACER_WINDOWS in line:
ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
output.write(ps_data)
snippet_names.append(b'powershell')
elif line.startswith(b'from ansible.module_utils.'):
tokens=line.split(b".")
import_error = False
if len(tokens) != 3:
import_error = True
if b" import *" not in line:
import_error = True
if import_error:
raise AnsibleError("error importing module in %s, expecting format like 'from ansible.module_utils.<lib name> import *'" % module_path)
snippet_name = tokens[2].split()[0]
snippet_names.append(snippet_name)
output.write(_slurp(os.path.join(_SNIPPET_PATH, to_unicode(snippet_name) + ".py")))
else:
if strip_comments and line.startswith(b"#") or line == b'':
pass
output.write(line)
output.write(b"\n")
if module_substyle == 'python':
# ziploader for new-style python classes
python_repred_args = to_bytes(repr(module_args_json))
constants = dict(
SELINUX_SPECIAL_FS=C.DEFAULT_SELINUX_SPECIAL_FS,
SYSLOG_FACILITY=_get_facility(task_vars),
)
python_repred_constants = to_bytes(repr(json.dumps(constants)), errors='strict')
if not module_path.endswith(".ps1"):
# Unixy modules
if len(snippet_names) > 0 and not b'basic' in snippet_names:
raise AnsibleError("missing required import in %s: from ansible.module_utils.basic import *" % module_path)
else:
# Windows modules
if len(snippet_names) > 0 and not b'powershell' in snippet_names:
try:
compression_method = getattr(zipfile, module_compression)
except AttributeError:
display.warning(u'Bad module compression string specified: %s. Using ZIP_STORED (no compression)' % module_compression)
compression_method = zipfile.ZIP_STORED
zipoutput = BytesIO()
zf = zipfile.ZipFile(zipoutput, mode='w', compression=compression_method)
zf.writestr('ansible/__init__.py', b''.join((b"__version__ = '", to_bytes(__version__), b"'\n")))
zf.writestr('ansible/module_utils/__init__.py', b'')
zf.writestr('ansible/module_exec/__init__.py', b'')
zf.writestr('ansible/module_exec/%s/__init__.py' % module_name, b"")
final_data = []
for line in lines:
if line.startswith(b'from ansible.module_utils.'):
tokens=line.split(b".")
snippet_name = tokens[2].split()[0]
snippet_names.add(snippet_name)
fname = to_unicode(snippet_name + b".py")
zf.writestr(os.path.join("ansible/module_utils", fname), _slurp(os.path.join(_SNIPPET_PATH, fname)))
final_data.append(line)
else:
final_data.append(line)
zf.writestr('ansible/module_exec/%s/__main__.py' % module_name, b"\n".join(final_data))
zf.close()
shebang = _get_shebang(u'/usr/bin/python', task_vars) or u'#!/usr/bin/python'
output.write(to_bytes(STRIPPED_ZIPLOADER_TEMPLATE % dict(
zipdata=base64.b64encode(zipoutput.getvalue()),
ansible_module=module_name,
args=python_repred_args,
constants=python_repred_constants,
shebang=shebang,
)))
module_data = output.getvalue()
# Sanity check from 1.x days. Maybe too strict. Some custom python
# modules that use ziploader may implement their own helpers and not
# need basic.py. All the constants that we substituted into basic.py
# for module_replacer are now available in other, better ways.
if b'basic' not in snippet_names:
raise AnsibleError("missing required import in %s: Did not import ansible.module_utils.basic for boilerplate helper code" % module_path)
elif module_substyle == 'powershell':
# Module replacer for jsonargs and windows
for line in lines:
if REPLACER_WINDOWS in line:
ps_data = _slurp(os.path.join(_SNIPPET_PATH, "powershell.ps1"))
output.write(ps_data)
snippet_names.add(b'powershell')
continue
output.write(line + b'\n')
module_data = output.getvalue()
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
# Sanity check from 1.x days. This is currently useless as we only
# get here if we are going to substitute powershell.ps1 into the
# module anyway. Leaving it for when/if we add other powershell
# module_utils files.
if b'powershell' not in snippet_names:
raise AnsibleError("missing required import in %s: # POWERSHELL_COMMON" % module_path)
return (output.getvalue(), module_style)
elif module_substyle == 'jsonargs':
# these strings could be included in a third-party module but
# officially they were included in the 'basic' snippet for new-style
# python modules (which has been replaced with something else in
# ziploader) If we remove them from jsonargs-style module replacer
# then we can remove them everywhere.
module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
# The main event -- substitute the JSON args string into the module
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
facility = b'syslog.' + to_bytes(_get_facility(task_vars), errors='strict')
module_data = module_data.replace(b'syslog.LOG_USER', facility)
return (module_data, module_style, shebang)
# ******************************************************************************
def modify_module(module_path, module_args, task_vars=dict(), strip_comments=False):
def modify_module(module_name, module_path, module_args, task_vars=dict(), module_compression='ZIP_STORED'):
"""
Used to insert chunks of code into modules before transfer rather than
doing regular python imports. This allows for more efficient transfer in
@ -163,43 +346,28 @@ def modify_module(module_path, module_args, task_vars=dict(), strip_comments=Fal
# read in the module source
module_data = f.read()
(module_data, module_style) = _find_snippet_imports(module_data, module_path, strip_comments)
(module_data, module_style, shebang) = _find_snippet_imports(module_name, module_data, module_path, module_args, task_vars, module_compression)
module_args_json = to_bytes(json.dumps(module_args))
python_repred_args = to_bytes(repr(module_args_json))
if shebang is None:
lines = module_data.split(b"\n", 1)
if lines[0].startswith(b"#!"):
shebang = lines[0].strip()
args = shlex.split(str(shebang[2:]))
interpreter = args[0]
interpreter = to_bytes(interpreter)
# these strings should be part of the 'basic' snippet which is required to be included
module_data = module_data.replace(REPLACER_VERSION, to_bytes(repr(__version__)))
module_data = module_data.replace(REPLACER_COMPLEX, python_repred_args)
module_data = module_data.replace(REPLACER_WINARGS, module_args_json)
module_data = module_data.replace(REPLACER_JSONARGS, module_args_json)
module_data = module_data.replace(REPLACER_SELINUX, to_bytes(','.join(C.DEFAULT_SELINUX_SPECIAL_FS)))
new_shebang = to_bytes(_get_shebang(interpreter, task_vars, args[1:]), errors='strict', nonstring='passthru')
if new_shebang:
lines[0] = shebang = new_shebang
if module_style == 'new':
facility = C.DEFAULT_SYSLOG_FACILITY
if 'ansible_syslog_facility' in task_vars:
facility = task_vars['ansible_syslog_facility']
module_data = module_data.replace(b'syslog.LOG_USER', to_bytes("syslog.%s" % facility))
if os.path.basename(interpreter).startswith(b'python'):
lines.insert(1, ENCODING_STRING)
else:
# No shebang, assume a binary module?
pass
lines = module_data.split(b"\n", 1)
shebang = None
if lines[0].startswith(b"#!"):
shebang = lines[0].strip()
args = shlex.split(str(shebang[2:]))
interpreter = args[0]
interpreter_config = 'ansible_%s_interpreter' % os.path.basename(interpreter)
interpreter = to_bytes(interpreter)
if interpreter_config in task_vars:
interpreter = to_bytes(task_vars[interpreter_config], errors='strict')
lines[0] = shebang = b"#!{0} {1}".format(interpreter, b" ".join(args[1:]))
if os.path.basename(interpreter).startswith(b'python'):
lines.insert(1, ENCODING_STRING)
module_data = b"\n".join(lines)
else:
# No shebang, assume a binary module?
pass
module_data = b"\n".join(lines)
shebang = to_bytes(shebang, errors='strict')
return (module_data, module_style, shebang)

4
lib/ansible/module_utils/a10.py
View file

@ -27,8 +27,8 @@
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import json
# Note: modules using this must have from ansible.module_utils.urls import *
# before this is imported
from ansible.module_utils.urls import fetch_url
AXAPI_PORT_PROTOCOLS = {
'tcp': 2,

70
lib/ansible/module_utils/basic.py
View file

@ -27,25 +27,13 @@
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# == BEGIN DYNAMICALLY INSERTED CODE ==
ANSIBLE_VERSION = "<<ANSIBLE_VERSION>>"
MODULE_ARGS = "<<INCLUDE_ANSIBLE_MODULE_ARGS>>"
MODULE_COMPLEX_ARGS = "<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>"
BOOLEANS_TRUE = ['yes', 'on', '1', 'true', 1, True]
BOOLEANS_FALSE = ['no', 'off', '0', 'false', 0, False]
BOOLEANS = BOOLEANS_TRUE + BOOLEANS_FALSE
SELINUX_SPECIAL_FS="<<SELINUX_SPECIAL_FILESYSTEMS>>"
# ansible modules can be written in any language. To simplify
# development of Python modules, the functions available here
# can be inserted in any module source automatically by including
# #<<INCLUDE_ANSIBLE_MODULE_COMMON>> on a blank line by itself inside
# of an ansible module. The source of this common code lives
# in ansible/executor/module_common.py
# development of Python modules, the functions available here can
# be used to do many common tasks
import locale
import os
@ -231,6 +219,27 @@ except ImportError:
_literal_eval = literal_eval
from ansible import __version__
# Backwards compat. New code should just import and use __version__
ANSIBLE_VERSION = __version__
try:
# MODULE_COMPLEX_ARGS is an old name kept for backwards compat
MODULE_COMPLEX_ARGS = os.environ.pop('ANSIBLE_MODULE_ARGS')
except KeyError:
# This file might be used for its utility functions. So don't fail if
# running outside of a module environment (will fail in _load_params()
# instead)
MODULE_COMPLEX_ARGS = None
try:
# ARGS are for parameters given in the playbook. Constants are for things
# that ansible needs to configure controller side but are passed to all
# modules.
MODULE_CONSTANTS = os.environ.pop('ANSIBLE_MODULE_CONSTANTS')
except KeyError:
MODULE_CONSTANTS = None
FILE_COMMON_ARGUMENTS=dict(
src = dict(),
mode = dict(type='raw'),
@ -539,7 +548,8 @@ class AnsibleModule(object):
if k not in self.argument_spec:
self.argument_spec[k] = v
self.params = self._load_params()
self._load_constants()
self._load_params()
# append to legal_inputs and then possibly check against them
try:
@ -754,7 +764,7 @@ class AnsibleModule(object):
(device, mount_point, fstype, options, rest) = line.split(' ', 4)
if path_mount_point == mount_point:
for fs in SELINUX_SPECIAL_FS.split(','):
for fs in self.constants['SELINUX_SPECIAL_FS']:
if fs in fstype:
special_context = self.selinux_context(path_mount_point)
return (True, special_context)
@ -1412,16 +1422,38 @@ class AnsibleModule(object):
self.params[k] = default
def _load_params(self):
''' read the input and return a dictionary and the arguments string '''
''' read the input and set the params attribute'''
if MODULE_COMPLEX_ARGS is None:
# This helper used too early for fail_json to work.
print('{"msg": "Error: ANSIBLE_MODULE_ARGS not found in environment. Unable to figure out what parameters were passed", "failed": true}')
sys.exit(1)
params = json_dict_unicode_to_bytes(json.loads(MODULE_COMPLEX_ARGS))
if params is None:
params = dict()
return params
self.params = params
def _load_constants(self):
''' read the input and set the constants attribute'''
if MODULE_CONSTANTS is None:
# This helper used too early for fail_json to work.
print('{"msg": "Error: ANSIBLE_MODULE_CONSTANTS not found in environment. Unable to figure out what constants were passed", "failed": true}')
sys.exit(1)
# Make constants into "native string"
if sys.version_info >= (3,):
constants = json_dict_bytes_to_unicode(json.loads(MODULE_CONSTANTS))
else:
constants = json_dict_unicode_to_bytes(json.loads(MODULE_CONSTANTS))
if constants is None:
constants = dict()
self.constants = constants
def _log_to_syslog(self, msg):
if HAS_SYSLOG:
module = 'ansible-%s' % os.path.basename(__file__)
syslog.openlog(str(module), 0, syslog.LOG_USER)
facility = getattr(syslog, self.constants.get('SYSLOG_FACILITY', 'LOG_USER'), syslog.LOG_USER)
syslog.openlog(str(module), 0, facility)
syslog.syslog(syslog.LOG_INFO, msg)
def debug(self, msg):

7
lib/ansible/module_utils/ec2.py
View file

@ -25,9 +25,16 @@
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import os
from time import sleep
try:
import boto
HAS_BOTO = True
except ImportError:
HAS_BOTO = False
try:
import boto3
import botocore

7
lib/ansible/module_utils/eos.py
View file

@ -18,6 +18,13 @@
#
import os
import re
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.shell import Shell, Command, HAS_PARAMIKO
from ansible.module_utils.netcfg import parse
from ansible.module_utils.urls import fetch_url
NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
NET_COMMON_ARGS = dict(

2
lib/ansible/module_utils/gce.py
View file

@ -29,6 +29,8 @@
import os
import traceback
from libcloud.compute.types import Provider
from libcloud.compute.providers import get_driver
USER_AGENT_PRODUCT="Ansible-gce"

6
lib/ansible/module_utils/ios.py
View file

@ -17,6 +17,12 @@
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
#
import re
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.shell import Shell, Command, HAS_PARAMIKO
from ansible.module_utils.netcfg import parse
NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
NET_COMMON_ARGS = dict(

6
lib/ansible/module_utils/iosxr.py
View file

@ -17,6 +17,12 @@
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
#
import re
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.shell import Shell, HAS_PARAMIKO
from ansible.module_utils.netcfg import parse
NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
NET_COMMON_ARGS = dict(

4
lib/ansible/module_utils/junos.py
View file

@ -17,6 +17,10 @@
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
#
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.shell import Shell, HAS_PARAMIKO
from ansible.module_utils.netcfg import parse
NET_COMMON_ARGS = dict(
host=dict(required=True),
port=dict(default=22, type='int'),

6
lib/ansible/module_utils/mysql.py
View file

@ -27,7 +27,13 @@
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
# USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import os
try:
import MySQLdb
mysqldb_found = True
except ImportError:
mysqldb_found = False
def mysql_connect(module, login_user=None, login_password=None, config_file='', ssl_cert=None, ssl_key=None, ssl_ca=None, db=None, cursor_class=None, connect_timeout=30):
config = {

3
lib/ansible/module_utils/netcfg.py
View file

@ -20,6 +20,9 @@
import re
import collections
import itertools
import shlex
from ansible.module_utils.basic import BOOLEANS_TRUE, BOOLEANS_FALSE
DEFAULT_COMMENT_TOKENS = ['#', '!']

10
lib/ansible/module_utils/nxos.py
View file

@ -16,6 +16,14 @@
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
#
import re
from ansible.module_utils.urls import fetch_url
from ansible.module_utils.shell import Shell, HAS_PARAMIKO
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.netcfg import parse
NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
NET_COMMON_ARGS = dict(
@ -117,8 +125,6 @@ class Nxapi(object):
(command_type, ','.join(NXAPI_COMMAND_TYPES))
self.module_fail_json(msg=msg)
debug = dict()
data = self._get_body(clist, command_type, encoding)
data = self.module.jsonify(data)

6
lib/ansible/module_utils/openswitch.py
View file

@ -16,6 +16,7 @@
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
#
import re
import time
import json
@ -28,6 +29,11 @@ try:
except ImportError:
HAS_OPS = False
from ansible.module_utils.basic import AnsibleModule
from ansible.module_utils.urls import fetch_url
from ansible.module_utils.shell import Shell, HAS_PARAMIKO
from ansible.module_utils.netcfg import parse
NET_PASSWD_RE = re.compile(r"[\r\n]?password: $", re.I)
NET_COMMON_ARGS = dict(

2
lib/ansible/module_utils/powershell.ps1
View file

@ -32,7 +32,7 @@ Set-StrictMode -Version 2.0
# JSON; assign them to an environment variable and redefine $args so existing
# modules will continue to work.
$complex_args = @'
<<INCLUDE_ANSIBLE_MODULE_WINDOWS_ARGS>>
<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>
'@
Set-Content env:MODULE_COMPLEX_ARGS -Value $complex_args
$args = @('env:MODULE_COMPLEX_ARGS')

4
lib/ansible/module_utils/rax.py
View file

@ -32,6 +32,8 @@ import os
import re
from uuid import UUID
from ansible import __version__
from ansible.module_utils.basic import BOOLEANS
FINAL_STATUSES = ('ACTIVE', 'ERROR')
VOLUME_STATUS = ('available', 'attaching', 'creating', 'deleting', 'in-use',
@ -262,7 +264,7 @@ def rax_required_together():
def setup_rax_module(module, rax_module, region_required=True):
"""Set up pyrax in a standard way for all modules"""
rax_module.USER_AGENT = 'ansible/%s %s' % (ANSIBLE_VERSION,
rax_module.USER_AGENT = 'ansible/%s %s' % (__version__,
rax_module.USER_AGENT)
api_key = module.params.get('api_key')

2
lib/ansible/module_utils/shell.py
View file

@ -146,7 +146,7 @@ class Shell(object):
cmd = '%s\r' % str(command)
self.shell.sendall(cmd)
responses.append(self.receive(command))
except socket.timeout, exc:
except socket.timeout:
raise ShellError("timeout trying to send command", cmd)
return responses

25
lib/ansible/module_utils/urls.py
View file

@ -81,6 +81,18 @@
# agrees to be bound by the terms and conditions of this License
# Agreement.
import httplib
import netrc
import os
import re
import sys
import socket
import platform
import tempfile
import base64
from ansible.module_utils.basic import get_distribution
try:
import urllib2
HAS_URLLIB2 = True
@ -151,8 +163,6 @@ if not HAS_MATCH_HOSTNAME:
"""The match_hostname() function from Python 3.4, essential when using SSL."""
import re
class CertificateError(ValueError):
pass
@ -257,17 +267,6 @@ if not HAS_MATCH_HOSTNAME:
HAS_MATCH_HOSTNAME = True
import httplib
import netrc
import os
import re
import sys
import socket
import platform
import tempfile
import base64
# This is a dummy cacert provided for Mac OS since you need at least 1
# ca cert, regardless of validity, for Python on Mac OS to use the
# keychain functionality in OpenSSL for validating SSL certificates.

2
lib/ansible/module_utils/vca.py
View file

@ -21,6 +21,8 @@ try:
except ImportError:
HAS_PYVCLOUD = False
from ansible.module_utils.basic import AnsibleModule
SERVICE_MAP = {'vca': 'ondemand', 'vchs': 'subscription', 'vcd': 'vcd'}
LOGIN_HOST = {'vca': 'vca.vmware.com', 'vchs': 'vchs.vmware.com'}

2
lib/ansible/module_utils/vmware.py
View file

@ -25,7 +25,7 @@ try:
# requests is required for exception handling of the ConnectionError
import requests
from pyVim import connect
from pyVmomi import vim, vmodl
from pyVmomi import vim
HAS_PYVMOMI = True
except ImportError:
HAS_PYVMOMI = False

2
lib/ansible/playbook/play_context.py
View file

@ -80,6 +80,7 @@ MAGIC_VARIABLE_MAPPING = dict(
su_exe = ('ansible_su_exe',),
su_flags = ('ansible_su_flags',),
executable = ('ansible_shell_executable',),
module_compression = ('ansible_module_compression',),
)
SU_PROMPT_LOCALIZATIONS = [
@ -169,6 +170,7 @@ class PlayContext(Base):
_accelerate_ipv6 = FieldAttribute(isa='bool', default=False, always_post_validate=True)
_accelerate_port = FieldAttribute(isa='int', default=C.ACCELERATE_PORT, always_post_validate=True)
_executable = FieldAttribute(isa='string', default=C.DEFAULT_EXECUTABLE)
_module_compression = FieldAttribute(isa='string', default=C.DEFAULT_MODULE_COMPRESSION)
# privilege escalation fields
_become = FieldAttribute(isa='bool')

2
lib/ansible/plugins/action/__init__.py
View file

@ -144,7 +144,7 @@ class ActionBase(with_metaclass(ABCMeta, object)):
"run 'git submodule update --init --recursive' to correct this problem." % (module_name))
# insert shared code and arguments into the module
(module_data, module_style, module_shebang) = modify_module(module_path, module_args, task_vars=task_vars)
(module_data, module_style, module_shebang) = modify_module(module_name, module_path, module_args, task_vars=task_vars, module_compression=self._play_context.module_compression)
return (module_style, module_shebang, module_data)

6
test/integration/roles/test_hash_behavior/tasks/main.yml
View file

@ -17,8 +17,12 @@
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
- name: get the hash behavior env setting
shell: env | grep ANSIBLE_HASH_BEHAVIOUR | cut -f2- -d'='
shell: env | grep ^ANSIBLE_HASH_BEHAVIOUR'=' | cut -f2- -d'='
register: hash_behavior
# This only works with the local connection. The way this test is run means the
connection: local
delegate_to: localhost
- name: debug hash behavior result
debug: var=hash_behavior.stdout

1
test/units/module_utils/basic/test__log_invocation.py
View file

@ -33,6 +33,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
# test basic log invocation
basic.MODULE_COMPLEX_ARGS = json.dumps(dict(foo=False, bar=[1,2,3], bam="bam", baz=u'baz'))
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec=dict(
foo = dict(default=True, type='bool'),

3
test/units/module_utils/basic/test_exit_json.py
View file

@ -39,6 +39,7 @@ class TestAnsibleModuleExitJson(unittest.TestCase):
def setUp(self):
self.COMPLEX_ARGS = basic.MODULE_COMPLEX_ARGS
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
self.old_stdout = sys.stdout
self.fake_stream = BytesIO()
@ -129,6 +130,7 @@ class TestAnsibleModuleExitValuesRemoved(unittest.TestCase):
for args, return_val, expected in self.dataset:
sys.stdout = BytesIO()
basic.MODULE_COMPLEX_ARGS = json.dumps(args)
basic.MODULE_CONSTANTS = '{}'
module = basic.AnsibleModule(
argument_spec = dict(
username=dict(),
@ -148,6 +150,7 @@ class TestAnsibleModuleExitValuesRemoved(unittest.TestCase):
expected['failed'] = True
sys.stdout = BytesIO()
basic.MODULE_COMPLEX_ARGS = json.dumps(args)
basic.MODULE_CONSTANTS = '{}'
module = basic.AnsibleModule(
argument_spec = dict(
username=dict(),

12
test/units/module_utils/basic/test_log.py
View file

@ -42,7 +42,9 @@ class TestAnsibleModuleSysLogSmokeTest(unittest.TestCase):
def setUp(self):
self.complex_args_token = basic.MODULE_COMPLEX_ARGS
self.constants_sentinel = basic.MODULE_CONSTANTS
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
self.am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -54,6 +56,7 @@ class TestAnsibleModuleSysLogSmokeTest(unittest.TestCase):
def tearDown(self):
basic.MODULE_COMPLEX_ARGS = self.complex_args_token
basic.MODULE_CONSTANTS = self.constants_sentinel
basic.has_journal = self.has_journal
def test_smoketest_syslog(self):
@ -73,13 +76,16 @@ class TestAnsibleModuleJournaldSmokeTest(unittest.TestCase):
def setUp(self):
self.complex_args_token = basic.MODULE_COMPLEX_ARGS
self.constants_sentinel = basic.MODULE_CONSTANTS
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
self.am = basic.AnsibleModule(
argument_spec = dict(),
)
def tearDown(self):
basic.MODULE_COMPLEX_ARGS = self.complex_args_token
basic.MODULE_CONSTANTS = self.constants_sentinel
@unittest.skipUnless(basic.has_journal, 'python systemd bindings not installed')
def test_smoketest_journal(self):
@ -116,7 +122,9 @@ class TestAnsibleModuleLogSyslog(unittest.TestCase):
def setUp(self):
self.complex_args_token = basic.MODULE_COMPLEX_ARGS
self.constants_sentinel = basic.MODULE_CONSTANTS
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
self.am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -127,6 +135,7 @@ class TestAnsibleModuleLogSyslog(unittest.TestCase):
def tearDown(self):
basic.MODULE_COMPLEX_ARGS = self.complex_args_token
basic.MODULE_CONSTANTS = self.constants_sentinel
basic.has_journal = self.has_journal
@patch('syslog.syslog', autospec=True)
@ -168,7 +177,9 @@ class TestAnsibleModuleLogJournal(unittest.TestCase):
def setUp(self):
self.complex_args_token = basic.MODULE_COMPLEX_ARGS
self.constants_sentinel = basic.MODULE_CONSTANTS
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
self.am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -188,6 +199,7 @@ class TestAnsibleModuleLogJournal(unittest.TestCase):
def tearDown(self):
basic.MODULE_COMPLEX_ARGS = self.complex_args_token
basic.MODULE_CONSTANTS = self.constants_sentinel
basic.has_journal = self.has_journal
if self.module_patcher:
self.module_patcher.stop()

1
test/units/module_utils/basic/test_run_command.py
View file

@ -62,6 +62,7 @@ class TestAnsibleModuleRunCommand(unittest.TestCase):
raise OSError(errno.EPERM, "Permission denied: '/inaccessible'")
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
self.module = AnsibleModule(argument_spec=dict())
self.module.fail_json = MagicMock(side_effect=SystemExit)

1
test/units/module_utils/basic/test_safe_eval.py
View file

@ -29,6 +29,7 @@ class TestAnsibleModuleExitJson(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec=dict(),
)

22
test/units/module_utils/test_basic.py
View file

@ -267,6 +267,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec=dict(),
)
@ -282,6 +283,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
# should test ok
basic.MODULE_COMPLEX_ARGS = '{"foo":"hello"}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = arg_spec,
mutually_exclusive = mut_ex,
@ -296,6 +298,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
# fail, because a required param was not specified
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
self.assertRaises(
SystemExit,
basic.AnsibleModule,
@ -310,6 +313,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
# fail because of mutually exclusive parameters
basic.MODULE_COMPLEX_ARGS = '{"foo":"hello", "bar": "bad", "bam": "bad"}'
basic.MODULE_CONSTANTS = '{}'
self.assertRaises(
SystemExit,
basic.AnsibleModule,
@ -324,6 +328,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
# fail because a param required due to another param was not specified
basic.MODULE_COMPLEX_ARGS = '{"bam":"bad"}'
basic.MODULE_CONSTANTS = '{}'
self.assertRaises(
SystemExit,
basic.AnsibleModule,
@ -340,6 +345,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -389,6 +395,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -409,6 +416,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -423,6 +431,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -455,6 +464,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -491,6 +501,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -533,10 +544,11 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.SELINUX_SPECIAL_FS = 'nfs,nfsd,foos'
basic.MODULE_CONSTANTS = '{"SELINUX_SPECIAL_FS": "nfs,nfsd,foos"}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
print(am.constants)
def _mock_find_mount_point(path):
if path.startswith('/some/path'):
@ -574,6 +586,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -585,6 +598,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -600,6 +614,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -624,6 +639,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -669,6 +685,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -747,6 +764,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -835,6 +853,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)
@ -1013,6 +1032,7 @@ class TestModuleUtilsBasic(unittest.TestCase):
from ansible.module_utils import basic
basic.MODULE_COMPLEX_ARGS = '{}'
basic.MODULE_CONSTANTS = '{}'
am = basic.AnsibleModule(
argument_spec = dict(),
)

3
test/units/plugins/action/test_action.py
View file

@ -52,7 +52,6 @@ python_module_replacers = b"""
#!/usr/bin/python
#ANSIBLE_VERSION = "<<ANSIBLE_VERSION>>"
#MODULE_ARGS = "<<INCLUDE_ANSIBLE_MODULE_ARGS>>"
#MODULE_COMPLEX_ARGS = "<<INCLUDE_ANSIBLE_MODULE_COMPLEX_ARGS>>"
#SELINUX_SPECIAL_FS="<<SELINUX_SPECIAL_FILESYSTEMS>>"
@ -61,7 +60,7 @@ from ansible.module_utils.basic import *
"""
powershell_module_replacers = b"""
WINDOWS_ARGS = "<<INCLUDE_ANSIBLE_MODULE_WINDOWS_ARGS>>"
WINDOWS_ARGS = "<<INCLUDE_ANSIBLE_MODULE_JSON_ARGS>>"
# POWERSHELL_COMMON
"""