wbrawner/ansible
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'svg/varsplugins_refactor' into svg_and_inventory_refactor

Browse source
This commit is contained in:
James Cammarata 2014-07-07 14:16:37 -05:00
commit fe892fccb1
7 changed files with 299 additions and 231 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/ansible
View file

@ -136,7 +136,7 @@ class Cli(object):
if not options.ask_vault_pass:
vault_pass = tmp_vault_pass
inventory_manager = inventory.Inventory(options.inventory)
inventory_manager = inventory.Inventory(options.inventory, vault_password=vault_pass)
if options.subset:
inventory_manager.subset(options.subset)
hosts = inventory_manager.list_hosts(pattern)

13
bin/ansible-playbook
View file

@ -100,11 +100,6 @@ def main(args):
if (options.ask_vault_pass and options.vault_password_file):
parser.error("--ask-vault-pass and --vault-password-file are mutually exclusive")
inventory = ansible.inventory.Inventory(options.inventory)
inventory.subset(options.subset)
if len(inventory.list_hosts()) == 0:
raise errors.AnsibleError("provided hosts list is empty")
sshpass = None
sudopass = None
su_pass = None
@ -160,12 +155,14 @@ def main(args):
if not (os.path.isfile(playbook) or stat.S_ISFIFO(os.stat(playbook).st_mode)):
raise errors.AnsibleError("the playbook: %s does not appear to be a file" % playbook)
inventory = ansible.inventory.Inventory(options.inventory, vault_password=vault_pass)
inventory.subset(options.subset)
if len(inventory.list_hosts()) == 0:
raise errors.AnsibleError("provided hosts list is empty")
# run all playbooks specified on the command line
for playbook in args:
# let inventory know which playbooks are using so it can know the basedirs
inventory.set_playbook_basedir(os.path.dirname(playbook))
stats = callbacks.AggregateStats()
playbook_cb = callbacks.PlaybookCallbacks(verbose=utils.VERBOSITY)
if options.step:

135
lib/ansible/inventory/__init__.py
View file

@ -38,13 +38,14 @@ class Inventory(object):
__slots__ = [ 'host_list', 'groups', '_restriction', '_also_restriction', '_subset',
'parser', '_vars_per_host', '_vars_per_group', '_hosts_cache', '_groups_list',
'_pattern_cache', '_vars_plugins', '_playbook_basedir']
'_pattern_cache', '_vault_password', '_vars_plugins', '_playbook_basedir']
def __init__(self, host_list=C.DEFAULT_HOST_LIST):
def __init__(self, host_list=C.DEFAULT_HOST_LIST, vault_password=None):
# the host file file, or script path, or list of hosts
# if a list, inventory data will NOT be loaded
self.host_list = host_list
self._vault_password=vault_password
# caching to avoid repeated calculations, particularly with
# external inventory scripts.
@ -55,7 +56,7 @@ class Inventory(object):
self._groups_list = {}
self._pattern_cache = {}
# to be set by calling set_playbook_basedir by ansible-playbook
# to be set by calling set_playbook_basedir by playbook code
self._playbook_basedir = None
# the inventory object holds a list of groups
@ -139,6 +140,14 @@ class Inventory(object):
self._vars_plugins = [ x for x in utils.plugins.vars_loader.all(self) ]
# get group vars from group_vars/ files and vars plugins
for group in self.groups:
group.vars = utils.combine_vars(group.vars, self.get_group_variables(group.name, self._vault_password))
# get host vars from host_vars/ files and vars plugins
for host in self.get_hosts():
host.vars = utils.combine_vars(host.vars, self.get_variables(host.name, self._vault_password))
def _match(self, str, pattern_str):
if pattern_str.startswith('~'):
@ -392,19 +401,35 @@ class Inventory(object):
return group
return None
def get_group_variables(self, groupname):
if groupname not in self._vars_per_group:
self._vars_per_group[groupname] = self._get_group_variables(groupname)
def get_group_variables(self, groupname, update_cached=False, vault_password=None):
if groupname not in self._vars_per_group or update_cached:
self._vars_per_group[groupname] = self._get_group_variables(groupname, vault_password=vault_password)
return self._vars_per_group[groupname]
def _get_group_variables(self, groupname):
def _get_group_variables(self, groupname, vault_password=None):
group = self.get_group(groupname)
if group is None:
raise Exception("group not found: %s" % groupname)
return group.get_variables()
def get_variables(self, hostname, vault_password=None):
if hostname not in self._vars_per_host:
vars = {}
# plugin.get_group_vars retrieves just vars for specific group
vars_results = [ plugin.get_group_vars(group, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_group_vars')]
for updated in vars_results:
if updated is not None:
vars = utils.combine_vars(vars, updated)
# get group variables set by Inventory Parsers
vars = utils.combine_vars(vars, group.get_variables())
# Read group_vars/ files
vars = utils.combine_vars(vars, self.get_group_vars(group))
return vars
def get_variables(self, hostname, update_cached=False, vault_password=None):
if hostname not in self._vars_per_host or update_cached:
self._vars_per_host[hostname] = self._get_variables(hostname, vault_password=vault_password)
return self._vars_per_host[hostname]
@ -415,14 +440,31 @@ class Inventory(object):
raise errors.AnsibleError("host not found: %s" % hostname)
vars = {}
vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins ]
# plugin.run retrieves all vars (also from groups) for host
vars_results = [ plugin.run(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'run')]
for updated in vars_results:
if updated is not None:
vars = utils.combine_vars(vars, updated)
# plugin.get_host_vars retrieves just vars for specific host
vars_results = [ plugin.get_host_vars(host, vault_password=vault_password) for plugin in self._vars_plugins if hasattr(plugin, 'get_host_vars')]
for updated in vars_results:
if updated is not None:
vars = utils.combine_vars(vars, updated)
# get host variables set by Inventory Parsers
vars = utils.combine_vars(vars, host.get_variables())
# still need to check InventoryParser per host vars
# which actually means InventoryScript per host,
# which is not performant
if self.parser is not None:
vars = utils.combine_vars(vars, self.parser.get_host_variables(host))
# Read host_vars/ files
vars = utils.combine_vars(vars, self.get_host_vars(host))
return vars
def add_group(self, group):
@ -525,10 +567,73 @@ class Inventory(object):
return self._playbook_basedir
def set_playbook_basedir(self, dir):
"""
sets the base directory of the playbook so inventory plugins can use it to find
variable files and other things.
"""
self._playbook_basedir = dir
sets the base directory of the playbook so inventory can use it as a
basedir for host_ and group_vars, and other things.
"""
# Only update things if dir is a different playbook basedir
if dir != self._playbook_basedir:
self._playbook_basedir = dir
# get group vars from group_vars/ files
for group in self.groups:
group.vars = utils.combine_vars(group.vars, self.get_group_vars(group, new_pb_basedir=True))
# get host vars from host_vars/ files
for host in self.get_hosts():
host.vars = utils.combine_vars(host.vars, self.get_host_vars(host, new_pb_basedir=True))
def get_host_vars(self, host, new_pb_basedir=False):
""" Read host_vars/ files """
return self._get_hostgroup_vars(host=host, group=None, new_pb_basedir=False)
def get_group_vars(self, group, new_pb_basedir=False):
""" Read group_vars/ files """
return self._get_hostgroup_vars(host=None, group=group, new_pb_basedir=False)
def _get_hostgroup_vars(self, host=None, group=None, new_pb_basedir=False):
"""
Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel
to the inventory base directory or in the same directory as the playbook. Variables in the playbook
dir will win over the inventory dir if files are in both.
"""
results = {}
scan_pass = 0
_basedir = self.basedir()
# look in both the inventory base directory and the playbook base directory
# unless we do an update for a new playbook base dir
if not new_pb_basedir:
basedirs = [_basedir, self._playbook_basedir]
else:
basedirs = [self._playbook_basedir]
for basedir in basedirs:
# this can happen from particular API usages, particularly if not run
# from /usr/bin/ansible-playbook
if basedir is None:
continue
scan_pass = scan_pass + 1
# it's not an eror if the directory does not exist, keep moving
if not os.path.exists(basedir):
continue
# save work of second scan if the directories are the same
if _basedir == self._playbook_basedir and scan_pass != 1:
continue
if group and host is None:
# load vars in dir/group_vars/name_of_group
base_path = os.path.join(basedir, "group_vars/%s" % group.name)
results = utils.load_vars(base_path, results, vault_password=self._vault_password)
elif host and group is None:
# same for hostvars in dir/host_vars/name_of_host
base_path = os.path.join(basedir, "host_vars/%s" % host.name)
results = utils.load_vars(base_path, results, vault_password=self._vault_password)
# all done, results is a dictionary of variables for this particular host.
return results

195
lib/ansible/inventory/vars_plugins/group_vars.py
View file

@ -1,195 +0,0 @@
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
import os
import stat
import errno
from ansible import errors
from ansible import utils
import ansible.constants as C
def _load_vars(basepath, results, vault_password=None):
"""
Load variables from any potential yaml filename combinations of basepath,
returning result.
"""
paths_to_check = [ "".join([basepath, ext])
for ext in C.YAML_FILENAME_EXTENSIONS ]
found_paths = []
for path in paths_to_check:
found, results = _load_vars_from_path(path, results, vault_password=vault_password)
if found:
found_paths.append(path)
# disallow the potentially confusing situation that there are multiple
# variable files for the same name. For example if both group_vars/all.yml
# and group_vars/all.yaml
if len(found_paths) > 1:
raise errors.AnsibleError("Multiple variable files found. "
"There should only be one. %s" % ( found_paths, ))
return results
def _load_vars_from_path(path, results, vault_password=None):
"""
Robustly access the file at path and load variables, carefully reporting
errors in a friendly/informative way.
Return the tuple (found, new_results, )
"""
try:
# in the case of a symbolic link, we want the stat of the link itself,
# not its target
pathstat = os.lstat(path)
except os.error, err:
# most common case is that nothing exists at that path.
if err.errno == errno.ENOENT:
return False, results
# otherwise this is a condition we should report to the user
raise errors.AnsibleError(
"%s is not accessible: %s."
" Please check its permissions." % ( path, err.strerror))
# symbolic link
if stat.S_ISLNK(pathstat.st_mode):
try:
target = os.path.realpath(path)
except os.error, err2:
raise errors.AnsibleError("The symbolic link at %s "
"is not readable: %s. Please check its permissions."
% (path, err2.strerror, ))
# follow symbolic link chains by recursing, so we repeat the same
# permissions checks above and provide useful errors.
return _load_vars_from_path(target, results)
# directory
if stat.S_ISDIR(pathstat.st_mode):
# support organizing variables across multiple files in a directory
return True, _load_vars_from_folder(path, results, vault_password=vault_password)
# regular file
elif stat.S_ISREG(pathstat.st_mode):
data = utils.parse_yaml_from_file(path, vault_password=vault_password)
if data and type(data) != dict:
raise errors.AnsibleError("%s must be stored as a dictionary/hash" % path)
elif data is None:
data = {}
# combine vars overrides by default but can be configured to do a
# hash merge in settings
results = utils.combine_vars(results, data)
return True, results
# something else? could be a fifo, socket, device, etc.
else:
raise errors.AnsibleError("Expected a variable file or directory "
"but found a non-file object at path %s" % (path, ))
def _load_vars_from_folder(folder_path, results, vault_password=None):
"""
Load all variables within a folder recursively.
"""
# this function and _load_vars_from_path are mutually recursive
try:
names = os.listdir(folder_path)
except os.error, err:
raise errors.AnsibleError(
"This folder cannot be listed: %s: %s."
% ( folder_path, err.strerror))
# evaluate files in a stable order rather than whatever order the
# filesystem lists them.
names.sort()
# do not parse hidden files or dirs, e.g. .svn/
paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')]
for path in paths:
_found, results = _load_vars_from_path(path, results, vault_password=vault_password)
return results
class VarsModule(object):
"""
Loads variables from group_vars/<groupname> and host_vars/<hostname> in directories parallel
to the inventory base directory or in the same directory as the playbook. Variables in the playbook
dir will win over the inventory dir if files are in both.
"""
def __init__(self, inventory):
""" constructor """
self.inventory = inventory
def run(self, host, vault_password=None):
""" main body of the plugin, does actual loading """
inventory = self.inventory
basedir = inventory.playbook_basedir()
if basedir is not None:
basedir = os.path.abspath(basedir)
self.pb_basedir = basedir
# sort groups by depth so deepest groups can override the less deep ones
groupz = sorted(inventory.groups_for_host(host.name), key=lambda g: g.depth)
groups = [ g.name for g in groupz ]
inventory_basedir = inventory.basedir()
results = {}
scan_pass = 0
# look in both the inventory base directory and the playbook base directory
for basedir in [ inventory_basedir, self.pb_basedir ]:
# this can happen from particular API usages, particularly if not run
# from /usr/bin/ansible-playbook
if basedir is None:
continue
scan_pass = scan_pass + 1
# it's not an eror if the directory does not exist, keep moving
if not os.path.exists(basedir):
continue
# save work of second scan if the directories are the same
if inventory_basedir == self.pb_basedir and scan_pass != 1:
continue
# load vars in dir/group_vars/name_of_group
for group in groups:
base_path = os.path.join(basedir, "group_vars/%s" % group)
results = _load_vars(base_path, results, vault_password=vault_password)
# same for hostvars in dir/host_vars/name_of_host
base_path = os.path.join(basedir, "host_vars/%s" % host.name)
results = _load_vars(base_path, results, vault_password=vault_password)
# all done, results is a dictionary of variables for this particular host.
return results

48
lib/ansible/inventory/vars_plugins/noop.py Normal file
View file

@ -0,0 +1,48 @@
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2014, Serge van Ginderachter <serge@vanginderachter.be>
#
# This file is part of Ansible
#
# Ansible is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Ansible is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
class VarsModule(object):
"""
Loads variables for groups and/or hosts
"""
def __init__(self, inventory):
""" constructor """
self.inventory = inventory
self.inventory_basedir = inventory.basedir()
def run(self, host, vault_password=None):
""" For backwards compatibility, when only vars per host were retrieved
This method should return both host specific vars as well as vars
calculated from groups it is a member of """
return {}
def get_host_vars(self, host, vault_password=None):
""" Get host specific variables. """
return {}
def get_group_vars(self, group, vault_password=None):
""" Get group specific variables. """
return {}

4
lib/ansible/playbook/__init__.py
View file

@ -164,6 +164,10 @@ class PlayBook(object):
self.basedir = os.path.dirname(playbook) or '.'
utils.plugins.push_basedir(self.basedir)
# let inventory know the playbook basedir so it can load more vars
self.inventory.set_playbook_basedir(self.basedir)
vars = extra_vars.copy()
vars['playbook_dir'] = self.basedir
if self.inventory.basedir() is not None:

133
lib/ansible/utils/__init__.py
View file

@ -1,4 +1,4 @@
# (c) 2012, Michael DeHaan <michael.dehaan@gmail.com>
# (c) 2012-2014, Michael DeHaan <michael.dehaan@gmail.com>
#
# This file is part of Ansible
#
@ -15,6 +15,7 @@
# You should have received a copy of the GNU General Public License
# along with Ansible. If not, see <http://www.gnu.org/licenses/>.
import errno
import sys
import re
import os
@ -620,18 +621,19 @@ def merge_hash(a, b):
''' recursively merges hash b into a
keys from b take precedence over keys from a '''
result = copy.deepcopy(a)
result = {}
# next, iterate over b keys and values
for k, v in b.iteritems():
# if there's already such key in a
# and that key contains dict
if k in result and isinstance(result[k], dict):
# merge those dicts recursively
result[k] = merge_hash(a[k], v)
else:
# otherwise, just copy a value from b to a
result[k] = v
for dicts in a, b:
# next, iterate over b keys and values
for k, v in dicts.iteritems():
# if there's already such key in a
# and that key contains dict
if k in result and isinstance(result[k], dict):
# merge those dicts recursively
result[k] = merge_hash(a[k], v)
else:
# otherwise, just copy a value from b to a
result[k] = v
return result
@ -1208,5 +1210,112 @@ def before_comment(msg):
msg = msg.replace("**NOT_A_COMMENT**","#")
return msg
def load_vars(basepath, results, vault_password=None):
"""
Load variables from any potential yaml filename combinations of basepath,
returning result.
"""
paths_to_check = [ "".join([basepath, ext])
for ext in C.YAML_FILENAME_EXTENSIONS ]
found_paths = []
for path in paths_to_check:
found, results = _load_vars_from_path(path, results, vault_password=vault_password)
if found:
found_paths.append(path)
# disallow the potentially confusing situation that there are multiple
# variable files for the same name. For example if both group_vars/all.yml
# and group_vars/all.yaml
if len(found_paths) > 1:
raise errors.AnsibleError("Multiple variable files found. "
"There should only be one. %s" % ( found_paths, ))
return results
## load variables from yaml files/dirs
# e.g. host/group_vars
#
def _load_vars_from_path(path, results, vault_password=None):
"""
Robustly access the file at path and load variables, carefully reporting
errors in a friendly/informative way.
Return the tuple (found, new_results, )
"""
try:
# in the case of a symbolic link, we want the stat of the link itself,
# not its target
pathstat = os.lstat(path)
except os.error, err:
# most common case is that nothing exists at that path.
if err.errno == errno.ENOENT:
return False, results
# otherwise this is a condition we should report to the user
raise errors.AnsibleError(
"%s is not accessible: %s."
" Please check its permissions." % ( path, err.strerror))
# symbolic link
if stat.S_ISLNK(pathstat.st_mode):
try:
target = os.path.realpath(path)
except os.error, err2:
raise errors.AnsibleError("The symbolic link at %s "
"is not readable: %s. Please check its permissions."
% (path, err2.strerror, ))
# follow symbolic link chains by recursing, so we repeat the same
# permissions checks above and provide useful errors.
return _load_vars_from_path(target, results)
# directory
if stat.S_ISDIR(pathstat.st_mode):
# support organizing variables across multiple files in a directory
return True, _load_vars_from_folder(path, results, vault_password=vault_password)
# regular file
elif stat.S_ISREG(pathstat.st_mode):
data = parse_yaml_from_file(path, vault_password=vault_password)
if type(data) != dict:
raise errors.AnsibleError(
"%s must be stored as a dictionary/hash" % path)
# combine vars overrides by default but can be configured to do a
# hash merge in settings
results = combine_vars(results, data)
return True, results
# something else? could be a fifo, socket, device, etc.
else:
raise errors.AnsibleError("Expected a variable file or directory "
"but found a non-file object at path %s" % (path, ))
def _load_vars_from_folder(folder_path, results, vault_password=None):
"""
Load all variables within a folder recursively.
"""
# this function and _load_vars_from_path are mutually recursive
try:
names = os.listdir(folder_path)
except os.error, err:
raise errors.AnsibleError(
"This folder cannot be listed: %s: %s."
% ( folder_path, err.strerror))
# evaluate files in a stable order rather than whatever order the
# filesystem lists them.
names.sort()
# do not parse hidden files or dirs, e.g. .svn/
paths = [os.path.join(folder_path, name) for name in names if not name.startswith('.')]
for path in paths:
_found, results = _load_vars_from_path(path, results, vault_password=vault_password)
return results