From 01436cf186e7a85295934f9e54cee582135a4dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Lobato=20Garc=C3=ADa?= Date: Tue, 21 Feb 2017 18:21:22 +0100 Subject: [PATCH] Add Foreman inventory (#19510) This commit adds the foreman inventory based on https://github.com/theforeman/foreman_ansible_inventory and its configuration file. --- contrib/inventory/foreman.ini | 122 +++++++++++ contrib/inventory/foreman.py | 379 ++++++++++++++++++++++++++++++++++ 2 files changed, 501 insertions(+) create mode 100644 contrib/inventory/foreman.ini create mode 100755 contrib/inventory/foreman.py diff --git a/contrib/inventory/foreman.ini b/contrib/inventory/foreman.ini new file mode 100644 index 0000000000..36d611d37b --- /dev/null +++ b/contrib/inventory/foreman.ini @@ -0,0 +1,122 @@ +# Foreman inventory (https://github.com/theforeman/foreman_ansible_inventory) +# +# This script can be used as an Ansible dynamic inventory. +# The connection parameters are set up via *foreman.ini* +# This is how the script founds the configuration file in +# order of discovery. +# +# * `/etc/ansible/foreman.ini` +# * Current directory of your inventory script. +# * `FOREMAN_INI_PATH` environment variable. +# +# ## Variables and Parameters +# +# The data returned from Foreman for each host is stored in a foreman +# hash so they're available as *host_vars* along with the parameters +# of the host and it's hostgroups: +# +# "foo.example.com": { +# "foreman": { +# "architecture_id": 1, +# "architecture_name": "x86_64", +# "build": false, +# "build_status": 0, +# "build_status_label": "Installed", +# "capabilities": [ +# "build", +# "image" +# ], +# "compute_profile_id": 4, +# "hostgroup_name": "webtier/myapp", +# "id": 70, +# "image_name": "debian8.1", +# ... +# "uuid": "50197c10-5ebb-b5cf-b384-a1e203e19e77" +# }, +# "foreman_params": { +# "testparam1": "foobar", +# "testparam2": "small", +# ... +# } +# +# and could therefore be used in Ansible like: +# +# - debug: msg="From Foreman host {{ foreman['uuid'] }}" +# +# Which yields +# +# TASK [test_foreman : debug] **************************************************** +# ok: [foo.example.com] => { +# "msg": "From Foreman host 50190bd1-052a-a34a-3c9c-df37a39550bf" +# } +# +# ## Automatic Ansible groups +# +# The inventory will provide a set of groups, by default prefixed by +# 'foreman_'. If you want to customize this prefix, change the +# group_prefix option in /etc/ansible/foreman.ini. The rest of this +# guide will assume the default prefix of 'foreman' +# +# The hostgroup, location, organization, content view, and lifecycle +# environment of each host are created as Ansible groups with a +# foreman_ prefix, all lowercase and problematic parameters +# removed. So e.g. the foreman hostgroup +# +# myapp / webtier / datacenter1 +# +# would turn into the Ansible group: +# +# foreman_hostgroup_myapp_webtier_datacenter1 +# +# Furthermore Ansible groups can be created on the fly using the +# *group_patterns* variable in *foreman.ini* so that you can build up +# hierarchies using parameters on the hostgroup and host variables. +# +# Lets assume you have a host that is built using this nested hostgroup: +# +# myapp / webtier / datacenter1 +# +# and each of the hostgroups defines a parameters respectively: +# +# myapp: app_param = myapp +# webtier: tier_param = webtier +# datacenter1: dc_param = datacenter1 +# +# The host is also in a subnet called "mysubnet" and provisioned via an image +# then *group_patterns* like: +# +# [ansible] +# group_patterns = ["{app_param}-{tier_param}-{dc_param}", +# "{app_param}-{tier_param}", +# "{app_param}", +# "{subnet_name}-{provision_method}"] +# +# would put the host into the additional Ansible groups: +# +# - myapp-webtier-datacenter1 +# - myapp-webtier +# - myapp +# - mysubnet-image +# +# by recursively resolving the hostgroups, getting the parameter keys +# and values and doing a Python *string.format()* like replacement on +# it. +# +[foreman] +url = http://localhost:3000/ +user = foreman +password = secret +ssl_verify = True + +[ansible] +group_patterns = ["{app}-{tier}-{color}", + "{app}-{color}", + "{app}", + "{tier}"] +group_prefix = foreman_ +# Whether to fetch facts from Foreman and store them on the host +want_facts = True + +[cache] +path = . +max_age = 60 diff --git a/contrib/inventory/foreman.py b/contrib/inventory/foreman.py new file mode 100755 index 0000000000..9a87397db4 --- /dev/null +++ b/contrib/inventory/foreman.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python +# vim: set fileencoding=utf-8 : +# +# Copyright (C) 2016 Guido Günther , +# Daniel Lobato Garcia +# +# This script is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Ansible is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with it. If not, see . +# +# This is somewhat based on cobbler inventory + +# Stdlib imports +# __future__ imports must occur at the beginning of file +from __future__ import print_function +try: + # Python 2 version + import ConfigParser +except ImportError: + # Python 3 version + import configparser as ConfigParser +import json +import argparse +import copy +import os +import re +import sys +from time import time +from collections import defaultdict +from distutils.version import LooseVersion, StrictVersion + +# 3rd party imports +import requests +if LooseVersion(requests.__version__) < LooseVersion('1.1.0'): + print('This script requires python-requests 1.1 as a minimum version') + sys.exit(1) + +from requests.auth import HTTPBasicAuth + +def json_format_dict(data, pretty=False): + """Converts a dict to a JSON object and dumps it as a formatted string""" + + if pretty: + return json.dumps(data, sort_keys=True, indent=2) + else: + return json.dumps(data) + +class ForemanInventory(object): + + def __init__(self): + self.inventory = defaultdict(list) # A list of groups and the hosts in that group + self.cache = dict() # Details about hosts in the inventory + self.params = dict() # Params of each host + self.facts = dict() # Facts of each host + self.hostgroups = dict() # host groups + self.session = None # Requests session + self.config_paths = [ + "/etc/ansible/foreman.ini", + os.path.dirname(os.path.realpath(__file__)) + '/foreman.ini', + ] + env_value = os.environ.get('FOREMAN_INI_PATH') + if env_value is not None: + self.config_paths.append(os.path.expanduser(os.path.expandvars(env_value))) + + def read_settings(self): + """Reads the settings from the foreman.ini file""" + + config = ConfigParser.SafeConfigParser() + config.read(self.config_paths) + + # Foreman API related + try: + self.foreman_url = config.get('foreman', 'url') + self.foreman_user = config.get('foreman', 'user') + self.foreman_pw = config.get('foreman', 'password') + self.foreman_ssl_verify = config.getboolean('foreman', 'ssl_verify') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError) as e: + print("Error parsing configuration: %s" % e, file=sys.stderr) + return False + + # Ansible related + try: + group_patterns = config.get('ansible', 'group_patterns') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + group_patterns = "[]" + + self.group_patterns = json.loads(group_patterns) + + try: + self.group_prefix = config.get('ansible', 'group_prefix') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.group_prefix = "foreman_" + + try: + self.want_facts = config.getboolean('ansible', 'want_facts') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.want_facts = True + + # Cache related + try: + cache_path = os.path.expanduser(config.get('cache', 'path')) + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + cache_path = '.' + (script, ext) = os.path.splitext(os.path.basename(__file__)) + self.cache_path_cache = cache_path + "/%s.cache" % script + self.cache_path_inventory = cache_path + "/%s.index" % script + self.cache_path_params = cache_path + "/%s.params" % script + self.cache_path_facts = cache_path + "/%s.facts" % script + try: + self.cache_max_age = config.getint('cache', 'max_age') + except (ConfigParser.NoOptionError, ConfigParser.NoSectionError): + self.cache_max_age = 60 + return True + + def parse_cli_args(self): + """Command line argument processing""" + + parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on foreman') + parser.add_argument('--list', action='store_true', default=True, help='List instances (default: True)') + parser.add_argument('--host', action='store', help='Get all the variables about a specific instance') + parser.add_argument('--refresh-cache', action='store_true', default=False, + help='Force refresh of cache by making API requests to foreman (default: False - use cache files)') + self.args = parser.parse_args() + + def _get_session(self): + if not self.session: + self.session = requests.session() + self.session.auth = HTTPBasicAuth(self.foreman_user, self.foreman_pw) + self.session.verify = self.foreman_ssl_verify + return self.session + + def _get_json(self, url, ignore_errors=None): + page = 1 + results = [] + s = self._get_session() + while True: + ret = s.get(url, params={'page': page, 'per_page': 250}) + if ignore_errors and ret.status_code in ignore_errors: + break + ret.raise_for_status() + json = ret.json() + # /hosts/:id has not results key + if 'results' not in json: + return json + # Facts are returned as dict in results not list + if isinstance(json['results'], dict): + return json['results'] + # List of all hosts is returned paginaged + results = results + json['results'] + if len(results) >= json['total']: + break + page += 1 + if len(json['results']) == 0: + print("Did not make any progress during loop. " + "expected %d got %d" % (json['total'], len(results)), + file=sys.stderr) + break + return results + + def _get_hosts(self): + return self._get_json("%s/api/v2/hosts" % self.foreman_url) + + def _get_all_params_by_id(self, hid): + url = "%s/api/v2/hosts/%s" % (self.foreman_url, hid) + ret = self._get_json(url, [404]) + if ret == []: + ret = {} + return ret.get('all_parameters', {}) + + def _resolve_params(self, host): + """Fetch host params and convert to dict""" + params = {} + + for param in self._get_all_params_by_id(host['id']): + name = param['name'] + params[name] = param['value'] + + return params + + def _get_facts_by_id(self, hid): + url = "%s/api/v2/hosts/%s/facts" % (self.foreman_url, hid) + return self._get_json(url) + + def _get_facts(self, host): + """Fetch all host facts of the host""" + if not self.want_facts: + return {} + + ret = self._get_facts_by_id(host['id']) + if len(ret.values()) == 0: + facts = {} + elif len(ret.values()) == 1: + facts = list(ret.values())[0] + else: + raise ValueError("More than one set of facts returned for '%s'" % host) + return facts + + def write_to_cache(self, data, filename): + """Write data in JSON format to a file""" + json_data = json_format_dict(data, True) + cache = open(filename, 'w') + cache.write(json_data) + cache.close() + + def _write_cache(self): + self.write_to_cache(self.cache, self.cache_path_cache) + self.write_to_cache(self.inventory, self.cache_path_inventory) + self.write_to_cache(self.params, self.cache_path_params) + self.write_to_cache(self.facts, self.cache_path_facts) + + def to_safe(self, word): + '''Converts 'bad' characters in a string to underscores + so they can be used as Ansible groups + + >>> ForemanInventory.to_safe("foo-bar baz") + 'foo_barbaz' + ''' + regex = "[^A-Za-z0-9\_]" + return re.sub(regex, "_", word.replace(" ", "")) + + def update_cache(self): + """Make calls to foreman and save the output in a cache""" + + self.groups = dict() + self.hosts = dict() + + for host in self._get_hosts(): + dns_name = host['name'] + + # Create ansible groups for hostgroup + group = 'hostgroup' + val = host.get('%s_title' % group) or host.get('%s_name' % group) + if val: + safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower())) + self.inventory[safe_key].append(dns_name) + + # Create ansible groups for environment, location and organization + for group in ['environment', 'location', 'organization']: + val = host.get('%s_name' % group) + if val: + safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower())) + self.inventory[safe_key].append(dns_name) + + for group in ['lifecycle_environment', 'content_view']: + val = host.get('content_facet_attributes', {}).get('%s_name' % group) + if val: + safe_key = self.to_safe('%s%s_%s' % (self.group_prefix, group, val.lower())) + self.inventory[safe_key].append(dns_name) + + params = self._resolve_params(host) + + # Ansible groups by parameters in host groups and Foreman host + # attributes. + groupby = copy.copy(params) + for k, v in host.items(): + if isinstance(v, str): + groupby[k] = self.to_safe(v) + elif isinstance(v, int): + groupby[k] = v + + # The name of the ansible groups is given by group_patterns: + for pattern in self.group_patterns: + try: + key = pattern.format(**groupby) + self.inventory[key].append(dns_name) + except KeyError: + pass # Host not part of this group + + self.cache[dns_name] = host + self.params[dns_name] = params + self.facts[dns_name] = self._get_facts(host) + self.inventory['all'].append(dns_name) + self._write_cache() + + def is_cache_valid(self): + """Determines if the cache is still valid""" + if os.path.isfile(self.cache_path_cache): + mod_time = os.path.getmtime(self.cache_path_cache) + current_time = time() + if (mod_time + self.cache_max_age) > current_time: + if (os.path.isfile(self.cache_path_inventory) and + os.path.isfile(self.cache_path_params) and + os.path.isfile(self.cache_path_facts)): + return True + return False + + def load_inventory_from_cache(self): + """Read the index from the cache file sets self.index""" + + cache = open(self.cache_path_inventory, 'r') + json_inventory = cache.read() + self.inventory = json.loads(json_inventory) + + def load_params_from_cache(self): + """Read the index from the cache file sets self.index""" + + cache = open(self.cache_path_params, 'r') + json_params = cache.read() + self.params = json.loads(json_params) + + def load_facts_from_cache(self): + """Read the index from the cache file sets self.facts""" + if not self.want_facts: + return + cache = open(self.cache_path_facts, 'r') + json_facts = cache.read() + self.facts = json.loads(json_facts) + + def load_cache_from_cache(self): + """Read the cache from the cache file sets self.cache""" + + cache = open(self.cache_path_cache, 'r') + json_cache = cache.read() + self.cache = json.loads(json_cache) + + def get_inventory(self): + if self.args.refresh_cache or not self.is_cache_valid(): + self.update_cache() + else: + self.load_inventory_from_cache() + self.load_params_from_cache() + self.load_facts_from_cache() + self.load_cache_from_cache() + + def get_host_info(self): + """Get variables about a specific host""" + + if not self.cache or len(self.cache) == 0: + # Need to load index from cache + self.load_cache_from_cache() + + if self.args.host not in self.cache: + # try updating the cache + self.update_cache() + + if self.args.host not in self.cache: + # host might not exist anymore + return json_format_dict({}, True) + + return json_format_dict(self.cache[self.args.host], True) + + def _print_data(self): + data_to_print = "" + if self.args.host: + data_to_print += self.get_host_info() + else: + self.inventory['_meta'] = {'hostvars': {}} + for hostname in self.cache: + self.inventory['_meta']['hostvars'][hostname] = { + 'foreman': self.cache[hostname], + 'foreman_params': self.params[hostname], + } + if self.want_facts: + self.inventory['_meta']['hostvars'][hostname]['foreman_facts'] = self.facts[hostname] + + data_to_print += json_format_dict(self.inventory, True) + + print(data_to_print) + + def run(self): + # Read settings and parse CLI arguments + if not self.read_settings(): + return False + self.parse_cli_args() + self.get_inventory() + self._print_data() + return True + +if __name__ == '__main__': + sys.exit(not ForemanInventory().run())