GCP Inventory Plugin (#36884)
* Inventory * Adding multi project support * Adding multi project support * Fixing PR comments and cleaning code * Adding cache support * Changed filter notation * Inventory changes * Better readability for zones, networks, subnetworks * Added project to inventory output * Using IP instead of hostname * Keyed_groups support * Use all zones when none provided. * PR changes * Doc changes * Accepts *gcp_compute.yaml file names * Added support for changing host naming precedent * PR changes round 2 * Cache changes * Changed verify_file function * Misc style changes * Cache fixes * Fix docs for `hostnames` option.
This commit is contained in:
Alex Stephen
Ryan Brown
parent
cba64f5869
commit
dc31809d2a
2 changed files with 355 additions and 4 deletions
|
|
@ -70,9 +70,10 @@ class GcpSession(object):
|
|||
self.product = product
|
||||
self._validate()
|
||||
|
||||
def get(self, url, body=None):
|
||||
def get(self, url, body=None, **kwargs):
|
||||
kwargs.update({'json': body, 'headers': self._headers()})
|
||||
try:
|
||||
return self.session().get(url, json=body, headers=self._headers())
|
||||
return self.session().get(url, **kwargs)
|
||||
except getattr(requests.exceptions, 'RequestException') as inst:
|
||||
self.module.fail_json(msg=inst.message)
|
||||
|
||||
|
|
@ -105,12 +106,12 @@ class GcpSession(object):
|
|||
if not HAS_GOOGLE_LIBRARIES:
|
||||
self.module.fail_json(msg="Please install the google-auth library")
|
||||
|
||||
if self.module.params['service_account_email'] is not None and self.module.params['auth_kind'] != 'machineaccount':
|
||||
if self.module.params.get('service_account_email') is not None and self.module.params['auth_kind'] != 'machineaccount':
|
||||
self.module.fail_json(
|
||||
msg="Service Acccount Email only works with Machine Account-based authentication"
|
||||
)
|
||||
|
||||
if self.module.params['service_account_file'] is not None and self.module.params['auth_kind'] != 'serviceaccount':
|
||||
if self.module.params.get('service_account_file') is not None and self.module.params['auth_kind'] != 'serviceaccount':
|
||||
self.module.fail_json(
|
||||
msg="Service Acccount File only works with Service Account-based authentication"
|
||||
)
|
||||
|
|
|
|||
350
lib/ansible/plugins/inventory/gcp_compute.py
Normal file
350
lib/ansible/plugins/inventory/gcp_compute.py
Normal file
|
|
@ -0,0 +1,350 @@
|
|||
# Copyright (c) 2017 Ansible Project
|
||||
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
|
||||
|
||||
from __future__ import (absolute_import, division, print_function)
|
||||
__metaclass__ = type
|
||||
|
||||
DOCUMENTATION = '''
|
||||
name: gcp_compute
|
||||
plugin_type: inventory
|
||||
short_description: Google Cloud Compute Engine inventory source
|
||||
requirements:
|
||||
- requests >= 2.18.4
|
||||
- google-auth >= 1.3.0
|
||||
extends_documentation_fragment:
|
||||
- gcp
|
||||
- constructed
|
||||
- inventory_cache
|
||||
description:
|
||||
- Get inventory hosts from Google Cloud Platform GCE.
|
||||
- Uses a <name>.gcp.yaml (or <name>.gcp.yml) YAML configuration file.
|
||||
options:
|
||||
zones:
|
||||
description: A list of regions in which to describe GCE instances.
|
||||
default: all zones available to a given project
|
||||
projects:
|
||||
description: A list of projects in which to describe GCE instances.
|
||||
filters:
|
||||
description: >
|
||||
A list of filter value pairs. Available filters are listed here
|
||||
U(https://cloud.google.com/compute/docs/reference/rest/v1/instances/list).
|
||||
Each additional filter in the list will act be added as an AND condition
|
||||
(filter1 and filter2)
|
||||
hostnames:
|
||||
description: A list of options that describe the ordering for which
|
||||
hostnames should be assigned. Currently supported hostnames are
|
||||
'public_ip', 'private_ip', or 'name'.
|
||||
default: ['public_ip', 'private_ip', 'name']
|
||||
'''
|
||||
|
||||
EXAMPLES = '''
|
||||
plugin: gcp_compute
|
||||
zones: # populate inventory with instances in these regions
|
||||
- us-east1-a
|
||||
projects:
|
||||
- gcp-prod-gke-100
|
||||
- gcp-cicd-101
|
||||
filters:
|
||||
- machineType = n1-standard-1
|
||||
- scheduling.automaticRestart = true AND machineType = n1-standard-1
|
||||
|
||||
scopes:
|
||||
- https://www.googleapis.com/auth/compute
|
||||
service_account_file: /tmp/service_account.json
|
||||
auth_kind: serviceaccount
|
||||
'''
|
||||
|
||||
from ansible.errors import AnsibleError, AnsibleParserError
|
||||
from ansible.module_utils._text import to_native, to_text
|
||||
from ansible.module_utils.six import string_types
|
||||
from ansible.module_utils.gcp_utils import GcpSession, navigate_hash, GcpRequestException
|
||||
from ansible.plugins.inventory import BaseInventoryPlugin, Constructable, Cacheable, to_safe_group_name
|
||||
import json
|
||||
|
||||
|
||||
# The mappings give an array of keys to get from the filter name to the value
|
||||
# returned by boto3's GCE describe_instances method.
|
||||
class GcpMockModule(object):
|
||||
def __init__(self, params):
|
||||
self.params = params
|
||||
|
||||
def fail_json(self, *args, **kwargs):
|
||||
raise AnsibleError(kwargs['msg'])
|
||||
|
||||
|
||||
class InventoryModule(BaseInventoryPlugin, Constructable, Cacheable):
|
||||
|
||||
NAME = 'gcp_compute'
|
||||
|
||||
def __init__(self):
|
||||
super(InventoryModule, self).__init__()
|
||||
|
||||
self.group_prefix = 'gcp_'
|
||||
|
||||
def _populate_host(self, item):
|
||||
'''
|
||||
:param item: A GCP instance
|
||||
'''
|
||||
hostname = self._get_hostname(item)
|
||||
self.inventory.add_host(hostname)
|
||||
for key in item:
|
||||
self.inventory.set_variable(hostname, key, item[key])
|
||||
self.inventory.add_child('all', hostname)
|
||||
|
||||
def verify_file(self, path):
|
||||
'''
|
||||
:param path: the path to the inventory config file
|
||||
:return the contents of the config file
|
||||
'''
|
||||
if super(InventoryModule, self).verify_file(path):
|
||||
if path.endswith('.gcp.yml') or path.endswith('.gcp.yaml'):
|
||||
return True
|
||||
elif path.endswith('.gcp_compute.yml') or path.endswith('.gcp_compute.yaml'):
|
||||
return True
|
||||
return False
|
||||
|
||||
def self_link(self, params):
|
||||
'''
|
||||
:param params: a dict containing all of the fields relevant to build URL
|
||||
:return the formatted URL as a string.
|
||||
'''
|
||||
return "https://www.googleapis.com/compute/v1/projects/{project}/zones/{zone}/instances".format(**params)
|
||||
|
||||
def fetch_list(self, params, link, query):
|
||||
'''
|
||||
:param params: a dict containing all of the fields relevant to build URL
|
||||
:param link: a formatted URL
|
||||
:param query: a formatted query string
|
||||
:return the JSON response containing a list of instances.
|
||||
'''
|
||||
module = GcpMockModule(params)
|
||||
auth = GcpSession(module, 'compute')
|
||||
response = auth.get(link, params={'filter': query})
|
||||
return self._return_if_object(module, response)
|
||||
|
||||
def _get_zones(self, config_data):
|
||||
'''
|
||||
:param config_data: dict of info from inventory file
|
||||
:return an array of zones that this project has access to
|
||||
'''
|
||||
link = "https://www.googleapis.com/compute/v1/projects/{project}/zones".format(**config_data)
|
||||
zones = []
|
||||
zones_response = self.fetch_list(config_data, link, '')
|
||||
for item in zones_response['items']:
|
||||
zones.append(item['name'])
|
||||
return zones
|
||||
|
||||
def _get_query_options(self, filters):
|
||||
'''
|
||||
:param config_data: contents of the inventory config file
|
||||
:return A fully built query string
|
||||
'''
|
||||
if not filters:
|
||||
return ''
|
||||
|
||||
if len(filters) == 1:
|
||||
return filters[0]
|
||||
else:
|
||||
queries = []
|
||||
for f in filters:
|
||||
# For multiple queries, all queries should have ()
|
||||
if f[0] != '(' and f[-1] != ')':
|
||||
queries.append("(%s)" % ''.join(f))
|
||||
else:
|
||||
queries.append(f)
|
||||
|
||||
return ' '.join(queries)
|
||||
|
||||
def _return_if_object(self, module, response):
|
||||
'''
|
||||
:param module: A GcpModule
|
||||
:param response: A Requests response object
|
||||
:return JSON response
|
||||
'''
|
||||
# If not found, return nothing.
|
||||
if response.status_code == 404:
|
||||
return None
|
||||
|
||||
# If no content, return nothing.
|
||||
if response.status_code == 204:
|
||||
return None
|
||||
|
||||
try:
|
||||
response.raise_for_status
|
||||
result = response.json()
|
||||
except getattr(json.decoder, 'JSONDecodeError', ValueError) as inst:
|
||||
module.fail_json(msg="Invalid JSON response with error: %s" % inst)
|
||||
except GcpRequestException as inst:
|
||||
module.fail_json(msg="Network error: %s" % inst)
|
||||
|
||||
if navigate_hash(result, ['error', 'errors']):
|
||||
module.fail_json(msg=navigate_hash(result, ['error', 'errors']))
|
||||
if result['kind'] != 'compute#instanceList' and result['kind'] != 'compute#zoneList':
|
||||
module.fail_json(msg="Incorrect result: {kind}".format(**result))
|
||||
|
||||
return result
|
||||
|
||||
def _format_items(self, items):
|
||||
'''
|
||||
:param items: A list of hosts
|
||||
'''
|
||||
for host in items:
|
||||
if 'zone' in host:
|
||||
host['zone_selflink'] = host['zone']
|
||||
host['zone'] = host['zone'].split('/')[-1]
|
||||
if 'machineType' in host:
|
||||
host['machineType_selflink'] = host['machineType']
|
||||
host['machineType'] = host['machineType'].split('/')[-1]
|
||||
|
||||
if 'networkInterfaces' in host:
|
||||
for network in host['networkInterfaces']:
|
||||
if 'network' in network:
|
||||
network['network'] = self._format_network_info(network['network'])
|
||||
if 'subnetwork' in network:
|
||||
network['subnetwork'] = self._format_network_info(network['subnetwork'])
|
||||
|
||||
host['project'] = host['selfLink'].split('/')[6]
|
||||
return items
|
||||
|
||||
def _add_hosts(self, items, config_data, format_items=True):
|
||||
'''
|
||||
:param items: A list of hosts
|
||||
:param config_data: configuration data
|
||||
:param format_items: format items or not
|
||||
'''
|
||||
if not items:
|
||||
return
|
||||
if format_items:
|
||||
items = self._format_items(items)
|
||||
|
||||
for host in items:
|
||||
self._populate_host(host)
|
||||
|
||||
hostname = self._get_hostname(host)
|
||||
self._set_composite_vars(self.get_option('compose'), host, hostname)
|
||||
self._add_host_to_composed_groups(self.get_option('groups'), host, hostname)
|
||||
self._add_host_to_keyed_groups(self.get_option('keyed_groups'), host, hostname)
|
||||
|
||||
def _format_network_info(self, address):
|
||||
'''
|
||||
:param address: A GCP network address
|
||||
:return a dict with network shortname and region
|
||||
'''
|
||||
split = address.split('/')
|
||||
region = ''
|
||||
if 'global' in split:
|
||||
region = 'global'
|
||||
else:
|
||||
region = split[8]
|
||||
return {
|
||||
'region': region,
|
||||
'name': split[-1],
|
||||
'selfLink': address
|
||||
}
|
||||
|
||||
def _get_hostname(self, item):
|
||||
'''
|
||||
:param item: A host response from GCP
|
||||
:return the hostname of this instance
|
||||
'''
|
||||
hostname_ordering = ['public_ip', 'private_ip', 'name']
|
||||
if self.get_option('hostnames'):
|
||||
hostname_ordering = self.get_option('hostnames')
|
||||
|
||||
for order in hostname_ordering:
|
||||
name = None
|
||||
if order == 'public_ip':
|
||||
name = self._get_publicip(item)
|
||||
elif order == 'private_ip':
|
||||
name = self._get_privateip(item)
|
||||
elif order == 'name':
|
||||
name = item[u'name']
|
||||
else:
|
||||
raise AnsibleParserError("%s is not a valid hostname precedent" % order)
|
||||
|
||||
if name:
|
||||
return name
|
||||
|
||||
raise AnsibleParserError("No valid name found for host")
|
||||
|
||||
def _get_publicip(self, item):
|
||||
'''
|
||||
:param item: A host response from GCP
|
||||
:return the publicIP of this instance or None
|
||||
'''
|
||||
# Get public IP if exists
|
||||
for interface in item['networkInterfaces']:
|
||||
if 'accessConfigs' in interface:
|
||||
for accessConfig in interface['accessConfigs']:
|
||||
if 'natIP' in accessConfig:
|
||||
return accessConfig[u'natIP']
|
||||
return None
|
||||
|
||||
def _get_privateip(self, item):
|
||||
'''
|
||||
:param item: A host response from GCP
|
||||
:return the privateIP of this instance or None
|
||||
'''
|
||||
# Fallback: Get private IP
|
||||
for interface in item[u'networkInterfaces']:
|
||||
if 'networkIP' in interface:
|
||||
return interface[u'networkIP']
|
||||
|
||||
def parse(self, inventory, loader, path, cache=True):
|
||||
super(InventoryModule, self).parse(inventory, loader, path)
|
||||
|
||||
config_data = {}
|
||||
if self.verify_file(path):
|
||||
config_data = self._read_config_data(path)
|
||||
|
||||
# get user specifications
|
||||
if 'zones' in config_data:
|
||||
if not isinstance(config_data['zones'], list):
|
||||
raise AnsibleParserError("Zones must be a list in GCP inventory YAML files")
|
||||
|
||||
# get user specifications
|
||||
if 'projects' not in config_data:
|
||||
raise AnsibleParserError("Projects must be included in inventory YAML file")
|
||||
|
||||
if not isinstance(config_data['projects'], list):
|
||||
raise AnsibleParserError("Projects must be a list in GCP inventory YAML files")
|
||||
|
||||
projects = config_data['projects']
|
||||
zones = config_data.get('zones')
|
||||
config_data['scopes'] = ['https://www.googleapis.com/auth/compute']
|
||||
|
||||
query = self._get_query_options(config_data['filters'])
|
||||
|
||||
# Cache logic
|
||||
if cache:
|
||||
cache = self.get_option('cache')
|
||||
cache_key = self.get_cache_key(path)
|
||||
else:
|
||||
cache_key = None
|
||||
|
||||
cache_needs_update = False
|
||||
if cache:
|
||||
try:
|
||||
results = self.cache.get(cache_key)
|
||||
for project in results:
|
||||
for zone in results[project]:
|
||||
self._add_hosts(results[project][zone], config_data, False)
|
||||
except KeyError:
|
||||
cache_needs_update = True
|
||||
|
||||
if not cache or cache_needs_update:
|
||||
cached_data = {}
|
||||
for project in projects:
|
||||
cached_data[project] = {}
|
||||
config_data['project'] = project
|
||||
if not zones:
|
||||
zones = self._get_zones(config_data)
|
||||
for zone in zones:
|
||||
config_data['zone'] = zone
|
||||
link = self.self_link(config_data)
|
||||
resp = self.fetch_list(config_data, link, query)
|
||||
self._add_hosts(resp.get('items'), config_data)
|
||||
cached_data[project][zone] = resp.get('items')
|
||||
|
||||
if cache_needs_update:
|
||||
self.cache.set(cache_key, cached_data)
|
||||
Loading…
Reference in a new issue