From 2ffe3c42bb15263ad67dd36842712d9d3ec99070 Mon Sep 17 00:00:00 2001 From: Brian Coca Date: Thu, 19 Oct 2017 14:23:22 -0400 Subject: [PATCH] fixed caches (#30667) --- lib/ansible/inventory/data.py | 4 ---- lib/ansible/inventory/manager.py | 9 ++++++++- lib/ansible/plugins/inventory/__init__.py | 9 ++++----- lib/ansible/plugins/inventory/constructed.py | 9 ++++++--- lib/ansible/plugins/inventory/openstack.py | 6 +++--- lib/ansible/plugins/inventory/script.py | 6 +++--- lib/ansible/plugins/inventory/virtualbox.py | 6 +++--- 7 files changed, 27 insertions(+), 22 deletions(-) diff --git a/lib/ansible/inventory/data.py b/lib/ansible/inventory/data.py index e828c1a8d2..9c292feff9 100644 --- a/lib/ansible/inventory/data.py +++ b/lib/ansible/inventory/data.py @@ -26,7 +26,6 @@ from ansible.errors import AnsibleError from ansible.inventory.group import Group from ansible.inventory.host import Host from ansible.module_utils.six import iteritems -from ansible.plugins.cache import FactCache from ansible.utils.vars import combine_vars from ansible.utils.path import basedir @@ -62,9 +61,6 @@ class InventoryData(object): self.add_group(group) self.add_child('all', 'ungrouped') - # prime cache - self.cache = FactCache() - def serialize(self): data = dict() return data diff --git a/lib/ansible/inventory/manager.py b/lib/ansible/inventory/manager.py index 6eab40ae91..c1aaebda18 100644 --- a/lib/ansible/inventory/manager.py +++ b/lib/ansible/inventory/manager.py @@ -24,6 +24,8 @@ import os import re import itertools +from copy import deepcopy + from ansible import constants as C from ansible.errors import AnsibleError, AnsibleOptionsError, AnsibleParserError from ansible.inventory.data import InventoryData @@ -256,7 +258,12 @@ class InventoryManager(object): # initialize if plugin.verify_file(source): try: - plugin.parse(self._inventory, self._loader, source, cache=cache) + # in case plugin fails 1/2 way we dont want partial inventory + inventory = deepcopy(self._inventory) + plugin.parse(inventory, self._loader, source, cache=cache) + + # plugin worked! so lets use the more complete inventory + self._inventory = inventory parsed = True display.vvv('Parsed %s inventory source with %s plugin' % (to_native(source), plugin_name)) break diff --git a/lib/ansible/plugins/inventory/__init__.py b/lib/ansible/plugins/inventory/__init__.py index 0c38c222ef..84513d8950 100644 --- a/lib/ansible/plugins/inventory/__init__.py +++ b/lib/ansible/plugins/inventory/__init__.py @@ -45,11 +45,11 @@ class BaseInventoryPlugin(object): TYPE = 'generator' - def __init__(self, cache=None): + def __init__(self): self.inventory = None self.display = display - self.cache = cache + self._cache = {} def parse(self, inventory, loader, path, cache=True): ''' Populates self.groups from the given data. Raises an error on any parse failure. ''' @@ -160,10 +160,9 @@ class BaseFileInventoryPlugin(BaseInventoryPlugin): TYPE = 'storage' - def __init__(self, cache=None): + def __init__(self): - # file based inventories are always local so no need for cache - super(BaseFileInventoryPlugin, self).__init__(cache=None) + super(BaseFileInventoryPlugin, self).__init__() # Helper methods diff --git a/lib/ansible/plugins/inventory/constructed.py b/lib/ansible/plugins/inventory/constructed.py index 65eef611c9..ce7a53db44 100644 --- a/lib/ansible/plugins/inventory/constructed.py +++ b/lib/ansible/plugins/inventory/constructed.py @@ -58,6 +58,7 @@ from collections import MutableMapping from ansible import constants as C from ansible.errors import AnsibleParserError +from ansible.plugins.cache import FactCache from ansible.plugins.inventory import BaseInventoryPlugin from ansible.module_utils._text import to_native from ansible.utils.vars import combine_vars @@ -72,6 +73,8 @@ class InventoryModule(BaseInventoryPlugin): super(InventoryModule, self).__init__() + self._cache = FactCache() + def verify_file(self, path): valid = False @@ -86,7 +89,7 @@ class InventoryModule(BaseInventoryPlugin): def parse(self, inventory, loader, path, cache=False): ''' parses the inventory file ''' - super(InventoryModule, self).parse(inventory, loader, path, cache=True) + super(InventoryModule, self).parse(inventory, loader, path, cache=cache) try: data = self.loader.load_from_file(path) @@ -107,8 +110,8 @@ class InventoryModule(BaseInventoryPlugin): # get available variables to templar hostvars = inventory.hosts[host].get_vars() - if host in inventory.cache: # adds facts if cache is active - hostvars = combine_vars(hostvars, inventory.cache[host]) + if host in self._cache: # adds facts if cache is active + hostvars = combine_vars(hostvars, self._cache[host]) # create composite vars self._set_composite_vars(data.get('compose'), hostvars, host, strict=strict) diff --git a/lib/ansible/plugins/inventory/openstack.py b/lib/ansible/plugins/inventory/openstack.py index fae78185bc..933cfef061 100644 --- a/lib/ansible/plugins/inventory/openstack.py +++ b/lib/ansible/plugins/inventory/openstack.py @@ -151,9 +151,9 @@ class InventoryModule(BaseInventoryPlugin): self._config_data = {} source_data = None - if cache and cache_key in inventory.cache: + if cache and cache_key in self._cache: try: - source_data = inventory.cache[cache_key] + source_data = self._cache[cache_key] except KeyError: pass @@ -189,7 +189,7 @@ class InventoryModule(BaseInventoryPlugin): source_data = cloud_inventory.list_hosts( expand=expand_hostvars, fail_on_cloud_config=fail_on_errors) - inventory.cache[cache_key] = source_data + self._cache[cache_key] = source_data self._populate_from_source(source_data) diff --git a/lib/ansible/plugins/inventory/script.py b/lib/ansible/plugins/inventory/script.py index e6767c9f78..131ff2b70e 100644 --- a/lib/ansible/plugins/inventory/script.py +++ b/lib/ansible/plugins/inventory/script.py @@ -72,7 +72,7 @@ class InventoryModule(BaseInventoryPlugin): try: cache_key = self.get_cache_prefix(path) - if not cache or cache_key not in inventory.cache: + if not cache or cache_key not in self._cache: try: sp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except OSError as e: @@ -93,11 +93,11 @@ class InventoryModule(BaseInventoryPlugin): raise AnsibleError("Inventory {0} contained characters that cannot be interpreted as UTF-8: {1}".format(path, to_native(e))) try: - inventory.cache[cache_key] = self.loader.load(data, file_name=path) + self._cache[cache_key] = self.loader.load(data, file_name=path) except Exception as e: raise AnsibleError("failed to parse executable inventory script results from {0}: {1}\n{2}".format(path, to_native(e), err)) - processed = inventory.cache[cache_key] + processed = self._cache[cache_key] if not isinstance(processed, Mapping): raise AnsibleError("failed to parse executable inventory script results from {0}: needs to be a json dict\n{1}".format(path, err)) diff --git a/lib/ansible/plugins/inventory/virtualbox.py b/lib/ansible/plugins/inventory/virtualbox.py index 2b033c0e76..e995bc3835 100644 --- a/lib/ansible/plugins/inventory/virtualbox.py +++ b/lib/ansible/plugins/inventory/virtualbox.py @@ -176,9 +176,9 @@ class InventoryModule(BaseInventoryPlugin): raise AnsibleParserError("Incorrect plugin name in file: %s" % config_data.get('plugin', 'none found')) source_data = None - if cache and cache_key in inventory.cache: + if cache and cache_key in self._cache: try: - source_data = inventory.cache[cache_key] + source_data = self._cache[cache_key] except KeyError: pass @@ -203,6 +203,6 @@ class InventoryModule(BaseInventoryPlugin): AnsibleParserError(to_native(e)) source_data = p.stdout.read() - inventory.cache[cache_key] = to_text(source_data, errors='surrogate_or_strict') + self._cache[cache_key] = to_text(source_data, errors='surrogate_or_strict') self._populate_from_source(source_data.splitlines(), config_data)