vultr: use exponential backoff for api query retries (#60529)

2019-08-19 16:09:09 +02:00 · 2019-08-19 16:09:09 +02:00 · 4b743a2721
commit 4b743a2721
parent f9f882a981
30 changed files with 195 additions and 3 deletions
--- a/changelogs/fragments/60529-vultry_retry_backoff.yml
+++ b/changelogs/fragments/60529-vultry_retry_backoff.yml
@ -0,0 +1,2 @@
+minor_changes:
+  - vultr - the retry on failure functionality was changed to use an exponential backoff behaviour.
--- a/lib/ansible/module_utils/vultr.py
+++ b/lib/ansible/module_utils/vultr.py
@ -7,6 +7,7 @@ __metaclass__ = type

 import os
 import time
+import random
 import urllib
 from ansible.module_utils.six.moves import configparser
 from ansible.module_utils._text import to_text, to_native
@ -22,6 +23,7 @@ def vultr_argument_spec():
        api_key=dict(type='str', default=os.environ.get('VULTR_API_KEY'), no_log=True),
        api_timeout=dict(type='int', default=os.environ.get('VULTR_API_TIMEOUT')),
        api_retries=dict(type='int', default=os.environ.get('VULTR_API_RETRIES')),
+        api_retry_max_delay=dict(type='int', default=os.environ.get('VULTR_API_RETRY_MAX_DELAY')),
        api_account=dict(type='str', default=os.environ.get('VULTR_API_ACCOUNT') or 'default'),
        api_endpoint=dict(type='str', default=os.environ.get('VULTR_API_ENDPOINT')),
        validate_certs=dict(type='bool', default=True),
@ -59,6 +61,7 @@ class Vultr:
                'api_key': self.module.params.get('api_key') or config.get('key'),
                'api_timeout': self.module.params.get('api_timeout') or int(config.get('timeout') or 60),
                'api_retries': self.module.params.get('api_retries') or int(config.get('retries') or 5),
+                'api_retry_max_delay': self.module.params.get('api_retries') or int(config.get('retry_max_delay') or 12),
                'api_endpoint': self.module.params.get('api_endpoint') or config.get('endpoint') or VULTR_API_ENDPOINT,
            }
        except ValueError as e:
@ -74,6 +77,7 @@ class Vultr:
            'api_account': self.module.params.get('api_account'),
            'api_timeout': self.api_config['api_timeout'],
            'api_retries': self.api_config['api_retries'],
+            'api_retry_max_delay': self.api_config['api_retry_max_delay'],
            'api_endpoint': self.api_config['api_endpoint'],
        }

@ -85,7 +89,7 @@ class Vultr:
        }

    def read_env_variables(self):
-        keys = ['key', 'timeout', 'retries', 'endpoint']
+        keys = ['key', 'timeout', 'retries', 'retry_max_delay', 'endpoint']
        env_conf = {}
        for key in keys:
            if 'VULTR_API_%s' % key.upper() not in os.environ:
@ -159,7 +163,10 @@ class Vultr:
            except AttributeError:
                data = urllib.parse.urlencode(data_encoded) + data_list

-        for s in range(0, self.api_config['api_retries']):
+        retry_max_delay = self.api_config['api_retry_max_delay']
+        randomness = random.randint(0, 1000) / 1000.0
+
+        for retry in range(0, self.api_config['api_retries']):
            response, info = fetch_url(
                module=self.module,
                url=url,
@ -173,7 +180,11 @@ class Vultr:
                break

            # Vultr has a rate limiting requests per second, try to be polite
-            time.sleep(1)
+            # Use exponential backoff plus a little bit of randomness
+            delay = 2 ** retry + randomness
+            if delay > retry_max_delay:
+                delay = retry_max_delay + randomness
+            time.sleep(delay)

        else:
            self.fail_json(msg="Reached API retries limit %s for URL %s, method %s with data %s. Returned %s, with body: %s %s" % (
--- a/lib/ansible/modules/cloud/vultr/_vultr_account_facts.py
+++ b/lib/ansible/modules/cloud/vultr/_vultr_account_facts.py
@ -58,6 +58,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/_vultr_block_storage_facts.py
+++ b/lib/ansible/modules/cloud/vultr/_vultr_block_storage_facts.py
@ -57,6 +57,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/_vultr_os_facts.py
+++ b/lib/ansible/modules/cloud/vultr/_vultr_os_facts.py
@ -58,6 +58,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/_vultr_ssh_key_facts.py
+++ b/lib/ansible/modules/cloud/vultr/_vultr_ssh_key_facts.py
@ -58,6 +58,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/_vultr_user_facts.py
+++ b/lib/ansible/modules/cloud/vultr/_vultr_user_facts.py
@ -57,6 +57,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_account_info.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_account_info.py
@ -54,6 +54,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_block_storage.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_block_storage.py
@ -78,6 +78,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_block_storage_info.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_block_storage_info.py
@ -57,6 +57,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_dns_domain.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_dns_domain.py
@ -74,6 +74,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_dns_domain_facts.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_dns_domain_facts.py
@ -53,6 +53,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_dns_record.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_dns_record.py
@ -139,6 +139,22 @@ vultr_api:
      returned: success
      type: int
      sample: 60
+    api_retries:
+      description: Amount of max retries for the API requests
+      returned: success
+      type: int
+      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
+    api_endpoint:
+      description: Endpoint used for the API requests
+      returned: success
+      type: str
+      sample: "https://api.vultr.com"
 vultr_dns_record:
  description: Response from Vultr API
  returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_firewall_group.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_firewall_group.py
@ -68,6 +68,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_firewall_group_facts.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_firewall_group_facts.py
@ -53,6 +53,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_firewall_rule.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_firewall_rule.py
@ -116,6 +116,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_network.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_network.py
@ -76,6 +76,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_network_facts.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_network_facts.py
@ -53,6 +53,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_os_info.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_os_info.py
@ -57,6 +57,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_plan_facts.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_plan_facts.py
@ -54,6 +54,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_region_facts.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_region_facts.py
@ -54,6 +54,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_server.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_server.py
@ -184,6 +184,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_server_facts.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_server_facts.py
@ -54,6 +54,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_ssh_key.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_ssh_key.py
@ -73,6 +73,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_ssh_key_info.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_ssh_key_info.py
@ -58,6 +58,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_startup_script.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_startup_script.py
@ -86,6 +86,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_startup_script_facts.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_startup_script_facts.py
@ -54,6 +54,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_user.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_user.py
@ -113,6 +113,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/modules/cloud/vultr/vultr_user_info.py
+++ b/lib/ansible/modules/cloud/vultr/vultr_user_info.py
@ -57,6 +57,12 @@ vultr_api:
      returned: success
      type: int
      sample: 5
+    api_retry_max_delay:
+      description: Exponential backoff delay in seconds between retries up to this max delay value.
+      returned: success
+      type: int
+      sample: 12
+      version_added: '2.9'
    api_endpoint:
      description: Endpoint used for the API requests
      returned: success
--- a/lib/ansible/plugins/doc_fragments/vultr.py
+++ b/lib/ansible/plugins/doc_fragments/vultr.py
@ -26,6 +26,13 @@ options:
      - The ENV variable C(VULTR_API_RETRIES) is used as default, when defined.
      - Fallback value is 5 retries if not specified.
    type: int
+  api_retry_max_delay:
+    description:
+      - Retry backoff delay in seconds is exponential up to this max. value, in seconds.
+      - The ENV variable C(VULTR_API_RETRY_MAX_DELAY) is used as default, when defined.
+      - Fallback value is 12 seconds.
+    type: int
+    version_added: '2.9'
  api_account:
    description:
      - Name of the ini section in the C(vultr.ini) file.