From 7da565b3aea86ef7b1ac304a423ca20b2c0e6c12 Mon Sep 17 00:00:00 2001 From: Sloane Hertel Date: Mon, 18 Mar 2019 08:29:03 -0500 Subject: [PATCH] parse botocore.endpoint logs into a list of AWS actions (#49312) * Add an option to parse botocore.endpoint logs for the AWS actions performed during a task Add a callback to consolidate all AWS actions used by modules Added some documentation to the AWS guidelines * Enable aws_resource_actions callback only for AWS tests * Add script to help generate policies * Set debug_botocore_endpoint_logs via environment variable for all AWS integration tests Ensure AWS tests inherit environment (also remove AWS CLI in aws_rds inventory tests and use the module) --- .../aws_config/build_iam_policy_framework.py | 327 ++++++++++++++++++ lib/ansible/module_utils/aws/core.py | 34 +- lib/ansible/module_utils/ec2.py | 3 +- .../modules/cloud/amazon/GUIDELINES.md | 68 +++- .../plugins/callback/aws_resource_actions.py | 72 ++++ lib/ansible/plugins/doc_fragments/aws.py | 8 + .../targets/aws_eks/playbooks/full_test.yml | 1 + .../targets/aws_eks/playbooks/old_version.yml | 1 + .../playbooks/full_test.yml | 1 + .../ec2_instance/playbooks/full_test.yml | 1 + .../ec2_instance/playbooks/version_fail.yml | 1 + .../playbooks/full_test.yml | 1 + .../playbooks/version_fail.yml | 1 + .../ecs_cluster/playbooks/full_test.yml | 1 + .../network_assign_public_ip_fail.yml | 1 + .../ecs_cluster/playbooks/network_fail.yml | 1 + .../network_force_new_deployment.yml | 1 + .../network_force_new_deployment_fail.yml | 1 + .../targets/efs/playbooks/full_test.yml | 1 + .../targets/efs/playbooks/version_fail.yml | 1 + .../elb_target/playbooks/full_test.yml | 1 + .../elb_target/playbooks/version_fail.yml | 1 + .../elb_target_facts/playbooks/full_test.yml | 1 + .../playbooks/populate_cache.yml | 1 + .../playbooks/test_populating_inventory.yml | 1 + ..._populating_inventory_with_constructed.yml | 1 + .../playbooks/populate_cache.yml | 81 ++--- .../playbooks/test_populating_inventory.yml | 102 ++---- ..._populating_inventory_with_constructed.yml | 82 ++--- .../playbooks/test_refresh_inventory.yml | 98 ++---- .../inventory_aws_rds/templates/inventory.yml | 2 + .../templates/inventory_with_cache.yml | 2 + .../templates/inventory_with_constructed.yml | 2 + test/runner/lib/cloud/aws.py | 4 + 34 files changed, 672 insertions(+), 233 deletions(-) create mode 100644 hacking/aws_config/build_iam_policy_framework.py create mode 100644 lib/ansible/plugins/callback/aws_resource_actions.py diff --git a/hacking/aws_config/build_iam_policy_framework.py b/hacking/aws_config/build_iam_policy_framework.py new file mode 100644 index 0000000000..39edae75fa --- /dev/null +++ b/hacking/aws_config/build_iam_policy_framework.py @@ -0,0 +1,327 @@ +# Requires pandas, bs4, html5lib, and lxml +# +# Call script with the output from aws_resource_actions callback, e.g. +# python build_iam_policy_framework.py ['ec2:AuthorizeSecurityGroupEgress', 'ec2:AuthorizeSecurityGroupIngress', 'sts:GetCallerIdentity'] +# +# The sample output: +# { +# "Version": "2012-10-17", +# "Statement": [ +# { +# "Sid": "AnsibleEditor0", +# "Effect": "Allow", +# "Action": [ +# "ec2:AuthorizeSecurityGroupEgress", +# "ec2:AuthorizeSecurityGroupIngress" +# ], +# "Resource": "arn:aws:ec2:${Region}:${Account}:security-group/${SecurityGroupId}" +# }, +# { +# "Sid": "AnsibleEditor1", +# "Effect": "Allow", +# "Action": [ +# "sts:GetCallerIdentity" +# ], +# "Resource": "*" +# } +# ] +# } +# +# Policy troubleshooting: +# - If there are more actions in the policy than you provided, AWS has documented dependencies for some of your actions and +# those have been added to the policy. +# - If there are fewer actions in the policy than you provided, some of your actions are not in the IAM table of actions for +# that service. For example, the API call s3:DeleteObjects does not actually correlate to the permission needed in a policy. +# In this case s3:DeleteObject is the permission required to allow both the s3:DeleteObjects action and the s3:DeleteObject action. +# - The policies output are only as accurate as the AWS documentation. If the policy does not permit the +# necessary actions, look for undocumented dependencies. For example, redshift:CreateCluster requires ec2:DescribeVpcs, +# ec2:DescribeSubnets, ec2:DescribeSecurityGroups, and ec2:DescribeInternetGateways, but AWS does not document this. +# + +import json +import requests +import sys + +missing_dependencies = [] +try: + import pandas as pd +except ImportError: + missing_dependencies.append('pandas') +try: + import bs4 +except ImportError: + missing_dependencies.append('bs4') +try: + import html5lib +except ImportError: + missing_dependencies.append('html5lib') +try: + import lxml +except ImportError: + missing_dependencies.append('lxml') + + +irregular_service_names = { + 'a4b': 'alexaforbusiness', + 'appstream': 'appstream2.0', + 'acm': 'certificatemanager', + 'acm-pca': 'certificatemanagerprivatecertificateauthority', + 'aws-marketplace-management': 'marketplacemanagementportal', + 'ce': 'costexplorerservice', + 'cognito-identity': 'cognitoidentity', + 'cognito-sync': 'cognitosync', + 'cognito-idp': 'cognitouserpools', + 'cur': 'costandusagereport', + 'dax': 'dynamodbacceleratordax', + 'dlm': 'datalifecyclemanager', + 'dms': 'databasemigrationservice', + 'ds': 'directoryservice', + 'ec2messages': 'messagedeliveryservice', + 'ecr': 'ec2containerregistry', + 'ecs': 'elasticcontainerservice', + 'eks': 'elasticcontainerserviceforkubernetes', + 'efs': 'elasticfilesystem', + 'es': 'elasticsearchservice', + 'events': 'cloudwatchevents', + 'firehose': 'kinesisfirehose', + 'fms': 'firewallmanager', + 'health': 'healthapisandnotifications', + 'importexport': 'importexportdiskservice', + 'iot1click': 'iot1-click', + 'kafka': 'managedstreamingforkafka', + 'kinesisvideo': 'kinesisvideostreams', + 'kms': 'keymanagementservice', + 'license-manager': 'licensemanager', + 'logs': 'cloudwatchlogs', + 'opsworks-cm': 'opsworksconfigurationmanagement', + 'mediaconnect': 'elementalmediaconnect', + 'mediaconvert': 'elementalmediaconvert', + 'medialive': 'elementalmedialive', + 'mediapackage': 'elementalmediapackage', + 'mediastore': 'elementalmediastore', + 'mgh': 'migrationhub', + 'mobiletargeting': 'pinpoint', + 'pi': 'performanceinsights', + 'pricing': 'pricelist', + 'ram': 'resourceaccessmanager', + 'resource-groups': 'resourcegroups', + 'sdb': 'simpledb', + 'servicediscovery': 'cloudmap', + 'serverlessrepo': 'serverlessapplicationrepository', + 'sms': 'servermigrationservice', + 'sms-voice': 'pinpointsmsandvoiceservice', + 'sso-directory': 'ssodirectory', + 'ssm': 'systemsmanager', + 'ssmmessages': 'sessionmanagermessagegatewayservice', + 'states': 'stepfunctions', + 'sts': 'securitytokenservice', + 'swf': 'simpleworkflowservice', + 'tag': 'resourcegrouptaggingapi', + 'transfer': 'transferforsftp', + 'waf-regional': 'wafregional', + 'wam': 'workspacesapplicationmanager', + 'xray': 'x-ray' +} + +irregular_service_links = { + 'apigateway': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_manageamazonapigateway.html' + ], + 'aws-marketplace': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsmarketplace.html', + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsmarketplacemeteringservice.html', + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_awsprivatemarketplace.html' + ], + 'discovery': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_applicationdiscovery.html' + ], + 'elasticloadbalancing': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_elasticloadbalancing.html', + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_elasticloadbalancingv2.html' + ], + 'globalaccelerator': [ + 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_globalaccelerator.html' + ] +} + + +def get_docs_by_prefix(prefix): + amazon_link_form = 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_amazon{0}.html' + aws_link_form = 'https://docs.aws.amazon.com/IAM/latest/UserGuide/list_aws{0}.html' + + if prefix in irregular_service_links: + links = irregular_service_links[prefix] + else: + if prefix in irregular_service_names: + prefix = irregular_service_names[prefix] + links = [amazon_link_form.format(prefix), aws_link_form.format(prefix)] + + return links + + +def get_html(links): + html_list = [] + for link in links: + html = requests.get(link).content + try: + parsed_html = pd.read_html(html) + html_list.append(parsed_html) + except ValueError as e: + if 'No tables found' in str(e): + pass + else: + raise e + + return html_list + + +def get_tables(service): + links = get_docs_by_prefix(service) + html_list = get_html(links) + action_tables = [] + arn_tables = [] + for df_list in html_list: + for df in df_list: + table = json.loads(df.to_json(orient='split')) + table_data = table['data'][0] + if 'Actions' in table_data and 'Resource Types (*required)' in table_data: + action_tables.append(table['data'][1::]) + elif 'Resource Types' in table_data and 'ARN' in table_data: + arn_tables.append(table['data'][1::]) + + # Action table indices: + # 0: Action, 1: Description, 2: Access level, 3: Resource type, 4: Condition keys, 5: Dependent actions + # ARN tables indices: + # 0: Resource type, 1: ARN template, 2: Condition keys + return action_tables, arn_tables + + +def add_dependent_action(resources, dependency): + resource, action = dependency.split(':') + if resource in resources: + resources[resource].append(action) + else: + resources[resource] = [action] + return resources + + +def get_dependent_actions(resources): + for service in dict(resources): + action_tables, arn_tables = get_tables(service) + for found_action_table in action_tables: + for action_stuff in found_action_table: + if action_stuff is None: + continue + if action_stuff[0] in resources[service] and action_stuff[5]: + dependencies = action_stuff[5].split() + if isinstance(dependencies, list): + for dependency in dependencies: + resources = add_dependent_action(resources, dependency) + else: + resources = add_dependent_action(resources, dependencies) + return resources + + +def get_actions_by_service(resources): + service_action_dict = {} + dependencies = {} + for service in resources: + action_tables, arn_tables = get_tables(service) + + # Create dict of the resource type to the corresponding ARN + arn_dict = {} + for found_arn_table in arn_tables: + for arn_stuff in found_arn_table: + arn_dict["{0}*".format(arn_stuff[0])] = arn_stuff[1] + + # Create dict of the action to the corresponding ARN + action_dict = {} + for found_action_table in action_tables: + for action_stuff in found_action_table: + if action_stuff[0] is None: + continue + if arn_dict.get(action_stuff[3]): + action_dict[action_stuff[0]] = arn_dict[action_stuff[3]] + else: + action_dict[action_stuff[0]] = None + service_action_dict[service] = action_dict + return service_action_dict + + +def get_resource_arns(aws_actions, action_dict): + resource_arns = {} + for resource_action in aws_actions: + resource, action = resource_action.split(':') + if action not in action_dict: + continue + if action_dict[action] is None: + resource = "*" + else: + resource = action_dict[action].replace("${Partition}", "aws") + if resource not in resource_arns: + resource_arns[resource] = [] + resource_arns[resource].append(resource_action) + return resource_arns + + +def get_resources(actions): + resources = {} + for action in actions: + resource, action = action.split(':') + if resource not in resources: + resources[resource] = [] + resources[resource].append(action) + return resources + + +def combine_arn_actions(resources, service_action_arn_dict): + arn_actions = {} + for service in service_action_arn_dict: + service_arn_actions = get_resource_arns(aws_actions, service_action_arn_dict[service]) + for resource in service_arn_actions: + if resource in arn_actions: + arn_actions[resource].extend(service_arn_actions[resource]) + else: + arn_actions[resource] = service_arn_actions[resource] + return arn_actions + + +def combine_actions_and_dependent_actions(resources): + aws_actions = [] + for resource in resources: + for action in resources[resource]: + aws_actions.append('{0}:{1}'.format(resource, action)) + return set(aws_actions) + + +def get_actions_restricted_by_arn(aws_actions): + resources = get_resources(aws_actions) + resources = get_dependent_actions(resources) + service_action_arn_dict = get_actions_by_service(resources) + aws_actions = combine_actions_and_dependent_actions(resources) + return combine_arn_actions(aws_actions, service_action_arn_dict) + + +def main(aws_actions): + arn_actions = get_actions_restricted_by_arn(aws_actions) + statement = [] + for resource_restriction in arn_actions: + statement.append({ + "Sid": "AnsibleEditor{0}".format(len(statement)), + "Effect": "Allow", + "Action": arn_actions[resource_restriction], + "Resource": resource_restriction + }) + + policy = {"Version": "2012-10-17", "Statement": statement} + print(json.dumps(policy, indent=4)) + + +if __name__ == '__main__': + if missing_dependencies: + sys.exit('Missing Python libraries: {0}'.format(', '.join(missing_dependencies))) + actions = sys.argv[1:] + if len(actions) == 1: + actions = sys.argv[1].split(',') + aws_actions = [action.strip('[], "\'') for action in actions] + main(aws_actions) diff --git a/lib/ansible/module_utils/aws/core.py b/lib/ansible/module_utils/aws/core.py index 289c64dd0a..2138572007 100644 --- a/lib/ansible/module_utils/aws/core.py +++ b/lib/ansible/module_utils/aws/core.py @@ -60,10 +60,18 @@ don't need to be wrapped in the backoff decorator. """ +import re +import logging import traceback from functools import wraps from distutils.version import LooseVersion +try: + from cStringIO import StringIO +except ImportError: + # Python 3 + from io import StringIO + from ansible.module_utils.basic import AnsibleModule, missing_required_lib from ansible.module_utils._text import to_native from ansible.module_utils.ec2 import HAS_BOTO3, camel_dict_to_snake_dict, ec2_argument_spec, boto3_conn, get_aws_connection_info @@ -120,14 +128,38 @@ class AnsibleAWSModule(object): self._diff = self._module._diff self._name = self._module._name + self._botocore_endpoint_log_stream = StringIO() + self.logger = None + if self.params.get('debug_botocore_endpoint_logs'): + self.logger = logging.getLogger('botocore.endpoint') + self.logger.setLevel(logging.DEBUG) + self.logger.addHandler(logging.StreamHandler(self._botocore_endpoint_log_stream)) + @property def params(self): return self._module.params + def _get_resource_action_list(self): + actions = [] + for ln in self._botocore_endpoint_log_stream.getvalue().split('\n'): + ln = ln.strip() + if not ln: + continue + found_operational_request = re.search(r"OperationModel\(name=.*?\)", ln) + if found_operational_request: + operation_request = found_operational_request.group(0)[20:-1] + resource = re.search(r"https://.*?\.", ln).group(0)[8:-1] + actions.append("{0}:{1}".format(resource, operation_request)) + return list(set(actions)) + def exit_json(self, *args, **kwargs): + if self.params.get('debug_botocore_endpoint_logs'): + kwargs['resource_actions'] = self._get_resource_action_list() return self._module.exit_json(*args, **kwargs) def fail_json(self, *args, **kwargs): + if self.params.get('debug_botocore_endpoint_logs'): + kwargs['resource_actions'] = self._get_resource_action_list() return self._module.fail_json(*args, **kwargs) def debug(self, *args, **kwargs): @@ -190,7 +222,7 @@ class AnsibleAWSModule(object): if response is not None: failure.update(**camel_dict_to_snake_dict(response)) - self._module.fail_json(**failure) + self.fail_json(**failure) def _gather_versions(self): """Gather AWS SDK (boto3 and botocore) dependency versions diff --git a/lib/ansible/module_utils/ec2.py b/lib/ansible/module_utils/ec2.py index f59448cafd..fc88063b31 100644 --- a/lib/ansible/module_utils/ec2.py +++ b/lib/ansible/module_utils/ec2.py @@ -31,7 +31,7 @@ import re import traceback from ansible.module_utils.ansible_release import __version__ -from ansible.module_utils.basic import missing_required_lib +from ansible.module_utils.basic import missing_required_lib, env_fallback from ansible.module_utils._text import to_native, to_text from ansible.module_utils.cloud import CloudRetry from ansible.module_utils.six import string_types, binary_type, text_type @@ -177,6 +177,7 @@ def boto_exception(err): def aws_common_argument_spec(): return dict( + debug_botocore_endpoint_logs=dict(fallback=(env_fallback, ['ANSIBLE_DEBUG_BOTOCORE_LOGS']), default=False, type='bool'), ec2_url=dict(), aws_secret_key=dict(aliases=['ec2_secret_key', 'secret_key'], no_log=True), aws_access_key=dict(aliases=['ec2_access_key', 'access_key']), diff --git a/lib/ansible/modules/cloud/amazon/GUIDELINES.md b/lib/ansible/modules/cloud/amazon/GUIDELINES.md index 6086240279..fadfe33ac0 100644 --- a/lib/ansible/modules/cloud/amazon/GUIDELINES.md +++ b/lib/ansible/modules/cloud/amazon/GUIDELINES.md @@ -633,14 +633,66 @@ for every call, it's preferrable to use [YAML Anchors](http://blog.daemonl.com/2 As explained in the [Integration Test guide](https://docs.ansible.com/ansible/latest/dev_guide/testing_integration.html#iam-policies-for-aws) there are defined IAM policies in `hacking/aws_config/testing_policies/` that contain the necessary permissions -to run the AWS integration test. +to run the AWS integration test. The permissions used by CI are more restrictive than those in `hacking/aws_config/testing_policies`; for CI we want +the most restrictive policy possible that still allows the given tests to pass. -If your module is interacting with a new service or otherwise requires new permissions you must update the -appropriate policy file to grant the permissions needed to run your integration test. +If your module interacts with a new service or otherwise requires new permissions, tests will fail when you submit a pull request and the +[Ansibullbot](https://github.com/ansible/ansibullbot/blob/master/ISSUE_HELP.md) will tag your PR as needing revision. +We do not automatically grant additional permissions to the roles used by the continuous integration builds. You must provide the minimum IAM permissions required to run your integration test. -There is no process for automatically granting additional permissions to the roles used by the continuous -integration builds, so the tests will initially fail when you submit a pull request and the -[Ansibullbot](https://github.com/ansible/ansibullbot/blob/master/ISSUE_HELP.md) will tag it as needing revision. - -Once you're certain the failure is only due to the missing permissions, add a comment with the `ready_for_review` +If your PR has test failures, check carefully to be certain the failure is only due to the missing permissions. If you've ruled out other sources of failure, add a comment with the `ready_for_review` tag and explain that it's due to missing permissions. + +Your pull request cannot be merged until the tests are passing. If your pull request is failing due to missing permissions, +you must collect the minimum IAM permissions required to +run the tests. + +There are two ways to figure out which IAM permissions you need for your PR to pass: + +* Start with the most permissive IAM policy, run the tests to collect information about which resources your tests actually use, then construct a policy based on that output. This approach only works on modules that use `AnsibleAWSModule`. +* Start with the least permissive IAM policy, run the tests to discover a failure, add permissions for the resource that addresses that failure, then repeat. If your module uses `AnsibleModule` instead of `AnsibleAWSModule`, you must use this approach. + +To start with the most permissive IAM policy: + +1) [Create an IAM policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start) that allows all actions (set `Action` and `Resource` to `*`). +2) Run your tests locally with this policy. On `AnsibleAWSModule`-based modules, the `debug_botocore_endpoint_logs` option is automatically set to `yes`, so you +should see a list of `AWS ACTIONS` after the `PLAY RECAP` showing all the permissions used. If your tests use a `boto`/`AnsibleModule` module, you must start with the least permissive policy (see below). +3) Modify your policy to allow only the actions your tests use. Restrict account, region, and prefix where possible. Wait a few minutes for your policy to update. +4) Run the tests again with a user or role that allows only the new policy. +5) If the tests fail, troubleshoot (see tips below), modify the policy, run the tests again, and repeat the process until the tests pass with a restrictive policy. +6) Share the minimum policy in a comment on your PR. + +To start from the least permissive IAM policy: + +1) Run the integration tests locally with no IAM permissions. +2) Examine the error when the tests reach a failure. + a) If the error message indicates the action used in the request, add the action to your policy. + b) If the error message does not indicate the action used in the request: + - Usually the action is a CamelCase version of the method name - for example, for an ec2 client the method `describe_security_groups` correlates to the action `ec2:DescribeSecurityGroups`. + - Refer to the documentation to identify the action. + c) If the error message indicates the resource ARN used in the request, limit the action to that resource. + d) If the error message does not indicate the resource ARN used: + - Determine if the action can be restricted to a resource by examining the documentation. + - If the action can be restricted, use the documentation to construct the ARN and add it to the policy. +3) Add the action or resource that caused the failure to [an IAM policy](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_policies_create.html#access_policies_create-start). Wait a few minutes for your policy to update. +4) Run the tests again with this policy attached to your user or role. +5) If the tests still fail at the same place with the same error you will need to troubleshoot (see tips below). If the first test passes, repeat steps 2 and 3 for the next error. Repeat the process until the tests pass with a restrictive policy. +6) Share the minimum policy in a comment on your PR. + +Troubleshooting IAM policies: + +- When you make changes to a policy, wait a few minutes for the policy to update before re-running the tests. +- Use the [policy simulator](https://policysim.aws.amazon.com/) to verify that each action (limited by resource when applicable) in your policy is allowed. +- If you're restricting actions to certain resources, replace resources temporarily with `*`. If the tests pass with wildcard resources, there is a problem with the resource definition in your policy. +- If the initial troubleshooting above doesn't provide any more insight, AWS may be using additional undisclosed resources and actions. +- Examine the AWS FullAccess policy for the service for clues. +- Re-read the AWS documentation, especially the [list of Actions, Resources and Condition Keys](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_actions-resources-contextkeys.html) for the various AWS services. +- Look at the [cloudonaut](https://iam.cloudonaut.io) documentation as a troubleshooting cross-reference. +- Use a search engine. +- Ask in the Ansible IRC channel #ansible-aws. + + +Some cases where tests should be marked as unsupported: +1) The tests take longer than 10 or 15 minutes to complete +2) The tests create expensive resources +3) The tests create inline policies diff --git a/lib/ansible/plugins/callback/aws_resource_actions.py b/lib/ansible/plugins/callback/aws_resource_actions.py new file mode 100644 index 0000000000..f871fe5479 --- /dev/null +++ b/lib/ansible/plugins/callback/aws_resource_actions.py @@ -0,0 +1,72 @@ +# (C) 2018 Ansible Project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) + +# Make coding more python3-ish +from __future__ import (absolute_import, division, print_function) +__metaclass__ = type + +DOCUMENTATION = ''' + callback: aws_resource_actions + type: aggregate + short_description: summarizes all "resource:actions" completed + version_added: "2.8" + description: + - Ansible callback plugin for collecting the AWS actions completed by all boto3 modules using + AnsibleAWSModule in a playbook. Botocore endpoint logs need to be enabled for those modules, which can + be done easily by setting debug_botocore_endpoint_logs to True for group/aws using module_defaults. + requirements: + - whitelisting in configuration - see examples section below for details. +''' + +EXAMPLES = ''' +example: > + To enable, add this to your ansible.cfg file in the defaults block + [defaults] + callback_whitelist = aws_resource_actions +sample output: > +# +# AWS ACTIONS: ['s3:PutBucketAcl', 's3:HeadObject', 's3:DeleteObject', 's3:PutObjectAcl', 's3:CreateMultipartUpload', +# 's3:DeleteBucket', 's3:GetObject', 's3:DeleteObjects', 's3:CreateBucket', 's3:CompleteMultipartUpload', +# 's3:ListObjectsV2', 's3:HeadBucket', 's3:UploadPart', 's3:PutObject'] +# +sample output: > +# +# AWS ACTIONS: ['ec2:DescribeVpcAttribute', 'ec2:DescribeVpcClassicLink', 'ec2:ModifyVpcAttribute', 'ec2:CreateTags', +# 'sts:GetCallerIdentity', 'ec2:DescribeSecurityGroups', 'ec2:DescribeTags', 'ec2:DescribeVpcs', 'ec2:CreateVpc'] +# +''' + +from ansible.plugins.callback import CallbackBase +from ansible.module_utils._text import to_native + + +class CallbackModule(CallbackBase): + CALLBACK_VERSION = 2.8 + CALLBACK_TYPE = 'aggregate' + CALLBACK_NAME = 'aws_resource_actions' + CALLBACK_NEEDS_WHITELIST = True + + def __init__(self): + self.aws_resource_actions = [] + super(CallbackModule, self).__init__() + + def extend_aws_resource_actions(self, result): + if result.get('resource_actions'): + self.aws_resource_actions.extend(result['resource_actions']) + + def runner_on_ok(self, host, res): + self.extend_aws_resource_actions(res) + + def runner_on_failed(self, host, res, ignore_errors=False): + self.extend_aws_resource_actions(res) + + def v2_runner_item_on_ok(self, result): + self.extend_aws_resource_actions(result._result) + + def v2_runner_item_on_failed(self, result): + self.extend_aws_resource_actions(result._result) + + def playbook_on_stats(self, stats): + if self.aws_resource_actions: + self.aws_resource_actions = sorted(list(to_native(action) for action in set(self.aws_resource_actions))) + self._display.display("AWS ACTIONS: {0}".format(self.aws_resource_actions)) diff --git a/lib/ansible/plugins/doc_fragments/aws.py b/lib/ansible/plugins/doc_fragments/aws.py index e754973443..84c3a4048a 100644 --- a/lib/ansible/plugins/doc_fragments/aws.py +++ b/lib/ansible/plugins/doc_fragments/aws.py @@ -9,6 +9,14 @@ class ModuleDocFragment(object): # AWS only documentation fragment DOCUMENTATION = r''' options: + debug_botocore_endpoint_logs: + description: + - Use a botocore.endpoint logger to parse the unique (rather than total) "resource:action" API calls made during a task, outputing + the set to the resource_actions key in the task results. Use the aws_resource_action callback to output to total list made during + a playbook. The ANSIBLE_DEBUG_BOTOCORE_LOGS environment variable may also be used. + type: bool + default: 'no' + version_added: "2.8" ec2_url: description: - Url to use to connect to EC2 or your Eucalyptus cloud (by default the module will use EC2 endpoints). diff --git a/test/integration/targets/aws_eks/playbooks/full_test.yml b/test/integration/targets/aws_eks/playbooks/full_test.yml index cd020be571..354dd744d3 100644 --- a/test/integration/targets/aws_eks/playbooks/full_test.yml +++ b/test/integration/targets/aws_eks/playbooks/full_test.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" roles: - aws_eks diff --git a/test/integration/targets/aws_eks/playbooks/old_version.yml b/test/integration/targets/aws_eks/playbooks/old_version.yml index d14dabcf00..c055cc68e2 100644 --- a/test/integration/targets/aws_eks/playbooks/old_version.yml +++ b/test/integration/targets/aws_eks/playbooks/old_version.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" tasks: - name: try and use aws_eks_cluster module diff --git a/test/integration/targets/cloudformation_stack_set/playbooks/full_test.yml b/test/integration/targets/cloudformation_stack_set/playbooks/full_test.yml index 46b314caed..257e1e48a5 100644 --- a/test/integration/targets/cloudformation_stack_set/playbooks/full_test.yml +++ b/test/integration/targets/cloudformation_stack_set/playbooks/full_test.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" roles: - ../../cloudformation_stack_set diff --git a/test/integration/targets/ec2_instance/playbooks/full_test.yml b/test/integration/targets/ec2_instance/playbooks/full_test.yml index e5266389c3..ca5606e6c8 100644 --- a/test/integration/targets/ec2_instance/playbooks/full_test.yml +++ b/test/integration/targets/ec2_instance/playbooks/full_test.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" roles: - ec2_instance diff --git a/test/integration/targets/ec2_instance/playbooks/version_fail.yml b/test/integration/targets/ec2_instance/playbooks/version_fail.yml index 6f20b64568..d084ae0c99 100644 --- a/test/integration/targets/ec2_instance/playbooks/version_fail.yml +++ b/test/integration/targets/ec2_instance/playbooks/version_fail.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" vars: resource_prefix: 'ansible-testing' diff --git a/test/integration/targets/ec2_launch_template/playbooks/full_test.yml b/test/integration/targets/ec2_launch_template/playbooks/full_test.yml index 7664841d14..ae375ac17b 100644 --- a/test/integration/targets/ec2_launch_template/playbooks/full_test.yml +++ b/test/integration/targets/ec2_launch_template/playbooks/full_test.yml @@ -1,4 +1,5 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" roles: - ec2_launch_template diff --git a/test/integration/targets/ec2_launch_template/playbooks/version_fail.yml b/test/integration/targets/ec2_launch_template/playbooks/version_fail.yml index d8a4461085..0db80028e0 100644 --- a/test/integration/targets/ec2_launch_template/playbooks/version_fail.yml +++ b/test/integration/targets/ec2_launch_template/playbooks/version_fail.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" vars: resource_prefix: 'ansible-testing' module_defaults: diff --git a/test/integration/targets/ecs_cluster/playbooks/full_test.yml b/test/integration/targets/ecs_cluster/playbooks/full_test.yml index 3219fa9472..867391114b 100644 --- a/test/integration/targets/ecs_cluster/playbooks/full_test.yml +++ b/test/integration/targets/ecs_cluster/playbooks/full_test.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" roles: - ecs_cluster diff --git a/test/integration/targets/ecs_cluster/playbooks/network_assign_public_ip_fail.yml b/test/integration/targets/ecs_cluster/playbooks/network_assign_public_ip_fail.yml index 79e931f7c2..e24dbaf73e 100644 --- a/test/integration/targets/ecs_cluster/playbooks/network_assign_public_ip_fail.yml +++ b/test/integration/targets/ecs_cluster/playbooks/network_assign_public_ip_fail.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" vars: resource_prefix: 'ansible-testing' diff --git a/test/integration/targets/ecs_cluster/playbooks/network_fail.yml b/test/integration/targets/ecs_cluster/playbooks/network_fail.yml index 7d026d806f..be8e47bf27 100644 --- a/test/integration/targets/ecs_cluster/playbooks/network_fail.yml +++ b/test/integration/targets/ecs_cluster/playbooks/network_fail.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" vars: resource_prefix: 'ansible-testing' diff --git a/test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment.yml b/test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment.yml index 07b013696b..dba9899daf 100644 --- a/test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment.yml +++ b/test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" vars: resource_prefix: 'ansible-testing-fnd' diff --git a/test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment_fail.yml b/test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment_fail.yml index ff9bd9f144..4259667dca 100644 --- a/test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment_fail.yml +++ b/test/integration/targets/ecs_cluster/playbooks/network_force_new_deployment_fail.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" vars: resource_prefix: 'ansible-testing-fndf' diff --git a/test/integration/targets/efs/playbooks/full_test.yml b/test/integration/targets/efs/playbooks/full_test.yml index d31f5a552c..6581151358 100644 --- a/test/integration/targets/efs/playbooks/full_test.yml +++ b/test/integration/targets/efs/playbooks/full_test.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" vars: resource_prefix: 'ansible-testing' diff --git a/test/integration/targets/efs/playbooks/version_fail.yml b/test/integration/targets/efs/playbooks/version_fail.yml index a8b923e9a7..49c94ae35c 100644 --- a/test/integration/targets/efs/playbooks/version_fail.yml +++ b/test/integration/targets/efs/playbooks/version_fail.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" vars: resource_prefix: 'ansible-testing' diff --git a/test/integration/targets/elb_target/playbooks/full_test.yml b/test/integration/targets/elb_target/playbooks/full_test.yml index 2e742d954b..03b1c4de02 100644 --- a/test/integration/targets/elb_target/playbooks/full_test.yml +++ b/test/integration/targets/elb_target/playbooks/full_test.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" roles: - elb_target diff --git a/test/integration/targets/elb_target/playbooks/version_fail.yml b/test/integration/targets/elb_target/playbooks/version_fail.yml index b2ce2a74bf..aad4376d38 100644 --- a/test/integration/targets/elb_target/playbooks/version_fail.yml +++ b/test/integration/targets/elb_target/playbooks/version_fail.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" tasks: - name: set up aws connection info diff --git a/test/integration/targets/elb_target_facts/playbooks/full_test.yml b/test/integration/targets/elb_target_facts/playbooks/full_test.yml index 6364c620c5..2cdf1d1a16 100644 --- a/test/integration/targets/elb_target_facts/playbooks/full_test.yml +++ b/test/integration/targets/elb_target_facts/playbooks/full_test.yml @@ -1,5 +1,6 @@ - hosts: localhost connection: local + environment: "{{ ansible_test.environment }}" roles: - elb_target_facts diff --git a/test/integration/targets/inventory_aws_ec2/playbooks/populate_cache.yml b/test/integration/targets/inventory_aws_ec2/playbooks/populate_cache.yml index 52790c1a10..07b0eec4c5 100644 --- a/test/integration/targets/inventory_aws_ec2/playbooks/populate_cache.yml +++ b/test/integration/targets/inventory_aws_ec2/playbooks/populate_cache.yml @@ -2,6 +2,7 @@ - hosts: 127.0.0.1 connection: local gather_facts: no + environment: "{{ ansible_test.environment }}" tasks: - block: diff --git a/test/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory.yml b/test/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory.yml index 004ff5773f..73a67db065 100644 --- a/test/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory.yml +++ b/test/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory.yml @@ -2,6 +2,7 @@ - hosts: 127.0.0.1 connection: local gather_facts: no + environment: "{{ ansible_test.environment }}" tasks: - block: diff --git a/test/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory_with_constructed.yml b/test/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory_with_constructed.yml index 8c6cc4b80a..fdeeeeff42 100644 --- a/test/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory_with_constructed.yml +++ b/test/integration/targets/inventory_aws_ec2/playbooks/test_populating_inventory_with_constructed.yml @@ -2,6 +2,7 @@ - hosts: 127.0.0.1 connection: local gather_facts: no + environment: "{{ ansible_test.environment }}" tasks: - block: diff --git a/test/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml b/test/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml index 8698e0a996..ba02428cc1 100644 --- a/test/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml +++ b/test/integration/targets/inventory_aws_rds/playbooks/populate_cache.yml @@ -2,11 +2,12 @@ - hosts: 127.0.0.1 connection: local gather_facts: no + environment: "{{ ansible_test.environment }}" tasks: - block: - set_fact: - instance_id: 'rds-mariadb-{{ resource_prefix }}' + instance_id: '{{ resource_prefix }}-mariadb' - name: assert group was populated with inventory but is empty assert: @@ -16,36 +17,28 @@ # Create new host, add it to inventory and then terminate it without updating the cache - # TODO: Uncomment once rds_instance has been added - #- name: set connection information for all tasks - # set_fact: - # aws_connection_info: &aws_connection_info - # aws_access_key: '{{ aws_access_key }}' - # aws_secret_key: '{{ aws_secret_key }}' - # security_token: '{{ security_token }}' - # region: '{{ aws_region }}' - # no_log: yes + - name: set connection information for all tasks + set_fact: + aws_connection_info: &aws_connection_info + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + region: '{{ aws_region }}' + no_log: yes - - name: Use AWS CLI to create an RDS DB instance - command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser'" - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" - - # TODO: Uncomment once rds_instance has been added - #- name: create minimal mariadb instance in default VPC and default subnet group - # rds_instance: - # state: present - # engine: mariadb - # instance_class: db.t2.micro - # storage: 20 - # instance_id: 'rds-mariadb-{{ resource_prefix }}' - # master_username: 'ansible-test-user' - # master_password: 'password-{{ resource_prefix }}' - # <<: *aws_connection_info - # register: setup_instance + - name: create minimal mariadb instance in default VPC and default subnet group + rds_instance: + state: present + engine: mariadb + db_instance_class: db.t2.micro + allocated_storage: 20 + instance_id: '{{ instance_id }}' + master_username: 'ansibletestuser' + master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}' + tags: + workload_type: other + <<: *aws_connection_info + register: setup_instance - meta: refresh_inventory @@ -55,22 +48,12 @@ always: - - name: Use AWS CLI to delete the DB instance - command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot" - ignore_errors: True - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" - - # TODO: Uncomment once rds_instance has been added - #- name: remove mariadb instance - # rds_instance: - # state: absent - # engine: mariadb - # skip_final_snapshot: yes - # instance_id: ansible-rds-mariadb-example - # <<: *aws_connection_info - # ignore_errors: yes - # when: setup_instance is defined + - name: remove mariadb instance + rds_instance: + state: absent + engine: mariadb + skip_final_snapshot: yes + instance_id: '{{ instance_id }}' + <<: *aws_connection_info + ignore_errors: yes + when: setup_instance is defined diff --git a/test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml b/test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml index 0b1ba33080..967249bdfd 100644 --- a/test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml +++ b/test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory.yml @@ -2,6 +2,7 @@ - hosts: 127.0.0.1 connection: local gather_facts: no + environment: "{{ ansible_test.environment }}" tasks: - block: @@ -18,35 +19,28 @@ # Create new host, refresh inventory, remove host, refresh inventory - #- name: set connection information for all tasks - # set_fact: - # aws_connection_info: &aws_connection_info - # aws_access_key: '{{ aws_access_key }}' - # aws_secret_key: '{{ aws_secret_key }}' - # security_token: '{{ security_token }}' - # region: '{{ aws_region }}' - # no_log: yes + - name: set connection information for all tasks + set_fact: + aws_connection_info: &aws_connection_info + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + region: '{{ aws_region }}' + no_log: yes - - name: Use AWS CLI to create an RDS DB instance - command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser'" - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" - - # TODO: Uncomment once rds_instance has been added - #- name: create minimal mariadb instance in default VPC and default subnet group - # rds_instance: - # state: present - # engine: mariadb - # instance_class: db.t2.micro - # storage: 20 - # instance_id: 'rds-mariadb-{{ resource_prefix }}' - # master_username: 'ansible-test-user' - # master_password: 'password-{{ resource_prefix }}' - # <<: *aws_connection_info - # register: setup_instance + - name: create minimal mariadb instance in default VPC and default subnet group + rds_instance: + state: present + engine: mariadb + db_instance_class: db.t2.micro + allocated_storage: 20 + instance_id: '{{ instance_id }}' + master_username: 'ansibletestuser' + master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}' + tags: + workload_type: other + <<: *aws_connection_info + register: setup_instance - meta: refresh_inventory @@ -57,23 +51,13 @@ - "groups.aws_rds | length == 1" - "groups.aws_rds.0 == '{{ instance_id }}'" - - name: Use AWS CLI to delete the DB instance - command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot" - ignore_errors: True - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" - - # TODO: Uncomment once rds_instance has been added - #- name: remove mariadb instance - # rds_instance: - # state: absent - # engine: mariadb - # skip_final_snapshot: yes - # instance_id: ansible-rds-mariadb-example - # <<: *aws_connection_info + - name: remove mariadb instance + rds_instance: + state: absent + engine: mariadb + skip_final_snapshot: yes + instance_id: '{{ instance_id }}' + <<: *aws_connection_info - meta: refresh_inventory @@ -85,22 +69,12 @@ always: - - name: Use AWS CLI to delete the DB instance - command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot" - ignore_errors: True - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" - - # TODO: Uncomment once rds_instance has been added - #- name: remove mariadb instance - # rds_instance: - # state: absent - # engine: mariadb - # skip_final_snapshot: yes - # instance_id: ansible-rds-mariadb-example - # <<: *aws_connection_info - # ignore_errors: yes - # when: setup_instance is defined + - name: remove mariadb instance + rds_instance: + state: absent + engine: mariadb + skip_final_snapshot: yes + instance_id: '{{ instance_id }}' + <<: *aws_connection_info + ignore_errors: yes + when: setup_instance is defined diff --git a/test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml b/test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml index c3ce5c008e..e2eb04118c 100644 --- a/test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml +++ b/test/integration/targets/inventory_aws_rds/playbooks/test_populating_inventory_with_constructed.yml @@ -2,44 +2,38 @@ - hosts: 127.0.0.1 connection: local gather_facts: no + environment: "{{ ansible_test.environment }}" tasks: - block: - set_fact: - instance_id: "{{ resource_prefix }}constructed" + instance_id: "{{ resource_prefix }}-mariadb" # Create new host, refresh inventory - #- name: set connection information for all tasks - # set_fact: - # aws_connection_info: &aws_connection_info - # aws_access_key: '{{ aws_access_key }}' - # aws_secret_key: '{{ aws_secret_key }}' - # security_token: '{{ security_token }}' - # region: '{{ aws_region }}' - # no_log: yes + - name: set connection information for all tasks + set_fact: + aws_connection_info: &aws_connection_info + aws_access_key: '{{ aws_access_key }}' + aws_secret_key: '{{ aws_secret_key }}' + security_token: '{{ security_token }}' + region: '{{ aws_region }}' + no_log: yes - # TODO: Uncomment once rds_instance has been added - #- name: create minimal mariadb instance in default VPC and default subnet group - # rds_instance: - # state: present - # engine: mariadb - # instance_class: db.t2.micro - # storage: 20 - # instance_id: 'rds-mariadb-{{ resource_prefix }}' - # master_username: 'ansible-test-user' - # master_password: 'password-{{ resource_prefix }}' - # <<: *aws_connection_info - # register: setup_instance - - - name: Use AWS CLI to create an RDS DB instance - command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser' --tags Key='workload_type',Value='other'" - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" + - name: create minimal mariadb instance in default VPC and default subnet group + rds_instance: + state: present + engine: mariadb + db_instance_class: db.t2.micro + allocated_storage: 20 + instance_id: '{{ resource_prefix }}-mariadb' + master_username: 'ansibletestuser' + master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}' + tags: + workload_type: other + <<: *aws_connection_info + register: setup_instance - meta: refresh_inventory - debug: var=groups @@ -51,26 +45,16 @@ - "groups | length == 6" - groups.tag_workload_type_other - groups.rds_mariadb - - groups.rds_parameter_group_default_mariadb10_0 + - groups.rds_parameter_group_default_mariadb10_3 always: - - name: Use AWS CLI to delete the DB instance - command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot" - ignore_errors: True - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" - - # TODO: Uncomment once rds_instance has been added - #- name: remove mariadb instance - # rds_instance: - # state: absent - # engine: mariadb - # skip_final_snapshot: yes - # instance_id: ansible-rds-mariadb-example - # <<: *aws_connection_info - # ignore_errors: yes - # when: setup_instance is defined + - name: remove mariadb instance + rds_instance: + state: absent + engine: mariadb + skip_final_snapshot: yes + instance_id: '{{ instance_id }}' + <<: *aws_connection_info + ignore_errors: yes + when: setup_instance is defined diff --git a/test/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml b/test/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml index de73edf373..4c3f335ead 100644 --- a/test/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml +++ b/test/integration/targets/inventory_aws_rds/playbooks/test_refresh_inventory.yml @@ -9,34 +9,28 @@ - "'aws_rds' in groups" - "not groups.aws_rds" - #- name: set connection information for all tasks - # set_fact: - # aws_connection_info: &aws_connection_info - # aws_access_key: "{{ aws_access_key }}" - # aws_secret_key: "{{ aws_secret_key }}" - # security_token: "{{ security_token }}" - # region: "{{ aws_region }}" - # no_log: yes + - name: set connection information for all tasks + set_fact: + aws_connection_info: &aws_connection_info + aws_access_key: "{{ aws_access_key }}" + aws_secret_key: "{{ aws_secret_key }}" + security_token: "{{ security_token }}" + region: "{{ aws_region }}" + no_log: yes - #- name: create minimal mariadb instance in default VPC and default subnet group - # rds_instance: - # state: present - # engine: mariadb - # instance_class: db.t2.micro - # storage: 20 - # instance_id: 'rds-mariadb-{{ resource_prefix }}' - # master_username: 'ansible-test-user' - # master_password: 'password-{{ resource_prefix }}' - # <<: *aws_connection_info - # register: setup_instance - - - name: Use AWS CLI to create an RDS DB instance - command: "aws rds create-db-instance --db-instance-identifier '{{ instance_id }}' --engine 'mariadb' --db-instance-class 'db.t2.micro' --allocated-storage 20 --master-user-password '{{ resource_prefix }}' --master-username 'ansibletestuser'" - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" + - name: create minimal mariadb instance in default VPC and default subnet group + rds_instance: + state: present + engine: mariadb + db_instance_class: db.t2.micro + allocated_storage: 20 + instance_id: 'rds-mariadb-{{ resource_prefix }}' + master_username: 'ansibletestuser' + master_user_password: 'password-{{ resource_prefix | regex_findall(".{8}$") | first }}' + tags: + workload_type: other + <<: *aws_connection_info + register: setup_instance - meta: refresh_inventory @@ -47,22 +41,13 @@ - "groups.aws_rds | length == 1" - "groups.aws_rds.0 == '{{ resource_prefix }}'" - #- name: remove mariadb instance - # rds_instance: - # state: absent - # engine: mariadb - # skip_final_snapshot: yes - # instance_id: ansible-rds-mariadb-example - # <<: *aws_connection_info - - - name: Use AWS CLI to delete the DB instance - command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot" - ignore_errors: True - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" + - name: remove mariadb instance + rds_instance: + state: absent + engine: mariadb + skip_final_snapshot: yes + instance_id: ansible-rds-mariadb-example + <<: *aws_connection_info - meta: refresh_inventory @@ -74,21 +59,12 @@ always: - - name: Use AWS CLI to delete the DB instance - command: "aws rds delete-db-instance --db-instance-identifier '{{ instance_id }}' --skip-final-snapshot" - ignore_errors: True - environment: - AWS_ACCESS_KEY_ID: "{{ aws_access_key }}" - AWS_SECRET_ACCESS_KEY: "{{ aws_secret_key }}" - AWS_SESSION_TOKEN: "{{ security_token }}" - AWS_DEFAULT_REGION: "{{ aws_region }}" - - #- name: remove mariadb instance - # rds_instance: - # state: absent - # engine: mariadb - # skip_final_snapshot: yes - # instance_id: ansible-rds-mariadb-example - # <<: *aws_connection_info - # ignore_errors: yes - # when: setup_instance is defined + - name: remove mariadb instance + rds_instance: + state: absent + engine: mariadb + skip_final_snapshot: yes + instance_id: ansible-rds-mariadb-example + <<: *aws_connection_info + ignore_errors: yes + when: setup_instance is defined diff --git a/test/integration/targets/inventory_aws_rds/templates/inventory.yml b/test/integration/targets/inventory_aws_rds/templates/inventory.yml index 1da89481c0..f9ac33468e 100644 --- a/test/integration/targets/inventory_aws_rds/templates/inventory.yml +++ b/test/integration/targets/inventory_aws_rds/templates/inventory.yml @@ -4,3 +4,5 @@ aws_secret_access_key: '{{ aws_secret_key }}' aws_security_token: '{{ security_token }}' regions: - '{{ aws_region }}' +filters: + db-instance-id: "{{ resource_prefix }}-mariadb" diff --git a/test/integration/targets/inventory_aws_rds/templates/inventory_with_cache.yml b/test/integration/targets/inventory_aws_rds/templates/inventory_with_cache.yml index 609f760b98..212ea8aca2 100644 --- a/test/integration/targets/inventory_aws_rds/templates/inventory_with_cache.yml +++ b/test/integration/targets/inventory_aws_rds/templates/inventory_with_cache.yml @@ -7,3 +7,5 @@ aws_secret_access_key: '{{ aws_secret_key }}' aws_security_token: '{{ security_token }}' regions: - '{{ aws_region }}' +filters: + db-instance-id: "{{ resource_prefix }}-mariadb" diff --git a/test/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.yml b/test/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.yml index 2362c75d85..5620e4c33b 100644 --- a/test/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.yml +++ b/test/integration/targets/inventory_aws_rds/templates/inventory_with_constructed.yml @@ -11,3 +11,5 @@ keyed_groups: prefix: tag - key: engine prefix: rds +filters: + db-instance-id: "{{ resource_prefix }}-mariadb" diff --git a/test/runner/lib/cloud/aws.py b/test/runner/lib/cloud/aws.py index b1ce4a9b62..9442beb902 100644 --- a/test/runner/lib/cloud/aws.py +++ b/test/runner/lib/cloud/aws.py @@ -99,8 +99,12 @@ class AwsCloudEnvironment(CloudEnvironment): ansible_vars.update(dict(parser.items('default'))) + env_vars = {'ANSIBLE_DEBUG_BOTOCORE_LOGS': 'True'} + return CloudEnvironmentConfig( + env_vars=env_vars, ansible_vars=ansible_vars, + callback_plugins=['aws_resource_actions'], ) def on_failure(self, target, tries):