#!/usr/bin/python # # This is a free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This Ansible library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this library. If not, see . from __future__ import (absolute_import, division, print_function) __metaclass__ = type DOCUMENTATION = r''' --- module: s3_bucket version_added: 1.0.0 short_description: Manage S3 buckets in AWS, DigitalOcean, Ceph, Walrus, FakeS3 and StorageGRID description: - Manage S3 buckets in AWS, DigitalOcean, Ceph, Walrus, FakeS3 and StorageGRID. author: - Rob White (@wimnat) - Aubin Bikouo (@abikouo) options: force: description: - When trying to delete a bucket, delete all keys (including versions and delete markers) in the bucket first (an S3 bucket must be empty for a successful deletion). type: bool default: 'no' name: description: - Name of the S3 bucket. required: true type: str policy: description: - The JSON policy as a string. Set to the string C("null") to force the absence of a policy. type: json s3_url: description: - S3 URL endpoint for usage with DigitalOcean, Ceph, Eucalyptus and FakeS3 etc. - Assumes AWS if not specified. - For Walrus, use FQDN of the endpoint without scheme nor path. aliases: [ S3_URL ] type: str ceph: description: - Enable API compatibility with Ceph. It takes into account the S3 API subset working with Ceph in order to provide the same module behaviour where possible. type: bool default: false requester_pays: description: - With Requester Pays buckets, the requester instead of the bucket owner pays the cost of the request and the data download from the bucket. type: bool state: description: - Create or remove the S3 bucket. required: false default: present choices: [ 'present', 'absent' ] type: str tags: description: - Tags dict to apply to bucket. type: dict purge_tags: description: - Whether to remove tags that aren't present in the I(tags) parameter. type: bool default: True versioning: description: - Whether versioning is enabled or disabled (note that once versioning is enabled, it can only be suspended). type: bool encryption: description: - Describes the default server-side encryption to apply to new objects in the bucket. In order to remove the server-side encryption, the encryption needs to be set to 'none' explicitly. choices: [ 'none', 'AES256', 'aws:kms' ] type: str encryption_key_id: description: KMS master key ID to use for the default encryption. This parameter is allowed if I(encryption) is C(aws:kms). If not specified then it will default to the AWS provided KMS key. type: str public_access: description: - Configure public access block for S3 bucket. - This option cannot be used together with I(delete_public_access). suboptions: block_public_acls: description: Sets BlockPublicAcls value. type: bool default: False block_public_policy: description: Sets BlockPublicPolicy value. type: bool default: False ignore_public_acls: description: Sets IgnorePublicAcls value. type: bool default: False restrict_public_buckets: description: Sets RestrictPublicAcls value. type: bool default: False type: dict version_added: 1.3.0 delete_public_access: description: - Delete public access block configuration from bucket. - This option cannot be used together with a I(public_access) definition. default: false type: bool version_added: 1.3.0 object_ownership: description: - Allow bucket's ownership controls. - C(BucketOwnerPreferred) - Objects uploaded to the bucket change ownership to the bucket owner if the objects are uploaded with the bucket-owner-full-control canned ACL. - C(ObjectWriter) - The uploading account will own the object if the object is uploaded with the bucket-owner-full-control canned ACL. - This option cannot be used together with a I(delete_object_ownership) definition. - Management of bucket ownership controls requires botocore>=1.18.11. choices: [ 'BucketOwnerPreferred', 'ObjectWriter' ] type: str version_added: 2.0.0 delete_object_ownership: description: - Delete bucket's ownership controls. - This option cannot be used together with a I(object_ownership) definition. - Management of bucket ownership controls requires botocore>=1.18.11. default: false type: bool version_added: 2.0.0 extends_documentation_fragment: - amazon.aws.aws - amazon.aws.ec2 notes: - If C(requestPayment), C(policy), C(tagging) or C(versioning) operations/API aren't implemented by the endpoint, module doesn't fail if each parameter satisfies the following condition. I(requester_pays) is C(False), I(policy), I(tags), and I(versioning) are C(None). ''' EXAMPLES = r''' # Note: These examples do not set authentication details, see the AWS Guide for details. # Create a simple S3 bucket - amazon.aws.s3_bucket: name: mys3bucket state: present # Create a simple S3 bucket on Ceph Rados Gateway - amazon.aws.s3_bucket: name: mys3bucket s3_url: http://your-ceph-rados-gateway-server.xxx ceph: true # Remove an S3 bucket and any keys it contains - amazon.aws.s3_bucket: name: mys3bucket state: absent force: yes # Create a bucket, add a policy from a file, enable requester pays, enable versioning and tag - amazon.aws.s3_bucket: name: mys3bucket policy: "{{ lookup('file','policy.json') }}" requester_pays: yes versioning: yes tags: example: tag1 another: tag2 # Create a simple DigitalOcean Spaces bucket using their provided regional endpoint - amazon.aws.s3_bucket: name: mydobucket s3_url: 'https://nyc3.digitaloceanspaces.com' # Create a bucket with AES256 encryption - amazon.aws.s3_bucket: name: mys3bucket state: present encryption: "AES256" # Create a bucket with aws:kms encryption, KMS key - amazon.aws.s3_bucket: name: mys3bucket state: present encryption: "aws:kms" encryption_key_id: "arn:aws:kms:us-east-1:1234/5678example" # Create a bucket with aws:kms encryption, default key - amazon.aws.s3_bucket: name: mys3bucket state: present encryption: "aws:kms" # Create a bucket with public policy block configuration - amazon.aws.s3_bucket: name: mys3bucket state: present public_access: block_public_acls: true ignore_public_acls: true ## keys == 'false' can be omitted, undefined keys defaults to 'false' # block_public_policy: false # restrict_public_buckets: false # Delete public policy block from bucket - amazon.aws.s3_bucket: name: mys3bucket state: present delete_public_access: true # Create a bucket with object ownership controls set to ObjectWriter - amazon.aws.s3_bucket: name: mys3bucket state: present object_ownership: ObjectWriter # Delete onwership controls from bucket - amazon.aws.s3_bucket: name: mys3bucket state: present delete_object_ownership: true # Delete a bucket policy from bucket - amazon.aws.s3_bucket: name: mys3bucket state: present policy: "null" ''' RETURN = r''' encryption: description: - Server-side encryption of the objects in the S3 bucket. type: str returned: I(state=present) sample: '' name: description: Name of the S3 bucket. type: str returned: I(state=present) sample: "2d3ce10a8210d36d6b4d23b822892074complex" object_ownership: description: S3 bucket's ownership controls. type: str returned: I(state=present) sample: "BucketOwnerPreferred" policy: description: S3 bucket's policy. type: dict returned: I(state=present) sample: { "Statement": [ { "Action": "s3:GetObject", "Effect": "Allow", "Principal": "*", "Resource": "arn:aws:s3:::2d3ce10a8210d36d6b4d23b822892074complex/*", "Sid": "AddPerm" } ], "Version": "2012-10-17" } requester_pays: description: - Indicates that the requester was successfully charged for the request. type: str returned: I(state=present) sample: '' tags: description: S3 bucket's tags. type: dict returned: I(state=present) sample: { "Tag1": "tag1", "Tag2": "tag2" } versioning: description: S3 bucket's versioning configuration. type: dict returned: I(state=present) sample: { "MfaDelete": "Disabled", "Versioning": "Enabled" } ''' import json import os import time try: import botocore except ImportError: pass # Handled by AnsibleAWSModule from ansible.module_utils.basic import to_text from ansible.module_utils.six import string_types from ansible.module_utils.six.moves.urllib.parse import urlparse from ..module_utils.core import AnsibleAWSModule from ..module_utils.core import is_boto3_error_code from ..module_utils.ec2 import AWSRetry from ..module_utils.ec2 import ansible_dict_to_boto3_tag_list from ..module_utils.ec2 import boto3_conn from ..module_utils.ec2 import boto3_tag_list_to_ansible_dict from ..module_utils.ec2 import compare_policies from ..module_utils.ec2 import get_aws_connection_info from ..module_utils.ec2 import snake_dict_to_camel_dict from ..module_utils.s3 import validate_bucket_name def create_or_update_bucket(s3_client, module, location): policy = module.params.get("policy") name = module.params.get("name") requester_pays = module.params.get("requester_pays") tags = module.params.get("tags") purge_tags = module.params.get("purge_tags") versioning = module.params.get("versioning") encryption = module.params.get("encryption") encryption_key_id = module.params.get("encryption_key_id") public_access = module.params.get("public_access") delete_public_access = module.params.get("delete_public_access") delete_object_ownership = module.params.get("delete_object_ownership") object_ownership = module.params.get("object_ownership") changed = False result = {} try: bucket_is_present = bucket_exists(s3_client, name) except botocore.exceptions.EndpointConnectionError as e: module.fail_json_aws(e, msg="Invalid endpoint provided: %s" % to_text(e)) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to check bucket presence") if not bucket_is_present: try: bucket_changed = create_bucket(s3_client, name, location) s3_client.get_waiter('bucket_exists').wait(Bucket=name) changed = changed or bucket_changed except botocore.exceptions.WaiterError as e: module.fail_json_aws(e, msg='An error occurred waiting for the bucket to become available') except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed while creating bucket") # Versioning try: versioning_status = get_bucket_versioning(s3_client, name) except is_boto3_error_code(['NotImplemented', 'XNotImplemented']) as e: if versioning is not None: module.fail_json_aws(e, msg="Failed to get bucket versioning") except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Failed to get bucket versioning") else: if versioning is not None: required_versioning = None if versioning and versioning_status.get('Status') != "Enabled": required_versioning = 'Enabled' elif not versioning and versioning_status.get('Status') == "Enabled": required_versioning = 'Suspended' if required_versioning: try: put_bucket_versioning(s3_client, name, required_versioning) changed = True except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to update bucket versioning") versioning_status = wait_versioning_is_applied(module, s3_client, name, required_versioning) # This output format is there to ensure compatibility with previous versions of the module result['versioning'] = { 'Versioning': versioning_status.get('Status', 'Disabled'), 'MfaDelete': versioning_status.get('MFADelete', 'Disabled'), } # Requester pays try: requester_pays_status = get_bucket_request_payment(s3_client, name) except is_boto3_error_code(['NotImplemented', 'XNotImplemented']) as e: if requester_pays is not None: module.fail_json_aws(e, msg="Failed to get bucket request payment") except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Failed to get bucket request payment") else: if requester_pays is not None: payer = 'Requester' if requester_pays else 'BucketOwner' if requester_pays_status != payer: put_bucket_request_payment(s3_client, name, payer) requester_pays_status = wait_payer_is_applied(module, s3_client, name, payer, should_fail=False) if requester_pays_status is None: # We have seen that it happens quite a lot of times that the put request was not taken into # account, so we retry one more time put_bucket_request_payment(s3_client, name, payer) requester_pays_status = wait_payer_is_applied(module, s3_client, name, payer, should_fail=True) changed = True result['requester_pays'] = requester_pays # Policy try: current_policy = get_bucket_policy(s3_client, name) except is_boto3_error_code(['NotImplemented', 'XNotImplemented']) as e: if policy is not None: module.fail_json_aws(e, msg="Failed to get bucket policy") except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Failed to get bucket policy") else: if policy is not None: if isinstance(policy, string_types): policy = json.loads(policy) if not policy and current_policy: try: delete_bucket_policy(s3_client, name) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to delete bucket policy") current_policy = wait_policy_is_applied(module, s3_client, name, policy) changed = True elif compare_policies(current_policy, policy): try: put_bucket_policy(s3_client, name, policy) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to update bucket policy") current_policy = wait_policy_is_applied(module, s3_client, name, policy, should_fail=False) if current_policy is None: # As for request payement, it happens quite a lot of times that the put request was not taken into # account, so we retry one more time put_bucket_policy(s3_client, name, policy) current_policy = wait_policy_is_applied(module, s3_client, name, policy, should_fail=True) changed = True result['policy'] = current_policy # Tags try: current_tags_dict = get_current_bucket_tags_dict(s3_client, name) except is_boto3_error_code(['NotImplemented', 'XNotImplemented']) as e: if tags is not None: module.fail_json_aws(e, msg="Failed to get bucket tags") except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Failed to get bucket tags") else: if tags is not None: # Tags are always returned as text tags = dict((to_text(k), to_text(v)) for k, v in tags.items()) if not purge_tags: # Ensure existing tags that aren't updated by desired tags remain current_copy = current_tags_dict.copy() current_copy.update(tags) tags = current_copy if current_tags_dict != tags: if tags: try: put_bucket_tagging(s3_client, name, tags) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to update bucket tags") else: if purge_tags: try: delete_bucket_tagging(s3_client, name) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to delete bucket tags") current_tags_dict = wait_tags_are_applied(module, s3_client, name, tags) changed = True result['tags'] = current_tags_dict # Encryption try: current_encryption = get_bucket_encryption(s3_client, name) except is_boto3_error_code(['NotImplemented', 'XNotImplemented']) as e: if encryption is not None: module.fail_json_aws(e, msg="Failed to get bucket encryption settings") except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Failed to get bucket encryption settings") else: if encryption is not None: current_encryption_algorithm = current_encryption.get('SSEAlgorithm') if current_encryption else None current_encryption_key = current_encryption.get('KMSMasterKeyID') if current_encryption else None if encryption == 'none': if current_encryption_algorithm is not None: try: delete_bucket_encryption(s3_client, name) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to delete bucket encryption") current_encryption = wait_encryption_is_applied(module, s3_client, name, None) changed = True else: if (encryption != current_encryption_algorithm) or (encryption == 'aws:kms' and current_encryption_key != encryption_key_id): expected_encryption = {'SSEAlgorithm': encryption} if encryption == 'aws:kms' and encryption_key_id is not None: expected_encryption.update({'KMSMasterKeyID': encryption_key_id}) current_encryption = put_bucket_encryption_with_retry(module, s3_client, name, expected_encryption) changed = True result['encryption'] = current_encryption # Public access clock configuration current_public_access = {} try: current_public_access = get_bucket_public_access(s3_client, name) except is_boto3_error_code(['NotImplemented', 'XNotImplemented']) as e: if public_access is not None: module.fail_json_aws(e, msg="Failed to get bucket public access configuration") except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Failed to get bucket public access configuration") else: # -- Create / Update public access block if public_access is not None: camel_public_block = snake_dict_to_camel_dict(public_access, capitalize_first=True) if current_public_access == camel_public_block: result['public_access_block'] = current_public_access else: put_bucket_public_access(s3_client, name, camel_public_block) changed = True result['public_access_block'] = camel_public_block # -- Delete public access block if delete_public_access: if current_public_access == {}: result['public_access_block'] = current_public_access else: delete_bucket_public_access(s3_client, name) changed = True result['public_access_block'] = {} # -- Bucket ownership try: bucket_ownership = get_bucket_ownership_cntrl(s3_client, module, name) result['object_ownership'] = bucket_ownership except KeyError as e: # Some non-AWS providers appear to return policy documents that aren't # compatible with AWS, cleanly catch KeyError so users can continue to use # other features. if delete_object_ownership or object_ownership is not None: module.fail_json_aws(e, msg="Failed to get bucket object ownership settings") except is_boto3_error_code(['NotImplemented', 'XNotImplemented']) as e: if delete_object_ownership or object_ownership is not None: module.fail_json_aws(e, msg="Failed to get bucket object ownership settings") except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Failed to get bucket bucket object ownership settings") else: if delete_object_ownership: # delete S3 buckect ownership if bucket_ownership is not None: delete_bucket_ownership(s3_client, name) changed = True result['object_ownership'] = None elif object_ownership is not None: # update S3 bucket ownership if bucket_ownership != object_ownership: put_bucket_ownership(s3_client, name, object_ownership) changed = True result['object_ownership'] = object_ownership # Module exit module.exit_json(changed=changed, name=name, **result) def bucket_exists(s3_client, bucket_name): try: s3_client.head_bucket(Bucket=bucket_name) bucket_exists = True except is_boto3_error_code('404'): bucket_exists = False return bucket_exists @AWSRetry.exponential_backoff(max_delay=120) def create_bucket(s3_client, bucket_name, location): try: configuration = {} if location not in ('us-east-1', None): configuration['LocationConstraint'] = location if len(configuration) > 0: s3_client.create_bucket(Bucket=bucket_name, CreateBucketConfiguration=configuration) else: s3_client.create_bucket(Bucket=bucket_name) return True except is_boto3_error_code('BucketAlreadyOwnedByYou'): # We should never get here since we check the bucket presence before calling the create_or_update_bucket # method. However, the AWS Api sometimes fails to report bucket presence, so we catch this exception return False @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def put_bucket_tagging(s3_client, bucket_name, tags): s3_client.put_bucket_tagging(Bucket=bucket_name, Tagging={'TagSet': ansible_dict_to_boto3_tag_list(tags)}) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def put_bucket_policy(s3_client, bucket_name, policy): s3_client.put_bucket_policy(Bucket=bucket_name, Policy=json.dumps(policy)) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def delete_bucket_policy(s3_client, bucket_name): s3_client.delete_bucket_policy(Bucket=bucket_name) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_policy(s3_client, bucket_name): try: current_policy = json.loads(s3_client.get_bucket_policy(Bucket=bucket_name).get('Policy')) except is_boto3_error_code('NoSuchBucketPolicy'): return None return current_policy @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def put_bucket_request_payment(s3_client, bucket_name, payer): s3_client.put_bucket_request_payment(Bucket=bucket_name, RequestPaymentConfiguration={'Payer': payer}) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_request_payment(s3_client, bucket_name): return s3_client.get_bucket_request_payment(Bucket=bucket_name).get('Payer') @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_versioning(s3_client, bucket_name): return s3_client.get_bucket_versioning(Bucket=bucket_name) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def put_bucket_versioning(s3_client, bucket_name, required_versioning): s3_client.put_bucket_versioning(Bucket=bucket_name, VersioningConfiguration={'Status': required_versioning}) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def get_bucket_encryption(s3_client, bucket_name): try: result = s3_client.get_bucket_encryption(Bucket=bucket_name) return result.get('ServerSideEncryptionConfiguration', {}).get('Rules', [])[0].get('ApplyServerSideEncryptionByDefault') except is_boto3_error_code('ServerSideEncryptionConfigurationNotFoundError'): return None except (IndexError, KeyError): return None def put_bucket_encryption_with_retry(module, s3_client, name, expected_encryption): max_retries = 3 for retries in range(1, max_retries + 1): try: put_bucket_encryption(s3_client, name, expected_encryption) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: # pylint: disable=duplicate-except module.fail_json_aws(e, msg="Failed to set bucket encryption") current_encryption = wait_encryption_is_applied(module, s3_client, name, expected_encryption, should_fail=(retries == max_retries), retries=5) if current_encryption == expected_encryption: return current_encryption # We shouldn't get here, the only time this should happen is if # current_encryption != expected_encryption and retries == max_retries # Which should use module.fail_json and fail out first. module.fail_json(msg='Failed to apply bucket encryption', current=current_encryption, expected=expected_encryption, retries=retries) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def put_bucket_encryption(s3_client, bucket_name, encryption): server_side_encryption_configuration = {'Rules': [{'ApplyServerSideEncryptionByDefault': encryption}]} s3_client.put_bucket_encryption(Bucket=bucket_name, ServerSideEncryptionConfiguration=server_side_encryption_configuration) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def delete_bucket_tagging(s3_client, bucket_name): s3_client.delete_bucket_tagging(Bucket=bucket_name) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def delete_bucket_encryption(s3_client, bucket_name): s3_client.delete_bucket_encryption(Bucket=bucket_name) @AWSRetry.exponential_backoff(max_delay=240, catch_extra_error_codes=['OperationAborted']) def delete_bucket(s3_client, bucket_name): try: s3_client.delete_bucket(Bucket=bucket_name) except is_boto3_error_code('NoSuchBucket'): # This means bucket should have been in a deleting state when we checked it existence # We just ignore the error pass @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def put_bucket_public_access(s3_client, bucket_name, public_acces): ''' Put new public access block to S3 bucket ''' s3_client.put_public_access_block(Bucket=bucket_name, PublicAccessBlockConfiguration=public_acces) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def delete_bucket_public_access(s3_client, bucket_name): ''' Delete public access block from S3 bucket ''' s3_client.delete_public_access_block(Bucket=bucket_name) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def delete_bucket_ownership(s3_client, bucket_name): ''' Delete bucket ownership controls from S3 bucket ''' s3_client.delete_bucket_ownership_controls(Bucket=bucket_name) @AWSRetry.exponential_backoff(max_delay=120, catch_extra_error_codes=['NoSuchBucket', 'OperationAborted']) def put_bucket_ownership(s3_client, bucket_name, target): ''' Put bucket ownership controls for S3 bucket ''' s3_client.put_bucket_ownership_controls( Bucket=bucket_name, OwnershipControls={ 'Rules': [{'ObjectOwnership': target}] }) def wait_policy_is_applied(module, s3_client, bucket_name, expected_policy, should_fail=True): for dummy in range(0, 12): try: current_policy = get_bucket_policy(s3_client, bucket_name) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to get bucket policy") if compare_policies(current_policy, expected_policy): time.sleep(5) else: return current_policy if should_fail: module.fail_json(msg="Bucket policy failed to apply in the expected time", requested_policy=expected_policy, live_policy=current_policy) else: return None def wait_payer_is_applied(module, s3_client, bucket_name, expected_payer, should_fail=True): for dummy in range(0, 12): try: requester_pays_status = get_bucket_request_payment(s3_client, bucket_name) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to get bucket request payment") if requester_pays_status != expected_payer: time.sleep(5) else: return requester_pays_status if should_fail: module.fail_json(msg="Bucket request payment failed to apply in the expected time", requested_status=expected_payer, live_status=requester_pays_status) else: return None def wait_encryption_is_applied(module, s3_client, bucket_name, expected_encryption, should_fail=True, retries=12): for dummy in range(0, retries): try: encryption = get_bucket_encryption(s3_client, bucket_name) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to get updated encryption for bucket") if encryption != expected_encryption: time.sleep(5) else: return encryption if should_fail: module.fail_json(msg="Bucket encryption failed to apply in the expected time", requested_encryption=expected_encryption, live_encryption=encryption) return encryption def wait_versioning_is_applied(module, s3_client, bucket_name, required_versioning): for dummy in range(0, 24): try: versioning_status = get_bucket_versioning(s3_client, bucket_name) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to get updated versioning for bucket") if versioning_status.get('Status') != required_versioning: time.sleep(8) else: return versioning_status module.fail_json(msg="Bucket versioning failed to apply in the expected time", requested_versioning=required_versioning, live_versioning=versioning_status) def wait_tags_are_applied(module, s3_client, bucket_name, expected_tags_dict): for dummy in range(0, 12): try: current_tags_dict = get_current_bucket_tags_dict(s3_client, bucket_name) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to get bucket policy") if current_tags_dict != expected_tags_dict: time.sleep(5) else: return current_tags_dict module.fail_json(msg="Bucket tags failed to apply in the expected time", requested_tags=expected_tags_dict, live_tags=current_tags_dict) def get_current_bucket_tags_dict(s3_client, bucket_name): try: current_tags = s3_client.get_bucket_tagging(Bucket=bucket_name).get('TagSet') except is_boto3_error_code('NoSuchTagSet'): return {} # The Ceph S3 API returns a different error code to AWS except is_boto3_error_code('NoSuchTagSetError'): # pylint: disable=duplicate-except return {} return boto3_tag_list_to_ansible_dict(current_tags) def get_bucket_public_access(s3_client, bucket_name): ''' Get current bucket public access block ''' try: bucket_public_access_block = s3_client.get_public_access_block(Bucket=bucket_name) return bucket_public_access_block['PublicAccessBlockConfiguration'] except is_boto3_error_code('NoSuchPublicAccessBlockConfiguration'): return {} def get_bucket_ownership_cntrl(s3_client, module, bucket_name): ''' Get current bucket public access block ''' if not module.botocore_at_least('1.18.11'): return None try: bucket_ownership = s3_client.get_bucket_ownership_controls(Bucket=bucket_name) return bucket_ownership['OwnershipControls']['Rules'][0]['ObjectOwnership'] except is_boto3_error_code(['OwnershipControlsNotFoundError', 'NoSuchOwnershipControls']): return None def paginated_list(s3_client, **pagination_params): pg = s3_client.get_paginator('list_objects_v2') for page in pg.paginate(**pagination_params): yield [data['Key'] for data in page.get('Contents', [])] def paginated_versions_list(s3_client, **pagination_params): try: pg = s3_client.get_paginator('list_object_versions') for page in pg.paginate(**pagination_params): # We have to merge the Versions and DeleteMarker lists here, as DeleteMarkers can still prevent a bucket deletion yield [(data['Key'], data['VersionId']) for data in (page.get('Versions', []) + page.get('DeleteMarkers', []))] except is_boto3_error_code('NoSuchBucket'): yield [] def destroy_bucket(s3_client, module): force = module.params.get("force") name = module.params.get("name") try: bucket_is_present = bucket_exists(s3_client, name) except botocore.exceptions.EndpointConnectionError as e: module.fail_json_aws(e, msg="Invalid endpoint provided: %s" % to_text(e)) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to check bucket presence") if not bucket_is_present: module.exit_json(changed=False) if force: # if there are contents then we need to delete them (including versions) before we can delete the bucket try: for key_version_pairs in paginated_versions_list(s3_client, Bucket=name): formatted_keys = [{'Key': key, 'VersionId': version} for key, version in key_version_pairs] for fk in formatted_keys: # remove VersionId from cases where they are `None` so that # unversioned objects are deleted using `DeleteObject` # rather than `DeleteObjectVersion`, improving backwards # compatibility with older IAM policies. if not fk.get('VersionId'): fk.pop('VersionId') if formatted_keys: resp = s3_client.delete_objects(Bucket=name, Delete={'Objects': formatted_keys}) if resp.get('Errors'): module.fail_json( msg='Could not empty bucket before deleting. Could not delete objects: {0}'.format( ', '.join([k['Key'] for k in resp['Errors']]) ), errors=resp['Errors'], response=resp ) except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed while deleting bucket") try: delete_bucket(s3_client, name) s3_client.get_waiter('bucket_not_exists').wait(Bucket=name, WaiterConfig=dict(Delay=5, MaxAttempts=60)) except botocore.exceptions.WaiterError as e: module.fail_json_aws(e, msg='An error occurred waiting for the bucket to be deleted.') except (botocore.exceptions.BotoCoreError, botocore.exceptions.ClientError) as e: module.fail_json_aws(e, msg="Failed to delete bucket") module.exit_json(changed=True) def is_fakes3(s3_url): """ Return True if s3_url has scheme fakes3:// """ if s3_url is not None: return urlparse(s3_url).scheme in ('fakes3', 'fakes3s') else: return False def get_s3_client(module, aws_connect_kwargs, location, ceph, s3_url): if s3_url and ceph: # TODO - test this ceph = urlparse(s3_url) params = dict(module=module, conn_type='client', resource='s3', use_ssl=ceph.scheme == 'https', region=location, endpoint=s3_url, **aws_connect_kwargs) elif is_fakes3(s3_url): fakes3 = urlparse(s3_url) port = fakes3.port if fakes3.scheme == 'fakes3s': protocol = "https" if port is None: port = 443 else: protocol = "http" if port is None: port = 80 params = dict(module=module, conn_type='client', resource='s3', region=location, endpoint="%s://%s:%s" % (protocol, fakes3.hostname, to_text(port)), use_ssl=fakes3.scheme == 'fakes3s', **aws_connect_kwargs) else: params = dict(module=module, conn_type='client', resource='s3', region=location, endpoint=s3_url, **aws_connect_kwargs) return boto3_conn(**params) def main(): argument_spec = dict( force=dict(default=False, type='bool'), policy=dict(type='json'), name=dict(required=True), requester_pays=dict(type='bool'), s3_url=dict(aliases=['S3_URL']), state=dict(default='present', choices=['present', 'absent']), tags=dict(type='dict'), purge_tags=dict(type='bool', default=True), versioning=dict(type='bool'), ceph=dict(default=False, type='bool'), encryption=dict(choices=['none', 'AES256', 'aws:kms']), encryption_key_id=dict(), public_access=dict(type='dict', options=dict( block_public_acls=dict(type='bool', default=False), ignore_public_acls=dict(type='bool', default=False), block_public_policy=dict(type='bool', default=False), restrict_public_buckets=dict(type='bool', default=False))), delete_public_access=dict(type='bool', default=False), object_ownership=dict(type='str', choices=['BucketOwnerPreferred', 'ObjectWriter']), delete_object_ownership=dict(type='bool', default=False), ) required_by = dict( encryption_key_id=('encryption',), ) mutually_exclusive = [ ['public_access', 'delete_public_access'], ['delete_object_ownership', 'object_ownership'] ] module = AnsibleAWSModule( argument_spec=argument_spec, required_by=required_by, mutually_exclusive=mutually_exclusive ) region, ec2_url, aws_connect_kwargs = get_aws_connection_info(module, boto3=True) validate_bucket_name(module, module.params["name"]) if region in ('us-east-1', '', None): # default to US Standard region location = 'us-east-1' else: # Boto uses symbolic names for locations but region strings will # actually work fine for everything except us-east-1 (US Standard) location = region s3_url = module.params.get('s3_url') ceph = module.params.get('ceph') # allow eucarc environment variables to be used if ansible vars aren't set if not s3_url and 'S3_URL' in os.environ: s3_url = os.environ['S3_URL'] if ceph and not s3_url: module.fail_json(msg='ceph flavour requires s3_url') # Look at s3_url and tweak connection settings # if connecting to Ceph RGW, Walrus or fakes3 if s3_url: for key in ['validate_certs', 'security_token', 'profile_name']: aws_connect_kwargs.pop(key, None) s3_client = get_s3_client(module, aws_connect_kwargs, location, ceph, s3_url) if s3_client is None: # this should never happen module.fail_json(msg='Unknown error, failed to create s3 connection, no information available.') state = module.params.get("state") encryption = module.params.get("encryption") encryption_key_id = module.params.get("encryption_key_id") delete_object_ownership = module.params.get('delete_object_ownership') object_ownership = module.params.get('object_ownership') if delete_object_ownership or object_ownership: module.require_botocore_at_least('1.18.11', reason='to manipulate bucket ownership controls') # Parameter validation if encryption_key_id is not None and encryption != 'aws:kms': module.fail_json(msg="Only 'aws:kms' is a valid option for encryption parameter when you specify encryption_key_id.") if state == 'present': create_or_update_bucket(s3_client, module, location) elif state == 'absent': destroy_bucket(s3_client, module) if __name__ == '__main__': main()