From 96be82e3ee27f8543eb6fed6d338b345dae561a7 Mon Sep 17 00:00:00 2001 From: Anton Babenko Date: Sun, 24 May 2020 22:27:00 +0200 Subject: [PATCH] feat: Added modules/notifications for S3 bucket notifications (#31) --- .pre-commit-config.yaml | 4 +- README.md | 26 ++++-- examples/complete/README.md | 4 + examples/notification/README.md | 46 ++++++++++ examples/notification/main.tf | 89 +++++++++++++++++++ examples/notification/outputs.tf | 39 ++++++++ examples/s3-replication/README.md | 4 + modules/notification/README.md | 33 +++++++ modules/notification/main.tf | 142 ++++++++++++++++++++++++++++++ modules/notification/outputs.tf | 4 + modules/notification/variables.tf | 35 ++++++++ 11 files changed, 416 insertions(+), 10 deletions(-) create mode 100644 examples/notification/README.md create mode 100644 examples/notification/main.tf create mode 100644 examples/notification/outputs.tf create mode 100644 modules/notification/README.md create mode 100644 modules/notification/main.tf create mode 100644 modules/notification/outputs.tf create mode 100644 modules/notification/variables.tf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 02854526..fc33f2c8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,10 +1,10 @@ repos: - repo: git://github.com/antonbabenko/pre-commit-terraform - rev: v1.27.0 + rev: v1.30.0 hooks: - id: terraform_fmt - id: terraform_docs - repo: git://github.com/pre-commit/pre-commit-hooks - rev: v2.5.0 + rev: v3.1.0 hooks: - id: check-merge-conflict diff --git a/README.md b/README.md index 39440e8f..d30a1df4 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ This type of resources are supported: * [S3 Bucket](https://www.terraform.io/docs/providers/aws/r/s3_bucket.html) * [S3 Bucket Policy](https://www.terraform.io/docs/providers/aws/r/s3_bucket_policy.html) +* [S3 Bucket Notification](https://www.terraform.io/docs/providers/aws/r/s3_bucket_notification.html) - use [modules/notification](https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/tree/master/modules/notification) to configure notifications to Lambda functions, SQS queues, and SNS topics. These features of S3 bucket configurations are supported: @@ -75,26 +76,35 @@ module "s3_bucket" { * [Complete](https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/tree/master/examples/complete) - Complete S3 bucket with most of supported features enabled * [Cross-Region Replication](https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/tree/master/examples/s3-replication) - S3 bucket with Cross-Region Replication (CRR) enabled +- [S3 bucket Notifications](https://github.com/terraform-aws-modules/terraform-aws-s3-bucket/tree/master/examples/notification) - S3 bucket notifications to Lambda functions, SQS queues, and SNS topics. + +## Requirements + +| Name | Version | +|------|---------| +| terraform | ~> 0.12.6 | +| aws | ~> 2.35 | + ## Providers | Name | Version | |------|---------| -| aws | n/a | +| aws | ~> 2.35 | ## Inputs | Name | Description | Type | Default | Required | -|------|-------------|------|---------|:-----:| -| acceleration\_status | (Optional) Sets the accelerate configuration of an existing bucket. Can be Enabled or Suspended. | `string` | n/a | yes | +|------|-------------|------|---------|:--------:| +| acceleration\_status | (Optional) Sets the accelerate configuration of an existing bucket. Can be Enabled or Suspended. | `string` | `null` | no | | acl | (Optional) The canned ACL to apply. Defaults to 'private'. | `string` | `"private"` | no | | attach\_elb\_log\_delivery\_policy | Controls if S3 bucket should have ELB log delivery policy attached | `bool` | `false` | no | | attach\_policy | Controls if S3 bucket should have bucket policy attached (set to `true` to use value of `policy` as bucket policy) | `bool` | `false` | no | | block\_public\_acls | Whether Amazon S3 should block public ACLs for this bucket. | `bool` | `false` | no | | block\_public\_policy | Whether Amazon S3 should block public bucket policies for this bucket. | `bool` | `false` | no | -| bucket | (Optional, Forces new resource) The name of the bucket. If omitted, Terraform will assign a random, unique name. | `string` | n/a | yes | -| bucket\_prefix | (Optional, Forces new resource) Creates a unique bucket name beginning with the specified prefix. Conflicts with bucket. | `string` | n/a | yes | +| bucket | (Optional, Forces new resource) The name of the bucket. If omitted, Terraform will assign a random, unique name. | `string` | `null` | no | +| bucket\_prefix | (Optional, Forces new resource) Creates a unique bucket name beginning with the specified prefix. Conflicts with bucket. | `string` | `null` | no | | cors\_rule | Map containing a rule of Cross-Origin Resource Sharing. | `any` | `{}` | no | | create\_bucket | Controls if S3 bucket should be created | `bool` | `true` | no | | force\_destroy | (Optional, Default:false ) A boolean that indicates all objects should be deleted from the bucket so that the bucket can be destroyed without error. These objects are not recoverable. | `bool` | `false` | no | @@ -102,10 +112,10 @@ module "s3_bucket" { | lifecycle\_rule | List of maps containing configuration of object lifecycle management. | `any` | `[]` | no | | logging | Map containing access bucket logging configuration. | `map(string)` | `{}` | no | | object\_lock\_configuration | Map containing S3 object locking configuration. | `any` | `{}` | no | -| policy | (Optional) A valid bucket policy JSON document. Note that if the policy document is not specific enough (but still valid), Terraform may view the policy as constantly changing in a terraform plan. In this case, please make sure you use the verbose/specific version of the policy. For more information about building AWS IAM policy documents with Terraform, see the AWS IAM Policy Document Guide. | `string` | n/a | yes | -| region | (Optional) If specified, the AWS region this bucket should reside in. Otherwise, the region used by the callee. | `string` | n/a | yes | +| policy | (Optional) A valid bucket policy JSON document. Note that if the policy document is not specific enough (but still valid), Terraform may view the policy as constantly changing in a terraform plan. In this case, please make sure you use the verbose/specific version of the policy. For more information about building AWS IAM policy documents with Terraform, see the AWS IAM Policy Document Guide. | `string` | `null` | no | +| region | (Optional) If specified, the AWS region this bucket should reside in. Otherwise, the region used by the callee. | `string` | `null` | no | | replication\_configuration | Map containing cross-region replication configuration. | `any` | `{}` | no | -| request\_payer | (Optional) Specifies who should bear the cost of Amazon S3 data transfer. Can be either BucketOwner or Requester. By default, the owner of the S3 bucket would incur the costs of any data transfer. See Requester Pays Buckets developer guide for more information. | `string` | n/a | yes | +| request\_payer | (Optional) Specifies who should bear the cost of Amazon S3 data transfer. Can be either BucketOwner or Requester. By default, the owner of the S3 bucket would incur the costs of any data transfer. See Requester Pays Buckets developer guide for more information. | `string` | `null` | no | | restrict\_public\_buckets | Whether Amazon S3 should restrict public bucket policies for this bucket. | `bool` | `false` | no | | server\_side\_encryption\_configuration | Map containing server-side encryption configuration. | `any` | `{}` | no | | tags | (Optional) A mapping of tags to assign to the bucket. | `map(string)` | `{}` | no | diff --git a/examples/complete/README.md b/examples/complete/README.md index 215bc0a5..47577788 100644 --- a/examples/complete/README.md +++ b/examples/complete/README.md @@ -24,6 +24,10 @@ $ terraform apply Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. +## Requirements + +No requirements. + ## Providers | Name | Version | diff --git a/examples/notification/README.md b/examples/notification/README.md new file mode 100644 index 00000000..02ca44e7 --- /dev/null +++ b/examples/notification/README.md @@ -0,0 +1,46 @@ +# S3 bucket notifications to Lambda functions, SQS queues, and SNS topics + +Configuration in this directory creates S3 bucket notifications to all supported destinations. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| aws | n/a | +| random | n/a | + +## Inputs + +No input. + +## Outputs + +| Name | Description | +|------|-------------| +| this\_s3\_bucket\_arn | The ARN of the bucket. Will be of format arn:aws:s3:::bucketname. | +| this\_s3\_bucket\_bucket\_domain\_name | The bucket domain name. Will be of format bucketname.s3.amazonaws.com. | +| this\_s3\_bucket\_bucket\_regional\_domain\_name | The bucket region-specific domain name. The bucket domain name including the region name, please refer here for format. Note: The AWS CloudFront allows specifying S3 region-specific endpoint when creating S3 origin, it will prevent redirect issues from CloudFront to S3 Origin URL. | +| this\_s3\_bucket\_hosted\_zone\_id | The Route 53 Hosted Zone ID for this bucket's region. | +| this\_s3\_bucket\_id | The name of the bucket. | +| this\_s3\_bucket\_region | The AWS region this bucket resides in. | +| this\_s3\_bucket\_website\_domain | The domain of the website endpoint, if the bucket is configured with a website. If not, this will be an empty string. This is used to create Route 53 alias records. | +| this\_s3\_bucket\_website\_endpoint | The website endpoint, if the bucket is configured with a website. If not, this will be an empty string. | + + diff --git a/examples/notification/main.tf b/examples/notification/main.tf new file mode 100644 index 00000000..43bcb552 --- /dev/null +++ b/examples/notification/main.tf @@ -0,0 +1,89 @@ +locals { + bucket_name = "s3-bucket-${random_pet.this.id}" +} + +resource "random_pet" "this" { + length = 2 +} + +module "s3_bucket" { + source = "../../" + + bucket = local.bucket_name + force_destroy = true +} + +module "lambda_function1" { + source = "terraform-aws-modules/cloudwatch/aws//examples/fixtures/aws_lambda_function" +} + +module "lambda_function2" { + source = "terraform-aws-modules/cloudwatch/aws//examples/fixtures/aws_lambda_function" +} + +module "sns_topic1" { + source = "terraform-aws-modules/cloudwatch/aws//examples/fixtures/aws_sns_topic" +} + +module "sns_topic2" { + source = "terraform-aws-modules/cloudwatch/aws//examples/fixtures/aws_sns_topic" +} + +resource "aws_sqs_queue" "this" { + count = 2 + name = "${random_pet.this.id}-${count.index}" +} + +module "all_notifications" { + source = "../../modules/notification" + + bucket = module.s3_bucket.this_s3_bucket_id + create = false + + // Common error - Error putting S3 notification configuration: InvalidArgument: Configuration is ambiguously defined. Cannot have overlapping suffixes in two rules if the prefixes are overlapping for the same event type. + + lambda_notifications = { + lambda1 = { + lambda_function_arn = module.lambda_function1.this_lambda_function_arn + events = ["s3:ObjectCreated:Put"] + filter_prefix = "prefix/" + filter_suffix = ".json" + } + + lambda2 = { + lambda_function_arn = module.lambda_function2.this_lambda_function_arn + events = ["s3:ObjectCreated:Post"] + } + } + + sqs_notifications = { + sqs1 = { + queue_arn = aws_sqs_queue.this[0].arn + events = ["s3:ObjectCreated:Put"] + filter_prefix = "prefix2/" + filter_suffix = ".txt" + + // queue_id = aws_sqs_queue.this[0].id // optional + } + + sqs2 = { + queue_arn = aws_sqs_queue.this[1].arn + events = ["s3:ObjectCreated:Copy"] + } + } + + sns_notifications = { + sns1 = { + topic_arn = module.sns_topic1.this_sns_topic_arn + events = ["s3:ObjectRemoved:Delete"] + filter_prefix = "prefix3/" + filter_suffix = ".csv" + } + + sns2 = { + topic_arn = module.sns_topic2.this_sns_topic_arn + events = ["s3:ObjectRemoved:DeleteMarkerCreated"] + } + } + +} diff --git a/examples/notification/outputs.tf b/examples/notification/outputs.tf new file mode 100644 index 00000000..d7a44021 --- /dev/null +++ b/examples/notification/outputs.tf @@ -0,0 +1,39 @@ +output "this_s3_bucket_id" { + description = "The name of the bucket." + value = module.s3_bucket.this_s3_bucket_id +} + +output "this_s3_bucket_arn" { + description = "The ARN of the bucket. Will be of format arn:aws:s3:::bucketname." + value = module.s3_bucket.this_s3_bucket_arn +} + +output "this_s3_bucket_bucket_domain_name" { + description = "The bucket domain name. Will be of format bucketname.s3.amazonaws.com." + value = module.s3_bucket.this_s3_bucket_bucket_domain_name +} + +output "this_s3_bucket_bucket_regional_domain_name" { + description = "The bucket region-specific domain name. The bucket domain name including the region name, please refer here for format. Note: The AWS CloudFront allows specifying S3 region-specific endpoint when creating S3 origin, it will prevent redirect issues from CloudFront to S3 Origin URL." + value = module.s3_bucket.this_s3_bucket_bucket_regional_domain_name +} + +output "this_s3_bucket_hosted_zone_id" { + description = "The Route 53 Hosted Zone ID for this bucket's region." + value = module.s3_bucket.this_s3_bucket_hosted_zone_id +} + +output "this_s3_bucket_region" { + description = "The AWS region this bucket resides in." + value = module.s3_bucket.this_s3_bucket_region +} + +output "this_s3_bucket_website_endpoint" { + description = "The website endpoint, if the bucket is configured with a website. If not, this will be an empty string." + value = module.s3_bucket.this_s3_bucket_website_endpoint +} + +output "this_s3_bucket_website_domain" { + description = "The domain of the website endpoint, if the bucket is configured with a website. If not, this will be an empty string. This is used to create Route 53 alias records. " + value = module.s3_bucket.this_s3_bucket_website_domain +} diff --git a/examples/s3-replication/README.md b/examples/s3-replication/README.md index b95b6d10..5eefe108 100644 --- a/examples/s3-replication/README.md +++ b/examples/s3-replication/README.md @@ -17,6 +17,10 @@ $ terraform apply Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. +## Requirements + +No requirements. + ## Providers | Name | Version | diff --git a/modules/notification/README.md b/modules/notification/README.md new file mode 100644 index 00000000..cdc6f82e --- /dev/null +++ b/modules/notification/README.md @@ -0,0 +1,33 @@ +# S3 bucket notification + +Creates S3 bucket notification resource with all supported types of deliveries: AWS Lambda, SQS Queue, SNS Topic. + + +## Requirements + +No requirements. + +## Providers + +| Name | Version | +|------|---------| +| aws | n/a | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| bucket | Name of S3 bucket to use | `string` | `""` | no | +| bucket\_arn | ARN of S3 bucket to use in policies | `string` | `null` | no | +| create | Whether to create this resource or not? | `bool` | `true` | no | +| lambda\_notifications | Map of S3 bucket notifications to Lambda function | `any` | `{}` | no | +| sns\_notifications | Map of S3 bucket notifications to SNS topic | `any` | `{}` | no | +| sqs\_notifications | Map of S3 bucket notifications to SQS queue | `any` | `{}` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| this\_s3\_bucket\_notification\_id | ID of S3 bucket | + + diff --git a/modules/notification/main.tf b/modules/notification/main.tf new file mode 100644 index 00000000..a0b38c37 --- /dev/null +++ b/modules/notification/main.tf @@ -0,0 +1,142 @@ +locals { + bucket_arn = coalesce(var.bucket_arn, "arn:aws:s3:::${var.bucket}") + + // Convert from "arn:aws:sqs:eu-west-1:835367859851:bold-starling-0" into "https://sqs.eu-west-1.amazonaws.com/835367859851/bold-starling-0" if queue_id was not specified + // queue_url used in aws_sqs_queue_policy is not the same as arn which is used in all other places + queue_ids = { for k, v in var.sqs_notifications : k => format("https://%s.%s.amazonaws.com/%s/%s", data.aws_arn.queue[k].service, data.aws_arn.queue[k].region, data.aws_arn.queue[k].account, data.aws_arn.queue[k].resource) if lookup(v, "queue_id", "") == "" } +} + +resource "aws_s3_bucket_notification" "this" { + count = var.create && (length(var.lambda_notifications) > 0 || length(var.sqs_notifications) > 0 || length(var.sns_notifications) > 0) ? 1 : 0 + + bucket = var.bucket + + dynamic "lambda_function" { + for_each = var.lambda_notifications + + content { + id = lambda_function.key + lambda_function_arn = lambda_function.value.lambda_function_arn + events = lambda_function.value.events + filter_prefix = lookup(lambda_function.value, "filter_prefix", null) + filter_suffix = lookup(lambda_function.value, "filter_suffix", null) + } + } + + dynamic "queue" { + for_each = var.sqs_notifications + + content { + id = queue.key + queue_arn = queue.value.queue_arn + events = queue.value.events + filter_prefix = lookup(queue.value, "filter_prefix", null) + filter_suffix = lookup(queue.value, "filter_suffix", null) + } + } + + dynamic "topic" { + for_each = var.sns_notifications + + content { + id = topic.key + topic_arn = topic.value.topic_arn + events = topic.value.events + filter_prefix = lookup(topic.value, "filter_prefix", null) + filter_suffix = lookup(topic.value, "filter_suffix", null) + } + } + + depends_on = [ + aws_lambda_permission.allow, + aws_sqs_queue_policy.allow, + aws_sns_topic_policy.allow, + ] +} + +// Lambda +resource "aws_lambda_permission" "allow" { + for_each = var.lambda_notifications + + statement_id_prefix = "AllowLambdaS3BucketNotification-" + action = "lambda:InvokeFunction" + function_name = each.value.lambda_function_arn + principal = "s3.amazonaws.com" + source_arn = local.bucket_arn +} + +// SQS Queue +data "aws_arn" "queue" { + for_each = var.sqs_notifications + + arn = each.value.queue_arn +} + +data "aws_iam_policy_document" "sqs" { + for_each = var.sqs_notifications + + statement { + sid = "AllowSQSS3BucketNotification" + + effect = "Allow" + + actions = [ + "sqs:SendMessage", + ] + + principals { + type = "Service" + identifiers = ["s3.amazonaws.com"] + } + + resources = [each.value.queue_arn] + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = [local.bucket_arn] + } + } +} + +resource "aws_sqs_queue_policy" "allow" { + for_each = var.sqs_notifications + + queue_url = lookup(each.value, "queue_id", lookup(local.queue_ids, each.key, null)) + policy = data.aws_iam_policy_document.sqs[each.key].json +} + +// SNS Topic +data "aws_iam_policy_document" "sns" { + for_each = var.sns_notifications + + statement { + sid = "AllowSNSS3BucketNotification" + + effect = "Allow" + + actions = [ + "sns:Publish", + ] + + principals { + type = "Service" + identifiers = ["s3.amazonaws.com"] + } + + resources = [each.value.topic_arn] + + condition { + test = "ArnEquals" + variable = "aws:SourceArn" + values = [local.bucket_arn] + } + } +} + +resource "aws_sns_topic_policy" "allow" { + for_each = var.sns_notifications + + arn = each.value.topic_arn + policy = data.aws_iam_policy_document.sns[each.key].json +} diff --git a/modules/notification/outputs.tf b/modules/notification/outputs.tf new file mode 100644 index 00000000..2c38f9da --- /dev/null +++ b/modules/notification/outputs.tf @@ -0,0 +1,4 @@ +output "this_s3_bucket_notification_id" { + description = "ID of S3 bucket" + value = element(concat(aws_s3_bucket_notification.this.*.id, [""]), 0) +} diff --git a/modules/notification/variables.tf b/modules/notification/variables.tf new file mode 100644 index 00000000..d0c4c537 --- /dev/null +++ b/modules/notification/variables.tf @@ -0,0 +1,35 @@ +variable "create" { + description = "Whether to create this resource or not?" + type = bool + default = true +} + +variable "bucket" { + description = "Name of S3 bucket to use" + type = string + default = "" +} + +variable "bucket_arn" { + description = "ARN of S3 bucket to use in policies" + type = string + default = null +} + +variable "lambda_notifications" { + description = "Map of S3 bucket notifications to Lambda function" + type = any # map(map(any)) is better, but Terraform 0.12.25 panics + default = {} +} + +variable "sqs_notifications" { + description = "Map of S3 bucket notifications to SQS queue" + type = any # map(map(any)) is better, but Terraform 0.12.25 panics + default = {} +} + +variable "sns_notifications" { + description = "Map of S3 bucket notifications to SNS topic" + type = any # map(map(any)) is better, but Terraform 0.12.25 panics + default = {} +}