enha: creating a deadletter for a queue is now optional (#1)

custom-terraform-aws-modules · Apr 3, 2024 · 7e4dc18 · 7e4dc18
1 parent 5a31e4a
commit 7e4dc18
Show file tree

Hide file tree

Showing 5 changed files with 223 additions and 32 deletions.
diff --git a/README.md b/README.md
@@ -32,12 +32,12 @@ This module provides a S3 bucket and multiple SQS queus which receive an event,
 
 ### `queues`
 
-| Name                       | Description                                                                                                                | Type     | Default | Required |
-| -------------------------- | -------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | :------: |
-| identifier                 | Unique identifier to differentiate global resources.                                                                       | `string` | n/a     |   yes    |
-| message_retention_seconds  | The number of seconds Amazon SQS retains a message. Integer representing seconds, from 60 (1 minute) to 1209600 (14 days). | `number` | n/a     |   yes    |
-| visibility_timeout_seconds | The visibility timeout for messages in the queue. An integer from 0 to 43200 (12 hours).                                   | `number` | n/a     |   yes    |
-| max_receive_count          | Specifies how many times the same message can be received before moved into the deadletter queue.                          | `number` | n/a     |   yes    |
+| Name                       | Description                                                                                                                                                                                 | Type     | Default | Required |
+| -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------- | ------- | :------: |
+| identifier                 | Unique identifier to differentiate global resources.                                                                                                                                        | `string` | n/a     |   yes    |
+| message_retention_seconds  | The number of seconds Amazon SQS retains a message. Integer representing seconds, from 60 (1 minute) to 1209600 (14 days).                                                                  | `number` | 345600  |    no    |
+| visibility_timeout_seconds | The visibility timeout for messages in the queue. An integer from 0 to 43200 (12 hours).                                                                                                    | `number` | 300     |    no    |
+| max_receive_count          | Specifies how many times the same message can be received before moved into the deadletter queue. Value '0' does not create a deadletter for the queue and the queue will retry infinitely. | `number` | 0       |    no    |
 
 ## Outputs
 
@@ -50,10 +50,13 @@ This module provides a S3 bucket and multiple SQS queus which receive an event,
 
 ### `queues`
 
-| Name | Description               |
-| ---- | ------------------------- |
-| url  | The URL of the SQS queue. |
-| arn  | The ARN of the SQS queue. |
+| Name            | Description                                            |
+| --------------- | ------------------------------------------------------ |
+| queue_arn       | The ARN of the SQS queue.                              |
+| queue_url       | The URL of the SQS queue.                              |
+| deadletter_arn  | The ARN of the deadletter SQS queue of the main queue. |
+| deadletter_url  | The URL of the deadletter SQS queue of the main queue. |
+
 
 ## Example
 
@@ -76,7 +79,7 @@ module "bucket" {
       identifier                 = "example-bucket-queue-two-dev"
       message_retention_seconds  = 345600
       visibility_timeout_seconds = 300
-      max_receive_count          = 4
+      max_receive_count          = 0
     }
   ]
 

diff --git a/main.tf b/main.tf
@@ -107,7 +107,7 @@ data "aws_iam_policy_document" "fanout" {
     }
 
     actions   = ["sqs:SendMessage"]
-    resources = ["arn:aws:sqs:*:*:${try(var.queues[count.index]["identifier"], null)}"]
+    resources = ["arn:aws:sqs:*:*:${var.queues[count.index]["identifier"]}"]
 
     condition {
       test     = "ArnEquals"
@@ -145,7 +145,7 @@ data "aws_iam_policy_document" "queue" {
 
     actions = ["sqs:SendMessage"]
 
-    resources = ["arn:aws:sqs:*:*:${try(var.queues[0]["identifier"], null)}"]
+    resources = ["arn:aws:sqs:*:*:${var.queues[0]["identifier"]}"]
 
     condition {
       test     = "ArnEquals"
@@ -155,35 +155,49 @@ data "aws_iam_policy_document" "queue" {
   }
 }
 
+# save indices of queues which have a related deadletter queue to later connect them
+locals {
+  deadletter_queues = [for i, v in var.queues : i if v["max_receive_count"] > 0]
+}
+
 resource "aws_sqs_queue" "deadletter" {
-  count = length(var.queues)
-  name  = "${try(var.queues[count.index]["identifier"], null)}-deadletter"
+  count = length(local.deadletter_queues)
+  name  = "${var.queues[local.deadletter_queues[count.index]]["identifier"]}-deadletter"
 
   tags = var.tags
 }
 
+locals {
+  deadletter_output = [for i, v in var.queues : {
+    arn         = try(aws_sqs_queue.deadletter[index(local.deadletter_queues, i)].arn, null)
+    url         = try(aws_sqs_queue.deadletter[index(local.deadletter_queues, i)].url, null)
+    queue_index = try(index(local.deadletter_queues, i), null)
+  }]
+}
+
 resource "aws_sqs_queue" "main" {
   count                      = length(var.queues)
-  name                       = try(var.queues[count.index]["identifier"], null)
-  message_retention_seconds  = try(var.queues[count.index]["message_retention_seconds"], null)
-  visibility_timeout_seconds = try(var.queues[count.index]["visibility_timeout_seconds"], null)
-  policy                     = length(var.queues) > 1 ? data.aws_iam_policy_document.fanout[count.index].json : data.aws_iam_policy_document.queue[0].json
-
-  redrive_policy = jsonencode({
-    deadLetterTargetArn = aws_sqs_queue.deadletter[count.index].arn
-    maxReceiveCount     = try(var.queues[count.index]["max_receive_count"], null)
-  })
+  name                       = var.queues[count.index]["identifier"]
+  message_retention_seconds  = var.queues[count.index]["message_retention_seconds"]
+  visibility_timeout_seconds = var.queues[count.index]["visibility_timeout_seconds"]
+  policy = length(var.queues) > 1 ? (
+  data.aws_iam_policy_document.fanout[count.index].json) : data.aws_iam_policy_document.queue[0].json
+
+  redrive_policy = var.queues[count.index]["max_receive_count"] > 0 ? jsonencode({
+    deadLetterTargetArn = local.deadletter_output[count.index]["arn"]
+    maxReceiveCount     = var.queues[count.index]["max_receive_count"]
+  }) : null
 
   tags = var.tags
 }
 
 resource "aws_sqs_queue_redrive_allow_policy" "main" {
-  count     = length(var.queues)
+  count     = length(local.deadletter_queues)
   queue_url = aws_sqs_queue.deadletter[count.index].id
 
   redrive_allow_policy = jsonencode({
     redrivePermission = "byQueue",
-    sourceQueueArns   = [aws_sqs_queue.main[count.index].arn]
+    sourceQueueArns   = [aws_sqs_queue.main[local.deadletter_queues[count.index]].arn]
   })
 }
 

diff --git a/outputs.tf b/outputs.tf
@@ -18,8 +18,10 @@ output "uri" {
 
 output "queues" {
   description = "List of objects of each created queue."
-  value = [for index, value in try(var.queues, []) : {
-    url = aws_sqs_queue.main[index].url
-    arn = aws_sqs_queue.main[index].arn
+  value = [for index, value in var.queues : {
+    queue_arn      = aws_sqs_queue.main[index].arn
+    queue_url      = aws_sqs_queue.main[index].url
+    deadletter_arn = local.deadletter_output[index]["arn"]
+    deadletter_url = local.deadletter_output[index]["url"]
   }]
 }
diff --git a/tests/queue.tftest.hcl b/tests/queue.tftest.hcl
@@ -239,3 +239,175 @@ run "multiple_queues" {
     error_message = "S3 to SNS IAM policy was not created"
   }
 }
+
+run "queue_without_deadletter" {
+  command = plan
+
+  variables {
+    identifier = "test-bucket"
+    queues = [
+      {
+        identifier                 = "test-queue-one"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 0
+      }
+    ]
+  }
+
+  assert {
+    condition     = length(aws_sqs_queue.main) == 1
+    error_message = "Main SQS queue was not created"
+  }
+
+  assert {
+    condition     = length(aws_sqs_queue.deadletter) == 0
+    error_message = "Deadletter SQS queue was created unexpectedly"
+  }
+
+  assert {
+    condition     = length(local.deadletter_queues) == 0
+    error_message = "Deadletter index list has an unexpected length"
+  }
+
+  assert {
+    condition     = length(local.deadletter_output) == 1
+    error_message = "Deadletter output list has an unexpected length"
+  }
+
+  assert {
+    condition = local.deadletter_output[0]["arn"] == null && (
+    local.deadletter_output[0]["url"] == null)
+    error_message = "Deadletter output at index '0' is not null"
+  }
+}
+
+run "multiple_queues_single_deadletter" {
+  command = plan
+
+  variables {
+    identifier = "test-bucket"
+    queues = [
+      {
+        identifier                 = "test-queue-one"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 0
+      },
+      {
+        identifier                 = "test-queue-two"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 0
+      },
+      {
+        identifier                 = "test-queue-three"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 4
+      },
+      {
+        identifier                 = "test-queue-four"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 0
+      }
+    ]
+  }
+
+  assert {
+    condition     = length(aws_sqs_queue.main) == length(var.queues)
+    error_message = "SQS queues were not created"
+  }
+
+  assert {
+    condition     = length(aws_sqs_queue.deadletter) == 1
+    error_message = "Unexpected amount of deadletter SQS queues were created"
+  }
+
+  assert {
+    condition     = length(local.deadletter_queues) == 1
+    error_message = "Deadletter index list has an unexpected length"
+  }
+
+  assert {
+    condition     = length(local.deadletter_output) == length(var.queues)
+    error_message = "Deadletter output list has an unexpected length"
+  }
+
+  assert {
+    condition     = [for i, v in local.deadletter_output : v["queue_index"]] == [null, null, 0, null]
+    error_message = "Unexpected deadletter output"
+  }
+}
+
+run "multiple_queues_multiple_deadletter" {
+  command = plan
+
+  variables {
+    identifier = "test-bucket"
+    queues = [
+      {
+        identifier                 = "test-queue-one"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 0
+      },
+      {
+        identifier                 = "test-queue-two"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 0
+      },
+      {
+        identifier                 = "test-queue-three"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 4
+      },
+      {
+        identifier                 = "test-queue-four"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 4
+      },
+      {
+        identifier                 = "test-queue-five"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 0
+      },
+      {
+        identifier                 = "test-queue-six"
+        message_retention_seconds  = 345600
+        visibility_timeout_seconds = 300
+        max_receive_count          = 4
+      }
+    ]
+  }
+
+  assert {
+    condition     = length(aws_sqs_queue.main) == length(var.queues)
+    error_message = "SQS queues were not created"
+  }
+
+  assert {
+    condition     = length(aws_sqs_queue.deadletter) == 3
+    error_message = "Unexpected amount of deadletter SQS queues were created"
+  }
+
+  assert {
+    condition     = length(local.deadletter_queues) == 3
+    error_message = "Deadletter index list has an unexpected length"
+  }
+
+  assert {
+    condition     = length(local.deadletter_output) == length(var.queues)
+    error_message = "Deadletter output list has an unexpected length"
+  }
+
+  assert {
+    condition     = [for i, v in local.deadletter_output : v["queue_index"]] == [null, null, 0, 1, null, 2]
+    error_message = "Unexpected deadletter output"
+  }
+}
diff --git a/variables.tf b/variables.tf
@@ -27,9 +27,9 @@ variable "queues" {
   description = "A list of object to define SQS queues."
   type = list(object({
     identifier                 = string
-    message_retention_seconds  = number
-    visibility_timeout_seconds = number
-    max_receive_count          = number
+    message_retention_seconds  = optional(number, 345600)
+    visibility_timeout_seconds = optional(number, 300)
+    max_receive_count          = optional(number, 0)
   }))
   default = []
   validation {