CCM-14203 - Callback Delivery Replay and Recovery

infrastructure/terraform/components/callbacks/cloudwatch_dashboard_dlq.tf

@@ -0,0 +1,105 @@
+locals {
+  dlq_overview_metrics = [
+    for idx, client_key in sort(keys(local.all_clients)) :
+    idx == 0 ? [
+      "AWS/SQS",
+      "ApproximateNumberOfMessagesVisible",
+      "QueueName",
+      "${local.csi}-${client_key}-dlq-queue",
+      { "label" : "${client_key} DLQ" }
+      ] : [
+      "...",
+      "${local.csi}-${client_key}-dlq-queue",
+      { "label" : "${client_key} DLQ" }
+    ]
+  ]
+
+  dlq_timeseries_widgets = [
+    for idx, client_key in sort(keys(local.all_clients)) : {
+      height = 6
+      width  = 12
+      y      = 5 + (floor(idx / 2) * 6)
+      x      = (idx % 2) * 12
+      type   = "metric"
+      properties = {
+        title   = "${client_key} DLQ Depth"
+        view    = "timeSeries"
+        stacked = false
+        region  = var.region
+        stat    = "Maximum"
+        period  = 300
+        metrics = [
+          [
+            "AWS/SQS",
+            "ApproximateNumberOfMessagesVisible",
+            "QueueName",
+            "${local.csi}-${client_key}-dlq-queue",
+            { "label" : "Messages awaiting redrive", "color" : "#d62728" }
+          ],
+          [
+            "AWS/SQS",
+            "NumberOfMessagesMoved",
+            "QueueName",
+            "${local.csi}-${client_key}-dlq-queue",
+            { "label" : "Messages being redriven", "color" : "#2ca02c" }
+          ]
+        ]
+        annotations = {
+          horizontal = [
+            {
+              label = "Alarm threshold"
+              value = 0
+              fill  = "above"
+            }
+          ]
+        }
+        yAxis = {
+          left = {
+            min       = 0
+            showUnits = false
+            label     = "Message count"
+          }
+        }
+      }
+    }
+  ]
+}
+
+resource "aws_cloudwatch_dashboard" "dlq" {
+  dashboard_name = "${local.csi}-dlq"
+
+  dashboard_body = jsonencode({
+    widgets = concat(
+      [
+        {
+          height = 1
+          width  = 24
+          y      = 0
+          x      = 0
+          type   = "text"
+          properties = {
+            markdown = "## DLQ Monitoring\nMessages in a DLQ indicate failed callback deliveries."
+          }
+        },
+        {
+          height = 4
+          width  = 24
+          y      = 1
+          x      = 0
+          type   = "metric"
+          properties = {
+            title                    = "DLQ Depth Overview"
+            view                     = "singleValue"
+            sparkline                = true
+            region                   = var.region
+            stat                     = "Maximum"
+            period                   = 300
+            singleValueFullPrecision = false
+            metrics                  = local.dlq_overview_metrics
+          }
+        }
+      ],
+      local.dlq_timeseries_widgets,
+    )
+  })
+}

infrastructure/terraform/components/callbacks/cloudwatch_metric_alarm_dlq_depth.tf

@@ -0,0 +1,31 @@
+resource "aws_cloudwatch_metric_alarm" "client_dlq_depth" {
+  for_each = local.all_clients
+
+  alarm_name = "${local.csi}-${each.key}-dlq-depth"
+  alarm_description = join(" ", [
+    "RELIABILITY: Messages are in DLQ for ${each.key}.",
+    "Failed callback deliveries require operator attention.",
+  ])
+
+  comparison_operator = "GreaterThanThreshold"
+  evaluation_periods  = 1
+  metric_name         = "ApproximateNumberOfMessagesVisible"
+  namespace           = "AWS/SQS"
+  period              = 300
+  statistic           = "Sum"
+  threshold           = 0
+  actions_enabled     = true
+  treat_missing_data  = "notBreaching"
+
+  dimensions = {
+    QueueName = "${local.csi}-${each.key}-dlq-queue"
+  }
+
+  tags = merge(
+    local.default_tags,
+    {
+      Name   = "${local.csi}-${each.key}-dlq-depth"
+      Client = each.key
+    },
+  )
+}