From f95fe291248c88428748444527cefdee43a71f49 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Tue, 25 Aug 2026 08:52:45 +0100 Subject: [PATCH 1/5] add new alarm and tweak existing alarm --- .../terraform/modules/alarms-ddb/README.md | 3 ++- .../cloudwatch_metric_alarm_write_capacity.tf | 22 +++++++++++++++++++ .../terraform/modules/alarms-ddb/variables.tf | 7 +++++- 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 infrastructure/terraform/modules/alarms-ddb/cloudwatch_metric_alarm_write_capacity.tf diff --git a/infrastructure/terraform/modules/alarms-ddb/README.md b/infrastructure/terraform/modules/alarms-ddb/README.md index b9c3b0c12..e23ba4734 100644 --- a/infrastructure/terraform/modules/alarms-ddb/README.md +++ b/infrastructure/terraform/modules/alarms-ddb/README.md @@ -12,11 +12,12 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [alarm\_prefix](#input\_alarm\_prefix) | n/a | `string` | n/a | yes | -| [evaluation\_periods](#input\_evaluation\_periods) | n/a | `number` | `1` | no | +| [evaluation\_periods](#input\_evaluation\_periods) | n/a | `number` | `2` | no | | [period\_seconds](#input\_period\_seconds) | n/a | `number` | `60` | no | | [read\_throttle\_threshold](#input\_read\_throttle\_threshold) | n/a | `number` | `0` | no | | [table\_name](#input\_table\_name) | n/a | `string` | n/a | yes | | [tags](#input\_tags) | n/a | `map(string)` | `{}` | no | +| [write\_capacity\_threshold](#input\_write\_capacity\_threshold) | n/a | `number` | `1000` | no | | [write\_throttle\_threshold](#input\_write\_throttle\_threshold) | n/a | `number` | `0` | no | ## Modules diff --git a/infrastructure/terraform/modules/alarms-ddb/cloudwatch_metric_alarm_write_capacity.tf b/infrastructure/terraform/modules/alarms-ddb/cloudwatch_metric_alarm_write_capacity.tf new file mode 100644 index 000000000..4a065fd09 --- /dev/null +++ b/infrastructure/terraform/modules/alarms-ddb/cloudwatch_metric_alarm_write_capacity.tf @@ -0,0 +1,22 @@ +resource "aws_cloudwatch_metric_alarm" "write_capacity" { + alarm_name = "${var.alarm_prefix}-ddb-${var.table_name}-write-capacity" + alarm_description = "RELIABILITY: DynamoDB consumed write capacity approaching limit" + + namespace = "AWS/DynamoDB" + metric_name = "ConsumedWriteCapacityUnits" + statistic = "Sum" + period = var.period_seconds + + evaluation_periods = var.evaluation_periods + threshold = var.write_capacity_threshold + comparison_operator = "GreaterThanThreshold" + treat_missing_data = "notBreaching" + + dimensions = { TableName = var.table_name } + + actions_enabled = false + alarm_actions = [] + ok_actions = [] + insufficient_data_actions = [] + tags = var.tags +} diff --git a/infrastructure/terraform/modules/alarms-ddb/variables.tf b/infrastructure/terraform/modules/alarms-ddb/variables.tf index 3895d21eb..cc9473fa3 100644 --- a/infrastructure/terraform/modules/alarms-ddb/variables.tf +++ b/infrastructure/terraform/modules/alarms-ddb/variables.tf @@ -18,7 +18,7 @@ variable "period_seconds" { variable "evaluation_periods" { type = number - default = 1 + default = 2 } variable "read_throttle_threshold" { @@ -30,3 +30,8 @@ variable "write_throttle_threshold" { type = number default = 0 } + +variable "write_capacity_threshold" { + type = number + default = 1000 # temporary value restore to 48000 80% of 60k/min DynamoDB limit +} From 7ceb46b22f8dc177cf020e11fc6636ae5ea53010 Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Tue, 25 Aug 2026 10:44:12 +0100 Subject: [PATCH 2/5] change write throttle alarm to a 3 in 5 --- .../components/api/module_ddb_alarms_letter_queue.tf | 3 +++ .../terraform/modules/alarms-ddb/README.md | 4 +++- .../cloudwatch_metric_alarm_write_throttle.tf | 3 ++- .../terraform/modules/alarms-ddb/variables.tf | 12 +++++++++++- 4 files changed, 19 insertions(+), 3 deletions(-) diff --git a/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf b/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf index 212b7d9ca..f099b6c54 100644 --- a/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf +++ b/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf @@ -4,4 +4,7 @@ module "ddb_alarms_letter_queue" { alarm_prefix = local.csi table_name = aws_dynamodb_table.letter_queue.name tags = local.default_tags + + write_throttle_evaluation_periods = 5 + write_throttle_datapoints_to_alarm = 3 } diff --git a/infrastructure/terraform/modules/alarms-ddb/README.md b/infrastructure/terraform/modules/alarms-ddb/README.md index e23ba4734..5ef6a72dd 100644 --- a/infrastructure/terraform/modules/alarms-ddb/README.md +++ b/infrastructure/terraform/modules/alarms-ddb/README.md @@ -12,12 +12,14 @@ | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [alarm\_prefix](#input\_alarm\_prefix) | n/a | `string` | n/a | yes | -| [evaluation\_periods](#input\_evaluation\_periods) | n/a | `number` | `2` | no | +| [evaluation\_periods](#input\_evaluation\_periods) | n/a | `number` | `1` | no | | [period\_seconds](#input\_period\_seconds) | n/a | `number` | `60` | no | | [read\_throttle\_threshold](#input\_read\_throttle\_threshold) | n/a | `number` | `0` | no | | [table\_name](#input\_table\_name) | n/a | `string` | n/a | yes | | [tags](#input\_tags) | n/a | `map(string)` | `{}` | no | | [write\_capacity\_threshold](#input\_write\_capacity\_threshold) | n/a | `number` | `1000` | no | +| [write\_throttle\_datapoints\_to\_alarm](#input\_write\_throttle\_datapoints\_to\_alarm) | n/a | `number` | `1` | no | +| [write\_throttle\_evaluation\_periods](#input\_write\_throttle\_evaluation\_periods) | n/a | `number` | `1` | no | | [write\_throttle\_threshold](#input\_write\_throttle\_threshold) | n/a | `number` | `0` | no | ## Modules diff --git a/infrastructure/terraform/modules/alarms-ddb/cloudwatch_metric_alarm_write_throttle.tf b/infrastructure/terraform/modules/alarms-ddb/cloudwatch_metric_alarm_write_throttle.tf index 8975e1efa..3ec913acd 100644 --- a/infrastructure/terraform/modules/alarms-ddb/cloudwatch_metric_alarm_write_throttle.tf +++ b/infrastructure/terraform/modules/alarms-ddb/cloudwatch_metric_alarm_write_throttle.tf @@ -7,7 +7,8 @@ resource "aws_cloudwatch_metric_alarm" "write_throttle" { statistic = "Sum" period = var.period_seconds - evaluation_periods = var.evaluation_periods + evaluation_periods = var.write_throttle_evaluation_periods + datapoints_to_alarm = var.write_throttle_datapoints_to_alarm threshold = var.write_throttle_threshold comparison_operator = "GreaterThanThreshold" treat_missing_data = "notBreaching" diff --git a/infrastructure/terraform/modules/alarms-ddb/variables.tf b/infrastructure/terraform/modules/alarms-ddb/variables.tf index cc9473fa3..cc3351ae8 100644 --- a/infrastructure/terraform/modules/alarms-ddb/variables.tf +++ b/infrastructure/terraform/modules/alarms-ddb/variables.tf @@ -18,7 +18,17 @@ variable "period_seconds" { variable "evaluation_periods" { type = number - default = 2 + default = 1 +} + +variable "write_throttle_evaluation_periods" { + type = number + default = 1 +} + +variable "write_throttle_datapoints_to_alarm" { + type = number + default = 1 } variable "read_throttle_threshold" { From 4caaa51d2fcdb934c7e8882a133d5bf959570f1c Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Wed, 26 Aug 2026 11:12:02 +0100 Subject: [PATCH 3/5] make threshold really lower threshold to see if we can trigger --- .../components/api/module_ddb_alarms_letter_queue.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf b/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf index f099b6c54..eec37cf44 100644 --- a/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf +++ b/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf @@ -5,6 +5,6 @@ module "ddb_alarms_letter_queue" { table_name = aws_dynamodb_table.letter_queue.name tags = local.default_tags - write_throttle_evaluation_periods = 5 - write_throttle_datapoints_to_alarm = 3 + write_throttle_evaluation_periods = 1 # should be 5 + write_throttle_datapoints_to_alarm = 1 # should be 3 } From 3aab7a5014116da2e05d1a906077f3835901b5bf Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 27 Aug 2026 12:55:45 +0100 Subject: [PATCH 4/5] add temporarily enable only the letter queue alarams for testing --- infrastructure/terraform/components/api/README.md | 1 + .../components/api/module_ddb_alarms_letter_queue.tf | 2 +- infrastructure/terraform/components/api/variables.tf | 7 +++++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index b269caeb3..3c24fc19f 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -54,6 +54,7 @@ No requirements. | [shared\_infra\_account\_id](#input\_shared\_infra\_account\_id) | The AWS Account ID of the shared infrastructure account | `string` | `"000000000000"` | no | | [sns\_success\_logging\_sample\_percent](#input\_sns\_success\_logging\_sample\_percent) | Enable SNS Delivery Successful Sample Percentage | `number` | `0` | no | | [supplier\_mock\_schedule\_expression](#input\_supplier\_mock\_schedule\_expression) | Schedule expression for supplier mock scheduler | `string` | `"rate(1 minute)"` | no | +| [test\_alarms\_enabled](#input\_test\_alarms\_enabled) | Enable letter queue alarms for testing when account-wide alarms are disabled | `bool` | `true` | no | ## Modules | Name | Source | Version | diff --git a/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf b/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf index eec37cf44..2f209f1ea 100644 --- a/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf +++ b/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf @@ -1,5 +1,5 @@ module "ddb_alarms_letter_queue" { - count = local.alarms_enabled ? 1 : 0 + count = local.alarms_enabled || var.test_alarms_enabled ? 1 : 0 source = "../../modules/alarms-ddb" alarm_prefix = local.csi table_name = aws_dynamodb_table.letter_queue.name diff --git a/infrastructure/terraform/components/api/variables.tf b/infrastructure/terraform/components/api/variables.tf index 3d16b8465..b7d0c0988 100644 --- a/infrastructure/terraform/components/api/variables.tf +++ b/infrastructure/terraform/components/api/variables.tf @@ -226,6 +226,13 @@ variable "enable_alarms" { description = "Enable CloudWatch alarms for this deployed environment" default = true } + +variable "test_alarms_enabled" { + type = bool + description = "Enable letter queue alarms for testing when account-wide alarms are disabled" + default = true +} + variable "deploy_supplier_mock_scheduler" { type = bool description = "Deploy EventBridge Scheduler trigger for supplier mock lambda" From c8b5d3dac0e79c209f1aea2433d70c46eecf2e3c Mon Sep 17 00:00:00 2001 From: "ross.faulds2" Date: Thu, 27 Aug 2026 13:46:40 +0100 Subject: [PATCH 5/5] removed test variable and reinstated correct values for alarm --- infrastructure/terraform/components/api/README.md | 1 - .../components/api/module_ddb_alarms_letter_queue.tf | 6 +++--- infrastructure/terraform/components/api/variables.tf | 6 ------ infrastructure/terraform/modules/alarms-ddb/README.md | 2 +- infrastructure/terraform/modules/alarms-ddb/variables.tf | 2 +- 5 files changed, 5 insertions(+), 12 deletions(-) diff --git a/infrastructure/terraform/components/api/README.md b/infrastructure/terraform/components/api/README.md index 3c24fc19f..b269caeb3 100644 --- a/infrastructure/terraform/components/api/README.md +++ b/infrastructure/terraform/components/api/README.md @@ -54,7 +54,6 @@ No requirements. | [shared\_infra\_account\_id](#input\_shared\_infra\_account\_id) | The AWS Account ID of the shared infrastructure account | `string` | `"000000000000"` | no | | [sns\_success\_logging\_sample\_percent](#input\_sns\_success\_logging\_sample\_percent) | Enable SNS Delivery Successful Sample Percentage | `number` | `0` | no | | [supplier\_mock\_schedule\_expression](#input\_supplier\_mock\_schedule\_expression) | Schedule expression for supplier mock scheduler | `string` | `"rate(1 minute)"` | no | -| [test\_alarms\_enabled](#input\_test\_alarms\_enabled) | Enable letter queue alarms for testing when account-wide alarms are disabled | `bool` | `true` | no | ## Modules | Name | Source | Version | diff --git a/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf b/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf index 2f209f1ea..f099b6c54 100644 --- a/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf +++ b/infrastructure/terraform/components/api/module_ddb_alarms_letter_queue.tf @@ -1,10 +1,10 @@ module "ddb_alarms_letter_queue" { - count = local.alarms_enabled || var.test_alarms_enabled ? 1 : 0 + count = local.alarms_enabled ? 1 : 0 source = "../../modules/alarms-ddb" alarm_prefix = local.csi table_name = aws_dynamodb_table.letter_queue.name tags = local.default_tags - write_throttle_evaluation_periods = 1 # should be 5 - write_throttle_datapoints_to_alarm = 1 # should be 3 + write_throttle_evaluation_periods = 5 + write_throttle_datapoints_to_alarm = 3 } diff --git a/infrastructure/terraform/components/api/variables.tf b/infrastructure/terraform/components/api/variables.tf index b7d0c0988..8782cc428 100644 --- a/infrastructure/terraform/components/api/variables.tf +++ b/infrastructure/terraform/components/api/variables.tf @@ -227,12 +227,6 @@ variable "enable_alarms" { default = true } -variable "test_alarms_enabled" { - type = bool - description = "Enable letter queue alarms for testing when account-wide alarms are disabled" - default = true -} - variable "deploy_supplier_mock_scheduler" { type = bool description = "Deploy EventBridge Scheduler trigger for supplier mock lambda" diff --git a/infrastructure/terraform/modules/alarms-ddb/README.md b/infrastructure/terraform/modules/alarms-ddb/README.md index 5ef6a72dd..ed912741c 100644 --- a/infrastructure/terraform/modules/alarms-ddb/README.md +++ b/infrastructure/terraform/modules/alarms-ddb/README.md @@ -17,7 +17,7 @@ | [read\_throttle\_threshold](#input\_read\_throttle\_threshold) | n/a | `number` | `0` | no | | [table\_name](#input\_table\_name) | n/a | `string` | n/a | yes | | [tags](#input\_tags) | n/a | `map(string)` | `{}` | no | -| [write\_capacity\_threshold](#input\_write\_capacity\_threshold) | n/a | `number` | `1000` | no | +| [write\_capacity\_threshold](#input\_write\_capacity\_threshold) | n/a | `number` | `48000` | no | | [write\_throttle\_datapoints\_to\_alarm](#input\_write\_throttle\_datapoints\_to\_alarm) | n/a | `number` | `1` | no | | [write\_throttle\_evaluation\_periods](#input\_write\_throttle\_evaluation\_periods) | n/a | `number` | `1` | no | | [write\_throttle\_threshold](#input\_write\_throttle\_threshold) | n/a | `number` | `0` | no | diff --git a/infrastructure/terraform/modules/alarms-ddb/variables.tf b/infrastructure/terraform/modules/alarms-ddb/variables.tf index cc3351ae8..2534bc678 100644 --- a/infrastructure/terraform/modules/alarms-ddb/variables.tf +++ b/infrastructure/terraform/modules/alarms-ddb/variables.tf @@ -43,5 +43,5 @@ variable "write_throttle_threshold" { variable "write_capacity_threshold" { type = number - default = 1000 # temporary value restore to 48000 80% of 60k/min DynamoDB limit + default = 48000 # 80% of 60k/min DynamoDB limit }