From 7343e1936485318e1e0cf5c538fd1fbcc6588689 Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Mon, 23 Feb 2026 12:11:43 +0000 Subject: [PATCH 1/6] Adopt existing conventions for new columns --- ...ll_resource_request_item_plan_status_table.tf | 4 ++-- .../sql/ingestion/request_item_plan_status.sql | 16 ++++++++-------- .../sql/tables/request_item_plan_status.sql | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf index d4ac9881..1c74729e 100644 --- a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf +++ b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf @@ -106,7 +106,7 @@ resource "null_resource" "request_item_plan_status_specificationid_column" { EOT } - depends_on = [null_resource.request_item_plan_status_templateid_column] + depends_on = [null_resource.request_item_plan_status_failedreasoncode_column] } resource "null_resource" "request_item_plan_status_specificationbillingid_column" { @@ -122,5 +122,5 @@ resource "null_resource" "request_item_plan_status_specificationbillingid_column EOT } - depends_on = [null_resource.request_item_plan_status_templateid_column] + depends_on = [null_resource.request_item_plan_status_specificationid_column] } diff --git a/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql b/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql index e371221a..109d5372 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql @@ -35,9 +35,9 @@ USING ( CAST(CAST(templates.suppliers AS json) AS map)[LOWER(supplier)] ) AS templateid, failedreasoncode, - CAST("$classification".timestamp AS BIGINT) AS timestamp, specificationid, - specificationbillingid + specificationbillingid, + CAST("$classification".timestamp AS BIGINT) AS timestamp FROM ${source_table} WHERE (sk LIKE 'REQUEST_ITEM_PLAN#%') AND ( @@ -72,9 +72,9 @@ WHEN MATCHED AND (source.timestamp > target.timestamp) THEN UPDATE SET recipientcontactid = source.recipientcontactid, templateid = source.templateid, failedreasoncode = source.failedreasoncode, - timestamp = source.timestamp, specificationid = source.specificationid, - specificationbillingid = source.specificationbillingid + specificationbillingid = source.specificationbillingid, + timestamp = source.timestamp WHEN NOT MATCHED THEN INSERT ( clientid, campaignid, @@ -98,9 +98,9 @@ WHEN NOT MATCHED THEN INSERT ( recipientcontactid, templateid, failedreasoncode, - timestamp, specificationid, - specificationbillingid + specificationbillingid, + timestamp ) VALUES ( source.clientid, @@ -125,7 +125,7 @@ VALUES ( source.recipientcontactid, source.templateid, source.failedreasoncode, - source.timestamp, source.specificationid, - source.specificationbillingid + source.specificationbillingid, + source.timestamp ) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql index d3aa14c2..5d2f1266 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql @@ -20,9 +20,9 @@ CREATE TABLE IF NOT EXISTS ${table_name} ( ordernumber int, recipientcontactid string, templateid string, - timestamp bigint, specificationid string, - specificationbillingid string + specificationbillingid string, + timestamp bigint ) PARTITIONED BY (bucket(32, clientid), month(createdtime), month(completedtime)) LOCATION '${s3_location}' From a77c4f8035a1d94c7135683a652c4a8b963ff95d Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Mon, 23 Feb 2026 12:19:57 +0000 Subject: [PATCH 2/6] Add message length fields for billing purposes --- ...resource_request_item_plan_status_table.tf | 32 +++++++++++++++++++ .../ingestion/request_item_plan_status.sql | 8 +++++ .../sql/tables/request_item_plan_status.sql | 2 ++ 3 files changed, 42 insertions(+) diff --git a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf index 1c74729e..428285de 100644 --- a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf +++ b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf @@ -124,3 +124,35 @@ resource "null_resource" "request_item_plan_status_specificationbillingid_column depends_on = [null_resource.request_item_plan_status_specificationid_column] } + +resource "null_resource" "request_item_plan_status_messagelength_column" { + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = < target.timestamp) THEN UPDATE SET failedreasoncode = source.failedreasoncode, specificationid = source.specificationid, specificationbillingid = source.specificationbillingid, + messagelength = source.messagelength, + messagelengthunits = source.messagelengthunits, timestamp = source.timestamp WHEN NOT MATCHED THEN INSERT ( clientid, @@ -100,6 +104,8 @@ WHEN NOT MATCHED THEN INSERT ( failedreasoncode, specificationid, specificationbillingid, + messagelength, + messagelengthunits, timestamp ) VALUES ( @@ -127,5 +133,7 @@ VALUES ( source.failedreasoncode, source.specificationid, source.specificationbillingid, + source.messagelength, + source.messagelengthunits, source.timestamp ) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql index 5d2f1266..9d591bfb 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql @@ -22,6 +22,8 @@ CREATE TABLE IF NOT EXISTS ${table_name} ( templateid string, specificationid string, specificationbillingid string, + messagelength int, + messagelengthunits string, timestamp bigint ) PARTITIONED BY (bucket(32, clientid), month(createdtime), month(completedtime)) From c1fbf1ac1b47f5c851f95d34d36c0688e16effd9 Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Mon, 23 Feb 2026 12:41:05 +0000 Subject: [PATCH 3/6] Fix incorrect data type --- .../reporting/null_resource_request_item_plan_status_table.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf index 428285de..63ea667f 100644 --- a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf +++ b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf @@ -134,7 +134,7 @@ resource "null_resource" "request_item_plan_status_messagelength_column" { ${path.module}/scripts/add_column.sh \ ${aws_athena_workgroup.setup.name} \ ${aws_glue_catalog_database.reporting.name} \ - request_item_plan_status messagelength string + request_item_plan_status messagelength int EOT } From e8243f339efd1a724597e91740abd534bd4d2650 Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Mon, 23 Feb 2026 15:49:22 +0000 Subject: [PATCH 4/6] Add senderodscode column --- ...ll_resource_request_item_plan_status_table.tf | 16 ++++++++++++++++ .../sql/ingestion/request_item_plan_status.sql | 4 ++++ .../sql/tables/request_item_plan_status.sql | 1 + 3 files changed, 21 insertions(+) diff --git a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf index 63ea667f..176c60c7 100644 --- a/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf +++ b/infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf @@ -156,3 +156,19 @@ resource "null_resource" "request_item_plan_status_messagelengthunits_column" { depends_on = [null_resource.request_item_plan_status_messagelength_column] } + +resource "null_resource" "request_item_plan_status_senderodscode_column" { + triggers = { + always_run = timestamp() + } + provisioner "local-exec" { + command = < target.timestamp) THEN UPDATE SET specificationbillingid = source.specificationbillingid, messagelength = source.messagelength, messagelengthunits = source.messagelengthunits, + senderodscode = source.senderodscode, timestamp = source.timestamp WHEN NOT MATCHED THEN INSERT ( clientid, @@ -106,6 +108,7 @@ WHEN NOT MATCHED THEN INSERT ( specificationbillingid, messagelength, messagelengthunits, + senderodscode, timestamp ) VALUES ( @@ -135,5 +138,6 @@ VALUES ( source.specificationbillingid, source.messagelength, source.messagelengthunits, + source.senderodscode, source.timestamp ) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql index 9d591bfb..36470063 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql @@ -24,6 +24,7 @@ CREATE TABLE IF NOT EXISTS ${table_name} ( specificationbillingid string, messagelength int, messagelengthunits string, + senderodscode string, timestamp bigint ) PARTITIONED BY (bucket(32, clientid), month(createdtime), month(completedtime)) From 1cf1e7fb287cee24175c62a400130cc26f32764e Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Thu, 26 Feb 2026 12:28:35 +0000 Subject: [PATCH 5/6] Small performance tweak --- .../reporting/scripts/sql/reports/pds_cleared_failures.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/infrastructure/terraform/components/reporting/scripts/sql/reports/pds_cleared_failures.sql b/infrastructure/terraform/components/reporting/scripts/sql/reports/pds_cleared_failures.sql index cb1e298d..cc431e65 100644 --- a/infrastructure/terraform/components/reporting/scripts/sql/reports/pds_cleared_failures.sql +++ b/infrastructure/terraform/components/reporting/scripts/sql/reports/pds_cleared_failures.sql @@ -4,7 +4,8 @@ FROM ( LAG(ri.requestitemid,1) OVER (PARTITION BY ri.nhsnumberhash, rip.communicationtype ORDER BY ri.createdtime) AS firstrequestitemid FROM request_item_status ri INNER JOIN request_item_plan_status rip ON - rip.requestitemid = ri.requestitemid + rip.requestitemid = ri.requestitemid AND + rip.clientid = ri.clientid WHERE (rip.communicationtype = 'EMAIL' OR rip.communicationtype = 'SMS') ) AS tx INNER JOIN request_item_plan_status rip_success ON From 78c157c442c45c20570751bc26a2350f1658c384 Mon Sep 17 00:00:00 2001 From: Mark Pullan Date: Mon, 2 Mar 2026 09:44:11 +0000 Subject: [PATCH 6/6] Fix secrets fingerprint --- .gitleaksignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitleaksignore b/.gitleaksignore index ec397aad..c1bc3735 100644 --- a/.gitleaksignore +++ b/.gitleaksignore @@ -6,5 +6,5 @@ cd9c0efec38c5d63053dd865e5d4e207c0760d91:docs/guides/Perform_static_analysis.md: e2fd9d99d561c274cfd11b85a99a0cb1ae00f97d:infrastructure/terraform/components/reporting/templates/cloudinit_config.ps1:ipv4:28 8cd75c04961cd6ce8477b7dad9c6daa67f03cd19:infrastructure/terraform/components/reporting/templates/cloudinit_config.tmpl:ipv4:32 9b7b5b4539e13109e15c474684911b5a349c1b6d:infrastructure/terraform/components/reporting/scripts/sql/views/request_item_status_summary_all_emailfilter.sql:generic-api-key:12 -e4f41f458ca66d94f1fcebdff9579e2ce81d1d5e:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29 -e4f41f458ca66d94f1fcebdff9579e2ce81d1d5e:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43 +d38af4e4f6c36ca9c3d843193b434386a9bad5ee:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29 +d38af4e4f6c36ca9c3d843193b434386a9bad5ee:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43