CCM-14728 ingest billing fields

.gitleaksignore

@@ -6,5 +6,5 @@ cd9c0efec38c5d63053dd865e5d4e207c0760d91:docs/guides/Perform_static_analysis.md:
 e2fd9d99d561c274cfd11b85a99a0cb1ae00f97d:infrastructure/terraform/components/reporting/templates/cloudinit_config.ps1:ipv4:28
 8cd75c04961cd6ce8477b7dad9c6daa67f03cd19:infrastructure/terraform/components/reporting/templates/cloudinit_config.tmpl:ipv4:32
 9b7b5b4539e13109e15c474684911b5a349c1b6d:infrastructure/terraform/components/reporting/scripts/sql/views/request_item_status_summary_all_emailfilter.sql:generic-api-key:12
-e4f41f458ca66d94f1fcebdff9579e2ce81d1d5e:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29
-e4f41f458ca66d94f1fcebdff9579e2ce81d1d5e:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43
+d38af4e4f6c36ca9c3d843193b434386a9bad5ee:infrastructure/terraform/etc/env_eu-west-2_int.tfvars:generic-api-key:29
+d38af4e4f6c36ca9c3d843193b434386a9bad5ee:infrastructure/terraform/etc/env_eu-west-2_prod.tfvars:generic-api-key:43

infrastructure/terraform/components/reporting/null_resource_request_item_plan_status_table.tf

@@ -106,7 +106,7 @@ resource "null_resource" "request_item_plan_status_specificationid_column" {
     EOT
   }
 
-  depends_on = [null_resource.request_item_plan_status_templateid_column]
+  depends_on = [null_resource.request_item_plan_status_failedreasoncode_column]
 }
 
 resource "null_resource" "request_item_plan_status_specificationbillingid_column" {
@@ -122,5 +122,53 @@ resource "null_resource" "request_item_plan_status_specificationbillingid_column
     EOT
   }
 
-  depends_on = [null_resource.request_item_plan_status_templateid_column]
+  depends_on = [null_resource.request_item_plan_status_specificationid_column]
+}
+
+resource "null_resource" "request_item_plan_status_messagelength_column" {
+  triggers = {
+    always_run = timestamp()
+  }
+  provisioner "local-exec" {
+    command = <<EOT
+      ${path.module}/scripts/add_column.sh \
+        ${aws_athena_workgroup.setup.name} \
+        ${aws_glue_catalog_database.reporting.name} \
+        request_item_plan_status messagelength int
+    EOT
+  }
+
+  depends_on = [null_resource.request_item_plan_status_specificationbillingid_column]
+}
+
+resource "null_resource" "request_item_plan_status_messagelengthunits_column" {
+  triggers = {
+    always_run = timestamp()
+  }
+  provisioner "local-exec" {
+    command = <<EOT
+      ${path.module}/scripts/add_column.sh \
+        ${aws_athena_workgroup.setup.name} \
+        ${aws_glue_catalog_database.reporting.name} \
+        request_item_plan_status messagelengthunits string
+    EOT
+  }
+
+  depends_on = [null_resource.request_item_plan_status_messagelength_column]
+}
+
+resource "null_resource" "request_item_plan_status_senderodscode_column" {
+  triggers = {
+    always_run = timestamp()
+  }
+  provisioner "local-exec" {
+    command = <<EOT
+      ${path.module}/scripts/add_column.sh \
+        ${aws_athena_workgroup.setup.name} \
+        ${aws_glue_catalog_database.reporting.name} \
+        request_item_plan_status senderodscode string
+    EOT
+  }
+
+  depends_on = [null_resource.request_item_plan_status_messagelengthunits_column]
 }

infrastructure/terraform/components/reporting/scripts/sql/ingestion/request_item_plan_status.sql

@@ -35,9 +35,12 @@ USING (
           CAST(CAST(templates.suppliers AS json) AS map<varchar, varchar>)[LOWER(supplier)]
         ) AS templateid,
         failedreasoncode,
-        CAST("$classification".timestamp AS BIGINT) AS timestamp,
         specificationid,
-        specificationbillingid
+        specificationbillingid,
+        messagelength,
+        messagelengthunits,
+        senderodscode,
+        CAST("$classification".timestamp AS BIGINT) AS timestamp
       FROM ${source_table}
       WHERE (sk LIKE 'REQUEST_ITEM_PLAN#%') AND
       (
@@ -72,9 +75,12 @@ WHEN MATCHED AND (source.timestamp > target.timestamp) THEN UPDATE SET
   recipientcontactid =  source.recipientcontactid,
   templateid = source.templateid,
   failedreasoncode = source.failedreasoncode,
-  timestamp = source.timestamp,
   specificationid = source.specificationid,
-  specificationbillingid = source.specificationbillingid
+  specificationbillingid = source.specificationbillingid,
+  messagelength = source.messagelength,
+  messagelengthunits = source.messagelengthunits,
+  senderodscode = source.senderodscode,
+  timestamp = source.timestamp
 WHEN NOT MATCHED THEN INSERT (
   clientid,
   campaignid,
@@ -98,9 +104,12 @@ WHEN NOT MATCHED THEN INSERT (
   recipientcontactid,
   templateid,
   failedreasoncode,
-  timestamp,
   specificationid,
-  specificationbillingid
+  specificationbillingid,
+  messagelength,
+  messagelengthunits,
+  senderodscode,
+  timestamp
 )
 VALUES (
   source.clientid,
@@ -125,7 +134,10 @@ VALUES (
   source.recipientcontactid,
   source.templateid,
   source.failedreasoncode,
-  source.timestamp,
   source.specificationid,
-  source.specificationbillingid
+  source.specificationbillingid,
+  source.messagelength,
+  source.messagelengthunits,
+  source.senderodscode,
+  source.timestamp
 )

infrastructure/terraform/components/reporting/scripts/sql/reports/pds_cleared_failures.sql

@@ -4,7 +4,8 @@ FROM (
       LAG(ri.requestitemid,1) OVER (PARTITION BY ri.nhsnumberhash, rip.communicationtype ORDER BY ri.createdtime) AS firstrequestitemid
     FROM request_item_status ri
     INNER JOIN request_item_plan_status rip ON
-      rip.requestitemid = ri.requestitemid
+      rip.requestitemid = ri.requestitemid AND
+      rip.clientid = ri.clientid
     WHERE (rip.communicationtype = 'EMAIL' OR rip.communicationtype = 'SMS')
 ) AS tx
 INNER JOIN request_item_plan_status rip_success ON

infrastructure/terraform/components/reporting/scripts/sql/tables/request_item_plan_status.sql

@@ -20,9 +20,12 @@ CREATE TABLE IF NOT EXISTS ${table_name} (
     ordernumber int,
     recipientcontactid string,
     templateid string,
-    timestamp bigint,
     specificationid string,
-    specificationbillingid string
+    specificationbillingid string,
+    messagelength int,
+    messagelengthunits string,
+    senderodscode string,
+    timestamp bigint
 )
 PARTITIONED BY (bucket(32, clientid), month(createdtime), month(completedtime))
 LOCATION '${s3_location}'