Re-engineered the interaction data population

nhsconnect · Jun 24, 2024 · ad7e0c1 · ad7e0c1
1 parent 81c7fdc
commit ad7e0c1
Show file tree

Hide file tree

Showing 17 changed files with 123 additions and 134 deletions.
diff --git a/database/functions/R__reporting.add_transient_data.sql b/database/functions/R__reporting.add_transient_data.sql
@@ -10,7 +10,7 @@ create function reporting.add_transient_data
 returns void
 as $$
 begin	
-	insert into reporting.transient(transient_id, transient_data, transient_report_id, transient_report_name)
-	values (_transient_id, _transient_data, _transient_report_id, _transient_report_name);
+	insert into reporting.transient(transient_id, transient_data, transient_report_id, transient_report_name, entry_date)
+	values (_transient_id, _transient_data, _transient_report_id, _transient_report_name, now());
 end;
 $$ language plpgsql;
diff --git a/database/functions/R__reporting.get_transient_data.sql b/database/functions/R__reporting.get_transient_data.sql
@@ -0,0 +1,29 @@
+drop function if exists reporting.get_transient_data;
+
+create function reporting.get_transient_data
+(
+	_transient_report_id varchar(100)
+)
+returns table
+(
+	transient_id varchar(100), 
+	transient_data json,
+	transient_report_id varchar(100),
+	transient_report_name varchar(100)
+)
+as $$
+begin	
+	return query
+	select
+		t.transient_id,
+		t.transient_data,
+		t.transient_report_id,
+		t.transient_report_name
+	from 
+		reporting.transient t
+	where 
+		t.transient_report_id = _transient_report_id
+		and entry_date - now() = 0;
+end;
+$$ language plpgsql;
+
diff --git a/database/functions/R__reporting.truncate_transient_data.sql b/database/functions/R__reporting.truncate_transient_data.sql
diff --git a/database/schema/V6.5__schema.sql b/database/schema/V6.5__schema.sql
@@ -4,7 +4,8 @@ create table reporting.transient
     transient_id varchar(100) not null,
     transient_data json not null,
 	transient_report_id varchar(100) not null,
-	transient_report_name varchar(100) not null
+	transient_report_name varchar(100) not null,
+	entry_date timestamp without time zone not null
 );
 
 grant select, insert, update, delete on table reporting.transient to app_user;
diff --git a/modules/api/src/Controllers/ReportingController.cs b/modules/api/src/Controllers/ReportingController.cs
@@ -66,13 +66,6 @@ public async Task<ActionResult> List()
         return Ok(reports);
     }
 
-    [HttpGet("truncateinteractionreportdata")]
-    public async Task<ActionResult> TruncateInteractionReportData()
-    {
-        await _service.TruncateInteractionReportData();
-        return Ok();
-    }
-
     [HttpGet("capabilitylist")]
     public async Task<ActionResult> CapabilityList()
     {

diff --git a/modules/api/src/Core/Mapping/MappingExtensions.cs b/modules/api/src/Core/Mapping/MappingExtensions.cs
@@ -26,6 +26,7 @@ public static void ConfigureMappingServices()
             config.AddMap(new ReportMap());
             config.AddMap(new CapabilityReportMap());
             config.AddMap(new SpineMessageMap());
+            config.AddMap(new TransientDataMap());
         });
     }
 }
diff --git a/modules/api/src/DAL/Mapping/SpineMap.cs b/modules/api/src/DAL/Mapping/SpineMap.cs
diff --git a/modules/api/src/DAL/Mapping/TransientDataMap.cs b/modules/api/src/DAL/Mapping/TransientDataMap.cs
@@ -0,0 +1,15 @@
+using Dapper.FluentMap.Mapping;
+using GpConnect.AppointmentChecker.Api.DTO.Response.Reporting;
+
+namespace GpConnect.AppointmentChecker.Api.DAL.Mapping;
+
+public class TransientDataMap : EntityMap<TransientData>
+{
+    public TransientDataMap()
+    {
+        Map(p => p.Id).ToColumn("transient_id");
+        Map(p => p.Data).ToColumn("transient_data");
+        Map(p => p.ReportId).ToColumn("transient_report_id");
+        Map(p => p.ReportName).ToColumn("transient_report_name");
+    }
+}
diff --git a/modules/api/src/DTO/Request/ReportCreationRequest.cs b/modules/api/src/DTO/Request/ReportCreationRequest.cs
@@ -2,7 +2,7 @@ namespace GpConnect.AppointmentChecker.Api.DTO.Request;
 
 public class ReportCreationRequest
 {
-    public string JsonData { get; set; }
     public string ReportName { get; set; }
+    public string ReportId { get; set; }
     public List<ReportFilterRequest>? ReportFilter { get; set; } = null;
 }
diff --git a/modules/api/src/DTO/Response/Reporting/TransientData.cs b/modules/api/src/DTO/Response/Reporting/TransientData.cs
@@ -0,0 +1,9 @@
+namespace GpConnect.AppointmentChecker.Api.DTO.Response.Reporting;
+
+public class TransientData
+{
+    public string Id { get; set; }
+    public string Data { get; set; }
+    public string ReportId { get; set; }
+    public string ReportName { get; set; }
+}
diff --git a/modules/api/src/Service/Interfaces/IReportingService.cs b/modules/api/src/Service/Interfaces/IReportingService.cs
@@ -7,10 +7,9 @@ namespace GpConnect.AppointmentChecker.Api.Service.Interfaces;
 public interface IReportingService
 {
     public Task<string> GetReport(string functionName);
-    public Task<Stream> ExportReport(ReportRequest reportRequest);
+    public Task<Stream?> ExportReport(ReportRequest reportRequest);
     public Task<Stream> CreateInteractionReport(ReportCreationRequest reportCreationRequest);
     public Task RouteReportRequest(RouteReportRequest routeReportRequest);
-    public Task TruncateInteractionReportData();
     public Task SendMessageToCreateInteractionReportContent(ReportInteractionRequest reportInteractionRequest);
     public Task<List<Report>> GetReports();    
     public Task<List<CapabilityReport>> GetCapabilityReports();

diff --git a/modules/api/src/Service/ReportingService.cs b/modules/api/src/Service/ReportingService.cs
@@ -54,9 +54,17 @@ public async Task<Stream> ExportReport(ReportRequest reportRequest)
         return memoryStream;
     }
 
-    public async Task<Stream> CreateInteractionReport(ReportCreationRequest reportCreationRequest)
+    public async Task<Stream?> CreateInteractionReport(ReportCreationRequest reportCreationRequest)
     {
-        return CreateReport(reportCreationRequest.JsonData.ConvertJsonDataToDataTable(), reportCreationRequest.ReportName, reportCreationRequest.ReportFilter);
+        var functionName = "reporting.get_transient_data";
+        var parameters = new DynamicParameters();
+        parameters.Add("_transient_report_id", reportCreationRequest.ReportId, DbType.String, ParameterDirection.Input);
+        var response = await _dataService.ExecuteQueryFirstOrDefault<TransientData>(functionName, parameters);
+        if(response.Data != null)
+        {
+            return CreateReport(response.Data.ConvertJsonDataToDataTable(), reportCreationRequest.ReportName, reportCreationRequest.ReportFilter);
+        }
+        return null;
     }
 
     public async Task RouteReportRequest(RouteReportRequest routeReportRequest)
@@ -94,11 +102,6 @@ public async Task RouteReportRequest(RouteReportRequest routeReportRequest)
         }
     }
 
-    public async Task TruncateInteractionReportData()
-    {
-        await _dataService.ExecuteQuery("reporting.truncate_transient_data");
-    }
-
     private async Task CreateTransientData(string transientData, RouteReportRequest routeReportRequest)
     {
         if (transientData != null)

diff --git a/modules/function/src/CapabilityReportEventFunction.cs b/modules/function/src/CapabilityReportEventFunction.cs
@@ -202,11 +202,5 @@ private async Task Reset(params string[] objectPrefix)
 
             response?.DeletedObjects.ForEach(x => _lambdaContext.Logger.LogLine(x.Key));
         }
-        var truncate = await _httpClient.GetWithHeadersAsync("/reporting/truncateinteractionreportdata", new Dictionary<string, string>()
-        {
-            [Headers.UserId] = _endUserConfiguration.UserId,
-            [Headers.ApiKey] = _endUserConfiguration.ApiKey
-        });
-        truncate.EnsureSuccessStatusCode();
     }
 }
diff --git a/modules/function/src/CompletionFunction.cs b/modules/function/src/CompletionFunction.cs
@@ -1,11 +1,10 @@
 using Amazon.Lambda.Core;
-using Amazon.S3.Model;
 using GpConnect.AppointmentChecker.Function.Configuration;
 using GpConnect.AppointmentChecker.Function.DTO.Request;
+using GpConnect.AppointmentChecker.Function.DTO.Response;
 using GpConnect.AppointmentChecker.Function.Helpers;
 using GpConnect.AppointmentChecker.Function.Helpers.Constants;
 using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
 using System.Diagnostics;
 using System.Net;
 using System.Net.Http.Headers;
@@ -58,91 +57,53 @@ public async Task<HttpStatusCode> FunctionHandler(ILambdaContext lambdaContext)
         {
             _reportFilterRequest = completionFunctionRequest.ReportFilter;
             _distributionList = completionFunctionRequest.DistributionList;
-            await BundleUpJsonResponsesAndSendReport();
+            await CreateReport();
             _stopwatch.Stop();
             _lambdaContext.Logger.LogInformation($"CompletionFunction took {_stopwatch.Elapsed:%m} minutes {_stopwatch.Elapsed:%s} seconds to process");
             return HttpStatusCode.OK;
         }
         return HttpStatusCode.BadRequest;
     }
 
-    private async Task BundleUpJsonResponsesAndSendReport()
+    private async Task CreateReport()
     {
         var keyObjects = await StorageManager.GetObjects(new StorageListRequest
         {
             BucketName = _storageConfiguration.BucketName,
             ObjectPrefix = $"{Objects.Key}"
         });
-        
+
         foreach (var keyObject in keyObjects)
         {
-            var responses = new List<string>();
-            var sourceKey = keyObject.Key.SearchAndReplace(new Dictionary<string, string>() { { ".json", string.Empty }, { "key_", string.Empty } });
-
-            var bucketObjects = await StorageManager.GetObjects(new StorageListRequest
-            {
-                BucketName = _storageConfiguration.BucketName,
-                ObjectPrefix = $"{Objects.Transient}_{sourceKey}_{DateTime.Now:yyyy_MM_dd}"
-            });
+            var key = await StorageManager.Get<InteractionKey>(new StorageDownloadRequest { BucketName = _storageConfiguration.BucketName, Key = keyObject.Key });#
 
-            while (bucketObjects != null && bucketObjects.Count > 0)
+            if (key != null)           
             {
-                var deleteRequest = new DeleteObjectsRequest() { BucketName = storageListRequest.BucketName };
-                foreach (S3Object s3Object in listResponse)
+                var reportCreationRequest = new ReportCreationRequest
                 {
-                    deleteRequest.AddKey(s3Object.Key);
-                }
-                var deleteResponse = await s3Client.DeleteObjectsAsync(deleteRequest);
-                if (deleteResponse.HttpStatusCode == HttpStatusCode.OK)
-                {
-                    listResponse = await GetObjects(storageListRequest);
-                }
-                else
-                {
-                    listResponse = null;
-                }
-            }
+                    ReportFilter = _reportFilterRequest,
+                    ReportName = key.ReportName,
+                    ReportId = key.ReportId
+                };
 
-            for ( var i = 0; i < bucketObjects.Count; i++ )
-            {
-                var jsonData = await StorageManager.Get(new StorageDownloadRequest { BucketName = bucketObjects[i].BucketName, Key = bucketObjects[i].Key });
-                responses.Add(jsonData);
-            }
-
-            var responseObject = responses.Select(JArray.Parse).SelectMany(token => token).DistinctBy(x => x["ODS_Code"]).OrderBy(x => x["ODS_Code"]);
-            string combinedJson = JsonConvert.SerializeObject(responseObject, Formatting.Indented);
-
-            var interactionObject = await StorageManager.Get<ReportInteraction>(new StorageDownloadRequest { BucketName = keyObject.BucketName, Key = keyObject.Key });
-            await CreateReport(combinedJson, interactionObject);
-        }        
-    }
+                var json = new StringContent(JsonConvert.SerializeObject(reportCreationRequest, null, _options),
+                       Encoding.UTF8,
+                       MediaTypeHeaderValue.Parse("application/json").MediaType);
 
-    private async Task CreateReport(string jsonData, ReportInteraction interactionObject)
-    {
-        var reportCreationRequest = new ReportCreationRequest
-        {
-            JsonData = jsonData,
-            ReportFilter = _reportFilterRequest,
-            ReportName = interactionObject.ReportName,
-            ReportId = interactionObject.ReportId
-        };
-
-        var json = new StringContent(JsonConvert.SerializeObject(reportCreationRequest, null, _options),
-               Encoding.UTF8,
-               MediaTypeHeaderValue.Parse("application/json").MediaType);
-
-        var response = await _httpClient.PostWithHeadersAsync("/reporting/createinteractionreport", new Dictionary<string, string>()
-        {
-            [Headers.UserId] = _endUserConfiguration.UserId,
-            [Headers.ApiKey] = _endUserConfiguration.ApiKey
-        }, json);        
-        response.EnsureSuccessStatusCode();
+                var response = await _httpClient.PostWithHeadersAsync("/reporting/createinteractionreport", new Dictionary<string, string>()
+                {
+                    [Headers.UserId] = _endUserConfiguration.UserId,
+                    [Headers.ApiKey] = _endUserConfiguration.ApiKey
+                }, json);
+                response.EnsureSuccessStatusCode();
 
-        var fileStream = await response.Content.ReadAsStreamAsync();
-        var byteArray = StreamExtensions.UseBufferedStream(fileStream);
+                var fileStream = await response.Content.ReadAsStreamAsync();
+                var byteArray = StreamExtensions.UseBufferedStream(fileStream);
 
-        reportCreationRequest.ReportBytes = byteArray;
-        await PostReport(reportCreationRequest);
+                reportCreationRequest.ReportBytes = byteArray;
+                await PostReport(reportCreationRequest);
+            }            
+        }        
     }
 
     private async Task PostReport(ReportCreationRequest reportCreationRequest)

diff --git a/modules/function/src/DTO/Request/ReportCreationRequest.cs b/modules/function/src/DTO/Request/ReportCreationRequest.cs
@@ -4,7 +4,6 @@ namespace GpConnect.AppointmentChecker.Function.DTO.Request;
 
 public class ReportCreationRequest
 {
-    public string JsonData { get; set; }
     public string ReportName { get; set; }
     public string ReportId { get; set; }
     public string ReportKey => $"{Helpers.Constants.Objects.GpConnect}_{DateTime.Now.ToString("s").ReplaceNonAlphanumeric()}_{ReportName?.ReplaceNonAlphanumeric()}.xlsx".ToLower();

diff --git a/modules/function/src/DTO/Response/InteractionKey.cs b/modules/function/src/DTO/Response/InteractionKey.cs
@@ -0,0 +1,8 @@
+namespace GpConnect.AppointmentChecker.Function.DTO.Response;
+
+public class InteractionKey
+{
+    public string InteractionId { get; set; }
+    public string ReportName { get; set; }
+    public string ReportId { get; set; }
+}
diff --git a/modules/function/src/DTO/Response/TransientData.cs b/modules/function/src/DTO/Response/TransientData.cs
@@ -0,0 +1,19 @@
+using Newtonsoft.Json;
+
+namespace GpConnect.AppointmentChecker.Function.DTO.Response;
+
+public class TransientData
+{
+    [JsonProperty("id")]
+    public string Id { get; set; }
+
+    [JsonProperty("data")]
+    public string Data { get; set; }
+
+    [JsonProperty("reportId")]
+    public string ReportId { get; set; }
+
+    [JsonProperty("reportName")]
+    public string ReportName { get; set; }
+
+}