Changes to the capability reporting logic to persist transient report…

…ing data

database/functions/R__reporting.add_transient_data.sql

@@ -0,0 +1,16 @@
+drop function if exists reporting.add_transient_data;
+
+create function reporting.add_transient_data
+(
+	_transient_id varchar(100),
+	_transient_data json,
+	_transient_report_id varchar(100),
+	_transient_report_name varchar(100)
+)
+returns void
+as $$
+begin	
+	insert into reporting.transient(transient_id, transient_data, transient_report_id, transient_report_name)
+	values (_transient_id, _transient_data, _transient_report_id, _transient_report_name);
+end;
+$$ language plpgsql;

database/functions/R__reporting.truncate_transient_data.sql

@@ -0,0 +1,11 @@
+drop function if exists reporting.truncate_transient_data;
+
+create function reporting.truncate_transient_data
+(
+)
+returns void
+as $$
+begin	
+	truncate table reporting.transient;
+end;
+$$ language plpgsql;

database/schema/V6.5__schema.sql

@@ -0,0 +1,10 @@
+drop table if exists reporting.transient;
+create table reporting.transient
+(
+    transient_id varchar(100) not null,
+    transient_data json not null,
+	transient_report_id varchar(100) not null,
+	transient_report_name varchar(100) not null
+);
+
+grant select, insert, update, delete on table reporting.transient to app_user;

modules/api/src/Controllers/ReportingController.cs

@@ -39,8 +39,8 @@ public async Task<IActionResult> Export([FromBody] ReportRequest reportRequest)
     [HttpPost("routereportrequest")]
     public async Task<IActionResult> RouteReportRequest([FromBody] RouteReportRequest routeReportRequest)
     {        
-        var result = await _service.RouteReportRequest(routeReportRequest);
-        return Ok(result);
+        await _service.RouteReportRequest(routeReportRequest);
+        return Ok();
     }
 
     [HttpPost("createinteractionmessage")]

modules/api/src/Controllers/SearchController.cs

@@ -18,6 +18,7 @@ public SearchController(ISearchService service)
     [HttpPost()]
     public async Task<ActionResult> ExecuteSearch([FromBody] SearchRequest searchRequest)
     {
+
         if (!searchRequest.ValidSearchCombination)
         {
             return BadRequest();

modules/api/src/DTO/Request/RouteReportRequest.cs

@@ -7,4 +7,5 @@ public class RouteReportRequest
     public List<string>? Workflow { get; set; }
     public string? ReportName { get; set; } = null;
     public string? ReportId { get; set; } = null;
+    public string? ObjectKeyJson { get; set; } = null;
 }

modules/api/src/Service/Interfaces/IReportingService.cs

@@ -9,7 +9,7 @@ public interface IReportingService
     public Task<string> GetReport(string functionName);
     public Task<Stream> ExportReport(ReportRequest reportRequest);
     public Task<Stream> CreateInteractionReport(ReportCreationRequest reportCreationRequest);
-    public Task<string> RouteReportRequest(RouteReportRequest routeReportRequest);
+    public Task RouteReportRequest(RouteReportRequest routeReportRequest);
     public Task SendMessageToCreateInteractionReportContent(ReportInteractionRequest reportInteractionRequest);
     public Task<List<Report>> GetReports();    
     public Task<List<CapabilityReport>> GetCapabilityReports();

modules/api/src/Service/ReportingService.cs

@@ -1,4 +1,5 @@
 using Amazon.SQS.Model;
+using Dapper;
 using DocumentFormat.OpenXml;
 using DocumentFormat.OpenXml.Packaging;
 using DocumentFormat.OpenXml.Spreadsheet;
@@ -58,25 +59,30 @@ public async Task<Stream> CreateInteractionReport(ReportCreationRequest reportCr
         return CreateReport(reportCreationRequest.JsonData.ConvertJsonDataToDataTable(), reportCreationRequest.ReportName, reportCreationRequest.ReportFilter);
     }
 
-    public async Task<string> RouteReportRequest(RouteReportRequest routeReportRequest)
+    public async Task RouteReportRequest(RouteReportRequest routeReportRequest)
     {
         try
         {
+            string? transientData = null;
             switch (routeReportRequest.ReportId.ToUpper())
             {
                 case "ACCESSRECORDSTRUCTURED":
-                    return await _interactionService.CreateInteractionData<AccessRecordStructuredReporting>(routeReportRequest);
+                    transientData = await _interactionService.CreateInteractionData<AccessRecordStructuredReporting>(routeReportRequest);
+                    break;
                 case "APPOINTMENTMANAGEMENT":
-                    return await _interactionService.CreateInteractionData<AppointmentManagementReporting>(routeReportRequest);
+                    transientData = await _interactionService.CreateInteractionData<AppointmentManagementReporting>(routeReportRequest);
+                    break;
                 case "ACCESSRECORDHTML":
-                    return await _interactionService.CreateInteractionData<AccessRecordHtmlReporting>(routeReportRequest);
+                    transientData = await _interactionService.CreateInteractionData<AccessRecordHtmlReporting>(routeReportRequest);
+                    break;
                 case "UPDATERECORD":
                 case "SENDDOCUMENT":
-                    return await _workflowService.CreateWorkflowData<MailboxReporting>(routeReportRequest);
+                    transientData = await _workflowService.CreateWorkflowData<MailboxReporting>(routeReportRequest);
+                    break;
                 default:
                     break;
             }
-            return null;
+            await CreateTransientData(transientData, routeReportRequest);
         }
         catch (Exception exc)
         {
@@ -85,6 +91,20 @@ public async Task<string> RouteReportRequest(RouteReportRequest routeReportReque
         }
     }
 
+    private async Task CreateTransientData(string? transientData, RouteReportRequest routeReportRequest)
+    {
+        if (transientData != null)
+        {
+            var functionName = "reporting.add_transient_data";
+            var parameters = new DynamicParameters();
+            parameters.Add("_transient_id", routeReportRequest.ObjectKeyJson);
+            parameters.Add("_transient_data", transientData);
+            parameters.Add("_transient_report_id", routeReportRequest.ReportId);
+            parameters.Add("_transient_report_name", routeReportRequest.ReportName);
+            await _dataService.ExecuteQuery(functionName, parameters);
+        }
+    }
+
     public async Task<MemoryStream> ExportBySpineMessage(int spineMessageId, string reportName)
     {
         var parameters = new Dictionary<string, int>

modules/function/src/CapabilityReportEventFunction.cs

@@ -59,7 +59,7 @@ public async Task<HttpStatusCode> FunctionHandler(ILambdaContext lambdaContext)
         _stopwatch.Start();
         _lambdaContext = lambdaContext;
         _lambdaContext.Logger.LogInformation("Purging json objects from S3");
-        await Reset(Objects.Transient, Objects.Key, Objects.Completion);
+        await Reset(Objects.Key, Objects.Completion);
         _lambdaContext.Logger.LogInformation("Obtaining roles data from S3");
         var rolesSource = await StorageManager.Get<List<string>>(new StorageDownloadRequest { BucketName = _storageConfiguration.BucketName, Key = _storageConfiguration.RolesObject });
         var odsList = await GetOdsData(rolesSource.ToArray());
@@ -99,7 +99,7 @@ private async Task<IEnumerable<MessagingRequest>> AddMessagesToQueue(string[] od
             var dataSourceCount = dataSource.Count;
             var capabilityReports = await GetCapabilityReports();
 
-            var batchSize = 50;
+            var batchSize = 20;
             var iterationCount = dataSourceCount / batchSize;
 
             await Parallel.ForEachAsync(capabilityReports, _parallelOptions, async (capabilityReport, ct) =>

modules/function/src/CompletionFunction.cs

@@ -1,4 +1,5 @@
 using Amazon.Lambda.Core;
+using Amazon.S3.Model;
 using GpConnect.AppointmentChecker.Function.Configuration;
 using GpConnect.AppointmentChecker.Function.DTO.Request;
 using GpConnect.AppointmentChecker.Function.Helpers;
@@ -84,7 +85,25 @@ private async Task BundleUpJsonResponsesAndSendReport()
                 ObjectPrefix = $"{Objects.Transient}_{sourceKey}_{DateTime.Now:yyyy_MM_dd}"
             });
 
-            for( var i = 0; i < bucketObjects.Count; i++ )
+            while (bucketObjects != null && bucketObjects.Count > 0)
+            {
+                var deleteRequest = new DeleteObjectsRequest() { BucketName = storageListRequest.BucketName };
+                foreach (S3Object s3Object in listResponse)
+                {
+                    deleteRequest.AddKey(s3Object.Key);
+                }
+                var deleteResponse = await s3Client.DeleteObjectsAsync(deleteRequest);
+                if (deleteResponse.HttpStatusCode == HttpStatusCode.OK)
+                {
+                    listResponse = await GetObjects(storageListRequest);
+                }
+                else
+                {
+                    listResponse = null;
+                }
+            }
+
+            for ( var i = 0; i < bucketObjects.Count; i++ )
             {
                 var jsonData = await StorageManager.Get(new StorageDownloadRequest { BucketName = bucketObjects[i].BucketName, Key = bucketObjects[i].Key });
                 responses.Add(jsonData);

modules/function/src/SQSEventFunction.cs

@@ -55,12 +55,7 @@ public async Task<SQSBatchResponse> FunctionHandler(SQSEvent evnt, ILambdaContex
             try
             {
                 var reportRequest = await ProcessMessageAsync(message);
-                var response = await RouteReportRequest(reportRequest);
-
-                if (response.IsSuccessStatusCode)
-                {
-                    await GenerateTransientJsonForReport(reportRequest.ObjectKeyJson, response);
-                }
+                await RouteReportRequest(reportRequest);
                 await Task.CompletedTask;
             }
             catch (Exception)
@@ -77,7 +72,6 @@ public async Task<SQSBatchResponse> FunctionHandler(SQSEvent evnt, ILambdaContex
     {
         try
         {
-            _lambdaContext.Logger.LogLine(message.Body);
             ReportInteraction? reportInteraction = null;
             var messageRequest = JsonConvert.DeserializeObject<MessagingRequest>(message.Body);
             if (messageRequest != null)
@@ -113,47 +107,29 @@ public async Task<SQSBatchResponse> FunctionHandler(SQSEvent evnt, ILambdaContex
         }
         catch (Exception e)
         {
-            _lambdaContext.Logger.LogError(e.Message);
+            _lambdaContext.Logger.LogError(e.StackTrace);
             throw;
         }
     }
 
-    private async Task<HttpResponseMessage?> RouteReportRequest(ReportInteraction reportInteraction)
+    private async Task RouteReportRequest(ReportInteraction reportInteraction)
     {
         try
         {
             var json = new StringContent(JsonConvert.SerializeObject(reportInteraction, null, _options),
                Encoding.UTF8,
                MediaTypeHeaderValue.Parse("application/json").MediaType);
 
-            var response = await _httpClient.PostWithHeadersAsync("/reporting/routereportrequest", new Dictionary<string, string>()
+            await _httpClient.PostWithHeadersAsync("/reporting/routereportrequest", new Dictionary<string, string>()
             {
                 [Headers.UserId] = _endUserConfiguration.UserId,
                 [Headers.ApiKey] = _endUserConfiguration.ApiKey
             }, json);
-
-            return response;
         }
         catch (Exception e)
         {
             _lambdaContext.Logger.LogError(e.Message);
             throw;
         }
     }
-
-    private async Task<string?> GenerateTransientJsonForReport(string objectKeyJson, HttpResponseMessage? httpResponseMessage)
-    {
-        if (httpResponseMessage != null)
-        {
-            var inputBytes = await StreamExtensions.GetByteArray(httpResponseMessage);
-            var url = await StorageManager.Post(new StorageUploadRequest()
-            {
-                BucketName = _storageConfiguration.BucketName,
-                Key = objectKeyJson,
-                InputBytes = inputBytes
-            });
-            return url;
-        }
-        return null;
-    }
 }