From 79da18fc2ae553e03f5588898d2cc27339b93eab Mon Sep 17 00:00:00 2001 From: Chris Morris Date: Fri, 21 Jun 2024 10:37:53 +0100 Subject: [PATCH] Changes to the capability reporting logic to persist transient reporting data --- .../R__reporting.add_transient_data.sql | 16 ++++++++++ .../R__reporting.truncate_transient_data.sql | 11 +++++++ database/schema/V6.5__schema.sql | 10 ++++++ .../src/Controllers/ReportingController.cs | 4 +-- .../api/src/Controllers/SearchController.cs | 1 + .../api/src/DTO/Request/RouteReportRequest.cs | 1 + .../Service/Interfaces/IReportingService.cs | 2 +- modules/api/src/Service/ReportingService.cs | 32 +++++++++++++++---- .../src/CapabilityReportEventFunction.cs | 4 +-- modules/function/src/CompletionFunction.cs | 21 +++++++++++- modules/function/src/SQSEventFunction.cs | 32 +++---------------- 11 files changed, 94 insertions(+), 40 deletions(-) create mode 100644 database/functions/R__reporting.add_transient_data.sql create mode 100644 database/functions/R__reporting.truncate_transient_data.sql create mode 100644 database/schema/V6.5__schema.sql diff --git a/database/functions/R__reporting.add_transient_data.sql b/database/functions/R__reporting.add_transient_data.sql new file mode 100644 index 00000000..432225ed --- /dev/null +++ b/database/functions/R__reporting.add_transient_data.sql @@ -0,0 +1,16 @@ +drop function if exists reporting.add_transient_data; + +create function reporting.add_transient_data +( + _transient_id varchar(100), + _transient_data json, + _transient_report_id varchar(100), + _transient_report_name varchar(100) +) +returns void +as $$ +begin + insert into reporting.transient(transient_id, transient_data, transient_report_id, transient_report_name) + values (_transient_id, _transient_data, _transient_report_id, _transient_report_name); +end; +$$ language plpgsql; \ No newline at end of file diff --git a/database/functions/R__reporting.truncate_transient_data.sql b/database/functions/R__reporting.truncate_transient_data.sql new file mode 100644 index 00000000..2343d7c3 --- /dev/null +++ b/database/functions/R__reporting.truncate_transient_data.sql @@ -0,0 +1,11 @@ +drop function if exists reporting.truncate_transient_data; + +create function reporting.truncate_transient_data +( +) +returns void +as $$ +begin + truncate table reporting.transient; +end; +$$ language plpgsql; \ No newline at end of file diff --git a/database/schema/V6.5__schema.sql b/database/schema/V6.5__schema.sql new file mode 100644 index 00000000..9ca35625 --- /dev/null +++ b/database/schema/V6.5__schema.sql @@ -0,0 +1,10 @@ +drop table if exists reporting.transient; +create table reporting.transient +( + transient_id varchar(100) not null, + transient_data json not null, + transient_report_id varchar(100) not null, + transient_report_name varchar(100) not null +); + +grant select, insert, update, delete on table reporting.transient to app_user; \ No newline at end of file diff --git a/modules/api/src/Controllers/ReportingController.cs b/modules/api/src/Controllers/ReportingController.cs index 0cfd4e1f..828e2b43 100644 --- a/modules/api/src/Controllers/ReportingController.cs +++ b/modules/api/src/Controllers/ReportingController.cs @@ -39,8 +39,8 @@ public async Task Export([FromBody] ReportRequest reportRequest) [HttpPost("routereportrequest")] public async Task RouteReportRequest([FromBody] RouteReportRequest routeReportRequest) { - var result = await _service.RouteReportRequest(routeReportRequest); - return Ok(result); + await _service.RouteReportRequest(routeReportRequest); + return Ok(); } [HttpPost("createinteractionmessage")] diff --git a/modules/api/src/Controllers/SearchController.cs b/modules/api/src/Controllers/SearchController.cs index f0e408a7..3721324a 100644 --- a/modules/api/src/Controllers/SearchController.cs +++ b/modules/api/src/Controllers/SearchController.cs @@ -18,6 +18,7 @@ public SearchController(ISearchService service) [HttpPost()] public async Task ExecuteSearch([FromBody] SearchRequest searchRequest) { + if (!searchRequest.ValidSearchCombination) { return BadRequest(); diff --git a/modules/api/src/DTO/Request/RouteReportRequest.cs b/modules/api/src/DTO/Request/RouteReportRequest.cs index 205529e5..120d8405 100644 --- a/modules/api/src/DTO/Request/RouteReportRequest.cs +++ b/modules/api/src/DTO/Request/RouteReportRequest.cs @@ -7,4 +7,5 @@ public class RouteReportRequest public List? Workflow { get; set; } public string? ReportName { get; set; } = null; public string? ReportId { get; set; } = null; + public string? ObjectKeyJson { get; set; } = null; } \ No newline at end of file diff --git a/modules/api/src/Service/Interfaces/IReportingService.cs b/modules/api/src/Service/Interfaces/IReportingService.cs index 919da840..9a6ede9c 100644 --- a/modules/api/src/Service/Interfaces/IReportingService.cs +++ b/modules/api/src/Service/Interfaces/IReportingService.cs @@ -9,7 +9,7 @@ public interface IReportingService public Task GetReport(string functionName); public Task ExportReport(ReportRequest reportRequest); public Task CreateInteractionReport(ReportCreationRequest reportCreationRequest); - public Task RouteReportRequest(RouteReportRequest routeReportRequest); + public Task RouteReportRequest(RouteReportRequest routeReportRequest); public Task SendMessageToCreateInteractionReportContent(ReportInteractionRequest reportInteractionRequest); public Task> GetReports(); public Task> GetCapabilityReports(); diff --git a/modules/api/src/Service/ReportingService.cs b/modules/api/src/Service/ReportingService.cs index f6516cc6..0fbe9aaf 100644 --- a/modules/api/src/Service/ReportingService.cs +++ b/modules/api/src/Service/ReportingService.cs @@ -1,4 +1,5 @@ using Amazon.SQS.Model; +using Dapper; using DocumentFormat.OpenXml; using DocumentFormat.OpenXml.Packaging; using DocumentFormat.OpenXml.Spreadsheet; @@ -58,25 +59,30 @@ public async Task CreateInteractionReport(ReportCreationRequest reportCr return CreateReport(reportCreationRequest.JsonData.ConvertJsonDataToDataTable(), reportCreationRequest.ReportName, reportCreationRequest.ReportFilter); } - public async Task RouteReportRequest(RouteReportRequest routeReportRequest) + public async Task RouteReportRequest(RouteReportRequest routeReportRequest) { try { + string? transientData = null; switch (routeReportRequest.ReportId.ToUpper()) { case "ACCESSRECORDSTRUCTURED": - return await _interactionService.CreateInteractionData(routeReportRequest); + transientData = await _interactionService.CreateInteractionData(routeReportRequest); + break; case "APPOINTMENTMANAGEMENT": - return await _interactionService.CreateInteractionData(routeReportRequest); + transientData = await _interactionService.CreateInteractionData(routeReportRequest); + break; case "ACCESSRECORDHTML": - return await _interactionService.CreateInteractionData(routeReportRequest); + transientData = await _interactionService.CreateInteractionData(routeReportRequest); + break; case "UPDATERECORD": case "SENDDOCUMENT": - return await _workflowService.CreateWorkflowData(routeReportRequest); + transientData = await _workflowService.CreateWorkflowData(routeReportRequest); + break; default: break; } - return null; + await CreateTransientData(transientData, routeReportRequest); } catch (Exception exc) { @@ -85,6 +91,20 @@ public async Task RouteReportRequest(RouteReportRequest routeReportReque } } + private async Task CreateTransientData(string? transientData, RouteReportRequest routeReportRequest) + { + if (transientData != null) + { + var functionName = "reporting.add_transient_data"; + var parameters = new DynamicParameters(); + parameters.Add("_transient_id", routeReportRequest.ObjectKeyJson); + parameters.Add("_transient_data", transientData); + parameters.Add("_transient_report_id", routeReportRequest.ReportId); + parameters.Add("_transient_report_name", routeReportRequest.ReportName); + await _dataService.ExecuteQuery(functionName, parameters); + } + } + public async Task ExportBySpineMessage(int spineMessageId, string reportName) { var parameters = new Dictionary diff --git a/modules/function/src/CapabilityReportEventFunction.cs b/modules/function/src/CapabilityReportEventFunction.cs index 5094c0fa..48c1f749 100644 --- a/modules/function/src/CapabilityReportEventFunction.cs +++ b/modules/function/src/CapabilityReportEventFunction.cs @@ -59,7 +59,7 @@ public async Task FunctionHandler(ILambdaContext lambdaContext) _stopwatch.Start(); _lambdaContext = lambdaContext; _lambdaContext.Logger.LogInformation("Purging json objects from S3"); - await Reset(Objects.Transient, Objects.Key, Objects.Completion); + await Reset(Objects.Key, Objects.Completion); _lambdaContext.Logger.LogInformation("Obtaining roles data from S3"); var rolesSource = await StorageManager.Get>(new StorageDownloadRequest { BucketName = _storageConfiguration.BucketName, Key = _storageConfiguration.RolesObject }); var odsList = await GetOdsData(rolesSource.ToArray()); @@ -99,7 +99,7 @@ private async Task> AddMessagesToQueue(string[] od var dataSourceCount = dataSource.Count; var capabilityReports = await GetCapabilityReports(); - var batchSize = 50; + var batchSize = 20; var iterationCount = dataSourceCount / batchSize; await Parallel.ForEachAsync(capabilityReports, _parallelOptions, async (capabilityReport, ct) => diff --git a/modules/function/src/CompletionFunction.cs b/modules/function/src/CompletionFunction.cs index 89ab4f90..6f6b5615 100644 --- a/modules/function/src/CompletionFunction.cs +++ b/modules/function/src/CompletionFunction.cs @@ -1,4 +1,5 @@ using Amazon.Lambda.Core; +using Amazon.S3.Model; using GpConnect.AppointmentChecker.Function.Configuration; using GpConnect.AppointmentChecker.Function.DTO.Request; using GpConnect.AppointmentChecker.Function.Helpers; @@ -84,7 +85,25 @@ private async Task BundleUpJsonResponsesAndSendReport() ObjectPrefix = $"{Objects.Transient}_{sourceKey}_{DateTime.Now:yyyy_MM_dd}" }); - for( var i = 0; i < bucketObjects.Count; i++ ) + while (bucketObjects != null && bucketObjects.Count > 0) + { + var deleteRequest = new DeleteObjectsRequest() { BucketName = storageListRequest.BucketName }; + foreach (S3Object s3Object in listResponse) + { + deleteRequest.AddKey(s3Object.Key); + } + var deleteResponse = await s3Client.DeleteObjectsAsync(deleteRequest); + if (deleteResponse.HttpStatusCode == HttpStatusCode.OK) + { + listResponse = await GetObjects(storageListRequest); + } + else + { + listResponse = null; + } + } + + for ( var i = 0; i < bucketObjects.Count; i++ ) { var jsonData = await StorageManager.Get(new StorageDownloadRequest { BucketName = bucketObjects[i].BucketName, Key = bucketObjects[i].Key }); responses.Add(jsonData); diff --git a/modules/function/src/SQSEventFunction.cs b/modules/function/src/SQSEventFunction.cs index 84ea68de..1b8864ec 100644 --- a/modules/function/src/SQSEventFunction.cs +++ b/modules/function/src/SQSEventFunction.cs @@ -55,12 +55,7 @@ public async Task FunctionHandler(SQSEvent evnt, ILambdaContex try { var reportRequest = await ProcessMessageAsync(message); - var response = await RouteReportRequest(reportRequest); - - if (response.IsSuccessStatusCode) - { - await GenerateTransientJsonForReport(reportRequest.ObjectKeyJson, response); - } + await RouteReportRequest(reportRequest); await Task.CompletedTask; } catch (Exception) @@ -77,7 +72,6 @@ public async Task FunctionHandler(SQSEvent evnt, ILambdaContex { try { - _lambdaContext.Logger.LogLine(message.Body); ReportInteraction? reportInteraction = null; var messageRequest = JsonConvert.DeserializeObject(message.Body); if (messageRequest != null) @@ -113,12 +107,12 @@ public async Task FunctionHandler(SQSEvent evnt, ILambdaContex } catch (Exception e) { - _lambdaContext.Logger.LogError(e.Message); + _lambdaContext.Logger.LogError(e.StackTrace); throw; } } - private async Task RouteReportRequest(ReportInteraction reportInteraction) + private async Task RouteReportRequest(ReportInteraction reportInteraction) { try { @@ -126,13 +120,11 @@ public async Task FunctionHandler(SQSEvent evnt, ILambdaContex Encoding.UTF8, MediaTypeHeaderValue.Parse("application/json").MediaType); - var response = await _httpClient.PostWithHeadersAsync("/reporting/routereportrequest", new Dictionary() + await _httpClient.PostWithHeadersAsync("/reporting/routereportrequest", new Dictionary() { [Headers.UserId] = _endUserConfiguration.UserId, [Headers.ApiKey] = _endUserConfiguration.ApiKey }, json); - - return response; } catch (Exception e) { @@ -140,20 +132,4 @@ public async Task FunctionHandler(SQSEvent evnt, ILambdaContex throw; } } - - private async Task GenerateTransientJsonForReport(string objectKeyJson, HttpResponseMessage? httpResponseMessage) - { - if (httpResponseMessage != null) - { - var inputBytes = await StreamExtensions.GetByteArray(httpResponseMessage); - var url = await StorageManager.Post(new StorageUploadRequest() - { - BucketName = _storageConfiguration.BucketName, - Key = objectKeyJson, - InputBytes = inputBytes - }); - return url; - } - return null; - } }