From f72361c0faf88b6460526287a4ae75c7dc1da2e5 Mon Sep 17 00:00:00 2001 From: Yohann Paris Date: Tue, 11 Feb 2025 15:35:44 -0500 Subject: [PATCH] Remove skema unified (#6568) --- packages/server/README.md | 2 +- .../knowledge/KnowledgeController.java | 455 +----------------- .../proxies/skema/SkemaUnifiedProxy.java | 89 ---- .../hmiserver/service/ExtractionService.java | 319 +----------- .../resources/application-staging.properties | 1 - .../src/main/resources/application.properties | 1 - .../src/main/resources/messages.properties | 11 +- .../knowledge/KnowledgeControllerTests.java | 289 ----------- .../service/ExtractionServiceTests.java | 64 --- .../test/resources/knowledge/equation1.png | Bin 2790 -> 0 bytes .../test/resources/knowledge/equation2.png | Bin 3240 -> 0 bytes .../test/resources/knowledge/equation3.png | Bin 2229 -> 0 bytes 12 files changed, 29 insertions(+), 1202 deletions(-) delete mode 100644 packages/server/src/main/java/software/uncharted/terarium/hmiserver/proxies/skema/SkemaUnifiedProxy.java delete mode 100644 packages/server/src/test/resources/knowledge/equation1.png delete mode 100644 packages/server/src/test/resources/knowledge/equation2.png delete mode 100644 packages/server/src/test/resources/knowledge/equation3.png diff --git a/packages/server/README.md b/packages/server/README.md index eb8c8f54a1..65f64e35a9 100644 --- a/packages/server/README.md +++ b/packages/server/README.md @@ -186,7 +186,7 @@ Error messages sent to the client should be easy to understand and provide meani * Use active voice instead of passive (instead of "the service was terminated…" try "We terminated the service…"). * Avoid software development jargon that users may not understand ("UUID," "curies"). * Don't introduce backend terms that users never see in the UI. For example, avoid terms like "artifacts" and "assets" if users only see "resources" in the UI. -* Consider whether it's necessary to name a backend service that users never see. Instead of referencing "SKEMA," determine whether it would be clearer to refer to our "model service." +* Consider whether it's necessary to name a backend service that users never see. Instead of referencing "MIRA," determine whether it would be clearer to refer to our "model service." ### Terms diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeController.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeController.java index 7698f3fea5..1d9bb5e038 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeController.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeController.java @@ -4,69 +4,42 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ObjectNode; -import feign.FeignException; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import jakarta.annotation.PostConstruct; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.concurrent.Future; import java.util.concurrent.TimeoutException; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.http.HttpEntity; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.entity.ContentType; import org.springframework.http.HttpStatus; -import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; import org.springframework.security.access.annotation.Secured; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RequestPart; import org.springframework.web.bind.annotation.RestController; -import org.springframework.web.multipart.MultipartFile; import org.springframework.web.server.ResponseStatusException; import software.uncharted.terarium.hmiserver.models.ClientEventType; -import software.uncharted.terarium.hmiserver.models.dataservice.code.Code; -import software.uncharted.terarium.hmiserver.models.dataservice.code.CodeFile; import software.uncharted.terarium.hmiserver.models.dataservice.document.DocumentAsset; import software.uncharted.terarium.hmiserver.models.dataservice.model.Model; -import software.uncharted.terarium.hmiserver.models.dataservice.modelparts.ModelHeader; -import software.uncharted.terarium.hmiserver.models.dataservice.modelparts.ModelMetadata; -import software.uncharted.terarium.hmiserver.models.dataservice.provenance.Provenance; -import software.uncharted.terarium.hmiserver.models.dataservice.provenance.ProvenanceRelationType; -import software.uncharted.terarium.hmiserver.models.dataservice.provenance.ProvenanceType; -import software.uncharted.terarium.hmiserver.models.extractionservice.ExtractionResponse; import software.uncharted.terarium.hmiserver.models.task.TaskRequest; import software.uncharted.terarium.hmiserver.models.task.TaskRequest.TaskType; import software.uncharted.terarium.hmiserver.models.task.TaskResponse; -import software.uncharted.terarium.hmiserver.proxies.skema.SkemaUnifiedProxy; import software.uncharted.terarium.hmiserver.security.Roles; import software.uncharted.terarium.hmiserver.service.ClientEventService; import software.uncharted.terarium.hmiserver.service.CurrentUserService; import software.uncharted.terarium.hmiserver.service.ExtractionService; -import software.uncharted.terarium.hmiserver.service.data.CodeService; import software.uncharted.terarium.hmiserver.service.data.ModelService; import software.uncharted.terarium.hmiserver.service.data.ProjectService; -import software.uncharted.terarium.hmiserver.service.data.ProvenanceService; import software.uncharted.terarium.hmiserver.service.notification.NotificationGroupInstance; import software.uncharted.terarium.hmiserver.service.notification.NotificationService; import software.uncharted.terarium.hmiserver.service.tasks.EquationsCleanupResponseHandler; @@ -74,7 +47,6 @@ import software.uncharted.terarium.hmiserver.service.tasks.SympyToAMRResponseHandler; import software.uncharted.terarium.hmiserver.service.tasks.TaskService; import software.uncharted.terarium.hmiserver.service.tasks.TaskService.TaskMode; -import software.uncharted.terarium.hmiserver.utils.ByteMultipartFile; import software.uncharted.terarium.hmiserver.utils.JsonUtil; import software.uncharted.terarium.hmiserver.utils.Messages; import software.uncharted.terarium.hmiserver.utils.rebac.Schema; @@ -87,12 +59,7 @@ public class KnowledgeController { private final ObjectMapper mapper; - private final SkemaUnifiedProxy skemaUnifiedProxy; - private final ModelService modelService; - private final ProvenanceService provenanceService; - - private final CodeService codeService; private final ExtractionService extractionService; private final TaskService taskService; @@ -168,7 +135,7 @@ public static class NotificationProperties { } /** - * Send the equations to the skema unified service to get the AMR + * Equations LaTeX to AMR * * @return UUID Model ID, or null if the model was not created or updated */ @@ -244,68 +211,38 @@ public ResponseEntity equationsToModel( } } - // Get an AMR from Skema Unified Service or MIRA final Model responseAMR; - String extractionService = "mira"; - if (req.get("extractionService") != null) { - extractionService = req.get("extractionService").asText(); - } + final TaskRequest latexToSympyRequest; + final TaskResponse latexToSympyResponse; + final TaskRequest sympyToAMRRequest; + final TaskResponse sympyToAMRResponse; - if (extractionService.equals("mira")) { - final TaskRequest latexToSympyRequest; - final TaskResponse latexToSympyResponse; - final TaskRequest sympyToAMRRequest; - final TaskResponse sympyToAMRResponse; + try { + // 1. LaTeX to sympy code + latexToSympyRequest = createLatexToSympyTask(equationsReq); + latexToSympyResponse = taskService.runTaskSync(latexToSympyRequest); - try { - // 1. LaTeX to sympy code - latexToSympyRequest = createLatexToSympyTask(equationsReq); - latexToSympyResponse = taskService.runTaskSync(latexToSympyRequest); - - // 2. hand off - final String code = extractCodeFromLatexToSympy(latexToSympyResponse); - - // 3. sympy code string to amr json - sympyToAMRRequest = createSympyToAMRTask(code); - sympyToAMRResponse = taskService.runTaskSync(sympyToAMRRequest); - - final JsonNode taskResponseJSON = mapper.readValue(sympyToAMRResponse.getOutput(), JsonNode.class); - final ObjectNode amrNode = taskResponseJSON.get("response").get("amr").deepCopy(); - responseAMR = mapper.convertValue(amrNode, Model.class); - } catch (final Exception e) { - log.error(messages.get("task.mira.internal-error"), e); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("task.mira.internal-error")); - } - } else { - try { - // Create a request for SKEMA with the cleaned-up equations. - final JsonNode skemaRequest = mapper.createObjectNode().put("model", "petrinet").set("equations", equationsReq); - responseAMR = skemaUnifiedProxy.consolidatedEquationsToAMR(skemaRequest).getBody(); - if (responseAMR == null) { - log.warn("Skema Unified Service did not return a valid AMR based on the provided equations"); - throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, messages.get("skema.bad-equations")); - } - } catch (final FeignException e) { - log.error( - "An exception occurred while Skema Unified Service was trying to produce an AMR based on the provided equations", - e - ); - throw handleSkemaFeignException(e); - } catch (final Exception e) { - log.error( - "An unhandled error occurred while Skema Unified Service was trying to produce an AMR based on the provided equations", - e - ); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("skema.internal-error")); - } + // 2. hand off + final String code = extractCodeFromLatexToSympy(latexToSympyResponse); + + // 3. sympy code string to amr json + sympyToAMRRequest = createSympyToAMRTask(code); + sympyToAMRResponse = taskService.runTaskSync(sympyToAMRRequest); + + final JsonNode taskResponseJSON = mapper.readValue(sympyToAMRResponse.getOutput(), JsonNode.class); + final ObjectNode amrNode = taskResponseJSON.get("response").get("amr").deepCopy(); + responseAMR = mapper.convertValue(amrNode, Model.class); + } catch (final Exception e) { + log.error(messages.get("task.mira.internal-error"), e); + throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("task.mira.internal-error")); } // We only handle Petri Net models if (!responseAMR.isPetrinet()) { - throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, messages.get("skema.bad-equations.petrinet")); + throw new ResponseStatusException(HttpStatus.UNPROCESSABLE_ENTITY, messages.get("bad-equations.petrinet")); } - notificationInterface.sendMessage("AMR extracted via SKEMA."); + notificationInterface.sendMessage("AMR extracted via MIRA."); // If no model id is provided, create a new model asset Model model; @@ -338,328 +275,6 @@ public ResponseEntity equationsToModel( return ResponseEntity.ok(model.getId()); } - @PostMapping("/base64-equations-to-model") - @Secured(Roles.USER) - public ResponseEntity base64EquationsToAMR(@RequestBody final JsonNode req) { - try { - return ResponseEntity.ok(skemaUnifiedProxy.base64EquationsToAMR(req).getBody()); - } catch (final FeignException e) { - log.error("Error with Skema Unified Service while converting base64 equations to AMR", e); - throw handleSkemaFeignException(e); - } - } - - @PostMapping("/base64-equations-to-latex") - @Secured(Roles.USER) - public ResponseEntity base64EquationsToLatex(@RequestBody final JsonNode req) { - try { - return ResponseEntity.ok(skemaUnifiedProxy.base64EquationsToLatex(req).getBody()); - } catch (final FeignException e) { - log.error("Error with Skema Unified Service while converting base64 equations to Latex", e); - throw handleSkemaFeignException(e); - } - } - - /** - * Transform source code to AMR - * - * @param codeId (String): id of the code artifact model - * @param dynamicsOnly (Boolean): whether to only run the amr extraction over - * specified dynamics from the code - * object in TDS - * @param llmAssisted (Boolean): whether amr extraction is llm assisted - * @return Model - */ - @PostMapping("/code-to-amr") - @Secured(Roles.USER) - ResponseEntity postCodeToAMR( - @RequestParam("code-id") final UUID codeId, - @RequestParam(name = "project-id", required = false) final UUID projectId, - @RequestParam(name = "name", required = false, defaultValue = "") final String name, - @RequestParam(name = "description", required = false, defaultValue = "") final String description, - @RequestParam(name = "dynamics-only", required = false, defaultValue = "false") Boolean dynamicsOnly, - @RequestParam(name = "llm-assisted", required = false, defaultValue = "false") final Boolean llmAssisted - ) { - final Schema.Permission permission = projectService.checkPermissionCanWrite( - currentUserService.get().getId(), - projectId - ); - - final Optional code = codeService.getAsset(codeId, permission); - if (code.isEmpty()) { - log.error(String.format("Unable to fetch the requested code asset with codeId: %s", codeId)); - throw new ResponseStatusException(HttpStatus.NOT_FOUND, messages.get("code.not-found")); - } - - final Map codeFiles = code.get().getFiles(); - - final Map codeContent = new HashMap<>(); - - for (final Entry file : codeFiles.entrySet()) { - final String filename = file.getKey(); - final CodeFile codeFile = file.getValue(); - - final String content; - try { - content = codeService.fetchFileAsString(codeId, filename).orElseThrow(); - } catch (final IOException e) { - log.error("Unable to fetch code as a string", e); - throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, messages.get("postgres.service-unavailable")); - } - - if (dynamicsOnly && codeFile.getDynamics() != null && codeFile.getDynamics().getBlock() != null) { - final List blocks = codeFile.getDynamics().getBlock(); - for (final String block : blocks) { - final String[] parts = block.split("-"); - final int startLine = Integer.parseInt(parts[0].substring(1)); - final int endLine = Integer.parseInt(parts[1].substring(1)); - - final String[] codeLines = content.split("\n"); - final List targetLines = Arrays.asList(codeLines).subList(startLine - 1, endLine); - - final String targetBlock = String.join("\n", targetLines); - - codeContent.put(filename, targetBlock); - } - } else { - codeContent.put(filename, content); - dynamicsOnly = false; - } - } - - final List files = new ArrayList<>(); - final List blobs = new ArrayList<>(); - - final ResponseEntity resp; - - if (dynamicsOnly) { - for (final Entry entry : codeContent.entrySet()) { - files.add(entry.getKey()); - blobs.add(entry.getValue()); - } - - resp = skemaUnifiedProxy.snippetsToAMR(files, blobs); - } else { - final ByteArrayOutputStream zipBuffer = new ByteArrayOutputStream(); - final ZipOutputStream zipf = new ZipOutputStream(zipBuffer, StandardCharsets.UTF_8); - - try { - for (final Map.Entry entry : codeContent.entrySet()) { - final String codeName = entry.getKey(); - final String content = entry.getValue(); - final ZipEntry zipEntry = new ZipEntry(codeName); - zipf.putNextEntry(zipEntry); - zipf.write(content.getBytes(StandardCharsets.UTF_8)); - zipf.closeEntry(); - } - } catch (final IOException e) { - log.error("Unable to write to zip file", e); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("generic.io-error.write")); - } finally { - try { - zipf.close(); - } catch (final IOException e) { - log.error("Unable to close zip file", e); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("generic.io-error.write")); - } - } - - final ByteMultipartFile file = new ByteMultipartFile(zipBuffer.toByteArray(), "zip_file.zip", "application/zip"); - - resp = llmAssisted ? skemaUnifiedProxy.llmCodebaseToAMR(file) : skemaUnifiedProxy.codebaseToAMR(file); - } - - if (resp.getStatusCode().is4xxClientError()) { - log.warn("Skema Unified Service did not return a valid AMR because the provided code was not valid"); - throw new ResponseStatusException(resp.getStatusCode(), messages.get("skema.bad-code")); - } else if (resp.getStatusCode() == HttpStatus.SERVICE_UNAVAILABLE) { - log.warn("Skema Unified Service is currently unavailable"); - throw new ResponseStatusException(resp.getStatusCode(), messages.get("skema.service-unavailable")); - } else if (!resp.getStatusCode().is2xxSuccessful()) { - log.error( - "An error occurred while Skema Unified Service was trying to produce an AMR based on the provided code" - ); - throw new ResponseStatusException(resp.getStatusCode(), messages.get("skema.internal-error")); - } - - Model model; - try { - model = mapper.treeToValue(resp.getBody(), Model.class); - } catch (final IOException e) { - log.error("Unable to convert response to model", e); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("generic.unknown")); - } - - if (model.getMetadata() == null) { - model.setMetadata(new ModelMetadata()); - } - - // create the model - if (!name.isEmpty()) { - model.setName(name); - } - if (model.getMetadata() == null) { - model.setMetadata(new ModelMetadata()); - } - model.getMetadata().setCodeId(codeId.toString()); - - if (!description.isEmpty()) { - if (model.getHeader() == null) { - model.setHeader(new ModelHeader()); - } - model.getHeader().setDescription(description); - } - - try { - model = modelService.createAsset(model, projectId, permission); - } catch (final IOException e) { - log.error("Unable to create model", e); - throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, messages.get("postgres.service-unavailable")); - } - - // update the code - if (code.get().getMetadata() == null) { - code.get().setMetadata(new HashMap<>()); - } - code.get().getMetadata().put("model_id", model.getId().toString()); - - try { - codeService.updateAsset(code.get(), projectId, permission); - } catch (final IOException e) { - log.error("Unable to update code", e); - throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, messages.get("postgres.service-unavailable")); - } - - // set the provenance - final Provenance provenancePayload = new Provenance( - ProvenanceRelationType.EXTRACTED_FROM, - model.getId(), - ProvenanceType.MODEL, - codeId, - ProvenanceType.CODE - ); - provenanceService.createProvenance(provenancePayload); - - return ResponseEntity.ok(model); - } - - // Create a model from code blocks - @Operation(summary = "Create a model from code blocks") - @ApiResponses( - value = { - @ApiResponse( - responseCode = "200", - description = "Return the extraction job for code to amr", - content = @Content( - mediaType = "application/json", - schema = @io.swagger.v3.oas.annotations.media.Schema(implementation = ExtractionResponse.class) - ) - ), - @ApiResponse( - responseCode = "400", - description = "Invalid input - code file may be missing name", - content = @Content - ), - @ApiResponse(responseCode = "500", description = "Error running code blocks to model", content = @Content) - } - ) - @PostMapping(value = "/code-blocks-to-model", consumes = MediaType.MULTIPART_FORM_DATA_VALUE) - @Secured(Roles.USER) - public ResponseEntity codeBlocksToModel( - @RequestParam(name = "project-id", required = false) final UUID projectId, - @RequestPart final Code code, - @RequestPart("file") final MultipartFile input - ) { - final Schema.Permission permission = projectService.checkPermissionCanWrite( - currentUserService.get().getId(), - projectId - ); - - // 1. create code asset from code blocks - final Code createdCode; - try { - createdCode = codeService.createAsset(code, projectId, permission); - } catch (final IOException e) { - log.error("Unable to create code asset", e); - throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, messages.get("postgres.service-unavailable")); - } - - // 2. upload file to code asset - final byte[] fileAsBytes; - try { - fileAsBytes = input.getBytes(); - } catch (final IOException e) { - log.error("Unable to read file", e); - throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("generic.io-error.read")); - } - final HttpEntity fileEntity = new ByteArrayEntity(fileAsBytes, ContentType.TEXT_PLAIN); - final String filename = input.getOriginalFilename(); - - if (filename == null) { - log.warn("Filename is missing from the file"); - throw new ResponseStatusException(HttpStatus.BAD_REQUEST, messages.get("code.filename-needed")); - } - - try { - codeService.uploadFile(code.getId(), filename, fileEntity); - } catch (final IOException e) { - log.error("Unable to upload file to code asset", e); - throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, messages.get("postgres.service-unavailable")); - } - - // add the code file to the code asset - final CodeFile codeFile = new CodeFile(); - codeFile.setFileNameAndProgrammingLanguage(filename); - - if (code.getFiles() == null) { - code.setFiles(new HashMap<>()); - } - code.getFiles().put(filename, codeFile); - - try { - codeService.updateAsset(code, projectId, permission); - } catch (final IOException e) { - log.error("Unable to update code asset", e); - throw new ResponseStatusException(HttpStatus.SERVICE_UNAVAILABLE, messages.get("postgres.service-unavailable")); - } - - // 3. create model from code asset - return postCodeToAMR(createdCode.getId(), projectId, "temp model", "temp model description", false, false); - } - - @PostMapping("/align-model") - @Secured(Roles.USER) - @ApiResponses( - value = { - @ApiResponse(responseCode = "204", description = "Model as been align with document", content = @Content), - @ApiResponse( - responseCode = "500", - description = "Error aligning model with variable extracted from document", - content = @Content - ) - } - ) - public ResponseEntity alignModel( - @RequestParam("document-id") final UUID documentId, - @RequestParam("model-id") final UUID modelId, - @RequestParam(name = "project-id", required = false) final UUID projectId - ) { - final Schema.Permission permission = projectService.checkPermissionCanWrite( - currentUserService.get().getId(), - projectId - ); - - try { - return ResponseEntity.ok(extractionService.alignAMR(projectId, documentId, modelId, permission).get()); - } catch (final InterruptedException | ExecutionException e) { - log.error("Error aligning model with document", e); - throw new ResponseStatusException( - org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, - messages.get("skema.error.align-model") - ); - } - } - /** * Document Extractions * @@ -828,28 +443,6 @@ public ResponseEntity latexToAMRDebug(@RequestBody final String latex) throw new ResponseStatusException(HttpStatus.INTERNAL_SERVER_ERROR, messages.get("generic.io-error.read")); } - private ResponseStatusException handleSkemaFeignException(final FeignException e) { - final HttpStatus statusCode = HttpStatus.resolve(e.status()); - if (statusCode != null && statusCode.is4xxClientError()) { - log.warn("Skema Unified Service did not return a valid AMR based on the provided resources"); - return new ResponseStatusException(statusCode, messages.get("skema.bad-equations")); - } else if (statusCode == HttpStatus.SERVICE_UNAVAILABLE) { - log.warn("Skema Unified Service is currently unavailable"); - return new ResponseStatusException(statusCode, messages.get("skema.service-unavailable")); - } else if (statusCode != null && statusCode.is5xxServerError()) { - log.error( - "An error occurred while Skema Unified Service was trying to produce an AMR based on the provided resources" - ); - return new ResponseStatusException(statusCode, messages.get("skema.internal-error")); - } - - final HttpStatus httpStatus = (statusCode == null) ? HttpStatus.INTERNAL_SERVER_ERROR : statusCode; - log.error( - "An unknown error occurred while Skema Unified Service was trying to produce an AMR based on the provided resources" - ); - return new ResponseStatusException(httpStatus, messages.get("generic.unknown")); - } - private TaskRequest cleanupEquationsTaskRequest(UUID projectId, List equations) { final EquationsCleanupResponseHandler.Input input = new EquationsCleanupResponseHandler.Input(); input.setEquations(equations); diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/proxies/skema/SkemaUnifiedProxy.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/proxies/skema/SkemaUnifiedProxy.java deleted file mode 100644 index b4fee58e37..0000000000 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/proxies/skema/SkemaUnifiedProxy.java +++ /dev/null @@ -1,89 +0,0 @@ -package software.uncharted.terarium.hmiserver.proxies.skema; - -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import lombok.Data; -import org.springframework.cloud.openfeign.FeignClient; -import org.springframework.http.ResponseEntity; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RequestPart; -import org.springframework.web.multipart.MultipartFile; -import software.uncharted.terarium.hmiserver.models.dataservice.model.Model; - -@FeignClient(name = "skema-unified", url = "${skema-unified.url}") -public interface SkemaUnifiedProxy { - @PostMapping("/workflows/latex/equations-to-amr") - ResponseEntity postLaTeXToAMR(@RequestBody JsonNode request); - - @PostMapping("/eqn2mml/image/base64/mml") - ResponseEntity postImageToEquations(@RequestBody String request); - - @PostMapping("/workflows/consolidated/equations-to-amr") - ResponseEntity consolidatedEquationsToAMR(@RequestBody JsonNode request); - - @PostMapping("/workflows/images/base64/equations-to-amr") - ResponseEntity base64EquationsToAMR(@RequestBody JsonNode request); - - @PostMapping("/workflows/images/base64/equations-to-latex") - ResponseEntity base64EquationsToLatex(@RequestBody JsonNode request); - - @PostMapping(value = "/workflows/code/llm-assisted-codebase-to-pn-amr", consumes = "multipart/form-data") - ResponseEntity llmCodebaseToAMR(@RequestPart("zip_file") MultipartFile file); - - @PostMapping(value = "/workflows/code/codebase-to-pn-amr", consumes = "multipart/form-data") - ResponseEntity codebaseToAMR(@RequestPart("zip_file") MultipartFile file); - - @PostMapping(value = "/workflows/code/snippets-to-amr", consumes = "multipart/form-data") - ResponseEntity snippetsToAMR( - @RequestPart("files") List files, - @RequestPart("blobs") List blobs - ); - - @PostMapping(value = "/metal/link_amr", consumes = "multipart/form-data") - ResponseEntity linkAMRFile( - @RequestPart("amr_file") MultipartFile amrFile, - @RequestPart("text_extractions_file") MultipartFile extractionsFile - ); - - @Data - class IntegratedTextExtractionsBody { - - public IntegratedTextExtractionsBody(final String text) { - this.texts = Arrays.asList(text); - this.amrs = new ArrayList<>(); - } - - public IntegratedTextExtractionsBody(final String text, final List amrs) { - this.texts = Arrays.asList(text); - this.amrs = amrs - .stream() - .map(amr -> { - try { - final ObjectMapper mapper = new ObjectMapper(); - return mapper.writeValueAsString(amr); - } catch (final JsonProcessingException e) { - e.printStackTrace(); - return null; - } - }) - .collect(Collectors.toList()); - } - - final List texts; - final List amrs; - } - - @PostMapping("/text-reading/integrated-text-extractions") - ResponseEntity integratedTextExtractions( - @RequestParam(value = "annotate_mit", defaultValue = "true") Boolean annotateMit, - @RequestParam(value = "annotate_skema", defaultValue = "false") Boolean annotateSkema, - @RequestBody IntegratedTextExtractionsBody body - ); -} diff --git a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/ExtractionService.java b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/ExtractionService.java index 33cf28a03c..aafdc8fb35 100644 --- a/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/ExtractionService.java +++ b/packages/server/src/main/java/software/uncharted/terarium/hmiserver/service/ExtractionService.java @@ -3,10 +3,8 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.ArrayNode; -import com.fasterxml.jackson.databind.node.ObjectNode; import feign.FeignException; import jakarta.annotation.PostConstruct; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.HttpURLConnection; import java.net.URI; @@ -28,45 +26,30 @@ import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.zip.ZipInputStream; import lombok.Data; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.apache.http.entity.ContentType; import org.redisson.api.RMapCache; import org.redisson.api.RedissonClient; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.env.Environment; import org.springframework.http.HttpStatus; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.web.server.ResponseStatusException; import software.uncharted.terarium.hmiserver.models.ClientEventType; import software.uncharted.terarium.hmiserver.models.dataservice.document.DocumentAsset; import software.uncharted.terarium.hmiserver.models.dataservice.document.ExtractedDocumentPage; -import software.uncharted.terarium.hmiserver.models.dataservice.model.Model; -import software.uncharted.terarium.hmiserver.models.dataservice.modelparts.ModelMetadata; -import software.uncharted.terarium.hmiserver.models.dataservice.provenance.Provenance; -import software.uncharted.terarium.hmiserver.models.dataservice.provenance.ProvenanceRelationType; -import software.uncharted.terarium.hmiserver.models.dataservice.provenance.ProvenanceType; import software.uncharted.terarium.hmiserver.models.dataservice.simulation.ProgressState; import software.uncharted.terarium.hmiserver.models.task.TaskRequest; import software.uncharted.terarium.hmiserver.models.task.TaskRequest.TaskType; import software.uncharted.terarium.hmiserver.models.task.TaskResponse; -import software.uncharted.terarium.hmiserver.proxies.skema.SkemaUnifiedProxy; -import software.uncharted.terarium.hmiserver.proxies.skema.SkemaUnifiedProxy.IntegratedTextExtractionsBody; import software.uncharted.terarium.hmiserver.service.data.DocumentAssetService; -import software.uncharted.terarium.hmiserver.service.data.ModelService; -import software.uncharted.terarium.hmiserver.service.data.ProvenanceService; -import software.uncharted.terarium.hmiserver.service.gollm.EmbeddingService; import software.uncharted.terarium.hmiserver.service.notification.NotificationGroupInstance; import software.uncharted.terarium.hmiserver.service.notification.NotificationService; import software.uncharted.terarium.hmiserver.service.tasks.ExtractEquationsResponseHandler; import software.uncharted.terarium.hmiserver.service.tasks.ExtractTablesResponseHandler; import software.uncharted.terarium.hmiserver.service.tasks.ExtractTextResponseHandler; import software.uncharted.terarium.hmiserver.service.tasks.TaskService; -import software.uncharted.terarium.hmiserver.utils.JsonUtil; -import software.uncharted.terarium.hmiserver.utils.StringMultipartFile; import software.uncharted.terarium.hmiserver.utils.rebac.Schema; @Service @@ -75,14 +58,11 @@ public class ExtractionService { private final DocumentAssetService documentService; - private final ModelService modelService; - private final SkemaUnifiedProxy skemaUnifiedProxy; private final ObjectMapper objectMapper; private final ClientEventService clientEventService; private final NotificationService notificationService; private final TaskService taskService; - private final ProvenanceService provenanceService; - private final EmbeddingService embeddingService; + // private final EmbeddingService embeddingService; private final CurrentUserService currentUserService; private static final String RESPONSE_CACHE_KEY = "extraction-service-response-cache"; @@ -137,27 +117,6 @@ private static class Properties { private final UUID documentId; } - public static String removeFileExtension(final String filename) { - final int lastIndexOfDot = filename.lastIndexOf("."); - if (lastIndexOfDot != -1) { - return filename.substring(0, lastIndexOfDot); - } - return filename; - } - - static class ExtractionFile { - - ExtractionFile(final String filename, final byte[] bytes, final ContentType contentType) { - this.filename = filename; - this.bytes = bytes; - this.contentType = contentType; - } - - ContentType contentType; - String filename; - byte[] bytes; - } - enum FailureType { TABLE_FAILURE("tables"), EQUATION_FAILURE("equations"), @@ -478,282 +437,6 @@ public Future extractPDFAndApplyToDocument( }); } - private DocumentAsset runVariableExtraction( - final NotificationGroupInstance notificationInterface, - final UUID projectId, - final UUID documentId, - final List modelIds, - final Schema.Permission hasWritePermission - ) { - notificationInterface.sendMessage("Starting variable extraction."); - try { - // Fetch the text from the document - final DocumentAsset document = documentService.getAsset(documentId, hasWritePermission).orElseThrow(); - notificationInterface.sendMessage("Document found, fetching text."); - if (document.getText() == null || document.getText().isEmpty()) { - throw new RuntimeException("No text found in paper document"); - } - - // add optional models - final List models = new ArrayList<>(); - for (final UUID modelId : modelIds) { - models.add(modelService.getAsset(modelId, hasWritePermission).orElseThrow()); - } - notificationInterface.sendMessage("Model(s) found, added to extraction request."); - - // Create a collection to hold the variable extractions - JsonNode collection = null; - - notificationInterface.sendMessage("Sending request to be processes by MIT."); - - final IntegratedTextExtractionsBody body = new IntegratedTextExtractionsBody(document.getText(), models); - - log.info("Sending variable extraction request to SKEMA using MIT only"); - final ResponseEntity resp = skemaUnifiedProxy.integratedTextExtractions(true, false, body); - - notificationInterface.sendMessage("Response received."); - if (resp.getStatusCode().is2xxSuccessful()) { - for (final JsonNode output : resp.getBody().get("outputs")) { - if (!output.has("errors") || output.get("errors").isEmpty()) { - collection = output.get("data"); - break; - } - } - } else { - throw new RuntimeException("non successful response."); - } - - if (collection == null) { - throw new RuntimeException("No variables extractions returned"); - } - - notificationInterface.sendMessage("Organizing and saving the extractions."); - final ArrayNode attributes = objectMapper.createArrayNode(); - for (final JsonNode attribute : collection.get("attributes")) { - attributes.add(attribute); - } - - // add the attributes to the metadata - if (document.getMetadata() == null) { - document.setMetadata(new HashMap<>()); - } - document.getMetadata().put("attributes", attributes); - - if (modelIds.size() > 0) { - for (final UUID modelId : modelIds) { - notificationInterface.sendMessage("Attempting to align models for model: " + modelId); - try { - runAlignAMR(notificationInterface, projectId, documentId, modelId, hasWritePermission); - notificationInterface.sendMessage("Model " + modelId + " aligned successfully"); - } catch (final Exception e) { - notificationInterface.sendMessage("Failed to align model: " + modelId + ", continuing..."); - } - } - } - - // update the document - return documentService.updateAsset(document, projectId, hasWritePermission).orElseThrow(); - } catch (final FeignException e) { - final String error = "Transitive service failure"; - log.error(error, e.contentUTF8(), e); - throw new ResponseStatusException( - e.status() < 100 ? HttpStatus.INTERNAL_SERVER_ERROR : HttpStatus.valueOf(e.status()), - error + ": " + e.getMessage() - ); - } catch (final Exception e) { - final String error = "Variable extraction on document: " + documentId + " failed."; - log.error(error, e); - notificationInterface.sendError(error + " — " + e.getMessage()); - throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error); - } - } - - public Future extractVariables( - final UUID projectId, - final UUID documentId, - final List modelIds, - final Schema.Permission hasWritePermission - ) { - // Set up the client interface - final NotificationGroupInstance notificationInterface = new NotificationGroupInstance<>( - clientEventService, - notificationService, - ClientEventType.EXTRACTION, - null, - new Properties(documentId), - currentUserService.get().getId() - ); - notificationInterface.sendMessage("Variable extraction task submitted..."); - - return executor.submit(() -> { - final DocumentAsset doc = runVariableExtraction( - notificationInterface, - projectId, - documentId, - modelIds, - hasWritePermission - ); - notificationInterface.sendFinalMessage("Extraction complete"); - return doc; - }); - } - - public Future alignAMR( - final UUID projectId, - final UUID documentId, - final UUID modelId, - final Schema.Permission hasWritePermission - ) { - final NotificationGroupInstance notificationInterface = new NotificationGroupInstance<>( - clientEventService, - notificationService, - ClientEventType.EXTRACTION, - null, - new Properties(documentId), - currentUserService.get().getId() - ); - - notificationInterface.sendMessage("Model alignment task submitted..."); - - return executor.submit(() -> { - final Model model = runAlignAMR(notificationInterface, projectId, documentId, modelId, hasWritePermission); - notificationInterface.sendFinalMessage("Alignment complete"); - return model; - }); - } - - public Model runAlignAMR( - final NotificationGroupInstance notificationInterface, - final UUID projectId, - final UUID documentId, - final UUID modelId, - final Schema.Permission hasWritePermission - ) { - try { - notificationInterface.sendMessage("Starting model alignment..."); - - final DocumentAsset document = documentService.getAsset(documentId, hasWritePermission).orElseThrow(); - - final Model model = modelService.getAsset(modelId, hasWritePermission).orElseThrow(); - - final String modelString = objectMapper.writeValueAsString(model); - - if (document.getMetadata() == null) { - document.setMetadata(new HashMap<>()); - } - if (document.getMetadata().get("attributes") == null) { - throw new RuntimeException("No attributes found in document"); - } - - final JsonNode attributes = objectMapper.valueToTree(document.getMetadata().get("attributes")); - - final ObjectNode extractions = objectMapper.createObjectNode(); - extractions.set("attributes", attributes); - - final String extractionsString = objectMapper.writeValueAsString(extractions); - - final StringMultipartFile amrFile = new StringMultipartFile(modelString, "amr.json", "application/json"); - final StringMultipartFile extractionFile = new StringMultipartFile( - extractionsString, - "extractions.json", - "application/json" - ); - - final ResponseEntity res; - try { - res = skemaUnifiedProxy.linkAMRFile(amrFile, extractionFile); - } catch (final FeignException e) { - final String error = "Transitive service failure"; - log.error(error, e.contentUTF8(), e); - throw new ResponseStatusException( - e.status() < 100 ? HttpStatus.INTERNAL_SERVER_ERROR : HttpStatus.valueOf(e.status()), - error + ": " + e.getMessage() - ); - } - if (!res.getStatusCode().is2xxSuccessful()) { - throw new ResponseStatusException(res.getStatusCode(), "Unable to link AMR file"); - } - - final JsonNode modelJson = objectMapper.valueToTree(model); - - // overwrite all updated fields - JsonUtil.recursiveSetAll((ObjectNode) modelJson, res.getBody()); - - final Model updatedModel = objectMapper.treeToValue(modelJson, Model.class); - - if (updatedModel.getMetadata() == null) { - updatedModel.setMetadata(new ModelMetadata()); - } - - // add document gollm card to model - final JsonNode card = document.getMetadata().get("gollmCard"); - if (card != null) { - updatedModel.getMetadata().setGollmCard(card); - } - - // update the model - modelService.updateAsset(model, projectId, hasWritePermission); - - // create provenance - final Provenance provenance = new Provenance( - ProvenanceRelationType.EXTRACTED_FROM, - modelId, - ProvenanceType.MODEL, - documentId, - ProvenanceType.DOCUMENT - ); - provenanceService.createProvenance(provenance); - - // update model embeddings - - // TODO: do you want to update the model asset embeddings in the project index? - - // if (card != null && model.getPublicAsset() && !model.getTemporary()) { - // final String cardText = objectMapper.writeValueAsString(card); - // if (cardText != null) { - // try { - // final TerariumAssetEmbeddings embeddings = embeddingService.generateEmbeddings(cardText); - - // modelService.uploadEmbeddings(modelId, embeddings, hasWritePermission); - // notificationInterface.sendMessage("Embeddings created"); - // } catch (final Exception e) { - // log.warn("Unable to generate embedding vectors for model"); - // } - // } - // } - - return model; - } catch (final FeignException e) { - final String error = "Transitive service failure"; - log.error(error, e.contentUTF8(), e); - throw new ResponseStatusException( - e.status() < 100 ? HttpStatus.INTERNAL_SERVER_ERROR : HttpStatus.valueOf(e.status()), - error + ": " + e.getMessage() - ); - } catch (final Exception e) { - final String error = "SKEMA link_amr request from document: " + documentId + " failed."; - log.error(error, e); - notificationInterface.sendError(error + " — " + e.getMessage()); - throw new ResponseStatusException(org.springframework.http.HttpStatus.INTERNAL_SERVER_ERROR, error); - } - } - - public static byte[] zipEntryToBytes(final ZipInputStream zipInputStream) throws IOException { - final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - final byte[] buffer = new byte[1024]; - int len; - while ((len = zipInputStream.read(buffer)) > 0) { - byteArrayOutputStream.write(buffer, 0, len); - } - - final byte[] bytes = byteArrayOutputStream.toByteArray(); - if (bytes.length == 0) { - throw new IOException("Empty file found in zip"); - } - - return bytes; - } - @Value("${terarium.taskrunner.equation_extraction.gpu-endpoint}") private String EQUATION_EXTRACTION_GPU_ENDPOINT; diff --git a/packages/server/src/main/resources/application-staging.properties b/packages/server/src/main/resources/application-staging.properties index 3fce103910..b3def20223 100644 --- a/packages/server/src/main/resources/application-staging.properties +++ b/packages/server/src/main/resources/application-staging.properties @@ -46,7 +46,6 @@ terarium.file-storage-s3-bucket-name=askem-staging-data-service ######################################################################################################################## # Microservice configuration ######################################################################################################################## -skema-unified.url=https://skema-unified.staging.terarium.ai ciemss-service.url=https://pyciemss.staging.terarium.ai ######################################################################################################################## diff --git a/packages/server/src/main/resources/application.properties b/packages/server/src/main/resources/application.properties index 27e03fe4e6..81f1298760 100644 --- a/packages/server/src/main/resources/application.properties +++ b/packages/server/src/main/resources/application.properties @@ -172,7 +172,6 @@ terarium.static-index-path=askem-static-index github.url=https://api.github.com jsdelivr.url=https://cdn.jsdelivr.net mira-api.url=http://mira-epi-dkg-lb-dc1e19b273dedaa2.elb.us-east-1.amazonaws.com -skema-unified.url=https://skema-unified.staging.terarium.ai ciemss-service.url=https://pyciemss.staging.terarium.ai ######################################################################################################################## diff --git a/packages/server/src/main/resources/messages.properties b/packages/server/src/main/resources/messages.properties index 147c3b44e7..710224d8e2 100644 --- a/packages/server/src/main/resources/messages.properties +++ b/packages/server/src/main/resources/messages.properties @@ -33,13 +33,13 @@ document.not-found = We couldn't find the requested document. Please verify that document.unable-to-delete = We couldn't delete the document. Please try again later. document.unable-to-update = We couldn't update the document. Please try again later. document.text-length-exceeded = This document exceeds our 600,000-word limit. If you still need to analyze this document, contact support. -document.extracton.failed = Document extraction failed. Please try again later. +document.extraction.failed = Document extraction failed. Please try again later. mira.concept.bad-curies = Our domain knowledge service (MIRA) couldn't find any valid concepts for the selected IDs. Please review the IDs that you selected and try again. mira.concept.bad-query = Our domain knowledge service (MIRA) couldn't find any valid concepts for the entered query. Please review your search criteria and try again. mira.ode.bad-model = Our domain knowledge service (MIRA) couldn't find a valid equation based on the selected model. Please verify the model you selected is valid and try again. mira.similarity.bad-curies = Our domain knowledge service (MIRA) couldn't find any valid entity similarities based on the selected IDs. Please review the IDs that you selected and try again. -mira.internal-error = Our model service (SKEMA) couldn't produce a valid model representation based on the selected resource. Please check that it's valid and try again. +mira.internal-error = Our model service (MIRA) couldn't produce a valid model representation based on the selected resource. Please check that it's valid and try again. mira.service-unavailable = Our domain knowledge service (MIRA) is temporarily unavailable. Please try again later. mira.bad-number = Comparison requires at least two models. If this problem continues, contact support. @@ -74,12 +74,7 @@ rebac.unauthorized-delete = You're not authorized to delete this resource. Conta rebac.unauthorized-read = You're not authorized to access this resource. Contact the project owner to upgrade your permissions. rebac.unauthorized-update = You're not authorized to update this resource. Contact the project owner to upgrade your permissions. -skema.bad-code = Our model service (SKEMA) couldn't find a valid model representation. This could be due to syntax or semantic errors in your code. Please review your code snippet and try again. -skema.bad-equations = Our model service (SKEMA) couldn't find a valid model representation. This could be due to invalid equations or an inability to parse them into the requested framework. Please review your equations and try again. -skema.bad-equations.petrinet = Our model service (SKEMA) couldn't create a valid Petrinet representation for the supplied equations. This could be due to invalid equations or an inability to parse them into the requested framework. Please review your equations and try again. -skema.error.align-model = We couldn't align the model with the selected document. Please verify that the model is valid and that the document has valid extractions. -skema.internal-error = We couldn't produce a valid model representation based on the selected resource. Please verify that the resource is valid and try again. -skema.service-unavailable = Our model service (SKEMA) is temporarily unavailable. Please try again later. +bad-equations.petrinet = Our model service could not create a valid Petri Net representation for the supplied equations. This could be due to invalid equations or an inability to parse them into the requested framework. Please review your equations and try again. task.gollm.json-processing = Our LLM service (GoLLM) couldn't process task data. Please verify that the data is valid and try again. task.gollm.timeout = We terminated our LLM service (GoLLM) because it took too long to complete. Please try again later. If this problem continues, contact support. diff --git a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeControllerTests.java b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeControllerTests.java index f53e8a6815..2e26e2f5a8 100644 --- a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeControllerTests.java +++ b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/controller/knowledge/KnowledgeControllerTests.java @@ -6,15 +6,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import java.io.IOException; import java.nio.file.Files; -import java.util.ArrayList; -import java.util.Base64; -import java.util.HashMap; -import java.util.Map; import java.util.UUID; import lombok.extern.slf4j.Slf4j; -import org.apache.http.HttpEntity; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.entity.ContentType; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; @@ -27,8 +20,6 @@ import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import software.uncharted.terarium.hmiserver.TerariumApplicationTests; import software.uncharted.terarium.hmiserver.configuration.MockUser; -import software.uncharted.terarium.hmiserver.models.dataservice.code.Code; -import software.uncharted.terarium.hmiserver.models.dataservice.code.CodeFile; import software.uncharted.terarium.hmiserver.models.dataservice.dataset.Dataset; import software.uncharted.terarium.hmiserver.models.dataservice.document.DocumentAsset; import software.uncharted.terarium.hmiserver.models.dataservice.model.Model; @@ -203,173 +194,6 @@ public void equationsToModelPetrinet() throws Exception { log.info(petrinetModelId.toString()); } - // @Test - @WithUserDetails(MockUser.URSULA) - public void base64EquationsToAMRTests() throws Exception { - final ClassPathResource resource1 = new ClassPathResource("knowledge/equation1.png"); - final byte[] content1 = Files.readAllBytes(resource1.getFile().toPath()); - final String encodedString1 = Base64.getEncoder().encodeToString(content1); - - final ClassPathResource resource2 = new ClassPathResource("knowledge/equation2.png"); - final byte[] content2 = Files.readAllBytes(resource2.getFile().toPath()); - final String encodedString2 = Base64.getEncoder().encodeToString(content2); - - final ClassPathResource resource3 = new ClassPathResource("knowledge/equation3.png"); - final byte[] content3 = Files.readAllBytes(resource3.getFile().toPath()); - final String encodedString3 = Base64.getEncoder().encodeToString(content3); - - final String payload = - "{\"images\": [" + - "\"" + - encodedString1 + - "\"," + - "\"" + - encodedString2 + - "\"," + - "\"" + - encodedString3 + - "\"],\"model\": \"regnet\"}"; - - final MvcResult res = mockMvc - .perform( - MockMvcRequestBuilders.post("/knowledge/base64-equations-to-model") - .contentType(MediaType.APPLICATION_JSON) - .content(payload) - .with(csrf()) - ) - .andExpect(status().isOk()) - .andReturn(); - - final Model amr = objectMapper.readValue(res.getResponse().getContentAsString(), Model.class); - log.info(amr.toString()); - } - - // @Test - @WithUserDetails(MockUser.URSULA) - public void base64EquationsToLatexTests() throws Exception { - final ClassPathResource resource1 = new ClassPathResource("knowledge/equation1.png"); - final byte[] content1 = Files.readAllBytes(resource1.getFile().toPath()); - final String encodedString1 = Base64.getEncoder().encodeToString(content1); - - final ClassPathResource resource2 = new ClassPathResource("knowledge/equation2.png"); - final byte[] content2 = Files.readAllBytes(resource2.getFile().toPath()); - final String encodedString2 = Base64.getEncoder().encodeToString(content2); - - final ClassPathResource resource3 = new ClassPathResource("knowledge/equation3.png"); - final byte[] content3 = Files.readAllBytes(resource3.getFile().toPath()); - final String encodedString3 = Base64.getEncoder().encodeToString(content3); - - final String payload = - "{\"images\": [" + - "\"" + - encodedString1 + - "\"," + - "\"" + - encodedString2 + - "\"," + - "\"" + - encodedString3 + - "\"],\"model\": \"regnet\"}"; - - final MvcResult res = mockMvc - .perform( - MockMvcRequestBuilders.post("/knowledge/base64-equations-to-latex") - .contentType(MediaType.APPLICATION_JSON) - .content(payload) - .with(csrf()) - ) - .andExpect(status().isOk()) - .andReturn(); - - final String latex = res.getResponse().getContentAsString(); - log.info(latex); - } - - // @Test - @WithUserDetails(MockUser.URSULA) - public void variableExtractionTests() throws Exception { - DocumentAsset documentAsset = (DocumentAsset) new DocumentAsset() - .setText("x = 0. y = 1. I = Infected population.") - .setName("test-document-name") - .setDescription("my description"); - - documentAsset = documentAssetService.createAsset(documentAsset, project.getId(), ASSUME_WRITE_PERMISSION); - - mockMvc - .perform( - MockMvcRequestBuilders.post("/knowledge/variable-extractions") - .contentType(MediaType.APPLICATION_JSON) - .param("document-id", documentAsset.getId().toString()) - .param("domain", "epi") - .with(csrf()) - ) - .andExpect(status().isAccepted()); - } - - // @Test - @WithUserDetails(MockUser.URSULA) - public void variableExtractionWithModelTests() throws Exception { - DocumentAsset documentAsset = (DocumentAsset) new DocumentAsset() - .setText("x = 0. y = 1. I = Infected population.") - .setName("test-document-name") - .setDescription("my description"); - - documentAsset = documentAssetService.createAsset(documentAsset, project.getId(), ASSUME_WRITE_PERMISSION); - - final ClassPathResource resource = new ClassPathResource("knowledge/sir.json"); - final byte[] content = Files.readAllBytes(resource.getFile().toPath()); - Model model = objectMapper.readValue(content, Model.class); - - model = modelService.createAsset(model, project.getId(), ASSUME_WRITE_PERMISSION); - - mockMvc - .perform( - MockMvcRequestBuilders.post("/knowledge/variable-extractions") - .contentType(MediaType.APPLICATION_JSON) - .param("document-id", documentAsset.getId().toString()) - .param("model-ids", model.getId().toString()) - .param("domain", "epi") - .with(csrf()) - ) - .andExpect(status().isAccepted()); - } - - // @Test - @WithUserDetails(MockUser.URSULA) - public void linkAmrTests() throws Exception { - DocumentAsset documentAsset = (DocumentAsset) new DocumentAsset() - .setText("x = 0. y = 1. I = Infected population.") - .setName("test-document-name") - .setDescription("my description"); - - documentAsset = documentAssetService.createAsset(documentAsset, project.getId(), ASSUME_WRITE_PERMISSION); - - documentAsset = extractionService - .extractVariables(project.getId(), documentAsset.getId(), new ArrayList<>(), ASSUME_WRITE_PERMISSION) - .get(); - - final ClassPathResource resource = new ClassPathResource("knowledge/sir.json"); - final byte[] content = Files.readAllBytes(resource.getFile().toPath()); - Model model = objectMapper.readValue(content, Model.class); - - model = modelService.createAsset(model, project.getId(), ASSUME_WRITE_PERMISSION); - - final MvcResult res = mockMvc - .perform( - MockMvcRequestBuilders.post("/knowledge/align-model") - .contentType(MediaType.APPLICATION_JSON) - .param("document-id", documentAsset.getId().toString()) - .param("model-id", model.getId().toString()) - .with(csrf()) - ) - .andExpect(status().isOk()) - .andReturn(); - - model = objectMapper.readValue(res.getResponse().getContentAsString(), Model.class); - - Assertions.assertNotNull(model); - } - // @Test @WithUserDetails(MockUser.URSULA) public void profileModel() throws Exception { @@ -497,117 +321,4 @@ public void profileDataset() throws Exception { Assertions.assertNotNull(dataset.getMetadata().get("dataCard")); } - - // @Test - @WithUserDetails(MockUser.URSULA) - public void codeToAmrTest() throws Exception { - final ClassPathResource resource = new ClassPathResource("knowledge/code.py"); - final byte[] content = Files.readAllBytes(resource.getFile().toPath()); - - final String filename = "code.py"; - - final CodeFile codeFile = new CodeFile(); - codeFile.setFileNameAndProgrammingLanguage(filename); - - final Map files = new HashMap<>(); - files.put(filename, codeFile); - - final Code code = codeService.createAsset( - (Code) new Code().setFiles(files).setName("test-code-name").setDescription("my description"), - project.getId(), - ASSUME_WRITE_PERMISSION - ); - - final HttpEntity fileEntity = new ByteArrayEntity(content, ContentType.TEXT_PLAIN); - codeService.uploadFile(code.getId(), filename, fileEntity); - - final MvcResult res = mockMvc - .perform( - MockMvcRequestBuilders.post("/knowledge/code-to-amr") - .contentType(MediaType.APPLICATION_JSON) - .param("code-id", code.getId().toString()) - .with(csrf()) - ) - .andExpect(status().isOk()) - .andReturn(); - - final Model model = objectMapper.readValue(res.getResponse().getContentAsString(), Model.class); - Assertions.assertNotNull(model); - } - - // @Test - @WithUserDetails(MockUser.URSULA) - public void codeToAmrTestLLM() throws Exception { - final ClassPathResource resource = new ClassPathResource("knowledge/code.py"); - final byte[] content = Files.readAllBytes(resource.getFile().toPath()); - - final String filename = "code.py"; - - final CodeFile codeFile = new CodeFile(); - codeFile.setFileNameAndProgrammingLanguage(filename); - - final Map files = new HashMap<>(); - files.put(filename, codeFile); - - final Code code = codeService.createAsset( - (Code) new Code().setFiles(files).setName("test-code-name").setDescription("my description"), - project.getId(), - ASSUME_WRITE_PERMISSION - ); - - final HttpEntity fileEntity = new ByteArrayEntity(content, ContentType.TEXT_PLAIN); - codeService.uploadFile(code.getId(), filename, fileEntity); - - final MvcResult res = mockMvc - .perform( - MockMvcRequestBuilders.post("/knowledge/code-to-amr") - .contentType(MediaType.APPLICATION_JSON) - .param("code-id", code.getId().toString()) - .param("llm-assisted", "true") - .with(csrf()) - ) - .andExpect(status().isOk()) - .andReturn(); - - final Model model = objectMapper.readValue(res.getResponse().getContentAsString(), Model.class); - Assertions.assertNotNull(model); - } - - // @Test - @WithUserDetails(MockUser.URSULA) - public void codeToAmrTestDynamicsOnly() throws Exception { - final ClassPathResource resource = new ClassPathResource("knowledge/code.py"); - final byte[] content = Files.readAllBytes(resource.getFile().toPath()); - - final String filename = "code.py"; - - final CodeFile codeFile = new CodeFile(); - codeFile.setFileNameAndProgrammingLanguage(filename); - - final Map files = new HashMap<>(); - files.put(filename, codeFile); - - final Code code = codeService.createAsset( - (Code) new Code().setFiles(files).setName("test-code-name").setDescription("my description"), - project.getId(), - ASSUME_WRITE_PERMISSION - ); - - final HttpEntity fileEntity = new ByteArrayEntity(content, ContentType.TEXT_PLAIN); - codeService.uploadFile(code.getId(), filename, fileEntity); - - final MvcResult res = mockMvc - .perform( - MockMvcRequestBuilders.post("/knowledge/code-to-amr") - .contentType(MediaType.APPLICATION_JSON) - .param("code-id", code.getId().toString()) - .param("dynamics-only", "true") - .with(csrf()) - ) - .andExpect(status().isOk()) - .andReturn(); - - final Model model = objectMapper.readValue(res.getResponse().getContentAsString(), Model.class); - Assertions.assertNotNull(model); - } } diff --git a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/ExtractionServiceTests.java b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/ExtractionServiceTests.java index 6b34dfcebb..c65a674055 100644 --- a/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/ExtractionServiceTests.java +++ b/packages/server/src/test/java/software/uncharted/terarium/hmiserver/service/ExtractionServiceTests.java @@ -65,70 +65,6 @@ public void teardown() throws IOException { projectSearchService.teardownIndexAndAlias(); } - // @Test - @WithUserDetails(MockUser.URSULA) - public void variableExtractionTests() throws Exception { - DocumentAsset documentAsset = (DocumentAsset) new DocumentAsset() - .setText("x = 0. y = 1. I = Infected population.") - .setName("test-document-name") - .setDescription("my description"); - - documentAsset = documentAssetService.createAsset(documentAsset, project.getId(), ASSUME_WRITE_PERMISSION); - - documentAsset = extractionService - .extractVariables(project.getId(), documentAsset.getId(), new ArrayList<>(), ASSUME_WRITE_PERMISSION) - .get(); - } - - // @Test - @WithUserDetails(MockUser.URSULA) - public void variableExtractionWithModelTests() throws Exception { - final ClassPathResource resource1 = new ClassPathResource("knowledge/extraction_text.txt"); - final byte[] content1 = Files.readAllBytes(resource1.getFile().toPath()); - - DocumentAsset documentAsset = (DocumentAsset) new DocumentAsset() - .setText(new String(content1)) - .setName("test-document-name") - .setDescription("my description"); - - documentAsset = documentAssetService.createAsset(documentAsset, project.getId(), ASSUME_WRITE_PERMISSION); - - final ClassPathResource resource2 = new ClassPathResource("knowledge/extraction_amr.json"); - final byte[] content2 = Files.readAllBytes(resource2.getFile().toPath()); - Model model = objectMapper.readValue(content2, Model.class); - - model = modelService.createAsset(model, project.getId(), ASSUME_WRITE_PERMISSION); - - documentAsset = extractionService - .extractVariables(project.getId(), documentAsset.getId(), List.of(model.getId()), ASSUME_WRITE_PERMISSION) - .get(); - } - - // @Test - @WithUserDetails(MockUser.URSULA) - public void linkAmrTests() throws Exception { - DocumentAsset documentAsset = (DocumentAsset) new DocumentAsset() - .setText("x = 0. y = 1. I = Infected population.") - .setName("test-document-name") - .setDescription("my description"); - - documentAsset = documentAssetService.createAsset(documentAsset, project.getId(), ASSUME_WRITE_PERMISSION); - - documentAsset = extractionService - .extractVariables(project.getId(), documentAsset.getId(), new ArrayList<>(), ASSUME_WRITE_PERMISSION) - .get(); - - final ClassPathResource resource = new ClassPathResource("knowledge/sir.json"); - final byte[] content = Files.readAllBytes(resource.getFile().toPath()); - Model model = objectMapper.readValue(content, Model.class); - - model = modelService.createAsset(model, project.getId(), ASSUME_WRITE_PERMISSION); - - model = extractionService - .alignAMR(project.getId(), documentAsset.getId(), model.getId(), ASSUME_WRITE_PERMISSION) - .get(); - } - // @Test @WithUserDetails(MockUser.URSULA) public void pdfExtraction() throws Exception { diff --git a/packages/server/src/test/resources/knowledge/equation1.png b/packages/server/src/test/resources/knowledge/equation1.png deleted file mode 100644 index 9c1c5436d00005b02e25e5991d1f09f0540c1e7f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2790 zcmVgwwE_xI=L=Q=t%rKP3h+4Wb8~b2{QNjLIPLB2baZs@@9%PQa_j5sX=!QZ=H~SD^r@+-=;-Ko zcXzqDx#{WYsHmvy?Ci6%v*qRGw6wJ2-YNw~CZEbDc-QBXXvTkl};NakEYip#W zq@$yw+uPeVHZ~$6BA}q4Vq#*qx3@h#J#1`j-{0RgH8tGa+}_^awY9b7J>vZU012W= zL_t(|ob8+om!ilO$B~Elg5lcQ>>^_$jBmvq=)`e5lb&pElHI%}o8tFZYDi8z?hr{7;I2;a#!{Kl^oM%(1?7S2uSyHOxNUfF9pQ}Ns zH<~R?Z)V<7-xvn8Z?sNVxHZ2it!G!kw?IkFXZ37+-6n4~-fGY*v2RyE|IQ?qa<;F# z8h|;Fs;>7AzyOd-?AsME?7AZWTZvyZD0cO?YOcP8V z_MIUseG1OPhM1gca_Kw8)OCkD(zm+97ipw7`0jl<9lHk9Udx>5L00;@GzQe<*a53( zT~00{NiNXYwC_;*g80p0808=nedTf0I{`^93`cb|>TZeL-M3+j?v&WQ3r8eoQ}Lii zhO3{vKMfP}f!!&ue-{qUw(vW8^d=*Hh;{3y_TxccKJpC?&S+X7ZWv_GDRXUW!H~}! zc?^loeoqfYG?iA}J1=?zs);tAKJxI0ZItBD&-qB@h7|q0V*7O^tRX3-3^LL;_>l27 zm&_^Le0h-RbrBtAq;IIhCi9^gfTIT-Fve z`B;$2mqasZv89G4{#aTMFPjF=3YHMLA($@C9}iUepuP1s7j5}RtIR%LFba;-M{B5R zU=l;FK(mPS(L%vo#JNQ_aNY>cf6>cQ6`xz29a?d{w^B%+lPwhkv{2C%6@Y`VI9#Gkg%wf5}TCX>q0Tp0%adGNY!8KdF9h26c?h%rp)9 z&ooDj-V&(M9-yh>(p+PB{{ww{eqXP*9*SNZpPQiHctJm1qyIuG8#LZ*EdMZRgZXKv zcyyM>v^WP(1rN3t#jzoz6BYWP#?*3UgW0FT7ZCroSp-|XU#>B?f>GidiOnd66zfuMGA)sYbR#U3qpetS6Pl4vs{I*+cwulDrEvmDd?By-aIQSIe?( zY{y&^z+lkbb#`wUF&~Tc#n(&s_gp8#5E|R#7Spe(XlcPipj&bnW7VOaS>(P_JUc0j zsW^>}zr5vggwC1<%HympzX&RQtR8z|yn*`mu#Vnzut10_F#FOR8#7I0O`n~NC3+0MZS{&IAm7d4+IZ(Xjt$#K5Fp4uh$3ZyTv&90-_&^F9A zEOKgOs8997cL`RsB&bb3){c7E@z$~QvBjCcG3-hI9Q5I28~p#Ydrl-V;5d>eF()2u zCQH)V`@;$QTF2Ne zNg4a;dzU7y8{Phi)?~>{D`W4wJkQXH|96mE)`DALr*X2B+2T*|NJgqXfPGK$*-@UK z1RFiE;9kh%8FA~|BdzIf0hSUC{_bbNbctQ6RxZf~>iKrJh!AO!lWiO{O}dK?4dJhk zjWTuFjH+FxyL%$rx|5Tc(d|)e>p6crI_ggFccx41P|?j?k^<2o56&)q+bw4+;=agaaI z)>0pnTH5+R{>qB_a72#sCj`?Rx#wGFzi5{E(eL-APo5c#Dk)+~LM=2#yAfJrC&paC zLH>|!-P&(0P+u=A;qB&!BbLP&gerDwK9Q`P?hdhlI7gF4GgW!dQP86X$qN#~Y5VdP zD92mBj#}S59aU9SuSwp9!Z3F=g4_)COY+A;S6t+e!~E8WT$=Q~9AgY?eLCJfQaR2I z&-8<6hCptH0xgOXEwr9}#1eYeNw-)VZ%+2a!+)gC$$;{*!Km*?6YhG01^&u*J%7zc zebjRV6)pCe%njkV*}_pjIp#;Xtw)*3?(PM)^3y11XC%rfr9R4r(4+W7eo>Zr!qmR0MPzmd+B|C?qoQ*EXf zvO|d0`d$YcNQyV)&?(}YMEZ)jP63egGLGWwMXlaBujX!bvS<}cU)NyUDe-)oujUy= z^Hlme7>8thz_3;c7nO>`rlV@`F2wHZungO)h*vRC9zZASV{0foD&CpWqhT8C3G@q{fqu#!asxsu|o7No?`c;uaJcI+S%+&-}2E*YkgMBDYJ8z4{^EZ z=HXcSK0`f<`bfa+1J626BBS<2Ci)mfeLhG+$B3lwe_-;*Xq<#RsD&1L$(>MmDSVs>E?+Z0;f3F~f$B%s9lGD6GcNi(@AwkL&ijCde#%_huGI(2IUl$j4u`|x sa5x+ehr{7;I2;a#!{Kl^oUfh#1CbOKA}lZF4FCWD07*qoM6N<$f@+$i_5c6? diff --git a/packages/server/src/test/resources/knowledge/equation2.png b/packages/server/src/test/resources/knowledge/equation2.png deleted file mode 100644 index 716d1d4d6f4b5e50248890b5000a28a9d2ab5f28..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3240 zcmV;Z3|I4sP) ztgNi6s;cen?XIq_udlD-;^K96b?4{jbaZq+K0dOtvOGLItE;QCv$OvG{^{xI>gwvT zv9W7wYpJQJ;Nal2w6tz+Zsq0W=H}*icXy|!r)_O*rlzLjToSdAYp`k-VL!h9bH8nLfG&Go)m}X{XYHDhoot-^BJ#1`jq@<+W+}wG2d7`4C zI2keD000YHNkld3&2k62>`#K%j#bVDPY#jLA*f^dz@p!V6o!N8t z`2OEyyBdTzZ6g7)t^WOy2V^xgeZ5uHg~Bit3WY+UP$(1%g+ifFC=?2XvMVgdzY!Xu zkoeYH+1;t5d6E{B}p1BS_Jv&@&j+~Jh z$_Ai-stX6K&vDa+m1%n_;8RsTTy>6|VTNo_0Y4N}Hl78_U_22mA{(9Uhael%1J>#| zY0Y$SER63mWP{HyMyUc$!#&Xu9O8A)x1WP-Xw=)N7Y=uW#s)HxlnojZsh19(8Hkv% zeGu)fY*+?9|Ndb0I!;b;o4+MToEnpj1xyw0K(#tjxO0U5*d~Wl-1?2}Uucl3e$Kbz zkl}4m*{GE!biohgP;z3jLA{Oo;V(8Co|FyHbd}0Rt$G@kACPY1V<=$IPM9{* z{M;#%%El8J@ufLss|jxO6~nY0KV)d4Q^vIgdjjYB<=SpOevKn%iLOe!7KbN+_D%{58}GKN*w^F|8eF5x z#{SetdYMDR+(GHz%phVV$ui5^e5=*66{W-*VWwxu(0-U12)!v<65DwCkJr$d zpR_{u1$!Hr>bJW|;pgabVaNWoEPeT?i>Zo4h`q1|`mtDM&{{?69L6B5ti|jL_lsw~ zCwrTw=j}MvEzfkoZg2QnvB=#r@o)Dqo(#PxEqTsDd2=mzSozZ}m zhi2hPM}I+E*K1^KLc~tiFw13yzupa>a%B{bW|onQG)J*K%m7Y} za4!p6g++S9a$-|SE4&F-MhdHgvgY%A6LSS-2$kr2RLcPMHWD|)dRRbDVNWFWhPH`HRg1r32a6 zGZ!gSHo~ZgJJB@b9)IQDFz&&UzQpt7_QvIwJsmlV)qeM`fH9hNt^;rOK4YoD=AoM7 z`IEgCKr@FqUxCuOj+e0irgFAy^pWo0v~wTv)j#63RaYU-^s|ZdukVpj$hX=N=_-#! zdi5L`$dWWtTimws(ZVg?jTi~n954MhlZ?#YtL>wn4U&HhOE+^$`zfsZImtveQjHo$ zMOL$hH^x13KWjAREk***3P0=xGT9-1Tyd{^KMnaS%Hckx3fM5m!>B!&H!k~lsJ@N5 zZzJqoFIilBqod%4bJB9Gc-w2Mi8k>|udvzcmrnb_qio}}5Kw_s#aKNI5MbJB81e2S zM}gkJrnm(uQguYKq_L9$ggL$R0uQpWIX2^oT7GL`r$Xc539`(R=0`2;j)XouUV2)> zrgCP7#-bu*0ta_F!_Nnf`MJRG`u?s?`HUm(UL#?i@FK3V`e*bD)Z84V3u8L{wR7fo z$1MJrXlXU8G~^=fnZinBW1jdh78N08dtmOaZtkjhqL6MPw;@TX~ktCr?TmQ4j* zCxZc4Hn!n3P}ODhI^a5KZEO{hhV91Z_dGTzA!HTzIFhAPennM5DlxXCg+Z#Ovhu4P zd)d2sxLkkZYqB>4HR3wqie5zQbl8Q$i?WM72C0el*od#s4!Vy_c)T8H@!op4bU7)k zBuJIGFt%W$(esVBv3lFH{c0DIjr~~(Ru04DP@HZzQc0X+5?jlZ*^o>h@Wt(UT;K5` zp>NxvRL#2b;Rk61bMn!x;HlTMt`1=M#HlAWj9fI+8c#@aWGc3`ZIF*H>rV4g! z&CGYETe(rL_5QK}FAb+;V;Ng*!A?YNvS5+g!rE3xJY*NJ`GI=AzQPu_nGT%b_Qaz* zqtRfI*hJl2e4=Kj{$!VyIud!ww99p*9hJkyB}qj8&8QQ-gc6I2kn#4f7UPOYlQ-3V zb{=>D_BMV;`izr5oWVwZrjIr{xzBJU$oh->Z*9x$2Yh^<6XD1#-Tc%u3?gKPqwQ$m zjJt+Geqhkho$H8nTqP8W{IVBsBFEZPyM&x_mX%L!q5t?73xT~2%Lx?T9Oi6~DRxHn zv1~X3iBaj0-gcKYs|a#N>2}nzSY{M@9O?JSC2qIvzw#0b(a3*3%^xk`rS6|S((39k z5Z;&kssMN!@}=i!+G72{hG_-@%zWk4REcx9C{{0NaI^UG{ct*+&aURg?i*q?KEAn} ze#F-hqal-6t_Xo0dgt1D;~EXLT@&ddaQ-0JGyI$c%LdFg*Bup3{aSJ4`I?;dW!1Sucp|)k%Jz?w zu)<6n%8D-=Bc9`i`0(GHz|oh+lgPK-YF!_8rbBw=-GBToD^FKd)sOJ#pDMlai-fIq zvcs#Y#`w-bJt7;;XmsYe28LfPJcPMh!Oje1kM;RQ|0_hnd7h^~s8BEPs}=_F_#iAF z_|4#W)MGhNqO2eo;{WtZf1BwbWbAiQs>ixR=)Zo*aTfD7=nq970N5uvCr7YJ67e=l z2~trC0Rj;u3xSFUvN14T&fB)H3G&m1{w$!fad03bB1onL*$8}oF(bQLr@PDUu01L0000`}6bj_xJbp_4V%V?yRh=aBy%rIXOByIy*Z%0002;^78rl`Kqd_adB~Tb91e& zt@ifz`1ts3ZEc~Up*=l4^z`)o{r%?V=BcTvZfhsCb+faxbaZsG zva)JwYU}Ik=;-L|?Cdu;H>9MbgwvFqocL8 zwP|T-l$4a9pr8~K6sD%8G&D4{w6u11c24p3#{d8agGod|RCt{2TMc{KHW0P}8_CGn z*iMHrarqpW43aEtNw##QP1=2Q+yDPL(@Dmb!AVoFG2X-b^l2qq!Ug! z;e-=jfLy886jd(TjBHUy7;6opvq{rH3?~1(WvO<-=IsXOi~uxlz4C@6@tsz07i{4! za64X?T8~^dlg}+5rUdzx=e_M|bl4vhY~pShQq!&ERIBl^Goq7Gtr)xZgHJ6VC#S}G zqQl00UxZbALPmr%Vy7Tux{k?uHbrn8gF+133pGj}(~YJ!aA~%lO<|3uQ;1=E;ey)T zil<{b#nCeC2A%#eEyAw-06bEY(}-5moFrXI31E*GV@KMc6%uL1NT!u3V$CkKD?+;0 zDYVkpGmB1sTohv7PN?{lF}P2o(KfBLX86nvoh-|=s5&+NMwV)D#7HN819s<4wOZB3 zi=_$5qC|^gKH(=Az;-m0==BN`Rp7`R7G~iNfOFPoqc-K18GCL`jn zY;lWcDYSBA*L9l)yi+a6!dy350&n4% zmPZ2G%Sh@utt71)f(X2Wtg$?+8c+{GxYSFb6<)_+@RM43Rt2;|dd@9oFo`kD%XU$^l{5G_y z@je@YGc=`uEV(RKS+okDVyl8Z^hw^qxROO$I`w!Unu0GR9!;WX@T$}l1i+#qy(2EvRuSmaR10$+-@{%X7I8}jVJxp4&ul%%Ep?$efhpKw7s_N~=iJlXjjjL5Y zppTaederVZE9BFkDbFrCkB_(XGat`Y{u%OxLG#KhGZR+1BOw>)Cftwf#mV=+n6CQ; zQ}@|t?$XcNQ-YZl79nxy6Yox{+jSErlphoSDbXr}KdOR76MTKc7-CXC0)itBl)<)2L zhabwy-1004)t?N@gXO43um7PcIg>dy(a{#g8Lh^fuSRNVCKe*;V+%}Zf`^_U1El{R ztlz2I&mdi50j=<+ilqs6=oRfx2*FmZ;POCIsv&Yd&|pR8=i%=}yUTYi5o)W(GdN5A zVi~PA)mc(4LGlwuEyX2P7Og@NSnI9=c4mQydVQixFJVG<#lEXbjaFXbObWD;np0^?&usry zdC=sqU9sn?Qbk}wElEr4%VfsTHU7PR;deryu1b zfjl>?x6of{m{whM%ggUZJMTN;iE`)+iI-Nn_52$MGRaF^!l z+l4=Sp~Q!!IorQ!(L6)YKUnvObw6+EPu0>vE)#tTB{GGI$tvdnm{d#oLAZKh&}2Mk zv0^0DQf?5gUf4J#8P67qmU~~8LAZJWVLj(-sT*o3HyAOkVECBphq+!##XS!ItwJu* zRc*$Sj6v1X-ofZAsOf{Y++5R2cHxFQ4k~Vru}ddjzPD~>m#r((7(MWOdcHDE2wXj? z1ahQSFwgZY1Xcrzo#-<$%s1lS(sbd8a`OpGE>~Ig#}Z8 zPt;}f0~Wa8cbHnj{kBp8>hrJw)-fZB)F%(zZ_uTllU`X}20eoBpY(qa8PWAw9kN>R z4IEaNq;aSl4w(a-1ffCK(+#HCujy-Uoeh`QMH{y}T=ieGXDk9~dX2iK&xjPvJ>0jB z_Ttd%PzgJ+ZY`?$xK4R~=jDVGPB`I&6HYkcgjD