Support for handling updates for requests and/or samples. (#594)

Added changes to addresses issues encountered when certain needs evolved related to the handling of requests and/or samples being updated. Mocked data has been updated as well. Signed-off-by: Divya Madala <[email protected]> Signed-off-by: Angelica Ochoa <[email protected]> Co-authored-by: divyamadala30 <[email protected]>
mskcc · Feb 10, 2022 · 6295d3f · 6295d3f
1 parent bbdb78e
commit 6295d3f
Show file tree

Hide file tree

Showing 23 changed files with 924 additions and 121 deletions.
diff --git a/model/src/main/java/org/mskcc/cmo/metadb/model/MetadbRequest.java b/model/src/main/java/org/mskcc/cmo/metadb/model/MetadbRequest.java
@@ -67,6 +67,18 @@ public class MetadbRequest implements Serializable {
 
     public MetadbRequest() {}
 
+    /**
+     * MetadbRequest constructor.
+     * @param requestMetadata
+     * @throws JsonProcessingException 
+     */
+    public MetadbRequest(RequestMetadata requestMetadata) throws JsonProcessingException {
+        this.igoRequestId = requestMetadata.getIgoRequestId();
+        this.igoProjectId = igoRequestId.split("_")[0];
+        this.metaDbProject = new MetadbProject(igoProjectId);
+        updateRequestMetadataByMetadata(requestMetadata);
+    }
+
     /**
      * MetadbRequest constructor.
      * @param igoRequest
@@ -365,8 +377,7 @@ public void updateRequestMetadataByMetadata(RequestMetadata requestMetadata)
         Map<String, Object> metadataMap =
                 mapper.readValue(requestMetadata.getRequestMetadataJson(), Map.class);
 
-        this.igoRequestId = String.valueOf(metadataMap.get("requestId"));
-        this.genePanel = String.valueOf(metadataMap.get("recipe"));
+        this.genePanel = resolveGenePanel(metadataMap);
         this.projectManagerName = String.valueOf(metadataMap.get("projectManagerName"));
         this.piEmail = String.valueOf(metadataMap.get("piEmail"));
         this.labHeadName = String.valueOf(metadataMap.get("labHeadName"));
@@ -381,10 +392,23 @@ public void updateRequestMetadataByMetadata(RequestMetadata requestMetadata)
         this.libraryType = String.valueOf(metadataMap.get("libraryType"));
         this.bicAnalysis = Boolean.parseBoolean(String.valueOf(metadataMap.get("bicAnalysis")));
         this.isCmoRequest = Boolean.parseBoolean(String.valueOf(metadataMap.get("isCmoRequest")));
-        this.metaDbProject = new MetadbProject(igoRequestId.split("_")[0]);
         addRequestMetadata(requestMetadata);
     }
 
+    /**
+     * Resolves gene panel from recipe or genePanel sample json field.
+     * @param metadataMap
+     * @return String
+     */
+    public String resolveGenePanel(Map<String, Object> metadataMap) {
+        Object genePanel = (metadataMap.containsKey("recipe"))
+                ? metadataMap.get("recipe") : metadataMap.get("genePanel");
+        if (genePanel != null) {
+            return String.valueOf(genePanel);
+        }
+        return null;
+    }
+
     /**
      * Returns the latest RequestMetadata.
      * Collections is sorting by date in ascending order so the last item

diff --git a/model/src/main/java/org/mskcc/cmo/metadb/model/SampleMetadata.java b/model/src/main/java/org/mskcc/cmo/metadb/model/SampleMetadata.java
@@ -1,6 +1,7 @@
 package org.mskcc.cmo.metadb.model;
 
 import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
 import com.fasterxml.jackson.annotation.JsonInclude;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import java.io.Serializable;
@@ -22,6 +23,7 @@
 import org.neo4j.ogm.annotation.typeconversion.Convert;
 
 @JsonInclude(JsonInclude.Include.NON_NULL)
+@JsonIgnoreProperties(ignoreUnknown = true)
 @NodeEntity
 public class SampleMetadata implements Serializable, Comparable<SampleMetadata> {
     @Id @GeneratedValue
@@ -109,15 +111,6 @@ public SampleMetadata(DmpSampleMetadata dmpSampleMetadata) {
         this.cmoSampleIdFields = new HashMap<>(); // default for dmp samples
     }
 
-    Map<String, String> setUpMetastasisValueMapping() {
-        Map<String, String> metastasisValueMapping = new HashMap<>();
-        metastasisValueMapping.put("0", "Primary");
-        metastasisValueMapping.put("1", "Metastasis");
-        metastasisValueMapping.put("2", "Local Recurrence");
-        metastasisValueMapping.put("127", "Unknown");
-        return metastasisValueMapping;
-    }
-
     public Long getId() {
         return id;
     }

diff --git a/model/src/main/java/org/mskcc/cmo/metadb/model/web/PublishedMetadbSample.java b/model/src/main/java/org/mskcc/cmo/metadb/model/web/PublishedMetadbSample.java
@@ -2,6 +2,7 @@
 
 import java.text.ParseException;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
@@ -52,6 +53,7 @@ public class PublishedMetadbSample {
     private List<Library> libraries;
     private List<SampleAlias> sampleAliases;
     private List<PatientAlias> patientAliases;
+    private Map<String, String> additionalProperties = new HashMap<>();
 
     public PublishedMetadbSample() {}
 
@@ -91,6 +93,7 @@ public PublishedMetadbSample(MetadbSample metaDbSample) throws ParseException {
         this.sampleAliases = metaDbSample.getSampleAliases();
         this.cmoSampleIdFields = latestSampleMetadata.getCmoSampleIdFields();
         this.patientAliases = metaDbSample.getPatient().getPatientAliases();
+        this.additionalProperties = latestSampleMetadata.getAdditionalProperties();
     }
 
     public UUID getMetaDbSampleId() {
@@ -347,6 +350,18 @@ public void setPatientAliases(List<PatientAlias> patientAliases) {
         this.patientAliases = patientAliases;
     }
 
+    public Map<String, String> getAdditionalProperties() {
+        return additionalProperties;
+    }
+
+    public void setAdditionalProperties(Map<String, String> additionalProperties) {
+        this.additionalProperties = additionalProperties;
+    }
+
+    public void addAdditionalProperty(String property, String value) {
+        this.additionalProperties.put(property, value);
+    }
+
     @Override
     public String toString() {
         return ToStringBuilder.reflectionToString(this);

diff --git a/persistence/src/main/java/org/mskcc/cmo/metadb/persistence/neo4j/MetadbSampleRepository.java b/persistence/src/main/java/org/mskcc/cmo/metadb/persistence/neo4j/MetadbSampleRepository.java
@@ -83,4 +83,5 @@ List<MetadbSample> findAllSamplesByCategoryAndCmoPatientId(
             + "RETURN sm")
     List<SampleMetadata> findSampleMetadataHistoryByNamespaceValue(
             @Param("namespace") String namespace, @Param("value") String value);
+
 }
diff --git a/pom.xml b/pom.xml
@@ -48,10 +48,10 @@
     <testcontainers.version>1.16.0</testcontainers.version>
     <!-- metadb messaging and shared entities dependency versions -->
     <cmo_metadb_messaging_java.group>com.github.mskcc</cmo_metadb_messaging_java.group>
-    <cmo_metadb_messaging_java.version>1.2.0.RELEASE</cmo_metadb_messaging_java.version>
+    <cmo_metadb_messaging_java.version>1.2.1.RELEASE</cmo_metadb_messaging_java.version>
     <!-- metadb common centralized config properties -->
     <cmo_metadb_common.group>com.github.mskcc</cmo_metadb_common.group>
-    <cmo_metadb_common.version>1.2.0.RELEASE</cmo_metadb_common.version>
+    <cmo_metadb_common.version>1.2.1.RELEASE</cmo_metadb_common.version>
     <!-- metadb expected schema version -->
     <metadb.schema_version>v2.0</metadb.schema_version>
   </properties>

diff --git a/service/src/main/java/org/mskcc/cmo/metadb/service/impl/MessageHandlingServiceImpl.java b/service/src/main/java/org/mskcc/cmo/metadb/service/impl/MessageHandlingServiceImpl.java
@@ -230,18 +230,9 @@ public void run() {
                         MetadbRequest existingRequest =
                                 requestService.getMetadbRequestById(requestMetadata.getIgoRequestId());
                         if (existingRequest == null) {
-                            LOG.info("Request does not already exist - saving to database: "
-                                    + requestMetadata.getIgoRequestId());
-                            // persist and handle new request
-                            MetadbRequest request =
-                                    RequestDataFactory.buildNewRequestFromMetadata(requestMetadata);
-
-                            LOG.info("Publishing new request metadata to " + CMO_REQUEST_UPDATE_TOPIC);
-                            requestService.saveRequest(request);
-                            messagingGateway.publish(request.getIgoRequestId(),
-                                    CMO_REQUEST_UPDATE_TOPIC,
-                                    mapper.writeValueAsString(
-                                          request.getRequestMetadataList()));
+                            LOG.warn("Request does not already exist in the database: "
+                                    + requestMetadata.getIgoRequestId()
+                                    + " - will not be persisting updates.");
                         } else if (requestService.requestHasMetadataUpdates(
                                 existingRequest.getLatestRequestMetadata(), requestMetadata)) {
                             // persist request-level metadata updates to database
@@ -310,7 +301,15 @@ public void run() {
                             messagingGateway.publish(CMO_SAMPLE_UPDATE_TOPIC,
                                     mapper.writeValueAsString(sample.getSampleMetadataList()));
                         } else if (sampleService.sampleHasMetadataUpdates(
-                                existingSample.getLatestSampleMetadata(), sampleMetadata)) {
+                                existingSample.getLatestSampleMetadata(), sampleMetadata)
+                                || (!sampleService.sampleHasMetadataUpdates(
+                                        existingSample.getLatestSampleMetadata(), sampleMetadata))
+                                && !existingSample.getLatestSampleMetadata().getCmoSampleName()
+                                        .equals(sampleMetadata.getCmoSampleName())) {
+                            // logic checks if 1. comparator detects changes or 2. comparator does not
+                            // detect changes because 'cmoSampleName' is ignored but the 'cmoSampleName'
+                            // for existing and current sample metadata clearly differ then proceed with
+                            // persisting updates for sample
                             LOG.info("Found updates for research sample - persisting to database: "
                                     + sampleMetadata.getPrimaryId());
                             // persist sample level updates to database and publish

diff --git a/service/src/main/java/org/mskcc/cmo/metadb/service/impl/RequestServiceImpl.java b/service/src/main/java/org/mskcc/cmo/metadb/service/impl/RequestServiceImpl.java
@@ -163,6 +163,9 @@ public PublishedMetadbRequest getPublishedMetadbRequestById(String requestId) th
         for (MetadbSample sample : request.getMetaDbSampleList()) {
             PublishedMetadbSample publishedSample = sampleService
                     .getPublishedMetadbSample(sample.getMetaDbSampleId());
+            // add request id and cmo indicator to sample's additional properties
+            publishedSample.addAdditionalProperty("igoRequestId", requestId);
+            publishedSample.addAdditionalProperty("isCmoSample", String.valueOf(request.getIsCmoRequest()));
             samples.add(publishedSample);
         }
         return new PublishedMetadbRequest(request, samples);

diff --git a/service/src/main/java/org/mskcc/cmo/metadb/service/util/RequestDataFactory.java b/service/src/main/java/org/mskcc/cmo/metadb/service/util/RequestDataFactory.java
@@ -38,20 +38,6 @@ public static MetadbRequest buildNewLimsRequestFromJson(String requestJson)
         return request;
     }
 
-    /**
-     * Method factory returns an instance of MetadbRequest built from
-     * an instance of RequestMetadata;
-     * @param requestMetadata
-     * @return MetadbRequest
-     * @throws JsonProcessingException
-     */
-    public static MetadbRequest buildNewRequestFromMetadata(RequestMetadata requestMetadata)
-            throws JsonProcessingException {
-        MetadbRequest request = new MetadbRequest();
-        request.updateRequestMetadataByMetadata(requestMetadata);
-        return request;
-    }
-
     /**
      * Method factory returns an instance of RequestMetadata built from
      * request metadata JSON string.
@@ -86,8 +72,11 @@ private static RequestMetadata extractRequestMetadataFromJson(String requestMeta
         if (requestMetadataMap.containsKey("samples")) {
             requestMetadataMap.remove("samples");
         }
+        Object requestId = requestMetadataMap.containsKey("requestId")
+                ? requestMetadataMap.get("requestId") : requestMetadataMap.get("igoRequestId");
+
         RequestMetadata requestMetadata = new RequestMetadata(
-                requestMetadataMap.get("requestId").toString(),
+                requestId.toString(),
                 mapper.writeValueAsString(requestMetadataMap),
                 LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE));
         return requestMetadata;

diff --git a/service/src/main/java/org/mskcc/cmo/metadb/service/util/SampleDataFactory.java b/service/src/main/java/org/mskcc/cmo/metadb/service/util/SampleDataFactory.java
@@ -116,6 +116,16 @@ public static SampleMetadata buildNewSampleMetadatafromJson(String sampleMetadat
                 mapper.readValue(sampleMetadataJson, SampleMetadata.class);
         sampleMetadata.setImportDate(
                 LocalDateTime.now().format(DateTimeFormatter.ISO_LOCAL_DATE));
+        // resolve igo request id if null from additionalProperties if possible
+        if (sampleMetadata.getIgoRequestId() == null
+                && !sampleMetadata.getAdditionalProperties().isEmpty()) {
+            Map<String, String> additionalProperties = sampleMetadata.getAdditionalProperties();
+            if (additionalProperties.containsKey("requestId")) {
+                sampleMetadata.setIgoRequestId(additionalProperties.get("requestId"));
+            } else if (additionalProperties.containsKey("igoRequestId")) {
+                sampleMetadata.setIgoRequestId(additionalProperties.get("igoRequestId"));
+            }
+        }
         return sampleMetadata;
     }
 

diff --git a/service/src/test/java/org/mskcc/cmo/metadb/service/RequestDataFactoryUtilTest.java b/service/src/test/java/org/mskcc/cmo/metadb/service/RequestDataFactoryUtilTest.java
@@ -0,0 +1,36 @@
+package org.mskcc.cmo.metadb.service;
+
+import java.util.Map;
+import org.junit.jupiter.api.Test;
+import org.mskcc.cmo.metadb.model.MetadbRequest;
+import org.mskcc.cmo.metadb.service.util.RequestDataFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.context.annotation.Import;
+
+/**
+ *
+ * @author ochoaa
+ */
+@SpringBootTest
+@Import(MockDataUtils.class)
+public class RequestDataFactoryUtilTest {
+    @Autowired
+    private MockDataUtils mockDataUtils;
+
+    /**
+     * Tests that deserialization of igo sample jsons succeeds and that the sample
+     * data factory can convert the igo sample into an instance of SampleMetadata.
+     * @throws Exception
+     */
+    @Test
+    public void testResearchSamplesAndRequestDataLoading() throws Exception {
+        for (Map.Entry<String, MockJsonTestData> entry : mockDataUtils.mockedRequestJsonDataMap.entrySet()) {
+            if (!entry.getKey().startsWith("mockIncoming")) {
+                continue;
+            }
+            String jsonString = entry.getValue().getJsonString();
+            MetadbRequest request = RequestDataFactory.buildNewLimsRequestFromJson(jsonString);
+        }
+    }
+}
diff --git a/service/src/test/java/org/mskcc/cmo/metadb/service/RequestServiceTest.java b/service/src/test/java/org/mskcc/cmo/metadb/service/RequestServiceTest.java
@@ -98,7 +98,6 @@ public void initializeMockDatabase() throws Exception {
         requestService.saveRequest(request5);
     }
 
-
     /**
      * Tests integrity of mock database by fetching request.
      * @throws Exception
@@ -162,6 +161,34 @@ public void testRequestHasMetadataUpdates() throws Exception {
         Boolean hasUpdates = requestService.requestHasUpdates(
                 origRequest, updatedRequest);
         Assertions.assertThat(hasUpdates).isEqualTo(Boolean.TRUE);
+
+
+        RequestMetadata updatedMetadata = RequestDataFactory.buildNewRequestMetadataFromMetadata(
+                updatedRequestData.getJsonString());
+        Boolean hasMetadataUpdates = requestService.requestHasMetadataUpdates(
+                origRequest.getLatestRequestMetadata(), updatedMetadata);
+        Assertions.assertThat(hasMetadataUpdates).isEqualTo(Boolean.TRUE);
+    }
+
+    /**
+     * Tests instance of RequestMetadata that's been updated against the latest request
+     * metadata persisted in the database for the matching Request ID.
+     * @throws Exception
+     */
+    @Test
+    public void testRequestHasUpdatesWithUniversalSchema() throws Exception {
+        String requestId = "MOCKREQUEST1_B";
+        MetadbRequest origRequest = requestService.getMetadbRequestById(requestId);
+        Assertions.assertThat(requestHasExpectedFieldsPopulated(origRequest)).isTrue();
+
+        MockJsonTestData updatedRequestMetadataData = mockDataUtils.mockedRequestJsonDataMap
+                .get("mockUpdatedPublishedRequest1Metadata");
+        RequestMetadata updatedRequestMetadata = RequestDataFactory
+                .buildNewRequestMetadataFromMetadata(updatedRequestMetadataData.getJsonString());
+
+        Boolean hasUpdates = requestService.requestHasMetadataUpdates(
+                origRequest.getLatestRequestMetadata(), updatedRequestMetadata);
+        Assertions.assertThat(hasUpdates).isEqualTo(Boolean.TRUE);
     }
 
     /**

diff --git a/service/src/test/java/org/mskcc/cmo/metadb/service/SampleDataFactoryUtilTest.java b/service/src/test/java/org/mskcc/cmo/metadb/service/SampleDataFactoryUtilTest.java
@@ -5,10 +5,8 @@
 import java.util.Map;
 import junit.framework.Assert;
 import org.junit.jupiter.api.Test;
-import org.mskcc.cmo.metadb.model.MetadbRequest;
 import org.mskcc.cmo.metadb.model.SampleMetadata;
 import org.mskcc.cmo.metadb.model.dmp.DmpSampleMetadata;
-import org.mskcc.cmo.metadb.service.util.RequestDataFactory;
 import org.mskcc.cmo.metadb.service.util.SampleDataFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.test.context.SpringBootTest;
@@ -27,22 +25,6 @@ public class SampleDataFactoryUtilTest {
     private Map<String, SampleMetadata> expectedConvertedDmpSampleValues
             = initExpectedConvertedDmpSampleValues();
 
-    /**
-     * Tests that deserialization of igo sample jsons succeeds and that the sample
-     * data factory can convert the igo sample into an instance of SampleMetadata.
-     * @throws Exception
-     */
-    @Test
-    public void testResearchSamplesAndRequestDataLoading() throws Exception {
-        for (Map.Entry<String, MockJsonTestData> entry : mockDataUtils.mockedRequestJsonDataMap.entrySet()) {
-            if (!entry.getKey().startsWith("mockIncoming")) {
-                continue;
-            }
-            String jsonString = entry.getValue().getJsonString();
-            MetadbRequest request = RequestDataFactory.buildNewLimsRequestFromJson(jsonString);
-        }
-    }
-
     /**
      * Tests that deserialization of dmp sample jsons succeeds and that the sample
      * data factory can convert the dmp sample into an instance of SampleMetadata.

diff --git a/service/src/test/resources/data/mocked_request_data_details.txt b/service/src/test/resources/data/mocked_request_data_details.txt
@@ -13,3 +13,4 @@ mockPublishedRequest2bJsonDataComplete	published_requests/outgoing_mocked_reques
 mockPublishedRequest3JsonDataPooledNormals	published_requests/outgoing_mocked_request3_pooled_normals.json	Mock published request with pooled normals, and metadb ids.
 mockPublishedRequest1JsonNullValues	published_requests/outgoing_mocked_request1_null_values.json	Mock published request with null values.
 mockPublishedRequest4JsonNullOrEmptyValues	published_requests/outgoing_mocked_request4_null_or_empty_values.json	Mock published request with null or empty values.
+mockUpdatedPublishedRequest1Metadata	other_mocked_data/outgoing_mocked_request1_updated_metadata.json	Mock published request metadata json with updates.
diff --git a/.../src/test/resources/data/other_mocked_data/outgoing_mocked_request1_updated_metadata.json b/.../src/test/resources/data/other_mocked_data/outgoing_mocked_request1_updated_metadata.json
@@ -0,0 +1,31 @@
+{
+  "metaDbRequestId": "6cc9f987-875a-11eb-ada3-acde48001122",
+  "igoRequestId": "MOCKREQUEST1_B",
+  "genePanel": "GENESET101_BAITS_NEWNAME",
+  "projectManagerName": "Bar, Foo",
+  "piEmail": "[email protected]",
+  "labHeadName": "Foo Bar",
+  "labHeadEmail": "[email protected]",
+  "investigatorName": "John Smith",
+  "investigatorEmail": "[email protected]",
+  "dataAnalystName": "Poin Dexter",
+  "dataAnalystEmail": "[email protected]",
+  "otherContactEmails": "[email protected]",
+  "dataAccessEmails": "",
+  "qcAccessEmails": "",
+  "isCmoRequest": true,
+  "bicAnalysis": false,
+  "pooledNormals": [
+    "/FASTQ/Project_POOLEDNORMALS/Sample_FFPEPOOLEDNORMAL_IGO_GENESET101_TAGCTTGA/FFPEPOOLEDNORMAL_IGO_GENESET101_TAGCTTGA_S23_R1_001.fastq.gz",
+    "/FASTQ/Project_POOLEDNORMALS/Sample_FFPEPOOLEDNORMAL_IGO_GENESET101_TAGCTTGA/FFPEPOOLEDNORMAL_IGO_GENESET101_TAGCTTGA_S23_R2_001.fastq.gz",
+    "/FASTQ/Project_POOLEDNORMALS/Sample_FROZENPOOLEDNORMAL_IGO_AMBIGUOUS_TTAGGCTG/FROZENPOOLEDNORMAL_IGO_AMBIGUOUS_TTAGGCTG_S86_R1_001.fastq.gz",
+    "/FASTQ/Project_POOLEDNORMALS/Sample_FROZENPOOLEDNORMAL_IGO_AMBIGUOUS_TTAGGCTG/FROZENPOOLEDNORMAL_IGO_AMBIGUOUS_TTAGGCTG_S86_R2_001.fastq.gz",
+    "/FASTQ/Project_POOLEDNORMALS/Sample_FROZENPOOLEDNORMAL_IGO_HEMESET_v1_TTAGGCTG/FROZENPOOLEDNORMAL_IGO_HEMESET_v1_TTAGGCTG_S147_R1_001.fastq.gz",
+    "/FASTQ/Project_POOLEDNORMALS/Sample_FROZENPOOLEDNORMAL_IGO_HEMESET_v1_TTAGGCTG/FROZENPOOLEDNORMAL_IGO_HEMESET_v1_TTAGGCTG_S147_R2_001.fastq.gz",
+    "/FASTQ/Project_POOLEDNORMALS/Sample_FROZENPOOLEDNORMAL_IGO_GENESET101_TTAGGCTG/FROZENPOOLEDNORMAL_IGO_GENESET101_TTAGGCTG_S196_R1_001.fastq.gz",
+    "/FASTQ/Project_POOLEDNORMALS/Sample_FROZENPOOLEDNORMAL_IGO_GENESET101_TTAGGCTG/FROZENPOOLEDNORMAL_IGO_GENESET101_TTAGGCTG_S196_R2_001.fastq.gz",
+    "/FASTQ/Project_POOLEDNORMALS/Sample_MOUSEPOOLEDNORMAL_IGO_AMBIGUOUS_GGCGTCAT/MOUSEPOOLEDNORMAL_IGO_AMBIGUOUS_GGCGTCAT_S61_R1_001.fastq.gz",
+    "/FASTQ/Project_POOLEDNORMALS/Sample_MOUSEPOOLEDNORMAL_IGO_AMBIGUOUS_GGCGTCAT/MOUSEPOOLEDNORMAL_IGO_AMBIGUOUS_GGCGTCAT_S61_R2_001.fastq.gz"
+  ],
+  "igoProjectId": "MOCKREQUEST1"
+}