MODLD-643: Preservation of MARC fields that are not rendered in the UI (

#94)

src/main/java/org/folio/linked/data/mapper/ResourceModelMapper.java

@@ -16,6 +16,7 @@
 import org.folio.ld.dictionary.ResourceTypeDictionary;
 import org.folio.linked.data.model.entity.FolioMetadata;
 import org.folio.linked.data.model.entity.PredicateEntity;
+import org.folio.linked.data.model.entity.RawMarc;
 import org.folio.linked.data.model.entity.Resource;
 import org.folio.linked.data.model.entity.ResourceTypeEntity;
 import org.mapstruct.BeforeMapping;
@@ -36,6 +37,8 @@ public Resource toEntity(org.folio.ld.dictionary.model.Resource model) {
 
   @Mapping(target = "folioMetadata",
     expression = "java(model.getFolioMetadata() != null ? mapFolioMetadata(model.getFolioMetadata(), resource) : null)")
+  @Mapping(target = "unmappedMarc",
+    expression = "java(model.getUnmappedMarc() != null ? mapUnmappedMarc(model.getUnmappedMarc(), resource) : null)")
   protected abstract Resource toEntity(org.folio.ld.dictionary.model.Resource model,
                                        @Context CyclicGraphContext cycleContext);
 
@@ -63,6 +66,9 @@ protected PredicateDictionary map(PredicateEntity predicateEntity) {
   protected abstract FolioMetadata mapFolioMetadata(org.folio.ld.dictionary.model.FolioMetadata folioMetadata,
                                                     Resource resource);
 
+  @Mapping(source = "resource", target = "resource")
+  protected abstract RawMarc mapUnmappedMarc(org.folio.ld.dictionary.model.RawMarc unmappedMarc, Resource resource);
+
   @Qualifier
   @Target(ElementType.METHOD)
   @Retention(RetentionPolicy.CLASS)

src/main/java/org/folio/linked/data/model/entity/RawMarc.java

@@ -0,0 +1,46 @@
+package org.folio.linked.data.model.entity;
+
+import static lombok.AccessLevel.PROTECTED;
+
+import com.vladmihalcea.hibernate.type.json.JsonBinaryType;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.JoinColumn;
+import jakarta.persistence.MapsId;
+import jakarta.persistence.OneToOne;
+import jakarta.persistence.Table;
+import lombok.Data;
+import lombok.EqualsAndHashCode;
+import lombok.NoArgsConstructor;
+import lombok.ToString;
+import lombok.experimental.Accessors;
+import org.hibernate.annotations.Type;
+
+@Entity
+@Data
+@NoArgsConstructor(access = PROTECTED)
+@Accessors(chain = true)
+@Table(name = "raw_marcs")
+@EqualsAndHashCode(of = "id")
+public class RawMarc {
+
+  @Id
+  @Column(name = "resource_hash")
+  private Long id;
+
+  @Column(columnDefinition = "json")
+  @Type(JsonBinaryType.class)
+  private String content;
+
+  @OneToOne
+  @MapsId
+  @JoinColumn(name = "resource_hash")
+  @ToString.Exclude
+  private Resource resource;
+
+  public RawMarc(Resource resource) {
+    this.resource = resource;
+    this.id = resource.getId();
+  }
+}

src/main/java/org/folio/linked/data/model/entity/Resource.java

@@ -102,6 +102,10 @@ public class Resource implements Persistable<Long> {
   @PrimaryKeyJoinColumn
   private FolioMetadata folioMetadata;
 
+  @OneToOne(cascade = ALL, mappedBy = "resource", orphanRemoval = true)
+  @PrimaryKeyJoinColumn
+  private RawMarc unmappedMarc;
+
   @Column(name = "created_date", updatable = false, nullable = false)
   private Timestamp createdDate;
 

src/main/java/org/folio/linked/data/service/resource/edge/ResourceEdgeServiceImpl.java

@@ -1,7 +1,12 @@
 package org.folio.linked.data.service.resource.edge;
 
 import static org.folio.ld.dictionary.PredicateDictionary.ADMIN_METADATA;
+import static org.folio.ld.dictionary.PredicateDictionary.DISSERTATION;
+import static org.folio.ld.dictionary.PredicateDictionary.GENRE;
+import static org.folio.ld.dictionary.PredicateDictionary.ILLUSTRATIONS;
+import static org.folio.ld.dictionary.PredicateDictionary.SUPPLEMENTARY_CONTENT;
 import static org.folio.ld.dictionary.ResourceTypeDictionary.INSTANCE;
+import static org.folio.ld.dictionary.ResourceTypeDictionary.WORK;
 
 import java.util.Map;
 import java.util.Set;
@@ -23,18 +28,21 @@
 public class ResourceEdgeServiceImpl implements ResourceEdgeService {
 
   private static final Map<ResourceTypeDictionary, Set<PredicateDictionary>> EDGES_TO_BE_COPIED = Map.of(
-    INSTANCE, Set.of(ADMIN_METADATA)
+    INSTANCE, Set.of(ADMIN_METADATA),
+    WORK, Set.of(ILLUSTRATIONS, SUPPLEMENTARY_CONTENT, DISSERTATION, GENRE)
   );
   private final ResourceRepository resourceRepository;
   private final ResourceModelMapper resourceModelMapper;
   private final ResourceEdgeRepository resourceEdgeRepository;
 
   @Override
   public void copyOutgoingEdges(Resource from, Resource to) {
-    getEdgesToBeCopied(from)
-      .stream()
-      .map(edge -> new ResourceEdge(to, edge.getTarget(), edge.getPredicate()))
-      .forEach(to::addOutgoingEdge);
+    if (from.getTypes().equals(to.getTypes())) {
+      getEdgesToBeCopied(from)
+        .stream()
+        .map(edge -> new ResourceEdge(to, edge.getTarget(), edge.getPredicate()))
+        .forEach(to::addOutgoingEdge);
+    }
   }
 
   @Override

src/main/resources/changelog/scripts/v-1.0.0/resource_graph/changelog.xml

@@ -9,6 +9,7 @@
   <include file="tables/type_lookup.sql" relativeToChangelogFile="true"/>
   <include file="tables/resources.sql" relativeToChangelogFile="true"/>
   <include file="tables/folio_metadata.sql" relativeToChangelogFile="true"/>
+  <include file="tables/raw_marcs.sql" relativeToChangelogFile="true"/>
   <include file="tables/resource_edges.sql" relativeToChangelogFile="true"/>
   <include file="tables/resource_type_map.sql" relativeToChangelogFile="true"/>
   <include file="tables/resource_context_map.sql" relativeToChangelogFile="true"/>

src/main/resources/changelog/scripts/v-1.0.0/resource_graph/tables/raw_marcs.sql

@@ -0,0 +1,8 @@
+create table if not exists raw_marcs (
+     resource_hash bigint primary key references resources(resource_hash),
+     content jsonb null
+  );
+
+comment on table raw_marcs is 'Store unmapped MARC records associated with an Instance resource. Applicable only for Instance resources';
+comment on column raw_marcs.resource_hash is 'The unique hash identifier for the resource';
+comment on column raw_marcs.content is 'JSON representation of MARC record';

src/test/java/org/folio/linked/data/e2e/resource/ResourceControllerITBase.java

@@ -618,8 +618,8 @@ void deleteResourceById_shouldDeleteRootInstanceAndRootEdges_reindexWork() throw
     var work = getSampleWork(null);
     var instance = resourceTestService.saveGraph(getSampleInstanceResource(null, work));
     assertThat(resourceTestService.findById(instance.getId())).isPresent();
-    assertThat(resourceTestService.countResources()).isEqualTo(59);
-    assertThat(resourceTestService.countEdges()).isEqualTo(61);
+    assertThat(resourceTestService.countResources()).isEqualTo(61);
+    assertThat(resourceTestService.countEdges()).isEqualTo(63);
     var requestBuilder = delete(RESOURCE_URL + "/" + instance.getId())
       .contentType(APPLICATION_JSON)
       .headers(defaultHeaders(env));
@@ -630,9 +630,9 @@ void deleteResourceById_shouldDeleteRootInstanceAndRootEdges_reindexWork() throw
     // then
     resultActions.andExpect(status().isNoContent());
     assertThat(resourceTestService.existsById(instance.getId())).isFalse();
-    assertThat(resourceTestService.countResources()).isEqualTo(58);
+    assertThat(resourceTestService.countResources()).isEqualTo(60);
     assertThat(resourceTestService.findEdgeById(instance.getOutgoingEdges().iterator().next().getId())).isNotPresent();
-    assertThat(resourceTestService.countEdges()).isEqualTo(43);
+    assertThat(resourceTestService.countEdges()).isEqualTo(45);
     checkSearchIndexMessage(work.getId(), UPDATE);
     checkIndexDate(work.getId().toString());
   }
@@ -642,8 +642,8 @@ void deleteResourceById_shouldDeleteRootWorkAndRootEdges() throws Exception {
     // given
     var existed = resourceTestService.saveGraph(getSampleWork(getSampleInstanceResource(null, null)));
     assertThat(resourceTestService.findById(existed.getId())).isPresent();
-    assertThat(resourceTestService.countResources()).isEqualTo(59);
-    assertThat(resourceTestService.countEdges()).isEqualTo(61);
+    assertThat(resourceTestService.countResources()).isEqualTo(61);
+    assertThat(resourceTestService.countEdges()).isEqualTo(63);
     var requestBuilder = delete(RESOURCE_URL + "/" + existed.getId())
       .contentType(APPLICATION_JSON)
       .headers(defaultHeaders(env));
@@ -654,9 +654,9 @@ void deleteResourceById_shouldDeleteRootWorkAndRootEdges() throws Exception {
     // then
     resultActions.andExpect(status().isNoContent());
     assertThat(resourceTestService.existsById(existed.getId())).isFalse();
-    assertThat(resourceTestService.countResources()).isEqualTo(58);
+    assertThat(resourceTestService.countResources()).isEqualTo(60);
     assertThat(resourceTestService.findEdgeById(existed.getOutgoingEdges().iterator().next().getId())).isNotPresent();
-    assertThat(resourceTestService.countEdges()).isEqualTo(30);
+    assertThat(resourceTestService.countEdges()).isEqualTo(32);
     checkSearchIndexMessage(existed.getId(), DELETE);
   }
 

src/test/java/org/folio/linked/data/service/resource/edge/ResourceEdgeServiceTest.java

@@ -1,20 +1,34 @@
 package org.folio.linked.data.service.resource.edge;
 
-import static org.assertj.core.api.AssertionsForClassTypes.assertThat;
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.folio.ld.dictionary.PredicateDictionary.ADMIN_METADATA;
+import static org.folio.ld.dictionary.PredicateDictionary.DISSERTATION;
+import static org.folio.ld.dictionary.PredicateDictionary.GENRE;
+import static org.folio.ld.dictionary.PredicateDictionary.ILLUSTRATIONS;
+import static org.folio.ld.dictionary.PredicateDictionary.SUPPLEMENTARY_CONTENT;
 import static org.folio.ld.dictionary.PredicateDictionary.TITLE;
 import static org.folio.linked.data.test.TestUtil.randomLong;
 import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.Mockito.doReturn;
 import static org.mockito.Mockito.when;
 
-import org.folio.ld.dictionary.model.Resource;
-import org.folio.ld.dictionary.model.ResourceEdge;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Stream;
+import org.folio.ld.dictionary.ResourceTypeDictionary;
 import org.folio.linked.data.mapper.ResourceModelMapper;
+import org.folio.linked.data.model.entity.PredicateEntity;
+import org.folio.linked.data.model.entity.Resource;
+import org.folio.linked.data.model.entity.ResourceEdge;
+import org.folio.linked.data.model.entity.ResourceTypeEntity;
 import org.folio.linked.data.repo.ResourceEdgeRepository;
 import org.folio.linked.data.repo.ResourceRepository;
 import org.folio.spring.testing.type.UnitTest;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.junit.jupiter.MockitoExtension;
@@ -37,7 +51,9 @@ class ResourceEdgeServiceTest {
   void saveNewResourceEdge_shouldSaveMappedEdgeResourceWithReferenceToSource() {
     // given
     var sourceId = randomLong();
-    var edgeModel = new ResourceEdge(new Resource().setId(sourceId), new Resource().setId(randomLong()), TITLE);
+    var edgeModel = new org.folio.ld.dictionary.model.ResourceEdge(
+      new org.folio.ld.dictionary.model.Resource().setId(sourceId),
+      new org.folio.ld.dictionary.model.Resource().setId(randomLong()), TITLE);
     var mappedEdgeResource = new org.folio.linked.data.model.entity.Resource().setId(edgeModel.getTarget().getId());
     doReturn(mappedEdgeResource).when(resourceModelMapper).toEntity(edgeModel.getTarget());
     doReturn(mappedEdgeResource).when(resourceRepository).save(mappedEdgeResource);
@@ -53,4 +69,59 @@ void saveNewResourceEdge_shouldSaveMappedEdgeResourceWithReferenceToSource() {
     assertThat(result.getPredicateHash()).isEqualTo(edgeModel.getPredicate().getHash());
   }
 
+  static Stream<Arguments> dataProvider() {
+    return Stream.of(
+      Arguments.of(getInstance(), getEmptyInstance(), List.of(ADMIN_METADATA.getUri())),
+      Arguments.of(getWork(), getEmptyWork(), List.of(ILLUSTRATIONS.getUri(), SUPPLEMENTARY_CONTENT.getUri(),
+        DISSERTATION.getUri(), GENRE.getUri())),
+      Arguments.of(getInstance(), getEmptyWork(), List.of()),
+      Arguments.of(getWork(), getEmptyInstance(), List.of())
+    );
+  }
+
+  @ParameterizedTest
+  @MethodSource("dataProvider")
+  void copyOutgoingEdges_shouldCopy_appropriateEdges(Resource from,
+                                                     Resource to,
+                                                     List<String> expectedPredicates) {
+    //when
+    resourceEdgeService.copyOutgoingEdges(from, to);
+
+    //then
+    assertThat(to.getOutgoingEdges()
+      .stream()
+      .map(ResourceEdge::getPredicate)
+      .map(PredicateEntity::getUri)
+      .toList()
+    ).containsExactlyInAnyOrderElementsOf(expectedPredicates);
+  }
+
+  private static Resource getEmptyInstance() {
+    var instance = new Resource();
+    instance.setTypes(Set.of(new ResourceTypeEntity(1L, ResourceTypeDictionary.INSTANCE.getUri(), "instance")));
+    return instance;
+  }
+
+  private static Resource getInstance() {
+    var instance = getEmptyInstance();
+    instance.addOutgoingEdge(new ResourceEdge(instance, new Resource(), TITLE));
+    instance.addOutgoingEdge(new ResourceEdge(instance, new Resource(), ADMIN_METADATA));
+    return instance;
+  }
+
+  private static Resource getEmptyWork() {
+    var work = new Resource();
+    work.setTypes(Set.of(new ResourceTypeEntity(2L, ResourceTypeDictionary.WORK.getUri(), "work")));
+    return work;
+  }
+
+  private static Resource getWork() {
+    var work = getEmptyWork();
+    work.addOutgoingEdge(new ResourceEdge(work, new Resource(), TITLE));
+    work.addOutgoingEdge(new ResourceEdge(work, new Resource(), ILLUSTRATIONS));
+    work.addOutgoingEdge(new ResourceEdge(work, new Resource(), SUPPLEMENTARY_CONTENT));
+    work.addOutgoingEdge(new ResourceEdge(work, new Resource(), DISSERTATION));
+    work.addOutgoingEdge(new ResourceEdge(work, new Resource(), GENRE));
+    return work;
+  }
 }

src/test/java/org/folio/linked/data/test/MonographTestUtil.java

@@ -523,26 +523,6 @@ public static Resource getSampleWork(Resource linkedInstance) {
       emptyMap()
     ).setLabel("eng");
 
-    var illustrations = createResource(
-      Map.of(
-        CODE, List.of("code"),
-        TERM, List.of("illustrations term"),
-        LINK, List.of("http://id.loc.gov/vocabulary/millus/code")
-      ),
-      Set.of(CATEGORY),
-      emptyMap()
-    ).setLabel("illustrations term");
-
-    var supplementaryContent = createResource(
-      Map.of(
-        CODE, List.of("code"),
-        TERM, List.of("supplementary content term"),
-        LINK, List.of("http://id.loc.gov/vocabulary/msupplcont/code")
-      ),
-      Set.of(CATEGORY),
-      emptyMap()
-    ).setLabel("supplementary content term");
-
     var pred2OutgoingResources = new LinkedHashMap<PredicateDictionary, List<Resource>>();
     pred2OutgoingResources.put(TITLE, List.of(primaryTitle, createParallelTitle(), createVariantTitle()));
     pred2OutgoingResources.put(CLASSIFICATION, List.of(createLcClassification(), createDeweyClassification()));
@@ -561,8 +541,8 @@ public static Resource getSampleWork(Resource linkedInstance) {
     pred2OutgoingResources.put(DISSERTATION, List.of(createDissertation()));
     pred2OutgoingResources.put(TARGET_AUDIENCE, List.of(createTargetAudience()));
     pred2OutgoingResources.put(LANGUAGE, List.of(language));
-    pred2OutgoingResources.put(ILLUSTRATIONS, List.of(illustrations));
-    pred2OutgoingResources.put(PredicateDictionary.SUPPLEMENTARY_CONTENT, List.of(supplementaryContent));
+    pred2OutgoingResources.put(ILLUSTRATIONS, List.of(createIllustrations()));
+    pred2OutgoingResources.put(PredicateDictionary.SUPPLEMENTARY_CONTENT, List.of(createSupplementaryContent()));
 
     var work = createResource(
       Map.ofEntries(
@@ -712,6 +692,50 @@ private static Resource createTargetAudience() {
     ).setLabel("Primary");
   }
 
+  private static Resource createIllustrations() {
+    var categorySet = createResource(
+      Map.of(
+        LINK, List.of("http://id.loc.gov/vocabulary/millus"),
+        LABEL, List.of("Illustrative Content")
+      ),
+      Set.of(CATEGORY_SET),
+      emptyMap())
+      .setLabel("Illustrative Content");
+    var pred2OutgoingResources = new LinkedHashMap<PredicateDictionary, List<Resource>>();
+    pred2OutgoingResources.put(IS_DEFINED_BY, List.of(categorySet));
+    return createResource(
+      Map.of(
+        CODE, List.of("code"),
+        TERM, List.of("illustrations term"),
+        LINK, List.of("http://id.loc.gov/vocabulary/millus/code")
+      ),
+      Set.of(CATEGORY),
+      pred2OutgoingResources
+    ).setLabel("illustrations term");
+  }
+
+  private static Resource createSupplementaryContent() {
+    var categorySet = createResource(
+      Map.of(
+        LINK, List.of("http://id.loc.gov/vocabulary/msupplcont"),
+        LABEL, List.of("Supplementary Content")
+      ),
+      Set.of(CATEGORY_SET),
+      emptyMap())
+      .setLabel("Supplementary Content");
+    var pred2OutgoingResources = new LinkedHashMap<PredicateDictionary, List<Resource>>();
+    pred2OutgoingResources.put(IS_DEFINED_BY, List.of(categorySet));
+    return createResource(
+      Map.of(
+        CODE, List.of("code"),
+        TERM, List.of("supplementary content term"),
+        LINK, List.of("http://id.loc.gov/vocabulary/msupplcont/code")
+      ),
+      Set.of(CATEGORY),
+      pred2OutgoingResources
+    ).setLabel("supplementary content term");
+  }
+
   private Resource status(String prefix) {
     return createResource(
       Map.of(