diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/exception/ExtractingPathVersionsException.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/exception/ExtractingPathVersionsException.java new file mode 100644 index 00000000..50d67374 --- /dev/null +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/exception/ExtractingPathVersionsException.java @@ -0,0 +1,9 @@ +package au.org.aodn.ardcvocabs.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) +public class ExtractingPathVersionsException extends RuntimeException { + public ExtractingPathVersionsException(String message) { super(message); }; +} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/exception/InvalidVersionFormatException.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/exception/InvalidVersionFormatException.java new file mode 100644 index 00000000..6d49abd0 --- /dev/null +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/exception/InvalidVersionFormatException.java @@ -0,0 +1,9 @@ +package au.org.aodn.ardcvocabs.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) +public class InvalidVersionFormatException extends RuntimeException { + public InvalidVersionFormatException(String message) { super(message); }; +} diff --git a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImpl.java b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImpl.java index ee562f8a..5417a3f1 100644 --- a/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImpl.java +++ b/ardcvocabs/src/main/java/au/org/aodn/ardcvocabs/service/ArdcVocabServiceImpl.java @@ -1,5 +1,7 @@ package au.org.aodn.ardcvocabs.service; +import au.org.aodn.ardcvocabs.exception.ExtractingPathVersionsException; +import au.org.aodn.ardcvocabs.exception.InvalidVersionFormatException; import au.org.aodn.ardcvocabs.model.ArdcCurrentPaths; import au.org.aodn.ardcvocabs.model.PathName; import au.org.aodn.ardcvocabs.model.VocabApiPaths; @@ -36,36 +38,47 @@ public class ArdcVocabServiceImpl implements ArdcVocabService { public Map> getResolvedPathCollection() { Map> resolvedPathCollection = new HashMap<>(); + for (ArdcCurrentPaths currentPath : ArdcCurrentPaths.values()) { try { + // Fetch current contents ObjectNode categoryCurrentContent = fetchCurrentContents(currentPath.getCategoryCurrent()); ObjectNode vocabCurrentContent = fetchCurrentContents(currentPath.getVocabCurrent()); + validateContentNotNull(currentPath, categoryCurrentContent, vocabCurrentContent); - if (categoryCurrentContent != null && vocabCurrentContent != null) { - // Extract versions - String categoryVersion = extractVersionFromCurrentContent(categoryCurrentContent); - String vocabVersion = extractVersionFromCurrentContent(vocabCurrentContent); + // Extract versions + String categoryVersion = extractVersionFromCurrentContent(categoryCurrentContent); + String vocabVersion = extractVersionFromCurrentContent(vocabCurrentContent); + validateVersionsNotNull(currentPath, categoryVersion, vocabVersion); - if (categoryVersion != null && vocabVersion != null) { - log.info("Fetched ARDC category version for {}: {}", currentPath.name(), categoryVersion); - log.info("Fetched ARDC vocab version for {}: {}", currentPath.name(), vocabVersion); + log.info("Fetched ARDC category version for {}: {}", currentPath.name(), categoryVersion); + log.info("Fetched ARDC vocab version for {}: {}", currentPath.name(), vocabVersion); + + // Build and store resolved paths + Map resolvedPaths = buildResolvedPaths(currentPath, categoryVersion, vocabVersion); + resolvedPathCollection.put(currentPath.name(), resolvedPaths); - // Build and store resolved paths - Map resolvedPaths = buildResolvedPaths(currentPath, categoryVersion, vocabVersion); - resolvedPathCollection.put(currentPath.name(), resolvedPaths); - } else { - log.error("Failed to extract versions for {}", currentPath.name()); - } - } else { - log.error("Failed to fetch HTML content for {}", currentPath.name()); - } } catch (Exception e) { log.error("Error initialising versions for {}: {}", currentPath.name(), e.getMessage(), e); + throw new ExtractingPathVersionsException(String.format("Error initialising versions for %s: %s", currentPath.name(), e.getMessage())); } } + return resolvedPathCollection; } + private void validateContentNotNull(ArdcCurrentPaths currentPath, ObjectNode categoryContent, ObjectNode vocabContent) { + if (categoryContent == null || vocabContent == null) { + throw new ExtractingPathVersionsException(String.format("Failed to fetch HTML content for %s", currentPath.name())); + } + } + + private void validateVersionsNotNull(ArdcCurrentPaths currentPath, String categoryVersion, String vocabVersion) { + if (categoryVersion == null || vocabVersion == null) { + throw new ExtractingPathVersionsException(String.format("Version extraction returned null for %s", currentPath.name())); + } + } + private ObjectNode fetchCurrentContents(String url) { try { return retryTemplate.execute(context -> restTemplate.getForObject(url, ObjectNode.class)); @@ -90,7 +103,7 @@ protected Map buildResolvedPaths(ArdcCurrentPaths currentPaths return resolvedPaths; } - protected String extractVersionFromCurrentContent(ObjectNode currentContent) { + protected String extractVersionFromCurrentContent(ObjectNode currentContent) throws InvalidPropertiesFormatException { if (currentContent != null && !currentContent.isEmpty()) { JsonNode node = currentContent.get("result"); if (!about.apply(node).isEmpty()) { @@ -102,10 +115,9 @@ protected String extractVersionFromCurrentContent(ObjectNode currentContent) { log.info("Valid Version Found: {}", version); return version; } else { - log.warn("Version does not match the required format: {}", about.apply(node)); + throw new InvalidVersionFormatException(String.format("Version does not match the required format: %s", about.apply(node))); } } - } else { log.warn("Current content is empty or null."); } diff --git a/indexer/src/main/java/au/org/aodn/esindexer/exception/IgnoreIndexingVocabsException.java b/indexer/src/main/java/au/org/aodn/esindexer/exception/IgnoreIndexingVocabsException.java new file mode 100644 index 00000000..f0c7841f --- /dev/null +++ b/indexer/src/main/java/au/org/aodn/esindexer/exception/IgnoreIndexingVocabsException.java @@ -0,0 +1,9 @@ +package au.org.aodn.esindexer.exception; + +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ResponseStatus; + +@ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) +public class IgnoreIndexingVocabsException extends RuntimeException { + public IgnoreIndexingVocabsException(String message) { super(message); } +} diff --git a/indexer/src/main/java/au/org/aodn/esindexer/service/VocabServiceImpl.java b/indexer/src/main/java/au/org/aodn/esindexer/service/VocabServiceImpl.java index 5c80fdab..09a89513 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/service/VocabServiceImpl.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/service/VocabServiceImpl.java @@ -7,6 +7,7 @@ import au.org.aodn.ardcvocabs.service.ArdcVocabService; import au.org.aodn.esindexer.configuration.AppConstants; import au.org.aodn.esindexer.exception.DocumentNotFoundException; +import au.org.aodn.esindexer.exception.IgnoreIndexingVocabsException; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ContactsModel; import au.org.aodn.stac.model.ThemesModel; @@ -378,6 +379,10 @@ public void populateVocabsData(Map> resolvedPathCo List platformVocabs = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.PLATFORM_VOCAB.name())); List organisationVocabs = ardcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get(VocabApiPaths.ORGANISATION_VOCAB.name())); + if (parameterVocabs.isEmpty() || platformVocabs.isEmpty() || organisationVocabs.isEmpty()) { + throw new IgnoreIndexingVocabsException("One or more vocab lists are empty. Skipping indexing."); + } + indexAllVocabs(parameterVocabs, platformVocabs, organisationVocabs); } @@ -408,10 +413,14 @@ public void populateVocabsDataAsync(Map> resolvedP } } - // Call indexAllVocabs only after all tasks are completed + // Validate allResults to ensure none of the lists are empty + if (allResults.stream().anyMatch(List::isEmpty)) { + throw new IgnoreIndexingVocabsException("One or more vocab tasks returned empty results. Skipping indexing."); + } + + // Call indexAllVocabs only after all tasks are completed and validated log.info("Indexing fetched vocabs to {}", vocabsIndexName); indexAllVocabs(allResults.get(0), allResults.get(1), allResults.get(2)); - } catch (InterruptedException | IOException e) { Thread.currentThread().interrupt(); // Restore interrupt status log.error("Thread was interrupted while processing vocab tasks", e); diff --git a/indexer/src/main/java/au/org/aodn/esindexer/utils/VocabsIndexUtils.java b/indexer/src/main/java/au/org/aodn/esindexer/utils/VocabsIndexUtils.java index 3dcb15e6..c5dfd0ee 100644 --- a/indexer/src/main/java/au/org/aodn/esindexer/utils/VocabsIndexUtils.java +++ b/indexer/src/main/java/au/org/aodn/esindexer/utils/VocabsIndexUtils.java @@ -1,7 +1,9 @@ package au.org.aodn.esindexer.utils; +import au.org.aodn.ardcvocabs.exception.ExtractingPathVersionsException; import au.org.aodn.ardcvocabs.model.PathName; import au.org.aodn.ardcvocabs.service.ArdcVocabService; +import au.org.aodn.esindexer.exception.IgnoreIndexingVocabsException; import au.org.aodn.esindexer.service.VocabService; import jakarta.annotation.PostConstruct; import lombok.extern.slf4j.Slf4j; @@ -45,9 +47,13 @@ public void setArdcVocabService(ArdcVocabService ardcVocabService) { public void init() throws IOException { // Check if the initialiseVocabsIndex flag is enabled if (initialiseVocabsIndex) { - log.info("Initialising {} asynchronously", vocabsIndexName); - storedResolvedPathCollection = ardcVocabService.getResolvedPathCollection(); - vocabService.populateVocabsDataAsync(storedResolvedPathCollection); + try { + log.info("Initialising {} asynchronously", vocabsIndexName); + storedResolvedPathCollection = ardcVocabService.getResolvedPathCollection(); + vocabService.populateVocabsDataAsync(storedResolvedPathCollection); + } catch (ExtractingPathVersionsException | IgnoreIndexingVocabsException e) { + log.warn("Skip initialising vocabs with error: {}", e.getMessage()); + } } } @@ -59,13 +65,16 @@ public void scheduledRefreshVocabsData() { if (!latestResolvedPathCollection.equals(storedResolvedPathCollection)) { log.info("Detected changes in the resolved path collection, updating vocabularies..."); - vocabService.populateVocabsData(latestResolvedPathCollection); - refreshCaches(); - - // update the head if there are new versions - synchronized (this) { - storedResolvedPathCollection = latestResolvedPathCollection; - log.info("Updated storedResolvedPathCollection with the latest data."); + try { + vocabService.populateVocabsData(latestResolvedPathCollection); + refreshCaches(); + // update the head if there are new versions + synchronized (this) { + storedResolvedPathCollection = latestResolvedPathCollection; + log.info("Updated storedResolvedPathCollection with the latest data."); + } + } catch (IgnoreIndexingVocabsException e) { + log.warn("Skip refreshing vocabs: {}", e.getMessage()); } } else { log.info("No changes detected in the resolved path collection. Skip updating caches"); diff --git a/indexer/src/test/java/au/org/aodn/esindexer/service/VocabServiceIT.java b/indexer/src/test/java/au/org/aodn/esindexer/service/VocabServiceIT.java index 4ad8454c..4e64dee6 100644 --- a/indexer/src/test/java/au/org/aodn/esindexer/service/VocabServiceIT.java +++ b/indexer/src/test/java/au/org/aodn/esindexer/service/VocabServiceIT.java @@ -1,16 +1,21 @@ package au.org.aodn.esindexer.service; +import au.org.aodn.ardcvocabs.model.PathName; import au.org.aodn.ardcvocabs.model.VocabApiPaths; import au.org.aodn.ardcvocabs.model.VocabModel; import au.org.aodn.ardcvocabs.service.ArdcVocabService; import au.org.aodn.esindexer.BaseTestClass; import au.org.aodn.esindexer.configuration.AppConstants; +import au.org.aodn.esindexer.exception.IgnoreIndexingVocabsException; import au.org.aodn.stac.model.ConceptModel; import au.org.aodn.stac.model.ThemesModel; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.json.JSONException; import org.junit.jupiter.api.*; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Spy; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.test.context.ActiveProfiles; @@ -24,6 +29,7 @@ import org.skyscreamer.jsonassert.JSONCompareMode; import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; @SpringBootTest( webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT @@ -33,9 +39,16 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) public class VocabServiceIT extends BaseTestClass { + @InjectMocks + @Spy + VocabServiceImpl mockVocabService; + @Autowired VocabService vocabService; + @Mock + ArdcVocabService mockArdcVocabService; + @Autowired ArdcVocabService ardcVocabService; @@ -115,6 +128,29 @@ void testProcessParameterVocabs() throws IOException, JSONException { ); } + @Test + void testSkipIndexingIfEmptyVocabs() throws IOException { + // Mock resolved path collection + Map> resolvedPathCollection = Map.of( + "PARAMETER_VOCAB", Map.of(), + "PLATFORM_VOCAB", Map.of(), + "ORGANISATION_VOCAB", Map.of() + ); + + // Mock service calls to return empty lists + when(mockArdcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get("PARAMETER_VOCAB"))).thenReturn(Collections.emptyList()); + when(mockArdcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get("PLATFORM_VOCAB"))).thenReturn(Collections.emptyList()); + when(mockArdcVocabService.getVocabTreeFromArdcByType(resolvedPathCollection.get("ORGANISATION_VOCAB"))).thenReturn(Collections.emptyList()); + + // Call the method + try { + mockVocabService.populateVocabsData(resolvedPathCollection); + } catch (IgnoreIndexingVocabsException e) { + // Verify that indexAllVocabs is not called + verify(mockVocabService, never()).indexAllVocabs(anyList(), anyList(), anyList()); + } + } + @Test void testProcessPlatformVocabs() throws IOException, JSONException { // read from ARDC