From cc5a2392a516c714f2f5f6abcbb5dcbab886936a Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Fri, 24 Sep 2021 10:48:48 +0200 Subject: [PATCH 1/5] doc: update roadmap --- docs/roadmap.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/roadmap.md b/docs/roadmap.md index 646b08a16..63f206383 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -14,15 +14,15 @@ guidelines only, and this section may be revised to provide newer information at ## Short term The following list of features are planned to be addressed in the short term, and incorporated in the next release of -the product planned for end of March 2021: +the product planned for end of November 2021: - Implement multi-attributes support for GeoProperties [#101](https://github.com/stellio-hub/stellio-context-broker/issues/101) -- Finish implementation of some missing common cross-cutting behaviors as defined in the NGSI-LD specification [#11](https://github.com/stellio-hub/stellio-context-broker/issues/11), [#12](https://github.com/stellio-hub/stellio-context-broker/issues/12), [#52](https://github.com/stellio-hub/stellio-context-broker/issues/52), [#146](https://github.com/stellio-hub/stellio-context-broker/issues/146), [#206](https://github.com/stellio-hub/stellio-context-broker/issues/206), [#287](https://github.com/stellio-hub/stellio-context-broker/issues/287) +- Finish implementation of some missing common cross-cutting behaviors as defined in the NGSI-LD specification [#12](https://github.com/stellio-hub/stellio-context-broker/issues/12), [#206](https://github.com/stellio-hub/stellio-context-broker/issues/206) - Implement the discovery endpoints introduced in version 1.3.1 of the NGSI-LD specification [#268](https://github.com/stellio-hub/stellio-context-broker/issues/268) - Implement support for the batch entities update endpoint [#62](https://github.com/stellio-hub/stellio-context-broker/issues/62) - Fix the currently [identified issues](https://github.com/stellio-hub/stellio-context-broker/issues?q=is%3Aissue+is%3Aopen+label%3Afix) - Implement support for the aggregated temporal representation of entities introduced in version 1.4.1 of the NGSI-LD specification -- Upgrade frameworks and libraries to their last released version (Spring Boot 2.4.x, Timescale 2.x, ...) +- Complete the requirements to become an approved full Generic Enabler ## Medium term @@ -33,7 +33,6 @@ release(s) generated in the next **9 months** after next planned release: - Implement support for the all the supported data types (e.g. structured property value) - Implement distributed capabilities (via support of Context Source as defined in the NGSI-LD specification) - Experiment with an alternative Graph database (namely Janus Graph) -- Complete the requirements to become an approved full Generic Enabler - Expose an API allowing the management of authorizations inside the information context ## Long term From 06290728c49bde48c77ecc02fc424fb1e60727bf Mon Sep 17 00:00:00 2001 From: Poulomi Nandy <60551269+PoulomiNandy@users.noreply.github.com> Date: Sun, 26 Sep 2021 17:27:16 +0200 Subject: [PATCH 2/5] feat(search): add support for pagination in temporal api #466 (#494) * implemented pagination behaviour in temporal API * refactor: common pagination processing and improve temporal entities queries * refactor: introduce a TemporaEntitiesQuery data class for a more typesafe way of handling queries Co-authored-by: Benoit Orihuela --- entity-service/config/detekt/baseline.xml | 1 - .../egm/stellio/entity/web/EntityHandler.kt | 31 +- search-service/config/detekt/baseline.xml | 2 + .../search/config/ApplicationProperties.kt | 7 +- .../search/model/TemporalEntitiesQuery.kt | 12 + .../stellio/search/service/QueryService.kt | 48 ++- .../service/TemporalEntityAttributeService.kt | 73 ++++- .../search/web/TemporalEntityHandler.kt | 32 +- .../web/TemporalEntityOperationsHandler.kt | 36 ++- .../src/main/resources/application.properties | 4 + .../search/service/QueryServiceTests.kt | 104 ++++--- .../TemporalEntityAttributeServiceTests.kt | 62 ++++ .../search/web/TemporalEntityHandlerTests.kt | 273 ++++++++++++++++-- .../TemporalEntityOperationsHandlerTests.kt | 40 ++- .../com/egm/stellio/shared/util/ApiUtils.kt | 18 ++ .../config/detekt/baseline.xml | 1 - .../subscription/web/SubscriptionHandler.kt | 23 +- 17 files changed, 596 insertions(+), 171 deletions(-) create mode 100644 search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntitiesQuery.kt diff --git a/entity-service/config/detekt/baseline.xml b/entity-service/config/detekt/baseline.xml index 676ec7fc6..d17dbd70a 100644 --- a/entity-service/config/detekt/baseline.xml +++ b/entity-service/config/detekt/baseline.xml @@ -2,7 +2,6 @@ - ComplexMethod:EntityHandler.kt$EntityHandler$ @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @RequestParam params: MultiValueMap<String, String> ): ResponseEntity<*> EmptyFunctionBlock:StandaloneAuthorizationService.kt$StandaloneAuthorizationService${} LargeClass:EntityHandlerTests.kt$EntityHandlerTests LargeClass:EntityOperationHandlerTests.kt$EntityOperationHandlerTests diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt index 5a94db2b0..0bf5a1a1c 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt @@ -85,8 +85,12 @@ class EntityHandler( @RequestParam params: MultiValueMap ): ResponseEntity<*> { val count = params.getFirst(QUERY_PARAM_COUNT)?.toBoolean() ?: false - val offset = params.getFirst(QUERY_PARAM_OFFSET)?.toIntOrNull() ?: 0 - val limit = params.getFirst(QUERY_PARAM_LIMIT)?.toIntOrNull() ?: applicationProperties.pagination.limitDefault + val (offset, limit) = extractAndValidatePaginationParameters( + params, + applicationProperties.pagination.limitDefault, + applicationProperties.pagination.limitMax, + count + ) val ids = params.getFirst(QUERY_PARAM_ID)?.split(",") val type = params.getFirst(QUERY_PARAM_TYPE) val idPattern = params.getFirst(QUERY_PARAM_ID_PATTERN) @@ -100,29 +104,6 @@ class EntityHandler( val mediaType = getApplicableMediaType(httpHeaders) val userId = extractSubjectOrEmpty().awaitFirst() - if (!count && (limit <= 0 || offset < 0)) - return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) - .body( - BadRequestDataResponse( - "Offset must be greater than zero and limit must be strictly greater than zero" - ) - ) - - if (count && (limit < 0 || offset < 0)) - return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) - .body( - BadRequestDataResponse("Offset and limit must be greater than zero") - ) - - if (limit > applicationProperties.pagination.limitMax) - return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) - .body( - BadRequestDataResponse( - "You asked for $limit results, " + - "but the supported maximum limit is ${applicationProperties.pagination.limitMax}" - ) - ) - if (q == null && type == null && attrs == null) return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) .body( diff --git a/search-service/config/detekt/baseline.xml b/search-service/config/detekt/baseline.xml index fef8ed57e..1fce44a44 100644 --- a/search-service/config/detekt/baseline.xml +++ b/search-service/config/detekt/baseline.xml @@ -5,8 +5,10 @@ LargeClass:EntityEventListenerServiceTest.kt$EntityEventListenerServiceTest LargeClass:TemporalEntityHandlerTests.kt$TemporalEntityHandlerTests LongMethod:ParameterizedTests.kt$ParameterizedTests.Companion$@JvmStatic fun rawResultsProvider(): Stream<Arguments> + LongMethod:QueryServiceTests.kt$QueryServiceTests$@Test fun `it should query temporal entities as requested by query params`() LongParameterList:AttributeInstance.kt$AttributeInstance.Companion$( temporalEntityAttribute: UUID, instanceId: URI? = null, observedAt: ZonedDateTime, value: String? = null, measuredValue: Double? = null, payload: Map<String, Any> ) LongParameterList:EntityEventListenerService.kt$EntityEventListenerService$( entityId: URI, expandedAttributeName: String, datasetId: URI?, attributeValuesNode: JsonNode, updatedEntity: String, contexts: List<String> ) + LongParameterList:TemporalEntityAttributeService.kt$TemporalEntityAttributeService$( limit: Int, offset: Int, ids: Set<URI>, types: Set<String>, attrs: Set<String>, withEntityPayload: Boolean = false ) ReturnCount:EntityEventListenerService.kt$EntityEventListenerService$internal fun toTemporalAttributeMetadata(jsonNode: JsonNode): Validated<String, AttributeMetadata> ReturnCount:TemporalEntityAttributeService.kt$TemporalEntityAttributeService$internal fun toTemporalAttributeMetadata( ngsiLdAttributeInstance: NgsiLdAttributeInstance ): Validated<String, AttributeMetadata> SwallowedException:TemporalEntityHandler.kt$catch (e: IllegalArgumentException) { "'timerel' is not valid, it should be one of 'before', 'between', or 'after'".left() } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/config/ApplicationProperties.kt b/search-service/src/main/kotlin/com/egm/stellio/search/config/ApplicationProperties.kt index c82104949..212e190a1 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/config/ApplicationProperties.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/config/ApplicationProperties.kt @@ -8,7 +8,8 @@ import java.net.URI @ConfigurationProperties("application") data class ApplicationProperties( val entity: Entity, - val authentication: Authentication + val authentication: Authentication, + val pagination: Pagination ) { data class Authentication( val enabled: Boolean @@ -18,4 +19,8 @@ data class ApplicationProperties( val serviceUrl: URI, val storePayloads: Boolean ) + data class Pagination( + val limitDefault: Int, + val limitMax: Int + ) } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntitiesQuery.kt b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntitiesQuery.kt new file mode 100644 index 000000000..7d0099897 --- /dev/null +++ b/search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntitiesQuery.kt @@ -0,0 +1,12 @@ +package com.egm.stellio.search.model + +import java.net.URI + +data class TemporalEntitiesQuery( + val ids: Set, + val types: Set, + val temporalQuery: TemporalQuery, + val withTemporalValues: Boolean, + val limit: Int, + val offset: Int +) diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt index 65acdea7c..280cf4c32 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/QueryService.kt @@ -1,6 +1,8 @@ package com.egm.stellio.search.service +import com.egm.stellio.search.config.ApplicationProperties import com.egm.stellio.search.model.AttributeInstanceResult +import com.egm.stellio.search.model.TemporalEntitiesQuery import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.web.buildTemporalQuery @@ -19,25 +21,36 @@ import java.util.* class QueryService( private val attributeInstanceService: AttributeInstanceService, private val temporalEntityAttributeService: TemporalEntityAttributeService, + private val applicationProperties: ApplicationProperties, private val temporalEntityService: TemporalEntityService ) { - fun parseAndCheckQueryParams(queryParams: MultiValueMap, contextLink: String): Map { + fun parseAndCheckQueryParams( + queryParams: MultiValueMap, + contextLink: String + ): TemporalEntitiesQuery { val withTemporalValues = hasValueInOptionsParam( Optional.ofNullable(queryParams.getFirst("options")), OptionsParamValue.TEMPORAL_VALUES ) val ids = parseRequestParameter(queryParams.getFirst(QUERY_PARAM_ID)).map { it.toUri() }.toSet() val types = parseAndExpandRequestParameter(queryParams.getFirst(QUERY_PARAM_TYPE), contextLink) val temporalQuery = buildTemporalQuery(queryParams, contextLink) + val (offset, limit) = extractAndValidatePaginationParameters( + queryParams, + applicationProperties.pagination.limitDefault, + applicationProperties.pagination.limitMax + ) if (types.isEmpty() && temporalQuery.expandedAttrs.isEmpty()) throw BadRequestDataException("Either type or attrs need to be present in request parameters") - return mapOf( - "ids" to ids, - "types" to types, - "temporalQuery" to temporalQuery, - "withTemporalValues" to withTemporalValues + return TemporalEntitiesQuery( + ids = ids, + types = types, + temporalQuery = temporalQuery, + withTemporalValues = withTemporalValues, + limit = limit, + offset = offset ) } @@ -74,20 +87,23 @@ class QueryService( } suspend fun queryTemporalEntities( - ids: Set, - types: Set, - temporalQuery: TemporalQuery, - withTemporalValues: Boolean, + temporalEntitiesQuery: TemporalEntitiesQuery, contextLink: String ): List { val temporalEntityAttributes = temporalEntityAttributeService.getForEntities( - ids, - types, - temporalQuery.expandedAttrs + temporalEntitiesQuery.limit, + temporalEntitiesQuery.offset, + temporalEntitiesQuery.ids, + temporalEntitiesQuery.types, + temporalEntitiesQuery.temporalQuery.expandedAttrs ).awaitFirstOrDefault(emptyList()) val temporalEntityAttributesWithMatchingInstances = - searchInstancesForTemporalEntityAttributes(temporalEntityAttributes, temporalQuery, withTemporalValues) + searchInstancesForTemporalEntityAttributes( + temporalEntityAttributes, + temporalEntitiesQuery.temporalQuery, + temporalEntitiesQuery.withTemporalValues + ) val temporalEntityAttributesWithInstances = fillWithTEAWithoutInstances(temporalEntityAttributes, temporalEntityAttributesWithMatchingInstances) @@ -106,9 +122,9 @@ class QueryService( return temporalEntityService.buildTemporalEntities( attributeInstancesPerEntityAndAttribute, - temporalQuery, + temporalEntitiesQuery.temporalQuery, listOf(contextLink), - withTemporalValues + temporalEntitiesQuery.withTemporalValues ) } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt index fd244eaf1..a1144e93e 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt @@ -244,9 +244,15 @@ class TemporalEntityAttributeService( ).valid() } - fun getForEntities(ids: Set, types: Set, attrs: Set, withEntityPayload: Boolean = false): - Mono> { - var selectQuery = if (withEntityPayload) + fun getForEntities( + limit: Int, + offset: Int, + ids: Set, + types: Set, + attrs: Set, + withEntityPayload: Boolean = false + ): Mono> { + val selectQuery = if (withEntityPayload) """ SELECT id, temporal_entity_attribute.entity_id, type, attribute_name, attribute_type, attribute_value_type, payload::TEXT, dataset_id @@ -261,20 +267,62 @@ class TemporalEntityAttributeService( WHERE """.trimIndent() - val formattedIds = ids.joinToString(",") { "'$it'" } - val formattedTypes = types.joinToString(",") { "'$it'" } - val formattedAttrs = attrs.joinToString(",") { "'$it'" } - if (ids.isNotEmpty()) selectQuery = "$selectQuery entity_id in ($formattedIds) AND" - if (types.isNotEmpty()) selectQuery = "$selectQuery type in ($formattedTypes) AND" - if (attrs.isNotEmpty()) selectQuery = "$selectQuery attribute_name in ($formattedAttrs) AND" + val filterQuery = buildEntitiesQueryFilter(ids, types, attrs) + val finalQuery = """ + $selectQuery + $filterQuery + ORDER BY entity_id + limit :limit + offset :offset + """.trimIndent() return databaseClient - .sql(selectQuery.removeSuffix("AND")) + .sql(finalQuery) + .bind("limit", limit) + .bind("offset", offset) .fetch() .all() .map { rowToTemporalEntityAttribute(it) } .collectList() } + fun getCountForEntities(ids: Set, types: Set, attrs: Set): Mono { + val selectStatement = + """ + SELECT count(distinct(entity_id)) as count_entity from temporal_entity_attribute + WHERE + """.trimIndent() + + val filterQuery = buildEntitiesQueryFilter(ids, types, attrs) + return databaseClient + .sql("$selectStatement $filterQuery") + .map(rowToTemporalCount) + .first() + } + + fun buildEntitiesQueryFilter( + ids: Set, + types: Set, + attrs: Set + ): String { + val formattedIds = ids.joinToString( + separator = ",", + prefix = "entity_id in(", + postfix = ")" + ) { "'$it'" } + val formattedTypes = types.joinToString( + separator = ",", + prefix = "type in (", + postfix = ")" + ) { "'$it'" } + val formattedAttrs = attrs.joinToString( + separator = ",", + prefix = "attribute_name in (", + postfix = ")" + ) { "'$it'" } + + return listOf(formattedIds, formattedTypes, formattedAttrs).joinToString(" AND ") + } + fun getForEntity(id: URI, attrs: Set, withEntityPayload: Boolean = false): Flux { val selectQuery = if (withEntityPayload) """ @@ -292,7 +340,6 @@ class TemporalEntityAttributeService( """.trimIndent() val expandedAttrsList = attrs.joinToString(",") { "'$it'" } - val finalQuery = if (attrs.isNotEmpty()) "$selectQuery AND attribute_name in ($expandedAttrsList)" @@ -361,4 +408,8 @@ class TemporalEntityAttributeService( private var rowToId: ((Row) -> UUID) = { row -> row.get("id", UUID::class.java)!! } + + private var rowToTemporalCount: ((Row) -> Int) = { row -> + row.get("count_entity", Integer::class.java)!!.toInt() + } } diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt index 71f363605..515ce6d8d 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt @@ -19,6 +19,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdFragment import com.egm.stellio.shared.util.JsonLdUtils.expandValueAsListOfMap import com.egm.stellio.shared.util.JsonUtils.serializeObject import com.egm.stellio.shared.util.OptionsParamValue +import com.egm.stellio.shared.util.PagingUtils import com.egm.stellio.shared.util.buildGetSuccessResponse import com.egm.stellio.shared.util.checkAndGetContext import com.egm.stellio.shared.util.getApplicableMediaType @@ -42,7 +43,6 @@ import org.springframework.web.bind.annotation.RequestMapping import org.springframework.web.bind.annotation.RequestParam import org.springframework.web.bind.annotation.RestController import reactor.core.publisher.Mono -import java.net.URI import java.time.ZonedDateTime import java.util.Optional @@ -101,18 +101,34 @@ class TemporalEntityHandler( ): ResponseEntity<*> { val contextLink = getContextFromLinkHeaderOrDefault(httpHeaders) val mediaType = getApplicableMediaType(httpHeaders) - val parsedParams = queryService.parseAndCheckQueryParams(params, contextLink) + val temporalEntitiesQuery = queryService.parseAndCheckQueryParams(params, contextLink) val temporalEntities = queryService.queryTemporalEntities( - parsedParams["ids"] as Set, - parsedParams["types"] as Set, - parsedParams["temporalQuery"] as TemporalQuery, - parsedParams["withTemporalValues"] as Boolean, + temporalEntitiesQuery, contextLink ) + val temporalEntityCount = temporalEntityAttributeService.getCountForEntities( + temporalEntitiesQuery.ids, + temporalEntitiesQuery.types, + temporalEntitiesQuery.temporalQuery.expandedAttrs + ).awaitFirst() + + val prevAndNextLinks = PagingUtils.getPagingLinks( + "/ngsi-ld/v1/temporal/entities", + params, + temporalEntityCount, + temporalEntitiesQuery.offset, + temporalEntitiesQuery.limit + ) - return buildGetSuccessResponse(mediaType, contextLink) - .body(serializeObject(temporalEntities.map { addContextsToEntity(it, listOf(contextLink), mediaType) })) + return PagingUtils.buildPaginationResponse( + serializeObject(temporalEntities.map { addContextsToEntity(it, listOf(contextLink), mediaType) }), + temporalEntityCount, + false, + prevAndNextLinks, + mediaType, + contextLink + ) } /** diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandler.kt index a500fafbe..4d69d1031 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandler.kt @@ -1,7 +1,7 @@ package com.egm.stellio.search.web -import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.service.QueryService +import com.egm.stellio.search.service.TemporalEntityAttributeService import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.JsonLdUtils.addContextsToEntity import com.egm.stellio.shared.util.JsonUtils.serializeObject @@ -12,12 +12,12 @@ import org.springframework.http.ResponseEntity import org.springframework.util.LinkedMultiValueMap import org.springframework.web.bind.annotation.* import reactor.core.publisher.Mono -import java.net.URI @RestController @RequestMapping("/ngsi-ld/v1/temporal/entityOperations") class TemporalEntityOperationsHandler( - private val queryService: QueryService + private val queryService: QueryService, + private val temporalEntityAttributeService: TemporalEntityAttributeService ) { /** @@ -40,16 +40,32 @@ class TemporalEntityOperationsHandler( queryParams.add(it.key, it.value.toString()) } - val parsedParams = queryService.parseAndCheckQueryParams(queryParams, contextLink) + val temporalEntitiesQuery = queryService.parseAndCheckQueryParams(queryParams, contextLink) val temporalEntities = queryService.queryTemporalEntities( - parsedParams["ids"] as Set, - parsedParams["types"] as Set, - parsedParams["temporalQuery"] as TemporalQuery, - parsedParams["withTemporalValues"] as Boolean, + temporalEntitiesQuery, contextLink ) + val temporalEntityCount = temporalEntityAttributeService.getCountForEntities( + temporalEntitiesQuery.ids, + temporalEntitiesQuery.types, + temporalEntitiesQuery.temporalQuery.expandedAttrs + ).awaitFirst() - return buildGetSuccessResponse(mediaType, contextLink) - .body(serializeObject(temporalEntities.map { addContextsToEntity(it, listOf(contextLink), mediaType) })) + val prevAndNextLinks = PagingUtils.getPagingLinks( + "/ngsi-ld/v1/temporal/entities", + queryParams, + temporalEntityCount, + temporalEntitiesQuery.offset, + temporalEntitiesQuery.limit + ) + + return PagingUtils.buildPaginationResponse( + (serializeObject(temporalEntities.map { addContextsToEntity(it, listOf(contextLink), mediaType) })), + temporalEntityCount, + false, + prevAndNextLinks, + mediaType, + contextLink + ) } } diff --git a/search-service/src/main/resources/application.properties b/search-service/src/main/resources/application.properties index 2b73f8ceb..10873b7b7 100644 --- a/search-service/src/main/resources/application.properties +++ b/search-service/src/main/resources/application.properties @@ -34,3 +34,7 @@ spring.mvc.log-request-details = true # application.graylog.host = localhost # application.graylog.port = 12201 # application.graylog.source = stellio-int + +# Pagination config for query resources endpoints +application.pagination.limit-default = 30 +application.pagination.limit-max = 100 diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt index 97c6de7b6..48b096207 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/QueryServiceTests.kt @@ -1,8 +1,10 @@ package com.egm.stellio.search.service +import com.egm.stellio.search.config.ApplicationProperties import com.egm.stellio.search.config.CoroutineTestRule import com.egm.stellio.search.model.FullAttributeInstanceResult import com.egm.stellio.search.model.SimplifiedAttributeInstanceResult +import com.egm.stellio.search.model.TemporalEntitiesQuery import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.shared.model.BadRequestDataException @@ -11,6 +13,7 @@ import com.egm.stellio.shared.util.APIC_COMPOUND_CONTEXT import com.egm.stellio.shared.util.toUri import com.ninjasquad.springmockk.MockkBean import io.mockk.every +import io.mockk.verify import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.runBlockingTest import org.junit.Rule @@ -46,6 +49,9 @@ class QueryServiceTests { @MockkBean private lateinit var temporalEntityService: TemporalEntityService + @MockkBean(relaxed = true) + private lateinit var applicationProperties: ApplicationProperties + private val entityUri = "urn:ngsi-ld:BeeHive:TESTC".toUri() private val secondEntityUri = "urn:ngsi-ld:BeeHive:TESTB".toUri() @@ -74,6 +80,9 @@ class QueryServiceTests { queryParams.add("timerel", "before") queryParams.add("time", "2019-10-17T07:31:39Z") + every { applicationProperties.pagination.limitDefault } returns 30 + every { applicationProperties.pagination.limitMax } returns 100 + val exception = assertThrows { queryService.parseAndCheckQueryParams(queryParams, APIC_COMPOUND_CONTEXT) } @@ -93,23 +102,28 @@ class QueryServiceTests { queryParams.add("attrs", "incoming,outgoing") queryParams.add("id", "$entityUri,$secondEntityUri") queryParams.add("options", "temporalValues") + queryParams.add("limit", "10") + queryParams.add("offset", "2") - val parsedParams = queryService.parseAndCheckQueryParams(queryParams, APIC_COMPOUND_CONTEXT) + every { applicationProperties.pagination.limitDefault } returns 30 + every { applicationProperties.pagination.limitMax } returns 100 + val temporalEntitiesQuery = queryService.parseAndCheckQueryParams(queryParams, APIC_COMPOUND_CONTEXT) + + assertEquals(setOf(entityUri, secondEntityUri), temporalEntitiesQuery.ids) + assertEquals(setOf(beehiveType, apiaryType), temporalEntitiesQuery.types) assertEquals( - parsedParams, - mapOf( - "ids" to setOf(entityUri, secondEntityUri), - "types" to setOf(beehiveType, apiaryType), - "temporalQuery" to TemporalQuery( - timerel = TemporalQuery.Timerel.BETWEEN, - time = ZonedDateTime.parse("2019-10-17T07:31:39Z"), - endTime = ZonedDateTime.parse("2019-10-18T07:31:39Z"), - expandedAttrs = setOf(incomingAttrExpandedName, outgoingAttrExpandedName) - ), - "withTemporalValues" to true - ) + TemporalQuery( + timerel = TemporalQuery.Timerel.BETWEEN, + time = ZonedDateTime.parse("2019-10-17T07:31:39Z"), + endTime = ZonedDateTime.parse("2019-10-18T07:31:39Z"), + expandedAttrs = setOf(incomingAttrExpandedName, outgoingAttrExpandedName) + ), + temporalEntitiesQuery.temporalQuery ) + assertTrue(temporalEntitiesQuery.withTemporalValues) + assertEquals(10, temporalEntitiesQuery.limit) + assertEquals(2, temporalEntitiesQuery.offset) } @Test @@ -176,11 +190,11 @@ class QueryServiceTests { APIC_COMPOUND_CONTEXT ) - io.mockk.verify { + verify { temporalEntityAttributeService.getForEntity(entityUri, emptySet(), false) } - io.mockk.verify { + verify { attributeInstanceService.search( match { temporalQuery -> temporalQuery.timerel == TemporalQuery.Timerel.AFTER && @@ -191,7 +205,7 @@ class QueryServiceTests { ) } - io.mockk.verify { + verify { temporalEntityService.buildTemporalEntity( entityUri, match { teaInstanceResult -> teaInstanceResult.size == 2 }, @@ -211,9 +225,8 @@ class QueryServiceTests { attributeName = "incoming", attributeValueType = TemporalEntityAttribute.AttributeValueType.MEASURE ) - every { temporalEntityAttributeService.getForEntities(any(), any(), any()) } returns Mono.just( - listOf(temporalEntityAttribute) - ) + every { temporalEntityAttributeService.getForEntities(any(), any(), any(), any(), any()) } returns + Mono.just(listOf(temporalEntityAttribute)) every { attributeInstanceService.search(any(), any>(), any()) } returns Mono.just( @@ -228,26 +241,32 @@ class QueryServiceTests { every { temporalEntityService.buildTemporalEntities(any(), any(), any(), any()) } returns emptyList() queryService.queryTemporalEntities( - emptySet(), - setOf(beehiveType, apiaryType), - TemporalQuery( - expandedAttrs = emptySet(), - timerel = TemporalQuery.Timerel.BEFORE, - time = ZonedDateTime.parse("2019-10-17T07:31:39Z") + TemporalEntitiesQuery( + emptySet(), + setOf(beehiveType, apiaryType), + TemporalQuery( + expandedAttrs = emptySet(), + timerel = TemporalQuery.Timerel.BEFORE, + time = ZonedDateTime.parse("2019-10-17T07:31:39Z") + ), + false, + 2, + 2 ), - false, APIC_COMPOUND_CONTEXT ) - io.mockk.verify { + verify { temporalEntityAttributeService.getForEntities( + 2, + 2, emptySet(), setOf(beehiveType, apiaryType), emptySet() ) } - io.mockk.verify { + verify { attributeInstanceService.search( match { temporalQuery -> temporalQuery.timerel == TemporalQuery.Timerel.BEFORE && @@ -258,7 +277,7 @@ class QueryServiceTests { ) } - io.mockk.verify { + verify { temporalEntityService.buildTemporalEntities( match { it.first().first == entityUri }, any(), @@ -277,7 +296,12 @@ class QueryServiceTests { attributeName = "incoming", attributeValueType = TemporalEntityAttribute.AttributeValueType.MEASURE ) - every { temporalEntityAttributeService.getForEntities(any(), any(), any()) } returns Mono.just( + every { + temporalEntityAttributeService.getForEntities( + any(), any(), any(), any(), + any() + ) + } returns Mono.just( listOf(temporalEntityAttribute) ) every { @@ -287,20 +311,24 @@ class QueryServiceTests { every { temporalEntityService.buildTemporalEntities(any(), any(), any(), any()) } returns emptyList() val entitiesList = queryService.queryTemporalEntities( - emptySet(), - setOf(beehiveType, apiaryType), - TemporalQuery( - expandedAttrs = emptySet(), - timerel = TemporalQuery.Timerel.BEFORE, - time = ZonedDateTime.parse("2019-10-17T07:31:39Z") + TemporalEntitiesQuery( + emptySet(), + setOf(beehiveType, apiaryType), + TemporalQuery( + expandedAttrs = emptySet(), + timerel = TemporalQuery.Timerel.BEFORE, + time = ZonedDateTime.parse("2019-10-17T07:31:39Z") + ), + false, + 2, + 2 ), - false, APIC_COMPOUND_CONTEXT ) assertTrue(entitiesList.isEmpty()) - io.mockk.verify { + verify { temporalEntityService.buildTemporalEntities( match { it.size == 1 && diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt index e7348f68e..262633a84 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt @@ -311,6 +311,8 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { val temporalEntityAttributes = temporalEntityAttributeService.getForEntities( + 10, + 0, setOf("urn:ngsi-ld:BeeHive:TESTD".toUri(), "urn:ngsi-ld:BeeHive:TESTC".toUri()), setOf("https://ontology.eglobalmark.com/apic#BeeHive"), setOf( @@ -330,6 +332,62 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { .expectComplete() .verify() } + @Test + fun `it should retrieve the temporal entities for the requested limit and offset`() { + val firstRawEntity = loadSampleData("beehive_two_temporal_properties.jsonld") + val secondRawEntity = loadSampleData("beehive.jsonld") + + every { attributeInstanceService.create(any()) } returns Mono.just(2) + + temporalEntityAttributeService.createEntityTemporalReferences(firstRawEntity, listOf(APIC_COMPOUND_CONTEXT)) + .block() + temporalEntityAttributeService.createEntityTemporalReferences(secondRawEntity, listOf(APIC_COMPOUND_CONTEXT)) + .block() + + val temporalEntityAttributes = + temporalEntityAttributeService.getForEntities( + 10, + 1, + setOf("urn:ngsi-ld:BeeHive:TESTD".toUri(), "urn:ngsi-ld:BeeHive:TESTC".toUri()), + setOf("https://ontology.eglobalmark.com/apic#BeeHive"), + setOf( + incomingAttrExpandedName, + outgoingAttrExpandedName + ) + ) + + StepVerifier.create(temporalEntityAttributes) + .expectNextCount(1) + .expectComplete() + .verify() + } + + @Test + fun `it should retrieve the persisted temporal entities count of the requested entities`() { + val rawEntity = loadSampleData("beehive_two_temporal_properties.jsonld") + + every { attributeInstanceService.create(any()) } returns Mono.just(1) + + temporalEntityAttributeService.createEntityTemporalReferences(rawEntity, listOf(APIC_COMPOUND_CONTEXT)) + .block() + + val temporalEntity = + temporalEntityAttributeService.getCountForEntities( + setOf( + "urn:ngsi-ld:BeeHive:TESTD".toUri(), "urn:ngsi-ld:BeeHive:TESTC".toUri(), + "urn:ngsi-ld:BeeHive:TESTD".toUri() + ), + setOf("https://ontology.eglobalmark.com/apic#BeeHive"), + setOf( + incomingAttrExpandedName, + outgoingAttrExpandedName + ) + ) + + StepVerifier.create(temporalEntity) + .expectNextCount(1) + .verifyComplete() + } @Test fun `it should return an empty list no temporal attribute matches the requested entities`() { @@ -345,6 +403,8 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { val temporalEntityAttributes = temporalEntityAttributeService.getForEntities( + 10, + 2, setOf("urn:ngsi-ld:BeeHive:TESTD".toUri(), "urn:ngsi-ld:BeeHive:TESTC".toUri()), setOf("https://ontology.eglobalmark.com/apic#UnknownType"), setOf( @@ -373,6 +433,8 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer { val temporalEntityAttributes = temporalEntityAttributeService.getForEntities( + 10, + 2, setOf("urn:ngsi-ld:BeeHive:TESTD".toUri(), "urn:ngsi-ld:BeeHive:TESTC".toUri()), setOf("https://ontology.eglobalmark.com/apic#BeeHive"), setOf("unknownAttribute") diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt index 751af6a2c..8f3dab8aa 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityHandlerTests.kt @@ -1,7 +1,9 @@ package com.egm.stellio.search.web +import arrow.core.extensions.listk.align.empty import com.egm.stellio.search.config.WebSecurityTestConfig import com.egm.stellio.search.model.SimplifiedAttributeInstanceResult +import com.egm.stellio.search.model.TemporalEntitiesQuery import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.service.AttributeInstanceService @@ -39,7 +41,6 @@ import org.springframework.util.LinkedMultiValueMap import org.springframework.web.reactive.function.BodyInserters import reactor.core.publisher.Flux import reactor.core.publisher.Mono -import java.net.URI import java.time.ZonedDateTime import java.util.UUID @@ -609,14 +610,14 @@ class TemporalEntityHandlerTests { time = ZonedDateTime.parse("2019-10-17T07:31:39Z"), endTime = ZonedDateTime.parse("2019-10-18T07:31:39Z") ) - every { queryService.parseAndCheckQueryParams(any(), any()) } returns mapOf( - "ids" to emptySet(), - "types" to setOf("BeeHive"), - "temporalQuery" to temporalQuery, - "withTemporalValues" to false - ) - coEvery { queryService.queryTemporalEntities(any(), any(), any(), any(), any()) } returns emptyList() + every { temporalEntityAttributeService.getCountForEntities(any(), any(), any()) } answers { Mono.just(2) } + every { queryService.parseAndCheckQueryParams(any(), any()) } returns + buildDefaultQueryParams().copy(types = setOf("BeeHive"), temporalQuery = temporalQuery) + + coEvery { + queryService.queryTemporalEntities(any(), any()) + } returns emptyList() webClient.get() .uri( @@ -627,6 +628,7 @@ class TemporalEntityHandlerTests { .exchange() .expectStatus().isOk .expectBody().json("[]") + verify { queryService.parseAndCheckQueryParams( match { @@ -641,11 +643,15 @@ class TemporalEntityHandlerTests { } coVerify { queryService.queryTemporalEntities( - emptySet(), - setOf("BeeHive"), - temporalQuery, - false, - APIC_COMPOUND_CONTEXT + match { temporalEntitiesQuery -> + temporalEntitiesQuery.limit == 30 && + temporalEntitiesQuery.offset == 0 && + temporalEntitiesQuery.ids.isEmpty() && + temporalEntitiesQuery.types == setOf("BeeHive") && + temporalEntitiesQuery.temporalQuery == temporalQuery && + !temporalEntitiesQuery.withTemporalValues + }, + eq(APIC_COMPOUND_CONTEXT) ) } @@ -659,14 +665,11 @@ class TemporalEntityHandlerTests { ).minus("@context") val secondTemporalEntity = deserializeObject(loadSampleData("beehive.jsonld")).minus("@context") - every { queryService.parseAndCheckQueryParams(any(), any()) } returns mapOf( - "ids" to emptySet(), - "types" to emptySet(), - "temporalQuery" to TemporalQuery(), - "withTemporalValues" to false - ) - coEvery { queryService.queryTemporalEntities(any(), any(), any(), any(), any()) } returns - listOf(firstTemporalEntity, secondTemporalEntity) + every { temporalEntityAttributeService.getCountForEntities(any(), any(), any()) } answers { Mono.just(2) } + every { queryService.parseAndCheckQueryParams(any(), any()) } returns buildDefaultQueryParams() + coEvery { + queryService.queryTemporalEntities(any(), any()) + } returns listOf(firstTemporalEntity, secondTemporalEntity) webClient.get() .uri( @@ -691,14 +694,11 @@ class TemporalEntityHandlerTests { ).minus("@context") val secondTemporalEntity = deserializeObject(loadSampleData("beehive.jsonld")).minus("@context") - every { queryService.parseAndCheckQueryParams(any(), any()) } returns mapOf( - "ids" to emptySet(), - "types" to emptySet(), - "temporalQuery" to TemporalQuery(), - "withTemporalValues" to false - ) - coEvery { queryService.queryTemporalEntities(any(), any(), any(), any(), any()) } returns - listOf(firstTemporalEntity, secondTemporalEntity) + every { temporalEntityAttributeService.getCountForEntities(any(), any(), any()) } answers { Mono.just(2) } + every { queryService.parseAndCheckQueryParams(any(), any()) } returns buildDefaultQueryParams() + coEvery { + queryService.queryTemporalEntities(any(), any()) + } returns listOf(firstTemporalEntity, secondTemporalEntity) webClient.get() .uri( @@ -816,4 +816,219 @@ class TemporalEntityHandlerTests { assertEquals(null, temporalQuery.time) assertEquals(null, temporalQuery.timerel) } + + @Test + fun `query temporal entity should return 200 with prev link header if exists`() { + + val firstTemporalEntity = deserializeObject( + loadSampleData("beehive_with_two_temporal_attributes_evolution.jsonld") + ).minus("@context") + val secondTemporalEntity = deserializeObject(loadSampleData("beehive.jsonld")).minus("@context") + + every { temporalEntityAttributeService.getCountForEntities(any(), any(), any()) } returns Mono.just(2) + every { queryService.parseAndCheckQueryParams(any(), any()) } returns + buildDefaultQueryParams().copy(limit = 1, offset = 2) + coEvery { + queryService.queryTemporalEntities(any(), any()) + } returns + listOf(firstTemporalEntity, secondTemporalEntity) + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&time=2019-10-17T07:31:39Z&endTime=2019-10-18T07:31:39Z&" + + "type=BeeHive&limit=1&offset=2" + ) + .exchange() + .expectStatus().isOk + .expectHeader() + .valueEquals( + "Link", + ";rel=\"prev\";type=\"application/ld+json\"" + ) + } + + @Test + fun `query temporal entity should return 200 and empty response if requested offset does not exists`() { + + every { temporalEntityAttributeService.getCountForEntities(any(), any(), any()) } returns Mono.just(2) + every { queryService.parseAndCheckQueryParams(any(), any()) } returns buildDefaultQueryParams() + coEvery { queryService.queryTemporalEntities(any(), any()) } returns empty() + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&time=2019-10-17T07:31:39Z&endTime=2019-10-18T07:31:39Z&" + + "type=BeeHive&limit=1&offset=9" + ) + .exchange() + .expectStatus().isOk + .expectBody().json("[]") + } + + @Test + fun `query temporal entity should return 200 with next link header if exists`() { + val firstTemporalEntity = deserializeObject( + loadSampleData("beehive_with_two_temporal_attributes_evolution.jsonld") + ).minus("@context") + val secondTemporalEntity = deserializeObject(loadSampleData("beehive.jsonld")).minus("@context") + + every { temporalEntityAttributeService.getCountForEntities(any(), any(), any()) } returns Mono.just(2) + every { queryService.parseAndCheckQueryParams(any(), any()) } returns + buildDefaultQueryParams().copy(limit = 1, offset = 0) + coEvery { + queryService.queryTemporalEntities(any(), any()) + } returns + listOf(firstTemporalEntity, secondTemporalEntity) + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&time=2019-10-17T07:31:39Z&endTime=2019-10-18T07:31:39Z&" + + "type=BeeHive&limit=1&offset=0" + ) + .exchange() + .expectStatus().isOk + .expectHeader() + .valueEquals( + "Link", + ";rel=\"next\";type=\"application/ld+json\"" + ) + } + + @Test + fun `query temporal entity should return 200 with prev and next link header if exists`() { + val firstTemporalEntity = deserializeObject( + loadSampleData("beehive_with_two_temporal_attributes_evolution.jsonld") + ).minus("@context") + val secondTemporalEntity = deserializeObject(loadSampleData("beehive.jsonld")).minus("@context") + + every { temporalEntityAttributeService.getCountForEntities(any(), any(), any()) } returns Mono.just(3) + every { queryService.parseAndCheckQueryParams(any(), any()) } returns + buildDefaultQueryParams().copy(limit = 1, offset = 1) + coEvery { + queryService.queryTemporalEntities(any(), any()) + } returns + listOf(firstTemporalEntity, secondTemporalEntity) + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&time=2019-10-17T07:31:39Z&endTime=2019-10-18T07:31:39Z&" + + "type=BeeHive&limit=1&offset=1" + ) + .exchange() + .expectStatus().isOk + .expectHeader() + .valueEquals( + "Link", + ";rel=\"prev\";type=\"application/ld+json\"", + ";rel=\"next\";type=\"application/ld+json\"" + ) + } + + @Test + fun `query temporal entity should return 400 if requested offset is less than zero`() { + + every { queryService.parseAndCheckQueryParams(any(), any()) } returns buildDefaultQueryParams() + coEvery { + queryService.queryTemporalEntities(any(), any()) + } throws BadRequestDataException( + "Offset must be greater than zero and limit must be strictly greater than zero" + ) + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&time=2019-10-17T07:31:39Z&endTime=2019-10-18T07:31:39Z&" + + "type=BeeHive&limit=1&offset=-1" + ) + .exchange() + .expectStatus().isBadRequest + .expectBody().json( + """ + { + "type":"https://uri.etsi.org/ngsi-ld/errors/BadRequestData", + "title":"The request includes input data which does not meet the requirements of the operation", + "detail":"Offset must be greater than zero and limit must be strictly greater than zero" + } + """.trimIndent() + ) + } + + @Test + fun `query temporal entity should return 400 if limit is equal or less than zero`() { + + every { queryService.parseAndCheckQueryParams(any(), any()) } returns buildDefaultQueryParams() + coEvery { + queryService.queryTemporalEntities(any(), any()) + } throws BadRequestDataException( + "Offset must be greater than zero and limit must be strictly greater than zero" + ) + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&time=2019-10-17T07:31:39Z&endTime=2019-10-18T07:31:39Z&" + + "type=BeeHive&limit=-1&offset=1" + ) + .exchange() + .expectStatus().isBadRequest + .expectBody().json( + """ + { + "type":"https://uri.etsi.org/ngsi-ld/errors/BadRequestData", + "title":"The request includes input data which does not meet the requirements of the operation", + "detail":"Offset must be greater than zero and limit must be strictly greater than zero" + } + """.trimIndent() + ) + } + + @Test + fun `query temporal entity should return 400 if limit is greater than the maximum authorized limit`() { + + every { queryService.parseAndCheckQueryParams(any(), any()) } returns buildDefaultQueryParams() + coEvery { + queryService.queryTemporalEntities(any(), any()) + } throws BadRequestDataException( + "You asked for 200 results, but the supported maximum limit is 100" + ) + + webClient.get() + .uri( + "/ngsi-ld/v1/temporal/entities?" + + "timerel=between&time=2019-10-17T07:31:39Z&endTime=2019-10-18T07:31:39Z&" + + "type=BeeHive&limit=200&offset=1" + ) + .exchange() + .expectStatus().isBadRequest + .expectBody().json( + """ + { + "type":"https://uri.etsi.org/ngsi-ld/errors/BadRequestData", + "title":"The request includes input data which does not meet the requirements of the operation", + "detail":"You asked for 200 results, but the supported maximum limit is 100" + } + """.trimIndent() + ) + } + + private fun buildDefaultQueryParams(): TemporalEntitiesQuery = + TemporalEntitiesQuery( + ids = emptySet(), + types = emptySet(), + temporalQuery = TemporalQuery(), + withTemporalValues = false, + offset = 0, + limit = 30 + ) } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt index 4816294fc..6249241ca 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/web/TemporalEntityOperationsHandlerTests.kt @@ -1,8 +1,10 @@ package com.egm.stellio.search.web import com.egm.stellio.search.config.WebSecurityTestConfig +import com.egm.stellio.search.model.TemporalEntitiesQuery import com.egm.stellio.search.model.TemporalQuery import com.egm.stellio.search.service.QueryService +import com.egm.stellio.search.service.TemporalEntityAttributeService import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.util.* import com.ninjasquad.springmockk.MockkBean @@ -18,7 +20,7 @@ import org.springframework.test.context.ActiveProfiles import org.springframework.test.web.reactive.server.WebTestClient import org.springframework.util.LinkedMultiValueMap import org.springframework.web.reactive.function.BodyInserters -import java.net.URI +import reactor.core.publisher.Mono import java.time.ZonedDateTime @ActiveProfiles("test") @@ -32,6 +34,9 @@ class TemporalEntityOperationsHandlerTests { @Autowired private lateinit var webClient: WebTestClient + @MockkBean(relaxed = true) + private lateinit var temporalEntityAttributeService: TemporalEntityAttributeService + @MockkBean(relaxed = true) private lateinit var queryService: QueryService @@ -57,13 +62,18 @@ class TemporalEntityOperationsHandlerTests { endTime = ZonedDateTime.parse("2019-10-18T07:31:39Z"), expandedAttrs = setOf(incomingAttrExpandedName, outgoingAttrExpandedName) ) - every { queryService.parseAndCheckQueryParams(any(), any()) } returns mapOf( - "ids" to emptySet(), - "types" to setOf("BeeHive", "Apiary"), - "temporalQuery" to temporalQuery, - "withTemporalValues" to true - ) - coEvery { queryService.queryTemporalEntities(any(), any(), any(), any(), any()) } returns emptyList() + + every { temporalEntityAttributeService.getCountForEntities(any(), any(), any()) } answers { Mono.just(2) } + every { queryService.parseAndCheckQueryParams(any(), any()) } returns + TemporalEntitiesQuery( + ids = emptySet(), + types = setOf("BeeHive", "Apiary"), + temporalQuery = temporalQuery, + withTemporalValues = true, + limit = 1, + offset = 0 + ) + coEvery { queryService.queryTemporalEntities(any(), any()) } returns emptyList() val queryParams = LinkedMultiValueMap() queryParams.add("options", "temporalValues") @@ -88,11 +98,15 @@ class TemporalEntityOperationsHandlerTests { } coVerify { queryService.queryTemporalEntities( - emptySet(), - setOf("BeeHive", "Apiary"), - temporalQuery, - true, - APIC_COMPOUND_CONTEXT + match { temporalEntitiesQuery -> + temporalEntitiesQuery.limit == 1 && + temporalEntitiesQuery.offset == 0 && + temporalEntitiesQuery.ids.isEmpty() && + temporalEntitiesQuery.types == setOf("BeeHive", "Apiary") && + temporalEntitiesQuery.temporalQuery == temporalQuery && + temporalEntitiesQuery.withTemporalValues + }, + eq(APIC_COMPOUND_CONTEXT) ) } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt index e542245e2..3d0e23280 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/ApiUtils.kt @@ -10,6 +10,7 @@ import org.springframework.http.HttpHeaders import org.springframework.http.HttpStatus import org.springframework.http.MediaType import org.springframework.http.ResponseEntity +import org.springframework.util.MultiValueMap import org.springframework.web.server.NotAcceptableStatusException import java.time.ZonedDateTime import java.time.format.DateTimeParseException @@ -112,6 +113,23 @@ fun parseAndExpandRequestParameter(requestParam: String?, contextLink: String): JsonLdUtils.expandJsonLdKey(it.trim(), contextLink)!! }.toSet() +fun extractAndValidatePaginationParameters( + queryParams: MultiValueMap, + limitDefault: Int, + limitMax: Int, + isAskingForCount: Boolean = false +): Pair { + val offset = queryParams.getFirst(QUERY_PARAM_OFFSET)?.toIntOrNull() ?: 0 + val limit = queryParams.getFirst(QUERY_PARAM_LIMIT)?.toIntOrNull() ?: limitDefault + if (!isAskingForCount && (limit <= 0 || offset < 0)) + throw BadRequestDataException("Offset must be greater than zero and limit must be strictly greater than zero") + if (isAskingForCount && (limit < 0 || offset < 0)) + throw BadRequestDataException("Offset and limit must be greater than zero") + if (limit > limitMax) + throw BadRequestDataException("You asked for $limit results, but the supported maximum limit is $limitMax") + return Pair(offset, limit) +} + fun getApplicableMediaType(httpHeaders: HttpHeaders): MediaType = httpHeaders.accept.getApplicable() diff --git a/subscription-service/config/detekt/baseline.xml b/subscription-service/config/detekt/baseline.xml index 8737f6423..57703896a 100644 --- a/subscription-service/config/detekt/baseline.xml +++ b/subscription-service/config/detekt/baseline.xml @@ -6,7 +6,6 @@ LongParameterList:FixtureUtils.kt$( withQueryAndGeoQuery: Pair<Boolean, Boolean> = Pair(true, true), withEndpointInfo: Boolean = true, withNotifParams: Pair<FormatType, List<String>> = Pair(FormatType.NORMALIZED, emptyList()), withModifiedAt: Boolean = false, georel: String = "within", coordinates: Any = "[[100.0, 0.0], [101.0, 0.0], [101.0, 1.0], [100.0, 1.0], [100.0, 0.0]]" ) MaxLineLength:SubscriptionService.kt$SubscriptionService$ AND ( string_to_array(watched_attributes, ',') && string_to_array(:updatedAttributes, ',') OR watched_attributes IS NULL ) ReturnCount:QueryUtils.kt$QueryUtils$fun extractGeorelParams(georel: String): Triple<String, String?, String?> - ReturnCount:SubscriptionHandler.kt$SubscriptionHandler$ @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getSubscriptions( @RequestHeader httpHeaders: HttpHeaders, @RequestParam params: MultiValueMap<String, String>, @RequestParam options: Optional<String> ): ResponseEntity<*> SwallowedException:ParsingUtils.kt$ParsingUtils$catch (e: Exception) { logger.error("Error while parsing a subscription: ${e.message}", e) throw BadRequestDataException(e.message ?: "Failed to parse subscription") } SwallowedException:QueryUtils.kt$QueryUtils$catch (e: Exception) { throw BadRequestDataException("Unmatched query since it contains an unknown attribute $it") } SwallowedException:SubscriptionService.kt$SubscriptionService$catch (e: Exception) { false } diff --git a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/web/SubscriptionHandler.kt b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/web/SubscriptionHandler.kt index ee1c3c17d..2a702ed83 100644 --- a/subscription-service/src/main/kotlin/com/egm/stellio/subscription/web/SubscriptionHandler.kt +++ b/subscription-service/src/main/kotlin/com/egm/stellio/subscription/web/SubscriptionHandler.kt @@ -77,27 +77,14 @@ class SubscriptionHandler( @RequestParam params: MultiValueMap, @RequestParam options: Optional ): ResponseEntity<*> { - val offset = params.getFirst(QUERY_PARAM_OFFSET)?.toIntOrNull() ?: 0 - val limit = params.getFirst(QUERY_PARAM_LIMIT)?.toIntOrNull() ?: applicationProperties.pagination.limitDefault + val (offset, limit) = extractAndValidatePaginationParameters( + params, + applicationProperties.pagination.limitDefault, + applicationProperties.pagination.limitMax + ) val includeSysAttrs = options.filter { it.contains("sysAttrs") }.isPresent val contextLink = getContextFromLinkHeaderOrDefault(httpHeaders) val mediaType = getApplicableMediaType(httpHeaders) - if (limit <= 0 || offset < 0) - return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) - .body( - BadRequestDataResponse( - "Offset must be greater than zero and limit must be strictly greater than zero" - ) - ) - - if (limit > applicationProperties.pagination.limitMax) - return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON) - .body( - BadRequestDataResponse( - "You asked for $limit results, " + - "but the supported maximum limit is ${applicationProperties.pagination.limitMax}" - ) - ) val userId = extractSubjectOrEmpty().awaitFirst() val subscriptions = subscriptionService.getSubscriptions(limit, offset, userId) From 0f73bb7200c259bd659ae1a000b5e7372c415e70 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 26 Sep 2021 17:37:02 +0200 Subject: [PATCH 3/5] feat(entity): minor perf improvements in queries for entities (node label uses) --- .../com/egm/stellio/entity/repository/QueryUtils.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt index 0d05e5164..d2afd580b 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt @@ -129,9 +129,9 @@ object QueryUtils { """.trimIndent() else """ - WITH collect(entity) as entities, count(entity) as count - UNWIND entities as entity - RETURN entity.id as id, count + WITH collect(entity.id) as entitiesIds, count(entity) as count + UNWIND entitiesIds as entityId + RETURN entityId as id, count ORDER BY id SKIP $offset LIMIT $limit """.trimIndent() @@ -168,7 +168,7 @@ object QueryUtils { if (parsedQueryTerm.third.isRelationshipTarget()) { """ EXISTS { - MATCH (entity)-[:HAS_OBJECT]-()-[:${parsedQueryTerm.first}]->(e) + MATCH (entity:Entity)-[:HAS_OBJECT]-()-[:${parsedQueryTerm.first}]->(e) WHERE e.id ${parsedQueryTerm.second} ${parsedQueryTerm.third} } """.trimIndent() @@ -192,7 +192,7 @@ object QueryUtils { else """ EXISTS { - MATCH (entity)-[:HAS_VALUE]->(p:Property) + MATCH (entity:Entity)-[:HAS_VALUE]->(p:Property) WHERE p.name = '${expandJsonLdKey(comparablePropertyPath[0], contexts)!!}' AND p.$comparablePropertyName ${parsedQueryTerm.second} $comparableValue } @@ -208,7 +208,7 @@ object QueryUtils { ) { expandedAttr -> """ EXISTS { - MATCH (entity) + MATCH (entity:Entity) WHERE ( (entity)-[:HAS_VALUE]->(:Property { name: '$expandedAttr' }) OR From 0b559035af93d5a1f7418e9b39cf6dbfd8c91c24 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Thu, 29 Apr 2021 18:46:21 +0200 Subject: [PATCH 4/5] feat(entity): improve performance of the query to create a relationship - add a separate query to search for the eventual target (node index scan) - create or just link to the target depending on the result of the search query --- .../entity/repository/Neo4jRepository.kt | 52 ++++++++++++++++--- .../entity/repository/Neo4jRepositoryTests.kt | 26 +++++++--- 2 files changed, 63 insertions(+), 15 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt index 279facebb..f09bcbae4 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt @@ -63,15 +63,51 @@ class Neo4jRepository( targetId: URI ): Boolean { val relationshipType = relationship.type[0].toRelationshipTypeName() - val query = + + // first search for an existing target entity or partial entity with this id + val queryForTargetExistence = """ - MATCH (subject:${subjectNodeInfo.label} { id: ${'$'}subjectId }) - MERGE (target { id: ${'$'}targetId }) - ON CREATE SET target:PartialEntity - CREATE (subject)-[:HAS_OBJECT]-> - (r:Attribute:Relationship:`${relationship.type[0]}` ${'$'}props)-[:$relationshipType]->(target) - RETURN r.id as id - """ + MATCH (target:Entity) + WHERE target.id = ${'$'}targetId + RETURN labels(target) as labels + UNION MATCH (target:PartialEntity) + WHERE target.id = ${'$'}targetId + RETURN labels(target) as labels + """.trimIndent() + val parametersForExistence = mapOf( + "targetId" to targetId + ) + + val resultForExistence = + neo4jClient.query(queryForTargetExistence).bindAll(parametersForExistence).fetch().first() + val targetAlreadyExists = resultForExistence.isPresent + // if the target exists, find whether it is an entity or a partial entity + val labelForExisting = + if (targetAlreadyExists) + resultForExistence + .get() + .values + .map { (it as Array).toList() } + .flatten() + .first { it == "Entity" || it == "PartialEntity" } + else "" + + val query = + if (targetAlreadyExists) + """ + MATCH (subject:${subjectNodeInfo.label} { id: ${'$'}subjectId }) + WITH subject + MATCH (target:$labelForExisting { id: ${'$'}targetId }) + CREATE (subject)-[:HAS_OBJECT]->(r:Attribute:Relationship:`${relationship.type[0]}` ${'$'}props) + -[:$relationshipType]->(target) + """ + else + """ + MATCH (subject:${subjectNodeInfo.label} { id: ${'$'}subjectId }) + MERGE (target:PartialEntity { id: ${'$'}targetId }) + CREATE (subject)-[:HAS_OBJECT]->(r:Attribute:Relationship:`${relationship.type[0]}` ${'$'}props) + -[:$relationshipType]->(target) + """.trimIndent() val parameters = mapOf( "props" to relationship.nodeProperties(), diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt index 10c1b4b6b..1b61563fd 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt @@ -327,6 +327,8 @@ class Neo4jRepositoryTests : WithNeo4jContainer { val somePartialEntity = partialEntityRepository.findById(partialTargetEntityUri) assertTrue(somePartialEntity.isPresent) + + neo4jRepository.deleteEntity(entity.id) } @Test @@ -345,7 +347,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer { mutableListOf(temperatureProperty) ) - createEntity("urn:ngsi-ld:Sensor:6789".toUri(), listOf("Sensor")) + val secondTargetEntity = createEntity("urn:ngsi-ld:Sensor:6789".toUri(), listOf("Sensor")) val newPropertyPayload = """ { @@ -376,6 +378,10 @@ class Neo4jRepositoryTests : WithNeo4jContainer { propertyRepository.findById(updatedPropertyId).get().relationships[0].type[0], "https://uri.etsi.org/ngsi-ld/default-context/newRel" ) + + neo4jRepository.deleteEntity(targetEntity.id) + neo4jRepository.deleteEntity(entity.id) + neo4jRepository.deleteEntity(secondTargetEntity.id) } @Test @@ -415,13 +421,13 @@ class Neo4jRepositoryTests : WithNeo4jContainer { @Test fun `it should update modifiedAt value when updating an entity`() { - createEntity( + val entity = createEntity( "urn:ngsi-ld:Beekeeper:1233".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = "name", value = "Scalpa")) ) val modifiedAt = entityRepository.findById("urn:ngsi-ld:Beekeeper:1233".toUri()).get().modifiedAt - createEntity( + val secondEntity = createEntity( "urn:ngsi-ld:Beekeeper:1233".toUri(), listOf("Beekeeper"), mutableListOf(Property(name = "name", value = "Demha")) @@ -429,6 +435,9 @@ class Neo4jRepositoryTests : WithNeo4jContainer { val updatedModifiedAt = entityRepository.findById("urn:ngsi-ld:Beekeeper:1233".toUri()).get().modifiedAt assertNotNull(updatedModifiedAt) assertThat(updatedModifiedAt).isAfter(modifiedAt) + + neo4jRepository.deleteEntity(entity.id) + neo4jRepository.deleteEntity(secondEntity.id) } @Test @@ -687,7 +696,10 @@ class Neo4jRepositoryTests : WithNeo4jContainer { neo4jRepository.deleteEntity(sensor.id) val entity = entityRepository.findById(device.id).get() - assertEquals(entity.relationships.size, 0) + assertEquals(0, entity.relationships.size) + + neo4jRepository.deleteEntity(sensor.id) + neo4jRepository.deleteEntity(device.id) } @Test @@ -968,7 +980,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer { val propertiesInformation = attributesInformation["properties"] as Set<*> - assertEquals(3, propertiesInformation.size) + assertEquals("Got the following attributes: $propertiesInformation", 3, propertiesInformation.size) assertTrue(propertiesInformation.containsAll(listOf("humidity", "temperature", "incoming"))) assertEquals(attributesInformation["relationships"], emptySet()) assertEquals(attributesInformation["geoProperties"], emptySet()) @@ -1078,7 +1090,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer { val entityTypesNames = neo4jRepository.getEntityTypesNames() - assertEquals(entityTypesNames.size, 2) + assertEquals(2, entityTypesNames.size) assertTrue( entityTypesNames.containsAll( listOf("https://ontology.eglobalmark.com/apic#Beehive", "https://ontology.eglobalmark.com/apic#Sensor") @@ -1116,7 +1128,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer { val entityTypes = neo4jRepository.getEntityTypes() - assertEquals(2, entityTypes.size) + assertEquals("Got the following types instead: $entityTypes", 2, entityTypes.size) assertTrue( entityTypes.containsAll( listOf( From 1da80005b1c8e887966a8f6261b188bc02885d49 Mon Sep 17 00:00:00 2001 From: Benoit Orihuela Date: Sun, 26 Sep 2021 18:41:40 +0200 Subject: [PATCH 5/5] fixup(entity): improve performance of the query to create a relationship --- .../com/egm/stellio/entity/repository/Neo4jRepository.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt index f09bcbae4..2ab19fcfe 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt @@ -75,7 +75,7 @@ class Neo4jRepository( RETURN labels(target) as labels """.trimIndent() val parametersForExistence = mapOf( - "targetId" to targetId + "targetId" to targetId.toString() ) val resultForExistence = @@ -87,7 +87,7 @@ class Neo4jRepository( resultForExistence .get() .values - .map { (it as Array).toList() } + .map { it as List } .flatten() .first { it == "Entity" || it == "PartialEntity" } else ""