Merge branch 'develop'

docs/roadmap.md

@@ -14,15 +14,15 @@ guidelines only, and this section may be revised to provide newer information at
 ## Short term
 
 The following list of features are planned to be addressed in the short term, and incorporated in the next release of
-the product planned for end of March 2021:
+the product planned for end of November 2021:
 
 - Implement multi-attributes support for GeoProperties [#101](https://github.com/stellio-hub/stellio-context-broker/issues/101)
-- Finish implementation of some missing common cross-cutting behaviors as defined in the NGSI-LD specification [#11](https://github.com/stellio-hub/stellio-context-broker/issues/11), [#12](https://github.com/stellio-hub/stellio-context-broker/issues/12), [#52](https://github.com/stellio-hub/stellio-context-broker/issues/52), [#146](https://github.com/stellio-hub/stellio-context-broker/issues/146), [#206](https://github.com/stellio-hub/stellio-context-broker/issues/206), [#287](https://github.com/stellio-hub/stellio-context-broker/issues/287)
+- Finish implementation of some missing common cross-cutting behaviors as defined in the NGSI-LD specification [#12](https://github.com/stellio-hub/stellio-context-broker/issues/12), [#206](https://github.com/stellio-hub/stellio-context-broker/issues/206)
 - Implement the discovery endpoints introduced in version 1.3.1 of the NGSI-LD specification [#268](https://github.com/stellio-hub/stellio-context-broker/issues/268)
 - Implement support for the batch entities update endpoint [#62](https://github.com/stellio-hub/stellio-context-broker/issues/62)
 - Fix the currently [identified issues](https://github.com/stellio-hub/stellio-context-broker/issues?q=is%3Aissue+is%3Aopen+label%3Afix)
 - Implement support for the aggregated temporal representation of entities introduced in version 1.4.1 of the NGSI-LD specification
-- Upgrade frameworks and libraries to their last released version (Spring Boot 2.4.x, Timescale 2.x, ...)
+- Complete the requirements to become an approved full Generic Enabler
 
 ## Medium term
 
@@ -33,7 +33,6 @@ release(s) generated in the next **9 months** after next planned release:
 - Implement support for the all the supported data types (e.g. structured property value)
 - Implement distributed capabilities (via support of Context Source as defined in the NGSI-LD specification)
 - Experiment with an alternative Graph database (namely Janus Graph)
-- Complete the requirements to become an approved full Generic Enabler
 - Expose an API allowing the management of authorizations inside the information context
 
 ## Long term

entity-service/config/detekt/baseline.xml

@@ -2,7 +2,6 @@
 <SmellBaseline>
   <ManuallySuppressedIssues></ManuallySuppressedIssues>
   <CurrentIssues>
-    <ID>ComplexMethod:EntityHandler.kt$EntityHandler$ @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @RequestParam params: MultiValueMap&lt;String, String&gt; ): ResponseEntity&lt;*&gt;</ID>
     <ID>EmptyFunctionBlock:StandaloneAuthorizationService.kt$StandaloneAuthorizationService${}</ID>
     <ID>LargeClass:EntityHandlerTests.kt$EntityHandlerTests</ID>
     <ID>LargeClass:EntityOperationHandlerTests.kt$EntityOperationHandlerTests</ID>

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt

@@ -63,15 +63,51 @@ class Neo4jRepository(
         targetId: URI
     ): Boolean {
         val relationshipType = relationship.type[0].toRelationshipTypeName()
-        val query =
+
+        // first search for an existing target entity or partial entity with this id
+        val queryForTargetExistence =
             """
-            MATCH (subject:${subjectNodeInfo.label} { id: ${'$'}subjectId })
-            MERGE (target { id: ${'$'}targetId })
-            ON CREATE SET target:PartialEntity
-            CREATE (subject)-[:HAS_OBJECT]->
-                (r:Attribute:Relationship:`${relationship.type[0]}` ${'$'}props)-[:$relationshipType]->(target)
-            RETURN r.id as id
-        """
+            MATCH (target:Entity)
+            WHERE target.id = ${'$'}targetId
+            RETURN labels(target) as labels
+            UNION MATCH (target:PartialEntity)
+            WHERE target.id = ${'$'}targetId
+            RETURN labels(target) as labels
+            """.trimIndent()
+        val parametersForExistence = mapOf(
+            "targetId" to targetId.toString()
+        )
+
+        val resultForExistence =
+            neo4jClient.query(queryForTargetExistence).bindAll(parametersForExistence).fetch().first()
+        val targetAlreadyExists = resultForExistence.isPresent
+        // if the target exists, find whether it is an entity or a partial entity
+        val labelForExisting =
+            if (targetAlreadyExists)
+                resultForExistence
+                    .get()
+                    .values
+                    .map { it as List<String> }
+                    .flatten()
+                    .first { it == "Entity" || it == "PartialEntity" }
+            else ""
+
+        val query =
+            if (targetAlreadyExists)
+                """
+                MATCH (subject:${subjectNodeInfo.label} { id: ${'$'}subjectId })
+                WITH subject
+                MATCH (target:$labelForExisting { id: ${'$'}targetId })
+                CREATE (subject)-[:HAS_OBJECT]->(r:Attribute:Relationship:`${relationship.type[0]}` ${'$'}props)
+                        -[:$relationshipType]->(target)
+                """
+            else
+                """
+                MATCH (subject:${subjectNodeInfo.label} { id: ${'$'}subjectId })
+                MERGE (target:PartialEntity { id: ${'$'}targetId })
+                CREATE (subject)-[:HAS_OBJECT]->(r:Attribute:Relationship:`${relationship.type[0]}` ${'$'}props)
+                        -[:$relationshipType]->(target)
+                """.trimIndent()
 
         val parameters = mapOf(
             "props" to relationship.nodeProperties(),

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/QueryUtils.kt

@@ -129,9 +129,9 @@ object QueryUtils {
             """.trimIndent()
         else
             """
-            WITH collect(entity) as entities, count(entity) as count
-            UNWIND entities as entity
-            RETURN entity.id as id, count
+            WITH collect(entity.id) as entitiesIds, count(entity) as count
+            UNWIND entitiesIds as entityId
+            RETURN entityId as id, count
             ORDER BY id
             SKIP $offset LIMIT $limit
             """.trimIndent()
@@ -168,7 +168,7 @@ object QueryUtils {
             if (parsedQueryTerm.third.isRelationshipTarget()) {
                 """
                     EXISTS {
-                        MATCH (entity)-[:HAS_OBJECT]-()-[:${parsedQueryTerm.first}]->(e)
+                        MATCH (entity:Entity)-[:HAS_OBJECT]-()-[:${parsedQueryTerm.first}]->(e)
                         WHERE e.id ${parsedQueryTerm.second} ${parsedQueryTerm.third}
                     }
                 """.trimIndent()
@@ -192,7 +192,7 @@ object QueryUtils {
                 else
                     """
                        EXISTS {
-                           MATCH (entity)-[:HAS_VALUE]->(p:Property)
+                           MATCH (entity:Entity)-[:HAS_VALUE]->(p:Property)
                            WHERE p.name = '${expandJsonLdKey(comparablePropertyPath[0], contexts)!!}'
                            AND p.$comparablePropertyName ${parsedQueryTerm.second} $comparableValue
                        }
@@ -208,7 +208,7 @@ object QueryUtils {
         ) { expandedAttr ->
             """
                EXISTS {
-                   MATCH (entity)
+                   MATCH (entity:Entity)
                    WHERE (
                         (entity)-[:HAS_VALUE]->(:Property { name: '$expandedAttr' })
                         OR 

entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityHandler.kt

@@ -85,8 +85,12 @@ class EntityHandler(
         @RequestParam params: MultiValueMap<String, String>
     ): ResponseEntity<*> {
         val count = params.getFirst(QUERY_PARAM_COUNT)?.toBoolean() ?: false
-        val offset = params.getFirst(QUERY_PARAM_OFFSET)?.toIntOrNull() ?: 0
-        val limit = params.getFirst(QUERY_PARAM_LIMIT)?.toIntOrNull() ?: applicationProperties.pagination.limitDefault
+        val (offset, limit) = extractAndValidatePaginationParameters(
+            params,
+            applicationProperties.pagination.limitDefault,
+            applicationProperties.pagination.limitMax,
+            count
+        )
         val ids = params.getFirst(QUERY_PARAM_ID)?.split(",")
         val type = params.getFirst(QUERY_PARAM_TYPE)
         val idPattern = params.getFirst(QUERY_PARAM_ID_PATTERN)
@@ -100,29 +104,6 @@ class EntityHandler(
         val mediaType = getApplicableMediaType(httpHeaders)
         val userId = extractSubjectOrEmpty().awaitFirst()
 
-        if (!count && (limit <= 0 || offset < 0))
-            return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON)
-                .body(
-                    BadRequestDataResponse(
-                        "Offset must be greater than zero and limit must be strictly greater than zero"
-                    )
-                )
-
-        if (count && (limit < 0 || offset < 0))
-            return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON)
-                .body(
-                    BadRequestDataResponse("Offset and limit must be greater than zero")
-                )
-
-        if (limit > applicationProperties.pagination.limitMax)
-            return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON)
-                .body(
-                    BadRequestDataResponse(
-                        "You asked for $limit results, " +
-                            "but the supported maximum limit is ${applicationProperties.pagination.limitMax}"
-                    )
-                )
-
         if (q == null && type == null && attrs == null)
             return ResponseEntity.status(HttpStatus.BAD_REQUEST).contentType(MediaType.APPLICATION_JSON)
                 .body(

entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt

@@ -327,6 +327,8 @@ class Neo4jRepositoryTests : WithNeo4jContainer {
 
         val somePartialEntity = partialEntityRepository.findById(partialTargetEntityUri)
         assertTrue(somePartialEntity.isPresent)
+
+        neo4jRepository.deleteEntity(entity.id)
     }
 
     @Test
@@ -345,7 +347,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer {
             mutableListOf(temperatureProperty)
         )
 
-        createEntity("urn:ngsi-ld:Sensor:6789".toUri(), listOf("Sensor"))
+        val secondTargetEntity = createEntity("urn:ngsi-ld:Sensor:6789".toUri(), listOf("Sensor"))
         val newPropertyPayload =
             """
             {
@@ -376,6 +378,10 @@ class Neo4jRepositoryTests : WithNeo4jContainer {
             propertyRepository.findById(updatedPropertyId).get().relationships[0].type[0],
             "https://uri.etsi.org/ngsi-ld/default-context/newRel"
         )
+
+        neo4jRepository.deleteEntity(targetEntity.id)
+        neo4jRepository.deleteEntity(entity.id)
+        neo4jRepository.deleteEntity(secondTargetEntity.id)
     }
 
     @Test
@@ -415,20 +421,23 @@ class Neo4jRepositoryTests : WithNeo4jContainer {
 
     @Test
     fun `it should update modifiedAt value when updating an entity`() {
-        createEntity(
+        val entity = createEntity(
             "urn:ngsi-ld:Beekeeper:1233".toUri(),
             listOf("Beekeeper"),
             mutableListOf(Property(name = "name", value = "Scalpa"))
         )
         val modifiedAt = entityRepository.findById("urn:ngsi-ld:Beekeeper:1233".toUri()).get().modifiedAt
-        createEntity(
+        val secondEntity = createEntity(
             "urn:ngsi-ld:Beekeeper:1233".toUri(),
             listOf("Beekeeper"),
             mutableListOf(Property(name = "name", value = "Demha"))
         )
         val updatedModifiedAt = entityRepository.findById("urn:ngsi-ld:Beekeeper:1233".toUri()).get().modifiedAt
         assertNotNull(updatedModifiedAt)
         assertThat(updatedModifiedAt).isAfter(modifiedAt)
+
+        neo4jRepository.deleteEntity(entity.id)
+        neo4jRepository.deleteEntity(secondEntity.id)
     }
 
     @Test
@@ -687,7 +696,10 @@ class Neo4jRepositoryTests : WithNeo4jContainer {
         neo4jRepository.deleteEntity(sensor.id)
 
         val entity = entityRepository.findById(device.id).get()
-        assertEquals(entity.relationships.size, 0)
+        assertEquals(0, entity.relationships.size)
+
+        neo4jRepository.deleteEntity(sensor.id)
+        neo4jRepository.deleteEntity(device.id)
     }
 
     @Test
@@ -968,7 +980,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer {
 
         val propertiesInformation = attributesInformation["properties"] as Set<*>
 
-        assertEquals(3, propertiesInformation.size)
+        assertEquals("Got the following attributes: $propertiesInformation", 3, propertiesInformation.size)
         assertTrue(propertiesInformation.containsAll(listOf("humidity", "temperature", "incoming")))
         assertEquals(attributesInformation["relationships"], emptySet<String>())
         assertEquals(attributesInformation["geoProperties"], emptySet<String>())
@@ -1078,7 +1090,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer {
 
         val entityTypesNames = neo4jRepository.getEntityTypesNames()
 
-        assertEquals(entityTypesNames.size, 2)
+        assertEquals(2, entityTypesNames.size)
         assertTrue(
             entityTypesNames.containsAll(
                 listOf("https://ontology.eglobalmark.com/apic#Beehive", "https://ontology.eglobalmark.com/apic#Sensor")
@@ -1116,7 +1128,7 @@ class Neo4jRepositoryTests : WithNeo4jContainer {
 
         val entityTypes = neo4jRepository.getEntityTypes()
 
-        assertEquals(2, entityTypes.size)
+        assertEquals("Got the following types instead: $entityTypes", 2, entityTypes.size)
         assertTrue(
             entityTypes.containsAll(
                 listOf(

search-service/config/detekt/baseline.xml

@@ -5,8 +5,10 @@
     <ID>LargeClass:EntityEventListenerServiceTest.kt$EntityEventListenerServiceTest</ID>
     <ID>LargeClass:TemporalEntityHandlerTests.kt$TemporalEntityHandlerTests</ID>
     <ID>LongMethod:ParameterizedTests.kt$ParameterizedTests.Companion$@JvmStatic fun rawResultsProvider(): Stream&lt;Arguments&gt;</ID>
+    <ID>LongMethod:QueryServiceTests.kt$QueryServiceTests$@Test fun `it should query temporal entities as requested by query params`()</ID>
     <ID>LongParameterList:AttributeInstance.kt$AttributeInstance.Companion$( temporalEntityAttribute: UUID, instanceId: URI? = null, observedAt: ZonedDateTime, value: String? = null, measuredValue: Double? = null, payload: Map&lt;String, Any&gt; )</ID>
     <ID>LongParameterList:EntityEventListenerService.kt$EntityEventListenerService$( entityId: URI, expandedAttributeName: String, datasetId: URI?, attributeValuesNode: JsonNode, updatedEntity: String, contexts: List&lt;String&gt; )</ID>
+    <ID>LongParameterList:TemporalEntityAttributeService.kt$TemporalEntityAttributeService$( limit: Int, offset: Int, ids: Set&lt;URI&gt;, types: Set&lt;String&gt;, attrs: Set&lt;String&gt;, withEntityPayload: Boolean = false )</ID>
     <ID>ReturnCount:EntityEventListenerService.kt$EntityEventListenerService$internal fun toTemporalAttributeMetadata(jsonNode: JsonNode): Validated&lt;String, AttributeMetadata&gt;</ID>
     <ID>ReturnCount:TemporalEntityAttributeService.kt$TemporalEntityAttributeService$internal fun toTemporalAttributeMetadata( ngsiLdAttributeInstance: NgsiLdAttributeInstance ): Validated&lt;String, AttributeMetadata&gt;</ID>
     <ID>SwallowedException:TemporalEntityHandler.kt$catch (e: IllegalArgumentException) { "'timerel' is not valid, it should be one of 'before', 'between', or 'after'".left() }</ID>

search-service/src/main/kotlin/com/egm/stellio/search/config/ApplicationProperties.kt

@@ -8,7 +8,8 @@ import java.net.URI
 @ConfigurationProperties("application")
 data class ApplicationProperties(
     val entity: Entity,
-    val authentication: Authentication
+    val authentication: Authentication,
+    val pagination: Pagination
 ) {
     data class Authentication(
         val enabled: Boolean
@@ -18,4 +19,8 @@ data class ApplicationProperties(
         val serviceUrl: URI,
         val storePayloads: Boolean
     )
+    data class Pagination(
+        val limitDefault: Int,
+        val limitMax: Int
+    )
 }

search-service/src/main/kotlin/com/egm/stellio/search/model/TemporalEntitiesQuery.kt

@@ -0,0 +1,12 @@
+package com.egm.stellio.search.model
+
+import java.net.URI
+
+data class TemporalEntitiesQuery(
+    val ids: Set<URI>,
+    val types: Set<String>,
+    val temporalQuery: TemporalQuery,
+    val withTemporalValues: Boolean,
+    val limit: Int,
+    val offset: Int
+)