Merge branch 'develop'

.github/fiware/config.json

@@ -0,0 +1,29 @@
+{
+    "enabler": "Stellio context broker",
+    "chapter": "core",
+    "academy": "",
+    "readthedocs": "stellio",
+    "helpdesk": "",
+    "coveralls": "",
+    "github": ["stellio-hub/stellio-context-broker"],
+    "dockerregistry": ["hub.docker.com"],
+    "docker": [
+      "stellio/stellio-search-service", 
+      "stellio/stellio-subscription-service", 
+      "stellio/stellio-entity-service", 
+      "stellio/stellio-api-gateway"
+    ],
+    "email": "[email protected]",
+    "status": "incubating",
+    "compose": "https://raw.githubusercontent.com/stellio-hub/stellio-context-broker/develop/docker-compose.yml",
+    "exclude": ["zookeeper", "kafka", "neo4j", "postgres"],
+    "stackexchange": ["https://stackoverflow.com/questions/tagged/fiware-stellio"],
+    "unit-test": "",
+    "smoke-test": "",
+    "dockerfile": [
+      "https://github.com/stellio-hub/stellio-context-broker/blob/develop/api-gateway",
+      "https://github.com/stellio-hub/stellio-context-broker/blob/develop/entity-service",
+      "https://github.com/stellio-hub/stellio-context-broker/blob/develop/search-service", 
+      "https://github.com/stellio-hub/stellio-context-broker/blob/develop/subscription-service"
+   ]
+}

entity-service/config/detekt/baseline.xml

@@ -8,9 +8,12 @@
     <ID>LargeClass:EntityServiceTests.kt$EntityServiceTests</ID>
     <ID>LargeClass:Neo4jRepositoryTests.kt$Neo4jRepositoryTests : WithNeo4jContainer</ID>
     <ID>LargeClass:StandaloneNeo4jSearchRepositoryTests.kt$StandaloneNeo4jSearchRepositoryTests : WithNeo4jContainer</ID>
+    <ID>LongMethod:EntityEventServiceTests.kt$EntityEventServiceTests$@Test fun `it should publish ATTRIBUTE_APPEND and ATTRIBUTE_REPLACE events if attributes were appended and replaced`()</ID>
     <ID>LongMethod:EntityHandler.kt$EntityHandler$ @GetMapping(produces = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun getEntities( @RequestHeader httpHeaders: HttpHeaders, @RequestParam params: MultiValueMap&lt;String, String&gt; ): ResponseEntity&lt;*&gt;</ID>
     <ID>LongMethod:EntityServiceTests.kt$EntityServiceTests$@Test fun `it should create a new multi attribute property`()</ID>
+    <ID>LongMethod:Neo4jRepositoryTests.kt$Neo4jRepositoryTests$@Test fun `it should retrieve details of attributes`()</ID>
     <ID>LongMethod:QueryUtils.kt$QueryUtils$fun prepareQueryForEntitiesWithAuthentication( queryParams: QueryParams, offset: Int, limit: Int, contexts: List&lt;String&gt; ): String</ID>
+    <ID>LongParameterList:EntityEventService.kt$EntityEventService$( entityId: URI, entityType: String, attributeName: String, datasetId: URI? = null, overwrite: Boolean, operationPayload: String, updateOperationResult: UpdateOperationResult, contexts: List&lt;String&gt; )</ID>
     <ID>LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, offset: Int, limit: Int, contextLink: String, includeSysAttrs: Boolean )</ID>
     <ID>LongParameterList:EntityService.kt$EntityService$( queryParams: QueryParams, userSub: String, offset: Int, limit: Int, contexts: List&lt;String&gt;, includeSysAttrs: Boolean )</ID>
     <ID>MaxLineLength:EntityHandlerTests.kt$EntityHandlerTests$ </ID>

entity-service/src/main/kotlin/com/egm/stellio/entity/EntityServiceApplication.kt

@@ -4,7 +4,10 @@ import com.egm.stellio.entity.config.ApplicationProperties
 import org.springframework.boot.autoconfigure.SpringBootApplication
 import org.springframework.boot.context.properties.EnableConfigurationProperties
 import org.springframework.boot.runApplication
+import org.springframework.context.annotation.Bean
 import org.springframework.scheduling.annotation.EnableAsync
+import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor
+import java.util.concurrent.Executor
 
 @SpringBootApplication(scanBasePackages = ["com.egm.stellio.entity", "com.egm.stellio.shared"])
 @EnableConfigurationProperties(ApplicationProperties::class)
@@ -15,3 +18,16 @@ class EntityServiceApplication
 fun main(args: Array<String>) {
     runApplication<EntityServiceApplication>(*args)
 }
+
+@Bean
+fun taskExecutor(
+    applicationProperties: ApplicationProperties
+): Executor {
+    val executor = ThreadPoolTaskExecutor().apply {
+        corePoolSize = applicationProperties.eventsThreadPool.corePoolSize
+        maxPoolSize = applicationProperties.eventsThreadPool.maxPoolSize
+        setThreadNamePrefix("entity-executor-")
+    }
+    executor.initialize()
+    return executor
+}

entity-service/src/main/kotlin/com/egm/stellio/entity/config/ApplicationProperties.kt

@@ -6,10 +6,16 @@ import org.springframework.boot.context.properties.ConstructorBinding
 @ConstructorBinding
 @ConfigurationProperties("application")
 class ApplicationProperties(
-    val pagination: Pagination
+    val pagination: Pagination,
+    val eventsThreadPool: EventsThreadPool = EventsThreadPool()
 ) {
     data class Pagination(
         val limitDefault: Int,
         val limitMax: Int
     )
+
+    data class EventsThreadPool(
+        val corePoolSize: Int = 10,
+        val maxPoolSize: Int = 100
+    )
 }

entity-service/src/main/kotlin/com/egm/stellio/entity/model/AttributeDetails.kt

@@ -0,0 +1,10 @@
+package com.egm.stellio.entity.model
+
+import java.net.URI
+
+data class AttributeDetails(
+    val id: URI,
+    val type: String = "Attribute",
+    val attributeName: String,
+    val typeNames: List<String>
+)

entity-service/src/main/kotlin/com/egm/stellio/entity/model/AttributeList.kt

@@ -0,0 +1,11 @@
+package com.egm.stellio.entity.model
+
+import com.egm.stellio.shared.util.toUri
+import java.net.URI
+import java.util.*
+
+class AttributeList(
+    val id: URI = "urn:ngsi-ld:AttributeList:${UUID.randomUUID()}".toUri(),
+    val type: String = "AttributeList",
+    val attributeList: List<String>
+)

entity-service/src/main/kotlin/com/egm/stellio/entity/model/AttributeTypeInfo.kt

@@ -0,0 +1,12 @@
+package com.egm.stellio.entity.model
+
+import java.net.URI
+
+class AttributeTypeInfo(
+    val id: URI,
+    val type: String = "Attribute",
+    val attributeName: String,
+    val attributeTypes: List<String>,
+    val typeNames: List<String>,
+    val attributeCount: Int
+)

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt

@@ -510,6 +510,120 @@ class Neo4jRepository(
         }.flatten()
     }
 
+    fun getAttributes(): List<String> {
+        val query =
+            """
+                MATCH (a:Attribute)
+                OPTIONAL MATCH (a:Attribute)<-[:HAS_OBJECT|:HAS_VALUE]-(entity:Entity)
+                RETURN DISTINCT(labels(a))+collect(DISTINCT(a.name)) as attribute,
+                    count(entity.location) as entityWithLocationCount
+            """.trimIndent()
+
+        val result = neo4jClient.query(query).fetch().all()
+        return result.map { rowResult ->
+            val entityWithLocationCount = (rowResult["entityWithLocationCount"] as Long).toInt()
+            (rowResult["attribute"] as List<String>)
+                // filter our internal labels
+                .filter { it !in listOf("Attribute", "Relationship", "Property") }
+                .let {
+                    if (entityWithLocationCount > 0)
+                        it.plus(NGSILD_LOCATION_PROPERTY)
+                    else it
+                }
+        }.flatten().sortedBy { it }
+    }
+
+    fun getAttributesDetails(): List<Map<String, Any>> {
+        val query =
+            """
+                MATCH (a:Attribute)<-[:HAS_VALUE|:HAS_OBJECT]-(entity:Entity)
+                RETURN DISTINCT(labels(a)) as relation, 
+                    labels(entity) as typeNames,
+                    collect(distinct a.name) as property,
+                    count(entity.location) as entityWithLocationCount
+            """.trimIndent()
+
+        val result = neo4jClient.query(query).fetch().all()
+        return result.map { rowResult ->
+            val names = (rowResult["property"] as List<String>)
+            val labels = (rowResult["relation"] as List<String>)
+            val entityWithLocationCount = (rowResult["entityWithLocationCount"] as Long).toInt()
+            names.plus(labels)
+                // filter our internal labels
+                .filter { it !in listOf("Attribute", "Relationship", "Property") }
+                .let {
+                    if (entityWithLocationCount > 0)
+                        it.plus(NGSILD_LOCATION_PROPERTY)
+                    else it
+                }
+                .map { attributeName ->
+                    val typeNames = (rowResult["typeNames"] as List<String>)
+                        .filter { !authorizationEntitiesTypes.plus("Entity").contains(it) }.toSet()
+                    attributeName to typeNames
+                }
+        }.flatten()
+            // an attribute can appear in many entities types so group them by name
+            .groupBy { it.first }
+            .mapValues {
+                mapOf(
+                    "attribute" to it.key,
+                    // and merge all the entities types in one set
+                    "typeNames" to it.value.fold(emptySet()) { acc: Set<String>, pair -> acc.plus(pair.second) }
+                )
+            }.values
+            .sortedBy { it["attribute"] as String }
+    }
+
+    fun getAttributeInformation(expandedType: String): Map<String, Any> {
+        val relationshipQuery =
+            """
+                MATCH (a:Relationship:`$expandedType`)<-[:HAS_OBJECT]-(entity:Entity)
+                RETURN DISTINCT (labels(a)) as attributeTypes,
+                    labels(entity) as typeNames,
+                    count(a) as attributeCount
+            """.trimIndent()
+        val propertyQuery =
+            """
+                MATCH (a:Attribute { name:"$expandedType" })<-[:HAS_VALUE]-(entity:Entity)
+                RETURN DISTINCT (labels(a)) as attributeTypes,
+                    labels(entity) as typeNames,
+                    count(a) as attributeCount
+            """.trimIndent()
+
+        // here we are making the assumption that an attribute won't be a property and a relationship at the same time
+        // it should if correctly used but nothing can prevent it
+        val results = neo4jClient.query(relationshipQuery).fetch().all()
+            .ifEmpty { neo4jClient.query(propertyQuery).fetch().all() }
+        if (results.isEmpty())
+            return emptyMap()
+
+        // merge the results from all the matched paths
+        val aggregatedResults = results.fold(
+            Triple<Set<String>, Set<String>, Int>(emptySet(), emptySet(), 0)
+        ) { acc, current ->
+            Triple(
+                acc.first.plus(
+                    (current["attributeTypes"] as List<String>).filter {
+                        it !in listOf("Attribute", expandedType)
+                    }.toSet()
+                ),
+                acc.second.plus(
+                    (current["typeNames"] as List<String>).filter {
+                        !authorizationEntitiesTypes.plus("Entity").contains(it)
+                    }.toSet()
+                ),
+                acc.third.plus((current["attributeCount"] as Long).toInt())
+            )
+        }
+
+        return mapOf(
+            "attributeName" to expandedType,
+            "attributeTypes" to aggregatedResults.first,
+            "typeNames" to aggregatedResults.second,
+            "attributeCount" to aggregatedResults.third
+        )
+    }
+
     fun filterExistingEntitiesAsIds(entitiesIds: List<URI>): List<URI> {
         if (entitiesIds.isEmpty()) {
             return emptyList()

entity-service/src/main/kotlin/com/egm/stellio/entity/service/AttributeService.kt

@@ -0,0 +1,41 @@
+package com.egm.stellio.entity.service
+
+import com.egm.stellio.entity.model.*
+import com.egm.stellio.entity.repository.Neo4jRepository
+import com.egm.stellio.shared.util.JsonLdUtils.compactTerm
+import com.egm.stellio.shared.util.toUri
+import org.springframework.stereotype.Component
+
+@Component
+class AttributeService(
+    private val neo4jRepository: Neo4jRepository
+) {
+
+    fun getAttributeList(contexts: List<String>): AttributeList =
+        AttributeList(
+            attributeList = neo4jRepository.getAttributes().map { compactTerm(it, contexts) }
+        )
+
+    fun getAttributeDetails(contexts: List<String>): List<AttributeDetails> =
+        neo4jRepository.getAttributesDetails().map {
+            val attribute = (it["attribute"] as String)
+            AttributeDetails(
+                id = attribute.toUri(),
+                attributeName = compactTerm(attribute, contexts),
+                typeNames = (it["typeNames"] as Set<String>).toList().map { compactTerm(it, contexts) }
+            )
+        }
+
+    fun getAttributeTypeInfo(expandedType: String): AttributeTypeInfo? {
+        val attributesInformation = neo4jRepository.getAttributeInformation(expandedType)
+        if (attributesInformation.isEmpty()) return null
+
+        return AttributeTypeInfo(
+            id = expandedType.toUri(),
+            attributeName = attributesInformation["attributeName"] as String,
+            attributeTypes = attributesInformation["attributeTypes"] as List<String>,
+            typeNames = attributesInformation["typeNames"] as List<String>,
+            attributeCount = attributesInformation["attributeCount"] as Int
+        )
+    }
+}