feat(entity): support entityOperations/upsert replace option #21 (#55)

* feat: support entityOperations/upsert replace option #21

entity-service/src/main/kotlin/com/egm/stellio/entity/repository/Neo4jRepository.kt

@@ -3,7 +3,11 @@ package com.egm.stellio.entity.repository
 import com.egm.stellio.entity.model.Entity
 import com.egm.stellio.entity.model.Property
 import com.egm.stellio.entity.model.Relationship
-import com.egm.stellio.shared.util.*
+import com.egm.stellio.shared.util.extractShortTypeFromExpanded
+import com.egm.stellio.shared.util.isDate
+import com.egm.stellio.shared.util.isDateTime
+import com.egm.stellio.shared.util.isFloat
+import com.egm.stellio.shared.util.isTime
 import org.neo4j.ogm.session.Session
 import org.neo4j.ogm.session.SessionFactory
 import org.neo4j.ogm.session.event.Event
@@ -155,6 +159,34 @@ class Neo4jRepository(
         return Pair(queryStatistics.nodesDeleted, queryStatistics.relationshipsDeleted)
     }
 
+    @Transactional
+    fun deleteEntityAttributes(entityId: String): Pair<Int, Int> {
+        /**
+         * Delete :
+         *
+         * 1. the properties
+         * 2. the properties of properties
+         * 3. the relationships of properties
+         * 4. the relationships
+         * 5. the properties of relationships
+         * 6. the relationships of relationships
+         */
+        val query = """
+            MATCH (n:Entity { id: '$entityId' }) 
+            OPTIONAL MATCH (n)-[:HAS_VALUE]->(prop)
+            OPTIONAL MATCH (prop)-[:HAS_OBJECT]->(relOfProp)
+            OPTIONAL MATCH (prop)-[:HAS_VALUE]->(propOfProp)
+            OPTIONAL MATCH (n)-[:HAS_OBJECT]->(rel)
+            OPTIONAL MATCH (rel)-[:HAS_VALUE]->(propOfRel)
+            OPTIONAL MATCH (rel)-[:HAS_OBJECT]->(relOfRel:Relationship)
+            DETACH DELETE prop,relOfProp,propOfProp,rel,propOfRel,relOfRel
+        """.trimIndent()
+
+        val queryStatistics = session.query(query, emptyMap<String, Any>()).queryStatistics()
+        logger.debug("Deleted entity $entityId attributes : deleted ${queryStatistics.nodesDeleted} nodes, ${queryStatistics.relationshipsDeleted} relations")
+        return Pair(queryStatistics.nodesDeleted, queryStatistics.relationshipsDeleted)
+    }
+
     @Transactional
     fun deleteEntityProperty(entityId: String, propertyName: String): Int {
         /**
@@ -177,9 +209,9 @@ class Neo4jRepository(
     }
 
     /**
-     Given an entity E1 having a relationship R1 with an entity E2
-     When matching the relationships of R1 (to be deleted with R1), a check on :Relationship is necessary since R1 has a link also called HAS_OBJECT with the target entity E2
-     Otherwise, it will delete not only the relationships of R1 but also the entity E2
+    Given an entity E1 having a relationship R1 with an entity E2
+    When matching the relationships of R1 (to be deleted with R1), a check on :Relationship is necessary since R1 has a link also called HAS_OBJECT with the target entity E2
+    Otherwise, it will delete not only the relationships of R1 but also the entity E2
      */
     @Transactional
     fun deleteEntityRelationship(entityId: String, relationshipType: String): Int {
@@ -202,7 +234,10 @@ class Neo4jRepository(
         return queryStatistics.nodesDeleted
     }
 
-    fun getEntitiesByTypeAndQuery(type: String, query: Pair<List<Triple<String, String, String>>, List<Triple<String, String, String>>>): List<String> {
+    fun getEntitiesByTypeAndQuery(
+        type: String,
+        query: Pair<List<Triple<String, String, String>>, List<Triple<String, String, String>>>
+    ): List<String> {
         val propertiesFilter =
             if (query.second.isNotEmpty())
                 query.second.joinToString(" AND ") {

entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityOperationService.kt

@@ -14,6 +14,7 @@ import org.jgrapht.Graphs
 import org.jgrapht.graph.DefaultEdge
 import org.jgrapht.graph.DirectedPseudograph
 import org.springframework.stereotype.Component
+import org.springframework.transaction.annotation.Transactional
 import kotlin.streams.toList
 
 /**
@@ -58,17 +59,36 @@ class EntityOperationService(
     }
 
     /**
-     * Update a batch of [entities].
+     * Replaces a batch of [entities]
+     * Only entities with relations linked to existing entities will be replaced.
+     *
+     * @return a [BatchOperationResult] with list of replaced ids and list of errors (either not replaced or
+     * linked to invalid entity).
+     */
+    fun replace(entities: List<ExpandedEntity>, createBatchResult: BatchOperationResult): BatchOperationResult {
+        return processEntities(entities, createBatchResult, ::replaceEntity)
+    }
+
+    /**
+     * Updates a batch of [entities].
      * Only entities with relations linked to existing entities will be updated.
      *
      * @return a [BatchOperationResult] with list of updated ids and list of errors (either not totally updated or
      * linked to invalid entity).
      */
     fun update(entities: List<ExpandedEntity>, createBatchResult: BatchOperationResult): BatchOperationResult {
+        return processEntities(entities, createBatchResult, ::updateEntity)
+    }
+
+    private fun processEntities(
+        entities: List<ExpandedEntity>,
+        createBatchResult: BatchOperationResult,
+        processor: (ExpandedEntity) -> Either<BatchEntityError, String>
+    ): BatchOperationResult {
         val existingEntitiesIds = createBatchResult.success.plus(entities.map { it.id })
         val nonExistingEntitiesIds = createBatchResult.errors.map { it.entityId }
-        return entities.parallelStream().map { entity ->
-            updateEntity(entity, existingEntitiesIds, nonExistingEntitiesIds)
+        return entities.parallelStream().map {
+            processEntity(it, processor, existingEntitiesIds, nonExistingEntitiesIds)
         }.collect(
             { BatchOperationResult() },
             { batchOperationResult, updateResult ->
@@ -82,8 +102,9 @@ class EntityOperationService(
         )
     }
 
-    private fun updateEntity(
+    private fun processEntity(
         entity: ExpandedEntity,
+        processor: (ExpandedEntity) -> Either<BatchEntityError, String>,
         existingEntitiesIds: List<String>,
         nonExistingEntitiesIds: List<String>
     ): Either<BatchEntityError, String> {
@@ -103,27 +124,46 @@ class EntityOperationService(
         }
 
         return try {
-            val (_, notUpdated) = entityService.appendEntityAttributes(
-                entity.id,
-                entity.attributes,
-                false
-            )
-
-            if (notUpdated.isEmpty()) {
-                Either.right(entity.id)
-            } else {
-                Either.left(
-                    BatchEntityError(
-                        entity.id,
-                        ArrayList(notUpdated.map { it.attributeName + " : " + it.reason })
-                    )
-                )
-            }
+            processor(entity)
         } catch (e: BadRequestDataException) {
             Either.left(BatchEntityError(entity.id, arrayListOf(e.message)))
         }
     }
 
+    /*
+     * Transactional because it should not delete entity attributes if new ones could not be appended.
+     */
+    @Transactional(rollbackFor = [BadRequestDataException::class])
+    @Throws(BadRequestDataException::class)
+    private fun replaceEntity(entity: ExpandedEntity): Either<BatchEntityError, String> {
+        neo4jRepository.deleteEntityAttributes(entity.id)
+        val (_, notUpdated) = entityService.appendEntityAttributes(entity.id, entity.attributes, false)
+        if (notUpdated.isEmpty()) {
+            return Either.right(entity.id)
+        } else {
+            throw BadRequestDataException(ArrayList(notUpdated.map { it.attributeName + " : " + it.reason }).joinToString())
+        }
+    }
+
+    private fun updateEntity(entity: ExpandedEntity): Either<BatchEntityError, String> {
+        val (_, notUpdated) = entityService.appendEntityAttributes(
+            entity.id,
+            entity.attributes,
+            false
+        )
+
+        return if (notUpdated.isEmpty()) {
+            Either.right(entity.id)
+        } else {
+            Either.left(
+                BatchEntityError(
+                    entity.id,
+                    ArrayList(notUpdated.map { it.attributeName + " : " + it.reason })
+                )
+            )
+        }
+    }
+
     private fun findInvalidEntityId(
         entitiesIds: List<String>,
         existingEntitiesIds: List<String>,

entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityOperationHandler.kt

@@ -8,10 +8,7 @@ import com.fasterxml.jackson.module.kotlin.jacksonObjectMapper
 import org.springframework.http.HttpStatus
 import org.springframework.http.MediaType
 import org.springframework.http.ResponseEntity
-import org.springframework.web.bind.annotation.PostMapping
-import org.springframework.web.bind.annotation.RequestBody
-import org.springframework.web.bind.annotation.RequestMapping
-import org.springframework.web.bind.annotation.RestController
+import org.springframework.web.bind.annotation.*
 import reactor.core.publisher.Mono
 
 @RestController
@@ -50,7 +47,10 @@ class EntityOperationHandler(
      * Implements 6.15.3.1 - Upsert Batch of Entities
      */
     @PostMapping("/upsert", consumes = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE])
-    fun upsert(@RequestBody body: Mono<String>): Mono<ResponseEntity<*>> {
+    fun upsert(
+        @RequestBody body: Mono<String>,
+        @RequestParam(required = false) options: String?
+    ): Mono<ResponseEntity<*>> {
         return body
             .map {
                 extractAndParseBatchOfEntities(it)
@@ -59,8 +59,11 @@ class EntityOperationHandler(
                 val (existingEntities, newEntities) = entityOperationService.splitEntitiesByExistence(it)
 
                 val createBatchOperationResult = entityOperationService.create(newEntities)
-                val updateBatchOperationResult =
-                    entityOperationService.update(existingEntities, createBatchOperationResult)
+
+                val updateBatchOperationResult = when (options) {
+                    "update" -> entityOperationService.update(existingEntities, createBatchOperationResult)
+                    else -> entityOperationService.replace(existingEntities, createBatchOperationResult)
+                }
 
                 BatchOperationResult(
                     ArrayList(createBatchOperationResult.success.plus(updateBatchOperationResult.success)),

entity-service/src/test/kotlin/com/egm/stellio/entity/repository/Neo4jRepositoryTests.kt

@@ -361,6 +361,26 @@ class Neo4jRepositoryTests {
         neo4jRepository.deleteEntity(entity.id)
     }
 
+    @Test
+    fun `it should delete entity attributes`() {
+        val sensor = createEntity(
+            "urn:ngsi-ld:Sensor:1233",
+            listOf("Sensor"),
+            mutableListOf(Property(name = "name", value = "Scalpa"))
+        )
+        val device = createEntity("urn:ngsi-ld:Device:1233", listOf("Device"), mutableListOf())
+        createRelationship(sensor, EGM_OBSERVED_BY, device.id)
+
+        neo4jRepository.deleteEntityAttributes(sensor.id)
+
+        val entity = entityRepository.findById(sensor.id).get()
+        assertEquals(entity.relationships.size, 0)
+        assertEquals(entity.properties.size, 0)
+
+        neo4jRepository.deleteEntity(sensor.id)
+        neo4jRepository.deleteEntity(device.id)
+    }
+
     fun createEntity(id: String, type: List<String>, properties: MutableList<Property>): Entity {
         val entity = Entity(id = id, type = type, properties = properties)
         return entityRepository.save(entity)

entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityOperationServiceTests.kt

@@ -11,7 +11,11 @@ import com.egm.stellio.entity.web.BatchOperationResult
 import com.egm.stellio.shared.model.BadRequestDataException
 import com.egm.stellio.shared.model.ExpandedEntity
 import com.ninjasquad.springmockk.MockkBean
-import io.mockk.*
+import io.mockk.Runs
+import io.mockk.every
+import io.mockk.just
+import io.mockk.mockk
+import io.mockk.mockkClass
 import org.jgrapht.graph.DefaultEdge
 import org.jgrapht.graph.DirectedPseudograph
 import org.junit.jupiter.api.Assertions.assertEquals
@@ -237,4 +241,96 @@ class EntityOperationServiceTests {
             batchOperationResult.errors
         )
     }
-}
+
+    @Test
+    fun `it should replace entities`() {
+        val firstEntity = mockkClass(ExpandedEntity::class, relaxed = true)
+        every { firstEntity.id } returns "1"
+        every { firstEntity.getLinkedEntitiesIds() } returns listOf()
+        val secondEntity = mockkClass(ExpandedEntity::class, relaxed = true)
+        every { secondEntity.id } returns "2"
+        every { secondEntity.getLinkedEntitiesIds() } returns listOf()
+
+        every { neo4jRepository.filterExistingEntitiesIds(listOf()) } returns listOf()
+        every { neo4jRepository.deleteEntityAttributes("1") } returns mockk()
+        every { entityService.appendEntityAttributes(eq("1"), any(), any()) } returns UpdateResult(
+            emptyList(),
+            emptyList()
+        )
+        every { neo4jRepository.deleteEntityAttributes("2") } returns mockk()
+        every { entityService.appendEntityAttributes(eq("2"), any(), any()) } returns UpdateResult(
+            emptyList(),
+            emptyList()
+        )
+
+        val batchOperationResult =
+            entityOperationService.replace(listOf(firstEntity, secondEntity), BatchOperationResult())
+
+        assertEquals(listOf("1", "2"), batchOperationResult.success)
+        assertTrue(batchOperationResult.errors.isEmpty())
+    }
+
+    @Test
+    fun `it should count as error entities that couldn't be replaced`() {
+        val firstEntity = mockkClass(ExpandedEntity::class, relaxed = true)
+        every { firstEntity.id } returns "1"
+        every { firstEntity.getLinkedEntitiesIds() } returns listOf()
+        val secondEntity = mockkClass(ExpandedEntity::class, relaxed = true)
+        every { secondEntity.id } returns "2"
+        every { secondEntity.getLinkedEntitiesIds() } returns listOf()
+
+        every { neo4jRepository.filterExistingEntitiesIds(listOf()) } returns listOf()
+        every { neo4jRepository.deleteEntityAttributes("1") } returns mockk()
+        every { entityService.appendEntityAttributes(eq("1"), any(), any()) } returns UpdateResult(
+            emptyList(),
+            emptyList()
+        )
+        every { neo4jRepository.deleteEntityAttributes("2") } returns mockk()
+        every { entityService.appendEntityAttributes(eq("2"), any(), any()) } throws BadRequestDataException("error")
+
+        val batchOperationResult =
+            entityOperationService.replace(listOf(firstEntity, secondEntity), BatchOperationResult())
+
+        assertEquals(listOf("1"), batchOperationResult.success)
+        assertEquals(
+            listOf(BatchEntityError("2", arrayListOf("error"))),
+            batchOperationResult.errors
+        )
+    }
+
+    @Test
+    fun `it should count as error entities that couldn't be replaced totally`() {
+        val firstEntity = mockkClass(ExpandedEntity::class, relaxed = true)
+        every { firstEntity.id } returns "1"
+        every { firstEntity.getLinkedEntitiesIds() } returns listOf()
+        val secondEntity = mockkClass(ExpandedEntity::class, relaxed = true)
+        every { secondEntity.id } returns "2"
+        every { secondEntity.getLinkedEntitiesIds() } returns listOf()
+
+        every { neo4jRepository.filterExistingEntitiesIds(listOf()) } returns listOf()
+        every { neo4jRepository.deleteEntityAttributes("1") } returns mockk()
+        every { entityService.appendEntityAttributes(eq("1"), any(), any()) } returns UpdateResult(
+            emptyList(),
+            emptyList()
+        )
+        every { neo4jRepository.deleteEntityAttributes("2") } returns mockk()
+        every { entityService.appendEntityAttributes(eq("2"), any(), any()) } returns UpdateResult(
+            emptyList(),
+            listOf(
+                NotUpdatedDetails("attribute#1", "reason"),
+                NotUpdatedDetails("attribute#2", "reason")
+            )
+        )
+
+        val batchOperationResult = entityOperationService.replace(
+            listOf(firstEntity, secondEntity),
+            BatchOperationResult()
+        )
+
+        assertEquals(listOf("1"), batchOperationResult.success)
+        assertEquals(
+            listOf(BatchEntityError("2", arrayListOf("attribute#1 : reason, attribute#2 : reason"))),
+            batchOperationResult.errors
+        )
+    }
+}