feat: merge patch for LangProperty (and other attributes types)

search-service/config/detekt/baseline.xml

@@ -10,6 +10,8 @@
     <ID>LongMethod:AttributeInstanceService.kt$AttributeInstanceService$@Transactional suspend fun create(attributeInstance: AttributeInstance): Either&lt;APIException, Unit&gt;</ID>
     <ID>LongMethod:EntityAccessControlHandler.kt$EntityAccessControlHandler$@PostMapping("/{subjectId}/attrs", consumes = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun addRightsOnEntities( @RequestHeader httpHeaders: HttpHeaders, @PathVariable subjectId: String, @RequestBody requestBody: Mono&lt;String&gt; ): ResponseEntity&lt;*&gt;</ID>
     <ID>LongMethod:EntityOperationHandlerTests.kt$EntityOperationHandlerTests$@Test fun `create batch entity should return a 207 when some entities already exist`()</ID>
+    <ID>LongMethod:PatchAttributeTests.kt$PatchAttributeTests.Companion$@JvmStatic fun mergePatchProvider(): Stream&lt;Arguments&gt;</ID>
+    <ID>LongMethod:PatchAttributeTests.kt$PatchAttributeTests.Companion$@JvmStatic fun partialUpdatePatchProvider(): Stream&lt;Arguments&gt;</ID>
     <ID>LongMethod:QueryServiceTests.kt$QueryServiceTests$@Test fun `it should query temporal entities as requested by query params`()</ID>
     <ID>LongMethod:QueryServiceTests.kt$QueryServiceTests$@Test fun `it should return an empty list for an attribute if it has no temporal values`()</ID>
     <ID>LongMethod:TemporalEntityBuilderTests.kt$TemporalEntityBuilderTests$@Test fun `it should return a temporal entity with values aggregated`()</ID>

search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt

@@ -21,9 +21,7 @@ import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_RELATIONSHIP_OBJECT
 import com.egm.stellio.shared.util.JsonLdUtils.buildNonReifiedTemporalValue
 import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdEntity
 import com.egm.stellio.shared.util.JsonUtils.serializeObject
-import com.savvasdalkitsis.jsonmerger.JsonMerger
 import io.r2dbc.postgresql.codec.Json
-import org.json.JSONObject
 import org.slf4j.LoggerFactory
 import org.springframework.r2dbc.core.DatabaseClient
 import org.springframework.r2dbc.core.bind
@@ -271,9 +269,10 @@ class TemporalEntityAttributeService(
             attributeMetadata,
             observedAt
         )
-        val (jsonTargetObject, updatedAttributeInstance) = mergeAttributePayload(tea, processedAttributePayload)
+        val (jsonTargetObject, updatedAttributeInstance) =
+            mergePatch(tea.payload.toExpandedAttributeInstance(), processedAttributePayload)
         val value = getValueFromPartialAttributePayload(tea, updatedAttributeInstance)
-        updateOnUpdate(tea.id, processedAttributeMetadata.valueType, mergedAt, jsonTargetObject.toString()).bind()
+        updateOnUpdate(tea.id, processedAttributeMetadata.valueType, mergedAt, jsonTargetObject).bind()
 
         val attributeInstance =
             createContextualAttributeInstance(tea, updatedAttributeInstance, value, mergedAt, sub)
@@ -658,10 +657,11 @@ class TemporalEntityAttributeService(
                         BadRequestDataException("The type of the attribute has to be the same as the existing one")
                     }
                 }
-                val (jsonTargetObject, updatedAttributeInstance) = mergeAttributePayload(tea, attributeValues)
+                val (jsonTargetObject, updatedAttributeInstance) =
+                    partialUpdatePatch(tea.payload.toExpandedAttributeInstance(), attributeValues)
                 val value = getValueFromPartialAttributePayload(tea, updatedAttributeInstance)
                 val attributeValueType = guessAttributeValueType(tea.attributeType, attributeValues)
-                updateOnUpdate(tea.id, attributeValueType, modifiedAt, jsonTargetObject.toString()).bind()
+                updateOnUpdate(tea.id, attributeValueType, modifiedAt, jsonTargetObject).bind()
 
                 // then update attribute instance
                 val attributeInstance = createContextualAttributeInstance(
@@ -872,31 +872,6 @@ class TemporalEntityAttributeService(
                 )
         }
 
-    suspend fun mergeAttributePayload(
-        tea: TemporalEntityAttribute,
-        expandedAttributeInstance: ExpandedAttributeInstance
-    ): Pair<JSONObject, ExpandedAttributeInstance> {
-        val jsonSourceObject = JSONObject(tea.payload.asString())
-        val jsonUpdateObject = JSONObject(expandedAttributeInstance)
-        // if the attribute is a JsonProperty, preserve its JSON value to avoid it being merged
-        // (the whole JSON value shall be replaced)
-        val preservedJsonValue = if (tea.attributeType == TemporalEntityAttribute.AttributeType.JsonProperty)
-            expandedAttributeInstance[NGSILD_JSONPROPERTY_VALUE]
-        else null
-        val jsonMerger = JsonMerger(
-            arrayMergeMode = JsonMerger.ArrayMergeMode.REPLACE_ARRAY,
-            objectMergeMode = JsonMerger.ObjectMergeMode.MERGE_OBJECT
-        )
-        val jsonTargetObject = jsonMerger.merge(jsonSourceObject, jsonUpdateObject)
-            .let {
-                if (preservedJsonValue != null)
-                    it.put(NGSILD_JSONPROPERTY_VALUE, preservedJsonValue)
-                else it
-            }
-        val updatedAttributeInstance = jsonTargetObject.toMap() as ExpandedAttributeInstance
-        return Pair(jsonTargetObject, updatedAttributeInstance)
-    }
-
     private fun createContextualAttributeInstance(
         tea: TemporalEntityAttribute,
         expandedAttributeInstance: ExpandedAttributeInstance,
@@ -933,16 +908,15 @@ class TemporalEntityAttributeService(
         attributePayload: ExpandedAttributeInstance,
         attributeMetadata: AttributeMetadata,
         observedAt: ZonedDateTime?
-    ): Pair<ExpandedAttributeInstance, AttributeMetadata> {
-        return if (
+    ): Pair<ExpandedAttributeInstance, AttributeMetadata> =
+        if (
             observedAt != null &&
             tea.payload.deserializeAsMap().containsKey(NGSILD_OBSERVED_AT_PROPERTY) &&
             !attributePayload.containsKey(NGSILD_OBSERVED_AT_PROPERTY)
-        ) {
+        )
             Pair(
                 attributePayload.plus(NGSILD_OBSERVED_AT_PROPERTY to buildNonReifiedTemporalValue(observedAt)),
                 attributeMetadata.copy(observedAt = observedAt)
             )
-        } else Pair(attributePayload, attributeMetadata)
-    }
+        else Pair(attributePayload, attributeMetadata)
 }

search-service/src/main/kotlin/com/egm/stellio/search/util/AttributeInstanceUtils.kt

@@ -8,8 +8,15 @@ import com.egm.stellio.search.model.AttributeMetadata
 import com.egm.stellio.search.model.TemporalEntityAttribute
 import com.egm.stellio.search.model.TemporalEntityAttribute.AttributeValueType
 import com.egm.stellio.shared.model.*
+import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_LANGUAGE
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_JSONPROPERTY_VALUE
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_LANGUAGEPROPERTY_VALUE
+import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_PROPERTY_VALUE
 import com.egm.stellio.shared.util.JsonLdUtils.logger
+import com.egm.stellio.shared.util.JsonUtils.deserializeAsMap
 import com.egm.stellio.shared.util.JsonUtils.serializeObject
+import com.savvasdalkitsis.jsonmerger.JsonMerger
+import io.r2dbc.postgresql.codec.Json
 import java.time.LocalDate
 import java.time.LocalTime
 import java.time.ZonedDateTime
@@ -121,3 +128,55 @@ fun guessPropertyValueType(
         is LocalTime -> Pair(AttributeValueType.TIME, Triple(value.toString(), null, null))
         else -> Pair(AttributeValueType.STRING, Triple(value.toString(), null, null))
     }
+
+fun Json.toExpandedAttributeInstance(): ExpandedAttributeInstance =
+    this.deserializeAsMap() as ExpandedAttributeInstance
+
+fun partialUpdatePatch(
+    source: ExpandedAttributeInstance,
+    update: ExpandedAttributeInstance
+): Pair<String, ExpandedAttributeInstance> {
+    val target = source.plus(update)
+    return Pair(serializeObject(target), target)
+}
+
+fun mergePatch(
+    source: ExpandedAttributeInstance,
+    update: ExpandedAttributeInstance
+): Pair<String, ExpandedAttributeInstance> {
+    val target = source.toMutableMap()
+    update.forEach { (attrName, attrValue) ->
+        if (!source.containsKey(attrName)) {
+            target[attrName] = attrValue
+        } else if (listOf(NGSILD_JSONPROPERTY_VALUE, NGSILD_PROPERTY_VALUE).contains(attrName)) {
+            if (attrValue.size > 1) {
+                // a Property holding an array of value or a JsonPropery holding an array of JSON objects
+                // cannot be safely merged patch, so copy the whole value from the update
+                target[attrName] = attrValue
+            } else {
+                target[attrName] = listOf(
+                    JsonMerger().merge(
+                        serializeObject(source[attrName]!![0]),
+                        serializeObject(attrValue[0])
+                    ).deserializeAsMap()
+                )
+            }
+        } else if (listOf(NGSILD_LANGUAGEPROPERTY_VALUE).contains(attrName)) {
+            val sourceLangEntries = source[attrName] as List<Map<String, String>>
+            val targetLangEntries = sourceLangEntries.toMutableList()
+            (attrValue as List<Map<String, String>>).forEach { langEntry ->
+                // remove any previously existing entry for this language
+                targetLangEntries.removeIf {
+                    it[JSONLD_LANGUAGE] == langEntry[JSONLD_LANGUAGE]
+                }
+                targetLangEntries.add(langEntry)
+            }
+
+            target[attrName] = targetLangEntries
+        } else {
+            target[attrName] = attrValue
+        }
+    }
+
+    return Pair(serializeObject(target), target)
+}

search-service/src/test/kotlin/com/egm/stellio/search/scope/ScopeServiceTests.kt

@@ -5,8 +5,7 @@ import com.egm.stellio.search.model.AttributeInstance.TemporalProperty
 import com.egm.stellio.search.service.EntityPayloadService
 import com.egm.stellio.search.support.WithKafkaContainer
 import com.egm.stellio.search.support.WithTimescaleContainer
-import com.egm.stellio.search.util.deserializeAsMap
-import com.egm.stellio.shared.model.ExpandedAttributeInstance
+import com.egm.stellio.search.util.toExpandedAttributeInstance
 import com.egm.stellio.shared.model.PaginationQuery
 import com.egm.stellio.shared.model.getScopes
 import com.egm.stellio.shared.util.*
@@ -111,7 +110,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer {
         entityPayloadService.retrieve(beehiveTestCId)
             .shouldSucceedWith {
                 assertEquals(expectedScopes, it.scopes)
-                val scopesInEntity = (it.payload.deserializeAsMap() as ExpandedAttributeInstance).getScopes()
+                val scopesInEntity = it.payload.toExpandedAttributeInstance().getScopes()
                 assertEquals(expectedScopes, scopesInEntity)
             }
     }
@@ -296,7 +295,7 @@ class ScopeServiceTests : WithTimescaleContainer, WithKafkaContainer {
         scopeService.retrieve(beehiveTestCId)
             .shouldSucceedWith {
                 assertNull(it.first)
-                assertNull((it.second.deserializeAsMap() as ExpandedAttributeInstance).getScopes())
+                assertNull(it.second.toExpandedAttributeInstance().getScopes())
             }
         val scopeHistoryEntries = scopeService.retrieveHistory(
             listOf(beehiveTestCId),

search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt

@@ -5,7 +5,6 @@ import com.egm.stellio.search.model.*
 import com.egm.stellio.search.support.EMPTY_JSON_PAYLOAD
 import com.egm.stellio.search.support.WithKafkaContainer
 import com.egm.stellio.search.support.WithTimescaleContainer
-import com.egm.stellio.search.util.toJson
 import com.egm.stellio.shared.model.ResourceNotFoundException
 import com.egm.stellio.shared.model.toNgsiLdAttribute
 import com.egm.stellio.shared.model.toNgsiLdAttributes
@@ -281,65 +280,6 @@ class TemporalEntityAttributeServiceTests : WithTimescaleContainer, WithKafkaCon
         }
     }
 
-    @Test
-    fun `it should merge an attribute payload for a JsonProperty`() = runTest {
-        val initialJsonValue = """
-            {
-                "incoming": {
-                    "type": "JsonProperty",
-                    "json": { "id": 1, "b": null, "c": 12.4 },
-                    "observedAt": "2022-12-24T14:01:22.066Z",
-                    "subAttribute": {
-                        "type": "Property",
-                        "value": "subAttribute"
-                    }
-                }
-            }
-        """.trimIndent()
-        val temporalEntityAttribute = TemporalEntityAttribute(
-            entityId = beehiveTestCId,
-            attributeName = INCOMING_PROPERTY,
-            attributeType = TemporalEntityAttribute.AttributeType.JsonProperty,
-            attributeValueType = TemporalEntityAttribute.AttributeValueType.JSON,
-            createdAt = ngsiLdDateTime(),
-            payload = expandAttribute(initialJsonValue, APIC_COMPOUND_CONTEXTS).second[0].toJson()
-        )
-
-        val newJsonValue = """
-            {
-                "incoming": {
-                    "type": "JsonProperty",
-                    "json": { "id": 2, "b": "something" },
-                    "observedAt": "2023-12-24T14:01:22.066Z"
-                }
-            }
-        """.trimIndent()
-        val newJsonExpandedAttribute = expandAttribute(newJsonValue, APIC_COMPOUND_CONTEXTS).second[0]
-        val (_, expandedAttributeInstance) = temporalEntityAttributeService.mergeAttributePayload(
-            temporalEntityAttribute,
-            newJsonExpandedAttribute
-        )
-
-        val expectedJsonValue = """
-            {
-                "incoming": {
-                    "type": "JsonProperty",
-                    "json": { "id": 2, "b": "something" },
-                    "observedAt": "2023-12-24T14:01:22.066Z",
-                    "subAttribute": {
-                        "type": "Property",
-                        "value": "subAttribute"
-                    }
-                }
-            }
-        """.trimIndent()
-        val expectedJsonExpandedAttribute = expandAttribute(expectedJsonValue, APIC_COMPOUND_CONTEXTS).second[0]
-        assertJsonPayloadsAreEqual(
-            serializeObject(expectedJsonExpandedAttribute),
-            serializeObject(expandedAttributeInstance)
-        )
-    }
-
     @Test
     fun `it should merge an entity attribute`() = runTest {
         val rawEntity = loadSampleData()