feat(common): switch to Titanium JSON-LD library (#1073)

- use the same (poorly typed) contract as with current lib
- integrate library into JsonLdUtils
- improve typing of some expanded form structures
- redo all the JSON-LD processing of temporal entities (do it once before rendering, as it is done for entities)
- use local JSON-LD context server extension for JUnit5 based on https://www.mock-server.com/
- refactor: improve expanded values builders
  - add a Value suffix to make clear they are just doing this
  - clean the documentation
  - add a specific extension functions to add non-reified properties (temporal and non-temporal ones)
- refactor: expanded / compacted entity wrappers
  - rename JsonLdEntity to ExpandedEntity
  - extract compacted entity processings in a new CompactedEntity file
- fix(temporal): sysAttrs handling for temporal queries
- fix(temporal): aggregated and simplified representation of attributes and scopes
- feat(common): try to extract the wrapped cause when JSON-LD processing fails
- refactor: introduce an ExpandedMembers file for manipulation of expanded members
- refactor: clean up all the expand / compact processing utilities
- more tests for JsonLdUtils / refactor geoJsonToWkt processing
- refactor: move some last expanded members functions
- feat(temporal): merge uri value type to string for properties
- refactor: extract some constants from JsonLdUtils
- refactor: rename extractRelationshipObject et getRelationshipObject
- refactor: extract creation of a simplified or aggregated temporal instance into an utility function
- chore(subscription): update contexts references
- fix(temporal): do not restore geo-properties in temporal or aggregated representation

build.gradle.kts

@@ -63,7 +63,9 @@ subprojects {
         implementation("org.springframework.kafka:spring-kafka")
 
         implementation("com.fasterxml.jackson.module:jackson-module-kotlin")
-        implementation("com.github.jsonld-java:jsonld-java:0.13.6")
+
+        implementation("com.apicatalog:titanium-json-ld:1.3.3")
+        implementation("org.glassfish:jakarta.json:2.0.1")
 
         implementation("io.arrow-kt:arrow-fx-coroutines:1.2.1")
 

config/detekt/detekt.yml

@@ -168,10 +168,10 @@ complexity:
   TooManyFunctions:
     active: true
     excludes: ['**/test/**', '**/androidTest/**', '**/commonTest/**', '**/jvmTest/**', '**/androidUnitTest/**', '**/androidInstrumentedTest/**', '**/jsTest/**', '**/iosTest/**']
-    thresholdInFiles: 20
-    thresholdInClasses: 20
-    thresholdInInterfaces: 20
-    thresholdInObjects: 20
+    thresholdInFiles: 30
+    thresholdInClasses: 30
+    thresholdInInterfaces: 30
+    thresholdInObjects: 30
     thresholdInEnums: 11
     ignoreDeprecated: false
     ignorePrivate: false

search-service/config/detekt/baseline.xml

@@ -10,6 +10,7 @@
     <ID>LongMethod:AttributeInstanceService.kt$AttributeInstanceService$@Transactional suspend fun create(attributeInstance: AttributeInstance): Either&lt;APIException, Unit&gt;</ID>
     <ID>LongMethod:EntityAccessControlHandler.kt$EntityAccessControlHandler$@PostMapping("/{subjectId}/attrs", consumes = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) suspend fun addRightsOnEntities( @RequestHeader httpHeaders: HttpHeaders, @PathVariable subjectId: String, @RequestBody requestBody: Mono&lt;String&gt; ): ResponseEntity&lt;*&gt;</ID>
     <ID>LongMethod:QueryServiceTests.kt$QueryServiceTests$@Test fun `it should query temporal entities as requested by query params`()</ID>
+    <ID>LongMethod:QueryServiceTests.kt$QueryServiceTests$@Test fun `it should return an empty list for an attribute if it has no temporal values`()</ID>
     <ID>LongMethod:TemporalEntityBuilderTests.kt$TemporalEntityBuilderTests$@Test fun `it should return a temporal entity with values aggregated`()</ID>
     <ID>LongMethod:TemporalScopeBuilderTests.kt$TemporalScopeBuilderTests$@Test fun `it should build an aggregated temporal representation of scopes`()</ID>
     <ID>LongMethod:V0_29__JsonLd_migration.kt$V0_29__JsonLd_migration$override fun migrate(context: Context)</ID>
@@ -28,6 +29,5 @@
     <ID>ReturnCount:EntitiesQueryUtils.kt$fun buildTemporalQuery( params: MultiValueMap&lt;String, String&gt;, inQueryEntities: Boolean = false, withAggregatedValues: Boolean = false ): Either&lt;APIException, TemporalQuery&gt;</ID>
     <ID>SwallowedException:EntitiesQueryUtils.kt$e: IllegalArgumentException</ID>
     <ID>TooManyFunctions:EntityPayloadService.kt$EntityPayloadService</ID>
-    <ID>TooManyFunctions:TemporalEntityAttributeService.kt$TemporalEntityAttributeService</ID>
   </CurrentIssues>
 </SmellBaseline>

search-service/src/main/kotlin/com/egm/stellio/search/authorization/AuthorizationService.kt

@@ -4,7 +4,7 @@ import arrow.core.Either
 import arrow.core.Option
 import com.egm.stellio.search.model.EntitiesQuery
 import com.egm.stellio.shared.model.APIException
-import com.egm.stellio.shared.model.JsonLdEntity
+import com.egm.stellio.shared.model.ExpandedEntity
 import com.egm.stellio.shared.util.Sub
 import java.net.URI
 
@@ -23,20 +23,20 @@ interface AuthorizationService {
 
     suspend fun getAuthorizedEntities(
         entitiesQuery: EntitiesQuery,
-        contextLink: String,
+        contexts: List<String>,
         sub: Option<Sub>
-    ): Either<APIException, Pair<Int, List<JsonLdEntity>>>
+    ): Either<APIException, Pair<Int, List<ExpandedEntity>>>
 
     suspend fun getGroupsMemberships(
         offset: Int,
         limit: Int,
-        contextLink: String,
+        contexts: List<String>,
         sub: Option<Sub>
-    ): Either<APIException, Pair<Int, List<JsonLdEntity>>>
+    ): Either<APIException, Pair<Int, List<ExpandedEntity>>>
 
     suspend fun getUsers(
         offset: Int,
         limit: Int,
-        contextLink: String,
-    ): Either<APIException, Pair<Int, List<JsonLdEntity>>>
+        contexts: List<String>,
+    ): Either<APIException, Pair<Int, List<ExpandedEntity>>>
 }

search-service/src/main/kotlin/com/egm/stellio/search/authorization/DisabledAuthorizationService.kt

@@ -5,7 +5,7 @@ import arrow.core.Option
 import arrow.core.right
 import com.egm.stellio.search.model.EntitiesQuery
 import com.egm.stellio.shared.model.APIException
-import com.egm.stellio.shared.model.JsonLdEntity
+import com.egm.stellio.shared.model.ExpandedEntity
 import com.egm.stellio.shared.util.Sub
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
 import org.springframework.stereotype.Component
@@ -41,20 +41,20 @@ class DisabledAuthorizationService : AuthorizationService {
 
     override suspend fun getAuthorizedEntities(
         entitiesQuery: EntitiesQuery,
-        contextLink: String,
+        contexts: List<String>,
         sub: Option<Sub>
-    ): Either<APIException, Pair<Int, List<JsonLdEntity>>> = Pair(-1, emptyList<JsonLdEntity>()).right()
+    ): Either<APIException, Pair<Int, List<ExpandedEntity>>> = Pair(-1, emptyList<ExpandedEntity>()).right()
 
     override suspend fun getGroupsMemberships(
         offset: Int,
         limit: Int,
-        contextLink: String,
+        contexts: List<String>,
         sub: Option<Sub>
-    ): Either<APIException, Pair<Int, List<JsonLdEntity>>> = Pair(-1, emptyList<JsonLdEntity>()).right()
+    ): Either<APIException, Pair<Int, List<ExpandedEntity>>> = Pair(-1, emptyList<ExpandedEntity>()).right()
 
     override suspend fun getUsers(
         offset: Int,
         limit: Int,
-        contextLink: String,
-    ): Either<APIException, Pair<Int, List<JsonLdEntity>>> = Pair(-1, emptyList<JsonLdEntity>()).right()
+        contexts: List<String>,
+    ): Either<APIException, Pair<Int, List<ExpandedEntity>>> = Pair(-1, emptyList<ExpandedEntity>()).right()
 }

search-service/src/main/kotlin/com/egm/stellio/search/authorization/EnabledAuthorizationService.kt

@@ -6,7 +6,7 @@ import arrow.fx.coroutines.parMap
 import com.egm.stellio.search.model.EntitiesQuery
 import com.egm.stellio.shared.model.APIException
 import com.egm.stellio.shared.model.AccessDeniedException
-import com.egm.stellio.shared.model.JsonLdEntity
+import com.egm.stellio.shared.model.ExpandedEntity
 import com.egm.stellio.shared.util.*
 import com.egm.stellio.shared.util.AuthContextModel.SpecificAccessPolicy
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty
@@ -95,9 +95,9 @@ class EnabledAuthorizationService(
 
     override suspend fun getAuthorizedEntities(
         entitiesQuery: EntitiesQuery,
-        contextLink: String,
+        contexts: List<String>,
         sub: Option<Sub>
-    ): Either<APIException, Pair<Int, List<JsonLdEntity>>> = either {
+    ): Either<APIException, Pair<Int, List<ExpandedEntity>>> = either {
         val accessRights = entitiesQuery.attrs.mapNotNull { AccessRight.forExpandedAttributeName(it).getOrNull() }
         val entitiesAccessControl = entityAccessRightsService.getSubjectAccessRights(
             sub,
@@ -127,8 +127,8 @@ class EnabledAuthorizationService(
                     )
                 } else entityAccessControl
             }
-            .map { it.serializeProperties(contextLink) }
-            .map { JsonLdEntity(it, listOf(contextLink)) }
+            .map { it.serializeProperties(contexts) }
+            .map { ExpandedEntity(it, contexts) }
 
         val count = entityAccessRightsService.getSubjectAccessRightsCount(
             sub,
@@ -143,9 +143,9 @@ class EnabledAuthorizationService(
     override suspend fun getGroupsMemberships(
         offset: Int,
         limit: Int,
-        contextLink: String,
+        contexts: List<String>,
         sub: Option<Sub>
-    ): Either<APIException, Pair<Int, List<JsonLdEntity>>> = either {
+    ): Either<APIException, Pair<Int, List<ExpandedEntity>>> = either {
         val groups =
             when (userIsAdmin(sub)) {
                 is Either.Left -> {
@@ -162,10 +162,7 @@ class EnabledAuthorizationService(
             }
 
         val jsonLdEntities = groups.second.map {
-            JsonLdEntity(
-                it.serializeProperties(),
-                listOf(contextLink)
-            )
+            ExpandedEntity(it.serializeProperties(), contexts)
         }
 
         Pair(groups.first, jsonLdEntities)
@@ -174,16 +171,13 @@ class EnabledAuthorizationService(
     override suspend fun getUsers(
         offset: Int,
         limit: Int,
-        contextLink: String
-    ): Either<APIException, Pair<Int, List<JsonLdEntity>>> = either {
+        contexts: List<String>,
+    ): Either<APIException, Pair<Int, List<ExpandedEntity>>> = either {
         val users = subjectReferentialService.getUsers(offset, limit)
         val usersCount = subjectReferentialService.getUsersCount().bind()
 
         val jsonLdEntities = users.map {
-            JsonLdEntity(
-                it.serializeProperties(contextLink),
-                listOf(contextLink)
-            )
+            ExpandedEntity(it.serializeProperties(contexts), contexts)
         }
 
         Pair(usersCount, jsonLdEntities)

search-service/src/main/kotlin/com/egm/stellio/search/authorization/EntityAccessRights.kt

@@ -1,20 +1,26 @@
 package com.egm.stellio.search.authorization
 
-import com.egm.stellio.shared.util.*
+import com.egm.stellio.shared.model.ExpandedAttributeInstances
+import com.egm.stellio.shared.model.ExpandedTerm
+import com.egm.stellio.shared.model.addNonReifiedProperty
+import com.egm.stellio.shared.model.addSubAttribute
+import com.egm.stellio.shared.util.AccessRight
+import com.egm.stellio.shared.util.AuthContextModel
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_RIGHT
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_SAP
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_SUBJECT_INFO
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_REL_CAN_ADMIN
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_REL_CAN_READ
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_REL_CAN_WRITE
-import com.egm.stellio.shared.util.JsonLdUtils.DATASET_ID_PREFIX
+import com.egm.stellio.shared.util.AuthContextModel.DATASET_ID_PREFIX
 import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID
 import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_DATASET_ID_PROPERTY
-import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedProperty
 import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedPropertyMapValue
-import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedRelationship
-import com.egm.stellio.shared.util.JsonLdUtils.buildNonReifiedProperty
+import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedPropertyValue
+import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedRelationshipValue
+import com.egm.stellio.shared.util.extractSub
+import com.egm.stellio.shared.util.toUri
 import java.net.URI
 
 data class EntityAccessRights(
@@ -33,39 +39,39 @@ data class EntityAccessRights(
     ) {
         val datasetId: URI = (DATASET_ID_PREFIX + uri.extractSub()).toUri()
 
-        suspend fun serializeProperties(contextLink: String): ExpandedAttributeInstances =
-            buildExpandedRelationship(uri)
-                .addSubAttribute(NGSILD_DATASET_ID_PROPERTY, buildNonReifiedProperty(datasetId.toString()))
+        suspend fun serializeProperties(contexts: List<String>): ExpandedAttributeInstances =
+            buildExpandedRelationshipValue(uri)
+                .addNonReifiedProperty(NGSILD_DATASET_ID_PROPERTY, datasetId.toString())
                 .addSubAttribute(
                     AUTH_PROP_SUBJECT_INFO,
-                    buildExpandedPropertyMapValue(subjectInfo, listOf(contextLink))
+                    buildExpandedPropertyMapValue(subjectInfo, contexts)
                 )
     }
 
-    suspend fun serializeProperties(contextLink: String): Map<String, Any> {
+    suspend fun serializeProperties(contexts: List<String>): Map<String, Any> {
         val resultEntity = mutableMapOf<String, Any>()
 
         resultEntity[JSONLD_ID] = id.toString()
         resultEntity[JSONLD_TYPE] = types
-        resultEntity[AUTH_PROP_RIGHT] = buildExpandedProperty(right.attributeName)
+        resultEntity[AUTH_PROP_RIGHT] = buildExpandedPropertyValue(right.attributeName)
 
         specificAccessPolicy?.run {
-            resultEntity[AUTH_PROP_SAP] = buildExpandedProperty(this)
+            resultEntity[AUTH_PROP_SAP] = buildExpandedPropertyValue(this)
         }
 
         rCanAdminUsers?.run {
             resultEntity[AUTH_REL_CAN_ADMIN] = this.map {
-                it.serializeProperties(contextLink)
+                it.serializeProperties(contexts)
             }.flatten()
         }
         rCanWriteUsers?.run {
             resultEntity[AUTH_REL_CAN_WRITE] = this.map {
-                it.serializeProperties(contextLink)
+                it.serializeProperties(contexts)
             }.flatten()
         }
         rCanReadUsers?.run {
             resultEntity[AUTH_REL_CAN_READ] = this.map {
-                it.serializeProperties(contextLink)
+                it.serializeProperties(contexts)
             }.flatten()
         }
         return resultEntity

search-service/src/main/kotlin/com/egm/stellio/search/authorization/Group.kt

@@ -1,11 +1,11 @@
 package com.egm.stellio.search.authorization
 
+import com.egm.stellio.shared.util.AuthContextModel.AUTH_PROP_NAME
 import com.egm.stellio.shared.util.AuthContextModel.AUTH_REL_IS_MEMBER_OF
 import com.egm.stellio.shared.util.AuthContextModel.GROUP_ENTITY_PREFIX
 import com.egm.stellio.shared.util.AuthContextModel.GROUP_TYPE
 import com.egm.stellio.shared.util.JsonLdUtils
-import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_NAME_PROPERTY
-import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedProperty
+import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedPropertyValue
 
 data class Group(
     val id: String,
@@ -18,10 +18,10 @@ data class Group(
         resultEntity[JsonLdUtils.JSONLD_ID] = GROUP_ENTITY_PREFIX + id
         resultEntity[JsonLdUtils.JSONLD_TYPE] = listOf(type)
 
-        resultEntity[NGSILD_NAME_PROPERTY] = buildExpandedProperty(name)
+        resultEntity[AUTH_PROP_NAME] = buildExpandedPropertyValue(name)
 
         isMember.run {
-            resultEntity[AUTH_REL_IS_MEMBER_OF] = buildExpandedProperty(isMember.toString())
+            resultEntity[AUTH_REL_IS_MEMBER_OF] = buildExpandedPropertyValue(isMember.toString())
         }
         return resultEntity
     }

search-service/src/main/kotlin/com/egm/stellio/search/authorization/User.kt

@@ -12,8 +12,8 @@ import com.egm.stellio.shared.util.AuthContextModel.USER_ENTITY_PREFIX
 import com.egm.stellio.shared.util.AuthContextModel.USER_TYPE
 import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_ID
 import com.egm.stellio.shared.util.JsonLdUtils.JSONLD_TYPE
-import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedProperty
 import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedPropertyMapValue
+import com.egm.stellio.shared.util.JsonLdUtils.buildExpandedPropertyValue
 
 data class User(
     val id: String,
@@ -23,27 +23,27 @@ data class User(
     val familyName: String? = null,
     val subjectInfo: Map<String, String>
 ) {
-    suspend fun serializeProperties(contextLink: String): Map<String, Any> {
+    suspend fun serializeProperties(contexts: List<String>): Map<String, Any> {
         val resultEntity = mutableMapOf<String, Any>()
         resultEntity[JSONLD_ID] = USER_ENTITY_PREFIX + id
         resultEntity[JSONLD_TYPE] = listOf(type)
 
-        resultEntity[AUTH_PROP_USERNAME] = buildExpandedProperty(username)
+        resultEntity[AUTH_PROP_USERNAME] = buildExpandedPropertyValue(username)
 
         givenName?.run {
-            resultEntity[AUTH_PROP_GIVEN_NAME] = buildExpandedProperty(givenName)
+            resultEntity[AUTH_PROP_GIVEN_NAME] = buildExpandedPropertyValue(givenName)
         }
 
         familyName?.run {
-            resultEntity[AUTH_PROP_FAMILY_NAME] = buildExpandedProperty(familyName)
+            resultEntity[AUTH_PROP_FAMILY_NAME] = buildExpandedPropertyValue(familyName)
         }
 
         subjectInfo.filterKeys {
             !setOf(AUTH_TERM_USERNAME, AUTH_TERM_GIVEN_NAME, AUTH_TERM_FAMILY_NAME, AUTH_TERM_KIND).contains(it)
         }.run {
             if (this.isNotEmpty())
                 resultEntity[AUTH_PROP_SUBJECT_INFO] =
-                    buildExpandedPropertyMapValue(this, listOf(contextLink))
+                    buildExpandedPropertyMapValue(this, contexts)
         }
 
         return resultEntity

search-service/src/main/kotlin/com/egm/stellio/search/listener/ObservationEventListener.kt

@@ -7,9 +7,9 @@ import com.egm.stellio.search.service.EntityEventService
 import com.egm.stellio.search.service.EntityPayloadService
 import com.egm.stellio.shared.model.*
 import com.egm.stellio.shared.util.JsonLdUtils.expandAttribute
+import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdEntity
 import com.egm.stellio.shared.util.JsonLdUtils.expandJsonLdTerms
 import com.egm.stellio.shared.util.JsonUtils.deserializeAs
-import com.egm.stellio.shared.util.toExpandedAttributes
 import com.egm.stellio.shared.web.NGSILD_TENANT_HEADER
 import kotlinx.coroutines.CoroutineScope
 import kotlinx.coroutines.Dispatchers
@@ -64,10 +64,16 @@ class ObservationEventListener(
         tenantUri: URI,
         observationEvent: EntityCreateEvent
     ): Either<APIException, Unit> = either {
+        val expandedEntity = expandJsonLdEntity(
+            observationEvent.operationPayload,
+            observationEvent.contexts
+        )
+        val ngsiLdEntity = expandedEntity.toNgsiLdEntity().bind()
+
         mono {
             entityPayloadService.createEntity(
-                observationEvent.operationPayload,
-                observationEvent.contexts,
+                ngsiLdEntity,
+                expandedEntity,
                 observationEvent.sub
             ).map {
                 entityEventService.publishEntityCreateEvent(

search-service/src/main/kotlin/com/egm/stellio/search/model/AttributeInstance.kt

@@ -1,11 +1,11 @@
 package com.egm.stellio.search.model
 
+import com.egm.stellio.shared.model.ExpandedAttributeInstance
 import com.egm.stellio.shared.model.WKTCoordinates
-import com.egm.stellio.shared.util.ExpandedAttributeInstance
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_INSTANCE_ID_PROPERTY
 import com.egm.stellio.shared.util.JsonLdUtils.NGSILD_MODIFIED_AT_PROPERTY
-import com.egm.stellio.shared.util.JsonLdUtils.buildNonReifiedDateTime
-import com.egm.stellio.shared.util.JsonLdUtils.buildNonReifiedProperty
+import com.egm.stellio.shared.util.JsonLdUtils.buildNonReifiedPropertyValue
+import com.egm.stellio.shared.util.JsonLdUtils.buildNonReifiedTemporalValue
 import com.egm.stellio.shared.util.JsonUtils.serializeObject
 import com.egm.stellio.shared.util.toUri
 import io.r2dbc.postgresql.codec.Json
@@ -72,10 +72,10 @@ data class AttributeInstance private constructor(
             instanceId: URI,
             modifiedAt: ZonedDateTime? = null
         ): ExpandedAttributeInstance =
-            this.plus(NGSILD_INSTANCE_ID_PROPERTY to buildNonReifiedProperty(instanceId.toString()))
+            this.plus(NGSILD_INSTANCE_ID_PROPERTY to buildNonReifiedPropertyValue(instanceId.toString()))
                 .let {
                     if (modifiedAt != null)
-                        it.plus(NGSILD_MODIFIED_AT_PROPERTY to buildNonReifiedDateTime(modifiedAt))
+                        it.plus(NGSILD_MODIFIED_AT_PROPERTY to buildNonReifiedTemporalValue(modifiedAt))
                     else it
                 }