From 314a1d0cb8c05efba79a7485fe25f7edbcefdcbe Mon Sep 17 00:00:00 2001 From: vraybaud Date: Wed, 20 May 2020 15:29:07 +0200 Subject: [PATCH] feat: add support for entityOperation/upsert with update option #21 --- build.gradle.kts | 7 + .../entity/service/EntityOperationService.kt | 88 +++++++++++- .../stellio/entity/service/EntityService.kt | 22 ++- .../service/RepositoryEventsListener.kt | 8 +- .../egm/stellio/entity/web/APIResponses.kt | 15 +- .../entity/web/EntityOperationHandler.kt | 26 ++++ .../service/EntityOperationServiceTests.kt | 118 +++++++++++++++- .../entity/web/EntityOperationHandlerTests.kt | 129 ++++++++++++++++-- ...CMR_test_file_invalid_relation_update.json | 36 +++++ .../service/TemporalEntityAttributeService.kt | 6 +- .../search/web/TemporalEntityHandler.kt | 10 +- .../TemporalEntityAttributeServiceTests.kt | 30 +++- .../stellio/shared/model/ExpandedEntity.kt | 20 ++- .../stellio/shared/util/NgsiLdParsingUtils.kt | 7 +- 14 files changed, 476 insertions(+), 46 deletions(-) create mode 100644 entity-service/src/test/resources/ngsild/hcmr/HCMR_test_file_invalid_relation_update.json diff --git a/build.gradle.kts b/build.gradle.kts index f0639aa58..e57324719 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,6 +14,7 @@ plugins { kotlin("plugin.spring") version "1.3.61" apply false id("org.jlleitschuh.gradle.ktlint") version "8.2.0" id("com.google.cloud.tools.jib") version "1.6.1" apply false + kotlin("kapt") version "1.3.61" apply false } subprojects { @@ -21,12 +22,14 @@ subprojects { mavenCentral() maven { url = uri("https://repo.spring.io/milestone") } jcenter() + maven { url = uri("https://dl.bintray.com/arrow-kt/arrow-kt/") } } apply(plugin = "io.spring.dependency-management") apply(plugin = "org.jetbrains.kotlin.jvm") apply(plugin = "org.jetbrains.kotlin.plugin.spring") apply(plugin = "org.jlleitschuh.gradle.ktlint") + apply(plugin = "kotlin-kapt") java.sourceCompatibility = JavaVersion.VERSION_11 @@ -55,6 +58,10 @@ subprojects { implementation("com.fasterxml.jackson.module:jackson-module-kotlin") implementation("com.github.jsonld-java:jsonld-java:0.13.0") + implementation("io.arrow-kt:arrow-fx:0.10.4") + implementation("io.arrow-kt:arrow-syntax:0.10.4") + "kapt"("io.arrow-kt:arrow-meta:0.10.4") + annotationProcessor("org.springframework.boot:spring-boot-configuration-processor") runtimeOnly("de.siegmar:logback-gelf:3.0.0") diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityOperationService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityOperationService.kt index ca6e2df1e..e94ef7d7a 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityOperationService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityOperationService.kt @@ -1,5 +1,6 @@ package com.egm.stellio.entity.service +import arrow.core.Either import com.egm.stellio.entity.model.Entity import com.egm.stellio.entity.repository.EntityRepository import com.egm.stellio.entity.repository.Neo4jRepository @@ -8,8 +9,6 @@ import com.egm.stellio.entity.web.BatchEntityError import com.egm.stellio.entity.web.BatchOperationResult import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.model.ExpandedEntity -import com.egm.stellio.shared.util.NgsiLdParsingUtils.NGSILD_ENTITY_ID -import com.egm.stellio.shared.util.NgsiLdParsingUtils.NGSILD_ENTITY_TYPE import org.jgrapht.Graph import org.jgrapht.Graphs import org.jgrapht.graph.DefaultEdge @@ -58,6 +57,87 @@ class EntityOperationService( return BatchOperationResult(ArrayList(success), ArrayList(errors)) } + /** + * Update a batch of [entities]. + * Only entities with relations linked to existing entities will be updated. + * + * @return a [BatchOperationResult] with list of updated ids and list of errors (either not totally updated or + * linked to invalid entity). + */ + fun update(entities: List, createBatchResult: BatchOperationResult): BatchOperationResult { + val existingEntitiesIds = createBatchResult.success.plus(entities.map { it.id }) + val nonExistingEntitiesIds = createBatchResult.errors.map { it.entityId } + return entities.parallelStream().map { entity -> + updateEntity(entity, existingEntitiesIds, nonExistingEntitiesIds) + }.collect( + { BatchOperationResult() }, + { batchOperationResult, updateResult -> + updateResult.fold({ + batchOperationResult.errors.add(it) + }, { + batchOperationResult.success.add(it) + }) + }, + BatchOperationResult::plusAssign + ) + } + + private fun updateEntity( + entity: ExpandedEntity, + existingEntitiesIds: List, + nonExistingEntitiesIds: List + ): Either { + // All new attributes linked entities should be existing in the DB. + val linkedEntitiesIds = entity.getLinkedEntitiesIds() + val invalidLinkedEntityId = + findInvalidEntityId(linkedEntitiesIds, existingEntitiesIds, nonExistingEntitiesIds) + + // If there's a link to an invalid entity, then avoid calling the processor and return an error + if (invalidLinkedEntityId != null) { + return Either.left( + BatchEntityError( + entity.id, + arrayListOf("Target entity $invalidLinkedEntityId does not exist.") + ) + ) + } + + return try { + val (_, notUpdated) = entityService.appendEntityAttributes( + entity.id, + entity.attributes, + false + ) + + if (notUpdated.isEmpty()) { + Either.right(entity.id) + } else { + Either.left( + BatchEntityError( + entity.id, + ArrayList(notUpdated.map { it.attributeName + " : " + it.reason }) + ) + ) + } + } catch (e: BadRequestDataException) { + Either.left(BatchEntityError(entity.id, arrayListOf(e.message))) + } + } + + private fun findInvalidEntityId( + entitiesIds: List, + existingEntitiesIds: List, + nonExistingEntitiesIds: List + ): String? { + val invalidEntityId = entitiesIds.intersect(nonExistingEntitiesIds).firstOrNull() + if (invalidEntityId == null) { + val unknownEntitiesIds = entitiesIds.minus(existingEntitiesIds) + return unknownEntitiesIds + .minus(neo4jRepository.filterExistingEntitiesIds(unknownEntitiesIds)).firstOrNull() + } + return invalidEntityId + } + private fun createEntitiesWithoutCircularDependencies(graph: Graph): Pair> { val batchOperationResult = BatchOperationResult(arrayListOf(), arrayListOf()) val temporaryGraph = DirectedPseudograph(DefaultEdge::class.java) @@ -110,9 +190,7 @@ class EntityOperationService( try { entityService.appendEntityAttributes( entity.id, - entity.attributes.filterKeys { - !listOf(NGSILD_ENTITY_ID, NGSILD_ENTITY_TYPE).contains(it) - }, + entity.attributes, false ) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt index 083c22e86..d575d8ef8 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/EntityService.kt @@ -65,9 +65,7 @@ class EntityService( val entity = entityRepository.save(rawEntity) // filter the unwanted entries and expand all attributes for easier later processing - val propertiesAndRelationshipsMap = expandedEntity.attributes.filterKeys { - !listOf(NGSILD_ENTITY_ID, NGSILD_ENTITY_TYPE).contains(it) - }.mapValues { + val propertiesAndRelationshipsMap = expandedEntity.attributes.mapValues { expandValueAsMap(it.value) } @@ -102,8 +100,14 @@ class EntityService( } fun publishCreationEvent(expandedEntity: ExpandedEntity) { - val entityType = extractShortTypeFromPayload(expandedEntity.attributes) - val entityEvent = EntityEvent(EventType.CREATE, expandedEntity.id, entityType, getSerializedEntityById(expandedEntity.id), null) + val entityType = extractShortTypeFromPayload(expandedEntity.rawJsonLdProperties) + val entityEvent = EntityEvent( + EventType.CREATE, + expandedEntity.id, + entityType, + getSerializedEntityById(expandedEntity.id), + null + ) applicationEventPublisher.publishEvent(entityEvent) } @@ -336,7 +340,13 @@ class EntityService( fun getSerializedEntityById(entityId: String): String { val mapper = jacksonObjectMapper().findAndRegisterModules().disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) val entity = getFullEntityById(entityId) - return mapper.writeValueAsString(JsonLdProcessor.compact(entity.attributes, mapOf("@context" to entity.contexts), JsonLdOptions())) + return mapper.writeValueAsString( + JsonLdProcessor.compact( + entity.rawJsonLdProperties, + mapOf("@context" to entity.contexts), + JsonLdOptions() + ) + ) } fun searchEntities(type: String, query: List, contextLink: String): List = diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/RepositoryEventsListener.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/RepositoryEventsListener.kt index 8b11b9dea..943edc0d5 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/service/RepositoryEventsListener.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/service/RepositoryEventsListener.kt @@ -72,6 +72,12 @@ class RepositoryEventsListener( private fun getEntityById(entityId: String): String { val entity = entityService.getFullEntityById(entityId) - return mapper.writeValueAsString(JsonLdProcessor.compact(entity.attributes, mapOf("@context" to entity.contexts), JsonLdOptions())) + return mapper.writeValueAsString( + JsonLdProcessor.compact( + entity.rawJsonLdProperties, + mapOf("@context" to entity.contexts), + JsonLdOptions() + ) + ) } } diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/APIResponses.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/APIResponses.kt index 9bee80dd7..92f4b3173 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/APIResponses.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/APIResponses.kt @@ -1,11 +1,18 @@ package com.egm.stellio.entity.web data class BatchOperationResult( - val success: ArrayList, - val errors: ArrayList -) + val success: ArrayList = arrayListOf(), + val errors: ArrayList = arrayListOf() +) { + + operator fun plusAssign(other: BatchOperationResult) { + success.addAll(other.success) + errors.addAll(other.errors) + } +} data class BatchEntityError( val entityId: String, val error: ArrayList -) \ No newline at end of file + +) diff --git a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityOperationHandler.kt b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityOperationHandler.kt index 4fdaee576..592edb634 100644 --- a/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityOperationHandler.kt +++ b/entity-service/src/main/kotlin/com/egm/stellio/entity/web/EntityOperationHandler.kt @@ -46,6 +46,32 @@ class EntityOperationHandler( } } + /** + * Implements 6.15.3.1 - Upsert Batch of Entities + */ + @PostMapping("/upsert", consumes = [MediaType.APPLICATION_JSON_VALUE, JSON_LD_CONTENT_TYPE]) + fun upsert(@RequestBody body: Mono): Mono> { + return body + .map { + extractAndParseBatchOfEntities(it) + } + .map { + val (existingEntities, newEntities) = entityOperationService.splitEntitiesByExistence(it) + + val createBatchOperationResult = entityOperationService.create(newEntities) + val updateBatchOperationResult = + entityOperationService.update(existingEntities, createBatchOperationResult) + + BatchOperationResult( + ArrayList(createBatchOperationResult.success.plus(updateBatchOperationResult.success)), + ArrayList(createBatchOperationResult.errors.plus(updateBatchOperationResult.errors)) + ) + } + .map { + ResponseEntity.status(HttpStatus.OK).body(it) + } + } + private fun extractAndParseBatchOfEntities(payload: String): List { val extractedEntities = extractEntitiesFromJsonPayload(payload) return NgsiLdParsingUtils.parseEntities(extractedEntities) diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityOperationServiceTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityOperationServiceTests.kt index d5abbb9d4..697596de0 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityOperationServiceTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/service/EntityOperationServiceTests.kt @@ -1,11 +1,13 @@ package com.egm.stellio.entity.service import com.egm.stellio.entity.model.Entity +import com.egm.stellio.entity.model.NotUpdatedDetails import com.egm.stellio.entity.model.UpdateResult import com.egm.stellio.entity.repository.EntityRepository import com.egm.stellio.entity.repository.Neo4jRepository import com.egm.stellio.entity.util.EntitiesGraphBuilder import com.egm.stellio.entity.web.BatchEntityError +import com.egm.stellio.entity.web.BatchOperationResult import com.egm.stellio.shared.model.BadRequestDataException import com.egm.stellio.shared.model.ExpandedEntity import com.ninjasquad.springmockk.MockkBean @@ -114,11 +116,125 @@ class EntityOperationServiceTests { every { entityService.appendEntityAttributes(eq("1"), any(), any()) } returns mockkClass(UpdateResult::class) every { entityService.appendEntityAttributes(eq("2"), any(), any()) } returns mockkClass(UpdateResult::class) every { entityService.publishCreationEvent(any()) } just Runs - every { entityRepository.save(any()) } returns mockk() + every { entityRepository.save(any()) } returns mockk() val batchOperationResult = entityOperationService.create(listOf(firstEntity, secondEntity)) assertEquals(arrayListOf("1", "2"), batchOperationResult.success) assertTrue(batchOperationResult.errors.isEmpty()) } + + @Test + fun `it should not update entities with relationships to invalid entity not found in DB`() { + val firstEntity = mockkClass(ExpandedEntity::class, relaxed = true) + every { firstEntity.id } returns "1" + every { firstEntity.getLinkedEntitiesIds() } returns emptyList() + val secondEntity = mockkClass(ExpandedEntity::class, relaxed = true) + every { secondEntity.id } returns "2" + every { secondEntity.getLinkedEntitiesIds() } returns listOf("3") + + every { neo4jRepository.filterExistingEntitiesIds(listOf()) } returns emptyList() + every { neo4jRepository.filterExistingEntitiesIds(listOf("3")) } returns emptyList() + every { entityService.appendEntityAttributes(eq("1"), any(), any()) } returns UpdateResult( + emptyList(), + emptyList() + ) + + val batchOperationResult = + entityOperationService.update(listOf(firstEntity, secondEntity), BatchOperationResult()) + + assertEquals(listOf("1"), batchOperationResult.success) + assertEquals( + listOf(BatchEntityError("2", arrayListOf("Target entity 3 does not exist."))), + batchOperationResult.errors + ) + } + + @Test + fun `it should not update entities with relationships to invalid entity given in BatchOperationResult`() { + val firstEntity = mockkClass(ExpandedEntity::class, relaxed = true) + every { firstEntity.id } returns "1" + every { firstEntity.getLinkedEntitiesIds() } returns emptyList() + val secondEntity = mockkClass(ExpandedEntity::class, relaxed = true) + every { secondEntity.id } returns "2" + every { secondEntity.getLinkedEntitiesIds() } returns listOf("3") + + every { neo4jRepository.filterExistingEntitiesIds(listOf()) } returns emptyList() + every { entityService.appendEntityAttributes(eq("1"), any(), any()) } returns UpdateResult( + emptyList(), + emptyList() + ) + + val batchOperationResult = + entityOperationService.update( + listOf(firstEntity, secondEntity), + BatchOperationResult( + errors = arrayListOf(BatchEntityError("3", arrayListOf(""))) + ) + ) + + assertEquals(listOf("1"), batchOperationResult.success) + assertEquals( + listOf(BatchEntityError("2", arrayListOf("Target entity 3 does not exist."))), + batchOperationResult.errors + ) + } + + @Test + fun `it should count as error updating which results in BadRequestDataException`() { + val firstEntity = mockkClass(ExpandedEntity::class, relaxed = true) + every { firstEntity.id } returns "1" + every { firstEntity.getLinkedEntitiesIds() } returns emptyList() + val secondEntity = mockkClass(ExpandedEntity::class, relaxed = true) + every { secondEntity.id } returns "2" + every { secondEntity.getLinkedEntitiesIds() } returns emptyList() + + every { neo4jRepository.filterExistingEntitiesIds(emptyList()) } returns emptyList() + every { entityService.appendEntityAttributes(eq("1"), any(), any()) } returns UpdateResult( + emptyList(), + emptyList() + ) + every { entityService.appendEntityAttributes(eq("2"), any(), any()) } throws BadRequestDataException("error") + + val batchOperationResult = + entityOperationService.update(listOf(firstEntity, secondEntity), BatchOperationResult()) + + assertEquals(listOf("1"), batchOperationResult.success) + assertEquals( + listOf(BatchEntityError("2", arrayListOf("error"))), + batchOperationResult.errors + ) + } + + @Test + fun `it should count as error not updated attributes in entities`() { + val firstEntity = mockkClass(ExpandedEntity::class, relaxed = true) + every { firstEntity.id } returns "1" + every { firstEntity.getLinkedEntitiesIds() } returns emptyList() + val secondEntity = mockkClass(ExpandedEntity::class, relaxed = true) + every { secondEntity.id } returns "2" + every { secondEntity.getLinkedEntitiesIds() } returns emptyList() + + every { neo4jRepository.filterExistingEntitiesIds(listOf()) } returns emptyList() + every { entityService.appendEntityAttributes(eq("1"), any(), any()) } returns UpdateResult( + emptyList(), + emptyList() + ) + every { entityService.appendEntityAttributes(eq("2"), any(), any()) } returns UpdateResult( + emptyList(), + listOf( + NotUpdatedDetails("attribute#1", "reason"), + NotUpdatedDetails("attribute#2", "reason") + ) + ) + + val batchOperationResult = + entityOperationService.update(listOf(firstEntity, secondEntity), BatchOperationResult()) + + assertEquals(listOf("1"), batchOperationResult.success) + assertEquals( + listOf(BatchEntityError("2", arrayListOf("attribute#1 : reason", "attribute#2 : reason"))), + batchOperationResult.errors + ) + } } \ No newline at end of file diff --git a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityOperationHandlerTests.kt b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityOperationHandlerTests.kt index 559672b33..4cbfe712d 100644 --- a/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityOperationHandlerTests.kt +++ b/entity-service/src/test/kotlin/com/egm/stellio/entity/web/EntityOperationHandlerTests.kt @@ -32,7 +32,7 @@ class EntityOperationHandlerTests { @Autowired private lateinit var webClient: WebTestClient - @MockkBean(relaxed = true) + @MockkBean private lateinit var entityOperationService: EntityOperationService @Test @@ -46,8 +46,8 @@ class EntityOperationHandlerTests { val expandedEntities = slot>() every { entityOperationService.splitEntitiesByExistence(capture(expandedEntities)) } returns Pair( - listOf(), - listOf() + emptyList(), + emptyList() ) every { entityOperationService.create(any()) } returns BatchOperationResult( entitiesIds, @@ -94,8 +94,8 @@ class EntityOperationHandlerTests { ) every { entityOperationService.splitEntitiesByExistence(any()) } returns Pair( - listOf(), - listOf() + emptyList(), + emptyList() ) every { entityOperationService.create(any()) } returns BatchOperationResult( createdEntitiesIds, @@ -145,7 +145,7 @@ class EntityOperationHandlerTests { every { entityOperationService.splitEntitiesByExistence(any()) } returns Pair( listOf(existingEntity), - listOf() + emptyList() ) every { entityOperationService.create(any()) } returns BatchOperationResult( createdEntitiesIds, @@ -178,12 +178,125 @@ class EntityOperationHandlerTests { ) } + @Test + fun `upsert batch entity should return a 200 if JSON-LD payload is correct`() { + val jsonLdFile = ClassPathResource("/ngsild/hcmr/HCMR_test_file.json") + val createdEntitiesIds = arrayListOf( + "urn:ngsi-ld:Sensor:HCMR-AQUABOX1temperature" + ) + val entitiesIds = arrayListOf( + "urn:ngsi-ld:Sensor:HCMR-AQUABOX1dissolvedOxygen", + "urn:ngsi-ld:Device:HCMR-AQUABOX1" + ) + val createdBatchResult = BatchOperationResult( + createdEntitiesIds, + arrayListOf() + ) + + val existingEntities = mockk>() + val nonExistingEntities = mockk>() + + every { entityOperationService.splitEntitiesByExistence(any()) } returns Pair( + existingEntities, + nonExistingEntities + ) + + every { entityOperationService.create(nonExistingEntities) } returns createdBatchResult + every { entityOperationService.update(existingEntities, createdBatchResult) } returns BatchOperationResult( + entitiesIds, + arrayListOf() + ) + webClient.post() + .uri("/ngsi-ld/v1/entityOperations/upsert") + .header("Link", "<$aquacContext>; rel=http://www.w3.org/ns/json-ld#context; type=application/ld+json") + .bodyValue(jsonLdFile) + .exchange() + .expectStatus().isOk + .expectBody().json( + """ + { + "errors": [], + success: [ + "urn:ngsi-ld:Sensor:HCMR-AQUABOX1temperature", + "urn:ngsi-ld:Sensor:HCMR-AQUABOX1dissolvedOxygen", + "urn:ngsi-ld:Device:HCMR-AQUABOX1" + ] + } + """.trimIndent() + ) + } + + @Test + fun `upsert batch entity should return a 200 if JSON-LD payload contains update errors`() { + val jsonLdFile = ClassPathResource("/ngsild/hcmr/HCMR_test_file_invalid_relation_update.json") + val errors = arrayListOf( + BatchEntityError( + "urn:ngsi-ld:Sensor:HCMR-AQUABOX1temperature", + arrayListOf("Target entity urn:ngsi-ld:Device:HCMR-AQUABOX2 does not exist.") + ), + BatchEntityError( + "urn:ngsi-ld:Sensor:HCMR-AQUABOX1dissolvedOxygen", + arrayListOf("Target entity urn:ngsi-ld:Device:HCMR-AQUABOX2 does not exist.") + ) + ) + + every { entityOperationService.splitEntitiesByExistence(any()) } returns Pair( + emptyList(), + emptyList() + ) + every { entityOperationService.create(any()) } returns BatchOperationResult( + arrayListOf(), + arrayListOf() + ) + every { entityOperationService.update(any(), any()) } returns BatchOperationResult( + arrayListOf(), + errors + ) + + webClient.post() + .uri("/ngsi-ld/v1/entityOperations/upsert") + .header("Link", "<$aquacContext>; rel=http://www.w3.org/ns/json-ld#context; type=application/ld+json") + .bodyValue(jsonLdFile) + .exchange() + .expectStatus().isOk + .expectBody().json( + """ + { + "errors": [ + { + "entityId": "urn:ngsi-ld:Sensor:HCMR-AQUABOX1temperature", + "error": [ + "Target entity urn:ngsi-ld:Device:HCMR-AQUABOX2 does not exist." + ] + }, + { + "entityId": "urn:ngsi-ld:Sensor:HCMR-AQUABOX1dissolvedOxygen", + "error": [ + "Target entity urn:ngsi-ld:Device:HCMR-AQUABOX2 does not exist." + ] + } + ], + "success": [] + } + """.trimIndent() + ) + } + @Test fun `create batch entity should return a 400 if JSON-LD payload is not correct`() { + shouldReturn400WithBadPayload("create") + } + + @Test + fun `upsert batch entity should return a 400 if JSON-LD payload is not correct`() { + shouldReturn400WithBadPayload("upsert") + } + + private fun shouldReturn400WithBadPayload(method: String) { val jsonLdFile = ClassPathResource("/ngsild/hcmr/HCMR_test_file_missing_context.json") webClient.post() - .uri("/ngsi-ld/v1/entityOperations/create") + .uri("/ngsi-ld/v1/entityOperations/" + method) .header( "Link", "; rel=http://www.w3.org/ns/json-ld#context; type=application/ld+json" @@ -199,4 +312,4 @@ class EntityOperationHandlerTests { """.trimIndent() ) } -} \ No newline at end of file +} diff --git a/entity-service/src/test/resources/ngsild/hcmr/HCMR_test_file_invalid_relation_update.json b/entity-service/src/test/resources/ngsild/hcmr/HCMR_test_file_invalid_relation_update.json new file mode 100644 index 000000000..8263d8623 --- /dev/null +++ b/entity-service/src/test/resources/ngsild/hcmr/HCMR_test_file_invalid_relation_update.json @@ -0,0 +1,36 @@ +[ + { + "id": "urn:ngsi-ld:Sensor:HCMR-AQUABOX1temperature", + "type": "Sensor", + "deviceParameter": { + "type": "Property", + "value": "temperature" + }, + "connectsTo": { + "type": "Relationship", + "object": "urn:ngsi-ld:Device:HCMR-AQUABOX2" + }, + "@context": [ + "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/shared-jsonld-contexts/egm.jsonld", + "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/aquac/jsonld-contexts/aquac.jsonld", + "http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" + ] + }, + { + "id": "urn:ngsi-ld:Sensor:HCMR-AQUABOX1dissolvedOxygen", + "type": "Sensor", + "deviceParameter": { + "type": "Property", + "value": "dissolvedOxygen" + }, + "connectsTo": { + "type": "Relationship", + "object": "urn:ngsi-ld:Device:HCMR-AQUABOX2" + }, + "@context": [ + "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/shared-jsonld-contexts/egm.jsonld", + "https://raw.githubusercontent.com/easy-global-market/ngsild-api-data-models/master/aquac/jsonld-contexts/aquac.jsonld", + "http://uri.etsi.org/ngsi-ld/v1/ngsi-ld-core-context.jsonld" + ] + } +] diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt b/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt index 6c5e8d642..7fa725c81 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeService.kt @@ -1,8 +1,8 @@ package com.egm.stellio.search.service import com.egm.stellio.search.model.AttributeInstance -import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.RawValue +import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalValue import com.egm.stellio.search.util.isAttributeOfMeasureType import com.egm.stellio.search.util.valueToDoubleOrNull @@ -63,7 +63,7 @@ class TemporalEntityAttributeService( fun createEntityTemporalReferences(payload: String): Mono { val entity = NgsiLdParsingUtils.parseEntity(payload) - val rawEntity = entity.attributes + val rawEntity = entity.rawJsonLdProperties val temporalProperties = rawEntity .filter { @@ -193,7 +193,7 @@ class TemporalEntityAttributeService( withTemporalValues: Boolean ): ExpandedEntity { - val entity = expandedEntity.attributes.toMutableMap() + val entity = expandedEntity.rawJsonLdProperties.toMutableMap() rawResults.filter { // filtering out empty lists or lists with an empty map of results diff --git a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt index 6e4bab887..9cccf832c 100644 --- a/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt +++ b/search-service/src/main/kotlin/com/egm/stellio/search/web/TemporalEntityHandler.kt @@ -2,10 +2,13 @@ package com.egm.stellio.search.web import com.egm.stellio.search.model.TemporalEntityAttribute import com.egm.stellio.search.model.TemporalQuery -import com.egm.stellio.search.service.TemporalEntityAttributeService import com.egm.stellio.search.service.AttributeInstanceService import com.egm.stellio.search.service.EntityService -import com.egm.stellio.shared.model.* +import com.egm.stellio.search.service.TemporalEntityAttributeService +import com.egm.stellio.shared.model.BadRequestDataException +import com.egm.stellio.shared.model.BadRequestDataResponse +import com.egm.stellio.shared.model.ExpandedEntity +import com.egm.stellio.shared.model.ResourceNotFoundException import com.egm.stellio.shared.util.* import com.egm.stellio.shared.util.ApiUtils.serializeObject import com.egm.stellio.shared.util.NgsiLdParsingUtils.NGSILD_CORE_CONTEXT @@ -21,7 +24,6 @@ import org.springframework.web.bind.annotation.* import reactor.core.publisher.Flux import reactor.core.publisher.Mono import reactor.kotlin.core.publisher.toMono -import java.lang.IllegalArgumentException import java.util.* @RestController @@ -128,7 +130,7 @@ class TemporalEntityHandler( temporalEntityAttribute.type != "https://uri.etsi.org/ngsi-ld/Subscription" -> Mono.just(parseEntity(temporalEntityAttribute.entityPayload)) else -> { val parsedEntity = parseEntity(temporalEntityAttribute.entityPayload, emptyList()) - Mono.just(ExpandedEntity(parsedEntity.attributes, listOf(NGSILD_CORE_CONTEXT))) + Mono.just(ExpandedEntity(parsedEntity.rawJsonLdProperties, listOf(NGSILD_CORE_CONTEXT))) } } } diff --git a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt index 767e9a7b0..db4ac4731 100644 --- a/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt +++ b/search-service/src/test/kotlin/com/egm/stellio/search/service/TemporalEntityAttributeServiceTests.kt @@ -95,7 +95,11 @@ class TemporalEntityAttributeServiceTests : TimescaleBasedTests() { ) val enrichedEntity = temporalEntityAttributeService.injectTemporalValues(rawEntity, rawResults, true) - val serializedEntity = JsonLdProcessor.compact(enrichedEntity.attributes, mapOf("@context" to enrichedEntity.contexts), JsonLdOptions()) + val serializedEntity = JsonLdProcessor.compact( + enrichedEntity.rawJsonLdProperties, + mapOf("@context" to enrichedEntity.contexts), + JsonLdOptions() + ) val finalEntity = JsonUtils.toPrettyString(serializedEntity) assertEquals(loadSampleData("expectations/beehive_with_incoming_temporal_values.jsonld").trim(), finalEntity) } @@ -119,7 +123,11 @@ class TemporalEntityAttributeServiceTests : TimescaleBasedTests() { ) val enrichedEntity = temporalEntityAttributeService.injectTemporalValues(rawEntity, rawResults, true) - val serializedEntity = JsonLdProcessor.compact(enrichedEntity.attributes, mapOf("@context" to enrichedEntity.contexts), JsonLdOptions()) + val serializedEntity = JsonLdProcessor.compact( + enrichedEntity.rawJsonLdProperties, + mapOf("@context" to enrichedEntity.contexts), + JsonLdOptions() + ) val finalEntity = JsonUtils.toPrettyString(serializedEntity) assertEquals(loadSampleData("expectations/subscription_with_notifications_temporal_values.jsonld").trim(), finalEntity) } @@ -145,7 +153,11 @@ class TemporalEntityAttributeServiceTests : TimescaleBasedTests() { ) val enrichedEntity = temporalEntityAttributeService.injectTemporalValues(rawEntity, rawResults, false) - val serializedEntity = JsonLdProcessor.compact(enrichedEntity.attributes, mapOf("@context" to enrichedEntity.contexts), JsonLdOptions()) + val serializedEntity = JsonLdProcessor.compact( + enrichedEntity.rawJsonLdProperties, + mapOf("@context" to enrichedEntity.contexts), + JsonLdOptions() + ) val finalEntity = JsonUtils.toPrettyString(serializedEntity) assertEquals(loadSampleData("expectations/subscription_with_notifications.jsonld").trim(), finalEntity) } @@ -156,7 +168,11 @@ class TemporalEntityAttributeServiceTests : TimescaleBasedTests() { val rawResults = emptyList>>() val enrichedEntity = temporalEntityAttributeService.injectTemporalValues(rawEntity, rawResults, true) - val serializedEntity = JsonLdProcessor.compact(enrichedEntity.attributes, mapOf("@context" to enrichedEntity.contexts), JsonLdOptions()) + val serializedEntity = JsonLdProcessor.compact( + enrichedEntity.rawJsonLdProperties, + mapOf("@context" to enrichedEntity.contexts), + JsonLdOptions() + ) val finalEntity = JsonUtils.toPrettyString(serializedEntity) assertEquals(loadSampleData("subscription.jsonld").trim(), finalEntity) } @@ -167,7 +183,11 @@ class TemporalEntityAttributeServiceTests : TimescaleBasedTests() { val rawResults = listOf(listOf(emptyMap())) val enrichedEntity = temporalEntityAttributeService.injectTemporalValues(rawEntity, rawResults, true) - val serializedEntity = JsonLdProcessor.compact(enrichedEntity.attributes, mapOf("@context" to enrichedEntity.contexts), JsonLdOptions()) + val serializedEntity = JsonLdProcessor.compact( + enrichedEntity.rawJsonLdProperties, + mapOf("@context" to enrichedEntity.contexts), + JsonLdOptions() + ) val finalEntity = JsonUtils.toPrettyString(serializedEntity) assertEquals(loadSampleData("subscription.jsonld").trim(), finalEntity) } diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedEntity.kt b/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedEntity.kt index cdf495b46..ca32a2d09 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedEntity.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/model/ExpandedEntity.kt @@ -8,7 +8,7 @@ import com.github.jsonldjava.core.JsonLdOptions import com.github.jsonldjava.core.JsonLdProcessor class ExpandedEntity private constructor( - val attributes: Map, + val rawJsonLdProperties: Map, val contexts: List ) { companion object { @@ -23,23 +23,29 @@ class ExpandedEntity private constructor( } } - val id = attributes["@id"]!! as String - val type = (attributes["@type"]!! as List)[0] + val id = rawJsonLdProperties[NgsiLdParsingUtils.NGSILD_ENTITY_ID]!! as String + val type = (rawJsonLdProperties[NgsiLdParsingUtils.NGSILD_ENTITY_TYPE]!! as List)[0] val relationships by lazy { getAttributesOfType(NGSILD_RELATIONSHIP_TYPE) } val properties by lazy { getAttributesOfType(NGSILD_PROPERTY_TYPE) } + val attributes by lazy { initAttributesWithoutTypeAndId() } fun compact(): Map = - JsonLdProcessor.compact(attributes, mapOf("@context" to contexts), JsonLdOptions()) + JsonLdProcessor.compact(rawJsonLdProperties, mapOf("@context" to contexts), JsonLdOptions()) private fun getAttributesOfType(type: AttributeType): Map>> = - attributes.filterKeys { - !listOf(NgsiLdParsingUtils.NGSILD_ENTITY_ID, NgsiLdParsingUtils.NGSILD_ENTITY_TYPE).contains(it) - }.mapValues { + attributes.mapValues { NgsiLdParsingUtils.expandValueAsMap(it.value) }.filter { NgsiLdParsingUtils.isAttributeOfType(it.value, type) } + private fun initAttributesWithoutTypeAndId(): Map { + val idAndTypeKeys = listOf(NgsiLdParsingUtils.NGSILD_ENTITY_ID, NgsiLdParsingUtils.NGSILD_ENTITY_TYPE) + return rawJsonLdProperties.filterKeys { + !idAndTypeKeys.contains(it) + } + } + /** * Gets linked entities ids. * Entities can be linked either by a relation or a property. diff --git a/shared/src/main/kotlin/com/egm/stellio/shared/util/NgsiLdParsingUtils.kt b/shared/src/main/kotlin/com/egm/stellio/shared/util/NgsiLdParsingUtils.kt index c1052f4ff..cca60cf58 100644 --- a/shared/src/main/kotlin/com/egm/stellio/shared/util/NgsiLdParsingUtils.kt +++ b/shared/src/main/kotlin/com/egm/stellio/shared/util/NgsiLdParsingUtils.kt @@ -1,6 +1,9 @@ package com.egm.stellio.shared.util -import com.egm.stellio.shared.model.* +import com.egm.stellio.shared.model.BadRequestDataException +import com.egm.stellio.shared.model.EntityEvent +import com.egm.stellio.shared.model.ExpandedEntity +import com.egm.stellio.shared.model.Observation import com.fasterxml.jackson.annotation.JsonInclude import com.fasterxml.jackson.databind.JsonNode import com.fasterxml.jackson.databind.ObjectMapper @@ -340,7 +343,7 @@ object NgsiLdParsingUtils { fun getLocationFromEntity(parsedEntity: ExpandedEntity): Map? { try { - val location = expandValueAsMap(parsedEntity.attributes[NGSILD_LOCATION_PROPERTY]!!) + val location = expandValueAsMap(parsedEntity.rawJsonLdProperties[NGSILD_LOCATION_PROPERTY]!!) val locationValue = expandValueAsMap(location[NGSILD_GEOPROPERTY_VALUE]!!) val geoPropertyType = locationValue["@type"]!![0] as String val geoPropertyValue = locationValue[NGSILD_COORDINATES_PROPERTY]!!