feat(SaaS-Import): add endpoints to query import identifiers mappings

* Endpoint for paginating the mappings between import identifier and BPN
* Endpoint for filtering BPN by import identifier

bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/saas/config/SaasAdapterConfigProperties.kt

@@ -36,6 +36,8 @@ class SaasAdapterConfigProperties(
     val addressType: String = "BP_ADDRESS",
     val parentRelationType: String = "PARENT",
     val bpnKey: String = "CX_BPN",
+    val treatInvalidBpnAsNew: Boolean = false,
+    val requestSizeLimit: Int = 500
 ) {
     private val exchangeApiUrl: String = "data-exchange/rest/v4"
     private val referenceApiUrl: String = "referencedata/rest/v3"

bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/saas/controller/SaasController.kt

@@ -23,14 +23,23 @@ import io.swagger.v3.oas.annotations.Operation
 import io.swagger.v3.oas.annotations.media.Content
 import io.swagger.v3.oas.annotations.responses.ApiResponse
 import io.swagger.v3.oas.annotations.responses.ApiResponses
+import org.eclipse.tractusx.bpdm.common.dto.response.PageResponse
+import org.eclipse.tractusx.bpdm.pool.component.saas.config.SaasAdapterConfigProperties
+import org.eclipse.tractusx.bpdm.pool.component.saas.dto.ImportIdEntry
+import org.eclipse.tractusx.bpdm.pool.component.saas.dto.ImportIdFilterRequest
+import org.eclipse.tractusx.bpdm.pool.component.saas.dto.ImportIdMappingResponse
 import org.eclipse.tractusx.bpdm.pool.component.saas.service.ImportStarterService
+import org.eclipse.tractusx.bpdm.pool.dto.request.PaginationRequest
 import org.eclipse.tractusx.bpdm.pool.dto.response.SyncResponse
+import org.eclipse.tractusx.bpdm.pool.exception.BpdmRequestSizeException
+import org.springdoc.core.annotations.ParameterObject
 import org.springframework.web.bind.annotation.*
 
 @RestController
 @RequestMapping("/api/saas")
 class SaasController(
-    val partnerImportService: ImportStarterService
+    private val partnerImportService: ImportStarterService,
+    private val adapterConfigProperties: SaasAdapterConfigProperties
 ) {
     @Operation(
         summary = "Import new business partner records from SaaS",
@@ -64,4 +73,38 @@ class SaasController(
     fun getSyncStatus(): SyncResponse {
         return partnerImportService.getImportStatus()
     }
+
+    @Operation(
+        summary = "Filter Identifier Mappings by CX-Pool Identifiers",
+        description = "Specify a range of CX-Pool Identifiers to get the corresponding mapping to their Business Partner Numbers"
+    )
+    @ApiResponses(
+        value = [
+            ApiResponse(responseCode = "200", description = "The found import identifier mappings"),
+            ApiResponse(responseCode = "400", description = "On malformed requests or exceeding the request size of \${bpdm.saas.request-size-limit}"),
+        ]
+    )
+    @PostMapping("/identifier-mappings/filter")
+    fun getImportEntries(@RequestBody importIdFilterRequest: ImportIdFilterRequest): ImportIdMappingResponse {
+        if(importIdFilterRequest.importIdentifiers.size > adapterConfigProperties.requestSizeLimit)
+             BpdmRequestSizeException(importIdFilterRequest.importIdentifiers.size, adapterConfigProperties.requestSizeLimit)
+        return partnerImportService.getImportIdEntries(importIdFilterRequest.importIdentifiers)
+    }
+
+    @Operation(
+        summary = "Paginate Identifier Mappings by CX-Pool Identifiers",
+        description = "Paginate through all CX-Pool Identifier and Business Partner Number mappings."
+    )
+    @ApiResponses(
+        value = [
+            ApiResponse(responseCode = "200", description = "The found import identifier mappings"),
+            ApiResponse(responseCode = "400", description = "On malformed requests or exceeding the request size of \${bpdm.saas.request-size-limit}"),
+        ]
+    )
+    @GetMapping("/identifier-mappings")
+    fun getImportEntries(@ParameterObject paginationRequest: PaginationRequest): PageResponse<ImportIdEntry> {
+        if(paginationRequest.size > adapterConfigProperties.requestSizeLimit)
+            BpdmRequestSizeException(paginationRequest.size, adapterConfigProperties.requestSizeLimit)
+        return partnerImportService.getImportIdEntries(paginationRequest)
+    }
 }

bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/saas/dto/ImportIdEntriesResponse.kt → bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/saas/dto/ImportIdFilterRequest.kt

@@ -19,8 +19,6 @@
 
 package org.eclipse.tractusx.bpdm.pool.component.saas.dto
 
-data class ImportIdEntriesResponse(
-    val entries: Collection<ImportIdEntry>
-) {
-    val size = entries.size
-}
+data class ImportIdFilterRequest(
+    val importIdentifiers: Collection<String> = emptyList()
+)

bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/saas/dto/ImportIdMappingResponse.kt

@@ -0,0 +1,27 @@
+/*******************************************************************************
+ * Copyright (c) 2021,2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package org.eclipse.tractusx.bpdm.pool.component.saas.dto
+
+data class ImportIdMappingResponse(
+    val entries: Collection<ImportIdEntry>,
+    val notFound: Collection<String>
+) {
+    val size = entries.size
+}

bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/saas/service/ImportStarterService.kt

@@ -20,13 +20,18 @@
 package org.eclipse.tractusx.bpdm.pool.component.saas.service
 
 import mu.KotlinLogging
-import org.eclipse.tractusx.bpdm.pool.component.saas.dto.ImportIdEntriesResponse
+import org.eclipse.tractusx.bpdm.common.dto.response.PageResponse
+import org.eclipse.tractusx.bpdm.pool.component.saas.dto.ImportIdMappingResponse
 import org.eclipse.tractusx.bpdm.pool.component.saas.dto.ImportIdEntry
+import org.eclipse.tractusx.bpdm.pool.dto.request.PaginationRequest
 import org.eclipse.tractusx.bpdm.pool.dto.response.SyncResponse
 import org.eclipse.tractusx.bpdm.pool.entity.SyncType
 import org.eclipse.tractusx.bpdm.pool.repository.ImportEntryRepository
 import org.eclipse.tractusx.bpdm.pool.service.SyncRecordService
 import org.eclipse.tractusx.bpdm.pool.service.toDto
+import org.springframework.data.domain.Page
+import org.springframework.data.domain.PageRequest
+import org.springframework.data.domain.Pageable
 import org.springframework.scheduling.annotation.Scheduled
 import org.springframework.stereotype.Service
 
@@ -64,12 +69,26 @@ class ImportStarterService(
         return syncRecordService.getOrCreateRecord(SyncType.SAAS_IMPORT).toDto()
     }
 
-    fun getImportIdEntries(importIdentifiers: Collection<String>): ImportIdEntriesResponse {
-        return ImportIdEntriesResponse(
-            importEntryRepository.findByImportIdentifierIn(importIdentifiers).map { ImportIdEntry(it.importIdentifier, it.bpn) }
-        )
+    /**
+     * Filter import entries by the given [importIdentifiers]
+     */
+    fun getImportIdEntries(importIdentifiers: Collection<String>): ImportIdMappingResponse {
+        val foundEntries = importEntryRepository.findByImportIdentifierIn(importIdentifiers).map { ImportIdEntry(it.importIdentifier, it.bpn) }
+        val missingEntries = importIdentifiers.minus(foundEntries.map { it.importId }.toSet())
+
+        return ImportIdMappingResponse(foundEntries, missingEntries)
     }
 
+    /**
+     * Paginate over import entries by [paginationRequest]
+     */
+    fun getImportIdEntries(paginationRequest: PaginationRequest): PageResponse<ImportIdEntry> {
+        val entriesPage = importEntryRepository.findAll(PageRequest.of(paginationRequest.page, paginationRequest.size))
+        return entriesPage.toDto(entriesPage.content.map { ImportIdEntry(it.importIdentifier, it.bpn) })
+    }
+
+
+
     private fun startImport(inSync: Boolean): SyncResponse {
         val record = syncRecordService.setSynchronizationStart(SyncType.SAAS_IMPORT)
         logger.debug { "Initializing SaaS import starting with ID ${record.errorSave}' for modified records from '${record.fromTime}' with async: ${!inSync}" }

bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/component/saas/service/PartnerImportPageService.kt

@@ -29,7 +29,10 @@ import org.eclipse.tractusx.bpdm.pool.dto.response.AddressPartnerCreateResponse
 import org.eclipse.tractusx.bpdm.pool.dto.response.LegalEntityPartnerCreateResponse
 import org.eclipse.tractusx.bpdm.pool.dto.response.SitePartnerCreateResponse
 import org.eclipse.tractusx.bpdm.pool.entity.ImportEntry
+import org.eclipse.tractusx.bpdm.pool.repository.AddressPartnerRepository
 import org.eclipse.tractusx.bpdm.pool.repository.ImportEntryRepository
+import org.eclipse.tractusx.bpdm.pool.repository.LegalEntityRepository
+import org.eclipse.tractusx.bpdm.pool.repository.SiteRepository
 import org.eclipse.tractusx.bpdm.pool.service.BusinessPartnerBuildService
 import org.eclipse.tractusx.bpdm.pool.service.MetadataService
 import org.springframework.data.domain.Pageable
@@ -44,7 +47,10 @@ class PartnerImportPageService(
     private val mappingService: SaasToRequestMapper,
     private val businessPartnerBuildService: BusinessPartnerBuildService,
     private val importEntryRepository: ImportEntryRepository,
-    private val saasClient: SaasClient
+    private val saasClient: SaasClient,
+    private val legalEntityRepository: LegalEntityRepository,
+    private val siteRepository: SiteRepository,
+    private val addressPartnerRepository: AddressPartnerRepository
 ) {
     private val logger = KotlinLogging.logger { }
 
@@ -65,12 +71,10 @@ class PartnerImportPageService(
         addNewMetadata(partnerCollection.values)
 
         val validPartners = partnerCollection.values.filter { isValid(it) }
+        val (noBpn, validBpn) = partitionHasNoBpnAndValidBpn(validPartners)
 
-        addMissingImportEntries(validPartners)
-        val (partnersWithBpn, partnersWithoutBpn) = partitionHasImportEntry(validPartners)
-
-        val (createdLegalEntities, createdSites, createdAddresses) = createPartners(partnersWithoutBpn)
-        val (updatedLegalEntities, updatedSites, updatedAddresses) = updatePartners(partnersWithBpn)
+        val (createdLegalEntities, createdSites, createdAddresses) = createPartners(noBpn)
+        val (updatedLegalEntities, updatedSites, updatedAddresses) = updatePartners(validBpn)
 
         return ImportResponsePage(
             partnerCollection.total,
@@ -163,28 +167,35 @@ class PartnerImportPageService(
         val parents = saasClient.readBusinessPartnersByExternalIds(childrenWithParentId.map { it.parentId }).values
         val validParents = filterValidRelations(parents.filter { isValid(it) }, childrenWithParentId)
 
-        addMissingImportEntries(validParents)
-        val (parentsWithBpn, parentsWithoutBpn) = partitionHasImportEntry(validParents)
-
-        val (newLegalEntities, newSites, _) = createPartners(parentsWithoutBpn)
-
-        val parentIdToBpn = newLegalEntities.map { Pair(it.index, it.bpn) }
-            .plus(newSites.map { Pair(it.index, it.bpn) })
-            .plus(parentsWithBpn.map { Pair(it.partner.externalId!!, it.bpn) })
-            .toMap()
-
+        val parentsByImportId = determineParentBPNs(validParents).associateBy { it.partner.externalId!! }
 
         return childrenWithParentId.mapNotNull { childWithParentId ->
-            val parentBpn = parentIdToBpn[childWithParentId.parentId]
-            if (parentBpn != null) {
-                BusinessPartnerWithParentBpn(childWithParentId.partner, parentBpn)
+            val parent = parentsByImportId[childWithParentId.parentId]
+            if (parent != null) {
+                BusinessPartnerWithParentBpn(childWithParentId.partner, parent.bpn)
             } else {
                 logger.warn { "Can not resolve parent with Import-ID ${childWithParentId.parentId} for SaaS record with ID ${childWithParentId.partner.id}" }
                 null
             }
         }
     }
 
+    private fun determineParentBPNs(parents: Collection<BusinessPartnerSaas>): Collection<BusinessPartnerWithBpn>{
+        val parentByImportId = parents.associateBy { it.externalId!! }
+
+        val (parentsWithoutBpn, parentsWithBpn) = partitionHasNoBpnAndValidBpn(parents)
+
+        //create missing parents in the Pool
+        val (newLegalEntities, newSites, _) = createPartners(parentsWithoutBpn)
+
+        val createdParents = newLegalEntities.map { Pair(parentByImportId[it.index], it.bpn) }
+            .plus(newSites.map { Pair(parentByImportId[it.index], it.bpn) })
+            .filter { (parent, _) -> parent != null }
+            .map { BusinessPartnerWithBpn(it.first!!, it.second) }
+
+        return parentsWithBpn + createdParents
+    }
+
     private fun determineParentId(children: Collection<BusinessPartnerSaas>): Collection<BusinessPartnerWithParentId> {
         return children.mapNotNull { child ->
             val parentId = child.relations.firstOrNull { id -> id.type?.technicalKey == adapterProperties.parentRelationType }?.startNode
@@ -247,7 +258,37 @@ class PartnerImportPageService(
         return type
     }
 
-    private fun partitionHasBpn(partners: Collection<BusinessPartnerSaas>): Pair<Collection<BusinessPartnerWithBpn>, Collection<BusinessPartnerSaas>> {
+    /**
+     * Partition business partner collection into records for which no BPN can be retrieved and records which have a BPN that also is found in the Pool
+     * BPN is determined in two priorities:
+     *      1. Try to get BPN from import entry
+     *      2. Try to get BPN from record in identifiers and check whether this BPN actually exists in the Pool (valid BPN)
+     * If we encounter valid BPNs with no import entry we create an import entry for it (as self correction logic)
+     * Optionally, depending on the configuration we either ignore records with invalid BPNs or we treat them as having no BPN
+     */
+    private fun partitionHasNoBpnAndValidBpn(partners: Collection<BusinessPartnerSaas>): Pair<Collection<BusinessPartnerSaas>, Collection<BusinessPartnerWithBpn>>{
+        //search BPN in import entry based on CX-Pool identifier
+        val (withEntry, withoutEntry) = partitionHasImportEntry(partners)
+        //if no entry has been found look for BPN in identifiers of records
+        val (hasBpnIdentifier, hasNoBpnIdentifier) = partitionContainsBpnIdentifier(withoutEntry)
+        //if BPN is identifiers but no import record exists, check whether the BPN is known to the BPDM Pool
+        val (bpnFound, bpnMissing) = partitionBpnFound(hasBpnIdentifier)
+
+        val consequence = if(adapterProperties.treatInvalidBpnAsNew) "Record will be treated as having no BPN." else "Record will be ignored."
+        bpnMissing.forEach {
+            logger.warn { "Business partner with Id ${it.partner.externalId} contains BPN ${it.bpn} but such BPN can't be found in the Pool." + consequence }
+        }
+
+        val hasValidBpn = withEntry + bpnFound
+        val hasNoBpn = if(adapterProperties.treatInvalidBpnAsNew) hasNoBpnIdentifier + bpnMissing.map { it.partner } else hasNoBpnIdentifier
+
+        //Create missing import entries for records which have known BPNs
+        importEntryRepository.saveAll(bpnFound.map { ImportEntry(it.partner.externalId!!, it.bpn) })
+
+        return Pair(hasNoBpn, hasValidBpn)
+    }
+
+    private fun partitionContainsBpnIdentifier(partners: Collection<BusinessPartnerSaas>): Pair<Collection<BusinessPartnerWithBpn>, Collection<BusinessPartnerSaas>> {
         val (hasBpn, nopBpn) = partners
             .map { Pair(it, it.extractId(adapterProperties.bpnKey)) }
             .partition { (_, bpn) -> bpn != null }
@@ -270,15 +311,38 @@ class PartnerImportPageService(
         )
     }
 
-    private fun addMissingImportEntries(partners: Collection<BusinessPartnerSaas>) {
-        val (hasBpn, _) = partitionHasBpn(partners)
-        val partnersByImportId = hasBpn.associateBy { it.partner.externalId!! }
-        val existingImportIds = importEntryRepository.findByImportIdentifierIn(partnersByImportId.keys).map { it.importIdentifier }.toSet()
+    private fun partitionBpnFound(partners: Collection<BusinessPartnerWithBpn>): Pair<Collection<BusinessPartnerWithBpn>, Collection<BusinessPartnerWithBpn>> {
+
+        val partnersByBpn = partners.associateBy { it.bpn }
+        val (bpnLs, bpnSs, bpnAs) = partitionLSA(partnersByBpn.keys)
 
-        val missingPartners = partnersByImportId.minus(existingImportIds)
-        val missingEntries = missingPartners.entries.map { ImportEntry(it.key, it.value.bpn) }
+        val foundBpnLs = legalEntityRepository.findDistinctByBpnIn(bpnLs).map { it.bpn }
+        val foundBpnSs = siteRepository.findDistinctByBpnIn(bpnSs).map { it.bpn }
+        val foundBpnAs = addressPartnerRepository.findDistinctByBpnIn(bpnAs).map { it.bpn }
+
+        val foundBpns = foundBpnLs + foundBpnSs + foundBpnAs
+        val bpnMissing = partnersByBpn - foundBpns.toSet()
+        val bpnExists = partnersByBpn - bpnMissing.keys
+
+        return Pair(bpnExists.values, bpnMissing.values)
+    }
+
+    private fun partitionLSA(bpns: Collection<String>): Triple<Collection<String>, Collection<String>, Collection<String>>{
+        val bpnLs = mutableListOf<String>()
+        val bpnSs = mutableListOf<String>()
+        val bpnAs = mutableListOf<String>()
+
+        bpns.map { it.uppercase()}.forEach {
+            when(it.take(4))
+            {
+                "BPNL" -> bpnLs.add(it)
+                "BPNS" -> bpnSs.add(it)
+                "BPNA" -> bpnAs.add(it)
+                else -> logger.warn { "Encountered non-valid BPN: $it" }
+            }
+        }
 
-        importEntryRepository.saveAll(missingEntries)
+        return Triple(bpnLs, bpnSs, bpnAs)
     }
 
     private fun filterValidRelations(

bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/entity/ImportEntry.kt

@@ -33,7 +33,7 @@ import jakarta.persistence.Table
     ]
 )
 class ImportEntry(
-    @Column(name = "import_id", nullable = false)
+    @Column(name = "import_id", nullable = false, unique = true)
     var importIdentifier: String,
     @Column(name = "bpn", nullable = false)
     var bpn: String

bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/exception/BpdmRequestSizeException.kt

@@ -0,0 +1,31 @@
+/*******************************************************************************
+ * Copyright (c) 2021,2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package org.eclipse.tractusx.bpdm.pool.exception
+
+import org.springframework.http.HttpStatus
+import org.springframework.web.bind.annotation.ResponseStatus
+
+@ResponseStatus(HttpStatus.BAD_REQUEST)
+class BpdmRequestSizeException(
+    val actualSize: Int,
+    val maxSize: Int
+            ): RuntimeException("Request size of $actualSize exceeds maximum size of $maxSize") {
+
+}

bpdm-pool/src/main/kotlin/org/eclipse/tractusx/bpdm/pool/repository/ImportEntryRepository.kt

@@ -20,9 +20,11 @@
 package org.eclipse.tractusx.bpdm.pool.repository
 
 import org.eclipse.tractusx.bpdm.pool.entity.ImportEntry
+import org.eclipse.tractusx.bpdm.pool.entity.LegalEntity
 import org.springframework.data.repository.CrudRepository
+import org.springframework.data.repository.PagingAndSortingRepository
 
-interface ImportEntryRepository : CrudRepository<ImportEntry, Long> {
+interface ImportEntryRepository : CrudRepository<ImportEntry, Long>, PagingAndSortingRepository<ImportEntry, Long> {
 
     fun findByImportIdentifierIn(importIdentifier: Collection<String>): Set<ImportEntry>
 }