Merge pull request #193 from catenax-ng/feat/bridge_pagination

Bridge: use pagination for querying

bpdm-bridge-dummy/src/main/kotlin/com/catenax/bpdm/bridge/dummy/config/BridgeConfigProperties.kt

@@ -0,0 +1,29 @@
+/*******************************************************************************
+ * Copyright (c) 2021,2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ******************************************************************************/
+
+package com.catenax.bpdm.bridge.dummy.config
+
+import org.springframework.boot.context.properties.ConfigurationProperties
+
+
+@ConfigurationProperties(prefix = "bpdm.bridge")
+data class BridgeConfigProperties(
+    // Page size for querying Gate
+    val queryPageSize: Int = 100
+)

bpdm-bridge-dummy/src/main/kotlin/com/catenax/bpdm/bridge/dummy/service/GateQueryService.kt

@@ -19,6 +19,7 @@
 
 package com.catenax.bpdm.bridge.dummy.service
 
+import com.catenax.bpdm.bridge.dummy.config.BridgeConfigProperties
 import com.catenax.bpdm.bridge.dummy.dto.GateAddressInfo
 import com.catenax.bpdm.bridge.dummy.dto.GateLegalEntityInfo
 import com.catenax.bpdm.bridge.dummy.dto.GateSiteInfo
@@ -27,28 +28,39 @@ import org.eclipse.tractusx.bpdm.common.dto.request.PaginationRequest
 import org.eclipse.tractusx.bpdm.gate.api.client.GateClient
 import org.eclipse.tractusx.bpdm.gate.api.model.AddressGateInputResponse
 import org.eclipse.tractusx.bpdm.gate.api.model.LegalEntityGateInputResponse
+import org.eclipse.tractusx.bpdm.gate.api.model.SharingStateDto
 import org.eclipse.tractusx.bpdm.gate.api.model.SiteGateInputResponse
 import org.eclipse.tractusx.bpdm.gate.api.model.request.PaginationStartAfterRequest
+import org.eclipse.tractusx.bpdm.gate.api.model.response.ChangelogResponse
 import org.eclipse.tractusx.bpdm.gate.api.model.response.LsaType
 import org.springframework.stereotype.Service
 import java.time.Instant
 
 @Service
 class GateQueryService(
-    val gateClient: GateClient
+    val gateClient: GateClient,
+    val bridgeConfigProperties: BridgeConfigProperties
 ) {
 
     private val logger = KotlinLogging.logger { }
 
     fun getChangedExternalIdsByLsaType(modifiedAfter: Instant?): Map<LsaType, Set<String>> {
-        // TODO use pagination properly
-        val entriesGate = gateClient.changelog().getChangelogEntriesLsaType(
-            lsaType = null,
-            fromTime = modifiedAfter,
-            paginationRequest = PaginationRequest(0, 100)
-        )
-
-        return entriesGate.content
+        var page = 0
+        var totalPages: Int
+        val content = mutableListOf<ChangelogResponse>()
+
+        do {
+            val pageResponse = gateClient.changelog().getChangelogEntriesLsaType(
+                lsaType = null,
+                fromTime = modifiedAfter,
+                paginationRequest = PaginationRequest(page, bridgeConfigProperties.queryPageSize)
+            )
+            page++
+            totalPages = pageResponse.totalPages
+            content.addAll(pageResponse.content)
+        } while (page < totalPages)
+
+        return content
             .groupBy { it.businessPartnerType }
             .mapValues { (_, list) -> list.map { it.externalId }.toSet() }
             .also {
@@ -107,13 +119,23 @@ class GateQueryService(
         if (externalIds.isEmpty()) {
             return emptyMap()
         }
-        // TODO use pagination properly
-        val page = gateClient.sharingState().getSharingStates(
-            lsaType = lsaType,
-            externalIds = externalIds,
-            paginationRequest = PaginationRequest(0, 100)
-        )
-        return page.content
+
+        var page = 0
+        var totalPages: Int
+        val content = mutableListOf<SharingStateDto>()
+
+        do {
+            val pageResponse = gateClient.sharingState().getSharingStates(
+                lsaType = lsaType,
+                externalIds = externalIds,
+                paginationRequest = PaginationRequest(page, bridgeConfigProperties.queryPageSize)
+            )
+            page++
+            totalPages = pageResponse.totalPages
+            content.addAll(pageResponse.content)
+        } while (page < totalPages)
+
+        return content
             .associateBy { it.externalId }
             .filter { it.value.bpn != null }
             .mapValues { it.value.bpn!! }
@@ -123,39 +145,69 @@ class GateQueryService(
         if (externalIds.isEmpty()) {
             return emptyList()
         }
-        // TODO use pagination properly
-        val response = gateClient.legalEntities().getLegalEntitiesByExternalIds(
-            externalIds = externalIds,
-            paginationRequest = PaginationStartAfterRequest(null, 100)
-        )
-        logger.info { "Gate returned ${response.content.size} valid legal entities, ${response.invalidEntries} were invalid" }
-        return response.content
+
+        var pageStartAfter: String? = null
+        val validContent = mutableListOf<LegalEntityGateInputResponse>()
+        var invalidEntries = 0
+
+        do {
+            val pageResponse = gateClient.legalEntities().getLegalEntitiesByExternalIds(
+                externalIds = externalIds,
+                paginationRequest = PaginationStartAfterRequest(pageStartAfter, bridgeConfigProperties.queryPageSize)
+            )
+            pageStartAfter = pageResponse.nextStartAfter
+            validContent.addAll(pageResponse.content)
+            invalidEntries += pageResponse.invalidEntries
+        } while (pageStartAfter != null)
+
+        logger.info { "Gate returned ${validContent.size} valid legal entities, $invalidEntries were invalid" }
+        return validContent
     }
 
     private fun getSitesInput(externalIds: Set<String>): Collection<SiteGateInputResponse> {
         if (externalIds.isEmpty()) {
             return emptyList()
         }
-        // TODO use pagination properly
-        val response = gateClient.sites().getSitesByExternalIds(
-            externalIds = externalIds,
-            paginationRequest = PaginationStartAfterRequest(null, 100)
-        )
-        logger.info { "Gate returned ${response.content.size} valid sites, ${response.invalidEntries} were invalid" }
-        return response.content
+
+        var pageStartAfter: String? = null
+        val validContent = mutableListOf<SiteGateInputResponse>()
+        var invalidEntries = 0
+
+        do {
+            val pageResponse = gateClient.sites().getSitesByExternalIds(
+                externalIds = externalIds,
+                paginationRequest = PaginationStartAfterRequest(pageStartAfter, bridgeConfigProperties.queryPageSize)
+            )
+            pageStartAfter = pageResponse.nextStartAfter
+            validContent.addAll(pageResponse.content)
+            invalidEntries += pageResponse.invalidEntries
+        } while (pageStartAfter != null)
+
+        logger.info { "Gate returned ${validContent.size} valid sites, $invalidEntries were invalid" }
+        return validContent
     }
 
     private fun getAddressesInput(externalIds: Set<String>): Collection<AddressGateInputResponse> {
         if (externalIds.isEmpty()) {
             return emptyList()
         }
-        // TODO use pagination properly
-        val response = gateClient.addresses().getAddressesByExternalIds(
-            externalIds = externalIds,
-            paginationRequest = PaginationStartAfterRequest(null, 100)
-        )
-        logger.info { "Gate returned ${response.content.size} valid addresses, ${response.invalidEntries} were invalid" }
-        return response.content
+
+        var pageStartAfter: String? = null
+        val validContent = mutableListOf<AddressGateInputResponse>()
+        var invalidEntries = 0
+
+        do {
+            val pageResponse = gateClient.addresses().getAddressesByExternalIds(
+                externalIds = externalIds,
+                paginationRequest = PaginationStartAfterRequest(pageStartAfter, bridgeConfigProperties.queryPageSize)
+            )
+            pageStartAfter = pageResponse.nextStartAfter
+            validContent.addAll(pageResponse.content)
+            invalidEntries += pageResponse.invalidEntries
+        } while (pageStartAfter != null)
+
+        logger.info { "Gate returned ${validContent.size} valid addresses, $invalidEntries were invalid" }
+        return validContent
     }
 
 }

bpdm-bridge-dummy/src/main/resources/application.properties

@@ -20,12 +20,15 @@ [email protected]@
 bpdm.description=@project.description@
 bpdm.version=@project.version@
 ##
-#Change default port
+# Change default port
 server.port=8083
 ##
-#Logging Configuration
+# Logging Configuration
 bpdm.logging.unknown-user=Anonymous
 ##
+# Page size for querying Gate
+bpdm.bridge.query-page-size=100
+##
 # Connection to Pool and Gate
 bpdm.pool.base-url=http://localhost:8080/
 bpdm.gate.base-url=http://localhost:8081/