From 6665430e6b3eb152a6fb0dd31ea7385d78334bcb Mon Sep 17 00:00:00 2001 From: adityagajbhiye9 <133367448+adityagajbhiye9@users.noreply.github.com> Date: Fri, 8 Sep 2023 16:03:08 +0530 Subject: [PATCH 01/23] Update Dockerfile --- build/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 07f2d414f..78fb00f51 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -53,5 +53,5 @@ COPY --from=build modules/sde-core/target/*.jar ./app.jar EXPOSE 8080 # set the startup command to run your binary - -CMD ["java", "-jar", "./app.jar","--spring.config.location=file:/app/configuration.properties"] +CMD ["java", "-jar", "./app.jar"] +#CMD ["java", "-jar", "./app.jar","--spring.config.location=file:/app/configuration.properties"] From 1561dd7a6d04c5e03134ee3b47df74646f07f3fe Mon Sep 17 00:00:00 2001 From: ChetanT-System Date: Sun, 10 Sep 2023 17:07:35 +0530 Subject: [PATCH 02/23] - Changes updated for consumer download history --- .../sde/core/service/ConsumerService.java | 120 ++++++++++++++---- .../edc/model/request/ConsumerRequest.java | 2 +- .../request/{OfferRequest.java => Offer.java} | 7 +- .../services/ConsumerControlPanelService.java | 49 +++---- 4 files changed, 127 insertions(+), 51 deletions(-) rename modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/{OfferRequest.java => Offer.java} (89%) diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java index 5288be266..23ee436c7 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java @@ -23,18 +23,25 @@ import static org.springframework.http.ResponseEntity.ok; import java.io.ByteArrayInputStream; +import java.io.IOException; import java.io.PrintWriter; +import java.time.LocalDateTime; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.Optional; import java.util.TreeMap; import java.util.UUID; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; import org.eclipse.tractusx.sde.common.model.Submodel; +import org.eclipse.tractusx.sde.core.failurelog.mapper.DownloadHistoryMapper; +import org.eclipse.tractusx.sde.core.failurelog.repository.ConsumerDownloadHistoryRepository; +import org.eclipse.tractusx.sde.core.processreport.model.ConsumerDownloadHistory; +import org.eclipse.tractusx.sde.core.role.entity.ConsumerDownloadHistoryEntity; import org.eclipse.tractusx.sde.core.utils.CsvUtil; import org.eclipse.tractusx.sde.edc.model.request.ConsumerRequest; import org.eclipse.tractusx.sde.edc.services.ConsumerControlPanelService; @@ -58,48 +65,96 @@ public class ConsumerService { private final SubmodelOrchestartorService submodelOrchestartorService; + private final ConsumerDownloadHistoryRepository consumerDownloadHistoryRepository; + ObjectMapper mapper = new ObjectMapper(); + private DownloadHistoryMapper downloadHistoryMapper; + @SneakyThrows public void subscribeAndDownloadDataOffers(ConsumerRequest consumerRequest, HttpServletResponse response) { - Map subscribeAndDownloadDataOffers = consumerControlPanelService - .subscribeAndDownloadDataOffers(consumerRequest); + ConsumerDownloadHistory consumerDownloadHistory = new ConsumerDownloadHistory(); + consumerDownloadHistory.setStartDate(LocalDateTime.now().toString()); String processId = UUID.randomUUID().toString(); - Map>> csvWithValue = new TreeMap<>(); - - subscribeAndDownloadDataOffers.entrySet().stream().forEach(entry -> { - - JsonNode node = mapper.convertValue(entry.getValue(), JsonNode.class); - JsonNode status = node.get("status"); - JsonNode csv = node.get("csv"); - if (status == null && csv != null) { - - List csvHeader = new ArrayList<>(); - List csvValues = new ArrayList<>(); + Map subscribeAndDownloadDataOffers = consumerControlPanelService + .subscribeAndDownloadDataOffers(consumerRequest); - JsonNode jsonNode = node.get("csv"); - jsonNode.fieldNames().forEachRemaining(csvHeader::add); - jsonNode.fields().forEachRemaining(obje -> csvValues.add(obje.getValue().asText())); + int failedCount = 0; + int successCount = 0; + consumerDownloadHistory.setConnectorId(consumerRequest.getConnectorId()); + consumerDownloadHistory.setProviderUrl(consumerRequest.getProviderUrl()); + consumerDownloadHistory.setNumberOfItems(consumerRequest.getOffers().size()); + consumerDownloadHistory.setProcessId(processId); - Submodel findSubmodel = submodelOrchestartorService.findSubmodel(csvHeader); - List> csvValueList = csvWithValue.get(findSubmodel.getId()); - if (csvValueList == null) { - csvValueList = new ArrayList<>(); - csvValueList.add(csvHeader); + Map>> csvWithValue = new TreeMap<>(); + + consumerRequest.getOffers().stream().forEach(offer -> { + + Object object = subscribeAndDownloadDataOffers.get(offer.getAssetId()); + if (object != null) { + + JsonNode node = mapper.convertValue(object, JsonNode.class); + JsonNode status = node.get("status"); + JsonNode dataNode = node.get("data"); + JsonNode edrNode = node.get("edr"); + + if (dataNode != null) { + JsonNode csvNode = dataNode.get("csv"); + if (csvNode != null) { + List csvHeader = new ArrayList<>(); + List csvValues = new ArrayList<>(); + + csvNode.fieldNames().forEachRemaining(csvHeader::add); + csvNode.fields().forEachRemaining(obje -> csvValues.add(obje.getValue().asText())); + + Submodel findSubmodel = submodelOrchestartorService.findSubmodel(csvHeader); + List> csvValueList = csvWithValue.get(findSubmodel.getId()); + if (csvValueList == null) { + csvValueList = new ArrayList<>(); + csvValueList.add(csvHeader); + } + + csvValueList.add(csvValues); + csvWithValue.put(findSubmodel.getId(), csvValueList); + consumerDownloadHistory.setDownloadSuccessed(successCount + 1); + offer.setStatus(status.asText()); + } else { + consumerDownloadHistory.setDownloadFailed(failedCount + 1); + offer.setStatus("FAILED"); + offer.setDownloadErrorMsg("The data does not found in csv type"); + } + } else { + offer.setStatus(status.asText()); + consumerDownloadHistory.setDownloadFailed(failedCount + 1); + offer.setDownloadErrorMsg(readFieldFromJsonNode(dataNode, "error")); + } + + if (edrNode != null) { + offer.setAgreementId(readFieldFromJsonNode(edrNode, "agreementId")); + offer.setExpirationDate(readFieldFromJsonNode(edrNode, "expirationDate")); + offer.setTransferProcessId(readFieldFromJsonNode(edrNode, "transferProcessId")); } - csvValueList.add(csvValues); - csvWithValue.put(findSubmodel.getId(), csvValueList); - } else { - /// handle error log } }); + consumerDownloadHistory.setEndDate(LocalDateTime.now().toString()); + consumerDownloadHistory.setOffers(mapper.writeValueAsString(consumerRequest.getOffers())); + consumerDownloadHistory.setPolicies(mapper.writeValueAsString(consumerRequest.getPolicies())); + + // Save consumer Download history in DB + saveConsumerDownloadHistory(consumerDownloadHistory); + + prepareHttpResponse(response, processId, csvWithValue); + } + + private void prepareHttpResponse(HttpServletResponse response, String processId, + Map>> csvWithValue) throws IOException { if (csvWithValue.isEmpty()) { Map errorResponse = new HashMap<>(); errorResponse.put("msg", - "Unable to complete subscribe and download Data Offers, please try again, if error persist contact to admin"); + "Unable to process your request, please try again, if error persist contact to admin"); response.setContentType("application/json"); response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); response.setCharacterEncoding("UTF-8"); @@ -115,6 +170,14 @@ public void subscribeAndDownloadDataOffers(ConsumerRequest consumerRequest, Http } } + private String readFieldFromJsonNode(JsonNode node, String fieldName) { + JsonNode jsonNode = node.get(fieldName); + if (Optional.of(jsonNode).isPresent()) + return jsonNode.asText(); + else + return null; + } + @SneakyThrows private void prepareZipFiles(HttpServletResponse response, Map>> csvWithValue, String processId) { @@ -148,7 +211,12 @@ private void prepareZipFiles(HttpServletResponse response, Map downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(List assetId) { + Map downloadFileFromEDCUsingifAlreadyTransferStatusCompleted = consumerControlPanelService.downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(assetId); return ok().body(consumerControlPanelService.downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(assetId)); } + public void saveConsumerDownloadHistory(ConsumerDownloadHistory input) { + ConsumerDownloadHistoryEntity entity = downloadHistoryMapper.mapFrom(input); + consumerDownloadHistoryRepository.save(entity); + } } diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/ConsumerRequest.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/ConsumerRequest.java index 915bd13cb..d72202b67 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/ConsumerRequest.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/ConsumerRequest.java @@ -48,7 +48,7 @@ public class ConsumerRequest { private String providerUrl; @NonNull @NotEmpty - private List offers; + private List offers; @NonNull private List policies; diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/OfferRequest.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/Offer.java similarity index 89% rename from modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/OfferRequest.java rename to modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/Offer.java index 4c51638b0..02886f99b 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/OfferRequest.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/request/Offer.java @@ -33,11 +33,16 @@ @AllArgsConstructor @Builder @JsonInclude(JsonInclude.Include.NON_NULL) -public class OfferRequest { +public class Offer { @NotNull private String offerId; @NotNull private String assetId; private String policyId; + private String agreementId; + private String expirationDate; + private String transferProcessId; + private String status; + private String downloadErrorMsg; } diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index 275dd9063..fbce20337 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -257,23 +257,27 @@ public Map subscribeAndDownloadDataOffers(@Valid ConsumerRequest ActionRequest action = policyConstraintBuilderService.getUsagePolicyConstraints(policies); consumerRequest.getOffers().parallelStream().forEach(offer -> { + Map resultFields = new ConcurrentHashMap<>(); try { edrRequestHelper.edrRequestInitiate(recipientURL, consumerRequest.getConnectorId(), offer.getOfferId(), offer.getAssetId(), action, extensibleProperty); EDRCachedResponse checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId()); + resultFields.put("edr", checkContractNegotiationStatus); + resultFields.put("data", downloadFile(checkContractNegotiationStatus)); + resultFields.put("status", "SUCCESS"); - response.put(offer.getAssetId(), downloadFile(checkContractNegotiationStatus)); } catch (FeignException e) { log.error("RequestBody: " + e.request()); - log.error("SubscribeAndDownloadDataOffers Oops! We have an FeignException - " + e.request().url() + "-" - + e.contentUTF8()); - response.put(offer.getAssetId(), - errorMap("Unable to complete subscribeAndDownloadDataOffers because: " + e.contentUTF8())); + String errorMsg = "Unable to complete subscribeAndDownloadDataOffers because: " + e.contentUTF8(); + log.error(errorMsg); + prepareErrorMap(resultFields, errorMsg); } catch (Exception e) { log.error("SubscribeAndDownloadDataOffers Oops! We have an Exception -" + e.getMessage()); - response.put(offer.getAssetId(), - errorMap("Unable to complete subscribeAndDownloadDataOffers because: " + e.getMessage())); + String errorMsg = "Unable to complete subscribeAndDownloadDataOffers because: " + e.getMessage(); + prepareErrorMap(resultFields, errorMsg); + } finally { + response.put(offer.getAssetId(), resultFields); } }); @@ -316,32 +320,31 @@ private EDRCachedByIdResponse getAuthorizationTokenForDataDownload(String transf public Map downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(List assetIdList) { Map response = new ConcurrentHashMap<>(); for (String assetId : assetIdList) { + Map resultFields = new ConcurrentHashMap<>(); try { - EDRCachedResponse verifyEDRRequestStatus = verifyEDRRequestStatus(assetId); - response.put(assetId, downloadFile(verifyEDRRequestStatus)); - + resultFields.put("edr", verifyEDRRequestStatus); + resultFields.put("data", downloadFile(verifyEDRRequestStatus)); + resultFields.put("status", "SUCCESS"); } catch (FeignException e) { log.error("RequestBody: " + e.request()); - log.error("downloadFileFromEDCUsingifAlreadyTransferStatusCompleted Oops! We have an FeignException - " - + e.request().url() + "-" + e.contentUTF8()); - response.put(assetId, - errorMap("Unable to download existing subcribe data offer because: " + e.contentUTF8())); + String errorMsg = "Unable to download existing subcribe data offer because: " + e.contentUTF8(); + log.error(errorMsg); + prepareErrorMap(resultFields, errorMsg); } catch (Exception e) { - log.error("downloadFileFromEDCUsingifAlreadyTransferStatusCompleted Oops! We have an Exception -" - + e.getMessage()); - response.put(assetId, - errorMap("Unable to download existing subcribe data offer because: " + e.getMessage())); + log.error("SubscribeAndDownloadDataOffers Oops! We have an Exception -" + e.getMessage()); + String errorMsg = "Unable to download existing subcribe data offer because: " + e.getMessage(); + prepareErrorMap(resultFields, errorMsg); + } finally { + response.put(assetId, resultFields); } } return response; } - private Map errorMap(String errorMsg) { - Map errorMaps = new ConcurrentHashMap<>(); - errorMaps.put("status", "FAILED"); - errorMaps.put("error", errorMsg); - return errorMaps; + private void prepareErrorMap(Map resultFields, String errorMsg) { + resultFields.put("status", "FAILED"); + resultFields.put("error", errorMsg); } private Object downloadFile(EDRCachedResponse verifyEDRRequestStatus) { From ee7af6957496383e976aa47530e24a095e655446 Mon Sep 17 00:00:00 2001 From: ChetanT-System Date: Mon, 11 Sep 2023 10:26:35 +0530 Subject: [PATCH 03/23] - Updated Unit test --- .../tractusx/sde/controllers/ConsumerControllerTest.java | 6 +++--- .../sde/service/ConsumerControlPanelServiceTest.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/sde-core/src/test/java/org/eclipse/tractusx/sde/controllers/ConsumerControllerTest.java b/modules/sde-core/src/test/java/org/eclipse/tractusx/sde/controllers/ConsumerControllerTest.java index d08987f53..c6473b7b1 100644 --- a/modules/sde-core/src/test/java/org/eclipse/tractusx/sde/controllers/ConsumerControllerTest.java +++ b/modules/sde-core/src/test/java/org/eclipse/tractusx/sde/controllers/ConsumerControllerTest.java @@ -33,7 +33,7 @@ import org.eclipse.tractusx.sde.core.controller.ConsumerController; import org.eclipse.tractusx.sde.core.service.ConsumerService; import org.eclipse.tractusx.sde.edc.model.request.ConsumerRequest; -import org.eclipse.tractusx.sde.edc.model.request.OfferRequest; +import org.eclipse.tractusx.sde.edc.model.request.Offer; import org.eclipse.tractusx.sde.edc.model.response.QueryDataOfferModel; import org.eclipse.tractusx.sde.edc.services.ConsumerControlPanelService; import org.junit.jupiter.api.Test; @@ -106,9 +106,9 @@ void testSubscribeDataOffersBadRequest() throws Exception { // @Test void testSubscribeDataOffers() throws Exception { doNothing().when(consumerControlPanelService).subscribeDataOffers((ConsumerRequest) any(), anyString()); - List offers = new ArrayList<>(); + List offers = new ArrayList<>(); List policies = new ArrayList<>(); - OfferRequest mockOffer = Mockito.mock(OfferRequest.class); + Offer mockOffer = Mockito.mock(Offer.class); offers.add(mockOffer); UsagePolicies mockPolicy = Mockito.mock(UsagePolicies.class); policies.add(mockPolicy); diff --git a/modules/sde-core/src/test/java/org/eclipse/tractusx/sde/service/ConsumerControlPanelServiceTest.java b/modules/sde-core/src/test/java/org/eclipse/tractusx/sde/service/ConsumerControlPanelServiceTest.java index f318ce086..ff0ac1cab 100644 --- a/modules/sde-core/src/test/java/org/eclipse/tractusx/sde/service/ConsumerControlPanelServiceTest.java +++ b/modules/sde-core/src/test/java/org/eclipse/tractusx/sde/service/ConsumerControlPanelServiceTest.java @@ -45,7 +45,7 @@ import org.eclipse.tractusx.sde.edc.gateways.database.ContractNegotiationInfoRepository; import org.eclipse.tractusx.sde.edc.model.contractoffers.ContractOfferRequestFactory; import org.eclipse.tractusx.sde.edc.model.request.ConsumerRequest; -import org.eclipse.tractusx.sde.edc.model.request.OfferRequest; +import org.eclipse.tractusx.sde.edc.model.request.Offer; import org.eclipse.tractusx.sde.edc.model.response.QueryDataOfferModel; import org.eclipse.tractusx.sde.edc.services.ConsumerControlPanelService; import org.eclipse.tractusx.sde.portal.api.IPartnerPoolExternalServiceApi; @@ -134,7 +134,7 @@ void testQueryOnDataOffersWithUsagePolicies() throws Exception { @Test void testSubscribeDataOffers1() { - ArrayList offerRequestList = new ArrayList<>(); + ArrayList offerRequestList = new ArrayList<>(); List usagePolicies = new ArrayList<>(); UsagePolicies usagePolicy = UsagePolicies.builder().type(UsagePolicyEnum.CUSTOM).value("Sample") .typeOfAccess(PolicyAccessEnum.RESTRICTED).build(); From 48748f078837f36f01d136d8e36b29b0a6a58b3b Mon Sep 17 00:00:00 2001 From: ChetanT-System Date: Mon, 11 Sep 2023 11:33:58 +0530 Subject: [PATCH 04/23] - Added new changes for download --- .../mapper/DownloadHistoryMapper.java | 31 +++++++++ .../ConsumerDownloadHistoryRepository.java | 34 +++++++++ .../model/ConsumerDownloadHistory.java | 47 +++++++++++++ .../entity/ConsumerDownloadHistoryEntity.java | 69 +++++++++++++++++++ 4 files changed, 181 insertions(+) create mode 100644 modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/mapper/DownloadHistoryMapper.java create mode 100644 modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/repository/ConsumerDownloadHistoryRepository.java create mode 100644 modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java create mode 100644 modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/role/entity/ConsumerDownloadHistoryEntity.java diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/mapper/DownloadHistoryMapper.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/mapper/DownloadHistoryMapper.java new file mode 100644 index 000000000..ce359fbcb --- /dev/null +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/mapper/DownloadHistoryMapper.java @@ -0,0 +1,31 @@ +/******************************************************************************** + * Copyright (c) 2022 T-Systems International GmbH + * Copyright (c) 2022 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +package org.eclipse.tractusx.sde.core.failurelog.mapper; + +import org.eclipse.tractusx.sde.core.processreport.model.ConsumerDownloadHistory; +import org.eclipse.tractusx.sde.core.role.entity.ConsumerDownloadHistoryEntity; +import org.mapstruct.Mapper; + +@Mapper(componentModel = "spring") +public interface DownloadHistoryMapper { + + ConsumerDownloadHistoryEntity mapFrom(ConsumerDownloadHistory consumerDownloadHistory); +} diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/repository/ConsumerDownloadHistoryRepository.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/repository/ConsumerDownloadHistoryRepository.java new file mode 100644 index 000000000..9db5327b7 --- /dev/null +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/repository/ConsumerDownloadHistoryRepository.java @@ -0,0 +1,34 @@ +/******************************************************************************** + * Copyright (c) 2023 T-Systems International GmbH + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +package org.eclipse.tractusx.sde.core.failurelog.repository; + + +import java.util.List; + +import org.eclipse.tractusx.sde.core.failurelog.entity.FailureLogEntity; +import org.eclipse.tractusx.sde.core.role.entity.ConsumerDownloadHistoryEntity; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface ConsumerDownloadHistoryRepository extends JpaRepository { + + List findByProcessId(String id); + +} diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java new file mode 100644 index 000000000..ee6c33c5a --- /dev/null +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java @@ -0,0 +1,47 @@ +/******************************************************************************** + * Copyright (c) 2023 T-Systems International GmbH + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +package org.eclipse.tractusx.sde.core.processreport.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@Builder +@AllArgsConstructor +@NoArgsConstructor +public class ConsumerDownloadHistory { + + + private String processId; + private String providerUrl; + private String connectorId; + private int numberOfItems; + private int downloadFailed; + private int downloadSuccessed; + private String status; + private String startDate; + private String endDate; + private String offers; + private String policies; + private String referenceProcessId; + +} diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/role/entity/ConsumerDownloadHistoryEntity.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/role/entity/ConsumerDownloadHistoryEntity.java new file mode 100644 index 000000000..db090dbe1 --- /dev/null +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/role/entity/ConsumerDownloadHistoryEntity.java @@ -0,0 +1,69 @@ +/******************************************************************************** + * Copyright (c) 2023 T-Systems International GmbH + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ +package org.eclipse.tractusx.sde.core.role.entity; + +import java.util.List; + +import org.eclipse.tractusx.sde.common.entities.UsagePolicies; +import org.eclipse.tractusx.sde.edc.model.request.Offer; + +import jakarta.persistence.Cacheable; +import jakarta.persistence.Column; +import jakarta.persistence.Entity; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Table(name = "consumer_download_history") +@Entity +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +@Cacheable(value = false) +public class ConsumerDownloadHistoryEntity { + @Id + @Column(name = "process_id") + private String processId; + @Column(name = "provider_url") + private String providerUrl; + @Column(name = "connector_id") + private String connectorId; + @Column(name = "number_of_items") + private String numberOfItems; + @Column(name = "download_failed") + private String downloadFailed; + @Column(name = "download_successed") + private String downloadSuccessed; + @Column(name = "status") + private String status; + @Column(name = "start_date") + private String startDate; + @Column(name = "end_date") + private String endDate; + @Column(name = "offers") + private String offers; + private String policies; + private String referenceProcessId; + +} From c3bdd915be64dcdb61224fcaa99794b7f4cc6c32 Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Mon, 11 Sep 2023 16:50:24 +0530 Subject: [PATCH 05/23] Stable download version with history --- .../sde/common/enums/ProgressStatusEnum.java | 3 +- .../core/controller/ConsumerController.java | 29 +- .../ConsumerDownloadHistoryRepository.java | 7 +- .../entity/ConsumerDownloadHistoryEntity.java | 17 +- .../ConsumerDownloadHistoryMapper.java} | 33 ++- .../model/ConsumerDownloadHistory.java | 10 +- .../sde/core/service/ConsumerService.java | 278 ++++++++++++------ .../flyway/V22__add_new_role_for_download.sql | 19 +- .../services/ConsumerControlPanelService.java | 62 +++- 9 files changed, 318 insertions(+), 140 deletions(-) rename modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/{role => processreport}/entity/ConsumerDownloadHistoryEntity.java (87%) rename modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/{failurelog/mapper/DownloadHistoryMapper.java => processreport/mapper/ConsumerDownloadHistoryMapper.java} (50%) diff --git a/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java index 9e61c258a..f169fb12f 100644 --- a/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java +++ b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java @@ -25,5 +25,6 @@ public enum ProgressStatusEnum { IN_PROGRESS, COMPLETED, - FAILED + FAILED, + PARTIALED_FAILED } diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/controller/ConsumerController.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/controller/ConsumerController.java index 956d8961b..4b8ceb0dc 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/controller/ConsumerController.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/controller/ConsumerController.java @@ -22,7 +22,6 @@ import static org.springframework.http.ResponseEntity.ok; -import java.util.List; import java.util.UUID; import org.eclipse.tractusx.sde.core.service.ConsumerService; @@ -75,18 +74,34 @@ public ResponseEntity subscribeDataOffers(@Valid @RequestBody ConsumerRe } @PostMapping(value = "/subscribe-download-data-offers") - @PreAuthorize("hasPermission('','consumer_download_data')") + @PreAuthorize("hasPermission('','consumer_subscribe_download_data_offers')") public void subscribeAndDownloadDataOffers(@Valid @RequestBody ConsumerRequest consumerRequest, HttpServletResponse response) { - log.info("Request recevied : /api/subscribe-download-data-edr"); + log.info("Request recevied : /api/subscribe-download-data-offers"); consumerService.subscribeAndDownloadDataOffers(consumerRequest, response); } @GetMapping(value = "/download-data-offers") - @PreAuthorize("hasPermission('','consumer_download_data')") - public ResponseEntity downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(@RequestParam List assetIdList) + @PreAuthorize("hasPermission('','consumer_download_data_offer')") + public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(@RequestParam("processId") String referenceProcessId, HttpServletResponse response) throws Exception { - log.info("Request received : /api/download-data-using-edr"); - return consumerService.downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(assetIdList); + log.info("Request received : /api/download-data-offers"); + consumerService.downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(referenceProcessId,response); + } + + @GetMapping(value = "/view-download-history") + @PreAuthorize("hasPermission('','consumer_view_download_history')") + public ResponseEntity viewConsumerDownloadHistory() + throws Exception { + log.info("Request received : /api/view-download-history"); + return ok().body(consumerService.viewDownloadHistory()); + } + + @GetMapping(value = "/view-download-history-details") + @PreAuthorize("hasPermission('','consumer_view_download_history')") + public ResponseEntity viewConsumerDownloadHistoryDetails(@RequestParam("processId") String processId) + throws Exception { + log.info("Request received : /api/view-download-history-details"); + return ok().body(consumerService.viewConsumerDownloadHistoryDetails(processId)); } } diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/repository/ConsumerDownloadHistoryRepository.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/repository/ConsumerDownloadHistoryRepository.java index 9db5327b7..b87b0f49b 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/repository/ConsumerDownloadHistoryRepository.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/repository/ConsumerDownloadHistoryRepository.java @@ -21,14 +21,11 @@ package org.eclipse.tractusx.sde.core.failurelog.repository; -import java.util.List; - -import org.eclipse.tractusx.sde.core.failurelog.entity.FailureLogEntity; -import org.eclipse.tractusx.sde.core.role.entity.ConsumerDownloadHistoryEntity; +import org.eclipse.tractusx.sde.core.processreport.entity.ConsumerDownloadHistoryEntity; import org.springframework.data.jpa.repository.JpaRepository; public interface ConsumerDownloadHistoryRepository extends JpaRepository { - List findByProcessId(String id); + ConsumerDownloadHistoryEntity findByProcessId(String id); } diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/role/entity/ConsumerDownloadHistoryEntity.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/entity/ConsumerDownloadHistoryEntity.java similarity index 87% rename from modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/role/entity/ConsumerDownloadHistoryEntity.java rename to modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/entity/ConsumerDownloadHistoryEntity.java index db090dbe1..b45fdf5af 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/role/entity/ConsumerDownloadHistoryEntity.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/entity/ConsumerDownloadHistoryEntity.java @@ -17,17 +17,13 @@ * * SPDX-License-Identifier: Apache-2.0 ********************************************************************************/ -package org.eclipse.tractusx.sde.core.role.entity; - -import java.util.List; - -import org.eclipse.tractusx.sde.common.entities.UsagePolicies; -import org.eclipse.tractusx.sde.edc.model.request.Offer; +package org.eclipse.tractusx.sde.core.processreport.entity; import jakarta.persistence.Cacheable; import jakarta.persistence.Column; import jakarta.persistence.Entity; import jakarta.persistence.Id; +import jakarta.persistence.Lob; import jakarta.persistence.Table; import lombok.AllArgsConstructor; import lombok.Builder; @@ -50,19 +46,22 @@ public class ConsumerDownloadHistoryEntity { @Column(name = "connector_id") private String connectorId; @Column(name = "number_of_items") - private String numberOfItems; + private Integer numberOfItems; @Column(name = "download_failed") - private String downloadFailed; + private Integer downloadFailed; @Column(name = "download_successed") - private String downloadSuccessed; + private Integer downloadSuccessed; @Column(name = "status") private String status; @Column(name = "start_date") private String startDate; @Column(name = "end_date") private String endDate; + + @Lob @Column(name = "offers") private String offers; + @Lob private String policies; private String referenceProcessId; diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/mapper/DownloadHistoryMapper.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/mapper/ConsumerDownloadHistoryMapper.java similarity index 50% rename from modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/mapper/DownloadHistoryMapper.java rename to modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/mapper/ConsumerDownloadHistoryMapper.java index ce359fbcb..067d38321 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/mapper/DownloadHistoryMapper.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/mapper/ConsumerDownloadHistoryMapper.java @@ -18,14 +18,39 @@ * SPDX-License-Identifier: Apache-2.0 ********************************************************************************/ -package org.eclipse.tractusx.sde.core.failurelog.mapper; +package org.eclipse.tractusx.sde.core.processreport.mapper; +import java.util.List; + +import org.eclipse.tractusx.sde.common.entities.UsagePolicies; +import org.eclipse.tractusx.sde.core.processreport.entity.ConsumerDownloadHistoryEntity; import org.eclipse.tractusx.sde.core.processreport.model.ConsumerDownloadHistory; -import org.eclipse.tractusx.sde.core.role.entity.ConsumerDownloadHistoryEntity; +import org.eclipse.tractusx.sde.edc.model.request.Offer; import org.mapstruct.Mapper; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; + +import lombok.SneakyThrows; + @Mapper(componentModel = "spring") -public interface DownloadHistoryMapper { +public interface ConsumerDownloadHistoryMapper { + + ObjectMapper mapper = new ObjectMapper(); + + ConsumerDownloadHistory mapFrom(ConsumerDownloadHistoryEntity entity); - ConsumerDownloadHistoryEntity mapFrom(ConsumerDownloadHistory consumerDownloadHistory); + @SneakyThrows + default ConsumerDownloadHistory mapFromCustom(ConsumerDownloadHistoryEntity entity) { + ConsumerDownloadHistory mapFrom = mapFrom(entity); + + if (entity.getOffers() != null) + mapFrom.setOffers(mapper.readValue(entity.getOffers(), new TypeReference>() { + })); + if (entity.getPolicies() != null) + mapFrom.setPolicies(mapper.readValue(entity.getPolicies(), new TypeReference>() { + })); + + return mapFrom; + } } diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java index ee6c33c5a..d3abc702e 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java @@ -34,14 +34,14 @@ public class ConsumerDownloadHistory { private String processId; private String providerUrl; private String connectorId; - private int numberOfItems; - private int downloadFailed; - private int downloadSuccessed; + private Integer numberOfItems; + private Integer downloadFailed; + private Integer downloadSuccessed; private String status; private String startDate; private String endDate; - private String offers; - private String policies; + private Object offers; + private Object policies; private String referenceProcessId; } diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java index 23ee436c7..772b6dbb2 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java @@ -20,8 +20,6 @@ package org.eclipse.tractusx.sde.core.service; -import static org.springframework.http.ResponseEntity.ok; - import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.PrintWriter; @@ -34,21 +32,25 @@ import java.util.Optional; import java.util.TreeMap; import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import org.eclipse.tractusx.sde.common.enums.ProgressStatusEnum; +import org.eclipse.tractusx.sde.common.exception.NoDataFoundException; import org.eclipse.tractusx.sde.common.model.Submodel; -import org.eclipse.tractusx.sde.core.failurelog.mapper.DownloadHistoryMapper; import org.eclipse.tractusx.sde.core.failurelog.repository.ConsumerDownloadHistoryRepository; +import org.eclipse.tractusx.sde.core.processreport.entity.ConsumerDownloadHistoryEntity; +import org.eclipse.tractusx.sde.core.processreport.mapper.ConsumerDownloadHistoryMapper; import org.eclipse.tractusx.sde.core.processreport.model.ConsumerDownloadHistory; -import org.eclipse.tractusx.sde.core.role.entity.ConsumerDownloadHistoryEntity; import org.eclipse.tractusx.sde.core.utils.CsvUtil; import org.eclipse.tractusx.sde.edc.model.request.ConsumerRequest; +import org.eclipse.tractusx.sde.edc.model.request.Offer; import org.eclipse.tractusx.sde.edc.services.ConsumerControlPanelService; -import org.springframework.http.ResponseEntity; import org.springframework.stereotype.Service; import org.springframework.util.StreamUtils; +import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; @@ -67,112 +69,209 @@ public class ConsumerService { private final ConsumerDownloadHistoryRepository consumerDownloadHistoryRepository; - ObjectMapper mapper = new ObjectMapper(); + private final ConsumerDownloadHistoryMapper consumerDownloadHistoryMapper; - private DownloadHistoryMapper downloadHistoryMapper; + + ObjectMapper mapper = new ObjectMapper(); @SneakyThrows public void subscribeAndDownloadDataOffers(ConsumerRequest consumerRequest, HttpServletResponse response) { - ConsumerDownloadHistory consumerDownloadHistory = new ConsumerDownloadHistory(); - consumerDownloadHistory.setStartDate(LocalDateTime.now().toString()); String processId = UUID.randomUUID().toString(); - + AtomicInteger failedCount = new AtomicInteger(); + AtomicInteger successCount = new AtomicInteger(); + String startDate = LocalDateTime.now().toString(); + + ConsumerDownloadHistoryEntity entity = ConsumerDownloadHistoryEntity.builder() + .startDate(startDate) + .endDate(LocalDateTime.now().toString()) + .connectorId(consumerRequest.getConnectorId()) + .providerUrl(consumerRequest.getProviderUrl()) + .numberOfItems(consumerRequest.getOffers().size()) + .downloadSuccessed(successCount.get()) + .downloadFailed(failedCount.get()) + .processId(processId) + .status(ProgressStatusEnum.IN_PROGRESS.toString()) + .build(); + + // Save consumer Download history in DB + consumerDownloadHistoryRepository.save(entity); + Map subscribeAndDownloadDataOffers = consumerControlPanelService .subscribeAndDownloadDataOffers(consumerRequest); - int failedCount = 0; - int successCount = 0; - consumerDownloadHistory.setConnectorId(consumerRequest.getConnectorId()); - consumerDownloadHistory.setProviderUrl(consumerRequest.getProviderUrl()); - consumerDownloadHistory.setNumberOfItems(consumerRequest.getOffers().size()); - consumerDownloadHistory.setProcessId(processId); Map>> csvWithValue = new TreeMap<>(); + + consumerRequest.getOffers().stream().forEach(offer -> prepareFromOfferResponse(subscribeAndDownloadDataOffers, + failedCount, successCount, csvWithValue, offer)); + - consumerRequest.getOffers().stream().forEach(offer -> { - - Object object = subscribeAndDownloadDataOffers.get(offer.getAssetId()); - if (object != null) { - - JsonNode node = mapper.convertValue(object, JsonNode.class); - JsonNode status = node.get("status"); - JsonNode dataNode = node.get("data"); - JsonNode edrNode = node.get("edr"); - - if (dataNode != null) { - JsonNode csvNode = dataNode.get("csv"); - if (csvNode != null) { - List csvHeader = new ArrayList<>(); - List csvValues = new ArrayList<>(); - - csvNode.fieldNames().forEachRemaining(csvHeader::add); - csvNode.fields().forEachRemaining(obje -> csvValues.add(obje.getValue().asText())); - - Submodel findSubmodel = submodelOrchestartorService.findSubmodel(csvHeader); - List> csvValueList = csvWithValue.get(findSubmodel.getId()); - if (csvValueList == null) { - csvValueList = new ArrayList<>(); - csvValueList.add(csvHeader); - } - - csvValueList.add(csvValues); - csvWithValue.put(findSubmodel.getId(), csvValueList); - consumerDownloadHistory.setDownloadSuccessed(successCount + 1); - offer.setStatus(status.asText()); - } else { - consumerDownloadHistory.setDownloadFailed(failedCount + 1); - offer.setStatus("FAILED"); - offer.setDownloadErrorMsg("The data does not found in csv type"); - } - } else { - offer.setStatus(status.asText()); - consumerDownloadHistory.setDownloadFailed(failedCount + 1); - offer.setDownloadErrorMsg(readFieldFromJsonNode(dataNode, "error")); - } + entity.setEndDate(LocalDateTime.now().toString()); + entity.setOffers(mapper.writeValueAsString(consumerRequest.getOffers())); + entity.setPolicies(mapper.writeValueAsString(consumerRequest.getPolicies())); + entity.setDownloadSuccessed(successCount.get()); + entity.setDownloadFailed(failedCount.get()); + + entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); + if (failedCount.get() > 0 && consumerRequest.getOffers().size() != failedCount.get()) + entity.setStatus(ProgressStatusEnum.PARTIALED_FAILED.toString()); + else + entity.setStatus(ProgressStatusEnum.FAILED.toString()); + + // Save consumer Download history in DB + consumerDownloadHistoryRepository.save(entity); + + prepareHttpResponse(response, processId, csvWithValue); + } + + @SneakyThrows + public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(String referenceProcessId, + HttpServletResponse response) { + + String processId = UUID.randomUUID().toString(); + + ConsumerDownloadHistoryEntity entity = consumerDownloadHistoryRepository.findByProcessId(referenceProcessId); + + if (entity.getOffers() != null) { + + List offerList = mapper.readValue(entity.getOffers(), new TypeReference>() { + }); + + if (!offerList.isEmpty()) { + AtomicInteger failedCount = new AtomicInteger(); + AtomicInteger successCount = new AtomicInteger(); + + entity.setStartDate(LocalDateTime.now().toString()); + entity.setNumberOfItems(offerList.size()); + entity.setDownloadSuccessed(successCount.get()); + entity.setDownloadFailed(failedCount.get()); + entity.setProcessId(processId); + entity.setStatus(ProgressStatusEnum.IN_PROGRESS.toString()); + entity.setReferenceProcessId(referenceProcessId); + + // Save consumer Download history in DB + consumerDownloadHistoryRepository.save(entity); + + Map>> csvWithValue = new TreeMap<>(); + + List assetIds = offerList.stream().map(Offer::getAssetId).toList(); + + Map downloadFileFromEDCUsingifAlreadyTransferStatusCompleted = consumerControlPanelService + .downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(assetIds); + + offerList.stream() + .forEach(offer -> prepareFromOfferResponse( + downloadFileFromEDCUsingifAlreadyTransferStatusCompleted, failedCount, successCount, + csvWithValue, offer)); + + entity.setEndDate(LocalDateTime.now().toString()); + entity.setOffers(mapper.writeValueAsString(offerList)); + entity.setDownloadSuccessed(successCount.get()); + entity.setDownloadFailed(failedCount.get()); + entity.setProcessId(processId); + entity.setReferenceProcessId(referenceProcessId); - if (edrNode != null) { - offer.setAgreementId(readFieldFromJsonNode(edrNode, "agreementId")); - offer.setExpirationDate(readFieldFromJsonNode(edrNode, "expirationDate")); - offer.setTransferProcessId(readFieldFromJsonNode(edrNode, "transferProcessId")); - } + entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); + if (failedCount.get() > 0 && offerList.size() != failedCount.get()) + entity.setStatus(ProgressStatusEnum.PARTIALED_FAILED.toString()); + else + entity.setStatus(ProgressStatusEnum.FAILED.toString()); + + // Save consumer Download history in DB + consumerDownloadHistoryRepository.save(entity); + + prepareHttpResponse(response, processId, csvWithValue); + } else { + generateFailureJsonResponse(response, "Unable to find data offer in SDE for redownload"); } - }); + } else { + generateFailureJsonResponse(response, "Unable to find data offer in SDE for redownload"); + } + } - consumerDownloadHistory.setEndDate(LocalDateTime.now().toString()); - consumerDownloadHistory.setOffers(mapper.writeValueAsString(consumerRequest.getOffers())); - consumerDownloadHistory.setPolicies(mapper.writeValueAsString(consumerRequest.getPolicies())); + private void prepareFromOfferResponse(Map subscribeAndDownloadDataOffers, AtomicInteger failedCount, + AtomicInteger successCount, Map>> csvWithValue, Offer offer) { + Object object = subscribeAndDownloadDataOffers.get(offer.getAssetId()); + if (object != null) { + + JsonNode node = mapper.convertValue(object, JsonNode.class); + JsonNode status = node.get("status"); + JsonNode dataNode = node.get("data"); + JsonNode edrNode = node.get("edr"); + + if (dataNode != null) { + processCSVDataObject(successCount, failedCount, csvWithValue, offer, status, dataNode); + } else { + offer.setStatus(status.asText()); + failedCount.getAndIncrement(); + offer.setDownloadErrorMsg(readFieldFromJsonNode(node, "error")); + } - // Save consumer Download history in DB - saveConsumerDownloadHistory(consumerDownloadHistory); + if (edrNode != null) { + offer.setAgreementId(readFieldFromJsonNode(edrNode, "agreementId")); + offer.setExpirationDate(readFieldFromJsonNode(edrNode, "expirationDate")); + offer.setTransferProcessId(readFieldFromJsonNode(edrNode, "transferProcessId")); + } + } + } - prepareHttpResponse(response, processId, csvWithValue); + private void processCSVDataObject(AtomicInteger successCount, AtomicInteger failedCount, + Map>> csvWithValue, Offer offer, JsonNode status, JsonNode dataNode) { + + JsonNode csvNode = dataNode.get("csv"); + if (csvNode != null) { + List csvHeader = new ArrayList<>(); + List csvValues = new ArrayList<>(); + + csvNode.fieldNames().forEachRemaining(csvHeader::add); + csvNode.fields().forEachRemaining(obje -> csvValues.add(obje.getValue().asText())); + + Submodel findSubmodel = submodelOrchestartorService.findSubmodel(csvHeader); + List> csvValueList = csvWithValue.get(findSubmodel.getId()); + if (csvValueList == null) { + csvValueList = new ArrayList<>(); + csvValueList.add(csvHeader); + } + + csvValueList.add(csvValues); + csvWithValue.put(findSubmodel.getId(), csvValueList); + offer.setStatus(status.asText()); + successCount.getAndIncrement(); + } else { + offer.setStatus("FAILED"); + offer.setDownloadErrorMsg("The csv type data does not found in response"); + failedCount.getAndIncrement(); + } } private void prepareHttpResponse(HttpServletResponse response, String processId, Map>> csvWithValue) throws IOException { if (csvWithValue.isEmpty()) { - Map errorResponse = new HashMap<>(); - errorResponse.put("msg", - "Unable to process your request, please try again, if error persist contact to admin"); - response.setContentType("application/json"); - response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - response.setCharacterEncoding("UTF-8"); - String jsonData = new Gson().toJson(errorResponse); - PrintWriter out = response.getWriter(); - try { - out.print(jsonData); - } finally { - out.close(); - } + generateFailureJsonResponse(response, "Unable to process your request, please try again, if error persist contact to admin"); } else { prepareZipFiles(response, csvWithValue, processId); } } + private void generateFailureJsonResponse(HttpServletResponse response, String errormsg) throws IOException { + Map errorResponse = new HashMap<>(); + errorResponse.put("msg", errormsg); + response.setContentType("application/json"); + response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + response.setCharacterEncoding("UTF-8"); + String jsonData = new Gson().toJson(errorResponse); + PrintWriter out = response.getWriter(); + try { + out.print(jsonData); + } finally { + out.close(); + } + } + private String readFieldFromJsonNode(JsonNode node, String fieldName) { JsonNode jsonNode = node.get(fieldName); - if (Optional.of(jsonNode).isPresent()) + if (Optional.ofNullable(jsonNode).isPresent()) return jsonNode.asText(); else return null; @@ -210,13 +309,16 @@ private void prepareZipFiles(HttpServletResponse response, Map downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(List assetId) { - Map downloadFileFromEDCUsingifAlreadyTransferStatusCompleted = consumerControlPanelService.downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(assetId); - return ok().body(consumerControlPanelService.downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(assetId)); + public List viewDownloadHistory() { + return consumerDownloadHistoryRepository.findAll().stream() + .map(consumerDownloadHistoryMapper::mapFromCustom).toList(); } - public void saveConsumerDownloadHistory(ConsumerDownloadHistory input) { - ConsumerDownloadHistoryEntity entity = downloadHistoryMapper.mapFrom(input); - consumerDownloadHistoryRepository.save(entity); + public ConsumerDownloadHistory viewConsumerDownloadHistoryDetails(String processId) { + ConsumerDownloadHistoryEntity entity = Optional + .ofNullable(consumerDownloadHistoryRepository.findByProcessId(processId)) + .orElseThrow(() -> new NoDataFoundException("No data found processId " + processId)); + return consumerDownloadHistoryMapper.mapFromCustom(entity); } + } diff --git a/modules/sde-core/src/main/resources/flyway/V22__add_new_role_for_download.sql b/modules/sde-core/src/main/resources/flyway/V22__add_new_role_for_download.sql index 27c4df696..c0add3e93 100644 --- a/modules/sde-core/src/main/resources/flyway/V22__add_new_role_for_download.sql +++ b/modules/sde-core/src/main/resources/flyway/V22__add_new_role_for_download.sql @@ -18,11 +18,20 @@ * SPDX-License-Identifier: Apache-2.0 ********************************************************************************/ INSERT INTO sde_permission (sde_permission,description) - VALUES ('consumer_download_data','Allows consumer user to download data'); + VALUES ('consumer_subscribe_download_data_offers','Allows consumer user to subscribe and download data'); +INSERT INTO sde_permission (sde_permission,description) + VALUES ('consumer_download_data_offer','Allows consumer user to download data again'); +INSERT INTO sde_permission (sde_permission,description) + VALUES ('consumer_view_download_history','Allows consumer user to view download data history'); + +INSERT INTO sde_role_permission_mapping (sde_permission,sde_role) + VALUES ('consumer_subscribe_download_data_offers','Creator'); +INSERT INTO sde_role_permission_mapping (sde_permission,sde_role) + VALUES ('consumer_download_data_offer','Creator'); +INSERT INTO sde_role_permission_mapping (sde_permission,sde_role) + VALUES ('consumer_view_download_history','Creator'); INSERT INTO sde_role_permission_mapping (sde_permission,sde_role) - VALUES ('consumer_download_data','User'); - + VALUES ('consumer_download_data_offer','User'); INSERT INTO sde_role_permission_mapping (sde_permission,sde_role) - VALUES ('consumer_download_data','Creator'); - + VALUES ('consumer_view_download_history','User'); \ No newline at end of file diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index fbce20337..17585b14d 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -34,6 +34,7 @@ import org.eclipse.tractusx.sde.common.entities.UsagePolicies; import org.eclipse.tractusx.sde.common.enums.PolicyAccessEnum; import org.eclipse.tractusx.sde.common.enums.UsagePolicyEnum; +import org.eclipse.tractusx.sde.common.exception.ServiceException; import org.eclipse.tractusx.sde.edc.api.ContractOfferCatalogApi; import org.eclipse.tractusx.sde.edc.constants.EDCAssetConstant; import org.eclipse.tractusx.sde.edc.entities.database.ContractNegotiationInfoEntity; @@ -66,6 +67,7 @@ @RequiredArgsConstructor public class ConsumerControlPanelService extends AbstractEDCStepsHelper { + private static final String STATUS = "status"; private final ContractOfferCatalogApi contractOfferCatalogApiProxy; private final ContractNegotiateManagementHelper contractNegotiateManagement; private final EDRRequestHelper edrRequestHelper; @@ -259,16 +261,25 @@ public Map subscribeAndDownloadDataOffers(@Valid ConsumerRequest consumerRequest.getOffers().parallelStream().forEach(offer -> { Map resultFields = new ConcurrentHashMap<>(); try { - edrRequestHelper.edrRequestInitiate(recipientURL, consumerRequest.getConnectorId(), offer.getOfferId(), - offer.getAssetId(), action, extensibleProperty); - + + //Verify if there already contract established then use then contract for download EDRCachedResponse checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId()); + + if (checkContractNegotiationStatus == null) { + + log.info("Ther is not contract exist for "+offer.getAssetId()+", Initiating contract process"); + + edrRequestHelper.edrRequestInitiate(recipientURL, consumerRequest.getConnectorId(), + offer.getOfferId(), offer.getAssetId(), action, extensibleProperty); + } + + checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId()); resultFields.put("edr", checkContractNegotiationStatus); resultFields.put("data", downloadFile(checkContractNegotiationStatus)); - resultFields.put("status", "SUCCESS"); + resultFields.put(STATUS, "SUCCESS"); } catch (FeignException e) { - log.error("RequestBody: " + e.request()); + log.error("Feign RequestBody: " + e.request()); String errorMsg = "Unable to complete subscribeAndDownloadDataOffers because: " + e.contentUTF8(); log.error(errorMsg); prepareErrorMap(resultFields, errorMsg); @@ -292,21 +303,40 @@ private EDRCachedResponse verifyEDRRequestStatus(String assetId) { int retry = 5; int counter = 1; do { - Thread.sleep(5000); + Thread.sleep(3000); eDRCachedResponseList = edrRequestHelper.getEDRCachedByAsset(assetId); counter++; - } while (eDRCachedResponseList != null && !eDRCachedResponseList.isEmpty() - && !eDRCachedResponseList.get(0).getEdrState().equals("NEGOTIATED") && counter <= retry); - if (eDRCachedResponseList != null && !eDRCachedResponseList.isEmpty()) - eDRCachedResponse = eDRCachedResponseList.get(0); + if (eDRCachedResponseList != null && !eDRCachedResponseList.isEmpty()) { + Optional findAny = eDRCachedResponseList.stream() + .filter(obj -> obj.getEdrState().equals("NEGOTIATED")).findAny(); + if (findAny.isPresent()) + eDRCachedResponse = findAny.get(); + } + + } while ((eDRCachedResponse == null && counter <= retry) + || (eDRCachedResponse != null && !eDRCachedResponse.getEdrState().equals("NEGOTIATED"))); + + if (eDRCachedResponse == null) + throw new ServiceException("Time out!! to verify contract negotiated for asset " + assetId); + + } catch (FeignException e) { + log.error("RequestBody: " + e.request()); + String errorMsg = "FeignExceptionton to verify negotiated contract for asset " + assetId + ",error: " + + e.contentUTF8(); + log.error("Response: " + errorMsg); + throw new ServiceException(errorMsg); } catch (InterruptedException ie) { - log.error("Exception in verifyEDRRequestStatus" + ie.getMessage()); Thread.currentThread().interrupt(); - throw ie; + String errorMsg = "InterruptedException to verify negotiated contract for asset " + assetId + ", error: " + + ie.getMessage(); + log.error(errorMsg); + throw new ServiceException(errorMsg); } catch (Exception e) { - log.error("Exception in verifyEDRRequestStatus" + e.getMessage()); - throw e; + String errorMsg = "Exception to verify negotiated contract for asset " + assetId + ",Exception error: " + + e.getMessage(); + log.error(errorMsg); + throw new ServiceException(errorMsg); } return eDRCachedResponse; } @@ -325,7 +355,7 @@ public Map downloadFileFromEDCUsingifAlreadyTransferStatusComple EDRCachedResponse verifyEDRRequestStatus = verifyEDRRequestStatus(assetId); resultFields.put("edr", verifyEDRRequestStatus); resultFields.put("data", downloadFile(verifyEDRRequestStatus)); - resultFields.put("status", "SUCCESS"); + resultFields.put(STATUS, "SUCCESS"); } catch (FeignException e) { log.error("RequestBody: " + e.request()); String errorMsg = "Unable to download existing subcribe data offer because: " + e.contentUTF8(); @@ -343,7 +373,7 @@ public Map downloadFileFromEDCUsingifAlreadyTransferStatusComple } private void prepareErrorMap(Map resultFields, String errorMsg) { - resultFields.put("status", "FAILED"); + resultFields.put(STATUS, "FAILED"); resultFields.put("error", errorMsg); } From ae584fa9d9417ec9b1f510ceb26eba3b37aa6ffc Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Mon, 11 Sep 2023 19:31:40 +0530 Subject: [PATCH 06/23] PCF schema corrections --- .../create/steps/impl/RecordProcessUtils.java | 18 ++++++++++-------- .../pcf/src/main/resources/pcf.json | 19 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/submodel/executor/create/steps/impl/RecordProcessUtils.java b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/submodel/executor/create/steps/impl/RecordProcessUtils.java index 83c58185a..db379c59b 100644 --- a/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/submodel/executor/create/steps/impl/RecordProcessUtils.java +++ b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/submodel/executor/create/steps/impl/RecordProcessUtils.java @@ -38,9 +38,13 @@ public void setFieldValue(ObjectNode rowjObject, String ele, JsonObject jObject, fieldValue = fieldValue.trim(); - if (isNumberTypeField(jObject, fieldValue)) - rowjObject.put(ele, Double.parseDouble(fieldValue)); - else if (isDateFormatField(jObject)) { + if (isNumberTypeField(jObject)) { + if (fieldValue.isBlank()) { + rowjObject.putNull(ele); + } else { + rowjObject.put(ele, Double.parseDouble(fieldValue)); + } + } else if (isDateFormatField(jObject)) { if (fieldValue.isBlank()) fieldValue = null; @@ -57,12 +61,10 @@ private boolean isDateFormatField(JsonObject jObject) { return jObject.get("format") != null && "date-time".equals(jObject.get("format").getAsString()); } - private boolean isNumberTypeField(JsonObject jObject, String fieldValue) { - - if (fieldValue != null && !fieldValue.isBlank() && jObject.get("type") != null - && jObject.get("type").isJsonArray()) { + private boolean isNumberTypeField(JsonObject jObject) { + JsonElement jsonElement = JsonParser.parseString("number"); + if (jObject.get("type") != null && jObject.get("type").isJsonArray()) { JsonArray types = jObject.get("type").getAsJsonArray(); - JsonElement jsonElement = JsonParser.parseString("number"); return types.contains(jsonElement); } return false; diff --git a/modules/sde-submodules/pcf/src/main/resources/pcf.json b/modules/sde-submodules/pcf/src/main/resources/pcf.json index 0b1a3dbce..a6f13f2f4 100644 --- a/modules/sde-submodules/pcf/src/main/resources/pcf.json +++ b/modules/sde-submodules/pcf/src/main/resources/pcf.json @@ -12,6 +12,7 @@ "items": { "type": "object", "required": [ + "id", "specVersion", "companyId", "companyName", @@ -37,19 +38,20 @@ "pcfExcludingBiogenic", "extWBCSD_packagingGhgEmissions", "precedingPfId", - "secondaryEmissionFactorSource" + "secondaryEmissionFactorSource", + "assetLifeCyclePhase" ], "dependentRequired": { }, "properties": { "id": { "type": [ - "string", - "null" + "string" ], "pattern": "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", "title": "Id", "description": "Constraint for defining a UUID v4 identifier.", + "minLength": 1, "examples": [ "3893bb5d-da16-4dc1-9185-11d97476c254" ] @@ -351,7 +353,7 @@ "null" ], "title": "GeographyCountry", - "pattern": "([A-Z]{2})", + "pattern": "([A-Z]{2}|)", "description": "Two letter country code that must conform to data type ISO 3166CC as specified in the Catena-X PCF Rulebook in accordance with the technical specifications for PCF Data Exchange (Version 2.0.0) from the WBCSD (World Business Council for Sustainable Development)/ PACT initiative.", "examples": [ "DE" @@ -363,6 +365,7 @@ "null" ], "enum": [ + "", "Africa", "Americas", "Asia", @@ -488,11 +491,11 @@ }, "extWBCSD_allocationRulesDescription": { "type": [ - "string" + "string", + "null" ], "title": "ExtWBCSD_allocationRulesDescription", "description": "Allocation rules used and underlying reasoning in context of a product carbon footprint as specified in the technical specifications for PCF Data Exchange (Version 2.0.0) from the WBCSD (World Business Council for Sustainable Development)/ PACT initiative. WBCSD specific extension, in Catena-X for example specified by default with value \"In accordance with Catena-X PCF Rulebook\".", - "minLength": 1, "examples": [ "In accordance with Catena-X PCF Rulebook" ] @@ -822,11 +825,11 @@ "AsMaintained", "AsRecycled" ], - "minLength": 1, "title": "Asset Life Cycle Phase", + "minLength": 1, "description": "The lifecycle context in which the child part was assembled into the parent part.", "examples": [ - "lifecycle" + "AsPlanned" ] } From 124a3f7e94662c2f67b920a1a9d35c93a8850d92 Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Mon, 11 Sep 2023 19:37:35 +0530 Subject: [PATCH 07/23] remove unwanted dependency --- pom.xml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pom.xml b/pom.xml index bf914f9aa..5570f0fbc 100644 --- a/pom.xml +++ b/pom.xml @@ -156,10 +156,6 @@ com.fasterxml.jackson.core jackson-databind - - com.fasterxml.jackson.dataformat - jackson-dataformat-csv - org.yaml snakeyaml From af7edde2ca97d0289acf9f700c967b6b6822bad3 Mon Sep 17 00:00:00 2001 From: ChetanT-System Date: Tue, 12 Sep 2023 10:15:15 +0530 Subject: [PATCH 08/23] - Updated code changes --- .../sde/edc/services/ConsumerControlPanelService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index 17585b14d..13140b021 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -349,7 +349,8 @@ private EDRCachedByIdResponse getAuthorizationTokenForDataDownload(String transf @SneakyThrows public Map downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(List assetIdList) { Map response = new ConcurrentHashMap<>(); - for (String assetId : assetIdList) { + assetIdList.parallelStream().forEach(assetId -> { + Map resultFields = new ConcurrentHashMap<>(); try { EDRCachedResponse verifyEDRRequestStatus = verifyEDRRequestStatus(assetId); @@ -368,7 +369,7 @@ public Map downloadFileFromEDCUsingifAlreadyTransferStatusComple } finally { response.put(assetId, resultFields); } - } + }); return response; } From d54d4d81f34e3b6622934c978c45dd35d5ebbf62 Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Tue, 12 Sep 2023 11:11:58 +0530 Subject: [PATCH 09/23] Correct contract verification step --- .../services/ConsumerControlPanelService.java | 46 +++++++++++-------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index 13140b021..a1488a2bc 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -49,6 +49,7 @@ import org.eclipse.tractusx.sde.edc.model.edr.EDRCachedByIdResponse; import org.eclipse.tractusx.sde.edc.model.edr.EDRCachedResponse; import org.eclipse.tractusx.sde.edc.model.request.ConsumerRequest; +import org.eclipse.tractusx.sde.edc.model.request.Offer; import org.eclipse.tractusx.sde.edc.model.response.QueryDataOfferModel; import org.eclipse.tractusx.sde.edc.util.UtilityFunctions; import org.springframework.scheduling.annotation.Async; @@ -77,6 +78,10 @@ public class ConsumerControlPanelService extends AbstractEDCStepsHelper { private final ContractOfferRequestFactory contractOfferRequestFactory; + private static final Integer RETRY = 5; + + private static final Integer THRED_SLEEP_TIME=5000; + public List queryOnDataOffers(String providerUrl, Integer offset, Integer limit, String filterExpression) { @@ -261,19 +266,9 @@ public Map subscribeAndDownloadDataOffers(@Valid ConsumerRequest consumerRequest.getOffers().parallelStream().forEach(offer -> { Map resultFields = new ConcurrentHashMap<>(); try { - - //Verify if there already contract established then use then contract for download - EDRCachedResponse checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId()); - - if (checkContractNegotiationStatus == null) { - - log.info("Ther is not contract exist for "+offer.getAssetId()+", Initiating contract process"); - - edrRequestHelper.edrRequestInitiate(recipientURL, consumerRequest.getConnectorId(), - offer.getOfferId(), offer.getAssetId(), action, extensibleProperty); - } + EDRCachedResponse checkContractNegotiationStatus = verifyOrCreateContractNegotiation(consumerRequest, + extensibleProperty, recipientURL, action, offer); - checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId()); resultFields.put("edr", checkContractNegotiationStatus); resultFields.put("data", downloadFile(checkContractNegotiationStatus)); resultFields.put(STATUS, "SUCCESS"); @@ -296,14 +291,29 @@ public Map subscribeAndDownloadDataOffers(@Valid ConsumerRequest } @SneakyThrows - private EDRCachedResponse verifyEDRRequestStatus(String assetId) { + private EDRCachedResponse verifyOrCreateContractNegotiation(ConsumerRequest consumerRequest, + HashMap extensibleProperty, String recipientURL, ActionRequest action, Offer offer) { + EDRCachedResponse checkContractNegotiationStatus = null; + try { + // Verify if there already contract established then use then contract for + // download + checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId(), RETRY); + } catch (Exception e) { + log.info("There is not contract exist for " + offer.getAssetId() + ", Initiating contract process"); + edrRequestHelper.edrRequestInitiate(recipientURL, consumerRequest.getConnectorId(), offer.getOfferId(), + offer.getAssetId(), action, extensibleProperty); + checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId(), 1); + } + return checkContractNegotiationStatus; + } + + @SneakyThrows + private EDRCachedResponse verifyEDRRequestStatus(String assetId, int counter) { EDRCachedResponse eDRCachedResponse = null; List eDRCachedResponseList = null; try { - int retry = 5; - int counter = 1; do { - Thread.sleep(3000); + Thread.sleep(THRED_SLEEP_TIME); eDRCachedResponseList = edrRequestHelper.getEDRCachedByAsset(assetId); counter++; @@ -314,7 +324,7 @@ private EDRCachedResponse verifyEDRRequestStatus(String assetId) { eDRCachedResponse = findAny.get(); } - } while ((eDRCachedResponse == null && counter <= retry) + } while ((eDRCachedResponse == null && counter <= RETRY) || (eDRCachedResponse != null && !eDRCachedResponse.getEdrState().equals("NEGOTIATED"))); if (eDRCachedResponse == null) @@ -353,7 +363,7 @@ public Map downloadFileFromEDCUsingifAlreadyTransferStatusComple Map resultFields = new ConcurrentHashMap<>(); try { - EDRCachedResponse verifyEDRRequestStatus = verifyEDRRequestStatus(assetId); + EDRCachedResponse verifyEDRRequestStatus = verifyEDRRequestStatus(assetId, 1); resultFields.put("edr", verifyEDRRequestStatus); resultFields.put("data", downloadFile(verifyEDRRequestStatus)); resultFields.put(STATUS, "SUCCESS"); From 44b19310afceb19d34757593acfe151bc490aa75 Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Tue, 12 Sep 2023 11:54:02 +0530 Subject: [PATCH 10/23] Correct status for download hostory --- .../sde/core/service/ConsumerService.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java index 772b6dbb2..3613c2c30 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java @@ -36,6 +36,7 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; +import org.apache.catalina.startup.FailedContext; import org.eclipse.tractusx.sde.common.enums.ProgressStatusEnum; import org.eclipse.tractusx.sde.common.exception.NoDataFoundException; import org.eclipse.tractusx.sde.common.model.Submodel; @@ -113,11 +114,11 @@ public void subscribeAndDownloadDataOffers(ConsumerRequest consumerRequest, Http entity.setDownloadSuccessed(successCount.get()); entity.setDownloadFailed(failedCount.get()); - entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); - if (failedCount.get() > 0 && consumerRequest.getOffers().size() != failedCount.get()) + entity.setStatus(ProgressStatusEnum.FAILED.toString()); + if (consumerRequest.getOffers().size() == successCount.get()) + entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); + else if(successCount.get()!=0 && failedCount.get()!=0 ) entity.setStatus(ProgressStatusEnum.PARTIALED_FAILED.toString()); - else - entity.setStatus(ProgressStatusEnum.FAILED.toString()); // Save consumer Download history in DB consumerDownloadHistoryRepository.save(entity); @@ -172,11 +173,11 @@ public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(String refe entity.setProcessId(processId); entity.setReferenceProcessId(referenceProcessId); - entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); - if (failedCount.get() > 0 && offerList.size() != failedCount.get()) + entity.setStatus(ProgressStatusEnum.FAILED.toString()); + if (offerList.size() == successCount.get()) + entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); + else if(successCount.get()!=0 && failedCount.get()!=0 ) entity.setStatus(ProgressStatusEnum.PARTIALED_FAILED.toString()); - else - entity.setStatus(ProgressStatusEnum.FAILED.toString()); // Save consumer Download history in DB consumerDownloadHistoryRepository.save(entity); From 298fb552eee2c11388288457d09fe53f56f07c48 Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Tue, 12 Sep 2023 13:33:47 +0530 Subject: [PATCH 11/23] Handle donwload case if history record not exist --- .../org/eclipse/tractusx/sde/core/service/ConsumerService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java index 3613c2c30..836df6457 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java @@ -36,7 +36,6 @@ import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; -import org.apache.catalina.startup.FailedContext; import org.eclipse.tractusx.sde.common.enums.ProgressStatusEnum; import org.eclipse.tractusx.sde.common.exception.NoDataFoundException; import org.eclipse.tractusx.sde.common.model.Submodel; @@ -134,7 +133,7 @@ public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(String refe ConsumerDownloadHistoryEntity entity = consumerDownloadHistoryRepository.findByProcessId(referenceProcessId); - if (entity.getOffers() != null) { + if (entity!=null && entity.getOffers() != null) { List offerList = mapper.readValue(entity.getOffers(), new TypeReference>() { }); From 8f199c943fa6f9e035a191a7a0ed8b644687aa7b Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Tue, 12 Sep 2023 14:03:24 +0530 Subject: [PATCH 12/23] Sonar code duplication issue --- .../services/ConsumerControlPanelService.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index a1488a2bc..df02deea9 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -361,23 +361,23 @@ public Map downloadFileFromEDCUsingifAlreadyTransferStatusComple Map response = new ConcurrentHashMap<>(); assetIdList.parallelStream().forEach(assetId -> { - Map resultFields = new ConcurrentHashMap<>(); + Map downloadResultFields = new ConcurrentHashMap<>(); try { EDRCachedResponse verifyEDRRequestStatus = verifyEDRRequestStatus(assetId, 1); - resultFields.put("edr", verifyEDRRequestStatus); - resultFields.put("data", downloadFile(verifyEDRRequestStatus)); - resultFields.put(STATUS, "SUCCESS"); + downloadResultFields.put("edr", verifyEDRRequestStatus); + downloadResultFields.put("data", downloadFile(verifyEDRRequestStatus)); + downloadResultFields.put(STATUS, "SUCCESS"); } catch (FeignException e) { - log.error("RequestBody: " + e.request()); + log.error("Re-download RequestBody: " + e.request()); String errorMsg = "Unable to download existing subcribe data offer because: " + e.contentUTF8(); log.error(errorMsg); - prepareErrorMap(resultFields, errorMsg); + prepareErrorMap(downloadResultFields, errorMsg); } catch (Exception e) { - log.error("SubscribeAndDownloadDataOffers Oops! We have an Exception -" + e.getMessage()); + log.error("DownloadFileFromEDCUsingifAlreadyTransferStatusCompleted Oops! We have an Exception -" + e.getMessage()); String errorMsg = "Unable to download existing subcribe data offer because: " + e.getMessage(); - prepareErrorMap(resultFields, errorMsg); + prepareErrorMap(downloadResultFields, errorMsg); } finally { - response.put(assetId, resultFields); + response.put(assetId, downloadResultFields); } }); return response; From b614453756b6e0026221bb1986f5af0235f80227 Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Tue, 12 Sep 2023 16:26:20 +0530 Subject: [PATCH 13/23] Change response in view download history --- .../sde/common/model/Acknowledgement.java | 32 ++++++ .../sde/common/model/PagingResponse.java | 34 ++++++ .../core/controller/ConsumerController.java | 25 ++-- .../processreport/ProcessReportUseCase.java | 2 +- .../entity/ConsumerDownloadHistoryEntity.java | 6 +- .../model/ConsumerDownloadHistory.java | 6 +- .../model/ProcessReportPageResponse.java | 2 +- .../sde/core/service/ConsumerService.java | 107 +++++++++++------- .../services/ConsumerControlPanelService.java | 8 +- 9 files changed, 163 insertions(+), 59 deletions(-) create mode 100644 modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/Acknowledgement.java create mode 100644 modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/PagingResponse.java diff --git a/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/Acknowledgement.java b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/Acknowledgement.java new file mode 100644 index 000000000..4f88f5e84 --- /dev/null +++ b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/Acknowledgement.java @@ -0,0 +1,32 @@ +/******************************************************************************** + * Copyright (c) 2023 T-Systems International GmbH + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +package org.eclipse.tractusx.sde.common.model; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class Acknowledgement { + + private String id; + +} diff --git a/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/PagingResponse.java b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/PagingResponse.java new file mode 100644 index 000000000..938a511fc --- /dev/null +++ b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/PagingResponse.java @@ -0,0 +1,34 @@ +/******************************************************************************** + * Copyright (c) 2023 T-Systems International GmbH + * Copyright (c) 2023 Contributors to the Eclipse Foundation + * + * See the NOTICE file(s) distributed with this work for additional + * information regarding copyright ownership. + * + * This program and the accompanying materials are made available under the + * terms of the Apache License, Version 2.0 which is available at + * https://www.apache.org/licenses/LICENSE-2.0. + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + * + * SPDX-License-Identifier: Apache-2.0 + ********************************************************************************/ + +package org.eclipse.tractusx.sde.common.model; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class PagingResponse { + + private int page; + private int pageSize; + private long totalItems; + private Object items; +} diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/controller/ConsumerController.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/controller/ConsumerController.java index 4b8ceb0dc..89cb2378f 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/controller/ConsumerController.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/controller/ConsumerController.java @@ -27,9 +27,11 @@ import org.eclipse.tractusx.sde.core.service.ConsumerService; import org.eclipse.tractusx.sde.edc.model.request.ConsumerRequest; import org.eclipse.tractusx.sde.edc.services.ConsumerControlPanelService; +import org.springframework.data.repository.query.Param; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestParam; @@ -72,12 +74,19 @@ public ResponseEntity subscribeDataOffers(@Valid @RequestBody ConsumerRe consumerControlPanelService.subscribeDataOffers(consumerRequest, processId); return ResponseEntity.ok().body(processId); } + + @PostMapping(value = "/subscribe-download-data-offers-async") + @PreAuthorize("hasPermission('','consumer_subscribe_download_data_offers')") + public ResponseEntity subscribeAndDownloadDataOffersAsync(@Valid @RequestBody ConsumerRequest consumerRequest) { + log.info("Request recevied : /api/subscribe-download-data-offers-async"); + return ResponseEntity.ok().body(consumerService.subscribeAndDownloadDataOffersAsync(consumerRequest)); + } @PostMapping(value = "/subscribe-download-data-offers") @PreAuthorize("hasPermission('','consumer_subscribe_download_data_offers')") - public void subscribeAndDownloadDataOffers(@Valid @RequestBody ConsumerRequest consumerRequest, HttpServletResponse response) { + public void subscribeAndDownloadDataOffersSynchronous(@Valid @RequestBody ConsumerRequest consumerRequest, HttpServletResponse response) { log.info("Request recevied : /api/subscribe-download-data-offers"); - consumerService.subscribeAndDownloadDataOffers(consumerRequest, response); + consumerService.subscribeAndDownloadDataOffersSynchronous(consumerRequest, response); } @GetMapping(value = "/download-data-offers") @@ -90,15 +99,17 @@ public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(@RequestPar @GetMapping(value = "/view-download-history") @PreAuthorize("hasPermission('','consumer_view_download_history')") - public ResponseEntity viewConsumerDownloadHistory() - throws Exception { + public ResponseEntity viewConsumerDownloadHistory(@Param("page") Integer page, + @Param("pageSize") Integer pageSize) throws Exception { + page = page == null ? 0 : page; + pageSize = pageSize == null ? 10 : pageSize; log.info("Request received : /api/view-download-history"); - return ok().body(consumerService.viewDownloadHistory()); + return ok().body(consumerService.viewDownloadHistory(page, pageSize)); } - @GetMapping(value = "/view-download-history-details") + @GetMapping(value = "/view-download-history/{processId}") @PreAuthorize("hasPermission('','consumer_view_download_history')") - public ResponseEntity viewConsumerDownloadHistoryDetails(@RequestParam("processId") String processId) + public ResponseEntity viewConsumerDownloadHistoryDetails(@PathVariable("processId") String processId) throws Exception { log.info("Request received : /api/view-download-history-details"); return ok().body(consumerService.viewConsumerDownloadHistoryDetails(processId)); diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/ProcessReportUseCase.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/ProcessReportUseCase.java index c82f6ffb5..07ac506b4 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/ProcessReportUseCase.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/ProcessReportUseCase.java @@ -103,7 +103,7 @@ public ProcessReportPageResponse listAllProcessReports(int page, int size) { .findAll(PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "startDate"))); List processReports = result.get().map(mapper::mapFrom).toList(); return ProcessReportPageResponse.builder().items(processReports).pageSize(result.getSize()) - .pageNumber(result.getNumber()).totalItems(result.getTotalElements()).build(); + .page(result.getNumber()).totalItems(result.getTotalElements()).build(); } public ProcessReport getProcessReportById(String id) { diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/entity/ConsumerDownloadHistoryEntity.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/entity/ConsumerDownloadHistoryEntity.java index b45fdf5af..92204bce4 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/entity/ConsumerDownloadHistoryEntity.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/entity/ConsumerDownloadHistoryEntity.java @@ -19,6 +19,8 @@ ********************************************************************************/ package org.eclipse.tractusx.sde.core.processreport.entity; +import java.time.LocalDateTime; + import jakarta.persistence.Cacheable; import jakarta.persistence.Column; import jakarta.persistence.Entity; @@ -54,9 +56,9 @@ public class ConsumerDownloadHistoryEntity { @Column(name = "status") private String status; @Column(name = "start_date") - private String startDate; + private LocalDateTime startDate; @Column(name = "end_date") - private String endDate; + private LocalDateTime endDate; @Lob @Column(name = "offers") diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java index d3abc702e..09a80872d 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ConsumerDownloadHistory.java @@ -19,6 +19,8 @@ ********************************************************************************/ package org.eclipse.tractusx.sde.core.processreport.model; +import java.time.LocalDateTime; + import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; @@ -38,8 +40,8 @@ public class ConsumerDownloadHistory { private Integer downloadFailed; private Integer downloadSuccessed; private String status; - private String startDate; - private String endDate; + private LocalDateTime startDate; + private LocalDateTime endDate; private Object offers; private Object policies; private String referenceProcessId; diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ProcessReportPageResponse.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ProcessReportPageResponse.java index 463f3b67d..6e6b1ee1f 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ProcessReportPageResponse.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/model/ProcessReportPageResponse.java @@ -35,7 +35,7 @@ @NoArgsConstructor public class ProcessReportPageResponse { - private int pageNumber; + private int page; private int pageSize; private long totalItems; private List items; diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java index 836df6457..243c30959 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java @@ -38,6 +38,8 @@ import org.eclipse.tractusx.sde.common.enums.ProgressStatusEnum; import org.eclipse.tractusx.sde.common.exception.NoDataFoundException; +import org.eclipse.tractusx.sde.common.model.Acknowledgement; +import org.eclipse.tractusx.sde.common.model.PagingResponse; import org.eclipse.tractusx.sde.common.model.Submodel; import org.eclipse.tractusx.sde.core.failurelog.repository.ConsumerDownloadHistoryRepository; import org.eclipse.tractusx.sde.core.processreport.entity.ConsumerDownloadHistoryEntity; @@ -47,6 +49,9 @@ import org.eclipse.tractusx.sde.edc.model.request.ConsumerRequest; import org.eclipse.tractusx.sde.edc.model.request.Offer; import org.eclipse.tractusx.sde.edc.services.ConsumerControlPanelService; +import org.springframework.data.domain.Page; +import org.springframework.data.domain.PageRequest; +import org.springframework.data.domain.Sort; import org.springframework.stereotype.Service; import org.springframework.util.StreamUtils; @@ -71,69 +76,75 @@ public class ConsumerService { private final ConsumerDownloadHistoryMapper consumerDownloadHistoryMapper; - ObjectMapper mapper = new ObjectMapper(); - @SneakyThrows - public void subscribeAndDownloadDataOffers(ConsumerRequest consumerRequest, HttpServletResponse response) { + public Acknowledgement subscribeAndDownloadDataOffersAsync(ConsumerRequest consumerRequest) { + String processId = UUID.randomUUID().toString(); + + Runnable runnable = () -> subscribeAndDownloadDataOffers(consumerRequest, processId, false); + new Thread(runnable).start(); + return Acknowledgement.builder().id(processId).build(); + } + + @SneakyThrows + public void subscribeAndDownloadDataOffersSynchronous(ConsumerRequest consumerRequest, HttpServletResponse response) { String processId = UUID.randomUUID().toString(); + prepareHttpResponse(response, processId, subscribeAndDownloadDataOffers(consumerRequest, processId, true)); + } + + @SneakyThrows + public Map>> subscribeAndDownloadDataOffers(ConsumerRequest consumerRequest, + String processId, boolean flagToDownloadImidiate) { + AtomicInteger failedCount = new AtomicInteger(); AtomicInteger successCount = new AtomicInteger(); - String startDate = LocalDateTime.now().toString(); - - ConsumerDownloadHistoryEntity entity = ConsumerDownloadHistoryEntity.builder() - .startDate(startDate) - .endDate(LocalDateTime.now().toString()) - .connectorId(consumerRequest.getConnectorId()) - .providerUrl(consumerRequest.getProviderUrl()) - .numberOfItems(consumerRequest.getOffers().size()) - .downloadSuccessed(successCount.get()) - .downloadFailed(failedCount.get()) - .processId(processId) - .status(ProgressStatusEnum.IN_PROGRESS.toString()) - .build(); - + + ConsumerDownloadHistoryEntity entity = ConsumerDownloadHistoryEntity.builder().startDate(LocalDateTime.now()) + .connectorId(consumerRequest.getConnectorId()).providerUrl(consumerRequest.getProviderUrl()) + .numberOfItems(consumerRequest.getOffers().size()).downloadSuccessed(successCount.get()) + .downloadFailed(failedCount.get()).processId(processId) + .status(ProgressStatusEnum.IN_PROGRESS.toString()).build(); + // Save consumer Download history in DB consumerDownloadHistoryRepository.save(entity); - - Map subscribeAndDownloadDataOffers = consumerControlPanelService - .subscribeAndDownloadDataOffers(consumerRequest); + Map subscribeAndDownloadDataOffers = consumerControlPanelService + .subscribeAndDownloadDataOffers(consumerRequest, flagToDownloadImidiate); Map>> csvWithValue = new TreeMap<>(); consumerRequest.getOffers().stream().forEach(offer -> prepareFromOfferResponse(subscribeAndDownloadDataOffers, - failedCount, successCount, csvWithValue, offer)); + failedCount, successCount, csvWithValue, offer, flagToDownloadImidiate)); - - entity.setEndDate(LocalDateTime.now().toString()); + entity.setEndDate(LocalDateTime.now()); entity.setOffers(mapper.writeValueAsString(consumerRequest.getOffers())); entity.setPolicies(mapper.writeValueAsString(consumerRequest.getPolicies())); entity.setDownloadSuccessed(successCount.get()); entity.setDownloadFailed(failedCount.get()); - + entity.setStatus(ProgressStatusEnum.FAILED.toString()); if (consumerRequest.getOffers().size() == successCount.get()) entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); - else if(successCount.get()!=0 && failedCount.get()!=0 ) + else if (successCount.get() != 0 && failedCount.get() != 0) entity.setStatus(ProgressStatusEnum.PARTIALED_FAILED.toString()); // Save consumer Download history in DB consumerDownloadHistoryRepository.save(entity); - prepareHttpResponse(response, processId, csvWithValue); + return csvWithValue; + } @SneakyThrows public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(String referenceProcessId, HttpServletResponse response) { - + String processId = UUID.randomUUID().toString(); - + ConsumerDownloadHistoryEntity entity = consumerDownloadHistoryRepository.findByProcessId(referenceProcessId); - - if (entity!=null && entity.getOffers() != null) { + + if (entity != null && entity.getOffers() != null) { List offerList = mapper.readValue(entity.getOffers(), new TypeReference>() { }); @@ -142,7 +153,7 @@ public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(String refe AtomicInteger failedCount = new AtomicInteger(); AtomicInteger successCount = new AtomicInteger(); - entity.setStartDate(LocalDateTime.now().toString()); + entity.setStartDate(LocalDateTime.now()); entity.setNumberOfItems(offerList.size()); entity.setDownloadSuccessed(successCount.get()); entity.setDownloadFailed(failedCount.get()); @@ -163,19 +174,19 @@ public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(String refe offerList.stream() .forEach(offer -> prepareFromOfferResponse( downloadFileFromEDCUsingifAlreadyTransferStatusCompleted, failedCount, successCount, - csvWithValue, offer)); + csvWithValue, offer, true)); - entity.setEndDate(LocalDateTime.now().toString()); + entity.setEndDate(LocalDateTime.now()); entity.setOffers(mapper.writeValueAsString(offerList)); entity.setDownloadSuccessed(successCount.get()); entity.setDownloadFailed(failedCount.get()); entity.setProcessId(processId); entity.setReferenceProcessId(referenceProcessId); - + entity.setStatus(ProgressStatusEnum.FAILED.toString()); if (offerList.size() == successCount.get()) entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); - else if(successCount.get()!=0 && failedCount.get()!=0 ) + else if (successCount.get() != 0 && failedCount.get() != 0) entity.setStatus(ProgressStatusEnum.PARTIALED_FAILED.toString()); // Save consumer Download history in DB @@ -183,15 +194,16 @@ else if(successCount.get()!=0 && failedCount.get()!=0 ) prepareHttpResponse(response, processId, csvWithValue); } else { - generateFailureJsonResponse(response, "Unable to find data offer in SDE for redownload"); + generateFailureJsonResponse(response, "Unable to find data offer in SDE for download"); } } else { - generateFailureJsonResponse(response, "Unable to find data offer in SDE for redownload"); + generateFailureJsonResponse(response, "Unable to find data offer in SDE for download"); } } private void prepareFromOfferResponse(Map subscribeAndDownloadDataOffers, AtomicInteger failedCount, - AtomicInteger successCount, Map>> csvWithValue, Offer offer) { + AtomicInteger successCount, Map>> csvWithValue, Offer offer, + boolean flagToDownloadImidiate) { Object object = subscribeAndDownloadDataOffers.get(offer.getAssetId()); if (object != null) { @@ -200,8 +212,11 @@ private void prepareFromOfferResponse(Map subscribeAndDownloadDa JsonNode dataNode = node.get("data"); JsonNode edrNode = node.get("edr"); - if (dataNode != null) { + if (dataNode != null && flagToDownloadImidiate) { processCSVDataObject(successCount, failedCount, csvWithValue, offer, status, dataNode); + } else if (!flagToDownloadImidiate && "SUCCESS".equals(status.asText())) { + offer.setStatus(status.asText()); + successCount.getAndIncrement(); } else { offer.setStatus(status.asText()); failedCount.getAndIncrement(); @@ -248,7 +263,8 @@ private void processCSVDataObject(AtomicInteger successCount, AtomicInteger fail private void prepareHttpResponse(HttpServletResponse response, String processId, Map>> csvWithValue) throws IOException { if (csvWithValue.isEmpty()) { - generateFailureJsonResponse(response, "Unable to process your request, please try again, if error persist contact to admin"); + generateFailureJsonResponse(response, + "Unable to process your request, please try again"); } else { prepareZipFiles(response, csvWithValue, processId); } @@ -309,9 +325,14 @@ private void prepareZipFiles(HttpServletResponse response, Map viewDownloadHistory() { - return consumerDownloadHistoryRepository.findAll().stream() - .map(consumerDownloadHistoryMapper::mapFromCustom).toList(); + public PagingResponse viewDownloadHistory(Integer page, Integer pageSize) { + Page result = consumerDownloadHistoryRepository + .findAll(PageRequest.of(page, pageSize, Sort.by(Sort.Direction.DESC, "endDate"))); + List processReports = result.stream().map(consumerDownloadHistoryMapper::mapFromCustom) + .toList(); + return PagingResponse.builder().items(processReports).pageSize(result.getSize()).page(result.getNumber()) + .totalItems(result.getTotalElements()).build(); + } public ConsumerDownloadHistory viewConsumerDownloadHistoryDetails(String processId) { diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index df02deea9..d254508a8 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -58,7 +58,6 @@ import com.fasterxml.jackson.databind.JsonNode; import feign.FeignException; -import jakarta.validation.Valid; import lombok.RequiredArgsConstructor; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; @@ -246,7 +245,7 @@ public void subscribeDataOffers(ConsumerRequest consumerRequest, String processI } - public Map subscribeAndDownloadDataOffers(@Valid ConsumerRequest consumerRequest) { + public Map subscribeAndDownloadDataOffers(ConsumerRequest consumerRequest, boolean flagToDownloadImidiate) { HashMap extensibleProperty = new HashMap<>(); Map response = new ConcurrentHashMap<>(); @@ -270,7 +269,10 @@ public Map subscribeAndDownloadDataOffers(@Valid ConsumerRequest extensibleProperty, recipientURL, action, offer); resultFields.put("edr", checkContractNegotiationStatus); - resultFields.put("data", downloadFile(checkContractNegotiationStatus)); + + if (flagToDownloadImidiate) + resultFields.put("data", downloadFile(checkContractNegotiationStatus)); + resultFields.put(STATUS, "SUCCESS"); } catch (FeignException e) { From c2570f8ca621fc4824963c85184a89774fdedec0 Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Tue, 12 Sep 2023 18:24:12 +0530 Subject: [PATCH 14/23] Add exception handling for download file --- .../sde/core/service/ConsumerService.java | 6 +- .../ContractOfferRequestFactory.java | 2 +- .../services/ConsumerControlPanelService.java | 86 ++++++++++++------- 3 files changed, 59 insertions(+), 35 deletions(-) diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java index 243c30959..80af888b4 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java @@ -224,9 +224,9 @@ private void prepareFromOfferResponse(Map subscribeAndDownloadDa } if (edrNode != null) { - offer.setAgreementId(readFieldFromJsonNode(edrNode, "agreementId")); - offer.setExpirationDate(readFieldFromJsonNode(edrNode, "expirationDate")); - offer.setTransferProcessId(readFieldFromJsonNode(edrNode, "transferProcessId")); + offer.setAgreementId(readFieldFromJsonNode(edrNode, "edc:agreementId")); + offer.setExpirationDate(readFieldFromJsonNode(edrNode, "tx:expirationDate")); + offer.setTransferProcessId(readFieldFromJsonNode(edrNode, "edc:transferProcessId")); } } } diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/contractoffers/ContractOfferRequestFactory.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/contractoffers/ContractOfferRequestFactory.java index 4462a5f24..714f537d8 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/contractoffers/ContractOfferRequestFactory.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/model/contractoffers/ContractOfferRequestFactory.java @@ -44,7 +44,7 @@ public ObjectNode getContractOfferRequest(String providerUrl, Integer limit, Int { "@context": {}, "protocol": "dataspace-protocol-http", - "providerUrl": "%s", + "counterPartyAddress": "%s", "querySpec": { "offset": %s, "limit": %s diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index d254508a8..2535607ad 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -29,6 +29,7 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.eclipse.tractusx.sde.common.entities.UsagePolicies; @@ -67,6 +68,7 @@ @RequiredArgsConstructor public class ConsumerControlPanelService extends AbstractEDCStepsHelper { + private static final String NEGOTIATED = "NEGOTIATED"; private static final String STATUS = "status"; private final ContractOfferCatalogApi contractOfferCatalogApiProxy; private final ContractNegotiateManagementHelper contractNegotiateManagement; @@ -78,9 +80,9 @@ public class ConsumerControlPanelService extends AbstractEDCStepsHelper { private final ContractOfferRequestFactory contractOfferRequestFactory; private static final Integer RETRY = 5; - - private static final Integer THRED_SLEEP_TIME=5000; - + + private static final Integer THRED_SLEEP_TIME = 5000; + public List queryOnDataOffers(String providerUrl, Integer offset, Integer limit, String filterExpression) { @@ -245,7 +247,8 @@ public void subscribeDataOffers(ConsumerRequest consumerRequest, String processI } - public Map subscribeAndDownloadDataOffers(ConsumerRequest consumerRequest, boolean flagToDownloadImidiate) { + public Map subscribeAndDownloadDataOffers(ConsumerRequest consumerRequest, + boolean flagToDownloadImidiate) { HashMap extensibleProperty = new HashMap<>(); Map response = new ConcurrentHashMap<>(); @@ -269,10 +272,10 @@ public Map subscribeAndDownloadDataOffers(ConsumerRequest consum extensibleProperty, recipientURL, action, offer); resultFields.put("edr", checkContractNegotiationStatus); - + if (flagToDownloadImidiate) resultFields.put("data", downloadFile(checkContractNegotiationStatus)); - + resultFields.put(STATUS, "SUCCESS"); } catch (FeignException e) { @@ -281,7 +284,7 @@ public Map subscribeAndDownloadDataOffers(ConsumerRequest consum log.error(errorMsg); prepareErrorMap(resultFields, errorMsg); } catch (Exception e) { - log.error("SubscribeAndDownloadDataOffers Oops! We have an Exception -" + e.getMessage()); + log.error("SubscribeAndDownloadDataOffers Oops! We have -" + e.getMessage()); String errorMsg = "Unable to complete subscribeAndDownloadDataOffers because: " + e.getMessage(); prepareErrorMap(resultFields, errorMsg); } finally { @@ -312,40 +315,40 @@ private EDRCachedResponse verifyOrCreateContractNegotiation(ConsumerRequest cons @SneakyThrows private EDRCachedResponse verifyEDRRequestStatus(String assetId, int counter) { EDRCachedResponse eDRCachedResponse = null; + AtomicInteger isRefresheEDRToken = new AtomicInteger(); List eDRCachedResponseList = null; try { do { + isRefresheEDRToken.set(0); Thread.sleep(THRED_SLEEP_TIME); eDRCachedResponseList = edrRequestHelper.getEDRCachedByAsset(assetId); counter++; - - if (eDRCachedResponseList != null && !eDRCachedResponseList.isEmpty()) { - Optional findAny = eDRCachedResponseList.stream() - .filter(obj -> obj.getEdrState().equals("NEGOTIATED")).findAny(); - if (findAny.isPresent()) - eDRCachedResponse = findAny.get(); - } - + eDRCachedResponse = verifyEDRResponse(eDRCachedResponse, isRefresheEDRToken, eDRCachedResponseList); } while ((eDRCachedResponse == null && counter <= RETRY) - || (eDRCachedResponse != null && !eDRCachedResponse.getEdrState().equals("NEGOTIATED"))); + || (eDRCachedResponse != null && !eDRCachedResponse.getEdrState().equals(NEGOTIATED))); + + if (eDRCachedResponseList != null && eDRCachedResponseList.size() >= isRefresheEDRToken.get() + && eDRCachedResponse == null) + throw new ServiceException( + "Time out!! EDC refreshing EDR tokens, unable to get EDR negotiated status"); if (eDRCachedResponse == null) - throw new ServiceException("Time out!! to verify contract negotiated for asset " + assetId); - + throw new ServiceException("Time out!! unable to get EDR negotiated status"); + } catch (FeignException e) { log.error("RequestBody: " + e.request()); - String errorMsg = "FeignExceptionton to verify negotiated contract for asset " + assetId + ",error: " + String errorMsg = "FeignExceptionton to verify EDR negotiated status for asset " + assetId + "," + e.contentUTF8(); log.error("Response: " + errorMsg); throw new ServiceException(errorMsg); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); - String errorMsg = "InterruptedException to verify negotiated contract for asset " + assetId + ", error: " + String errorMsg = "InterruptedException to verify EDR negotiated status for asset " + assetId + "," + ie.getMessage(); log.error(errorMsg); throw new ServiceException(errorMsg); } catch (Exception e) { - String errorMsg = "Exception to verify negotiated contract for asset " + assetId + ",Exception error: " + String errorMsg = "Exception to verify EDR negotiated status for asset " + assetId + "," + e.getMessage(); log.error(errorMsg); throw new ServiceException(errorMsg); @@ -353,6 +356,22 @@ private EDRCachedResponse verifyEDRRequestStatus(String assetId, int counter) { return eDRCachedResponse; } + private EDRCachedResponse verifyEDRResponse(EDRCachedResponse eDRCachedResponse, AtomicInteger isRefresheEDRToken, + List eDRCachedResponseList) { + if (eDRCachedResponseList != null && !eDRCachedResponseList.isEmpty()) { + for (EDRCachedResponse edrCachedResponseObj : eDRCachedResponseList) { + String edrState = edrCachedResponseObj.getEdrState(); + if (edrState.equals(NEGOTIATED)) { + eDRCachedResponse = edrCachedResponseObj; + } + if ("REFRESHING".equalsIgnoreCase(edrState)) { + isRefresheEDRToken.getAndIncrement(); + } + } + } + return eDRCachedResponse; + } + @SneakyThrows private EDRCachedByIdResponse getAuthorizationTokenForDataDownload(String transferProcessId) { return edrRequestHelper.getEDRCachedByTransferProcessId(transferProcessId); @@ -369,14 +388,8 @@ public Map downloadFileFromEDCUsingifAlreadyTransferStatusComple downloadResultFields.put("edr", verifyEDRRequestStatus); downloadResultFields.put("data", downloadFile(verifyEDRRequestStatus)); downloadResultFields.put(STATUS, "SUCCESS"); - } catch (FeignException e) { - log.error("Re-download RequestBody: " + e.request()); - String errorMsg = "Unable to download existing subcribe data offer because: " + e.contentUTF8(); - log.error(errorMsg); - prepareErrorMap(downloadResultFields, errorMsg); } catch (Exception e) { - log.error("DownloadFileFromEDCUsingifAlreadyTransferStatusCompleted Oops! We have an Exception -" + e.getMessage()); - String errorMsg = "Unable to download existing subcribe data offer because: " + e.getMessage(); + String errorMsg = "DownloadFileFromEDCUsingifAlreadyTransferStatusCompleted Oops! We have -" + e.getMessage(); prepareErrorMap(downloadResultFields, errorMsg); } finally { response.put(assetId, downloadResultFields); @@ -390,11 +403,22 @@ private void prepareErrorMap(Map resultFields, String errorMsg) resultFields.put("error", errorMsg); } + @SneakyThrows private Object downloadFile(EDRCachedResponse verifyEDRRequestStatus) { if (verifyEDRRequestStatus != null) { - EDRCachedByIdResponse authorizationToken = getAuthorizationTokenForDataDownload( - verifyEDRRequestStatus.getTransferProcessId()); - return edrRequestHelper.getDataFromProvider(authorizationToken); + try { + EDRCachedByIdResponse authorizationToken = getAuthorizationTokenForDataDownload( + verifyEDRRequestStatus.getTransferProcessId()); + return edrRequestHelper.getDataFromProvider(authorizationToken); + } catch (FeignException e) { + log.error("Download RequestBody: " + e.request()); + String errorMsg = "Unable to download subcribe data offer because: " + e.contentUTF8(); + throw new ServiceException(errorMsg); + } catch (Exception e) { + log.error("DownloadFileFromEDCUsingifAlreadyTransferStatusCompleted Oops! We have -" + e.getMessage()); + String errorMsg = "Unable to download subcribe data offer because: " + e.getMessage(); + throw new ServiceException(errorMsg); + } } return null; } From 3e34ce06dce398191b8d429b71226b59619c6704 Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Wed, 13 Sep 2023 10:09:01 +0530 Subject: [PATCH 15/23] Corrected process status spell --- .../eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java | 2 +- .../eclipse/tractusx/sde/core/service/ConsumerService.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java index f169fb12f..fa1e003ee 100644 --- a/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java +++ b/modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java @@ -26,5 +26,5 @@ public enum ProgressStatusEnum { IN_PROGRESS, COMPLETED, FAILED, - PARTIALED_FAILED + PARTIALLY_FAILED } diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java index 80af888b4..0eab8bcda 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java @@ -127,7 +127,7 @@ public Map>> subscribeAndDownloadDataOffers(ConsumerRe if (consumerRequest.getOffers().size() == successCount.get()) entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); else if (successCount.get() != 0 && failedCount.get() != 0) - entity.setStatus(ProgressStatusEnum.PARTIALED_FAILED.toString()); + entity.setStatus(ProgressStatusEnum.PARTIALLY_FAILED.toString()); // Save consumer Download history in DB consumerDownloadHistoryRepository.save(entity); @@ -187,7 +187,7 @@ public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(String refe if (offerList.size() == successCount.get()) entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); else if (successCount.get() != 0 && failedCount.get() != 0) - entity.setStatus(ProgressStatusEnum.PARTIALED_FAILED.toString()); + entity.setStatus(ProgressStatusEnum.PARTIALLY_FAILED.toString()); // Save consumer Download history in DB consumerDownloadHistoryRepository.save(entity); From 0772f878a5df3632d08db7b1a01e98d15566f06b Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Wed, 13 Sep 2023 23:26:17 +0530 Subject: [PATCH 16/23] Update depenfdency and change zip download log --- DEPENDENCIES | 55 ++++++++++++++++++- modules/sde-core/pom.xml | 7 +++ .../sde/core/service/ConsumerService.java | 28 ++++++---- 3 files changed, 76 insertions(+), 14 deletions(-) diff --git a/DEPENDENCIES b/DEPENDENCIES index d16024b26..b6557e09d 100644 --- a/DEPENDENCIES +++ b/DEPENDENCIES @@ -1,16 +1,33 @@ maven/mavencentral/ch.qos.logback/logback-classic/1.4.7, EPL-1.0 OR LGPL-2.1-only, approved, #3435 maven/mavencentral/ch.qos.logback/logback-core/1.4.7, EPL-1.0 OR LGPL-2.1-only, approved, #3373 +maven/mavencentral/com.ethlo.time/itu/1.7.0, Apache-2.0, approved, clearlydefined maven/mavencentral/com.fasterxml.jackson.core/jackson-annotations/2.15.0, Apache-2.0, approved, #7947 maven/mavencentral/com.fasterxml.jackson.core/jackson-core/2.15.0, MIT AND Apache-2.0, approved, #7932 maven/mavencentral/com.fasterxml.jackson.core/jackson-databind/2.15.0, Apache-2.0, approved, #7934 +maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-toml/2.15.0, Apache-2.0, approved, #9160 +maven/mavencentral/com.fasterxml.jackson.dataformat/jackson-dataformat-yaml/2.15.0, Apache-2.0, approved, #8802 maven/mavencentral/com.fasterxml.jackson.datatype/jackson-datatype-jdk8/2.15.0, Apache-2.0, approved, #8808 maven/mavencentral/com.fasterxml.jackson.datatype/jackson-datatype-jsr310/2.15.0, Apache-2.0, approved, #7930 maven/mavencentral/com.fasterxml.jackson.module/jackson-module-parameter-names/2.15.0, Apache-2.0, approved, #8803 maven/mavencentral/com.fasterxml/classmate/1.5.1, Apache-2.0, approved, clearlydefined +maven/mavencentral/com.github.ben-manes.caffeine/caffeine/3.1.6, Apache-2.0, approved, clearlydefined +maven/mavencentral/com.github.stephenc.jcip/jcip-annotations/1.0-1, Apache-2.0, approved, CQ21949 +maven/mavencentral/com.google.code.gson/gson/2.10, Apache-2.0, approved, #6159 +maven/mavencentral/com.google.code.gson/gson/2.10.1, Apache-2.0, approved, #6159 +maven/mavencentral/com.google.errorprone/error_prone_annotations/2.18.0, Apache-2.0, approved, clearlydefined maven/mavencentral/com.jayway.jsonpath/json-path/2.8.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/com.networknt/json-schema-validator/1.0.72, Apache-2.0, approved, CQ22638 +maven/mavencentral/com.nimbusds/nimbus-jose-jwt/9.31, Apache-2.0, approved, clearlydefined +maven/mavencentral/com.opencsv/opencsv/5.8, Apache-2.0, approved, clearlydefined maven/mavencentral/com.sun.istack/istack-commons-runtime/4.1.1, BSD-3-Clause, approved, #2590 maven/mavencentral/com.vaadin.external.google/android-json/0.0.20131108.vaadin1, Apache-2.0, approved, CQ21310 maven/mavencentral/com.zaxxer/HikariCP/5.0.1, Apache-2.0, approved, clearlydefined +maven/mavencentral/commons-beanutils/commons-beanutils/1.9.4, Apache-2.0, approved, CQ12654 +maven/mavencentral/commons-codec/commons-codec/1.15, Apache-2.0 AND BSD-3-Clause AND LicenseRef-Public-Domain, approved, CQ22641 +maven/mavencentral/commons-collections/commons-collections/3.2.2, Apache-2.0, approved, CQ10385 +maven/mavencentral/commons-io/commons-io/2.7, Apache-2.0, approved, clearlydefined +maven/mavencentral/commons-logging/commons-logging/1.2, Apache-2.0, approved, CQ10162 +maven/mavencentral/io.github.classgraph/classgraph/4.8.149, MIT, approved, CQ22530 maven/mavencentral/io.github.openfeign.form/feign-form-spring/3.8.0, Apache-2.0, approved, clearlydefined maven/mavencentral/io.github.openfeign.form/feign-form/3.8.0, Apache-2.0, approved, clearlydefined maven/mavencentral/io.github.openfeign/feign-core/12.3, Apache-2.0, approved, clearlydefined @@ -18,6 +35,9 @@ maven/mavencentral/io.github.openfeign/feign-slf4j/12.3, Apache-2.0, approved, c maven/mavencentral/io.micrometer/micrometer-commons/1.11.0, Apache-2.0 AND (Apache-2.0 AND MIT), approved, #9243 maven/mavencentral/io.micrometer/micrometer-observation/1.11.0, Apache-2.0, approved, #9242 maven/mavencentral/io.smallrye/jandex/3.0.5, Apache-2.0, approved, clearlydefined +maven/mavencentral/io.swagger.core.v3/swagger-annotations-jakarta/2.2.7, Apache-2.0, approved, #5947 +maven/mavencentral/io.swagger.core.v3/swagger-core-jakarta/2.2.7, Apache-2.0, approved, #5929 +maven/mavencentral/io.swagger.core.v3/swagger-models-jakarta/2.2.7, Apache-2.0, approved, #5919 maven/mavencentral/jakarta.activation/jakarta.activation-api/2.1.2, EPL-2.0 OR BSD-3-Clause OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.jaf maven/mavencentral/jakarta.annotation/jakarta.annotation-api/2.1.1, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.ca maven/mavencentral/jakarta.inject/jakarta.inject-api/2.0.1, Apache-2.0, approved, clearlydefined @@ -30,6 +50,10 @@ maven/mavencentral/net.bytebuddy/byte-buddy/1.14.4, Apache-2.0 AND BSD-3-Clause, maven/mavencentral/net.minidev/accessors-smart/2.4.9, Apache-2.0, approved, #7515 maven/mavencentral/net.minidev/json-smart/2.4.10, Apache-2.0, approved, #3288 maven/mavencentral/org.antlr/antlr4-runtime/4.10.1, BSD-3-Clause AND LicenseRef-Public-domain AND MIT AND LicenseRef-Unicode-TOU, approved, #7065 +maven/mavencentral/org.apache.commons/commons-collections4/4.4, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.apache.commons/commons-csv/1.8, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.apache.commons/commons-lang3/3.12.0, Apache-2.0, approved, clearlydefined +maven/mavencentral/org.apache.commons/commons-text/1.10.0, Apache-2.0, approved, clearlydefined maven/mavencentral/org.apache.logging.log4j/log4j-api/2.17.1, Apache-2.0, approved, clearlydefined maven/mavencentral/org.apache.logging.log4j/log4j-to-slf4j/2.17.1, Apache-2.0, approved, #2163 maven/mavencentral/org.apache.tomcat.embed/tomcat-embed-core/10.1.8, Apache-2.0 AND (EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0) AND (CDDL-1.0 OR GPL-2.0-only WITH Classpath-exception-2.0) AND W3C AND CC0-1.0, approved, #5949 @@ -41,7 +65,23 @@ maven/mavencentral/org.assertj/assertj-core/3.24.2, Apache-2.0, approved, #6161 maven/mavencentral/org.bouncycastle/bcpkix-jdk15on/1.69, MIT, approved, clearlydefined maven/mavencentral/org.bouncycastle/bcprov-jdk15on/1.69, MIT, approved, clearlydefined maven/mavencentral/org.bouncycastle/bcutil-jdk15on/1.69, MIT, approved, clearlydefined +maven/mavencentral/org.checkerframework/checker-qual/3.33.0, MIT, approved, clearlydefined maven/mavencentral/org.eclipse.angus/angus-activation/2.0.0, EPL-2.0 OR GPL-2.0-only with Classpath-exception-2.0, approved, ee4j.angus +maven/mavencentral/org.eclipse.persistence/eclipselink/3.0.3, EPL-2.0 OR BSD-3-Clause, approved, ee4j.eclipselink +maven/mavencentral/org.eclipse.tractusx/assembly-part-relationship/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/batch/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/bpn-discovery/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/digital-twins/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/edc/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/part-as-planned/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/part-site-information-as-planned/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/pcf/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/portal/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/sde-common/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/serial-part-typization/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/single-level-bom-as-planned/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.eclipse.tractusx/single-level-usage-as-built/0.0.1, Apache-2.0, approved, automotive.tractusx +maven/mavencentral/org.flywaydb/flyway-core/9.16.3, Apache-2.0, approved, #7935 maven/mavencentral/org.glassfish.jaxb/jaxb-core/4.0.2, BSD-3-Clause, approved, ee4j.jaxb maven/mavencentral/org.glassfish.jaxb/jaxb-runtime/4.0.2, BSD-3-Clause, approved, ee4j.jaxb maven/mavencentral/org.glassfish.jaxb/txw2/4.0.2, BSD-3-Clause, approved, ee4j.jaxb @@ -62,17 +102,23 @@ maven/mavencentral/org.mockito/mockito-junit-jupiter/4.8.1, MIT, approved, clear maven/mavencentral/org.objenesis/objenesis/3.2, Apache-2.0, approved, clearlydefined maven/mavencentral/org.opentest4j/opentest4j/1.2.0, Apache-2.0, approved, clearlydefined maven/mavencentral/org.ow2.asm/asm/9.3, BSD-3-Clause, approved, clearlydefined +maven/mavencentral/org.postgresql/postgresql/42.6.0, BSD-2-Clause AND Apache-2.0, approved, #9159 maven/mavencentral/org.projectlombok/lombok/1.18.26, MIT AND LicenseRef-Public-Domain, approved, CQ23907 maven/mavencentral/org.skyscreamer/jsonassert/1.5.1, Apache-2.0, approved, clearlydefined maven/mavencentral/org.slf4j/jul-to-slf4j/2.0.7, MIT, approved, #7698 maven/mavencentral/org.slf4j/slf4j-api/2.0.7, MIT, approved, #5915 +maven/mavencentral/org.springdoc/springdoc-openapi-starter-common/2.0.2, Apache-2.0, approved, #5920 +maven/mavencentral/org.springdoc/springdoc-openapi-starter-webmvc-api/2.0.2, Apache-2.0, approved, #5950 +maven/mavencentral/org.springdoc/springdoc-openapi-starter-webmvc-ui/2.0.2, Apache-2.0, approved, #5923 maven/mavencentral/org.springframework.boot/spring-boot-autoconfigure/3.1.0, Apache-2.0, approved, #9341 maven/mavencentral/org.springframework.boot/spring-boot-devtools/3.1.0, Apache-2.0, approved, clearlydefined maven/mavencentral/org.springframework.boot/spring-boot-starter-aop/3.1.0, Apache-2.0, approved, #9338 +maven/mavencentral/org.springframework.boot/spring-boot-starter-cache/3.1.0, Apache-2.0, approved, #9653 maven/mavencentral/org.springframework.boot/spring-boot-starter-data-jpa/3.1.0, Apache-2.0, approved, #9733 maven/mavencentral/org.springframework.boot/spring-boot-starter-jdbc/3.1.0, Apache-2.0, approved, #9737 maven/mavencentral/org.springframework.boot/spring-boot-starter-json/3.1.0, Apache-2.0, approved, #9336 maven/mavencentral/org.springframework.boot/spring-boot-starter-logging/3.1.0, Apache-2.0, approved, #9343 +maven/mavencentral/org.springframework.boot/spring-boot-starter-oauth2-resource-server/3.1.0, Apache-2.0, approved, #8804 maven/mavencentral/org.springframework.boot/spring-boot-starter-security/3.1.0, Apache-2.0, approved, #9337 maven/mavencentral/org.springframework.boot/spring-boot-starter-test/3.1.0, Apache-2.0, approved, #9353 maven/mavencentral/org.springframework.boot/spring-boot-starter-tomcat/3.1.0, Apache-2.0, approved, #9351 @@ -89,15 +135,18 @@ maven/mavencentral/org.springframework.cloud/spring-cloud-starter-openfeign/4.0. maven/mavencentral/org.springframework.cloud/spring-cloud-starter/4.0.3, Apache-2.0, approved, #7299 maven/mavencentral/org.springframework.data/spring-data-commons/3.1.0, Apache-2.0, approved, #8805 maven/mavencentral/org.springframework.data/spring-data-jpa/3.1.0, Apache-2.0, approved, #9120 -maven/mavencentral/org.springframework.security/spring-security-config/6.1.0, Apache-2.0, approved, #9736 -maven/mavencentral/org.springframework.security/spring-security-core/6.1.0, Apache-2.0, approved, #9801 +maven/mavencentral/org.springframework.security/spring-security-config/6.1.2, Apache-2.0, approved, #9736 maven/mavencentral/org.springframework.security/spring-security-core/6.1.2, Apache-2.0, approved, #9801 maven/mavencentral/org.springframework.security/spring-security-crypto/6.1.0, Apache-2.0 AND ISC, approved, #9735 +maven/mavencentral/org.springframework.security/spring-security-oauth2-core/6.1.0, Apache-2.0, approved, #9741 +maven/mavencentral/org.springframework.security/spring-security-oauth2-jose/6.1.0, Apache-2.0, approved, #9345 +maven/mavencentral/org.springframework.security/spring-security-oauth2-resource-server/6.1.0, Apache-2.0, approved, #8798 maven/mavencentral/org.springframework.security/spring-security-rsa/1.0.11.RELEASE, Apache-2.0, approved, CQ20647 maven/mavencentral/org.springframework.security/spring-security-web/6.1.0, Apache-2.0, approved, #9800 maven/mavencentral/org.springframework/spring-aop/6.0.9, Apache-2.0, approved, #5940 maven/mavencentral/org.springframework/spring-aspects/6.0.9, Apache-2.0, approved, #5930 maven/mavencentral/org.springframework/spring-beans/6.0.9, Apache-2.0, approved, #5937 +maven/mavencentral/org.springframework/spring-context-support/6.0.9, Apache-2.0, approved, #6960 maven/mavencentral/org.springframework/spring-context/6.0.9, Apache-2.0, approved, #5936 maven/mavencentral/org.springframework/spring-core/6.0.9, Apache-2.0 AND BSD-3-Clause, approved, #5948 maven/mavencentral/org.springframework/spring-expression/6.0.9, Apache-2.0, approved, #3284 @@ -108,5 +157,7 @@ maven/mavencentral/org.springframework/spring-test/6.0.9, Apache-2.0, approved, maven/mavencentral/org.springframework/spring-tx/6.0.9, Apache-2.0, approved, #5926 maven/mavencentral/org.springframework/spring-web/6.0.9, Apache-2.0, approved, #5942 maven/mavencentral/org.springframework/spring-webmvc/6.0.9, Apache-2.0, approved, #5944 +maven/mavencentral/org.webjars/swagger-ui/4.15.5, Apache-2.0 AND MIT, approved, #5921 +maven/mavencentral/org.webjars/webjars-locator-core/0.52, MIT, approved, clearlydefined maven/mavencentral/org.xmlunit/xmlunit-core/2.9.1, Apache-2.0, approved, #6272 maven/mavencentral/org.yaml/snakeyaml/2.0, Apache-2.0 AND (Apache-2.0 OR BSD-3-Clause OR EPL-1.0 OR GPL-2.0-or-later OR LGPL-2.1-or-later), approved, #7275 diff --git a/modules/sde-core/pom.xml b/modules/sde-core/pom.xml index 3d0cb2080..42327ea30 100644 --- a/modules/sde-core/pom.xml +++ b/modules/sde-core/pom.xml @@ -177,6 +177,13 @@ pcf 0.0.1 + + + com.opencsv + opencsv + 5.8 + + diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java index 0eab8bcda..65e0178ca 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java @@ -20,8 +20,8 @@ package org.eclipse.tractusx.sde.core.service; -import java.io.ByteArrayInputStream; import java.io.IOException; +import java.io.OutputStreamWriter; import java.io.PrintWriter; import java.time.LocalDateTime; import java.util.ArrayList; @@ -45,7 +45,6 @@ import org.eclipse.tractusx.sde.core.processreport.entity.ConsumerDownloadHistoryEntity; import org.eclipse.tractusx.sde.core.processreport.mapper.ConsumerDownloadHistoryMapper; import org.eclipse.tractusx.sde.core.processreport.model.ConsumerDownloadHistory; -import org.eclipse.tractusx.sde.core.utils.CsvUtil; import org.eclipse.tractusx.sde.edc.model.request.ConsumerRequest; import org.eclipse.tractusx.sde.edc.model.request.Offer; import org.eclipse.tractusx.sde.edc.services.ConsumerControlPanelService; @@ -53,12 +52,12 @@ import org.springframework.data.domain.PageRequest; import org.springframework.data.domain.Sort; import org.springframework.stereotype.Service; -import org.springframework.util.StreamUtils; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.gson.Gson; +import com.opencsv.CSVWriter; import jakarta.servlet.http.HttpServletResponse; import lombok.AllArgsConstructor; @@ -306,17 +305,22 @@ private void prepareZipFiles(HttpServletResponse response, Map> value = entry.getValue(); - ByteArrayInputStream file = CsvUtil.writeCsv(value); + // create a zip entry and add it to ZipOutputStream ZipEntry e = new ZipEntry(fileName + ".csv"); - // Configure the zip entry, the properties of the file - e.setSize(file.read()); - - e.setTime(System.currentTimeMillis()); - // etc. zippedOut.putNextEntry(e); - // And the content of the resource: - StreamUtils.copy(file, zippedOut); - + // There is no need for staging the CSV on filesystem or reading bytes into + // memory. Directly write bytes to the output stream. + CSVWriter writer = new CSVWriter(new OutputStreamWriter(zippedOut)); + for (List list : value) { + String[] strarray = new String[list.size()]; + list.toArray(strarray); + // write the contents + writer.writeNext(strarray, false); + } + // flush the writer. Very important! + writer.flush(); + // close the entry. Note : we are not closing the zos just yet as we need to add + // more files to our ZIP zippedOut.closeEntry(); } zippedOut.finish(); From d8881ca5c1abff24d39f4992ef97bfcc66b3ddfe Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Wed, 13 Sep 2023 23:38:19 +0530 Subject: [PATCH 17/23] Update download error log --- .../tractusx/sde/edc/services/ConsumerControlPanelService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index 2535607ad..6e3c0ba88 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -389,7 +389,7 @@ public Map downloadFileFromEDCUsingifAlreadyTransferStatusComple downloadResultFields.put("data", downloadFile(verifyEDRRequestStatus)); downloadResultFields.put(STATUS, "SUCCESS"); } catch (Exception e) { - String errorMsg = "DownloadFileFromEDCUsingifAlreadyTransferStatusCompleted Oops! We have -" + e.getMessage(); + String errorMsg = "We have -" + e.getMessage(); prepareErrorMap(downloadResultFields, errorMsg); } finally { response.put(assetId, downloadResultFields); From 3b46ebfbb8bdfe238684e1e04244d546be34f7e0 Mon Sep 17 00:00:00 2001 From: ChetanT-System Date: Thu, 14 Sep 2023 13:24:20 +0530 Subject: [PATCH 18/23] - PCF submodel response changes --- .../sde/submodels/pcf/mapper/PcfMapper.java | 13 +++++++------ .../submodels/pcf/model/PcfSubmodelResponse.java | 8 +++++--- .../pcf/model/ProductOrSectorSpecificRules.java | 4 +++- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/mapper/PcfMapper.java b/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/mapper/PcfMapper.java index 23394f5c6..463466965 100644 --- a/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/mapper/PcfMapper.java +++ b/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/mapper/PcfMapper.java @@ -99,9 +99,9 @@ public JsonObject mapToResponse(PcfEntity entity) { .productMassPerDeclaredUnit(entity.getProductMassPerDeclaredUnit()) .productOrSectorSpecificRules(List.of(ProductOrSectorSpecificRules.builder() .extWBCSDOperator(entity.getExtWBCSDOperator()) - .productOrSectorSpecificRulesObj(ProductOrSectorSpecificRule.builder() + .productOrSectorSpecificRulesObj(List.of(ProductOrSectorSpecificRule.builder() .ruleName(entity.getRuleName()) - .build()) + .build())) .extWBCSDOtherOperatorName(entity.getExtWBCSDOtherOperatorName()) .build())) .extTFSAllocationWasteIncineration(entity.getExtTFSAllocationWasteIncineration()) @@ -142,18 +142,19 @@ public JsonObject mapToResponse(PcfEntity entity) { .build(); PcfSubmodelResponse build = PcfSubmodelResponse.builder().specVersion(entity.getSpecVersion()) - .companyIds(CompanyIds.builder().companyId(entity.getCompanyId()).build()) + .companyIds(List.of(CompanyIds.builder().companyId(entity.getCompanyId()).build())) .extWBCSDProductCodeCpc(entity.getExtWBCSDProductCodeCpc()).created(entity.getCreated()) .companyName(entity.getCompanyName()).extWBCSDPfStatus(entity.getExtWBCSDPfStatus()) - .version(entity.getVersion()).productName(entity.getProductName()).pcf(pcfResponse) + .version(entity.getVersion()) + .productName(entity.getProductName()).pcf(pcfResponse) .partialFullPcf(entity.getPartialFullPcf()) - .productIds(ProductIds.builder().productId(entity.getProductId()).build()) + .productIds(List.of(ProductIds.builder().productId(entity.getProductId()).build())) .validityPeriodStart(entity.getValidityPeriodStart()).comment(entity.getComment()).id(entity.getId()) .validityPeriodEnd(entity.getValidityPeriodEnd()).pcfLegalStatement(entity.getPcfLegalStatement()) .productDescription(entity.getProductDescription()) - .precedingPfIds(PrecedingPfIds.builder().id(entity.getPrecedingPfId()).build()).build(); + .precedingPfIds(List.of(PrecedingPfIds.builder().id(entity.getPrecedingPfId()).build())).build(); PcfAspect csv = mapFrom(entity); diff --git a/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/model/PcfSubmodelResponse.java b/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/model/PcfSubmodelResponse.java index 66b28af42..88b06c8e1 100644 --- a/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/model/PcfSubmodelResponse.java +++ b/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/model/PcfSubmodelResponse.java @@ -19,6 +19,8 @@ ********************************************************************************/ package org.eclipse.tractusx.sde.submodels.pcf.model; +import java.util.List; + import com.google.gson.annotations.SerializedName; import lombok.AllArgsConstructor; @@ -31,7 +33,7 @@ public class PcfSubmodelResponse { private String specVersion; - private CompanyIds companyIds; + private List companyIds; @SerializedName(value = "extWBCSD_productCodeCpc") private String extWBCSDProductCodeCpc; @@ -44,13 +46,13 @@ public class PcfSubmodelResponse { private double version; private Pcf pcf; private String partialFullPcf; - private ProductIds productIds; + private List productIds; private String validityPeriodStart; private String comment; private String id; private String validityPeriodEnd; private String pcfLegalStatement; private String productDescription; - private PrecedingPfIds precedingPfIds; + private List precedingPfIds; } diff --git a/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/model/ProductOrSectorSpecificRules.java b/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/model/ProductOrSectorSpecificRules.java index 4e5937714..79d64dd82 100644 --- a/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/model/ProductOrSectorSpecificRules.java +++ b/modules/sde-submodules/pcf/src/main/java/org/eclipse/tractusx/sde/submodels/pcf/model/ProductOrSectorSpecificRules.java @@ -19,6 +19,8 @@ ********************************************************************************/ package org.eclipse.tractusx.sde.submodels.pcf.model; +import java.util.List; + import com.google.gson.annotations.SerializedName; import lombok.AllArgsConstructor; @@ -34,7 +36,7 @@ public class ProductOrSectorSpecificRules { private String extWBCSDOperator; @SerializedName(value = "productOrSectorSpecificRules") - private ProductOrSectorSpecificRule productOrSectorSpecificRulesObj; + private List productOrSectorSpecificRulesObj; @SerializedName(value = "extWBCSD_otherOperatorName") private String extWBCSDOtherOperatorName; From bd7bdf4c9721e54d55b1263e0d556bad9adfa3c5 Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Thu, 14 Sep 2023 16:03:27 +0530 Subject: [PATCH 19/23] Correction error message --- .../sde/core/service/ConsumerService.java | 16 ++-- .../services/ConsumerControlPanelService.java | 85 +++++++++++-------- 2 files changed, 61 insertions(+), 40 deletions(-) diff --git a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java index 65e0178ca..1b791de51 100644 --- a/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java +++ b/modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/service/ConsumerService.java @@ -20,6 +20,10 @@ package org.eclipse.tractusx.sde.core.service; +import static org.eclipse.tractusx.sde.common.enums.ProgressStatusEnum.COMPLETED; +import static org.eclipse.tractusx.sde.common.enums.ProgressStatusEnum.FAILED; +import static org.eclipse.tractusx.sde.common.enums.ProgressStatusEnum.PARTIALLY_FAILED; + import java.io.IOException; import java.io.OutputStreamWriter; import java.io.PrintWriter; @@ -182,11 +186,11 @@ public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(String refe entity.setProcessId(processId); entity.setReferenceProcessId(referenceProcessId); - entity.setStatus(ProgressStatusEnum.FAILED.toString()); + entity.setStatus(FAILED.toString()); if (offerList.size() == successCount.get()) - entity.setStatus(ProgressStatusEnum.COMPLETED.toString()); + entity.setStatus(COMPLETED.toString()); else if (successCount.get() != 0 && failedCount.get() != 0) - entity.setStatus(ProgressStatusEnum.PARTIALLY_FAILED.toString()); + entity.setStatus(PARTIALLY_FAILED.toString()); // Save consumer Download history in DB consumerDownloadHistoryRepository.save(entity); @@ -214,10 +218,10 @@ private void prepareFromOfferResponse(Map subscribeAndDownloadDa if (dataNode != null && flagToDownloadImidiate) { processCSVDataObject(successCount, failedCount, csvWithValue, offer, status, dataNode); } else if (!flagToDownloadImidiate && "SUCCESS".equals(status.asText())) { - offer.setStatus(status.asText()); + offer.setStatus("SUCCESS"); successCount.getAndIncrement(); } else { - offer.setStatus(status.asText()); + offer.setStatus(FAILED.toString()); failedCount.getAndIncrement(); offer.setDownloadErrorMsg(readFieldFromJsonNode(node, "error")); } @@ -253,7 +257,7 @@ private void processCSVDataObject(AtomicInteger successCount, AtomicInteger fail offer.setStatus(status.asText()); successCount.getAndIncrement(); } else { - offer.setStatus("FAILED"); + offer.setStatus(FAILED.toString()); offer.setDownloadErrorMsg("The csv type data does not found in response"); failedCount.getAndIncrement(); } diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index 6e3c0ba88..bf17af895 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -68,6 +68,7 @@ @RequiredArgsConstructor public class ConsumerControlPanelService extends AbstractEDCStepsHelper { + private static final String REFRESHING = "REFRESHING"; private static final String NEGOTIATED = "NEGOTIATED"; private static final String STATUS = "status"; private final ContractOfferCatalogApi contractOfferCatalogApiProxy; @@ -273,6 +274,12 @@ public Map subscribeAndDownloadDataOffers(ConsumerRequest consum resultFields.put("edr", checkContractNegotiationStatus); + if (!NEGOTIATED.equalsIgnoreCase(checkContractNegotiationStatus.getEdrState())) { + throw new ServiceException( + "Time out!! to get 'NEGOTIATED' EDC EDR status to download data, The current status is '" + + checkContractNegotiationStatus.getEdrState() + "'"); + } + if (flagToDownloadImidiate) resultFields.put("data", downloadFile(checkContractNegotiationStatus)); @@ -298,74 +305,75 @@ public Map subscribeAndDownloadDataOffers(ConsumerRequest consum @SneakyThrows private EDRCachedResponse verifyOrCreateContractNegotiation(ConsumerRequest consumerRequest, HashMap extensibleProperty, String recipientURL, ActionRequest action, Offer offer) { - EDRCachedResponse checkContractNegotiationStatus = null; - try { - // Verify if there already contract established then use then contract for - // download - checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId(), RETRY); - } catch (Exception e) { - log.info("There is not contract exist for " + offer.getAssetId() + ", Initiating contract process"); + // Verify if there already EDR process initiated then skip t for again download + List eDRCachedResponseList = edrRequestHelper.getEDRCachedByAsset(offer.getAssetId()); + EDRCachedResponse checkContractNegotiationStatus = verifyEDRResponse(eDRCachedResponseList); + + if (checkContractNegotiationStatus == null) { + log.info("There was no EDR process initiated " + offer.getAssetId() + ", so initiating EDR process"); edrRequestHelper.edrRequestInitiate(recipientURL, consumerRequest.getConnectorId(), offer.getOfferId(), offer.getAssetId(), action, extensibleProperty); - checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId(), 1); + } else { + log.info("There was EDR process initiated " + offer.getAssetId() + ", so ignoring EDR process initiation"); } + + checkContractNegotiationStatus = verifyEDRRequestStatus(offer.getAssetId()); return checkContractNegotiationStatus; } @SneakyThrows - private EDRCachedResponse verifyEDRRequestStatus(String assetId, int counter) { + private EDRCachedResponse verifyEDRRequestStatus(String assetId) { EDRCachedResponse eDRCachedResponse = null; - AtomicInteger isRefresheEDRToken = new AtomicInteger(); + String edrStatus = "NewToSDE"; List eDRCachedResponseList = null; + int counter = 1; try { do { - isRefresheEDRToken.set(0); - Thread.sleep(THRED_SLEEP_TIME); + if (counter > 1) + Thread.sleep(THRED_SLEEP_TIME); eDRCachedResponseList = edrRequestHelper.getEDRCachedByAsset(assetId); - counter++; - eDRCachedResponse = verifyEDRResponse(eDRCachedResponse, isRefresheEDRToken, eDRCachedResponseList); - } while ((eDRCachedResponse == null && counter <= RETRY) - || (eDRCachedResponse != null && !eDRCachedResponse.getEdrState().equals(NEGOTIATED))); + eDRCachedResponse = verifyEDRResponse(eDRCachedResponseList); - if (eDRCachedResponseList != null && eDRCachedResponseList.size() >= isRefresheEDRToken.get() - && eDRCachedResponse == null) - throw new ServiceException( - "Time out!! EDC refreshing EDR tokens, unable to get EDR negotiated status"); + if (eDRCachedResponse != null) + edrStatus = eDRCachedResponse.getEdrState(); + + log.info("Verifying 'NEGOTIATED' EDC EDR status to download data for '" + assetId + + "', The current status is '" + edrStatus + "', Attempt " + counter); + counter++; + } while (counter <= RETRY && !NEGOTIATED.equals(edrStatus)); if (eDRCachedResponse == null) throw new ServiceException("Time out!! unable to get EDR negotiated status"); } catch (FeignException e) { log.error("RequestBody: " + e.request()); - String errorMsg = "FeignExceptionton to verify EDR negotiated status for asset " + assetId + "," - + e.contentUTF8(); + String errorMsg = "FeignExceptionton for asset " + assetId + "," + e.contentUTF8(); log.error("Response: " + errorMsg); throw new ServiceException(errorMsg); } catch (InterruptedException ie) { Thread.currentThread().interrupt(); - String errorMsg = "InterruptedException to verify EDR negotiated status for asset " + assetId + "," - + ie.getMessage(); + String errorMsg = "InterruptedException for asset " + assetId + "," + ie.getMessage(); log.error(errorMsg); throw new ServiceException(errorMsg); } catch (Exception e) { - String errorMsg = "Exception to verify EDR negotiated status for asset " + assetId + "," - + e.getMessage(); + String errorMsg = "Exception for asset " + assetId + "," + e.getMessage(); log.error(errorMsg); throw new ServiceException(errorMsg); } return eDRCachedResponse; } - private EDRCachedResponse verifyEDRResponse(EDRCachedResponse eDRCachedResponse, AtomicInteger isRefresheEDRToken, - List eDRCachedResponseList) { + private EDRCachedResponse verifyEDRResponse(List eDRCachedResponseList) { + EDRCachedResponse eDRCachedResponse = null; if (eDRCachedResponseList != null && !eDRCachedResponseList.isEmpty()) { for (EDRCachedResponse edrCachedResponseObj : eDRCachedResponseList) { String edrState = edrCachedResponseObj.getEdrState(); - if (edrState.equals(NEGOTIATED)) { + + if (NEGOTIATED.equalsIgnoreCase(edrState)) { + eDRCachedResponse = edrCachedResponseObj; + break; + } else { eDRCachedResponse = edrCachedResponseObj; - } - if ("REFRESHING".equalsIgnoreCase(edrState)) { - isRefresheEDRToken.getAndIncrement(); } } } @@ -384,9 +392,18 @@ public Map downloadFileFromEDCUsingifAlreadyTransferStatusComple Map downloadResultFields = new ConcurrentHashMap<>(); try { - EDRCachedResponse verifyEDRRequestStatus = verifyEDRRequestStatus(assetId, 1); + EDRCachedResponse verifyEDRRequestStatus = verifyEDRRequestStatus(assetId); + downloadResultFields.put("edr", verifyEDRRequestStatus); + + if (!NEGOTIATED.equalsIgnoreCase(verifyEDRRequestStatus.getEdrState())) { + throw new ServiceException( + "Time out!! to get 'NEGOTIATED' EDC EDR status to download data, The current status is '" + + verifyEDRRequestStatus.getEdrState() + "'"); + } + downloadResultFields.put("data", downloadFile(verifyEDRRequestStatus)); + downloadResultFields.put(STATUS, "SUCCESS"); } catch (Exception e) { String errorMsg = "We have -" + e.getMessage(); @@ -405,7 +422,7 @@ private void prepareErrorMap(Map resultFields, String errorMsg) @SneakyThrows private Object downloadFile(EDRCachedResponse verifyEDRRequestStatus) { - if (verifyEDRRequestStatus != null) { + if (verifyEDRRequestStatus != null && NEGOTIATED.equalsIgnoreCase(verifyEDRRequestStatus.getEdrState())) { try { EDRCachedByIdResponse authorizationToken = getAuthorizationTokenForDataDownload( verifyEDRRequestStatus.getTransferProcessId()); From 426119a973f64bafdfeeafa478765b569e811d0d Mon Sep 17 00:00:00 2001 From: Sachin Argade Date: Thu, 14 Sep 2023 16:20:43 +0530 Subject: [PATCH 20/23] Correction error message --- .../tractusx/sde/edc/services/ConsumerControlPanelService.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java index bf17af895..224595d18 100644 --- a/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java +++ b/modules/sde-external-services/edc/src/main/java/org/eclipse/tractusx/sde/edc/services/ConsumerControlPanelService.java @@ -29,7 +29,6 @@ import java.util.Optional; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.eclipse.tractusx.sde.common.entities.UsagePolicies; @@ -68,7 +67,6 @@ @RequiredArgsConstructor public class ConsumerControlPanelService extends AbstractEDCStepsHelper { - private static final String REFRESHING = "REFRESHING"; private static final String NEGOTIATED = "NEGOTIATED"; private static final String STATUS = "status"; private final ContractOfferCatalogApi contractOfferCatalogApiProxy; From 6da2532a46be2bb4c77e2e24f8c6e7d02560b88c Mon Sep 17 00:00:00 2001 From: adityagajbhiye9 <133367448+adityagajbhiye9@users.noreply.github.com> Date: Wed, 20 Sep 2023 11:42:24 +0530 Subject: [PATCH 21/23] [feat|sde|major]updated changelog.md - major release for consume data v2.3.0 - Support_edc_edr_api_use. - Support combine zip download. - Maintain download history for the re-download. --- CHANGELOG.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a4c43d1c..845d001e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +## [2.3.0] - 2023-09- +### Added +- Support_edc_edr_api_use. +- Support combine zip download. +- Maintain download history for the re-download. +### Fixed +- Update pcf schema. + ## [2.2.0] - 2023-09-20 ### Added - Pcf model schema and model registration. @@ -224,7 +232,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Compliance with Catena-X Guidelines - Integration with Digital Twin registry service. -[unreleased]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.2.0...main +[unreleased]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.3.0...main +[2.3.0]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.2.0...v2.3.0 [2.2.0]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.1.1...v2.2.0 [2.1.1]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.1.0...v2.1.1 [2.1.0]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.0.11...v2.1.0 From 387cc5dbfb85cb0cec000b9d52a6ea00ba281756 Mon Sep 17 00:00:00 2001 From: adityagajbhiye9 <133367448+adityagajbhiye9@users.noreply.github.com> Date: Wed, 20 Sep 2023 11:45:16 +0530 Subject: [PATCH 22/23] Update Dockerfile --- build/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index 78fb00f51..615124580 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -53,5 +53,4 @@ COPY --from=build modules/sde-core/target/*.jar ./app.jar EXPOSE 8080 # set the startup command to run your binary -CMD ["java", "-jar", "./app.jar"] -#CMD ["java", "-jar", "./app.jar","--spring.config.location=file:/app/configuration.properties"] +CMD ["java", "-jar", "./app.jar","--spring.config.location=file:/app/configuration.properties"] From d49d900ba8fca7a9303d6071ae268e935f8f9b8c Mon Sep 17 00:00:00 2001 From: adityagajbhiye9 <133367448+adityagajbhiye9@users.noreply.github.com> Date: Wed, 20 Sep 2023 18:55:24 +0530 Subject: [PATCH 23/23] update change log .md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 845d001e6..0e8b7589a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), -## [2.3.0] - 2023-09- +## [2.3.0] - 2023-09-20 ### Added - Support_edc_edr_api_use. - Support combine zip download.