[feat|sde-download history|major] consume data and maintain download history.

CHANGELOG.md

@@ -2,6 +2,14 @@
 All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
+## [2.3.0] - 2023-09-20
+### Added
+- Support_edc_edr_api_use.
+- Support combine zip download.
+- Maintain download history for the re-download.
+### Fixed
+- Update pcf schema.
+
 ## [2.2.0] - 2023-09-20
 ### Added
 - Pcf model schema and model registration.
@@ -224,7 +232,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Compliance with Catena-X Guidelines
 - Integration with Digital Twin registry service.
 
-[unreleased]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.2.0...main
+[unreleased]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.3.0...main
+[2.3.0]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.2.0...v2.3.0
 [2.2.0]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.1.1...v2.2.0
 [2.1.1]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.1.0...v2.1.1
 [2.1.0]: https://github.com/eclipse-tractusx/managed-simple-data-exchanger-backend/compare/v2.0.11...v2.1.0

DEPENDENCIES

@@ -18,11 +18,15 @@ maven/mavencentral/com.google.errorprone/error_prone_annotations/2.18.0, Apache-
 maven/mavencentral/com.jayway.jsonpath/json-path/2.8.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.networknt/json-schema-validator/1.0.72, Apache-2.0, approved, CQ22638
 maven/mavencentral/com.nimbusds/nimbus-jose-jwt/9.31, Apache-2.0, approved, clearlydefined
+maven/mavencentral/com.opencsv/opencsv/5.8, Apache-2.0, approved, clearlydefined
 maven/mavencentral/com.sun.istack/istack-commons-runtime/4.1.1, BSD-3-Clause, approved, #2590
 maven/mavencentral/com.vaadin.external.google/android-json/0.0.20131108.vaadin1, Apache-2.0, approved, CQ21310
 maven/mavencentral/com.zaxxer/HikariCP/5.0.1, Apache-2.0, approved, clearlydefined
+maven/mavencentral/commons-beanutils/commons-beanutils/1.9.4, Apache-2.0, approved, CQ12654
 maven/mavencentral/commons-codec/commons-codec/1.15, Apache-2.0 AND BSD-3-Clause AND LicenseRef-Public-Domain, approved, CQ22641
+maven/mavencentral/commons-collections/commons-collections/3.2.2, Apache-2.0, approved, CQ10385
 maven/mavencentral/commons-io/commons-io/2.7, Apache-2.0, approved, clearlydefined
+maven/mavencentral/commons-logging/commons-logging/1.2, Apache-2.0, approved, CQ10162
 maven/mavencentral/io.github.classgraph/classgraph/4.8.149, MIT, approved, CQ22530
 maven/mavencentral/io.github.openfeign.form/feign-form-spring/3.8.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/io.github.openfeign.form/feign-form/3.8.0, Apache-2.0, approved, clearlydefined
@@ -46,6 +50,7 @@ maven/mavencentral/net.bytebuddy/byte-buddy/1.14.4, Apache-2.0 AND BSD-3-Clause,
 maven/mavencentral/net.minidev/accessors-smart/2.4.9, Apache-2.0, approved, #7515
 maven/mavencentral/net.minidev/json-smart/2.4.10, Apache-2.0, approved, #3288
 maven/mavencentral/org.antlr/antlr4-runtime/4.10.1, BSD-3-Clause AND LicenseRef-Public-domain AND MIT AND LicenseRef-Unicode-TOU, approved, #7065
+maven/mavencentral/org.apache.commons/commons-collections4/4.4, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.apache.commons/commons-csv/1.8, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.apache.commons/commons-lang3/3.12.0, Apache-2.0, approved, clearlydefined
 maven/mavencentral/org.apache.commons/commons-text/1.10.0, Apache-2.0, approved, clearlydefined

build/Dockerfile

@@ -53,5 +53,4 @@ COPY --from=build modules/sde-core/target/*.jar ./app.jar
 
 EXPOSE 8080
 # set the startup command to run your binary
-
 CMD ["java", "-jar", "./app.jar","--spring.config.location=file:/app/configuration.properties"]

modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/enums/ProgressStatusEnum.java

@@ -25,5 +25,6 @@
 public enum ProgressStatusEnum {
     IN_PROGRESS,
     COMPLETED,
-    FAILED
+    FAILED,
+    PARTIALLY_FAILED
 }

modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/Acknowledgement.java

@@ -0,0 +1,32 @@
+/********************************************************************************
+ * Copyright (c) 2023 T-Systems International GmbH
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+package org.eclipse.tractusx.sde.common.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class Acknowledgement {
+
+	private String id;
+
+}

modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/model/PagingResponse.java

@@ -0,0 +1,34 @@
+/********************************************************************************
+ * Copyright (c) 2023 T-Systems International GmbH
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+package org.eclipse.tractusx.sde.common.model;
+
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class PagingResponse {
+
+	private int page;
+	private int pageSize;
+	private long totalItems;
+	private Object items;
+}

modules/sde-common/src/main/java/org/eclipse/tractusx/sde/common/submodel/executor/create/steps/impl/RecordProcessUtils.java

@@ -38,9 +38,13 @@ public void setFieldValue(ObjectNode rowjObject, String ele, JsonObject jObject,
 
 		fieldValue = fieldValue.trim();
 
-		if (isNumberTypeField(jObject, fieldValue))
-			rowjObject.put(ele, Double.parseDouble(fieldValue));
-		else if (isDateFormatField(jObject)) {
+		if (isNumberTypeField(jObject)) {
+			if (fieldValue.isBlank()) {
+				rowjObject.putNull(ele);
+			} else {
+				rowjObject.put(ele, Double.parseDouble(fieldValue));
+			}
+		} else if (isDateFormatField(jObject)) {
 
 			if (fieldValue.isBlank())
 				fieldValue = null;
@@ -57,12 +61,10 @@ private boolean isDateFormatField(JsonObject jObject) {
 		return jObject.get("format") != null && "date-time".equals(jObject.get("format").getAsString());
 	}
 
-	private boolean isNumberTypeField(JsonObject jObject, String fieldValue) {
-
-		if (fieldValue != null && !fieldValue.isBlank() && jObject.get("type") != null
-				&& jObject.get("type").isJsonArray()) {
+	private boolean isNumberTypeField(JsonObject jObject) {
+		JsonElement jsonElement = JsonParser.parseString("number");
+		if (jObject.get("type") != null && jObject.get("type").isJsonArray()) {
 			JsonArray types = jObject.get("type").getAsJsonArray();
-			JsonElement jsonElement = JsonParser.parseString("number");
 			return types.contains(jsonElement);
 		}
 		return false;

modules/sde-core/pom.xml

@@ -177,6 +177,13 @@
 			<artifactId>pcf</artifactId>
 			<version>0.0.1</version>
 		</dependency>
+
+		<dependency>
+		    <groupId>com.opencsv</groupId>
+		    <artifactId>opencsv</artifactId>
+		    <version>5.8</version>
+		</dependency>
+
 	</dependencies>
 
 	<build>

modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/controller/ConsumerController.java

@@ -22,15 +22,16 @@
 
 import static org.springframework.http.ResponseEntity.ok;
 
-import java.util.List;
 import java.util.UUID;
 
 import org.eclipse.tractusx.sde.core.service.ConsumerService;
 import org.eclipse.tractusx.sde.edc.model.request.ConsumerRequest;
 import org.eclipse.tractusx.sde.edc.services.ConsumerControlPanelService;
+import org.springframework.data.repository.query.Param;
 import org.springframework.http.ResponseEntity;
 import org.springframework.security.access.prepost.PreAuthorize;
 import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PathVariable;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestParam;
@@ -73,20 +74,46 @@ public ResponseEntity<Object> subscribeDataOffers(@Valid @RequestBody ConsumerRe
 		consumerControlPanelService.subscribeDataOffers(consumerRequest, processId);
 		return ResponseEntity.ok().body(processId);
 	}
+
+	@PostMapping(value = "/subscribe-download-data-offers-async")
+	@PreAuthorize("hasPermission('','consumer_subscribe_download_data_offers')")
+	public ResponseEntity<Object> subscribeAndDownloadDataOffersAsync(@Valid @RequestBody ConsumerRequest consumerRequest) {
+		log.info("Request recevied : /api/subscribe-download-data-offers-async");
+		return ResponseEntity.ok().body(consumerService.subscribeAndDownloadDataOffersAsync(consumerRequest));
+	}
 
 	@PostMapping(value = "/subscribe-download-data-offers")
-	@PreAuthorize("hasPermission('','consumer_download_data')")
-	public void subscribeAndDownloadDataOffers(@Valid @RequestBody ConsumerRequest consumerRequest, HttpServletResponse response) {
-		log.info("Request recevied : /api/subscribe-download-data-edr");
-		consumerService.subscribeAndDownloadDataOffers(consumerRequest, response);
+	@PreAuthorize("hasPermission('','consumer_subscribe_download_data_offers')")
+	public void subscribeAndDownloadDataOffersSynchronous(@Valid @RequestBody ConsumerRequest consumerRequest, HttpServletResponse response) {
+		log.info("Request recevied : /api/subscribe-download-data-offers");
+		consumerService.subscribeAndDownloadDataOffersSynchronous(consumerRequest, response);
 	}
 
 	@GetMapping(value = "/download-data-offers")
-	@PreAuthorize("hasPermission('','consumer_download_data')")
-	public ResponseEntity<Object> downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(@RequestParam List<String> assetIdList)
+	@PreAuthorize("hasPermission('','consumer_download_data_offer')")
+	public void downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(@RequestParam("processId") String referenceProcessId, HttpServletResponse response)
 			throws Exception {
-		log.info("Request received : /api/download-data-using-edr");
-		return consumerService.downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(assetIdList);
+		log.info("Request received : /api/download-data-offers");
+		consumerService.downloadFileFromEDCUsingifAlreadyTransferStatusCompleted(referenceProcessId,response);
+	}
+
+	@GetMapping(value = "/view-download-history")
+	@PreAuthorize("hasPermission('','consumer_view_download_history')")
+	public ResponseEntity<Object> viewConsumerDownloadHistory(@Param("page") Integer page,
+			@Param("pageSize") Integer pageSize) throws Exception {
+		page = page == null ? 0 : page;
+		pageSize = pageSize == null ? 10 : pageSize;
+		log.info("Request received : /api/view-download-history");
+		return ok().body(consumerService.viewDownloadHistory(page, pageSize));
+	}
+
+	@GetMapping(value = "/view-download-history/{processId}")
+	@PreAuthorize("hasPermission('','consumer_view_download_history')")
+	public ResponseEntity<Object> viewConsumerDownloadHistoryDetails(@PathVariable("processId") String processId)
+			throws Exception {
+		log.info("Request received : /api/view-download-history-details");
+		return ok().body(consumerService.viewConsumerDownloadHistoryDetails(processId));
+
 	}
 
 }

modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/failurelog/repository/ConsumerDownloadHistoryRepository.java

@@ -0,0 +1,31 @@
+/********************************************************************************
+ * Copyright (c) 2023 T-Systems International GmbH
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+
+package org.eclipse.tractusx.sde.core.failurelog.repository;
+
+
+import org.eclipse.tractusx.sde.core.processreport.entity.ConsumerDownloadHistoryEntity;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface ConsumerDownloadHistoryRepository extends JpaRepository<ConsumerDownloadHistoryEntity, String> {
+
+	ConsumerDownloadHistoryEntity findByProcessId(String id);
+
+}

modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/ProcessReportUseCase.java

@@ -103,7 +103,7 @@ public ProcessReportPageResponse listAllProcessReports(int page, int size) {
 				.findAll(PageRequest.of(page, size, Sort.by(Sort.Direction.DESC, "startDate")));
 		List<ProcessReport> processReports = result.get().map(mapper::mapFrom).toList();
 		return ProcessReportPageResponse.builder().items(processReports).pageSize(result.getSize())
-				.pageNumber(result.getNumber()).totalItems(result.getTotalElements()).build();
+				.page(result.getNumber()).totalItems(result.getTotalElements()).build();
 	}
 
 	public ProcessReport getProcessReportById(String id) {

modules/sde-core/src/main/java/org/eclipse/tractusx/sde/core/processreport/entity/ConsumerDownloadHistoryEntity.java

@@ -0,0 +1,70 @@
+/********************************************************************************
+ * Copyright (c) 2023 T-Systems International GmbH
+ * Copyright (c) 2023 Contributors to the Eclipse Foundation
+ *
+ * See the NOTICE file(s) distributed with this work for additional
+ * information regarding copyright ownership.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Apache License, Version 2.0 which is available at
+ * https://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ ********************************************************************************/
+package org.eclipse.tractusx.sde.core.processreport.entity;
+
+import java.time.LocalDateTime;
+
+import jakarta.persistence.Cacheable;
+import jakarta.persistence.Column;
+import jakarta.persistence.Entity;
+import jakarta.persistence.Id;
+import jakarta.persistence.Lob;
+import jakarta.persistence.Table;
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Table(name = "consumer_download_history")
+@Entity
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+@Cacheable(value = false)
+public class ConsumerDownloadHistoryEntity {
+	@Id
+	@Column(name = "process_id")
+	private String processId;
+	@Column(name = "provider_url")
+	private String providerUrl;
+	@Column(name = "connector_id")
+	private String connectorId;
+	@Column(name = "number_of_items")
+	private Integer numberOfItems;
+	@Column(name = "download_failed")
+	private Integer downloadFailed;
+	@Column(name = "download_successed")
+	private Integer downloadSuccessed;
+	@Column(name = "status")
+	private String status;
+	@Column(name = "start_date")
+	private LocalDateTime startDate;
+	@Column(name = "end_date")
+	private LocalDateTime endDate;
+
+	@Lob
+	@Column(name = "offers")
+	private String offers;
+	@Lob
+	private String policies;
+	private String referenceProcessId;
+
+}