diff --git a/src/main/java/app/coronawarn/quicktest/domain/Cancellation.java b/src/main/java/app/coronawarn/quicktest/domain/Cancellation.java index ac2615cb..0a503a17 100644 --- a/src/main/java/app/coronawarn/quicktest/domain/Cancellation.java +++ b/src/main/java/app/coronawarn/quicktest/domain/Cancellation.java @@ -76,6 +76,18 @@ public class Cancellation { @Column(name = "bucket_object_id") private String bucketObjectId; + @Column(name = "csv_entity_count") + @JsonIgnore + private Integer csvEntityCount; + + @Column(name = "csv_hash") + @JsonIgnore + private String csvHash; + + @Column(name = "csv_size") + @JsonIgnore + private Integer csvSize; + @Column(name = "data_export_error") @JsonIgnore private String dataExportError; diff --git a/src/main/java/app/coronawarn/quicktest/service/ArchiveSchedulingService.java b/src/main/java/app/coronawarn/quicktest/service/ArchiveSchedulingService.java index 023eac10..462a9c52 100644 --- a/src/main/java/app/coronawarn/quicktest/service/ArchiveSchedulingService.java +++ b/src/main/java/app/coronawarn/quicktest/service/ArchiveSchedulingService.java @@ -9,14 +9,17 @@ import com.opencsv.bean.StatefulBeanToCsv; import com.opencsv.bean.StatefulBeanToCsvBuilder; import java.io.ByteArrayInputStream; -import java.io.InputStream; import java.io.StringWriter; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; import java.time.ZonedDateTime; import java.util.List; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import net.javacrumbs.shedlock.spring.annotation.SchedulerLock; import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.security.crypto.codec.Hex; import org.springframework.stereotype.Service; @Slf4j @@ -85,25 +88,37 @@ private void processCsvUploadBatchRecursion(List cancellations) { for (Cancellation cancellation : cancellations) { try { List quicktests = - archiveService.getQuicktestsFromLongtermByTenantId(cancellation.getPartnerId()); + archiveService.getQuicktestsFromLongtermByTenantId(cancellation.getPartnerId()); + StringWriter stringWriter = new StringWriter(); CSVWriter csvWriter = - new CSVWriter(stringWriter, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER, - CSVWriter.DEFAULT_ESCAPE_CHARACTER, CSVWriter.DEFAULT_LINE_END); + new CSVWriter(stringWriter, '\t', CSVWriter.NO_QUOTE_CHARACTER, + CSVWriter.DEFAULT_ESCAPE_CHARACTER, CSVWriter.DEFAULT_LINE_END); StatefulBeanToCsv beanToCsv = - new StatefulBeanToCsvBuilder(csvWriter) - .build(); + new StatefulBeanToCsvBuilder(csvWriter) + .build(); beanToCsv.write(quicktests); - InputStream inputStream = new ByteArrayInputStream(stringWriter.toString().getBytes()); - String id = cancellation.getPartnerId() + ".csv"; + byte[] csvBytes = stringWriter.toString().getBytes(StandardCharsets.UTF_8); + + String objectId = cancellation.getPartnerId() + ".csv"; + ObjectMetadata metadata = new ObjectMetadata(); - metadata.setContentLength(stringWriter.toString().getBytes().length); - s3Client.putObject(s3Config.getBucketName(), id, inputStream, metadata); - log.info("File stored to S3 with id {}", id); - cancellationService.updateCsvCreated(cancellation, ZonedDateTime.now(), id); + metadata.setContentLength(csvBytes.length); + + s3Client.putObject( + s3Config.getBucketName(), + objectId, + new ByteArrayInputStream(csvBytes), metadata); + + log.info("File stored to S3 with id {}", objectId); + + cancellationService.updateCsvCreated(cancellation, ZonedDateTime.now(), objectId, + getHash(csvBytes), quicktests.size(), csvBytes.length); } catch (Exception e) { - log.error("Could not convert Quicktest to CSV: " + e.getLocalizedMessage()); - cancellationService.updateDataExportError(cancellation, e.getLocalizedMessage()); + String errorMessage = e.getClass().getName() + ": " + e.getMessage(); + + log.error("Could not convert Quicktest to CSV: " + errorMessage); + cancellationService.updateDataExportError(cancellation, errorMessage); } } @@ -112,4 +127,14 @@ private void processCsvUploadBatchRecursion(List cancellations) { processCsvUploadBatchRecursion(nextBatch); } } + + private String getHash(byte[] bytes) { + try { + MessageDigest sha256 = MessageDigest.getInstance("SHA-256"); + byte[] hashBytes = sha256.digest(bytes); + return String.valueOf(Hex.encode(hashBytes)); + } catch (NoSuchAlgorithmException e) { + throw new RuntimeException("Failed to load SHA-256 Message Digest"); + } + } } diff --git a/src/main/java/app/coronawarn/quicktest/service/CancellationService.java b/src/main/java/app/coronawarn/quicktest/service/CancellationService.java index 03c8e8d2..3db2dfd6 100644 --- a/src/main/java/app/coronawarn/quicktest/service/CancellationService.java +++ b/src/main/java/app/coronawarn/quicktest/service/CancellationService.java @@ -120,9 +120,13 @@ public void updateMovedToLongterm(Cancellation cancellation, ZonedDateTime moved * @param cancellation Cancellation Entity * @param csvCreated timestamp of job completion */ - public void updateCsvCreated(Cancellation cancellation, ZonedDateTime csvCreated, String bucketObjectId) { + public void updateCsvCreated(Cancellation cancellation, ZonedDateTime csvCreated, String bucketObjectId, + String hash, int entityCount, int fileSize) { cancellation.setCsvCreated(csvCreated); cancellation.setBucketObjectId(bucketObjectId); + cancellation.setCsvHash(hash); + cancellation.setCsvEntityCount(entityCount); + cancellation.setCsvSize(fileSize); cancellationRepository.save(cancellation); } diff --git a/src/main/resources/db/changelog.yml b/src/main/resources/db/changelog.yml index e911bf2a..1fb74851 100644 --- a/src/main/resources/db/changelog.yml +++ b/src/main/resources/db/changelog.yml @@ -65,3 +65,6 @@ databaseChangeLog: - include: file: changelog/V022_update_cancellationTable_3.yml relativeToChangelogFile: true + - include: + file: changelog/V023_update_cancellationTable_4.yml + relativeToChangelogFile: true diff --git a/src/main/resources/db/changelog/V023_update_cancellationTable_4.yml b/src/main/resources/db/changelog/V023_update_cancellationTable_4.yml new file mode 100644 index 00000000..a26190b9 --- /dev/null +++ b/src/main/resources/db/changelog/V023_update_cancellationTable_4.yml @@ -0,0 +1,26 @@ +databaseChangeLog: + - changeSet: + id: update-cancellation-table-4 + author: f11h + changes: + - addColumn: + tableName: cancellation + column: + name: csv_size + type: int + constraints: + nullable: true + - addColumn: + tableName: cancellation + column: + name: csv_hash + type: varchar(64) + constraints: + nullable: true + - addColumn: + tableName: cancellation + column: + name: csv_entity_count + type: int + constraints: + nullable: true diff --git a/src/test/java/app/coronawarn/quicktest/service/CancellationCsvTest.java b/src/test/java/app/coronawarn/quicktest/service/CancellationCsvTest.java new file mode 100644 index 00000000..6ad48711 --- /dev/null +++ b/src/test/java/app/coronawarn/quicktest/service/CancellationCsvTest.java @@ -0,0 +1,183 @@ +/*- + * ---license-start + * Corona-Warn-App / cwa-quick-test-backend + * --- + * Copyright (C) 2021 T-Systems International GmbH and all other contributors + * --- + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * ---license-end + */ + +package app.coronawarn.quicktest.service; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import app.coronawarn.quicktest.archive.repository.ArchiveRepository; +import app.coronawarn.quicktest.domain.Cancellation; +import app.coronawarn.quicktest.domain.QuickTestArchive; +import app.coronawarn.quicktest.model.Sex; +import app.coronawarn.quicktest.repository.CancellationRepository; +import app.coronawarn.quicktest.repository.QuickTestArchiveRepository; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.PutObjectResult; +import com.opencsv.CSVParser; +import com.opencsv.CSVParserBuilder; +import com.opencsv.CSVReader; +import com.opencsv.CSVReaderBuilder; +import com.opencsv.exceptions.CsvException; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringReader; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; +import java.util.List; +import org.apache.commons.lang3.RandomUtils; +import org.apache.tomcat.util.buf.HexUtils; +import org.bouncycastle.util.encoders.Hex; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.mockito.ArgumentCaptor; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.data.domain.Pageable; +import org.springframework.transaction.annotation.Transactional; + +@SpringBootTest +class CancellationCsvTest { + + @Autowired + private ArchiveSchedulingService archiveSchedulingService; + + @Autowired + private QuickTestArchiveRepository shortTermArchiveRepository; + + @Autowired + private ArchiveRepository longTermArchiveRepository; + + @Autowired + private CancellationRepository cancellationRepository; + + @MockBean + private AmazonS3 s3Client; + + @BeforeEach + void setUp() { + shortTermArchiveRepository.deleteAll(); + longTermArchiveRepository.deleteAllByTenantId(PARTNER_ID); + cancellationRepository.deleteAll(); + } + + public static final ZonedDateTime CANCELLATION_DATE = ZonedDateTime.now().minusHours(49).truncatedTo(ChronoUnit.MINUTES); + public static final String PARTNER_ID = "P10000"; + public static final String PARTNER_ID_HASH = "212e58b487b6d6b486b71c6ebb3fedc0db3c69114f125fb3cd2fbc72e6ffc25f"; + + @ParameterizedTest + @ValueSource(ints = {1, 5000}) + @Transactional + void testCsvExport(int n) throws IOException, NoSuchAlgorithmException, CsvException { + Cancellation cancellation = new Cancellation(); + cancellation.setPartnerId(PARTNER_ID); + cancellation.setCancellationDate(CANCELLATION_DATE); + cancellationRepository.save(cancellation); + + for (int i = 0; i < n; i++) { + shortTermArchiveRepository.save(buildQuickTestArchive(PARTNER_ID)); + } + + Assertions.assertEquals(n, shortTermArchiveRepository.findAllByTenantId(PARTNER_ID, Pageable.unpaged()).count()); + Assertions.assertEquals(0, longTermArchiveRepository.findAllByTenantId(PARTNER_ID_HASH).size()); + + archiveSchedulingService.cancellationArchiveJob(); + + Assertions.assertEquals(0, shortTermArchiveRepository.findAllByTenantId(PARTNER_ID, Pageable.unpaged()).count()); + Assertions.assertEquals(n, longTermArchiveRepository.findAllByTenantId(PARTNER_ID_HASH).size()); + + ArgumentCaptor inputStreamArgumentCaptor = ArgumentCaptor.forClass(InputStream.class); + String expectedFileName = PARTNER_ID + ".csv"; + when(s3Client.putObject(anyString(), eq(expectedFileName), inputStreamArgumentCaptor.capture(), any())) + .thenReturn(new PutObjectResult()); + + archiveSchedulingService.csvUploadJob(); + + verify(s3Client).putObject(anyString(), eq(expectedFileName), any(), any()); + + byte[] csvBytes = inputStreamArgumentCaptor.getValue().readAllBytes(); + String csv = new String(csvBytes, StandardCharsets.UTF_8); + + cancellation = cancellationRepository.findById(PARTNER_ID).orElseThrow(); + + Assertions.assertEquals(n, cancellation.getCsvEntityCount()); + Assertions.assertEquals(csvBytes.length, cancellation.getCsvSize()); + Assertions.assertEquals(getHash(csvBytes), cancellation.getCsvHash()); + + CSVParser csvParser = new CSVParserBuilder() + .withSeparator('\t') + .build(); + + try (CSVReader csvReader = new CSVReaderBuilder(new StringReader(csv)) + .withCSVParser(csvParser) + .build() + ) { + List csvEntries = csvReader.readAll(); + Assertions.assertEquals(n + 1, csvEntries.size()); + Assertions.assertEquals(27, csvEntries.get(0).length); + } + + longTermArchiveRepository.deleteAllByTenantId(PARTNER_ID_HASH); + } + + private String getHash(byte[] bytes) throws NoSuchAlgorithmException { + return Hex.toHexString(MessageDigest.getInstance("SHA-256").digest(bytes)); + } + private QuickTestArchive buildQuickTestArchive(String tenantId) { + QuickTestArchive qta = new QuickTestArchive(); + qta.setShortHashedGuid(HexUtils.toHexString(RandomUtils.nextBytes(4))); + qta.setHashedGuid(HexUtils.toHexString(RandomUtils.nextBytes(32))); + qta.setTenantId(tenantId); + qta.setPocId("poc_id"); + qta.setCreatedAt(LocalDateTime.now().minusMonths(3)); + qta.setUpdatedAt(LocalDateTime.now().minusMonths(2)); + qta.setConfirmationCwa(Boolean.TRUE); + qta.setTestResult(Short.valueOf("6")); + qta.setPrivacyAgreement(Boolean.TRUE); + qta.setLastName("last_name"); + qta.setFirstName("first_name"); + qta.setEmail("email"); + qta.setPhoneNumber("phone_number"); + qta.setSex(Sex.MALE); + qta.setStreet("street"); + qta.setHouseNumber("house_number"); + qta.setZipCode("zip_code"); + qta.setCity("city"); + qta.setTestBrandId("test_brand_id"); + qta.setTestBrandName("test_brand_name, Ltd, another_part_of_test_brand_name"); + qta.setBirthday("2000-01-01"); + qta.setPdf("PDF".getBytes()); + qta.setTestResultServerHash("test_result_server_hash"); + qta.setDcc("dcc"); + qta.setAdditionalInfo("additional_info"); + qta.setGroupName("group_name"); + return qta; + } +} diff --git a/src/test/java/app/coronawarn/quicktest/service/CancellationServiceTest.java b/src/test/java/app/coronawarn/quicktest/service/CancellationServiceTest.java index a95bd871..9d9d4800 100644 --- a/src/test/java/app/coronawarn/quicktest/service/CancellationServiceTest.java +++ b/src/test/java/app/coronawarn/quicktest/service/CancellationServiceTest.java @@ -24,6 +24,7 @@ import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.endsWith; import static org.mockito.Mockito.doNothing; import app.coronawarn.quicktest.domain.Cancellation; @@ -155,7 +156,8 @@ void testUpdateCsvCreated() { Cancellation cancellation = cancellationService.createCancellation(PARTNER_ID, CANCELLATION_DATE); Assertions.assertEquals(FINAL_DELETION, cancellation.getFinalDeletion()); - cancellationService.updateCsvCreated(cancellation, NEW_STATE_DATE, PARTNER_ID + ".csv"); + cancellationService.updateCsvCreated(cancellation, NEW_STATE_DATE, PARTNER_ID + ".csv", + "hash", 10, 200); Optional updatedCancellation = cancellationRepository.findById(PARTNER_ID); Assertions.assertTrue(updatedCancellation.isPresent()); @@ -167,6 +169,9 @@ void testUpdateCsvCreated() { Assertions.assertEquals(PARTNER_ID + ".csv", updatedCancellation.get().getBucketObjectId()); assertNull(updatedCancellation.get().getDownloadLinkRequested()); Assertions.assertEquals(NEW_STATE_DATE, updatedCancellation.get().getCsvCreated()); + Assertions.assertEquals("hash", updatedCancellation.get().getCsvHash()); + Assertions.assertEquals(10, updatedCancellation.get().getCsvEntityCount()); + Assertions.assertEquals(200, updatedCancellation.get().getCsvSize()); } @Test