Feat: Cancellation Add Extra Logging

src/main/java/app/coronawarn/quicktest/domain/Cancellation.java

@@ -76,6 +76,18 @@ public class Cancellation {
     @Column(name = "bucket_object_id")
     private String bucketObjectId;
 
+    @Column(name = "csv_entity_count")
+    @JsonIgnore
+    private Integer csvEntityCount;
+
+    @Column(name = "csv_hash")
+    @JsonIgnore
+    private String csvHash;
+
+    @Column(name = "csv_size")
+    @JsonIgnore
+    private Integer csvSize;
+
     @Column(name = "data_export_error")
     @JsonIgnore
     private String dataExportError;

src/main/java/app/coronawarn/quicktest/service/ArchiveSchedulingService.java

@@ -9,14 +9,17 @@
 import com.opencsv.bean.StatefulBeanToCsv;
 import com.opencsv.bean.StatefulBeanToCsvBuilder;
 import java.io.ByteArrayInputStream;
-import java.io.InputStream;
 import java.io.StringWriter;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.time.ZonedDateTime;
 import java.util.List;
 import lombok.RequiredArgsConstructor;
 import lombok.extern.slf4j.Slf4j;
 import net.javacrumbs.shedlock.spring.annotation.SchedulerLock;
 import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.security.crypto.codec.Hex;
 import org.springframework.stereotype.Service;
 
 @Slf4j
@@ -85,25 +88,37 @@ private void processCsvUploadBatchRecursion(List<Cancellation> cancellations) {
         for (Cancellation cancellation : cancellations) {
             try {
                 List<ArchiveCipherDtoV1> quicktests =
-                        archiveService.getQuicktestsFromLongtermByTenantId(cancellation.getPartnerId());
+                  archiveService.getQuicktestsFromLongtermByTenantId(cancellation.getPartnerId());
+
                 StringWriter stringWriter = new StringWriter();
                 CSVWriter csvWriter =
-                        new CSVWriter(stringWriter, CSVWriter.DEFAULT_SEPARATOR, CSVWriter.NO_QUOTE_CHARACTER,
-                                CSVWriter.DEFAULT_ESCAPE_CHARACTER, CSVWriter.DEFAULT_LINE_END);
+                  new CSVWriter(stringWriter, '\t', CSVWriter.NO_QUOTE_CHARACTER,
+                    CSVWriter.DEFAULT_ESCAPE_CHARACTER, CSVWriter.DEFAULT_LINE_END);
                 StatefulBeanToCsv<ArchiveCipherDtoV1> beanToCsv =
-                        new StatefulBeanToCsvBuilder<ArchiveCipherDtoV1>(csvWriter)
-                                .build();
+                  new StatefulBeanToCsvBuilder<ArchiveCipherDtoV1>(csvWriter)
+                    .build();
                 beanToCsv.write(quicktests);
-                InputStream inputStream = new ByteArrayInputStream(stringWriter.toString().getBytes());
-                String id = cancellation.getPartnerId() + ".csv";
+                byte[] csvBytes = stringWriter.toString().getBytes(StandardCharsets.UTF_8);
+
+                String objectId = cancellation.getPartnerId() + ".csv";
+
                 ObjectMetadata metadata = new ObjectMetadata();
-                metadata.setContentLength(stringWriter.toString().getBytes().length);
-                s3Client.putObject(s3Config.getBucketName(), id, inputStream, metadata);
-                log.info("File stored to S3 with id {}", id);
-                cancellationService.updateCsvCreated(cancellation, ZonedDateTime.now(), id);
+                metadata.setContentLength(csvBytes.length);
+
+                s3Client.putObject(
+                    s3Config.getBucketName(),
+                    objectId,
+                    new ByteArrayInputStream(csvBytes), metadata);
+
+                log.info("File stored to S3 with id {}", objectId);
+
+                cancellationService.updateCsvCreated(cancellation, ZonedDateTime.now(), objectId,
+                    getHash(csvBytes), quicktests.size(), csvBytes.length);
             } catch (Exception e) {
-                log.error("Could not convert Quicktest to CSV: " + e.getLocalizedMessage());
-                cancellationService.updateDataExportError(cancellation, e.getLocalizedMessage());
+                String errorMessage = e.getClass().getName() + ": " + e.getMessage();
+
+                log.error("Could not convert Quicktest to CSV: " + errorMessage);
+                cancellationService.updateDataExportError(cancellation, errorMessage);
             }
         }
 
@@ -112,4 +127,14 @@ private void processCsvUploadBatchRecursion(List<Cancellation> cancellations) {
             processCsvUploadBatchRecursion(nextBatch);
         }
     }
+
+    private String getHash(byte[] bytes) {
+        try {
+            MessageDigest sha256 = MessageDigest.getInstance("SHA-256");
+            byte[] hashBytes = sha256.digest(bytes);
+            return String.valueOf(Hex.encode(hashBytes));
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException("Failed to load SHA-256 Message Digest");
+        }
+    }
 }

src/main/java/app/coronawarn/quicktest/service/CancellationService.java

@@ -120,9 +120,13 @@ public void updateMovedToLongterm(Cancellation cancellation, ZonedDateTime moved
      * @param cancellation Cancellation Entity
      * @param csvCreated   timestamp of job completion
      */
-    public void updateCsvCreated(Cancellation cancellation, ZonedDateTime csvCreated, String bucketObjectId) {
+    public void updateCsvCreated(Cancellation cancellation, ZonedDateTime csvCreated, String bucketObjectId,
+                                 String hash, int entityCount, int fileSize) {
         cancellation.setCsvCreated(csvCreated);
         cancellation.setBucketObjectId(bucketObjectId);
+        cancellation.setCsvHash(hash);
+        cancellation.setCsvEntityCount(entityCount);
+        cancellation.setCsvSize(fileSize);
         cancellationRepository.save(cancellation);
     }
 

src/main/resources/db/changelog.yml

@@ -65,3 +65,6 @@ databaseChangeLog:
   - include:
       file: changelog/V022_update_cancellationTable_3.yml
       relativeToChangelogFile: true
+  - include:
+      file: changelog/V023_update_cancellationTable_4.yml
+      relativeToChangelogFile: true

src/main/resources/db/changelog/V023_update_cancellationTable_4.yml

@@ -0,0 +1,26 @@
+databaseChangeLog:
+  - changeSet:
+      id: update-cancellation-table-4
+      author: f11h
+      changes:
+        - addColumn:
+            tableName: cancellation
+            column:
+              name: csv_size
+              type: int
+              constraints:
+                nullable: true
+        - addColumn:
+            tableName: cancellation
+            column:
+              name: csv_hash
+              type: varchar(64)
+              constraints:
+                nullable: true
+        - addColumn:
+            tableName: cancellation
+            column:
+              name: csv_entity_count
+              type: int
+              constraints:
+                nullable: true

src/test/java/app/coronawarn/quicktest/service/CancellationCsvTest.java

@@ -0,0 +1,183 @@
+/*-
+ * ---license-start
+ * Corona-Warn-App / cwa-quick-test-backend
+ * ---
+ * Copyright (C) 2021 T-Systems International GmbH and all other contributors
+ * ---
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ---license-end
+ */
+
+package app.coronawarn.quicktest.service;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import app.coronawarn.quicktest.archive.repository.ArchiveRepository;
+import app.coronawarn.quicktest.domain.Cancellation;
+import app.coronawarn.quicktest.domain.QuickTestArchive;
+import app.coronawarn.quicktest.model.Sex;
+import app.coronawarn.quicktest.repository.CancellationRepository;
+import app.coronawarn.quicktest.repository.QuickTestArchiveRepository;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.model.PutObjectResult;
+import com.opencsv.CSVParser;
+import com.opencsv.CSVParserBuilder;
+import com.opencsv.CSVReader;
+import com.opencsv.CSVReaderBuilder;
+import com.opencsv.exceptions.CsvException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.time.LocalDateTime;
+import java.time.ZonedDateTime;
+import java.time.temporal.ChronoUnit;
+import java.util.List;
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.tomcat.util.buf.HexUtils;
+import org.bouncycastle.util.encoders.Hex;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.ArgumentCaptor;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.boot.test.mock.mockito.MockBean;
+import org.springframework.data.domain.Pageable;
+import org.springframework.transaction.annotation.Transactional;
+
+@SpringBootTest
+class CancellationCsvTest {
+
+    @Autowired
+    private ArchiveSchedulingService archiveSchedulingService;
+
+    @Autowired
+    private QuickTestArchiveRepository shortTermArchiveRepository;
+
+    @Autowired
+    private ArchiveRepository longTermArchiveRepository;
+
+    @Autowired
+    private CancellationRepository cancellationRepository;
+
+    @MockBean
+    private AmazonS3 s3Client;
+
+    @BeforeEach
+    void setUp() {
+        shortTermArchiveRepository.deleteAll();
+        longTermArchiveRepository.deleteAllByTenantId(PARTNER_ID);
+        cancellationRepository.deleteAll();
+    }
+
+    public static final ZonedDateTime CANCELLATION_DATE = ZonedDateTime.now().minusHours(49).truncatedTo(ChronoUnit.MINUTES);
+    public static final String PARTNER_ID = "P10000";
+    public static final String PARTNER_ID_HASH = "212e58b487b6d6b486b71c6ebb3fedc0db3c69114f125fb3cd2fbc72e6ffc25f";
+
+    @ParameterizedTest
+    @ValueSource(ints = {1, 5000})
+    @Transactional
+    void testCsvExport(int n) throws IOException, NoSuchAlgorithmException, CsvException {
+        Cancellation cancellation = new Cancellation();
+        cancellation.setPartnerId(PARTNER_ID);
+        cancellation.setCancellationDate(CANCELLATION_DATE);
+        cancellationRepository.save(cancellation);
+
+        for (int i = 0; i < n; i++) {
+            shortTermArchiveRepository.save(buildQuickTestArchive(PARTNER_ID));
+        }
+
+        Assertions.assertEquals(n, shortTermArchiveRepository.findAllByTenantId(PARTNER_ID, Pageable.unpaged()).count());
+        Assertions.assertEquals(0, longTermArchiveRepository.findAllByTenantId(PARTNER_ID_HASH).size());
+
+        archiveSchedulingService.cancellationArchiveJob();
+
+        Assertions.assertEquals(0, shortTermArchiveRepository.findAllByTenantId(PARTNER_ID, Pageable.unpaged()).count());
+        Assertions.assertEquals(n, longTermArchiveRepository.findAllByTenantId(PARTNER_ID_HASH).size());
+
+        ArgumentCaptor<InputStream> inputStreamArgumentCaptor = ArgumentCaptor.forClass(InputStream.class);
+        String expectedFileName = PARTNER_ID + ".csv";
+        when(s3Client.putObject(anyString(), eq(expectedFileName), inputStreamArgumentCaptor.capture(), any()))
+          .thenReturn(new PutObjectResult());
+
+        archiveSchedulingService.csvUploadJob();
+
+        verify(s3Client).putObject(anyString(), eq(expectedFileName), any(), any());
+
+        byte[] csvBytes = inputStreamArgumentCaptor.getValue().readAllBytes();
+        String csv = new String(csvBytes, StandardCharsets.UTF_8);
+
+        cancellation = cancellationRepository.findById(PARTNER_ID).orElseThrow();
+
+        Assertions.assertEquals(n, cancellation.getCsvEntityCount());
+        Assertions.assertEquals(csvBytes.length, cancellation.getCsvSize());
+        Assertions.assertEquals(getHash(csvBytes), cancellation.getCsvHash());
+
+        CSVParser csvParser = new CSVParserBuilder()
+                .withSeparator('\t')
+                .build();
+
+        try (CSVReader csvReader = new CSVReaderBuilder(new StringReader(csv))
+                .withCSVParser(csvParser)
+                .build()
+        ) {
+            List<String[]> csvEntries = csvReader.readAll();
+            Assertions.assertEquals(n + 1, csvEntries.size());
+            Assertions.assertEquals(27, csvEntries.get(0).length);
+        }
+
+        longTermArchiveRepository.deleteAllByTenantId(PARTNER_ID_HASH);
+    }
+
+    private String getHash(byte[] bytes) throws NoSuchAlgorithmException {
+        return Hex.toHexString(MessageDigest.getInstance("SHA-256").digest(bytes));
+    }
+    private QuickTestArchive buildQuickTestArchive(String tenantId) {
+        QuickTestArchive qta = new QuickTestArchive();
+        qta.setShortHashedGuid(HexUtils.toHexString(RandomUtils.nextBytes(4)));
+        qta.setHashedGuid(HexUtils.toHexString(RandomUtils.nextBytes(32)));
+        qta.setTenantId(tenantId);
+        qta.setPocId("poc_id");
+        qta.setCreatedAt(LocalDateTime.now().minusMonths(3));
+        qta.setUpdatedAt(LocalDateTime.now().minusMonths(2));
+        qta.setConfirmationCwa(Boolean.TRUE);
+        qta.setTestResult(Short.valueOf("6"));
+        qta.setPrivacyAgreement(Boolean.TRUE);
+        qta.setLastName("last_name");
+        qta.setFirstName("first_name");
+        qta.setEmail("email");
+        qta.setPhoneNumber("phone_number");
+        qta.setSex(Sex.MALE);
+        qta.setStreet("street");
+        qta.setHouseNumber("house_number");
+        qta.setZipCode("zip_code");
+        qta.setCity("city");
+        qta.setTestBrandId("test_brand_id");
+        qta.setTestBrandName("test_brand_name, Ltd, another_part_of_test_brand_name");
+        qta.setBirthday("2000-01-01");
+        qta.setPdf("PDF".getBytes());
+        qta.setTestResultServerHash("test_result_server_hash");
+        qta.setDcc("dcc");
+        qta.setAdditionalInfo("additional_info");
+        qta.setGroupName("group_name");
+        return qta;
+    }
+}

src/test/java/app/coronawarn/quicktest/service/CancellationServiceTest.java

@@ -24,6 +24,7 @@
 import static org.junit.jupiter.api.Assertions.assertNull;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.endsWith;
 import static org.mockito.Mockito.doNothing;
 
 import app.coronawarn.quicktest.domain.Cancellation;
@@ -155,7 +156,8 @@ void testUpdateCsvCreated() {
         Cancellation cancellation = cancellationService.createCancellation(PARTNER_ID, CANCELLATION_DATE);
         Assertions.assertEquals(FINAL_DELETION, cancellation.getFinalDeletion());
 
-        cancellationService.updateCsvCreated(cancellation, NEW_STATE_DATE, PARTNER_ID + ".csv");
+        cancellationService.updateCsvCreated(cancellation, NEW_STATE_DATE, PARTNER_ID + ".csv",
+          "hash", 10, 200);
 
         Optional<Cancellation> updatedCancellation = cancellationRepository.findById(PARTNER_ID);
         Assertions.assertTrue(updatedCancellation.isPresent());
@@ -167,6 +169,9 @@ void testUpdateCsvCreated() {
         Assertions.assertEquals(PARTNER_ID + ".csv", updatedCancellation.get().getBucketObjectId());
         assertNull(updatedCancellation.get().getDownloadLinkRequested());
         Assertions.assertEquals(NEW_STATE_DATE, updatedCancellation.get().getCsvCreated());
+        Assertions.assertEquals("hash", updatedCancellation.get().getCsvHash());
+        Assertions.assertEquals(10, updatedCancellation.get().getCsvEntityCount());
+        Assertions.assertEquals(200, updatedCancellation.get().getCsvSize());
     }
 
     @Test