From 4897cbe6c18cd4f73c561956d4972a42af7153e9 Mon Sep 17 00:00:00 2001 From: Krishna Bottla <40598480+kbottla@users.noreply.github.com> Date: Mon, 18 Sep 2023 14:01:08 +0100 Subject: [PATCH] PP-11232 Add method to find transactions for redaction - Added method to find transactions between a date range for redaction. This is to incrementally find transactions and redact PII from transactions. --- .../transaction/dao/TransactionDao.java | 108 +++++++++++------- .../transaction/dao/TransactionDaoIT.java | 72 ++++++++++-- 2 files changed, 129 insertions(+), 51 deletions(-) diff --git a/src/main/java/uk/gov/pay/ledger/transaction/dao/TransactionDao.java b/src/main/java/uk/gov/pay/ledger/transaction/dao/TransactionDao.java index e787302f3..6f23deb22 100644 --- a/src/main/java/uk/gov/pay/ledger/transaction/dao/TransactionDao.java +++ b/src/main/java/uk/gov/pay/ledger/transaction/dao/TransactionDao.java @@ -26,63 +26,68 @@ public class TransactionDao { private static final String FIND_TRANSACTION_BY_EXTERNAL_ID = "SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " + - "LEFT OUTER JOIN payout po on " + - "t.gateway_payout_id = po.gateway_payout_id " + - ":payoutJoinOnGatewayIdField " + - "WHERE t.external_id = :externalId " + - "AND (:gatewayAccountId is NULL OR t.gateway_account_id = :gatewayAccountId)"; + "LEFT OUTER JOIN payout po on " + + "t.gateway_payout_id = po.gateway_payout_id " + + ":payoutJoinOnGatewayIdField " + + "WHERE t.external_id = :externalId " + + "AND (:gatewayAccountId is NULL OR t.gateway_account_id = :gatewayAccountId)"; private static final String FIND_TRANSACTION_BY_EXTERNAL_ID_AND_GATEWAY_ACCOUNT_ID = "SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " + - "LEFT OUTER JOIN payout po on " + - "t.gateway_payout_id = po.gateway_payout_id " + - "AND po.gateway_account_id = :gatewayAccountId " + - "WHERE t.external_id = :externalId " + - "AND t.gateway_account_id = :gatewayAccountId " + - "AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type) " + - "AND (:parentExternalId is NULL OR t.parent_external_id = :parentExternalId)"; + "LEFT OUTER JOIN payout po on " + + "t.gateway_payout_id = po.gateway_payout_id " + + "AND po.gateway_account_id = :gatewayAccountId " + + "WHERE t.external_id = :externalId " + + "AND t.gateway_account_id = :gatewayAccountId " + + "AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type) " + + "AND (:parentExternalId is NULL OR t.parent_external_id = :parentExternalId)"; private static final String FIND_TRANSACTIONS_BY_EXTERNAL_OR_PARENT_ID_AND_GATEWAY_ACCOUNT_ID = "SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " + - "LEFT OUTER JOIN payout po on " + - "t.gateway_payout_id = po.gateway_payout_id " + - "AND po.gateway_account_id = :gatewayAccountId " + - "WHERE (t.external_id = :externalId or t.parent_external_id = :externalId) " + - "AND t.gateway_account_id = :gatewayAccountId"; + "LEFT OUTER JOIN payout po on " + + "t.gateway_payout_id = po.gateway_payout_id " + + "AND po.gateway_account_id = :gatewayAccountId " + + "WHERE (t.external_id = :externalId or t.parent_external_id = :externalId) " + + "AND t.gateway_account_id = :gatewayAccountId"; private static final String FIND_TRANSACTIONS_BY_PARENT_EXT_ID_AND_GATEWAY_ACCOUNT_ID = "SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " + - "LEFT OUTER JOIN payout po on " + - "t.gateway_payout_id = po.gateway_payout_id " + - ":payoutJoinOnGatewayIdField " + - "WHERE t.parent_external_id = :parentExternalId " + - "AND t.gateway_account_id = :gatewayAccountId " + - "AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type)"; + "LEFT OUTER JOIN payout po on " + + "t.gateway_payout_id = po.gateway_payout_id " + + ":payoutJoinOnGatewayIdField " + + "WHERE t.parent_external_id = :parentExternalId " + + "AND t.gateway_account_id = :gatewayAccountId " + + "AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type)"; private static final String FIND_TRANSACTIONS_BY_PARENT_EXT_ID = "SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " + - "LEFT OUTER JOIN payout po on " + - "t.gateway_payout_id = po.gateway_payout_id " + - "WHERE t.parent_external_id = :parentExternalId"; + "LEFT OUTER JOIN payout po on " + + "t.gateway_payout_id = po.gateway_payout_id " + + "WHERE t.parent_external_id = :parentExternalId"; private static final String SEARCH_TRANSACTIONS = "SELECT :distinctClauseWhenSearchingByMetadataValue t.*, po.paid_out_date AS paid_out_date FROM transaction t " + - " :transactionMetadataJoin " + - " LEFT OUTER JOIN payout po on " + - "t.gateway_payout_id = po.gateway_payout_id " + - ":payoutJoinOnGatewayIdField " + - ":searchExtraFields " + - "ORDER BY t.created_date DESC OFFSET :offset LIMIT :limit"; + " :transactionMetadataJoin " + + " LEFT OUTER JOIN payout po on " + + "t.gateway_payout_id = po.gateway_payout_id " + + ":payoutJoinOnGatewayIdField " + + ":searchExtraFields " + + "ORDER BY t.created_date DESC OFFSET :offset LIMIT :limit"; private static final String SEARCH_TRANSACTIONS_CURSOR = "SELECT :distinctClauseWhenSearchingByMetadataValue t.*, po.paid_out_date AS paid_out_date FROM transaction t " + - " :transactionMetadataJoin " + - "LEFT OUTER JOIN payout po on " + - "t.gateway_payout_id = po.gateway_payout_id " + - ":payoutJoinOnGatewayIdField " + - ":searchExtraFields " + - ":cursorFields " + - "ORDER BY t.created_date DESC, t.id DESC LIMIT :limit"; + " :transactionMetadataJoin " + + "LEFT OUTER JOIN payout po on " + + "t.gateway_payout_id = po.gateway_payout_id " + + ":payoutJoinOnGatewayIdField " + + ":searchExtraFields " + + ":cursorFields " + + "ORDER BY t.created_date DESC, t.id DESC LIMIT :limit"; + + private static final String SEARCH_TRANSACTIONS_FOR_REDACTION = + "SELECT t.* FROM transaction t " + + " WHERE t.created_date >= :dateOfLastProcessedTransaction AND t.created_date < :redactTransactionsUpToDate " + + "ORDER BY t.created_date ASC LIMIT :limit"; private static final String COUNT_TRANSACTIONS = "SELECT count(:distinctClauseWhenSearchingByMetadataValue t.id) " + "FROM transaction t " + @@ -283,9 +288,9 @@ public List findTransactionsByParentIdAndGatewayAccountId(Str public List findTransactionByParentId(String parentExternalId) { return jdbi.withHandle(handle -> handle.createQuery(FIND_TRANSACTIONS_BY_PARENT_EXT_ID) - .bind("parentExternalId", parentExternalId) - .map(new TransactionMapper()) - .stream().collect(Collectors.toList()) + .bind("parentExternalId", parentExternalId) + .map(new TransactionMapper()) + .stream().collect(Collectors.toList()) ); } @@ -357,6 +362,21 @@ public List cursorTransactionSearch(TransactionSearchParams s }); } + public List findTransactionsForRedaction(ZonedDateTime dateOfLastProcessedTransaction, + ZonedDateTime redactTransactionsUpToDate, + int noOfTransactionsToReturn) { + return jdbi.withHandle(handle -> { + Query query = handle.createQuery(SEARCH_TRANSACTIONS_FOR_REDACTION); + query.bind("dateOfLastProcessedTransaction", dateOfLastProcessedTransaction); + query.bind("redactTransactionsUpToDate", redactTransactionsUpToDate); + query.bind("limit", noOfTransactionsToReturn); + + return query + .map(new TransactionMapper()) + .list(); + }); + } + private String createSearchTemplate(TransactionSearchParams searchParams, String baseQueryString) { String searchClauseTemplate = String.join(" AND ", searchParams.getFilterTemplates()); searchClauseTemplate = StringUtils.isNotBlank(searchClauseTemplate) ? @@ -365,7 +385,7 @@ private String createSearchTemplate(TransactionSearchParams searchParams, String baseQueryString = baseQueryString .replace(":payoutJoinOnGatewayIdField", (searchParams.getAccountIds() != null && !searchParams.getAccountIds().isEmpty()) - ? SEARCH_CLAUSE_TRANSACTIONS_WITH_PAYOUT : ""); + ? SEARCH_CLAUSE_TRANSACTIONS_WITH_PAYOUT : ""); baseQueryString = baseQueryString .replace(":transactionMetadataJoin", @@ -401,4 +421,4 @@ public List getSourceTypeValues() { .mapTo(String.class) .collect(Collectors.toList())); } -} \ No newline at end of file +} diff --git a/src/test/java/uk/gov/pay/ledger/transaction/dao/TransactionDaoIT.java b/src/test/java/uk/gov/pay/ledger/transaction/dao/TransactionDaoIT.java index 0f328791e..bd274e816 100644 --- a/src/test/java/uk/gov/pay/ledger/transaction/dao/TransactionDaoIT.java +++ b/src/test/java/uk/gov/pay/ledger/transaction/dao/TransactionDaoIT.java @@ -1,29 +1,35 @@ package uk.gov.pay.ledger.transaction.dao; import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.RegisterExtension; import uk.gov.pay.ledger.app.LedgerConfig; -import uk.gov.service.payments.commons.model.Source; import uk.gov.pay.ledger.extension.AppWithPostgresAndSqsExtension; import uk.gov.pay.ledger.transaction.entity.TransactionEntity; import uk.gov.pay.ledger.transaction.model.TransactionType; import uk.gov.pay.ledger.transaction.state.TransactionState; import uk.gov.pay.ledger.util.fixture.TransactionFixture; +import uk.gov.service.payments.commons.model.Source; import java.time.ZonedDateTime; +import java.time.temporal.ChronoUnit; import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; +import static java.time.ZonedDateTime.parse; import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric; import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.hasItems; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.CoreMatchers.notNullValue; import static org.hamcrest.CoreMatchers.nullValue; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.hasProperty; +import static org.hamcrest.Matchers.not; import static org.mockito.Mockito.mock; import static uk.gov.pay.ledger.transaction.service.TransactionService.REDACTED_REFERENCE_NUMBER; import static uk.gov.pay.ledger.util.fixture.PayoutFixture.PayoutFixtureBuilder.aPayoutFixture; @@ -84,7 +90,7 @@ void shouldInsertTransactionWithSourceCardApi() { @Test void shouldRetrieveTransactionByExternalIdAndGatewayAccount() { - ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z"); + ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z"); String payOutId = randomAlphanumeric(20); TransactionFixture fixture = aTransactionFixture() @@ -113,7 +119,7 @@ void shouldRetrieveTransactionByExternalIdAndGatewayAccount() { @Test void shouldRetrieveTransactionWithPayoutDateByExternalIdAndNoGatewayAccount() { - ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z"); + ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z"); String payOutId = randomAlphanumeric(20); TransactionFixture fixture = aTransactionFixture() @@ -169,7 +175,7 @@ private void assertTransactionEntity(TransactionEntity transaction, TransactionF @Test void shouldRetrieveTransactionByExternalIdAndGatewayAccountId() { String payOutId = randomAlphanumeric(20); - ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z"); + ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z"); TransactionFixture fixture = aTransactionFixture() .withDefaultCardDetails() @@ -284,7 +290,7 @@ void shouldNotOverwriteTransactionIfItConsistsOfFewerEvents() { @Test void shouldFilterTransactionByExternalIdOrParentExternalIdAndGatewayAccountId() { String payOutId = randomAlphanumeric(20); - ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z"); + ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z"); TransactionEntity transaction1 = aTransactionFixture() .withState(TransactionState.CREATED) @@ -406,7 +412,7 @@ void findTransactionByParentIdAndGatewayAccountId_shouldFilterByParentExternalId @Test void findTransactionByParentId_shouldFilterByParentExternalId() { String payOutId = randomAlphanumeric(20); - ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z"); + ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z"); TransactionEntity transaction1 = aTransactionFixture() .withState(TransactionState.CREATED) @@ -447,4 +453,56 @@ void sourceTypeInDatabase_shouldMatchValuesInEnum() { transactionDao.getSourceTypeValues().forEach(x -> assertThat(sourceArray.contains(x), is(true))); sourceArray.forEach(x -> assertThat(transactionDao.getSourceTypeValues().contains(x), is(true))); } -} \ No newline at end of file + + @Nested + @DisplayName("TestFindTransactionsForRedaction") + class TestFindTransactionsForRedaction { + + @Test + void shouldReturnTransactionsCorrectlyForDateRangesAndNumberOfTransactions() { + TransactionEntity transactionToReturnToDelete1 = aTransactionFixture() + .withCreatedDate(parse("2016-01-01T00:00:00Z")) + .insert(rule.getJdbi()) + .toEntity(); + TransactionEntity transactionToReturnToDelete2 = aTransactionFixture() + .withCreatedDate(parse("2016-01-01T01:00:00Z")) + .insert(rule.getJdbi()) + .toEntity(); + TransactionEntity transactionEligibleForSearchButNotReturnedDueToLimit = aTransactionFixture() + .withCreatedDate(parse("2016-01-01T02:00:00Z")) + .insert(rule.getJdbi()) + .toEntity(); + + TransactionEntity transactionToExclude1 = aTransactionFixture() + .withCreatedDate(parse("2016-01-02T00:00:00Z")) + .insert(rule.getJdbi()) + .toEntity(); + TransactionEntity transactionToExclude2 = aTransactionFixture() + .withCreatedDate(parse("2016-01-03T00:00:00Z")) + .insert(rule.getJdbi()) + .toEntity(); + + List transactionsForRedaction = transactionDao.findTransactionsForRedaction( + parse("2015-12-31T00:00:00Z"), + parse("2016-01-02T00:00:00Z"), + 2 + ); + + List transactionIdsReturned = transactionsForRedaction + .stream() + .map(TransactionEntity::getExternalId) + .collect(Collectors.toList()); + + assertThat(transactionsForRedaction.size(), is(2)); + assertThat(transactionIdsReturned, hasItems( + transactionToReturnToDelete1.getExternalId(), + transactionToReturnToDelete2.getExternalId()) + ); + assertThat(transactionIdsReturned, not(hasItems( + transactionEligibleForSearchButNotReturnedDueToLimit.getExternalId(), + transactionToExclude1.getExternalId(), + transactionToExclude2.getExternalId()) + )); + } + } +}