Skip to content

Commit

Permalink
Merge pull request #2885 from alphagov/pp_11232_add_method_to_find_tr…
Browse files Browse the repository at this point in the history
…xs_to_redact

PP-11232 Add method to find transactions for redaction
  • Loading branch information
kbottla authored Sep 19, 2023
2 parents 0c25c45 + 4897cbe commit 69c3839
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 51 deletions.
108 changes: 64 additions & 44 deletions src/main/java/uk/gov/pay/ledger/transaction/dao/TransactionDao.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,63 +26,68 @@ public class TransactionDao {

private static final String FIND_TRANSACTION_BY_EXTERNAL_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
"WHERE t.external_id = :externalId " +
"AND (:gatewayAccountId is NULL OR t.gateway_account_id = :gatewayAccountId)";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
"WHERE t.external_id = :externalId " +
"AND (:gatewayAccountId is NULL OR t.gateway_account_id = :gatewayAccountId)";

private static final String FIND_TRANSACTION_BY_EXTERNAL_ID_AND_GATEWAY_ACCOUNT_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"AND po.gateway_account_id = :gatewayAccountId " +
"WHERE t.external_id = :externalId " +
"AND t.gateway_account_id = :gatewayAccountId " +
"AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type) " +
"AND (:parentExternalId is NULL OR t.parent_external_id = :parentExternalId)";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"AND po.gateway_account_id = :gatewayAccountId " +
"WHERE t.external_id = :externalId " +
"AND t.gateway_account_id = :gatewayAccountId " +
"AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type) " +
"AND (:parentExternalId is NULL OR t.parent_external_id = :parentExternalId)";

private static final String FIND_TRANSACTIONS_BY_EXTERNAL_OR_PARENT_ID_AND_GATEWAY_ACCOUNT_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"AND po.gateway_account_id = :gatewayAccountId " +
"WHERE (t.external_id = :externalId or t.parent_external_id = :externalId) " +
"AND t.gateway_account_id = :gatewayAccountId";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"AND po.gateway_account_id = :gatewayAccountId " +
"WHERE (t.external_id = :externalId or t.parent_external_id = :externalId) " +
"AND t.gateway_account_id = :gatewayAccountId";

private static final String FIND_TRANSACTIONS_BY_PARENT_EXT_ID_AND_GATEWAY_ACCOUNT_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
"WHERE t.parent_external_id = :parentExternalId " +
"AND t.gateway_account_id = :gatewayAccountId " +
"AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type)";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
"WHERE t.parent_external_id = :parentExternalId " +
"AND t.gateway_account_id = :gatewayAccountId " +
"AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type)";

private static final String FIND_TRANSACTIONS_BY_PARENT_EXT_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"WHERE t.parent_external_id = :parentExternalId";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"WHERE t.parent_external_id = :parentExternalId";

private static final String SEARCH_TRANSACTIONS =
"SELECT :distinctClauseWhenSearchingByMetadataValue t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
" :transactionMetadataJoin " +
" LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
":searchExtraFields " +
"ORDER BY t.created_date DESC OFFSET :offset LIMIT :limit";
" :transactionMetadataJoin " +
" LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
":searchExtraFields " +
"ORDER BY t.created_date DESC OFFSET :offset LIMIT :limit";

private static final String SEARCH_TRANSACTIONS_CURSOR =
"SELECT :distinctClauseWhenSearchingByMetadataValue t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
" :transactionMetadataJoin " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
":searchExtraFields " +
":cursorFields " +
"ORDER BY t.created_date DESC, t.id DESC LIMIT :limit";
" :transactionMetadataJoin " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
":searchExtraFields " +
":cursorFields " +
"ORDER BY t.created_date DESC, t.id DESC LIMIT :limit";

private static final String SEARCH_TRANSACTIONS_FOR_REDACTION =
"SELECT t.* FROM transaction t " +
" WHERE t.created_date >= :dateOfLastProcessedTransaction AND t.created_date < :redactTransactionsUpToDate " +
"ORDER BY t.created_date ASC LIMIT :limit";

private static final String COUNT_TRANSACTIONS = "SELECT count(:distinctClauseWhenSearchingByMetadataValue t.id) " +
"FROM transaction t " +
Expand Down Expand Up @@ -283,9 +288,9 @@ public List<TransactionEntity> findTransactionsByParentIdAndGatewayAccountId(Str
public List<TransactionEntity> findTransactionByParentId(String parentExternalId) {
return jdbi.withHandle(handle ->
handle.createQuery(FIND_TRANSACTIONS_BY_PARENT_EXT_ID)
.bind("parentExternalId", parentExternalId)
.map(new TransactionMapper())
.stream().collect(Collectors.toList())
.bind("parentExternalId", parentExternalId)
.map(new TransactionMapper())
.stream().collect(Collectors.toList())
);
}

Expand Down Expand Up @@ -357,6 +362,21 @@ public List<TransactionEntity> cursorTransactionSearch(TransactionSearchParams s
});
}

public List<TransactionEntity> findTransactionsForRedaction(ZonedDateTime dateOfLastProcessedTransaction,
ZonedDateTime redactTransactionsUpToDate,
int noOfTransactionsToReturn) {
return jdbi.withHandle(handle -> {
Query query = handle.createQuery(SEARCH_TRANSACTIONS_FOR_REDACTION);
query.bind("dateOfLastProcessedTransaction", dateOfLastProcessedTransaction);
query.bind("redactTransactionsUpToDate", redactTransactionsUpToDate);
query.bind("limit", noOfTransactionsToReturn);

return query
.map(new TransactionMapper())
.list();
});
}

private String createSearchTemplate(TransactionSearchParams searchParams, String baseQueryString) {
String searchClauseTemplate = String.join(" AND ", searchParams.getFilterTemplates());
searchClauseTemplate = StringUtils.isNotBlank(searchClauseTemplate) ?
Expand All @@ -365,7 +385,7 @@ private String createSearchTemplate(TransactionSearchParams searchParams, String
baseQueryString = baseQueryString
.replace(":payoutJoinOnGatewayIdField",
(searchParams.getAccountIds() != null && !searchParams.getAccountIds().isEmpty())
? SEARCH_CLAUSE_TRANSACTIONS_WITH_PAYOUT : "");
? SEARCH_CLAUSE_TRANSACTIONS_WITH_PAYOUT : "");

baseQueryString = baseQueryString
.replace(":transactionMetadataJoin",
Expand Down Expand Up @@ -401,4 +421,4 @@ public List<String> getSourceTypeValues() {
.mapTo(String.class)
.collect(Collectors.toList()));
}
}
}
Original file line number Diff line number Diff line change
@@ -1,29 +1,35 @@
package uk.gov.pay.ledger.transaction.dao;

import com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import uk.gov.pay.ledger.app.LedgerConfig;
import uk.gov.service.payments.commons.model.Source;
import uk.gov.pay.ledger.extension.AppWithPostgresAndSqsExtension;
import uk.gov.pay.ledger.transaction.entity.TransactionEntity;
import uk.gov.pay.ledger.transaction.model.TransactionType;
import uk.gov.pay.ledger.transaction.state.TransactionState;
import uk.gov.pay.ledger.util.fixture.TransactionFixture;
import uk.gov.service.payments.commons.model.Source;

import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;

import static java.time.ZonedDateTime.parse;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.hasItems;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.hasProperty;
import static org.hamcrest.Matchers.not;
import static org.mockito.Mockito.mock;
import static uk.gov.pay.ledger.transaction.service.TransactionService.REDACTED_REFERENCE_NUMBER;
import static uk.gov.pay.ledger.util.fixture.PayoutFixture.PayoutFixtureBuilder.aPayoutFixture;
Expand Down Expand Up @@ -84,7 +90,7 @@ void shouldInsertTransactionWithSourceCardApi() {

@Test
void shouldRetrieveTransactionByExternalIdAndGatewayAccount() {
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");
String payOutId = randomAlphanumeric(20);

TransactionFixture fixture = aTransactionFixture()
Expand Down Expand Up @@ -113,7 +119,7 @@ void shouldRetrieveTransactionByExternalIdAndGatewayAccount() {

@Test
void shouldRetrieveTransactionWithPayoutDateByExternalIdAndNoGatewayAccount() {
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");
String payOutId = randomAlphanumeric(20);

TransactionFixture fixture = aTransactionFixture()
Expand Down Expand Up @@ -169,7 +175,7 @@ private void assertTransactionEntity(TransactionEntity transaction, TransactionF
@Test
void shouldRetrieveTransactionByExternalIdAndGatewayAccountId() {
String payOutId = randomAlphanumeric(20);
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");

TransactionFixture fixture = aTransactionFixture()
.withDefaultCardDetails()
Expand Down Expand Up @@ -284,7 +290,7 @@ void shouldNotOverwriteTransactionIfItConsistsOfFewerEvents() {
@Test
void shouldFilterTransactionByExternalIdOrParentExternalIdAndGatewayAccountId() {
String payOutId = randomAlphanumeric(20);
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");

TransactionEntity transaction1 = aTransactionFixture()
.withState(TransactionState.CREATED)
Expand Down Expand Up @@ -406,7 +412,7 @@ void findTransactionByParentIdAndGatewayAccountId_shouldFilterByParentExternalId
@Test
void findTransactionByParentId_shouldFilterByParentExternalId() {
String payOutId = randomAlphanumeric(20);
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");

TransactionEntity transaction1 = aTransactionFixture()
.withState(TransactionState.CREATED)
Expand Down Expand Up @@ -447,4 +453,56 @@ void sourceTypeInDatabase_shouldMatchValuesInEnum() {
transactionDao.getSourceTypeValues().forEach(x -> assertThat(sourceArray.contains(x), is(true)));
sourceArray.forEach(x -> assertThat(transactionDao.getSourceTypeValues().contains(x), is(true)));
}
}

@Nested
@DisplayName("TestFindTransactionsForRedaction")
class TestFindTransactionsForRedaction {

@Test
void shouldReturnTransactionsCorrectlyForDateRangesAndNumberOfTransactions() {
TransactionEntity transactionToReturnToDelete1 = aTransactionFixture()
.withCreatedDate(parse("2016-01-01T00:00:00Z"))
.insert(rule.getJdbi())
.toEntity();
TransactionEntity transactionToReturnToDelete2 = aTransactionFixture()
.withCreatedDate(parse("2016-01-01T01:00:00Z"))
.insert(rule.getJdbi())
.toEntity();
TransactionEntity transactionEligibleForSearchButNotReturnedDueToLimit = aTransactionFixture()
.withCreatedDate(parse("2016-01-01T02:00:00Z"))
.insert(rule.getJdbi())
.toEntity();

TransactionEntity transactionToExclude1 = aTransactionFixture()
.withCreatedDate(parse("2016-01-02T00:00:00Z"))
.insert(rule.getJdbi())
.toEntity();
TransactionEntity transactionToExclude2 = aTransactionFixture()
.withCreatedDate(parse("2016-01-03T00:00:00Z"))
.insert(rule.getJdbi())
.toEntity();

List<TransactionEntity> transactionsForRedaction = transactionDao.findTransactionsForRedaction(
parse("2015-12-31T00:00:00Z"),
parse("2016-01-02T00:00:00Z"),
2
);

List<String> transactionIdsReturned = transactionsForRedaction
.stream()
.map(TransactionEntity::getExternalId)
.collect(Collectors.toList());

assertThat(transactionsForRedaction.size(), is(2));
assertThat(transactionIdsReturned, hasItems(
transactionToReturnToDelete1.getExternalId(),
transactionToReturnToDelete2.getExternalId())
);
assertThat(transactionIdsReturned, not(hasItems(
transactionEligibleForSearchButNotReturnedDueToLimit.getExternalId(),
transactionToExclude1.getExternalId(),
transactionToExclude2.getExternalId())
));
}
}
}

0 comments on commit 69c3839

Please sign in to comment.