Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PP-11232 Add method to find transactions for redaction #2885

Merged
merged 1 commit into from
Sep 19, 2023
Merged

PP-11232 Add method to find transactions for redaction #2885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
108 changes: 64 additions & 44 deletions src/main/java/uk/gov/pay/ledger/transaction/dao/TransactionDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,63 +26,68 @@ public class TransactionDao {

private static final String FIND_TRANSACTION_BY_EXTERNAL_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
"WHERE t.external_id = :externalId " +
"AND (:gatewayAccountId is NULL OR t.gateway_account_id = :gatewayAccountId)";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
"WHERE t.external_id = :externalId " +
"AND (:gatewayAccountId is NULL OR t.gateway_account_id = :gatewayAccountId)";

private static final String FIND_TRANSACTION_BY_EXTERNAL_ID_AND_GATEWAY_ACCOUNT_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"AND po.gateway_account_id = :gatewayAccountId " +
"WHERE t.external_id = :externalId " +
"AND t.gateway_account_id = :gatewayAccountId " +
"AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type) " +
"AND (:parentExternalId is NULL OR t.parent_external_id = :parentExternalId)";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"AND po.gateway_account_id = :gatewayAccountId " +
"WHERE t.external_id = :externalId " +
"AND t.gateway_account_id = :gatewayAccountId " +
"AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type) " +
"AND (:parentExternalId is NULL OR t.parent_external_id = :parentExternalId)";

private static final String FIND_TRANSACTIONS_BY_EXTERNAL_OR_PARENT_ID_AND_GATEWAY_ACCOUNT_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"AND po.gateway_account_id = :gatewayAccountId " +
"WHERE (t.external_id = :externalId or t.parent_external_id = :externalId) " +
"AND t.gateway_account_id = :gatewayAccountId";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"AND po.gateway_account_id = :gatewayAccountId " +
"WHERE (t.external_id = :externalId or t.parent_external_id = :externalId) " +
"AND t.gateway_account_id = :gatewayAccountId";

private static final String FIND_TRANSACTIONS_BY_PARENT_EXT_ID_AND_GATEWAY_ACCOUNT_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
"WHERE t.parent_external_id = :parentExternalId " +
"AND t.gateway_account_id = :gatewayAccountId " +
"AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type)";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
"WHERE t.parent_external_id = :parentExternalId " +
"AND t.gateway_account_id = :gatewayAccountId " +
"AND (:transactionType::transaction_type is NULL OR type = :transactionType::transaction_type)";

private static final String FIND_TRANSACTIONS_BY_PARENT_EXT_ID =
"SELECT t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"WHERE t.parent_external_id = :parentExternalId";
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
"WHERE t.parent_external_id = :parentExternalId";

private static final String SEARCH_TRANSACTIONS =
"SELECT :distinctClauseWhenSearchingByMetadataValue t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
" :transactionMetadataJoin " +
" LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
":searchExtraFields " +
"ORDER BY t.created_date DESC OFFSET :offset LIMIT :limit";
" :transactionMetadataJoin " +
" LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
":searchExtraFields " +
"ORDER BY t.created_date DESC OFFSET :offset LIMIT :limit";

private static final String SEARCH_TRANSACTIONS_CURSOR =
"SELECT :distinctClauseWhenSearchingByMetadataValue t.*, po.paid_out_date AS paid_out_date FROM transaction t " +
" :transactionMetadataJoin " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
":searchExtraFields " +
":cursorFields " +
"ORDER BY t.created_date DESC, t.id DESC LIMIT :limit";
" :transactionMetadataJoin " +
"LEFT OUTER JOIN payout po on " +
"t.gateway_payout_id = po.gateway_payout_id " +
":payoutJoinOnGatewayIdField " +
":searchExtraFields " +
":cursorFields " +
"ORDER BY t.created_date DESC, t.id DESC LIMIT :limit";

private static final String SEARCH_TRANSACTIONS_FOR_REDACTION =
"SELECT t.* FROM transaction t " +
" WHERE t.created_date >= :dateOfLastProcessedTransaction AND t.created_date < :redactTransactionsUpToDate " +
"ORDER BY t.created_date ASC LIMIT :limit";

private static final String COUNT_TRANSACTIONS = "SELECT count(:distinctClauseWhenSearchingByMetadataValue t.id) " +
"FROM transaction t " +
Expand Down Expand Up @@ -283,9 +288,9 @@ public List<TransactionEntity> findTransactionsByParentIdAndGatewayAccountId(Str
public List<TransactionEntity> findTransactionByParentId(String parentExternalId) {
return jdbi.withHandle(handle ->
handle.createQuery(FIND_TRANSACTIONS_BY_PARENT_EXT_ID)
.bind("parentExternalId", parentExternalId)
.map(new TransactionMapper())
.stream().collect(Collectors.toList())
.bind("parentExternalId", parentExternalId)
.map(new TransactionMapper())
.stream().collect(Collectors.toList())
);
}

Expand Down Expand Up @@ -357,6 +362,21 @@ public List<TransactionEntity> cursorTransactionSearch(TransactionSearchParams s
});
}

public List<TransactionEntity> findTransactionsForRedaction(ZonedDateTime dateOfLastProcessedTransaction,
ZonedDateTime redactTransactionsUpToDate,
int noOfTransactionsToReturn) {
return jdbi.withHandle(handle -> {
Query query = handle.createQuery(SEARCH_TRANSACTIONS_FOR_REDACTION);
query.bind("dateOfLastProcessedTransaction", dateOfLastProcessedTransaction);
query.bind("redactTransactionsUpToDate", redactTransactionsUpToDate);
query.bind("limit", noOfTransactionsToReturn);

return query
.map(new TransactionMapper())
.list();
});
}

private String createSearchTemplate(TransactionSearchParams searchParams, String baseQueryString) {
String searchClauseTemplate = String.join(" AND ", searchParams.getFilterTemplates());
searchClauseTemplate = StringUtils.isNotBlank(searchClauseTemplate) ?
Expand All @@ -365,7 +385,7 @@ private String createSearchTemplate(TransactionSearchParams searchParams, String
baseQueryString = baseQueryString
.replace(":payoutJoinOnGatewayIdField",
(searchParams.getAccountIds() != null && !searchParams.getAccountIds().isEmpty())
? SEARCH_CLAUSE_TRANSACTIONS_WITH_PAYOUT : "");
? SEARCH_CLAUSE_TRANSACTIONS_WITH_PAYOUT : "");

baseQueryString = baseQueryString
.replace(":transactionMetadataJoin",
Expand Down Expand Up @@ -401,4 +421,4 @@ public List<String> getSourceTypeValues() {
.mapTo(String.class)
.collect(Collectors.toList()));
}
}
}
72 changes: 65 additions & 7 deletions src/test/java/uk/gov/pay/ledger/transaction/dao/TransactionDaoIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,29 +1,35 @@
package uk.gov.pay.ledger.transaction.dao;

import com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.RegisterExtension;
import uk.gov.pay.ledger.app.LedgerConfig;
import uk.gov.service.payments.commons.model.Source;
import uk.gov.pay.ledger.extension.AppWithPostgresAndSqsExtension;
import uk.gov.pay.ledger.transaction.entity.TransactionEntity;
import uk.gov.pay.ledger.transaction.model.TransactionType;
import uk.gov.pay.ledger.transaction.state.TransactionState;
import uk.gov.pay.ledger.util.fixture.TransactionFixture;
import uk.gov.service.payments.commons.model.Source;

import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you need this? I can't see any uses of it in the tests? I was wondering if bringing it in causes some other behaviour?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good spot! it is not not required. I have used it in the test but removed it later. Will remove it in the next PR.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't cause any behavioural changes.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have to remember I'm not in ruby land anymore, I'm so used to things in ruby being require'd and then changing the behaviour of everything without ever being referenced directly!

import java.util.Arrays;
import java.util.List;
import java.util.stream.Collectors;

import static java.time.ZonedDateTime.parse;
import static org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric;
import static org.hamcrest.CoreMatchers.containsString;
import static org.hamcrest.CoreMatchers.hasItems;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.notNullValue;
import static org.hamcrest.CoreMatchers.nullValue;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.hasProperty;
import static org.hamcrest.Matchers.not;
import static org.mockito.Mockito.mock;
import static uk.gov.pay.ledger.transaction.service.TransactionService.REDACTED_REFERENCE_NUMBER;
import static uk.gov.pay.ledger.util.fixture.PayoutFixture.PayoutFixtureBuilder.aPayoutFixture;
Expand Down Expand Up @@ -84,7 +90,7 @@ void shouldInsertTransactionWithSourceCardApi() {

@Test
void shouldRetrieveTransactionByExternalIdAndGatewayAccount() {
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");
String payOutId = randomAlphanumeric(20);

TransactionFixture fixture = aTransactionFixture()
Expand Down Expand Up @@ -113,7 +119,7 @@ void shouldRetrieveTransactionByExternalIdAndGatewayAccount() {

@Test
void shouldRetrieveTransactionWithPayoutDateByExternalIdAndNoGatewayAccount() {
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");
String payOutId = randomAlphanumeric(20);

TransactionFixture fixture = aTransactionFixture()
Expand Down Expand Up @@ -169,7 +175,7 @@ private void assertTransactionEntity(TransactionEntity transaction, TransactionF
@Test
void shouldRetrieveTransactionByExternalIdAndGatewayAccountId() {
String payOutId = randomAlphanumeric(20);
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");

TransactionFixture fixture = aTransactionFixture()
.withDefaultCardDetails()
Expand Down Expand Up @@ -284,7 +290,7 @@ void shouldNotOverwriteTransactionIfItConsistsOfFewerEvents() {
@Test
void shouldFilterTransactionByExternalIdOrParentExternalIdAndGatewayAccountId() {
String payOutId = randomAlphanumeric(20);
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");

TransactionEntity transaction1 = aTransactionFixture()
.withState(TransactionState.CREATED)
Expand Down Expand Up @@ -406,7 +412,7 @@ void findTransactionByParentIdAndGatewayAccountId_shouldFilterByParentExternalId
@Test
void findTransactionByParentId_shouldFilterByParentExternalId() {
String payOutId = randomAlphanumeric(20);
ZonedDateTime paidOutDate = ZonedDateTime.parse("2019-12-12T10:00:00Z");
ZonedDateTime paidOutDate = parse("2019-12-12T10:00:00Z");

TransactionEntity transaction1 = aTransactionFixture()
.withState(TransactionState.CREATED)
Expand Down Expand Up @@ -447,4 +453,56 @@ void sourceTypeInDatabase_shouldMatchValuesInEnum() {
transactionDao.getSourceTypeValues().forEach(x -> assertThat(sourceArray.contains(x), is(true)));
sourceArray.forEach(x -> assertThat(transactionDao.getSourceTypeValues().contains(x), is(true)));
}
}

@Nested
@DisplayName("TestFindTransactionsForRedaction")
class TestFindTransactionsForRedaction {

@Test
void shouldReturnTransactionsCorrectlyForDateRangesAndNumberOfTransactions() {
TransactionEntity transactionToReturnToDelete1 = aTransactionFixture()
.withCreatedDate(parse("2016-01-01T00:00:00Z"))
.insert(rule.getJdbi())
.toEntity();
TransactionEntity transactionToReturnToDelete2 = aTransactionFixture()
.withCreatedDate(parse("2016-01-01T01:00:00Z"))
.insert(rule.getJdbi())
.toEntity();
TransactionEntity transactionEligibleForSearchButNotReturnedDueToLimit = aTransactionFixture()
.withCreatedDate(parse("2016-01-01T02:00:00Z"))
.insert(rule.getJdbi())
.toEntity();

TransactionEntity transactionToExclude1 = aTransactionFixture()
.withCreatedDate(parse("2016-01-02T00:00:00Z"))
.insert(rule.getJdbi())
.toEntity();
TransactionEntity transactionToExclude2 = aTransactionFixture()
.withCreatedDate(parse("2016-01-03T00:00:00Z"))
.insert(rule.getJdbi())
.toEntity();

List<TransactionEntity> transactionsForRedaction = transactionDao.findTransactionsForRedaction(
parse("2015-12-31T00:00:00Z"),
parse("2016-01-02T00:00:00Z"),
2
);

List<String> transactionIdsReturned = transactionsForRedaction
.stream()
.map(TransactionEntity::getExternalId)
.collect(Collectors.toList());

assertThat(transactionsForRedaction.size(), is(2));
assertThat(transactionIdsReturned, hasItems(
transactionToReturnToDelete1.getExternalId(),
transactionToReturnToDelete2.getExternalId())
);
assertThat(transactionIdsReturned, not(hasItems(
transactionEligibleForSearchButNotReturnedDueToLimit.getExternalId(),
transactionToExclude1.getExternalId(),
transactionToExclude2.getExternalId())
));
}
}
}