diff --git a/README.md b/README.md index 67d92937a..1eac545e1 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,8 @@ The [API Specification](/openapi/adminusers_spec.yaml) provides more detail on t | `RUN_MIGRATION` | Set to `true` to run a database migration. Defaults to `false`. | | `SELFSERVICE_URL` | The URL to the admin portal. Defaults to `https://selfservice.pymnt.localdomain`. | | `SUPPORT_URL` | The URL users can visit to get support. Defaults to `https://frontend.pymnt.localdomain/contact/`. | - +| `EXPUNGE_AND_ARCHIVE_HISTORICAL_DATA_ENABLED` | Set to `true` to enable expunging (user related data) and archiving (services) historical data. Default is `false`. | +| `EXPUNGE_USER_DATA_AFTER_DAYS` | Number of days after which users (not attached to any service) not logged in or created but never logged in deleted. Default is 4 years. | ----------------------------------------------------------------------------------------------------------- ## Maven profiles diff --git a/src/main/java/uk/gov/pay/adminusers/app/config/AdminUsersConfig.java b/src/main/java/uk/gov/pay/adminusers/app/config/AdminUsersConfig.java index b0c56ec3d..4d041180d 100644 --- a/src/main/java/uk/gov/pay/adminusers/app/config/AdminUsersConfig.java +++ b/src/main/java/uk/gov/pay/adminusers/app/config/AdminUsersConfig.java @@ -48,6 +48,9 @@ public class AdminUsersConfig extends Configuration { @NotNull private EventSubscriberQueueConfig eventSubscriberQueueConfig; + @NotNull + private ExpungeAndArchiveDataConfig expungeAndArchiveDataConfig; + @NotNull @JsonProperty("ledgerBaseURL") private String ledgerBaseUrl; @@ -118,6 +121,10 @@ public RestClientConfig getRestClientConfig() { return restClientConfig; } + public ExpungeAndArchiveDataConfig getExpungeAndArchiveDataConfig() { + return expungeAndArchiveDataConfig; + } + public Optional getEcsContainerMetadataUriV4() { return Optional.ofNullable(ecsContainerMetadataUriV4); } diff --git a/src/main/java/uk/gov/pay/adminusers/app/config/ExpungeAndArchiveDataConfig.java b/src/main/java/uk/gov/pay/adminusers/app/config/ExpungeAndArchiveDataConfig.java new file mode 100644 index 000000000..87d67777e --- /dev/null +++ b/src/main/java/uk/gov/pay/adminusers/app/config/ExpungeAndArchiveDataConfig.java @@ -0,0 +1,20 @@ +package uk.gov.pay.adminusers.app.config; + +import javax.validation.constraints.NotNull; + +public class ExpungeAndArchiveDataConfig { + + @NotNull + private boolean expungeAndArchiveHistoricalDataEnabled; + + @NotNull + private int expungeUserDataAfterDays; + + public boolean isExpungeAndArchiveHistoricalDataEnabled() { + return expungeAndArchiveHistoricalDataEnabled; + } + + public int getExpungeUserDataAfterDays() { + return expungeUserDataAfterDays; + } +} diff --git a/src/main/java/uk/gov/pay/adminusers/expungeandarchive/service/ExpungeAndArchiveHistoricalDataService.java b/src/main/java/uk/gov/pay/adminusers/expungeandarchive/service/ExpungeAndArchiveHistoricalDataService.java new file mode 100644 index 000000000..cf082b5e7 --- /dev/null +++ b/src/main/java/uk/gov/pay/adminusers/expungeandarchive/service/ExpungeAndArchiveHistoricalDataService.java @@ -0,0 +1,76 @@ +package uk.gov.pay.adminusers.expungeandarchive.service; + +import com.google.inject.Inject; +import io.prometheus.client.Histogram; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import uk.gov.pay.adminusers.app.config.AdminUsersConfig; +import uk.gov.pay.adminusers.app.config.ExpungeAndArchiveDataConfig; +import uk.gov.pay.adminusers.persistence.dao.ForgottenPasswordDao; +import uk.gov.pay.adminusers.persistence.dao.InviteDao; +import uk.gov.pay.adminusers.persistence.dao.UserDao; + +import java.time.Clock; +import java.time.ZonedDateTime; + +import static java.time.ZoneOffset.UTC; +import static java.time.temporal.ChronoUnit.DAYS; +import static net.logstash.logback.argument.StructuredArguments.kv; + +public class ExpungeAndArchiveHistoricalDataService { + + private static final Logger LOGGER = LoggerFactory.getLogger(ExpungeAndArchiveHistoricalDataService.class); + private final UserDao userDao; + private final InviteDao inviteDao; + private final ForgottenPasswordDao forgottenPasswordDao; + private final ExpungeAndArchiveDataConfig expungeAndArchiveDataConfig; + private final Clock clock; + + private static final Histogram duration = Histogram.build() + .name("expunge_and_archive_historical_data_job_duration_seconds") + .help("Duration of expunge and archive historical data job in seconds") + .unit("seconds") + .register(); + + @Inject + public ExpungeAndArchiveHistoricalDataService(UserDao userDao, InviteDao inviteDao, + ForgottenPasswordDao forgottenPasswordDao, + AdminUsersConfig adminUsersConfig, + Clock clock) { + this.userDao = userDao; + this.inviteDao = inviteDao; + this.forgottenPasswordDao = forgottenPasswordDao; + expungeAndArchiveDataConfig = adminUsersConfig.getExpungeAndArchiveDataConfig(); + this.clock = clock; + } + + public void expungeAndArchiveHistoricalData() { + Histogram.Timer responseTimeTimer = duration.startTimer(); + + try { + if (expungeAndArchiveDataConfig.isExpungeAndArchiveHistoricalDataEnabled()) { + ZonedDateTime deleteUsersAndRelatedDataBeforeDate = getDeleteUsersAndRelatedDataBeforeDate(); + + int noOfUsersDeleted = userDao.deleteUsersNotAssociatedWithAnyService(deleteUsersAndRelatedDataBeforeDate.toInstant()); + int noOfInvitesDeleted = inviteDao.deleteInvites(deleteUsersAndRelatedDataBeforeDate); + int noOfForgottenPasswordsDeleted = forgottenPasswordDao.deleteForgottenPasswords(deleteUsersAndRelatedDataBeforeDate); + + LOGGER.info("Completed expunging and archiving historical data", + kv("no_of_users_deleted", noOfUsersDeleted), + kv("no_of_forgotten_passwords_deleted", noOfForgottenPasswordsDeleted), + kv("no_of_invites_deleted", noOfInvitesDeleted)); + } else { + LOGGER.info("Expunging and archiving historical data is not enabled"); + } + } finally { + responseTimeTimer.observeDuration(); + } + } + + private ZonedDateTime getDeleteUsersAndRelatedDataBeforeDate() { + ZonedDateTime expungeAndArchiveDateBeforeDate = clock.instant() + .minus(expungeAndArchiveDataConfig.getExpungeUserDataAfterDays(), DAYS) + .atZone(UTC); + return expungeAndArchiveDateBeforeDate; + } +} diff --git a/src/main/resources/config/config.yaml b/src/main/resources/config/config.yaml index 4333838b7..b66d9fe96 100644 --- a/src/main/resources/config/config.yaml +++ b/src/main/resources/config/config.yaml @@ -138,3 +138,7 @@ restClientConfig: disabledSecureConnection: ${DISABLE_INTERNAL_HTTPS:-false} ecsContainerMetadataUriV4: ${ECS_CONTAINER_METADATA_URI_V4:-} + +expungeAndArchiveDataConfig: + expungeAndArchiveHistoricalDataEnabled: ${EXPUNGE_AND_ARCHIVE_HISTORICAL_DATA_ENABLED:-false} + expungeUserDataAfterDays: ${EXPUNGE_USER_DATA_AFTER_DAYS:-1460} diff --git a/src/test/java/uk/gov/pay/adminusers/expungeandarchive/service/ExpungeAndArchiveHistoricalDataServiceTest.java b/src/test/java/uk/gov/pay/adminusers/expungeandarchive/service/ExpungeAndArchiveHistoricalDataServiceTest.java new file mode 100644 index 000000000..5013d0283 --- /dev/null +++ b/src/test/java/uk/gov/pay/adminusers/expungeandarchive/service/ExpungeAndArchiveHistoricalDataServiceTest.java @@ -0,0 +1,121 @@ +package uk.gov.pay.adminusers.expungeandarchive.service; + +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.LoggingEvent; +import ch.qos.logback.core.Appender; +import io.prometheus.client.CollectorRegistry; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.ArgumentCaptor; +import org.mockito.Captor; +import org.mockito.Mock; +import org.mockito.junit.jupiter.MockitoExtension; +import org.slf4j.LoggerFactory; +import uk.gov.pay.adminusers.app.config.AdminUsersConfig; +import uk.gov.pay.adminusers.app.config.ExpungeAndArchiveDataConfig; +import uk.gov.pay.adminusers.persistence.dao.ForgottenPasswordDao; +import uk.gov.pay.adminusers.persistence.dao.InviteDao; +import uk.gov.pay.adminusers.persistence.dao.UserDao; + +import java.time.Clock; +import java.time.Instant; +import java.util.List; +import java.util.Optional; + +import static ch.qos.logback.classic.Level.INFO; +import static java.time.ZoneOffset.UTC; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +@ExtendWith(MockitoExtension.class) +class ExpungeAndArchiveHistoricalDataServiceTest { + + @Mock + UserDao mockUserDao; + + @Mock + InviteDao mockInviteDao; + + @Mock + ForgottenPasswordDao mockForgottenPasswordDao; + + ExpungeAndArchiveHistoricalDataService expungeAndArchiveHistoricalDataService; + + @Mock + AdminUsersConfig mockAdminUsersConfig; + + @Mock + ExpungeAndArchiveDataConfig mockExpungeAndArchiveConfig; + + @Mock + private Appender mockAppender; + + @Captor + private ArgumentCaptor loggingEventArgumentCaptor; + + private final CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry; + + String SYSTEM_INSTANT = "2022-03-03T10:15:30Z"; + Clock clock; + + @BeforeEach + void setUp() { + clock = Clock.fixed(Instant.parse(SYSTEM_INSTANT), UTC); + when(mockAdminUsersConfig.getExpungeAndArchiveDataConfig()).thenReturn(mockExpungeAndArchiveConfig); + expungeAndArchiveHistoricalDataService = new ExpungeAndArchiveHistoricalDataService(mockUserDao, + mockInviteDao, mockForgottenPasswordDao, mockAdminUsersConfig, clock); + } + + @Test + void shouldNotDeleteHistoricalDataIfFlagIsNotEnabled() { + Logger root = (Logger) LoggerFactory.getLogger(ExpungeAndArchiveHistoricalDataService.class); + root.addAppender(mockAppender); + root.setLevel(INFO); + + when(mockExpungeAndArchiveConfig.isExpungeAndArchiveHistoricalDataEnabled()).thenReturn(false); + expungeAndArchiveHistoricalDataService.expungeAndArchiveHistoricalData(); + + verifyNoInteractions(mockUserDao); + verifyNoInteractions(mockInviteDao); + verifyNoInteractions(mockForgottenPasswordDao); + + verify(mockAppender).doAppend(loggingEventArgumentCaptor.capture()); + List loggingEvents = loggingEventArgumentCaptor.getAllValues(); + assertThat(loggingEvents.get(0).getFormattedMessage(), is("Expunging and archiving historical data is not enabled")); + } + + @Test + void shouldDeleteHistoricalDataIfFlagIsEnabled() { + Logger root = (Logger) LoggerFactory.getLogger(ExpungeAndArchiveHistoricalDataService.class); + root.addAppender(mockAppender); + root.setLevel(INFO); + + when(mockExpungeAndArchiveConfig.isExpungeAndArchiveHistoricalDataEnabled()).thenReturn(true); + + expungeAndArchiveHistoricalDataService.expungeAndArchiveHistoricalData(); + + verify(mockUserDao).deleteUsersNotAssociatedWithAnyService(clock.instant()); + verify(mockInviteDao).deleteInvites(clock.instant().atZone(UTC)); + verify(mockForgottenPasswordDao).deleteForgottenPasswords(clock.instant().atZone(UTC)); + + verify(mockAppender).doAppend(loggingEventArgumentCaptor.capture()); + List loggingEvents = loggingEventArgumentCaptor.getAllValues(); + assertThat(loggingEvents.get(0).getFormattedMessage(), is("Completed expunging and archiving historical data")); + } + + @Test + void shouldObserveJobDurationForMetrics() { + Double initialDuration = Optional.ofNullable(collectorRegistry.getSampleValue("expunge_and_archive_historical_data_job_duration_seconds_sum")).orElse(0.0); + + expungeAndArchiveHistoricalDataService.expungeAndArchiveHistoricalData(); + + Double duration = collectorRegistry.getSampleValue("expunge_and_archive_historical_data_job_duration_seconds_sum"); + assertThat(duration, greaterThan(initialDuration)); + } +} diff --git a/src/test/resources/config/test-it-config.yaml b/src/test/resources/config/test-it-config.yaml index 5639c61fa..3988f32a9 100644 --- a/src/test/resources/config/test-it-config.yaml +++ b/src/test/resources/config/test-it-config.yaml @@ -130,3 +130,7 @@ restClientConfig: disabledSecureConnection: ${DISABLE_INTERNAL_HTTPS:-false} ecsContainerMetadataUriV4: ${ECS_CONTAINER_METADATA_URI_V4:-} + +expungeAndArchiveDataConfig: + expungeAndArchiveHistoricalDataEnabled: ${EXPUNGE_AND_ARCHIVE_HISTORICAL_DATA_ENABLED:-false} + expungeUserDataAfterDays: ${EXPUNGE_USER_DATA_AFTER_DAYS:-1460}