PP-11205 Add service method to expunge data

README.md

@@ -50,7 +50,8 @@ The [API Specification](/openapi/adminusers_spec.yaml) provides more detail on t
 | `RUN_MIGRATION`                                                               | Set to `true` to run a database migration. Defaults to `false`. |
 | `SELFSERVICE_URL`                                                             | The URL to the admin portal. Defaults to `https://selfservice.pymnt.localdomain`. |
 | `SUPPORT_URL`                                                                 | The URL users can visit to get support. Defaults to `https://frontend.pymnt.localdomain/contact/`. |
-
+| `EXPUNGE_AND_ARCHIVE_HISTORICAL_DATA_ENABLED`                                 | Set to `true` to enable expunging (user related data) and archiving (services) historical data. Default is `false`. |
+| `EXPUNGE_USER_DATA_AFTER_DAYS`                                                | Number of days after which users (not attached to any service) not logged in or created but never logged in deleted. Default is 4 years. |
 -----------------------------------------------------------------------------------------------------------
 
 ## Maven profiles

src/main/java/uk/gov/pay/adminusers/app/config/AdminUsersConfig.java

@@ -48,6 +48,9 @@ public class AdminUsersConfig extends Configuration {
     @NotNull
     private EventSubscriberQueueConfig eventSubscriberQueueConfig;
 
+    @NotNull
+    private ExpungeAndArchiveDataConfig expungeAndArchiveDataConfig;
+
     @NotNull
     @JsonProperty("ledgerBaseURL")
     private String ledgerBaseUrl;
@@ -118,6 +121,10 @@ public RestClientConfig getRestClientConfig() {
         return restClientConfig;
     }
 
+    public ExpungeAndArchiveDataConfig getExpungeAndArchiveDataConfig() {
+        return expungeAndArchiveDataConfig;
+    }
+
     public Optional<URI> getEcsContainerMetadataUriV4() {
         return Optional.ofNullable(ecsContainerMetadataUriV4);
     }

src/main/java/uk/gov/pay/adminusers/app/config/ExpungeAndArchiveDataConfig.java

@@ -0,0 +1,20 @@
+package uk.gov.pay.adminusers.app.config;
+
+import javax.validation.constraints.NotNull;
+
+public class ExpungeAndArchiveDataConfig {
+
+    @NotNull
+    private boolean expungeAndArchiveHistoricalDataEnabled;
+
+    @NotNull
+    private int expungeUserDataAfterDays;
+
+    public boolean isExpungeAndArchiveHistoricalDataEnabled() {
+        return expungeAndArchiveHistoricalDataEnabled;
+    }
+
+    public int getExpungeUserDataAfterDays() {
+        return expungeUserDataAfterDays;
+    }
+}

src/main/java/uk/gov/pay/adminusers/expungeandarchive/service/ExpungeAndArchiveHistoricalDataService.java

@@ -0,0 +1,76 @@
+package uk.gov.pay.adminusers.expungeandarchive.service;
+
+import com.google.inject.Inject;
+import io.prometheus.client.Histogram;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import uk.gov.pay.adminusers.app.config.AdminUsersConfig;
+import uk.gov.pay.adminusers.app.config.ExpungeAndArchiveDataConfig;
+import uk.gov.pay.adminusers.persistence.dao.ForgottenPasswordDao;
+import uk.gov.pay.adminusers.persistence.dao.InviteDao;
+import uk.gov.pay.adminusers.persistence.dao.UserDao;
+
+import java.time.Clock;
+import java.time.ZonedDateTime;
+
+import static java.time.ZoneOffset.UTC;
+import static java.time.temporal.ChronoUnit.DAYS;
+import static net.logstash.logback.argument.StructuredArguments.kv;
+
+public class ExpungeAndArchiveHistoricalDataService {
+
+    private static final Logger LOGGER = LoggerFactory.getLogger(ExpungeAndArchiveHistoricalDataService.class);
+    private final UserDao userDao;
+    private final InviteDao inviteDao;
+    private final ForgottenPasswordDao forgottenPasswordDao;
+    private final ExpungeAndArchiveDataConfig expungeAndArchiveDataConfig;
+    private final Clock clock;
+
+    private static final Histogram duration = Histogram.build()
+            .name("expunge_and_archive_historical_data_job_duration_seconds")
+            .help("Duration of expunge and archive historical data job in seconds")
+            .unit("seconds")
+            .register();
+
+    @Inject
+    public ExpungeAndArchiveHistoricalDataService(UserDao userDao, InviteDao inviteDao,
+                                                  ForgottenPasswordDao forgottenPasswordDao,
+                                                  AdminUsersConfig adminUsersConfig,
+                                                  Clock clock) {
+        this.userDao = userDao;
+        this.inviteDao = inviteDao;
+        this.forgottenPasswordDao = forgottenPasswordDao;
+        expungeAndArchiveDataConfig = adminUsersConfig.getExpungeAndArchiveDataConfig();
+        this.clock = clock;
+    }
+
+    public void expungeAndArchiveHistoricalData() {
+        Histogram.Timer responseTimeTimer = duration.startTimer();
+
+        try {
+            if (expungeAndArchiveDataConfig.isExpungeAndArchiveHistoricalDataEnabled()) {
+                ZonedDateTime deleteUsersAndRelatedDataBeforeDate = getDeleteUsersAndRelatedDataBeforeDate();
+
+                int noOfUsersDeleted = userDao.deleteUsersNotAssociatedWithAnyService(deleteUsersAndRelatedDataBeforeDate.toInstant());
+                int noOfInvitesDeleted = inviteDao.deleteInvites(deleteUsersAndRelatedDataBeforeDate);
+                int noOfForgottenPasswordsDeleted = forgottenPasswordDao.deleteForgottenPasswords(deleteUsersAndRelatedDataBeforeDate);
+
+                LOGGER.info("Completed expunging and archiving historical data",
+                        kv("no_of_users_deleted", noOfUsersDeleted),
+                        kv("no_of_forgotten_passwords_deleted", noOfForgottenPasswordsDeleted),
+                        kv("no_of_invites_deleted", noOfInvitesDeleted));
+            } else {
+                LOGGER.info("Expunging and archiving historical data is not enabled");
+            }
+        } finally {
+            responseTimeTimer.observeDuration();
+        }
+    }
+
+    private ZonedDateTime getDeleteUsersAndRelatedDataBeforeDate() {
+        ZonedDateTime expungeAndArchiveDateBeforeDate = clock.instant()
+                .minus(expungeAndArchiveDataConfig.getExpungeUserDataAfterDays(), DAYS)
+                .atZone(UTC);
+        return expungeAndArchiveDateBeforeDate;
+    }
+}

src/main/resources/config/config.yaml

@@ -138,3 +138,7 @@ restClientConfig:
   disabledSecureConnection: ${DISABLE_INTERNAL_HTTPS:-false}
 
 ecsContainerMetadataUriV4: ${ECS_CONTAINER_METADATA_URI_V4:-}
+
+expungeAndArchiveDataConfig:
+  expungeAndArchiveHistoricalDataEnabled: ${EXPUNGE_AND_ARCHIVE_HISTORICAL_DATA_ENABLED:-false}
+  expungeUserDataAfterDays: ${EXPUNGE_USER_DATA_AFTER_DAYS:-1460}

src/test/java/uk/gov/pay/adminusers/expungeandarchive/service/ExpungeAndArchiveHistoricalDataServiceTest.java

@@ -0,0 +1,121 @@
+package uk.gov.pay.adminusers.expungeandarchive.service;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.LoggingEvent;
+import ch.qos.logback.core.Appender;
+import io.prometheus.client.CollectorRegistry;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.slf4j.LoggerFactory;
+import uk.gov.pay.adminusers.app.config.AdminUsersConfig;
+import uk.gov.pay.adminusers.app.config.ExpungeAndArchiveDataConfig;
+import uk.gov.pay.adminusers.persistence.dao.ForgottenPasswordDao;
+import uk.gov.pay.adminusers.persistence.dao.InviteDao;
+import uk.gov.pay.adminusers.persistence.dao.UserDao;
+
+import java.time.Clock;
+import java.time.Instant;
+import java.util.List;
+import java.util.Optional;
+
+import static ch.qos.logback.classic.Level.INFO;
+import static java.time.ZoneOffset.UTC;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.greaterThan;
+import static org.hamcrest.Matchers.is;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyNoInteractions;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class ExpungeAndArchiveHistoricalDataServiceTest {
+
+    @Mock
+    UserDao mockUserDao;
+
+    @Mock
+    InviteDao mockInviteDao;
+
+    @Mock
+    ForgottenPasswordDao mockForgottenPasswordDao;
+
+    ExpungeAndArchiveHistoricalDataService expungeAndArchiveHistoricalDataService;
+
+    @Mock
+    AdminUsersConfig mockAdminUsersConfig;
+
+    @Mock
+    ExpungeAndArchiveDataConfig mockExpungeAndArchiveConfig;
+
+    @Mock
+    private Appender<ILoggingEvent> mockAppender;
+
+    @Captor
+    private ArgumentCaptor<LoggingEvent> loggingEventArgumentCaptor;
+
+    private final CollectorRegistry collectorRegistry = CollectorRegistry.defaultRegistry;
+
+    String SYSTEM_INSTANT = "2022-03-03T10:15:30Z";
+    Clock clock;
+
+    @BeforeEach
+    void setUp() {
+        clock = Clock.fixed(Instant.parse(SYSTEM_INSTANT), UTC);
+        when(mockAdminUsersConfig.getExpungeAndArchiveDataConfig()).thenReturn(mockExpungeAndArchiveConfig);
+        expungeAndArchiveHistoricalDataService = new ExpungeAndArchiveHistoricalDataService(mockUserDao,
+                mockInviteDao, mockForgottenPasswordDao, mockAdminUsersConfig, clock);
+    }
+
+    @Test
+    void shouldNotDeleteHistoricalDataIfFlagIsNotEnabled() {
+        Logger root = (Logger) LoggerFactory.getLogger(ExpungeAndArchiveHistoricalDataService.class);
+        root.addAppender(mockAppender);
+        root.setLevel(INFO);
+
+        when(mockExpungeAndArchiveConfig.isExpungeAndArchiveHistoricalDataEnabled()).thenReturn(false);
+        expungeAndArchiveHistoricalDataService.expungeAndArchiveHistoricalData();
+
+        verifyNoInteractions(mockUserDao);
+        verifyNoInteractions(mockInviteDao);
+        verifyNoInteractions(mockForgottenPasswordDao);
+
+        verify(mockAppender).doAppend(loggingEventArgumentCaptor.capture());
+        List<LoggingEvent> loggingEvents = loggingEventArgumentCaptor.getAllValues();
+        assertThat(loggingEvents.get(0).getFormattedMessage(), is("Expunging and archiving historical data is not enabled"));
+    }
+
+    @Test
+    void shouldDeleteHistoricalDataIfFlagIsEnabled() {
+        Logger root = (Logger) LoggerFactory.getLogger(ExpungeAndArchiveHistoricalDataService.class);
+        root.addAppender(mockAppender);
+        root.setLevel(INFO);
+
+        when(mockExpungeAndArchiveConfig.isExpungeAndArchiveHistoricalDataEnabled()).thenReturn(true);
+
+        expungeAndArchiveHistoricalDataService.expungeAndArchiveHistoricalData();
+
+        verify(mockUserDao).deleteUsersNotAssociatedWithAnyService(clock.instant());
+        verify(mockInviteDao).deleteInvites(clock.instant().atZone(UTC));
+        verify(mockForgottenPasswordDao).deleteForgottenPasswords(clock.instant().atZone(UTC));
+
+        verify(mockAppender).doAppend(loggingEventArgumentCaptor.capture());
+        List<LoggingEvent> loggingEvents = loggingEventArgumentCaptor.getAllValues();
+        assertThat(loggingEvents.get(0).getFormattedMessage(), is("Completed expunging and archiving historical data"));
+    }
+
+    @Test
+    void shouldObserveJobDurationForMetrics() {
+        Double initialDuration = Optional.ofNullable(collectorRegistry.getSampleValue("expunge_and_archive_historical_data_job_duration_seconds_sum")).orElse(0.0);
+
+        expungeAndArchiveHistoricalDataService.expungeAndArchiveHistoricalData();
+
+        Double duration = collectorRegistry.getSampleValue("expunge_and_archive_historical_data_job_duration_seconds_sum");
+        assertThat(duration, greaterThan(initialDuration));
+    }
+}

src/test/resources/config/test-it-config.yaml

@@ -130,3 +130,7 @@ restClientConfig:
   disabledSecureConnection: ${DISABLE_INTERNAL_HTTPS:-false}
 
 ecsContainerMetadataUriV4: ${ECS_CONTAINER_METADATA_URI_V4:-}
+
+expungeAndArchiveDataConfig:
+  expungeAndArchiveHistoricalDataEnabled: ${EXPUNGE_AND_ARCHIVE_HISTORICAL_DATA_ENABLED:-false}
+  expungeUserDataAfterDays: ${EXPUNGE_USER_DATA_AFTER_DAYS:-1460}