From ac5209105bbdf882fcfdb6bee039fd2777203d74 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 31 Jan 2022 09:48:32 +0000 Subject: [PATCH 01/18] Revert "Remove wiring stubs from this PR" This reverts commit d94823a075509ac9cd9c11c149c40ed426a93541. --- .../timelock/paxos/TimeLockAgent.java | 2 + .../main/conjure/timelock-management-api.yml | 18 ++++++++- .../TimeLockManagementResource.java | 38 +++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java index b96bd825cbd..ebca0c9b51b 100644 --- a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java +++ b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java @@ -177,6 +177,8 @@ public static TimeLockAgent create( metricsManager, Suppliers.compose(TimeLockRuntimeConfiguration::paxos, restrictedRuntime::get)); + // TODO(gs): set up AllNodesUpdater + TimeLockAgent agent = new TimeLockAgent( metricsManager, install, diff --git a/timelock-api/src/main/conjure/timelock-management-api.yml b/timelock-api/src/main/conjure/timelock-management-api.yml index c2ab0ad642d..9f1ac6e0c54 100644 --- a/timelock-api/src/main/conjure/timelock-management-api.yml +++ b/timelock-api/src/main/conjure/timelock-management-api.yml @@ -101,7 +101,23 @@ services: operation is atomic for each namespace (e.g. users will not see two different lock services servicing the same startTransactions request), but not atomic as a whole. Additionally, if this method throws, it is nondeterministic which, if any, namespaces have been invalidated; some may even be invalidated only on a - subset of nodes. This state can be cleared by re-enabling all namespaces. + subset od nodes. This state can be cleared by re-enabling all namespaces. + + disableTimelock: + http: POST /disable + args: + request: set + returns: DisableNamespacesResponse + docs: | + Disables the TimeLock server in a persistant way for the specified namespaces. This includes invalidating + all currently held locks, all lock watch state, as well as refusing to serve new requests. + reenableTimelock: + http: POST /reenable + args: + request: ReenableNamespacesRequest + returns: ReenableNamespacesResponse + docs: | + Allows the TimeLock server to once again serve requests for these namespaces. getServerLifecycleId: http: POST /getServerLifecycleId diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java index ce651f337c9..91a14c0c10e 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java @@ -24,6 +24,11 @@ import com.palantir.atlasdb.keyvalue.api.TimestampSeries; import com.palantir.atlasdb.timelock.ConjureResourceExceptionHandler; import com.palantir.atlasdb.timelock.TimelockNamespaces; +import com.palantir.atlasdb.timelock.api.DisableNamespacesResponse; +import com.palantir.atlasdb.timelock.api.Namespace; +import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest; +import com.palantir.atlasdb.timelock.api.ReenableNamespacesResponse; +import com.palantir.atlasdb.timelock.api.UnsuccessfulDisableNamespacesResponse; import com.palantir.atlasdb.timelock.api.management.TimeLockManagementService; import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceEndpoints; import com.palantir.atlasdb.timelock.api.management.UndertowTimeLockManagementService; @@ -116,6 +121,29 @@ public ListenableFuture invalidateResources(AuthHeader authHeader, Set disableTimelock( + AuthHeader authHeader, Set namespaces) { + return handleExceptions(() -> disableInternal(namespaces)); + } + + private ListenableFuture disableInternal(Set namespaces) { + // todo(gs): wire up ANDNU + return Futures.immediateFuture( + DisableNamespacesResponse.unsuccessful(UnsuccessfulDisableNamespacesResponse.of(ImmutableSet.of()))); + } + + @Override + public ListenableFuture reenableTimelock( + AuthHeader authHeader, ReenableNamespacesRequest request) { + return handleExceptions(() -> reenableInternal(request)); + } + + public ListenableFuture reenableInternal(ReenableNamespacesRequest request) { + // todo(gs): wire up ANDNU + return Futures.immediateFuture(ReenableNamespacesResponse.builder().build()); + } + @Override public ListenableFuture getServerLifecycleId(AuthHeader authHeader) { return Futures.immediateFuture(serviceLifecycleController.getServerId()); @@ -168,6 +196,16 @@ public void invalidateResources(AuthHeader authHeader, Set namespaces) { unwrap(resource.invalidateResources(authHeader, namespaces)); } + @Override + public DisableNamespacesResponse disableTimelock(AuthHeader authHeader, Set request) { + return unwrap(resource.disableTimelock(authHeader, request)); + } + + @Override + public ReenableNamespacesResponse reenableTimelock(AuthHeader authHeader, ReenableNamespacesRequest request) { + return unwrap(resource.reenableTimelock(authHeader, request)); + } + @Override public UUID getServerLifecycleId(AuthHeader authHeader) { return unwrap(resource.getServerLifecycleId(authHeader)); From 00865772b0f87499dba48d5bc9eec520a644d1df Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 31 Jan 2022 09:49:45 +0000 Subject: [PATCH 02/18] fix typo --- timelock-api/src/main/conjure/timelock-management-api.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/timelock-api/src/main/conjure/timelock-management-api.yml b/timelock-api/src/main/conjure/timelock-management-api.yml index 9f1ac6e0c54..6b8212fb5e2 100644 --- a/timelock-api/src/main/conjure/timelock-management-api.yml +++ b/timelock-api/src/main/conjure/timelock-management-api.yml @@ -101,7 +101,7 @@ services: operation is atomic for each namespace (e.g. users will not see two different lock services servicing the same startTransactions request), but not atomic as a whole. Additionally, if this method throws, it is nondeterministic which, if any, namespaces have been invalidated; some may even be invalidated only on a - subset od nodes. This state can be cleared by re-enabling all namespaces. + subset of nodes. This state can be cleared by re-enabling all namespaces. disableTimelock: http: POST /disable From 127b688b87dc6eacc4f5245aa2598cc5a03cb544 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 31 Jan 2022 10:05:14 +0000 Subject: [PATCH 03/18] wire up local updater --- .../main/java/com/palantir/timelock/paxos/TimeLockAgent.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java index ebca0c9b51b..d00775355c4 100644 --- a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java +++ b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java @@ -51,6 +51,7 @@ import com.palantir.atlasdb.timelock.lock.LockLog; import com.palantir.atlasdb.timelock.lock.v1.ConjureLockV1Resource; import com.palantir.atlasdb.timelock.management.DisabledNamespaces; +import com.palantir.atlasdb.timelock.management.DisabledNamespacesUpdaterResource; import com.palantir.atlasdb.timelock.management.PersistentNamespaceContexts; import com.palantir.atlasdb.timelock.management.ServiceLifecycleController; import com.palantir.atlasdb.timelock.management.TimeLockManagementResource; @@ -367,6 +368,9 @@ private void createAndRegisterResources() { presentUndertowRegistrar, AtlasRestoreResource.undertow( permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); + registerCorruptionHandlerWrappedService( + presentUndertowRegistrar, + DisabledNamespacesUpdaterResource.undertow(redirectRetryTargeter, namespaces)); } else { registrar.accept(ConjureTimelockResource.jersey(redirectRetryTargeter, asyncTimelockServiceGetter)); registrar.accept(ConjureLockWatchingResource.jersey(redirectRetryTargeter, asyncTimelockServiceGetter)); @@ -378,6 +382,7 @@ private void createAndRegisterResources() { permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); registrar.accept(AtlasRestoreResource.jersey( permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); + registrar.accept(DisabledNamespacesUpdaterResource.jersey(redirectRetryTargeter, namespaces)); } } From 034fc3ac53a754c509ad4b12977e892944ab3e81 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 31 Jan 2022 10:38:34 +0000 Subject: [PATCH 04/18] Wire ANDNU up to TimeLockManagementResource --- ...NodesDisabledNamespacesUpdaterFactory.java | 66 +++++++++++++++++++ .../timelock/paxos/PaxosRemoteClients.java | 7 ++ .../timelock/paxos/TimeLockAgent.java | 15 ++++- .../TimeLockManagementResource.java | 27 +++++--- .../management/DiskNamespaceLoaderTest.java | 1 + 5 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java diff --git a/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java b/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java new file mode 100644 index 00000000000..f8bb0096fa2 --- /dev/null +++ b/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java @@ -0,0 +1,66 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.timelock.management; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.palantir.atlasdb.timelock.TimelockNamespaces; +import com.palantir.atlasdb.timelock.api.DisabledNamespacesUpdaterService; +import com.palantir.atlasdb.timelock.paxos.ImmutablePaxosRemoteClients; +import com.palantir.atlasdb.timelock.paxos.PaxosRemoteClients; +import com.palantir.atlasdb.timelock.paxos.PaxosResourcesFactory.TimelockPaxosInstallationContext; +import com.palantir.atlasdb.timelock.paxos.WithDedicatedExecutor; +import com.palantir.atlasdb.util.MetricsManager; +import com.palantir.common.concurrent.CheckedRejectionExecutorService; +import com.palantir.common.streams.KeyedStream; +import com.palantir.tokens.auth.AuthHeader; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public final class AllNodesDisabledNamespacesUpdaterFactory { + private final AuthHeader authHeader; + private final TimelockPaxosInstallationContext install; + private final MetricsManager metricsManager; + + public AllNodesDisabledNamespacesUpdaterFactory( + AuthHeader authHeader, TimelockPaxosInstallationContext install, MetricsManager metricsManager) { + this.authHeader = authHeader; + this.install = install; + this.metricsManager = metricsManager; + } + + public AllNodesDisabledNamespacesUpdater create(TimelockNamespaces localNamespaces) { + PaxosRemoteClients remoteClients = ImmutablePaxosRemoteClients.of(install, metricsManager); + + List> remoteUpdaters = remoteClients.updaters(); + + // TODO(gs): include local one here?? + Map executorMap = + ImmutableMap.builder() + .putAll(KeyedStream.of(remoteUpdaters) + .mapKeys(WithDedicatedExecutor::service) + .map(WithDedicatedExecutor::executor) + .collectToMap()) + .build(); + + ImmutableList services = ImmutableList.copyOf( + remoteUpdaters.stream().map(WithDedicatedExecutor::service).collect(Collectors.toList())); + + return AllNodesDisabledNamespacesUpdater.create(authHeader, services, executorMap, localNamespaces); + } +} diff --git a/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/paxos/PaxosRemoteClients.java b/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/paxos/PaxosRemoteClients.java index ceecaa5df4a..74851b1ef9f 100644 --- a/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/paxos/PaxosRemoteClients.java +++ b/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/paxos/PaxosRemoteClients.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.net.HostAndPort; import com.palantir.atlasdb.AtlasDbMetricNames; +import com.palantir.atlasdb.timelock.api.DisabledNamespacesUpdaterService; import com.palantir.atlasdb.util.AtlasDbMetrics; import com.palantir.atlasdb.util.MetricsManager; import com.palantir.common.concurrent.CheckedRejectionExecutorService; @@ -121,6 +122,12 @@ public List> batchLearner() { return createInstrumentedRemoteProxiesAndAssignDedicatedPaxosExecutors(BatchPaxosLearnerRpcClient.class, true); } + @Value.Derived + public List> updaters() { + return createInstrumentedRemoteProxiesAndAssignDedicatedPaxosExecutors( + DisabledNamespacesUpdaterService.class, true); + } + @Value.Derived public List nonBatchPingableLeaders() { return nonBatchPingableLeadersWithContext().stream() diff --git a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java index d00775355c4..112473ea4ef 100644 --- a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java +++ b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java @@ -50,6 +50,8 @@ import com.palantir.atlasdb.timelock.batch.MultiClientConjureTimelockResource; import com.palantir.atlasdb.timelock.lock.LockLog; import com.palantir.atlasdb.timelock.lock.v1.ConjureLockV1Resource; +import com.palantir.atlasdb.timelock.management.AllNodesDisabledNamespacesUpdater; +import com.palantir.atlasdb.timelock.management.AllNodesDisabledNamespacesUpdaterFactory; import com.palantir.atlasdb.timelock.management.DisabledNamespaces; import com.palantir.atlasdb.timelock.management.DisabledNamespacesUpdaterResource; import com.palantir.atlasdb.timelock.management.PersistentNamespaceContexts; @@ -96,6 +98,7 @@ import com.palantir.timelock.store.PersistenceConfigStore; import com.palantir.timelock.store.SqliteBlobStore; import com.palantir.timestamp.ManagedTimestampService; +import com.palantir.tokens.auth.AuthHeader; import com.palantir.tokens.auth.BearerToken; import com.zaxxer.hikari.HikariDataSource; import java.net.URL; @@ -125,6 +128,7 @@ public class TimeLockAgent { private final TimeLockServicesCreator timelockCreator; private final NoSimultaneousServiceCheck noSimultaneousServiceCheck; private final PersistedSchemaVersion persistedSchemaVersion; + private final AllNodesDisabledNamespacesUpdaterFactory updaterFactory; private final HikariDataSource sqliteDataSource; private final FeedbackHandler feedbackHandler; private final LeaderElectionMetricAggregator leaderElectionAggregator; @@ -178,7 +182,8 @@ public static TimeLockAgent create( metricsManager, Suppliers.compose(TimeLockRuntimeConfiguration::paxos, restrictedRuntime::get)); - // TODO(gs): set up AllNodesUpdater + AllNodesDisabledNamespacesUpdaterFactory updaterFactory = new AllNodesDisabledNamespacesUpdaterFactory( + AuthHeader.valueOf("WIRE_ME_UP_SCOTTY"), installationContext, metricsManager); TimeLockAgent agent = new TimeLockAgent( metricsManager, @@ -190,6 +195,7 @@ public static TimeLockAgent create( blockingTimeoutMs, registrar, paxosResources, + updaterFactory, userAgent, persistedSchemaVersion, installationContext.sqliteDataSource(), @@ -235,6 +241,7 @@ private TimeLockAgent( long blockingTimeoutMs, Consumer registrar, PaxosResources paxosResources, + AllNodesDisabledNamespacesUpdaterFactory updaterFactory, UserAgent userAgent, PersistedSchemaVersion persistedSchemaVersion, HikariDataSource sqliteDataSource, @@ -246,6 +253,7 @@ private TimeLockAgent( this.undertowRegistrar = undertowRegistrar; this.registrar = registrar; this.paxosResources = paxosResources; + this.updaterFactory = updaterFactory; this.sqliteDataSource = sqliteDataSource; this.serviceStopper = serviceStopper; this.lockCreator = new LockCreator(runtime, threadPoolSize, blockingTimeoutMs); @@ -329,6 +337,7 @@ private void createAndRegisterResources() { this::createInvalidatingTimeLockServices, Suppliers.compose(TimeLockRuntimeConfiguration::maxNumberOfClients, runtime::get), DisabledNamespaces.create(sqliteDataSource)); + registerManagementResource(); // Finally, register the health check, and endpoints associated with the clients. TimeLockResource resource = TimeLockResource.create(namespaces); @@ -419,18 +428,22 @@ private boolean isLeaderForClient(Client client) { private void registerManagementResource() { ServiceLifecycleController serviceLifecycleController = new ServiceLifecycleController(serviceStopper, PTExecutors.newSingleThreadScheduledExecutor()); + AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater = updaterFactory.create(namespaces); + if (undertowRegistrar.isPresent()) { registerCorruptionHandlerWrappedService( undertowRegistrar.get(), TimeLockManagementResource.undertow( timestampStorage.persistentNamespaceContext(), namespaces, + allNodesDisabledNamespacesUpdater, redirectRetryTargeter(), serviceLifecycleController)); } else { registrar.accept(TimeLockManagementResource.jersey( timestampStorage.persistentNamespaceContext(), namespaces, + allNodesDisabledNamespacesUpdater, redirectRetryTargeter(), serviceLifecycleController)); } diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java index 91a14c0c10e..f9e5b9324d9 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java @@ -28,7 +28,6 @@ import com.palantir.atlasdb.timelock.api.Namespace; import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest; import com.palantir.atlasdb.timelock.api.ReenableNamespacesResponse; -import com.palantir.atlasdb.timelock.api.UnsuccessfulDisableNamespacesResponse; import com.palantir.atlasdb.timelock.api.management.TimeLockManagementService; import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceEndpoints; import com.palantir.atlasdb.timelock.api.management.UndertowTimeLockManagementService; @@ -47,16 +46,19 @@ public final class TimeLockManagementResource implements UndertowTimeLockManagem private static final SafeLogger log = SafeLoggerFactory.get(TimeLockManagementResource.class); private final Set namespaceLoaders; + private final AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater; private final TimelockNamespaces timelockNamespaces; private final ConjureResourceExceptionHandler exceptionHandler; private final ServiceLifecycleController serviceLifecycleController; private TimeLockManagementResource( Set namespaceLoaders, + AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater, TimelockNamespaces timelockNamespaces, RedirectRetryTargeter redirectRetryTargeter, ServiceLifecycleController serviceLifecycleController) { this.namespaceLoaders = namespaceLoaders; + this.allNodesDisabledNamespacesUpdater = allNodesDisabledNamespacesUpdater; this.timelockNamespaces = timelockNamespaces; this.exceptionHandler = new ConjureResourceExceptionHandler(redirectRetryTargeter); this.serviceLifecycleController = serviceLifecycleController; @@ -65,10 +67,12 @@ private TimeLockManagementResource( public static TimeLockManagementResource create( PersistentNamespaceContext persistentNamespaceContext, TimelockNamespaces timelockNamespaces, + AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater, RedirectRetryTargeter redirectRetryTargeter, ServiceLifecycleController serviceLifecycleController) { return new TimeLockManagementResource( createNamespaceLoaders(persistentNamespaceContext), + allNodesDisabledNamespacesUpdater, timelockNamespaces, redirectRetryTargeter, serviceLifecycleController); @@ -77,19 +81,29 @@ public static TimeLockManagementResource create( public static UndertowService undertow( PersistentNamespaceContext persistentNamespaceContext, TimelockNamespaces timelockNamespaces, + AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater, RedirectRetryTargeter redirectRetryTargeter, ServiceLifecycleController serviceLifecycleController) { return TimeLockManagementServiceEndpoints.of(TimeLockManagementResource.create( - persistentNamespaceContext, timelockNamespaces, redirectRetryTargeter, serviceLifecycleController)); + persistentNamespaceContext, + timelockNamespaces, + allNodesDisabledNamespacesUpdater, + redirectRetryTargeter, + serviceLifecycleController)); } public static TimeLockManagementService jersey( PersistentNamespaceContext persistentNamespaceContext, TimelockNamespaces timelockNamespaces, + AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater, RedirectRetryTargeter redirectRetryTargeter, ServiceLifecycleController serviceLifecycleController) { return new JerseyAdapter(TimeLockManagementResource.create( - persistentNamespaceContext, timelockNamespaces, redirectRetryTargeter, serviceLifecycleController)); + persistentNamespaceContext, + timelockNamespaces, + allNodesDisabledNamespacesUpdater, + redirectRetryTargeter, + serviceLifecycleController)); } @Override @@ -128,9 +142,7 @@ public ListenableFuture disableTimelock( } private ListenableFuture disableInternal(Set namespaces) { - // todo(gs): wire up ANDNU - return Futures.immediateFuture( - DisableNamespacesResponse.unsuccessful(UnsuccessfulDisableNamespacesResponse.of(ImmutableSet.of()))); + return Futures.immediateFuture(allNodesDisabledNamespacesUpdater.disableOnAllNodes(namespaces)); } @Override @@ -140,8 +152,7 @@ public ListenableFuture reenableTimelock( } public ListenableFuture reenableInternal(ReenableNamespacesRequest request) { - // todo(gs): wire up ANDNU - return Futures.immediateFuture(ReenableNamespacesResponse.builder().build()); + return Futures.immediateFuture(allNodesDisabledNamespacesUpdater.reEnableOnAllNodes(request)); } @Override diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/DiskNamespaceLoaderTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/DiskNamespaceLoaderTest.java index 05b0b239f5e..37d312270d0 100644 --- a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/DiskNamespaceLoaderTest.java +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/DiskNamespaceLoaderTest.java @@ -78,6 +78,7 @@ public void setup() throws MalformedURLException { timeLockManagementResource = TimeLockManagementResource.create( persistentNamespaceContext, namespaces, + mock(AllNodesDisabledNamespacesUpdater.class), redirectRetryTargeter, new ServiceLifecycleController(serviceStopper, PTExecutors.newSingleThreadScheduledExecutor())); From 318f302bb2f1e178d3da13086ec388b7e11fd557 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 31 Jan 2022 13:36:31 +0000 Subject: [PATCH 05/18] wire up to AtlasRestoreService --- .../atlasdb/backup/AtlasRestoreService.java | 13 ++++++++++++- .../atlasdb/backup/AtlasRestoreServiceTest.java | 8 ++++++-- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java index 323540b9a40..8da03f1d116 100644 --- a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java +++ b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java @@ -30,6 +30,7 @@ import com.palantir.atlasdb.internalschema.InternalSchemaMetadataState; import com.palantir.atlasdb.keyvalue.api.KeyValueService; import com.palantir.atlasdb.timelock.api.Namespace; +import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceBlocking; import com.palantir.common.streams.KeyedStream; import com.palantir.conjure.java.api.config.service.ServicesConfigBlock; import com.palantir.dialogue.clients.DialogueClients; @@ -52,6 +53,7 @@ public class AtlasRestoreService { private final AuthHeader authHeader; private final AtlasRestoreClientBlocking atlasRestoreClientBlocking; + private final TimeLockManagementServiceBlocking timeLockManagementService; private final BackupPersister backupPersister; private final CassandraRepairHelper cassandraRepairHelper; @@ -59,10 +61,12 @@ public class AtlasRestoreService { AtlasRestoreService( AuthHeader authHeader, AtlasRestoreClientBlocking atlasRestoreClientBlocking, + TimeLockManagementServiceBlocking timeLockManagementService, BackupPersister backupPersister, CassandraRepairHelper cassandraRepairHelper) { this.authHeader = authHeader; this.atlasRestoreClientBlocking = atlasRestoreClientBlocking; + this.timeLockManagementService = timeLockManagementService; this.backupPersister = backupPersister; this.cassandraRepairHelper = cassandraRepairHelper; } @@ -77,10 +81,17 @@ public static AtlasRestoreService create( DialogueClients.ReloadingFactory reloadingFactory = DialogueClients.create(servicesConfigBlock); AtlasRestoreClientBlocking atlasRestoreClientBlocking = reloadingFactory.get(AtlasRestoreClientBlocking.class, serviceName); + TimeLockManagementServiceBlocking timeLockManagementService = + reloadingFactory.get(TimeLockManagementServiceBlocking.class, serviceName); CassandraRepairHelper cassandraRepairHelper = new CassandraRepairHelper(keyValueServiceConfigFactory, keyValueServiceFactory); - return new AtlasRestoreService(authHeader, atlasRestoreClientBlocking, backupPersister, cassandraRepairHelper); + return new AtlasRestoreService( + authHeader, + atlasRestoreClientBlocking, + timeLockManagementService, + backupPersister, + cassandraRepairHelper); } /** diff --git a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java index f353ca6327c..a4e9d784f9a 100644 --- a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java +++ b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java @@ -32,6 +32,7 @@ import com.palantir.atlasdb.cassandra.backup.CassandraRepairHelper; import com.palantir.atlasdb.cassandra.backup.RangesForRepair; import com.palantir.atlasdb.timelock.api.Namespace; +import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceBlocking; import com.palantir.tokens.auth.AuthHeader; import java.util.Optional; import java.util.Set; @@ -56,6 +57,9 @@ public class AtlasRestoreServiceTest { @Mock private AtlasRestoreClientBlocking atlasRestoreClient; + @Mock + private TimeLockManagementServiceBlocking timeLockManagementService; + @Mock private CassandraRepairHelper cassandraRepairHelper; @@ -65,8 +69,8 @@ public class AtlasRestoreServiceTest { @Before public void setup() { backupPersister = new InMemoryBackupPersister(); - atlasRestoreService = - new AtlasRestoreService(authHeader, atlasRestoreClient, backupPersister, cassandraRepairHelper); + atlasRestoreService = new AtlasRestoreService( + authHeader, atlasRestoreClient, timeLockManagementService, backupPersister, cassandraRepairHelper); storeCompletedBackup(WITH_BACKUP); storeCompletedBackup(FAILING_NAMESPACE); From bbc919d566b13a704600c0fd73dc7e09b093744e Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 31 Jan 2022 13:58:17 +0000 Subject: [PATCH 06/18] disable/reenable during restore process --- .../atlasdb/backup/AtlasRestoreService.java | 26 +++++++--- .../backup/AtlasRestoreServiceTest.java | 49 ++++++++++++++++++- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java index 8da03f1d116..bca3769756d 100644 --- a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java +++ b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java @@ -29,7 +29,9 @@ import com.palantir.atlasdb.cassandra.backup.transaction.TransactionsTableInteraction; import com.palantir.atlasdb.internalschema.InternalSchemaMetadataState; import com.palantir.atlasdb.keyvalue.api.KeyValueService; +import com.palantir.atlasdb.timelock.api.DisableNamespacesResponse; import com.palantir.atlasdb.timelock.api.Namespace; +import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest; import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceBlocking; import com.palantir.common.streams.KeyedStream; import com.palantir.conjure.java.api.config.service.ServicesConfigBlock; @@ -48,6 +50,7 @@ import java.util.function.Function; import java.util.stream.Collectors; +// TODO(gs): retry logic for disable/re-enable public class AtlasRestoreService { private static final SafeLogger log = SafeLoggerFactory.get(AtlasRestoreService.class); @@ -105,11 +108,16 @@ public static AtlasRestoreService create( * * @return the set of namespaces for which we issued a repair command via the provided Consumer. */ - public Set repairInternalTables( + // TODO(gs): rename response object? + public DisableNamespacesResponse repairInternalTables( Set namespaces, BiConsumer repairTable) { Map completedBackups = getCompletedBackups(namespaces); Set namespacesToRepair = completedBackups.keySet(); + // Disable timelock + DisableNamespacesResponse response = timeLockManagementService.disableTimelock(authHeader, namespacesToRepair); + // TODO(gs): bail out if unsuccessful + // ConsistentCasTablesTask namespacesToRepair.forEach(namespace -> cassandraRepairHelper.repairInternalTables(namespace, repairTable)); @@ -118,7 +126,7 @@ public Set repairInternalTables( .forEach((namespace, completedBackup) -> restoreTransactionsTables(namespace, completedBackup, repairTable)); - return namespacesToRepair; + return response; } private void restoreTransactionsTables( @@ -143,8 +151,8 @@ private Map getCoordinationMap( schemaMetadataState, fastForwardTs, immutableTs); } - public Set completeRestore(Set namespaces) { - Set completedBackups = namespaces.stream() + public Set completeRestore(ReenableNamespacesRequest request) { + Set completedBackups = request.getNamespaces().stream() .map(backupPersister::getCompletedBackup) .flatMap(Optional::stream) .collect(Collectors.toSet()); @@ -152,13 +160,19 @@ public Set completeRestore(Set namespaces) { if (completedBackups.isEmpty()) { log.info( "Attempted to complete restore, but no completed backups were found", - SafeArg.of("namespaces", namespaces)); + SafeArg.of("namespaces", request.getNamespaces())); return ImmutableSet.of(); } CompleteRestoreResponse response = atlasRestoreClientBlocking.completeRestore(authHeader, CompleteRestoreRequest.of(completedBackups)); - return response.getSuccessfulNamespaces(); + + // TODO(gs): what to do about failed namespaces? + Set successfulNamespaces = response.getSuccessfulNamespaces(); + timeLockManagementService.reenableTimelock( + authHeader, ReenableNamespacesRequest.of(successfulNamespaces, request.getLockId())); + + return successfulNamespaces; } private Map getCompletedBackups(Set namespaces) { diff --git a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java index a4e9d784f9a..baa4f93b1b8 100644 --- a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java +++ b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java @@ -32,16 +32,20 @@ import com.palantir.atlasdb.cassandra.backup.CassandraRepairHelper; import com.palantir.atlasdb.cassandra.backup.RangesForRepair; import com.palantir.atlasdb.timelock.api.Namespace; +import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest; import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceBlocking; import com.palantir.tokens.auth.AuthHeader; import java.util.Optional; import java.util.Set; +import java.util.UUID; import java.util.function.BiConsumer; import java.util.stream.Collectors; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.InOrder; import org.mockito.Mock; +import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @RunWith(MockitoJUnitRunner.class) @@ -50,6 +54,7 @@ public class AtlasRestoreServiceTest { private static final Namespace NO_BACKUP = Namespace.of("no-backup"); private static final Namespace FAILING_NAMESPACE = Namespace.of("failing"); private static final long BACKUP_START_TIMESTAMP = 2L; + private static final UUID LOCK_ID = new UUID(12, 9); @Mock private AuthHeader authHeader; @@ -89,6 +94,9 @@ private void storeCompletedBackup(Namespace namespace) { @Test public void repairsOnlyWhenBackupPresent() { BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; + + // TODO(gs): validate response object + // TODO(gs): test: calls CassandraRepairHelper only when repair was successful atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); verify(cassandraRepairHelper).repairInternalTables(WITH_BACKUP, doNothingConsumer); @@ -97,12 +105,47 @@ public void repairsOnlyWhenBackupPresent() { verifyNoMoreInteractions(cassandraRepairHelper); } + @Test + public void disablesTimelockBeforeRepairing() { + BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; + atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); + + InOrder inOrder = Mockito.inOrder(timeLockManagementService, cassandraRepairHelper); + inOrder.verify(timeLockManagementService).disableTimelock(authHeader, ImmutableSet.of(WITH_BACKUP)); + inOrder.verify(cassandraRepairHelper).repairInternalTables(WITH_BACKUP, doNothingConsumer); + } + + @Test + public void completesRestoreAfterFastForwardingTimestamp() { + Set namespaces = ImmutableSet.of(WITH_BACKUP); + Set completedBackups = namespaces.stream() + .map(backupPersister::getCompletedBackup) + .flatMap(Optional::stream) + .collect(Collectors.toSet()); + + CompleteRestoreRequest completeRequest = CompleteRestoreRequest.of(completedBackups); + when(atlasRestoreClient.completeRestore(authHeader, completeRequest)) + .thenReturn(CompleteRestoreResponse.of(ImmutableSet.of(WITH_BACKUP))); + + ReenableNamespacesRequest reenableRequest = ReenableNamespacesRequest.of(namespaces, LOCK_ID); + + Set successfulNamespaces = atlasRestoreService.completeRestore(reenableRequest); + assertThat(successfulNamespaces).containsExactly(WITH_BACKUP); + + InOrder inOrder = Mockito.inOrder(atlasRestoreClient, timeLockManagementService); + inOrder.verify(atlasRestoreClient).completeRestore(authHeader, completeRequest); + inOrder.verify(timeLockManagementService).reenableTimelock(authHeader, reenableRequest); + } + @Test public void completeRestoreDoesNotRunNamespacesWithoutCompletedBackup() { - Set namespaces = atlasRestoreService.completeRestore(ImmutableSet.of(NO_BACKUP)); + ReenableNamespacesRequest reenableRequest = ReenableNamespacesRequest.of(ImmutableSet.of(NO_BACKUP), LOCK_ID); + + Set namespaces = atlasRestoreService.completeRestore(reenableRequest); assertThat(namespaces).isEmpty(); verifyNoInteractions(atlasRestoreClient); + verifyNoInteractions(timeLockManagementService); } @Test @@ -117,7 +160,9 @@ public void completeRestoreReturnsSuccessfulNamespaces() { when(atlasRestoreClient.completeRestore(authHeader, request)) .thenReturn(CompleteRestoreResponse.of(ImmutableSet.of(WITH_BACKUP))); - Set successfulNamespaces = atlasRestoreService.completeRestore(namespaces); + ReenableNamespacesRequest reenableRequest = ReenableNamespacesRequest.of(namespaces, LOCK_ID); + + Set successfulNamespaces = atlasRestoreService.completeRestore(reenableRequest); assertThat(successfulNamespaces).containsExactly(WITH_BACKUP); verify(atlasRestoreClient).completeRestore(authHeader, request); } From d96a048041c00589d6decc5afa8c958bf9ffbc69 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 31 Jan 2022 18:33:36 +0000 Subject: [PATCH 07/18] bail out if disable unsuccessful --- .../atlasdb/backup/AtlasRestoreService.java | 35 +++++++++++++++-- .../backup/AtlasRestoreServiceTest.java | 38 ++++++++++++++++--- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java index bca3769756d..128b5fde893 100644 --- a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java +++ b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java @@ -30,13 +30,17 @@ import com.palantir.atlasdb.internalschema.InternalSchemaMetadataState; import com.palantir.atlasdb.keyvalue.api.KeyValueService; import com.palantir.atlasdb.timelock.api.DisableNamespacesResponse; +import com.palantir.atlasdb.timelock.api.DisableNamespacesResponse.Visitor; import com.palantir.atlasdb.timelock.api.Namespace; import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest; +import com.palantir.atlasdb.timelock.api.SuccessfulDisableNamespacesResponse; +import com.palantir.atlasdb.timelock.api.UnsuccessfulDisableNamespacesResponse; import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceBlocking; import com.palantir.common.streams.KeyedStream; import com.palantir.conjure.java.api.config.service.ServicesConfigBlock; import com.palantir.dialogue.clients.DialogueClients; import com.palantir.logsafe.SafeArg; +import com.palantir.logsafe.exceptions.SafeIllegalStateException; import com.palantir.logsafe.logger.SafeLogger; import com.palantir.logsafe.logger.SafeLoggerFactory; import com.palantir.refreshable.Refreshable; @@ -114,10 +118,35 @@ public DisableNamespacesResponse repairInternalTables( Map completedBackups = getCompletedBackups(namespaces); Set namespacesToRepair = completedBackups.keySet(); - // Disable timelock DisableNamespacesResponse response = timeLockManagementService.disableTimelock(authHeader, namespacesToRepair); - // TODO(gs): bail out if unsuccessful + return response.accept(new Visitor<>() { + @Override + public DisableNamespacesResponse visitSuccessful(SuccessfulDisableNamespacesResponse value) { + restoreTables(repairTable, completedBackups, namespacesToRepair); + return response; + } + + @Override + public DisableNamespacesResponse visitUnsuccessful(UnsuccessfulDisableNamespacesResponse value) { + log.error( + "Failed to disable namespaces prior to restore", + SafeArg.of("namespaces", namespaces), + SafeArg.of("response", value)); + return response; + } + + @Override + public DisableNamespacesResponse visitUnknown(String unknownType) { + throw new SafeIllegalStateException( + "Unknown DisableNamespacesResponse", SafeArg.of("unknownType", unknownType)); + } + }); + } + private void restoreTables( + BiConsumer repairTable, + Map completedBackups, + Set namespacesToRepair) { // ConsistentCasTablesTask namespacesToRepair.forEach(namespace -> cassandraRepairHelper.repairInternalTables(namespace, repairTable)); @@ -125,8 +154,6 @@ public DisableNamespacesResponse repairInternalTables( KeyedStream.stream(completedBackups) .forEach((namespace, completedBackup) -> restoreTransactionsTables(namespace, completedBackup, repairTable)); - - return response; } private void restoreTransactionsTables( diff --git a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java index baa4f93b1b8..3e3be917024 100644 --- a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java +++ b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java @@ -31,8 +31,11 @@ import com.palantir.atlasdb.backup.api.CompletedBackup; import com.palantir.atlasdb.cassandra.backup.CassandraRepairHelper; import com.palantir.atlasdb.cassandra.backup.RangesForRepair; +import com.palantir.atlasdb.timelock.api.DisableNamespacesResponse; import com.palantir.atlasdb.timelock.api.Namespace; import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest; +import com.palantir.atlasdb.timelock.api.SuccessfulDisableNamespacesResponse; +import com.palantir.atlasdb.timelock.api.UnsuccessfulDisableNamespacesResponse; import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceBlocking; import com.palantir.tokens.auth.AuthHeader; import java.util.Optional; @@ -92,12 +95,17 @@ private void storeCompletedBackup(Namespace namespace) { } @Test - public void repairsOnlyWhenBackupPresent() { + public void repairsOnlyWhenBackupPresentAndDisableSuccessful() { BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; + DisableNamespacesResponse successfulDisable = + DisableNamespacesResponse.successful(SuccessfulDisableNamespacesResponse.of(LOCK_ID)); + when(timeLockManagementService.disableTimelock(authHeader, ImmutableSet.of(WITH_BACKUP))) + .thenReturn(successfulDisable); - // TODO(gs): validate response object - // TODO(gs): test: calls CassandraRepairHelper only when repair was successful - atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); + DisableNamespacesResponse actualDisable = + atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); + + assertThat(actualDisable).isEqualTo(successfulDisable); verify(cassandraRepairHelper).repairInternalTables(WITH_BACKUP, doNothingConsumer); verify(cassandraRepairHelper).repairTransactionsTables(eq(WITH_BACKUP), anyList(), eq(doNothingConsumer)); @@ -106,8 +114,13 @@ public void repairsOnlyWhenBackupPresent() { } @Test - public void disablesTimelockBeforeRepairing() { + public void disablesTimeLockBeforeRepairing() { BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; + DisableNamespacesResponse successfulDisable = + DisableNamespacesResponse.successful(SuccessfulDisableNamespacesResponse.of(LOCK_ID)); + when(timeLockManagementService.disableTimelock(authHeader, ImmutableSet.of(WITH_BACKUP))) + .thenReturn(successfulDisable); + atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); InOrder inOrder = Mockito.inOrder(timeLockManagementService, cassandraRepairHelper); @@ -115,6 +128,21 @@ public void disablesTimelockBeforeRepairing() { inOrder.verify(cassandraRepairHelper).repairInternalTables(WITH_BACKUP, doNothingConsumer); } + @Test + public void doesNotRepairIfDisableFails() { + BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; + DisableNamespacesResponse failedDisable = DisableNamespacesResponse.unsuccessful( + UnsuccessfulDisableNamespacesResponse.of(ImmutableSet.of(WITH_BACKUP), ImmutableSet.of())); + when(timeLockManagementService.disableTimelock(authHeader, ImmutableSet.of(WITH_BACKUP))) + .thenReturn(failedDisable); + + DisableNamespacesResponse actualDisable = + atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); + assertThat(actualDisable).isEqualTo(failedDisable); + + verifyNoInteractions(cassandraRepairHelper); + } + @Test public void completesRestoreAfterFastForwardingTimestamp() { Set namespaces = ImmutableSet.of(WITH_BACKUP); From 86d814288dcfea4b15ef2f5d5a89aa34d19a2d2f Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 7 Feb 2022 11:11:35 +0000 Subject: [PATCH 08/18] separate prepare method --- .../atlasdb/backup/AtlasRestoreService.java | 85 ++++++++++--------- .../backup/AtlasRestoreServiceTest.java | 45 ++++------ 2 files changed, 61 insertions(+), 69 deletions(-) diff --git a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java index 128b5fde893..5c32b592968 100644 --- a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java +++ b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java @@ -30,7 +30,6 @@ import com.palantir.atlasdb.internalschema.InternalSchemaMetadataState; import com.palantir.atlasdb.keyvalue.api.KeyValueService; import com.palantir.atlasdb.timelock.api.DisableNamespacesResponse; -import com.palantir.atlasdb.timelock.api.DisableNamespacesResponse.Visitor; import com.palantir.atlasdb.timelock.api.Namespace; import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest; import com.palantir.atlasdb.timelock.api.SuccessfulDisableNamespacesResponse; @@ -101,28 +100,14 @@ public static AtlasRestoreService create( cassandraRepairHelper); } - /** - * Returns the set of namespaces for which we successfully repaired internal tables. - * Only namespaces for which a known backup exists will be repaired. - * Namespaces are repaired serially. If repairTable throws an exception, then this will propagate back to the - * caller. In such cases, some namespaces may not have been repaired. - * - * @param namespaces the namespaces to repair. - * @param repairTable supplied function which is expected to repair the given ranges. - * - * @return the set of namespaces for which we issued a repair command via the provided Consumer. - */ - // TODO(gs): rename response object? - public DisableNamespacesResponse repairInternalTables( - Set namespaces, BiConsumer repairTable) { + public DisableNamespacesResponse prepareRestore(Set namespaces) { Map completedBackups = getCompletedBackups(namespaces); Set namespacesToRepair = completedBackups.keySet(); DisableNamespacesResponse response = timeLockManagementService.disableTimelock(authHeader, namespacesToRepair); - return response.accept(new Visitor<>() { + return response.accept(new DisableNamespacesResponse.Visitor<>() { @Override public DisableNamespacesResponse visitSuccessful(SuccessfulDisableNamespacesResponse value) { - restoreTables(repairTable, completedBackups, namespacesToRepair); return response; } @@ -143,6 +128,48 @@ public DisableNamespacesResponse visitUnknown(String unknownType) { }); } + /** + * Returns the set of namespaces for which we successfully repaired internal tables. + * Only namespaces for which a known backup exists will be repaired. + * Namespaces are repaired serially. If repairTable throws an exception, then this will propagate back to the + * caller. In such cases, some namespaces may not have been repaired. + * + * @param namespaces the namespaces to repair. + * @param repairTable supplied function which is expected to repair the given ranges. + * + * @return the set of namespaces for which we issued a repair command via the provided Consumer. + */ + // TODO(gs): response object? + public void repairInternalTables(Set namespaces, BiConsumer repairTable) { + Map completedBackups = getCompletedBackups(namespaces); + Set namespacesToRepair = completedBackups.keySet(); + restoreTables(repairTable, completedBackups, namespacesToRepair); + } + + public Set completeRestore(ReenableNamespacesRequest request) { + Set completedBackups = request.getNamespaces().stream() + .map(backupPersister::getCompletedBackup) + .flatMap(Optional::stream) + .collect(Collectors.toSet()); + + if (completedBackups.isEmpty()) { + log.info( + "Attempted to complete restore, but no completed backups were found", + SafeArg.of("namespaces", request.getNamespaces())); + return ImmutableSet.of(); + } + + CompleteRestoreResponse response = + atlasRestoreClientBlocking.completeRestore(authHeader, CompleteRestoreRequest.of(completedBackups)); + + // TODO(gs): what to do about failed namespaces? + Set successfulNamespaces = response.getSuccessfulNamespaces(); + timeLockManagementService.reenableTimelock( + authHeader, ReenableNamespacesRequest.of(successfulNamespaces, request.getLockId())); + + return successfulNamespaces; + } + private void restoreTables( BiConsumer repairTable, Map completedBackups, @@ -178,30 +205,6 @@ private Map getCoordinationMap( schemaMetadataState, fastForwardTs, immutableTs); } - public Set completeRestore(ReenableNamespacesRequest request) { - Set completedBackups = request.getNamespaces().stream() - .map(backupPersister::getCompletedBackup) - .flatMap(Optional::stream) - .collect(Collectors.toSet()); - - if (completedBackups.isEmpty()) { - log.info( - "Attempted to complete restore, but no completed backups were found", - SafeArg.of("namespaces", request.getNamespaces())); - return ImmutableSet.of(); - } - - CompleteRestoreResponse response = - atlasRestoreClientBlocking.completeRestore(authHeader, CompleteRestoreRequest.of(completedBackups)); - - // TODO(gs): what to do about failed namespaces? - Set successfulNamespaces = response.getSuccessfulNamespaces(); - timeLockManagementService.reenableTimelock( - authHeader, ReenableNamespacesRequest.of(successfulNamespaces, request.getLockId())); - - return successfulNamespaces; - } - private Map getCompletedBackups(Set namespaces) { return KeyedStream.of(namespaces) .map(backupPersister::getCompletedBackup) diff --git a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java index 3e3be917024..8b506a07121 100644 --- a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java +++ b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java @@ -59,6 +59,7 @@ public class AtlasRestoreServiceTest { private static final long BACKUP_START_TIMESTAMP = 2L; private static final UUID LOCK_ID = new UUID(12, 9); + // TODO(gs): auth header tests @Mock private AuthHeader authHeader; @@ -94,53 +95,41 @@ private void storeCompletedBackup(Namespace namespace) { backupPersister.storeCompletedBackup(completedBackup); } + // TODO(gs): should return namespaces too @Test - public void repairsOnlyWhenBackupPresentAndDisableSuccessful() { - BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; + public void prepareReturnsOnlyCompletedBackups() { DisableNamespacesResponse successfulDisable = DisableNamespacesResponse.successful(SuccessfulDisableNamespacesResponse.of(LOCK_ID)); when(timeLockManagementService.disableTimelock(authHeader, ImmutableSet.of(WITH_BACKUP))) .thenReturn(successfulDisable); DisableNamespacesResponse actualDisable = - atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); - + atlasRestoreService.prepareRestore(ImmutableSet.of(WITH_BACKUP, NO_BACKUP)); assertThat(actualDisable).isEqualTo(successfulDisable); - - verify(cassandraRepairHelper).repairInternalTables(WITH_BACKUP, doNothingConsumer); - verify(cassandraRepairHelper).repairTransactionsTables(eq(WITH_BACKUP), anyList(), eq(doNothingConsumer)); - verify(cassandraRepairHelper).cleanTransactionsTables(eq(WITH_BACKUP), eq(BACKUP_START_TIMESTAMP), anyList()); - verifyNoMoreInteractions(cassandraRepairHelper); } @Test - public void disablesTimeLockBeforeRepairing() { - BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; - DisableNamespacesResponse successfulDisable = - DisableNamespacesResponse.successful(SuccessfulDisableNamespacesResponse.of(LOCK_ID)); - when(timeLockManagementService.disableTimelock(authHeader, ImmutableSet.of(WITH_BACKUP))) - .thenReturn(successfulDisable); - - atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); - - InOrder inOrder = Mockito.inOrder(timeLockManagementService, cassandraRepairHelper); - inOrder.verify(timeLockManagementService).disableTimelock(authHeader, ImmutableSet.of(WITH_BACKUP)); - inOrder.verify(cassandraRepairHelper).repairInternalTables(WITH_BACKUP, doNothingConsumer); - } - - @Test - public void doesNotRepairIfDisableFails() { - BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; + public void prepareBackupFailsIfDisableFails() { DisableNamespacesResponse failedDisable = DisableNamespacesResponse.unsuccessful( UnsuccessfulDisableNamespacesResponse.of(ImmutableSet.of(WITH_BACKUP), ImmutableSet.of())); when(timeLockManagementService.disableTimelock(authHeader, ImmutableSet.of(WITH_BACKUP))) .thenReturn(failedDisable); DisableNamespacesResponse actualDisable = - atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); + atlasRestoreService.prepareRestore(ImmutableSet.of(WITH_BACKUP, NO_BACKUP)); assertThat(actualDisable).isEqualTo(failedDisable); + } - verifyNoInteractions(cassandraRepairHelper); + @Test + public void repairsOnlyWhenBackupPresentAndDisableSuccessful() { + BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; + + atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); + + verify(cassandraRepairHelper).repairInternalTables(WITH_BACKUP, doNothingConsumer); + verify(cassandraRepairHelper).repairTransactionsTables(eq(WITH_BACKUP), anyList(), eq(doNothingConsumer)); + verify(cassandraRepairHelper).cleanTransactionsTables(eq(WITH_BACKUP), eq(BACKUP_START_TIMESTAMP), anyList()); + verifyNoMoreInteractions(cassandraRepairHelper); } @Test From b07e20892cdd2994defabc44c0273da6e269f3ca Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 7 Feb 2022 12:27:21 +0000 Subject: [PATCH 09/18] pull out AuthHeaderValidator --- .../atlasdb/backup/AtlasRestoreResource.java | 13 ++----- .../atlasdb/backup/AuthHeaderValidator.java | 37 +++++++++++++++++++ 2 files changed, 40 insertions(+), 10 deletions(-) create mode 100644 timelock-impl/src/main/java/com/palantir/atlasdb/backup/AuthHeaderValidator.java diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreResource.java b/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreResource.java index 404cd76e1a4..dfe0740e3e4 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreResource.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreResource.java @@ -51,14 +51,14 @@ public class AtlasRestoreResource implements UndertowAtlasRestoreClient { private final Function timelockServices; private final ConjureResourceExceptionHandler exceptionHandler; - private final Supplier> permittedToken; + private final AuthHeaderValidator authHeaderValidator; @VisibleForTesting AtlasRestoreResource( Supplier> permittedToken, RedirectRetryTargeter redirectRetryTargeter, Function timelockServices) { - this.permittedToken = permittedToken; + this.authHeaderValidator = new AuthHeaderValidator(permittedToken); this.exceptionHandler = new ConjureResourceExceptionHandler(redirectRetryTargeter); this.timelockServices = timelockServices; } @@ -87,7 +87,7 @@ public ListenableFuture completeRestore( private ListenableFuture completeRestoreInternal( AuthHeader authHeader, CompleteRestoreRequest request) { - if (!suppliedTokenIsValid(authHeader)) { + if (!authHeaderValidator.suppliedTokenIsValid(authHeader)) { log.error( "Attempted to complete restore with an invalid auth header. " + "The provided token must match the configured permitted-backup-token.", @@ -113,13 +113,6 @@ private ListenableFuture> completeRestoreAsync(CompletedBack return Futures.immediateFuture(Optional.of(namespace)); } - private Boolean suppliedTokenIsValid(AuthHeader suppliedAuthHeader) { - return permittedToken - .get() - .map(token -> token.equals(suppliedAuthHeader.getBearerToken())) - .orElse(false); - } - private AsyncTimelockService timelock(Namespace namespace) { return timelockServices.apply(namespace.get()); } diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AuthHeaderValidator.java b/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AuthHeaderValidator.java new file mode 100644 index 00000000000..1151031d983 --- /dev/null +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AuthHeaderValidator.java @@ -0,0 +1,37 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.backup; + +import com.palantir.tokens.auth.AuthHeader; +import com.palantir.tokens.auth.BearerToken; +import java.util.Optional; +import java.util.function.Supplier; + +public final class AuthHeaderValidator { + private final Supplier> permittedToken; + + public AuthHeaderValidator(Supplier> permittedToken) { + this.permittedToken = permittedToken; + } + + public boolean suppliedTokenIsValid(AuthHeader suppliedAuthHeader) { + return permittedToken + .get() + .map(token -> token.equals(suppliedAuthHeader.getBearerToken())) + .orElse(false); + } +} From 36ba1b35826684b00565b1c6000a0572241ef8d7 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 7 Feb 2022 12:36:06 +0000 Subject: [PATCH 10/18] validate auth headers in mgmt resource --- .../timelock/paxos/TimeLockAgent.java | 6 +++ .../TimeLockManagementResource.java | 27 ++++++++++++ ...va => TimeLockManagementResourceTest.java} | 44 ++++++++++++++++++- 3 files changed, 75 insertions(+), 2 deletions(-) rename timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/{DiskNamespaceLoaderTest.java => TimeLockManagementResourceTest.java} (67%) diff --git a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java index 112473ea4ef..b464deeba3f 100644 --- a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java +++ b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java @@ -26,6 +26,7 @@ import com.palantir.atlasdb.AtlasDbConstants; import com.palantir.atlasdb.backup.AtlasBackupResource; import com.palantir.atlasdb.backup.AtlasRestoreResource; +import com.palantir.atlasdb.backup.AuthHeaderValidator; import com.palantir.atlasdb.config.AuxiliaryRemotingParameters; import com.palantir.atlasdb.config.ImmutableLeaderConfig; import com.palantir.atlasdb.config.ImmutableServerListConfig; @@ -429,6 +430,9 @@ private void registerManagementResource() { ServiceLifecycleController serviceLifecycleController = new ServiceLifecycleController(serviceStopper, PTExecutors.newSingleThreadScheduledExecutor()); AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater = updaterFactory.create(namespaces); + Refreshable> permittedBackupToken = + runtime.map(TimeLockRuntimeConfiguration::permittedBackupToken); + AuthHeaderValidator authHeaderValidator = new AuthHeaderValidator(permittedBackupToken); if (undertowRegistrar.isPresent()) { registerCorruptionHandlerWrappedService( @@ -437,6 +441,7 @@ private void registerManagementResource() { timestampStorage.persistentNamespaceContext(), namespaces, allNodesDisabledNamespacesUpdater, + authHeaderValidator, redirectRetryTargeter(), serviceLifecycleController)); } else { @@ -444,6 +449,7 @@ private void registerManagementResource() { timestampStorage.persistentNamespaceContext(), namespaces, allNodesDisabledNamespacesUpdater, + authHeaderValidator, redirectRetryTargeter(), serviceLifecycleController)); } diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java index f9e5b9324d9..bfc00fb5ba0 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.palantir.atlasdb.backup.AuthHeaderValidator; import com.palantir.atlasdb.futures.AtlasFutures; import com.palantir.atlasdb.http.RedirectRetryTargeter; import com.palantir.atlasdb.keyvalue.api.TimestampSeries; @@ -32,7 +33,10 @@ import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceEndpoints; import com.palantir.atlasdb.timelock.api.management.UndertowTimeLockManagementService; import com.palantir.atlasdb.timelock.paxos.PaxosTimeLockConstants; +import com.palantir.conjure.java.api.errors.ErrorType; +import com.palantir.conjure.java.api.errors.ServiceException; import com.palantir.conjure.java.undertow.lib.UndertowService; +import com.palantir.logsafe.SafeArg; import com.palantir.logsafe.logger.SafeLogger; import com.palantir.logsafe.logger.SafeLoggerFactory; import com.palantir.paxos.Client; @@ -48,6 +52,7 @@ public final class TimeLockManagementResource implements UndertowTimeLockManagem private final Set namespaceLoaders; private final AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater; private final TimelockNamespaces timelockNamespaces; + private final AuthHeaderValidator authHeaderValidator; private final ConjureResourceExceptionHandler exceptionHandler; private final ServiceLifecycleController serviceLifecycleController; @@ -55,11 +60,13 @@ private TimeLockManagementResource( Set namespaceLoaders, AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater, TimelockNamespaces timelockNamespaces, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, ServiceLifecycleController serviceLifecycleController) { this.namespaceLoaders = namespaceLoaders; this.allNodesDisabledNamespacesUpdater = allNodesDisabledNamespacesUpdater; this.timelockNamespaces = timelockNamespaces; + this.authHeaderValidator = authHeaderValidator; this.exceptionHandler = new ConjureResourceExceptionHandler(redirectRetryTargeter); this.serviceLifecycleController = serviceLifecycleController; } @@ -68,12 +75,14 @@ public static TimeLockManagementResource create( PersistentNamespaceContext persistentNamespaceContext, TimelockNamespaces timelockNamespaces, AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, ServiceLifecycleController serviceLifecycleController) { return new TimeLockManagementResource( createNamespaceLoaders(persistentNamespaceContext), allNodesDisabledNamespacesUpdater, timelockNamespaces, + authHeaderValidator, redirectRetryTargeter, serviceLifecycleController); } @@ -82,12 +91,14 @@ public static UndertowService undertow( PersistentNamespaceContext persistentNamespaceContext, TimelockNamespaces timelockNamespaces, AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, ServiceLifecycleController serviceLifecycleController) { return TimeLockManagementServiceEndpoints.of(TimeLockManagementResource.create( persistentNamespaceContext, timelockNamespaces, allNodesDisabledNamespacesUpdater, + authHeaderValidator, redirectRetryTargeter, serviceLifecycleController)); } @@ -96,12 +107,14 @@ public static TimeLockManagementService jersey( PersistentNamespaceContext persistentNamespaceContext, TimelockNamespaces timelockNamespaces, AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, ServiceLifecycleController serviceLifecycleController) { return new JerseyAdapter(TimeLockManagementResource.create( persistentNamespaceContext, timelockNamespaces, allNodesDisabledNamespacesUpdater, + authHeaderValidator, redirectRetryTargeter, serviceLifecycleController)); } @@ -138,6 +151,13 @@ public ListenableFuture invalidateResources(AuthHeader authHeader, Set disableTimelock( AuthHeader authHeader, Set namespaces) { + if (!authHeaderValidator.suppliedTokenIsValid(authHeader)) { + log.error( + "Attempted to disable TimeLock with an invalid auth header. " + + "The provided token must match the configured permitted-backup-token.", + SafeArg.of("namespaces", namespaces)); + throw new ServiceException(ErrorType.PERMISSION_DENIED); + } return handleExceptions(() -> disableInternal(namespaces)); } @@ -148,6 +168,13 @@ private ListenableFuture disableInternal(Set reenableTimelock( AuthHeader authHeader, ReenableNamespacesRequest request) { + if (!authHeaderValidator.suppliedTokenIsValid(authHeader)) { + log.error( + "Attempted to re-enable TimeLock with an invalid auth header. " + + "The provided token must match the configured permitted-backup-token.", + SafeArg.of("request", request)); + throw new ServiceException(ErrorType.PERMISSION_DENIED); + } return handleExceptions(() -> reenableInternal(request)); } diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/DiskNamespaceLoaderTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java similarity index 67% rename from timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/DiskNamespaceLoaderTest.java rename to timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java index 37d312270d0..1f0cd250665 100644 --- a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/DiskNamespaceLoaderTest.java +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java @@ -16,23 +16,33 @@ package com.palantir.atlasdb.timelock.management; +import static com.palantir.conjure.java.api.testing.Assertions.assertThatServiceExceptionThrownBy; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.palantir.atlasdb.backup.AuthHeaderValidator; +import com.palantir.atlasdb.futures.AtlasFutures; import com.palantir.atlasdb.http.RedirectRetryTargeter; import com.palantir.atlasdb.timelock.TimeLockServices; import com.palantir.atlasdb.timelock.TimelockNamespaces; +import com.palantir.atlasdb.timelock.api.Namespace; +import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest; import com.palantir.atlasdb.timelock.paxos.PaxosTimeLockConstants; import com.palantir.atlasdb.util.MetricsManager; import com.palantir.atlasdb.util.MetricsManagers; import com.palantir.common.concurrent.PTExecutors; +import com.palantir.conjure.java.api.errors.ErrorType; import com.palantir.paxos.SqliteConnections; import com.palantir.tokens.auth.AuthHeader; +import com.palantir.tokens.auth.BearerToken; import java.net.MalformedURLException; import java.net.URL; import java.nio.file.Path; import java.util.Set; +import java.util.UUID; import java.util.concurrent.ExecutionException; import java.util.function.Function; import java.util.function.Supplier; @@ -40,12 +50,23 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; import org.mockito.Mock; +import org.mockito.junit.MockitoJUnitRunner; -public class DiskNamespaceLoaderTest { +@RunWith(MockitoJUnitRunner.class) +public class TimeLockManagementResourceTest { private static final String NAMESPACE_1 = "namespace_1"; private static final String NAMESPACE_2 = "namespace_2"; - private static final AuthHeader AUTH_HEADER = AuthHeader.valueOf("Bearer omitted"); + private static final Set NAMESPACES = + ImmutableSet.of(Namespace.of(NAMESPACE_1), Namespace.of(NAMESPACE_2)); + private static final BearerToken BEARER_TOKEN = BearerToken.valueOf("bear"); + private static final BearerToken WRONG_BEARER_TOKEN = BearerToken.valueOf("tiger"); + private static final AuthHeader AUTH_HEADER = AuthHeader.of(BEARER_TOKEN); + private static final AuthHeader WRONG_AUTH_HEADER = AuthHeader.of(WRONG_BEARER_TOKEN); + + @Mock + private AuthHeaderValidator authHeaderValidator; @Mock private Function serviceFactory; @@ -75,10 +96,13 @@ public void setup() throws MalformedURLException { TimelockNamespaces namespaces = new TimelockNamespaces(metricsManager, serviceFactory, maxNumberOfClientsSupplier, disabledNamespaces); + when(authHeaderValidator.suppliedTokenIsValid(AUTH_HEADER)).thenReturn(true); + when(authHeaderValidator.suppliedTokenIsValid(WRONG_AUTH_HEADER)).thenReturn(false); timeLockManagementResource = TimeLockManagementResource.create( persistentNamespaceContext, namespaces, mock(AllNodesDisabledNamespacesUpdater.class), + authHeaderValidator, redirectRetryTargeter, new ServiceLifecycleController(serviceStopper, PTExecutors.newSingleThreadScheduledExecutor())); @@ -86,6 +110,22 @@ public void setup() throws MalformedURLException { createDirectoryInRootDataDirectory(NAMESPACE_2); } + @Test + public void disableTimeLockThrowsIfAuthHeaderIsWrong() { + assertThatServiceExceptionThrownBy(() -> AtlasFutures.getUnchecked( + timeLockManagementResource.disableTimelock(WRONG_AUTH_HEADER, NAMESPACES))) + .hasType(ErrorType.PERMISSION_DENIED); + } + // TODO(gs): dis/reenable tests with valid auth header + + @Test + public void reEnableTimeLockThrowsIfAuthHeaderIsWrong() { + ReenableNamespacesRequest request = ReenableNamespacesRequest.of(NAMESPACES, UUID.randomUUID()); + assertThatServiceExceptionThrownBy(() -> AtlasFutures.getUnchecked( + timeLockManagementResource.reenableTimelock(WRONG_AUTH_HEADER, request))) + .hasType(ErrorType.PERMISSION_DENIED); + } + @Test public void doesNotLoadLeaderPaxosAsNamespace() throws ExecutionException, InterruptedException { Set namespaces = From 77347543054cb96c7a86071d6be40f200c0107c0 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 7 Feb 2022 12:38:50 +0000 Subject: [PATCH 11/18] more tests --- .../TimeLockManagementResourceTest.java | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java index 1f0cd250665..191d84c153d 100644 --- a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java @@ -19,6 +19,8 @@ import static com.palantir.conjure.java.api.testing.Assertions.assertThatServiceExceptionThrownBy; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableList; @@ -68,6 +70,9 @@ public class TimeLockManagementResourceTest { @Mock private AuthHeaderValidator authHeaderValidator; + @Mock + private AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater; + @Mock private Function serviceFactory; @@ -101,7 +106,7 @@ public void setup() throws MalformedURLException { timeLockManagementResource = TimeLockManagementResource.create( persistentNamespaceContext, namespaces, - mock(AllNodesDisabledNamespacesUpdater.class), + allNodesDisabledNamespacesUpdater, authHeaderValidator, redirectRetryTargeter, new ServiceLifecycleController(serviceStopper, PTExecutors.newSingleThreadScheduledExecutor())); @@ -115,8 +120,14 @@ public void disableTimeLockThrowsIfAuthHeaderIsWrong() { assertThatServiceExceptionThrownBy(() -> AtlasFutures.getUnchecked( timeLockManagementResource.disableTimelock(WRONG_AUTH_HEADER, NAMESPACES))) .hasType(ErrorType.PERMISSION_DENIED); + verifyNoInteractions(allNodesDisabledNamespacesUpdater); + } + + @Test + public void disableTimeLockCallsUpdater() { + timeLockManagementResource.disableTimelock(AUTH_HEADER, NAMESPACES); + verify(allNodesDisabledNamespacesUpdater).disableOnAllNodes(NAMESPACES); } - // TODO(gs): dis/reenable tests with valid auth header @Test public void reEnableTimeLockThrowsIfAuthHeaderIsWrong() { @@ -124,6 +135,14 @@ public void reEnableTimeLockThrowsIfAuthHeaderIsWrong() { assertThatServiceExceptionThrownBy(() -> AtlasFutures.getUnchecked( timeLockManagementResource.reenableTimelock(WRONG_AUTH_HEADER, request))) .hasType(ErrorType.PERMISSION_DENIED); + verifyNoInteractions(allNodesDisabledNamespacesUpdater); + } + + @Test + public void reEnableTimeLockCallsUpdater() { + ReenableNamespacesRequest request = ReenableNamespacesRequest.of(NAMESPACES, UUID.randomUUID()); + timeLockManagementResource.reenableTimelock(AUTH_HEADER, request); + verify(allNodesDisabledNamespacesUpdater).reEnableOnAllNodes(request); } @Test From be642181a7b87a6de0c482648147f08c45a4a8fe Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 7 Feb 2022 13:37:27 +0000 Subject: [PATCH 12/18] compile --- .../com/palantir/atlasdb/backup/AtlasRestoreService.java | 5 +++-- .../timelock/management/TimeLockManagementResourceTest.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java index 5c32b592968..d368e1781ae 100644 --- a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java +++ b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java @@ -139,11 +139,12 @@ public DisableNamespacesResponse visitUnknown(String unknownType) { * * @return the set of namespaces for which we issued a repair command via the provided Consumer. */ - // TODO(gs): response object? - public void repairInternalTables(Set namespaces, BiConsumer repairTable) { + public Set repairInternalTables( + Set namespaces, BiConsumer repairTable) { Map completedBackups = getCompletedBackups(namespaces); Set namespacesToRepair = completedBackups.keySet(); restoreTables(repairTable, completedBackups, namespacesToRepair); + return namespacesToRepair; } public Set completeRestore(ReenableNamespacesRequest request) { diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java index 191d84c153d..413a4be6978 100644 --- a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java @@ -60,7 +60,7 @@ public class TimeLockManagementResourceTest { private static final String NAMESPACE_1 = "namespace_1"; private static final String NAMESPACE_2 = "namespace_2"; - private static final Set NAMESPACES = + private static final ImmutableSet NAMESPACES = ImmutableSet.of(Namespace.of(NAMESPACE_1), Namespace.of(NAMESPACE_2)); private static final BearerToken BEARER_TOKEN = BearerToken.valueOf("bear"); private static final BearerToken WRONG_BEARER_TOKEN = BearerToken.valueOf("tiger"); From 48c6834edfdf6e0296d75aa9af714d0e9826f99a Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 7 Feb 2022 13:49:22 +0000 Subject: [PATCH 13/18] cleanup --- .../com/palantir/atlasdb/backup/AtlasRestoreService.java | 6 +++--- .../palantir/atlasdb/backup/AtlasRestoreServiceTest.java | 6 +++--- .../AllNodesDisabledNamespacesUpdaterFactory.java | 7 +++---- .../management/AllNodesDisabledNamespacesUpdater.java | 7 ++++--- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java index d368e1781ae..80a459a0aa4 100644 --- a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java +++ b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java @@ -53,7 +53,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -// TODO(gs): retry logic for disable/re-enable public class AtlasRestoreService { private static final SafeLogger log = SafeLoggerFactory.get(AtlasRestoreService.class); @@ -160,11 +159,12 @@ public Set completeRestore(ReenableNamespacesRequest request) { return ImmutableSet.of(); } + // Fast forward timestamps CompleteRestoreResponse response = atlasRestoreClientBlocking.completeRestore(authHeader, CompleteRestoreRequest.of(completedBackups)); - - // TODO(gs): what to do about failed namespaces? Set successfulNamespaces = response.getSuccessfulNamespaces(); + + // Re-enable timelock timeLockManagementService.reenableTimelock( authHeader, ReenableNamespacesRequest.of(successfulNamespaces, request.getLockId())); diff --git a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java index 8b506a07121..ee0581d271f 100644 --- a/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java +++ b/atlasdb-backup/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreServiceTest.java @@ -59,7 +59,6 @@ public class AtlasRestoreServiceTest { private static final long BACKUP_START_TIMESTAMP = 2L; private static final UUID LOCK_ID = new UUID(12, 9); - // TODO(gs): auth header tests @Mock private AuthHeader authHeader; @@ -95,7 +94,6 @@ private void storeCompletedBackup(Namespace namespace) { backupPersister.storeCompletedBackup(completedBackup); } - // TODO(gs): should return namespaces too @Test public void prepareReturnsOnlyCompletedBackups() { DisableNamespacesResponse successfulDisable = @@ -124,7 +122,9 @@ public void prepareBackupFailsIfDisableFails() { public void repairsOnlyWhenBackupPresentAndDisableSuccessful() { BiConsumer doNothingConsumer = (_unused1, _unused2) -> {}; - atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); + Set repairedNamespaces = + atlasRestoreService.repairInternalTables(ImmutableSet.of(WITH_BACKUP, NO_BACKUP), doNothingConsumer); + assertThat(repairedNamespaces).containsExactly(WITH_BACKUP); verify(cassandraRepairHelper).repairInternalTables(WITH_BACKUP, doNothingConsumer); verify(cassandraRepairHelper).repairTransactionsTables(eq(WITH_BACKUP), anyList(), eq(doNothingConsumer)); diff --git a/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java b/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java index f8bb0096fa2..aa1b3ce7a25 100644 --- a/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java +++ b/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java @@ -49,8 +49,7 @@ public AllNodesDisabledNamespacesUpdater create(TimelockNamespaces localNamespac List> remoteUpdaters = remoteClients.updaters(); - // TODO(gs): include local one here?? - Map executorMap = + Map remoteExecutors = ImmutableMap.builder() .putAll(KeyedStream.of(remoteUpdaters) .mapKeys(WithDedicatedExecutor::service) @@ -58,9 +57,9 @@ public AllNodesDisabledNamespacesUpdater create(TimelockNamespaces localNamespac .collectToMap()) .build(); - ImmutableList services = ImmutableList.copyOf( + ImmutableList remoteServices = ImmutableList.copyOf( remoteUpdaters.stream().map(WithDedicatedExecutor::service).collect(Collectors.toList())); - return AllNodesDisabledNamespacesUpdater.create(authHeader, services, executorMap, localNamespaces); + return AllNodesDisabledNamespacesUpdater.create(authHeader, remoteServices, remoteExecutors, localNamespaces); } } diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdater.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdater.java index 358085ef8df..6945deec647 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdater.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdater.java @@ -79,10 +79,11 @@ public class AllNodesDisabledNamespacesUpdater { public static AllNodesDisabledNamespacesUpdater create( AuthHeader authHeader, - ImmutableList updaters, - Map executors, + ImmutableList remoteUpdaters, + Map remoteExecutors, TimelockNamespaces localUpdater) { - return new AllNodesDisabledNamespacesUpdater(authHeader, updaters, executors, localUpdater, UUID::randomUUID); + return new AllNodesDisabledNamespacesUpdater( + authHeader, remoteUpdaters, remoteExecutors, localUpdater, UUID::randomUUID); } /** From 0b7cb0ecc4b48e4089bb0dfd5709fffd2806948e Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 7 Feb 2022 13:51:15 +0000 Subject: [PATCH 14/18] javadoc --- .../atlasdb/backup/AtlasRestoreService.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java index 80a459a0aa4..c73c10525a0 100644 --- a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java +++ b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java @@ -99,6 +99,14 @@ public static AtlasRestoreService create( cassandraRepairHelper); } + /** + * Disables TimeLock on all nodes for the given namespaces. + * Should be called exactly once prior to a restore operation. Calling this on multiple nodes will cause conflicts. + * + * @param namespaces the namespaces to disable + * + * @return the result of the request, including a lock ID which must later be passed to completeRestore. + */ public DisableNamespacesResponse prepareRestore(Set namespaces) { Map completedBackups = getCompletedBackups(namespaces); Set namespacesToRepair = completedBackups.keySet(); @@ -146,6 +154,13 @@ public Set repairInternalTables( return namespacesToRepair; } + /** + * Completes the restore process for the requested namespaces. + * This includes fast-forwarding the timestamp, and then re-enabling the TimeLock namespaces. + * + * @param request the request object, which must include the lock ID returned by {@link #prepareRestore(Set)} + * @return the set of namespaces that were successfully fast-forwarded and re-enabled. + */ public Set completeRestore(ReenableNamespacesRequest request) { Set completedBackups = request.getNamespaces().stream() .map(backupPersister::getCompletedBackup) From bad7a50b76a44dd041be4ff5fedf760147de8114 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 7 Feb 2022 14:19:38 +0000 Subject: [PATCH 15/18] authenticate internal calls too --- .../atlasdb/backup/AtlasRestoreService.java | 13 ++++++ ...NodesDisabledNamespacesUpdaterFactory.java | 7 +--- .../timelock/paxos/TimeLockAgent.java | 11 ++--- .../AllNodesDisabledNamespacesUpdater.java | 37 +++++++++-------- .../DisabledNamespacesUpdaterResource.java | 41 ++++++++++++++++--- .../TimeLockManagementResource.java | 14 ++++--- ...AllNodesDisabledNamespacesUpdaterTest.java | 31 +++++++------- .../TimeLockManagementResourceTest.java | 4 +- 8 files changed, 102 insertions(+), 56 deletions(-) diff --git a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java index c73c10525a0..0794c96742f 100644 --- a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java +++ b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java @@ -19,6 +19,7 @@ import com.datastax.driver.core.policies.DefaultRetryPolicy; import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Sets; import com.palantir.atlasdb.backup.api.AtlasRestoreClientBlocking; import com.palantir.atlasdb.backup.api.CompleteRestoreRequest; import com.palantir.atlasdb.backup.api.CompleteRestoreResponse; @@ -178,10 +179,22 @@ public Set completeRestore(ReenableNamespacesRequest request) { CompleteRestoreResponse response = atlasRestoreClientBlocking.completeRestore(authHeader, CompleteRestoreRequest.of(completedBackups)); Set successfulNamespaces = response.getSuccessfulNamespaces(); + Set failedNamespaces = Sets.difference(request.getNamespaces(), successfulNamespaces); + if (!failedNamespaces.isEmpty()) { + log.error( + "Failed to fast-forward timestamp for some namespaces. These will not be re-enabled.", + SafeArg.of("failedNamespaces", failedNamespaces), + SafeArg.of("fastForwardedNamespaces", successfulNamespaces)); + } // Re-enable timelock timeLockManagementService.reenableTimelock( authHeader, ReenableNamespacesRequest.of(successfulNamespaces, request.getLockId())); + if (successfulNamespaces.containsAll(request.getNamespaces())) { + log.info( + "Successfully completed restore for all namespaces", + SafeArg.of("namespaces", successfulNamespaces)); + } return successfulNamespaces; } diff --git a/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java b/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java index aa1b3ce7a25..8190ae01eb0 100644 --- a/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java +++ b/timelock-agent/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterFactory.java @@ -27,19 +27,16 @@ import com.palantir.atlasdb.util.MetricsManager; import com.palantir.common.concurrent.CheckedRejectionExecutorService; import com.palantir.common.streams.KeyedStream; -import com.palantir.tokens.auth.AuthHeader; import java.util.List; import java.util.Map; import java.util.stream.Collectors; public final class AllNodesDisabledNamespacesUpdaterFactory { - private final AuthHeader authHeader; private final TimelockPaxosInstallationContext install; private final MetricsManager metricsManager; public AllNodesDisabledNamespacesUpdaterFactory( - AuthHeader authHeader, TimelockPaxosInstallationContext install, MetricsManager metricsManager) { - this.authHeader = authHeader; + TimelockPaxosInstallationContext install, MetricsManager metricsManager) { this.install = install; this.metricsManager = metricsManager; } @@ -60,6 +57,6 @@ public AllNodesDisabledNamespacesUpdater create(TimelockNamespaces localNamespac ImmutableList remoteServices = ImmutableList.copyOf( remoteUpdaters.stream().map(WithDedicatedExecutor::service).collect(Collectors.toList())); - return AllNodesDisabledNamespacesUpdater.create(authHeader, remoteServices, remoteExecutors, localNamespaces); + return AllNodesDisabledNamespacesUpdater.create(remoteServices, remoteExecutors, localNamespaces); } } diff --git a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java index b464deeba3f..e34cec14ae6 100644 --- a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java +++ b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java @@ -99,7 +99,6 @@ import com.palantir.timelock.store.PersistenceConfigStore; import com.palantir.timelock.store.SqliteBlobStore; import com.palantir.timestamp.ManagedTimestampService; -import com.palantir.tokens.auth.AuthHeader; import com.palantir.tokens.auth.BearerToken; import com.zaxxer.hikari.HikariDataSource; import java.net.URL; @@ -183,8 +182,8 @@ public static TimeLockAgent create( metricsManager, Suppliers.compose(TimeLockRuntimeConfiguration::paxos, restrictedRuntime::get)); - AllNodesDisabledNamespacesUpdaterFactory updaterFactory = new AllNodesDisabledNamespacesUpdaterFactory( - AuthHeader.valueOf("WIRE_ME_UP_SCOTTY"), installationContext, metricsManager); + AllNodesDisabledNamespacesUpdaterFactory updaterFactory = + new AllNodesDisabledNamespacesUpdaterFactory(installationContext, metricsManager); TimeLockAgent agent = new TimeLockAgent( metricsManager, @@ -353,6 +352,7 @@ private void createAndRegisterResources() { Refreshable> permittedBackupToken = runtime.map(TimeLockRuntimeConfiguration::permittedBackupToken); + AuthHeaderValidator authHeaderValidator = new AuthHeaderValidator(permittedBackupToken); RedirectRetryTargeter redirectRetryTargeter = redirectRetryTargeter(); if (undertowRegistrar.isPresent()) { Consumer presentUndertowRegistrar = undertowRegistrar.get(); @@ -380,7 +380,7 @@ private void createAndRegisterResources() { permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); registerCorruptionHandlerWrappedService( presentUndertowRegistrar, - DisabledNamespacesUpdaterResource.undertow(redirectRetryTargeter, namespaces)); + DisabledNamespacesUpdaterResource.undertow(authHeaderValidator, redirectRetryTargeter, namespaces)); } else { registrar.accept(ConjureTimelockResource.jersey(redirectRetryTargeter, asyncTimelockServiceGetter)); registrar.accept(ConjureLockWatchingResource.jersey(redirectRetryTargeter, asyncTimelockServiceGetter)); @@ -392,7 +392,8 @@ private void createAndRegisterResources() { permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); registrar.accept(AtlasRestoreResource.jersey( permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); - registrar.accept(DisabledNamespacesUpdaterResource.jersey(redirectRetryTargeter, namespaces)); + registrar.accept( + DisabledNamespacesUpdaterResource.jersey(authHeaderValidator, redirectRetryTargeter, namespaces)); } } diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdater.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdater.java index 6945deec647..8a2c2cc8e49 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdater.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdater.java @@ -55,7 +55,6 @@ public class AllNodesDisabledNamespacesUpdater { private static final SafeLogger log = SafeLoggerFactory.get(AllNodesDisabledNamespacesUpdater.class); - private final AuthHeader authHeader; private final ImmutableList remoteUpdaters; private final Map remoteExecutors; private final TimelockNamespaces localUpdater; @@ -64,12 +63,10 @@ public class AllNodesDisabledNamespacesUpdater { @VisibleForTesting AllNodesDisabledNamespacesUpdater( - AuthHeader authHeader, ImmutableList remoteUpdaters, Map remoteExecutors, TimelockNamespaces localUpdater, Supplier lockIdSupplier) { - this.authHeader = authHeader; this.remoteUpdaters = remoteUpdaters; this.remoteExecutors = remoteExecutors; this.localUpdater = localUpdater; @@ -78,12 +75,10 @@ public class AllNodesDisabledNamespacesUpdater { } public static AllNodesDisabledNamespacesUpdater create( - AuthHeader authHeader, ImmutableList remoteUpdaters, Map remoteExecutors, TimelockNamespaces localUpdater) { - return new AllNodesDisabledNamespacesUpdater( - authHeader, remoteUpdaters, remoteExecutors, localUpdater, UUID::randomUUID); + return new AllNodesDisabledNamespacesUpdater(remoteUpdaters, remoteExecutors, localUpdater, UUID::randomUUID); } /** @@ -101,13 +96,13 @@ public static AllNodesDisabledNamespacesUpdater create( * If we fail to roll back for some node (e.g. it becomes unreachable), then we're left in an inconsistent state * which will need manual remediation. */ - public DisableNamespacesResponse disableOnAllNodes(Set namespaces) { - if (anyNodeIsUnreachable()) { + public DisableNamespacesResponse disableOnAllNodes(AuthHeader authHeader, Set namespaces) { + if (anyNodeIsUnreachable(authHeader)) { return DisableNamespaceResponses.unsuccessfulDueToPingFailure(namespaces); } UUID lockId = lockIdSupplier.get(); - AllNodesUpdateResponse responses = disableNamespacesOnAllNodes(namespaces, lockId); + AllNodesUpdateResponse responses = disableNamespacesOnAllNodes(authHeader, namespaces, lockId); if (updateWasSuccessfulOnAllNodes(responses.allResponses())) { return DisableNamespacesResponse.successful(SuccessfulDisableNamespacesResponse.of(lockId)); } @@ -129,7 +124,7 @@ public DisableNamespacesResponse disableOnAllNodes(Set namespaces) { || remoteResponses.get(node).isSuccessful()) .collect(Collectors.toList()); - boolean rollbackSuccess = attemptReEnableOnNodes(namespaces, lockId, successfulOrUnreachableNodes); + boolean rollbackSuccess = attemptReEnableOnNodes(authHeader, namespaces, lockId, successfulOrUnreachableNodes); if (rollbackSuccess) { return DisableNamespaceResponses.unsuccessfulButRolledBack(namespaces, lockId); } @@ -142,9 +137,9 @@ public DisableNamespacesResponse disableOnAllNodes(Set namespaces) { * If some namespaces are locked with a different lock ID, then these namespaces will remain disabled, however * we will still re-enable those that were locked with the provided lock ID. */ - public ReenableNamespacesResponse reEnableOnAllNodes(ReenableNamespacesRequest request) { + public ReenableNamespacesResponse reEnableOnAllNodes(AuthHeader authHeader, ReenableNamespacesRequest request) { List responses = - reEnableNamespacesOnAllNodes(request).allResponses(); + reEnableNamespacesOnAllNodes(authHeader, request).allResponses(); if (updateWasSuccessfulOnAllNodes(responses)) { return ReenableNamespacesResponse.successful(SuccessfulReenableNamespacesResponse.of(true)); } @@ -160,7 +155,7 @@ public ReenableNamespacesResponse reEnableOnAllNodes(ReenableNamespacesRequest r } // Ping - private boolean anyNodeIsUnreachable() { + private boolean anyNodeIsUnreachable(AuthHeader authHeader) { return !isSuccessfulOnAllRemoteNodes(service -> new BooleanPaxosResponse(service.ping(authHeader))); } @@ -171,7 +166,8 @@ private boolean isSuccessfulOnAllRemoteNodes(Function namespaces, UUID lockId) { + private AllNodesUpdateResponse disableNamespacesOnAllNodes( + AuthHeader authHeader, Set namespaces, UUID lockId) { DisableNamespacesRequest request = DisableNamespacesRequest.of(namespaces, lockId); Function update = service -> service.disable(authHeader, request); @@ -182,16 +178,21 @@ private AllNodesUpdateResponse disableNamespacesOnAllNodes(Set namesp // ReEnable private boolean attemptReEnableOnNodes( - Set namespaces, UUID lockId, List successfulNodes) { + AuthHeader authHeader, + Set namespaces, + UUID lockId, + List successfulNodes) { ReenableNamespacesRequest request = ReenableNamespacesRequest.builder() .namespaces(namespaces) .lockId(lockId) .build(); - Collection responses = reEnableNamespacesOnRemoteNodes(request, successfulNodes); + Collection responses = + reEnableNamespacesOnRemoteNodes(authHeader, request, successfulNodes); return responses.stream().filter(SingleNodeUpdateResponse::isSuccessful).count() == successfulNodes.size(); } - private AllNodesUpdateResponse reEnableNamespacesOnAllNodes(ReenableNamespacesRequest request) { + private AllNodesUpdateResponse reEnableNamespacesOnAllNodes( + AuthHeader authHeader, ReenableNamespacesRequest request) { Set namespaces = request.getNamespaces(); UUID lockId = request.getLockId(); Function update = @@ -202,7 +203,7 @@ private AllNodesUpdateResponse reEnableNamespacesOnAllNodes(ReenableNamespacesRe } private Collection reEnableNamespacesOnRemoteNodes( - ReenableNamespacesRequest request, List nodes) { + AuthHeader authHeader, ReenableNamespacesRequest request, List nodes) { Function update = node -> node.reenable(authHeader, request); diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/DisabledNamespacesUpdaterResource.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/DisabledNamespacesUpdaterResource.java index f2e960ff725..f8e05753d72 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/DisabledNamespacesUpdaterResource.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/DisabledNamespacesUpdaterResource.java @@ -18,6 +18,7 @@ import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.palantir.atlasdb.backup.AuthHeaderValidator; import com.palantir.atlasdb.futures.AtlasFutures; import com.palantir.atlasdb.http.RedirectRetryTargeter; import com.palantir.atlasdb.timelock.ConjureResourceExceptionHandler; @@ -28,29 +29,45 @@ import com.palantir.atlasdb.timelock.api.ReenableNamespacesRequest; import com.palantir.atlasdb.timelock.api.SingleNodeUpdateResponse; import com.palantir.atlasdb.timelock.api.UndertowDisabledNamespacesUpdaterService; +import com.palantir.conjure.java.api.errors.ErrorType; +import com.palantir.conjure.java.api.errors.ServiceException; import com.palantir.conjure.java.undertow.lib.UndertowService; +import com.palantir.logsafe.SafeArg; +import com.palantir.logsafe.logger.SafeLogger; +import com.palantir.logsafe.logger.SafeLoggerFactory; import com.palantir.tokens.auth.AuthHeader; import java.util.function.Supplier; public final class DisabledNamespacesUpdaterResource implements UndertowDisabledNamespacesUpdaterService { + private static final SafeLogger log = SafeLoggerFactory.get(DisabledNamespacesUpdaterResource.class); + + private final AuthHeaderValidator authHeaderValidator; private final ConjureResourceExceptionHandler exceptionHandler; private final TimelockNamespaces timelockNamespaces; private DisabledNamespacesUpdaterResource( - RedirectRetryTargeter redirectRetryTargeter, TimelockNamespaces timelockNamespaces) { + AuthHeaderValidator authHeaderValidator, + RedirectRetryTargeter redirectRetryTargeter, + TimelockNamespaces timelockNamespaces) { + this.authHeaderValidator = authHeaderValidator; this.exceptionHandler = new ConjureResourceExceptionHandler(redirectRetryTargeter); this.timelockNamespaces = timelockNamespaces; } public static UndertowService undertow( - RedirectRetryTargeter redirectRetryTargeter, TimelockNamespaces timelockNamespaces) { + AuthHeaderValidator authHeaderValidator, + RedirectRetryTargeter redirectRetryTargeter, + TimelockNamespaces timelockNamespaces) { return DisabledNamespacesUpdaterServiceEndpoints.of( - new DisabledNamespacesUpdaterResource(redirectRetryTargeter, timelockNamespaces)); + new DisabledNamespacesUpdaterResource(authHeaderValidator, redirectRetryTargeter, timelockNamespaces)); } public static DisabledNamespacesUpdaterService jersey( - RedirectRetryTargeter redirectRetryTargeter, TimelockNamespaces timelockNamespaces) { - return new JerseyAdapter(new DisabledNamespacesUpdaterResource(redirectRetryTargeter, timelockNamespaces)); + AuthHeaderValidator authHeaderValidator, + RedirectRetryTargeter redirectRetryTargeter, + TimelockNamespaces timelockNamespaces) { + return new JerseyAdapter( + new DisabledNamespacesUpdaterResource(authHeaderValidator, redirectRetryTargeter, timelockNamespaces)); } @Override @@ -60,12 +77,26 @@ public ListenableFuture ping(AuthHeader _authHeader) { @Override public ListenableFuture disable(AuthHeader authHeader, DisableNamespacesRequest request) { + if (!authHeaderValidator.suppliedTokenIsValid(authHeader)) { + log.error( + "Attempted to disable TimeLock with an invalid auth header. " + + "The provided token must match the configured permitted-backup-token.", + SafeArg.of("request", request)); + throw new ServiceException(ErrorType.PERMISSION_DENIED); + } return handleExceptions(() -> Futures.immediateFuture(timelockNamespaces.disable(request))); } @Override public ListenableFuture reenable( AuthHeader authHeader, ReenableNamespacesRequest request) { + if (!authHeaderValidator.suppliedTokenIsValid(authHeader)) { + log.error( + "Attempted to re-enable TimeLock with an invalid auth header. " + + "The provided token must match the configured permitted-backup-token.", + SafeArg.of("request", request)); + throw new ServiceException(ErrorType.PERMISSION_DENIED); + } return handleExceptions(() -> Futures.immediateFuture(timelockNamespaces.reEnable(request))); } diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java index bfc00fb5ba0..1c1452117d4 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResource.java @@ -158,11 +158,12 @@ public ListenableFuture disableTimelock( SafeArg.of("namespaces", namespaces)); throw new ServiceException(ErrorType.PERMISSION_DENIED); } - return handleExceptions(() -> disableInternal(namespaces)); + return handleExceptions(() -> disableInternal(authHeader, namespaces)); } - private ListenableFuture disableInternal(Set namespaces) { - return Futures.immediateFuture(allNodesDisabledNamespacesUpdater.disableOnAllNodes(namespaces)); + private ListenableFuture disableInternal( + AuthHeader authHeader, Set namespaces) { + return Futures.immediateFuture(allNodesDisabledNamespacesUpdater.disableOnAllNodes(authHeader, namespaces)); } @Override @@ -175,11 +176,12 @@ public ListenableFuture reenableTimelock( SafeArg.of("request", request)); throw new ServiceException(ErrorType.PERMISSION_DENIED); } - return handleExceptions(() -> reenableInternal(request)); + return handleExceptions(() -> reenableInternal(authHeader, request)); } - public ListenableFuture reenableInternal(ReenableNamespacesRequest request) { - return Futures.immediateFuture(allNodesDisabledNamespacesUpdater.reEnableOnAllNodes(request)); + public ListenableFuture reenableInternal( + AuthHeader authHeader, ReenableNamespacesRequest request) { + return Futures.immediateFuture(allNodesDisabledNamespacesUpdater.reEnableOnAllNodes(authHeader, request)); } @Override diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterTest.java index 909cb016c17..7a47140eb99 100644 --- a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterTest.java +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/AllNodesDisabledNamespacesUpdaterTest.java @@ -95,7 +95,7 @@ remote1, new CheckedRejectionExecutorService(Executors.newSingleThreadExecutor() when(remote2.reenable(any(), any())).thenReturn(SUCCESSFUL_SINGLE_NODE_UPDATE); when(localUpdater.reEnable(any())).thenReturn(SUCCESSFUL_SINGLE_NODE_UPDATE); - updater = new AllNodesDisabledNamespacesUpdater(AUTH_HEADER, remotes, executors, localUpdater, () -> LOCK_ID); + updater = new AllNodesDisabledNamespacesUpdater(remotes, executors, localUpdater, () -> LOCK_ID); } @Test @@ -104,7 +104,7 @@ public void canDisableSingleNamespace() { when(remote2.disable(any(), any())).thenReturn(SUCCESSFUL_SINGLE_NODE_UPDATE); when(localUpdater.disable(any(DisableNamespacesRequest.class))).thenReturn(SUCCESSFUL_SINGLE_NODE_UPDATE); - DisableNamespacesResponse response = updater.disableOnAllNodes(ImmutableSet.of(NAMESPACE)); + DisableNamespacesResponse response = updater.disableOnAllNodes(AUTH_HEADER, ImmutableSet.of(NAMESPACE)); assertThat(response).isEqualTo(successfulDisableResponse()); } @@ -113,7 +113,7 @@ public void canDisableSingleNamespace() { public void doesNotDisableIfPingFailsOnOneNode() { when(remote2.ping(any())).thenThrow(new SafeRuntimeException("unreachable")); - DisableNamespacesResponse response = updater.disableOnAllNodes(ImmutableSet.of(NAMESPACE)); + DisableNamespacesResponse response = updater.disableOnAllNodes(AUTH_HEADER, ImmutableSet.of(NAMESPACE)); assertThat(response).isEqualTo(DISABLE_FAILED_SUCCESSFULLY); verify(remote1, never()).disable(any(), any()); @@ -132,7 +132,7 @@ public void rollsBackDisabledNamespacesAfterPartialFailure() { when(localUpdater.getNamespacesLockedWithDifferentLockId(any(), any())).thenReturn(ImmutableMap.of()); - DisableNamespacesResponse response = updater.disableOnAllNodes(BOTH_NAMESPACES); + DisableNamespacesResponse response = updater.disableOnAllNodes(AUTH_HEADER, BOTH_NAMESPACES); ReenableNamespacesRequest rollbackRequest = ReenableNamespacesRequest.of(BOTH_NAMESPACES, LOCK_ID); verify(remote1).reenable(AUTH_HEADER, rollbackRequest); @@ -151,7 +151,7 @@ public void rollsBackIfLocalUpdateFails() { when(localUpdater.disable(DisableNamespacesRequest.of(BOTH_NAMESPACES, LOCK_ID))) .thenReturn(singleNodeUpdateFailure(failedNamespaces)); - DisableNamespacesResponse response = updater.disableOnAllNodes(BOTH_NAMESPACES); + DisableNamespacesResponse response = updater.disableOnAllNodes(AUTH_HEADER, BOTH_NAMESPACES); assertThat(response).isEqualTo(partiallyDisabled(BOTH_NAMESPACES)); ReenableNamespacesRequest rollbackRequest = ReenableNamespacesRequest.of(BOTH_NAMESPACES, LOCK_ID); @@ -176,7 +176,7 @@ public void doesNotDisableIfSomeNamespaceAlreadyDisabled() { when(localUpdater.getNamespacesLockedWithDifferentLockId(any(), any())) .thenReturn(unsuccessfulResponse.lockedNamespaces()); - DisableNamespacesResponse response = updater.disableOnAllNodes(BOTH_NAMESPACES); + DisableNamespacesResponse response = updater.disableOnAllNodes(AUTH_HEADER, BOTH_NAMESPACES); assertThat(response).isEqualTo(consistentlyDisabled(disabledNamespaces)); @@ -202,7 +202,8 @@ public void rollsBackDisableIfInconsistentStateIsFound() { when(localUpdater.getNamespacesLockedWithDifferentLockId(any(), any())).thenReturn(ImmutableMap.of()); - DisableNamespacesResponse response = updater.disableOnAllNodes(ImmutableSet.of(NAMESPACE, OTHER_NAMESPACE)); + DisableNamespacesResponse response = + updater.disableOnAllNodes(AUTH_HEADER, ImmutableSet.of(NAMESPACE, OTHER_NAMESPACE)); ReenableNamespacesRequest rollbackRequest = ReenableNamespacesRequest.of(BOTH_NAMESPACES, LOCK_ID); verify(remote1, never()).reenable(any(), any()); @@ -232,7 +233,7 @@ public void disableDoesNotReportConsistentStateWhenNamespacesAreLockedWithDiffer when(localUpdater.getNamespacesLockedWithDifferentLockId(any(), any())) .thenReturn(lockedWithYetAnotherLock.lockedNamespaces()); - DisableNamespacesResponse response = updater.disableOnAllNodes(namespaces); + DisableNamespacesResponse response = updater.disableOnAllNodes(AUTH_HEADER, namespaces); verify(remote1, never()).reenable(any(), any()); verify(remote2).reenable(any(), any()); assertThat(response).isEqualTo(partiallyDisabled(namespaces)); @@ -258,7 +259,7 @@ public void reEnableDoesNotReportConsistentStateWhenNamespacesAreLockedWithDiffe when(localUpdater.reEnable(any())).thenReturn(lockedWithYetAnotherLock); ReenableNamespacesResponse reenableResponse = - updater.reEnableOnAllNodes(ReenableNamespacesRequest.of(namespaces, LOCK_ID)); + updater.reEnableOnAllNodes(AUTH_HEADER, ReenableNamespacesRequest.of(namespaces, LOCK_ID)); assertThat(reenableResponse).isEqualTo(partiallyLocked(namespaces)); verify(remote1, never()).disable(any(), any()); verify(remote2, never()).disable(any(), any()); @@ -275,7 +276,7 @@ public void reportsRollbackFailures() { when(remote1.reenable(any(), any())).thenReturn(singleNodeUpdateFailure(failedNamespaces)); - DisableNamespacesResponse response = updater.disableOnAllNodes(BOTH_NAMESPACES); + DisableNamespacesResponse response = updater.disableOnAllNodes(AUTH_HEADER, BOTH_NAMESPACES); verify(remote1).reenable(any(), any()); verify(remote2, never()).reenable(any(), any()); @@ -293,7 +294,7 @@ public void handlesNodesBecomingUnreachableDuringDisable() { when(localUpdater.getNamespacesLockedWithDifferentLockId(any(), any())).thenReturn(ImmutableMap.of()); - DisableNamespacesResponse response = updater.disableOnAllNodes(BOTH_NAMESPACES); + DisableNamespacesResponse response = updater.disableOnAllNodes(AUTH_HEADER, BOTH_NAMESPACES); // We don't know if the request succeeded or failed on remote2, so we should try our best to roll back verify(remote2).reenable(any(), any()); @@ -311,7 +312,7 @@ public void handlesNodesBecomingUnreachableDuringReEnable() { when(localUpdater.reEnable(any())).thenReturn(SUCCESSFUL_SINGLE_NODE_UPDATE); ReenableNamespacesRequest request = ReenableNamespacesRequest.of(BOTH_NAMESPACES, LOCK_ID); - ReenableNamespacesResponse response = updater.reEnableOnAllNodes(request); + ReenableNamespacesResponse response = updater.reEnableOnAllNodes(AUTH_HEADER, request); assertThat(response).isEqualTo(partiallyLocked(ImmutableSet.of())); } @@ -323,7 +324,7 @@ public void canReEnableSingleNamespace() { when(remote2.reenable(AUTH_HEADER, request)).thenReturn(SUCCESSFUL_SINGLE_NODE_UPDATE); when(localUpdater.reEnable(request)).thenReturn(SUCCESSFUL_SINGLE_NODE_UPDATE); - ReenableNamespacesResponse response = updater.reEnableOnAllNodes(request); + ReenableNamespacesResponse response = updater.reEnableOnAllNodes(AUTH_HEADER, request); assertThat(response).isEqualTo(REENABLED_SUCCESSFULLY); } @@ -337,7 +338,7 @@ public void reEnableCanPartiallyFail() { when(remote2.reenable(AUTH_HEADER, request)).thenReturn(singleNodeUpdateFailure(oneNamespace)); when(localUpdater.reEnable(request)).thenReturn(SingleNodeUpdateResponse.successful()); - ReenableNamespacesResponse response = updater.reEnableOnAllNodes(request); + ReenableNamespacesResponse response = updater.reEnableOnAllNodes(AUTH_HEADER, request); assertThat(response).isEqualTo(partiallyLocked(oneNamespace)); // verify we still unlocked locally verify(localUpdater).reEnable(request); @@ -354,7 +355,7 @@ public void doesNotReEnableIfSomeNamespaceDisabledWithOtherLock() { when(localUpdater.reEnable(any())).thenReturn(unsuccessfulResponse); ReenableNamespacesResponse response = - updater.reEnableOnAllNodes(ReenableNamespacesRequest.of(BOTH_NAMESPACES, LOCK_ID)); + updater.reEnableOnAllNodes(AUTH_HEADER, ReenableNamespacesRequest.of(BOTH_NAMESPACES, LOCK_ID)); assertThat(response).isEqualTo(consistentlyLocked(disabledNamespaces)); } diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java index 413a4be6978..e3598e0b1e6 100644 --- a/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/timelock/management/TimeLockManagementResourceTest.java @@ -126,7 +126,7 @@ public void disableTimeLockThrowsIfAuthHeaderIsWrong() { @Test public void disableTimeLockCallsUpdater() { timeLockManagementResource.disableTimelock(AUTH_HEADER, NAMESPACES); - verify(allNodesDisabledNamespacesUpdater).disableOnAllNodes(NAMESPACES); + verify(allNodesDisabledNamespacesUpdater).disableOnAllNodes(AUTH_HEADER, NAMESPACES); } @Test @@ -142,7 +142,7 @@ public void reEnableTimeLockThrowsIfAuthHeaderIsWrong() { public void reEnableTimeLockCallsUpdater() { ReenableNamespacesRequest request = ReenableNamespacesRequest.of(NAMESPACES, UUID.randomUUID()); timeLockManagementResource.reenableTimelock(AUTH_HEADER, request); - verify(allNodesDisabledNamespacesUpdater).reEnableOnAllNodes(request); + verify(allNodesDisabledNamespacesUpdater).reEnableOnAllNodes(AUTH_HEADER, request); } @Test From d136888a536d8ae1ffb374facdc32480f785bb63 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Mon, 7 Feb 2022 14:47:58 +0000 Subject: [PATCH 16/18] refactor usages of AuthHeaderValidator --- .../timelock/paxos/TimeLockAgent.java | 27 +++++----- .../atlasdb/backup/AtlasBackupResource.java | 26 ++++------ .../atlasdb/backup/AtlasRestoreResource.java | 13 +++-- .../backup/AtlasBackupResourceTest.java | 28 +++++------ .../backup/AtlasRestoreResourceTest.java | 32 ++++++------ .../backup/AuthHeaderValidatorTest.java | 50 +++++++++++++++++++ 6 files changed, 109 insertions(+), 67 deletions(-) create mode 100644 timelock-impl/src/test/java/com/palantir/atlasdb/backup/AuthHeaderValidatorTest.java diff --git a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java index e34cec14ae6..d22b610cf0c 100644 --- a/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java +++ b/timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java @@ -350,9 +350,7 @@ private void createAndRegisterResources() { Function lockServiceGetter = namespace -> namespaces.get(namespace).getLockService(); - Refreshable> permittedBackupToken = - runtime.map(TimeLockRuntimeConfiguration::permittedBackupToken); - AuthHeaderValidator authHeaderValidator = new AuthHeaderValidator(permittedBackupToken); + AuthHeaderValidator authHeaderValidator = getAuthHeaderValidator(); RedirectRetryTargeter redirectRetryTargeter = redirectRetryTargeter(); if (undertowRegistrar.isPresent()) { Consumer presentUndertowRegistrar = undertowRegistrar.get(); @@ -373,11 +371,11 @@ private void createAndRegisterResources() { registerCorruptionHandlerWrappedService( presentUndertowRegistrar, AtlasBackupResource.undertow( - permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); + authHeaderValidator, redirectRetryTargeter, asyncTimelockServiceGetter)); registerCorruptionHandlerWrappedService( presentUndertowRegistrar, AtlasRestoreResource.undertow( - permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); + authHeaderValidator, redirectRetryTargeter, asyncTimelockServiceGetter)); registerCorruptionHandlerWrappedService( presentUndertowRegistrar, DisabledNamespacesUpdaterResource.undertow(authHeaderValidator, redirectRetryTargeter, namespaces)); @@ -388,10 +386,10 @@ private void createAndRegisterResources() { registrar.accept(TimeLockPaxosHistoryProviderResource.jersey(corruptionComponents.localHistoryLoader())); registrar.accept( MultiClientConjureTimelockResource.jersey(redirectRetryTargeter, asyncTimelockServiceGetter)); - registrar.accept(AtlasBackupResource.jersey( - permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); + registrar.accept( + AtlasBackupResource.jersey(authHeaderValidator, redirectRetryTargeter, asyncTimelockServiceGetter)); registrar.accept(AtlasRestoreResource.jersey( - permittedBackupToken, redirectRetryTargeter, asyncTimelockServiceGetter)); + authHeaderValidator, redirectRetryTargeter, asyncTimelockServiceGetter)); registrar.accept( DisabledNamespacesUpdaterResource.jersey(authHeaderValidator, redirectRetryTargeter, namespaces)); } @@ -431,9 +429,6 @@ private void registerManagementResource() { ServiceLifecycleController serviceLifecycleController = new ServiceLifecycleController(serviceStopper, PTExecutors.newSingleThreadScheduledExecutor()); AllNodesDisabledNamespacesUpdater allNodesDisabledNamespacesUpdater = updaterFactory.create(namespaces); - Refreshable> permittedBackupToken = - runtime.map(TimeLockRuntimeConfiguration::permittedBackupToken); - AuthHeaderValidator authHeaderValidator = new AuthHeaderValidator(permittedBackupToken); if (undertowRegistrar.isPresent()) { registerCorruptionHandlerWrappedService( @@ -442,7 +437,7 @@ private void registerManagementResource() { timestampStorage.persistentNamespaceContext(), namespaces, allNodesDisabledNamespacesUpdater, - authHeaderValidator, + getAuthHeaderValidator(), redirectRetryTargeter(), serviceLifecycleController)); } else { @@ -450,12 +445,18 @@ private void registerManagementResource() { timestampStorage.persistentNamespaceContext(), namespaces, allNodesDisabledNamespacesUpdater, - authHeaderValidator, + getAuthHeaderValidator(), redirectRetryTargeter(), serviceLifecycleController)); } } + private AuthHeaderValidator getAuthHeaderValidator() { + Refreshable> permittedBackupToken = + runtime.map(TimeLockRuntimeConfiguration::permittedBackupToken); + return new AuthHeaderValidator(permittedBackupToken); + } + private void registerTimeLockCorruptionJerseyFilter() { if (!undertowRegistrar.isPresent()) { registrar.accept(new JerseyCorruptionFilter(corruptionComponents.timeLockCorruptionHealthCheck())); diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasBackupResource.java b/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasBackupResource.java index 660a0168731..98b1bb69182 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasBackupResource.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasBackupResource.java @@ -45,7 +45,6 @@ import com.palantir.logsafe.logger.SafeLogger; import com.palantir.logsafe.logger.SafeLoggerFactory; import com.palantir.tokens.auth.AuthHeader; -import com.palantir.tokens.auth.BearerToken; import java.util.Map; import java.util.Optional; import java.util.Set; @@ -56,34 +55,34 @@ public class AtlasBackupResource implements UndertowAtlasBackupClient { private static final SafeLogger log = SafeLoggerFactory.get(AtlasBackupResource.class); - private final Supplier> permittedBackupToken; + private final AuthHeaderValidator authHeaderValidator; private final Function timelockServices; private final ConjureResourceExceptionHandler exceptionHandler; @VisibleForTesting AtlasBackupResource( - Supplier> permittedBackupToken, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, Function timelockServices) { - this.permittedBackupToken = permittedBackupToken; + this.authHeaderValidator = authHeaderValidator; this.exceptionHandler = new ConjureResourceExceptionHandler(redirectRetryTargeter); this.timelockServices = timelockServices; } public static UndertowService undertow( - Supplier> permittedAuthHeader, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, Function timelockServices) { return AtlasBackupClientEndpoints.of( - new AtlasBackupResource(permittedAuthHeader, redirectRetryTargeter, timelockServices)); + new AtlasBackupResource(authHeaderValidator, redirectRetryTargeter, timelockServices)); } public static AtlasBackupClient jersey( - Supplier> permittedAuthHeader, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, Function timelockServices) { return new JerseyAtlasBackupClientAdapter( - new AtlasBackupResource(permittedAuthHeader, redirectRetryTargeter, timelockServices)); + new AtlasBackupResource(authHeaderValidator, redirectRetryTargeter, timelockServices)); } @Override @@ -92,7 +91,7 @@ public ListenableFuture prepareBackup(AuthHeader authHead } private PrepareBackupResponse prepareBackupInternal(AuthHeader authHeader, PrepareBackupRequest request) { - if (!suppliedTokenIsValid(authHeader)) { + if (!authHeaderValidator.suppliedTokenIsValid(authHeader)) { log.error( "Attempted to prepare backup with an invalid auth header. " + "The provided token must match the configured permitted-backup-token.", @@ -127,7 +126,7 @@ public ListenableFuture completeBackup( @SuppressWarnings("ConstantConditions") private ListenableFuture completeBackupInternal( AuthHeader authHeader, CompleteBackupRequest request) { - if (!suppliedTokenIsValid(authHeader)) { + if (!authHeaderValidator.suppliedTokenIsValid(authHeader)) { log.error( "Attempted to complete backup with an invalid auth header. " + "The provided token must match the configured permitted-backup-token.", @@ -173,13 +172,6 @@ private CompletedBackup fetchFastForwardTimestamp(InProgressBackupToken backupTo .build(); } - private Boolean suppliedTokenIsValid(AuthHeader suppliedAuthHeader) { - return permittedBackupToken - .get() - .map(token -> token.equals(suppliedAuthHeader.getBearerToken())) - .orElse(false); - } - private AsyncTimelockService timelock(Namespace namespace) { return timelockServices.apply(namespace.get()); } diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreResource.java b/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreResource.java index dfe0740e3e4..6397e6b0204 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreResource.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreResource.java @@ -39,7 +39,6 @@ import com.palantir.logsafe.logger.SafeLogger; import com.palantir.logsafe.logger.SafeLoggerFactory; import com.palantir.tokens.auth.AuthHeader; -import com.palantir.tokens.auth.BearerToken; import java.util.Map; import java.util.Optional; import java.util.function.Function; @@ -55,28 +54,28 @@ public class AtlasRestoreResource implements UndertowAtlasRestoreClient { @VisibleForTesting AtlasRestoreResource( - Supplier> permittedToken, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, Function timelockServices) { - this.authHeaderValidator = new AuthHeaderValidator(permittedToken); + this.authHeaderValidator = authHeaderValidator; this.exceptionHandler = new ConjureResourceExceptionHandler(redirectRetryTargeter); this.timelockServices = timelockServices; } public static UndertowService undertow( - Supplier> permittedToken, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, Function timelockServices) { return AtlasRestoreClientEndpoints.of( - new AtlasRestoreResource(permittedToken, redirectRetryTargeter, timelockServices)); + new AtlasRestoreResource(authHeaderValidator, redirectRetryTargeter, timelockServices)); } public static AtlasRestoreClient jersey( - Supplier> permittedToken, + AuthHeaderValidator authHeaderValidator, RedirectRetryTargeter redirectRetryTargeter, Function timelockServices) { return new JerseyAtlasRestoreClientAdapter( - new AtlasRestoreResource(permittedToken, redirectRetryTargeter, timelockServices)); + new AtlasRestoreResource(authHeaderValidator, redirectRetryTargeter, timelockServices)); } @Override diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AtlasBackupResourceTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AtlasBackupResourceTest.java index 949d97154d4..29a28d687b0 100644 --- a/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AtlasBackupResourceTest.java +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AtlasBackupResourceTest.java @@ -20,6 +20,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableSet; @@ -42,9 +43,9 @@ import com.palantir.tokens.auth.BearerToken; import java.net.URL; import java.util.List; -import java.util.Optional; import java.util.Set; import java.util.UUID; +import org.junit.Before; import org.junit.Test; public class AtlasBackupResourceTest { @@ -66,11 +67,19 @@ public class AtlasBackupResourceTest { private static final CompleteBackupResponse EMPTY_COMPLETE_BACKUP_RESPONSE = CompleteBackupResponse.of(ImmutableSet.of()); + private final AuthHeaderValidator authHeaderValidator = mock(AuthHeaderValidator.class); + private final AsyncTimelockService mockTimelock = mock(AsyncTimelockService.class); private final AsyncTimelockService otherTimelock = mock(AsyncTimelockService.class); private final AtlasBackupResource atlasBackupService = new AtlasBackupResource( - () -> Optional.of(BEARER_TOKEN), TARGETER, str -> str.equals("test") ? mockTimelock : otherTimelock); + authHeaderValidator, TARGETER, str -> str.equals("test") ? mockTimelock : otherTimelock); + + @Before + public void setUp() { + when(authHeaderValidator.suppliedTokenIsValid(AUTH_HEADER)).thenReturn(true); + when(authHeaderValidator.suppliedTokenIsValid(WRONG_AUTH_HEADER)).thenReturn(false); + } @Test public void prepareBackupThrowsIfAuthHeaderIsWrong() { @@ -86,19 +95,6 @@ WRONG_AUTH_HEADER, completeBackupRequest(validBackupToken())))) .hasType(ErrorType.PERMISSION_DENIED); } - @Test - public void emptyBearerTokenInConfigWillCauseBackupOperationsToFail() { - AtlasBackupResource emptyTokenResource = new AtlasBackupResource( - Optional::empty, TARGETER, str -> str.equals("test") ? mockTimelock : otherTimelock); - - assertThatServiceExceptionThrownBy(() -> AtlasFutures.getUnchecked( - emptyTokenResource.prepareBackup(AUTH_HEADER, PREPARE_BACKUP_REQUEST))) - .hasType(ErrorType.PERMISSION_DENIED); - assertThatServiceExceptionThrownBy(() -> AtlasFutures.getUnchecked( - emptyTokenResource.completeBackup(AUTH_HEADER, completeBackupRequest(validBackupToken())))) - .hasType(ErrorType.PERMISSION_DENIED); - } - @Test public void preparesBackupSuccessfully() { LockToken lockToken = lockToken(); @@ -110,6 +106,7 @@ public void preparesBackupSuccessfully() { assertThat(AtlasFutures.getUnchecked(atlasBackupService.prepareBackup(AUTH_HEADER, PREPARE_BACKUP_REQUEST))) .isEqualTo(prepareBackupResponseWith(expectedBackupToken)); + verify(authHeaderValidator).suppliedTokenIsValid(AUTH_HEADER); } @Test @@ -122,6 +119,7 @@ public void completeBackupContainsNamespaceWhenLockIsHeld() { assertThat(AtlasFutures.getUnchecked( atlasBackupService.completeBackup(AUTH_HEADER, completeBackupRequest(backupToken)))) .isEqualTo(completeBackupResponseWith(expected)); + verify(authHeaderValidator).suppliedTokenIsValid(AUTH_HEADER); } @Test diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreResourceTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreResourceTest.java index 8ad66721556..19d26011bfc 100644 --- a/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreResourceTest.java +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AtlasRestoreResourceTest.java @@ -18,7 +18,10 @@ import static com.palantir.conjure.java.api.testing.Assertions.assertThatServiceExceptionThrownBy; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.AdditionalMatchers.not; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableSet; import com.palantir.atlasdb.backup.api.CompleteRestoreRequest; @@ -34,7 +37,7 @@ import com.palantir.tokens.auth.BearerToken; import java.net.URL; import java.util.List; -import java.util.Optional; +import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; @@ -58,8 +61,19 @@ public class AtlasRestoreResourceTest { @Mock private AsyncTimelockService otherTimelock; - private final AtlasRestoreResource atlasRestoreResource = new AtlasRestoreResource( - () -> Optional.of(BEARER_TOKEN), TARGETER, str -> str.equals("test") ? mockTimelock : otherTimelock); + @Mock + private AuthHeaderValidator authHeaderValidator; + + private AtlasRestoreResource atlasRestoreResource; + + @Before + public void setUp() { + when(authHeaderValidator.suppliedTokenIsValid(AUTH_HEADER)).thenReturn(true); + when(authHeaderValidator.suppliedTokenIsValid(not(eq(AUTH_HEADER)))).thenReturn(false); + + atlasRestoreResource = new AtlasRestoreResource( + authHeaderValidator, TARGETER, str -> str.equals("test") ? mockTimelock : otherTimelock); + } @Test public void throwsIfWrongAuthHeaderIsProvided() { @@ -71,18 +85,6 @@ public void throwsIfWrongAuthHeaderIsProvided() { .hasType(ErrorType.PERMISSION_DENIED); } - @Test - public void emptyBearerTokenInConfigWillCauseRestoreOperationsToFail() { - AtlasRestoreResource emptyTokenResource = new AtlasRestoreResource( - Optional::empty, TARGETER, str -> str.equals("test") ? mockTimelock : otherTimelock); - - CompletedBackup completedBackup = completedBackup(); - CompleteRestoreRequest request = CompleteRestoreRequest.of(ImmutableSet.of(completedBackup)); - assertThatServiceExceptionThrownBy( - () -> AtlasFutures.getUnchecked(emptyTokenResource.completeRestore(AUTH_HEADER, request))) - .hasType(ErrorType.PERMISSION_DENIED); - } - @Test public void completesRestoreSuccessfully() { CompletedBackup completedBackup = completedBackup(); diff --git a/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AuthHeaderValidatorTest.java b/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AuthHeaderValidatorTest.java new file mode 100644 index 00000000000..f624922238b --- /dev/null +++ b/timelock-impl/src/test/java/com/palantir/atlasdb/backup/AuthHeaderValidatorTest.java @@ -0,0 +1,50 @@ +/* + * (c) Copyright 2022 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.atlasdb.backup; + +import static com.palantir.logsafe.testing.Assertions.assertThat; + +import com.palantir.tokens.auth.AuthHeader; +import com.palantir.tokens.auth.BearerToken; +import java.util.Optional; +import org.junit.Test; + +public class AuthHeaderValidatorTest { + private static final BearerToken BEARER_TOKEN = BearerToken.valueOf("bear"); + private static final BearerToken WRONG_TOKEN = BearerToken.valueOf("tiger"); + + private final AuthHeaderValidator authHeaderValidator = new AuthHeaderValidator(() -> Optional.of(BEARER_TOKEN)); + + @Test + public void succeedsWithMatchingHeader() { + assertThat(authHeaderValidator.suppliedTokenIsValid(AuthHeader.of(BEARER_TOKEN))) + .isTrue(); + } + + @Test + public void failsWithWrongHeader() { + assertThat(authHeaderValidator.suppliedTokenIsValid(AuthHeader.of(WRONG_TOKEN))) + .isFalse(); + } + + @Test + public void failsIfSupplierYieldsEmpty() { + AuthHeaderValidator emptyValidator = new AuthHeaderValidator(Optional::empty); + assertThat(emptyValidator.suppliedTokenIsValid(AuthHeader.of(BEARER_TOKEN))) + .isFalse(); + } +} From b38c83f5731777d5dddbac8faa5976d253abec88 Mon Sep 17 00:00:00 2001 From: svc-changelog Date: Mon, 7 Feb 2022 14:54:07 +0000 Subject: [PATCH 17/18] Add generated changelog entries --- changelog/@unreleased/pr-5892.v2.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 changelog/@unreleased/pr-5892.v2.yml diff --git a/changelog/@unreleased/pr-5892.v2.yml b/changelog/@unreleased/pr-5892.v2.yml new file mode 100644 index 00000000000..eac6dcc019a --- /dev/null +++ b/changelog/@unreleased/pr-5892.v2.yml @@ -0,0 +1,7 @@ +type: feature +feature: + description: Added AtlasRestoreService methods `prepareRestore` and `completeRestore`, + which will respectively disable and re-enable TimeLock, and should be called during + the restore process. + links: + - https://github.com/palantir/atlasdb/pull/5892 From 4ba9f67cd5f52f9ccd4242f6a7006a8de16822d5 Mon Sep 17 00:00:00 2001 From: Glenn Sheasby Date: Tue, 8 Feb 2022 15:52:08 +0000 Subject: [PATCH 18/18] nits --- .../atlasdb/backup/AtlasRestoreService.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java index 0794c96742f..b18a101b802 100644 --- a/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java +++ b/atlasdb-backup/src/main/java/com/palantir/atlasdb/backup/AtlasRestoreService.java @@ -36,6 +36,7 @@ import com.palantir.atlasdb.timelock.api.SuccessfulDisableNamespacesResponse; import com.palantir.atlasdb.timelock.api.UnsuccessfulDisableNamespacesResponse; import com.palantir.atlasdb.timelock.api.management.TimeLockManagementServiceBlocking; +import com.palantir.common.annotation.NonIdempotent; import com.palantir.common.streams.KeyedStream; import com.palantir.conjure.java.api.config.service.ServicesConfigBlock; import com.palantir.dialogue.clients.DialogueClients; @@ -108,11 +109,12 @@ public static AtlasRestoreService create( * * @return the result of the request, including a lock ID which must later be passed to completeRestore. */ + @NonIdempotent public DisableNamespacesResponse prepareRestore(Set namespaces) { Map completedBackups = getCompletedBackups(namespaces); - Set namespacesToRepair = completedBackups.keySet(); + Set namespacesToRestore = completedBackups.keySet(); - DisableNamespacesResponse response = timeLockManagementService.disableTimelock(authHeader, namespacesToRepair); + DisableNamespacesResponse response = timeLockManagementService.disableTimelock(authHeader, namespacesToRestore); return response.accept(new DisableNamespacesResponse.Visitor<>() { @Override public DisableNamespacesResponse visitSuccessful(SuccessfulDisableNamespacesResponse value) { @@ -151,7 +153,7 @@ public Set repairInternalTables( Set namespaces, BiConsumer repairTable) { Map completedBackups = getCompletedBackups(namespaces); Set namespacesToRepair = completedBackups.keySet(); - restoreTables(repairTable, completedBackups, namespacesToRepair); + repairTables(repairTable, completedBackups, namespacesToRepair); return namespacesToRepair; } @@ -162,6 +164,7 @@ public Set repairInternalTables( * @param request the request object, which must include the lock ID returned by {@link #prepareRestore(Set)} * @return the set of namespaces that were successfully fast-forwarded and re-enabled. */ + @NonIdempotent public Set completeRestore(ReenableNamespacesRequest request) { Set completedBackups = request.getNamespaces().stream() .map(backupPersister::getCompletedBackup) @@ -199,7 +202,7 @@ public Set completeRestore(ReenableNamespacesRequest request) { return successfulNamespaces; } - private void restoreTables( + private void repairTables( BiConsumer repairTable, Map completedBackups, Set namespacesToRepair) { @@ -209,10 +212,10 @@ private void restoreTables( // RepairTransactionsTablesTask KeyedStream.stream(completedBackups) .forEach((namespace, completedBackup) -> - restoreTransactionsTables(namespace, completedBackup, repairTable)); + repairTransactionsTables(namespace, completedBackup, repairTable)); } - private void restoreTransactionsTables( + private void repairTransactionsTables( Namespace namespace, CompletedBackup completedBackup, BiConsumer repairTable) { Map coordinationMap = getCoordinationMap(namespace, completedBackup); List transactionsTableInteractions =