ABR1: AtlasBackupService

atlasdb-backups/build.gradle

@@ -0,0 +1,10 @@
+apply from: "../gradle/shared.gradle"
+
+dependencies {
+    implementation project(':atlasdb-config')
+    implementation project(':timelock-api:timelock-api-objects')
+
+    implementation 'com.palantir.safe-logging:safe-logging'
+
+    testImplementation 'org.mockito:mockito-core'
+}

atlasdb-backups/src/main/java/com/palantir/atlasdb/backup/AtlasBackupTask.java

@@ -0,0 +1,73 @@
+/*
+ * (c) Copyright 2021 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.backup;
+
+import com.palantir.atlasdb.backup.api.AtlasBackupServiceBlocking;
+import com.palantir.atlasdb.timelock.api.BackupToken;
+import com.palantir.atlasdb.timelock.api.CompleteBackupRequest;
+import com.palantir.atlasdb.timelock.api.CompleteBackupResponse;
+import com.palantir.atlasdb.timelock.api.Namespace;
+import com.palantir.atlasdb.timelock.api.PrepareBackupRequest;
+import com.palantir.atlasdb.timelock.api.PrepareBackupResponse;
+import com.palantir.conjure.java.api.config.service.ServicesConfigBlock;
+import com.palantir.dialogue.clients.DialogueClients;
+import com.palantir.dialogue.clients.DialogueClients.ReloadingFactory;
+import com.palantir.refreshable.Refreshable;
+import com.palantir.tokens.auth.AuthHeader;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public final class AtlasBackupTask {
+    private final AtlasBackupServiceBlocking atlasBackupServiceBlocking;
+    private final BackupTokenPersister backupTokenPersister;
+
+    private AtlasBackupTask(AtlasBackupServiceBlocking atlasBackupServiceBlocking) {
+        this.atlasBackupServiceBlocking = atlasBackupServiceBlocking;
+        this.backupTokenPersister = new InMemoryBackupTokenPersister();
+    }
+
+    public static AtlasBackupTask create(Refreshable<ServicesConfigBlock> servicesConfigBlock, String serviceName) {
+        ReloadingFactory reloadingFactory = DialogueClients.create(servicesConfigBlock);
+        AtlasBackupServiceBlocking atlasBackupServiceBlocking =
+                reloadingFactory.get(AtlasBackupServiceBlocking.class, serviceName);
+        return new AtlasBackupTask(atlasBackupServiceBlocking);
+    }
+
+    public Set<Namespace> prepareBackup(AuthHeader authHeader, Set<Namespace> namespaces) {
+        PrepareBackupRequest request = PrepareBackupRequest.of(namespaces);
+        PrepareBackupResponse response = atlasBackupServiceBlocking.prepareBackup(authHeader, request);
+
+        return response.getSuccessful().stream()
+                .filter(backupTokenPersister::storeBackupToken)
+                .map(BackupToken::getNamespace)
+                .collect(Collectors.toSet());
+    }
+
+    public Set<Namespace> completeBackup(AuthHeader authHeader, Set<Namespace> namespaces) {
+        Set<BackupToken> tokens = namespaces.stream()
+                .map(backupTokenPersister::retrieveBackupToken)
+                .flatMap(Optional::stream)
+                .collect(Collectors.toSet());
+        CompleteBackupRequest request = CompleteBackupRequest.of(tokens);
+        CompleteBackupResponse response = atlasBackupServiceBlocking.completeBackup(authHeader, request);
+
+        return response.getSuccessfulBackups().stream()
+                .map(BackupToken::getNamespace)
+                .collect(Collectors.toSet());
+    }
+}

atlasdb-backups/src/main/java/com/palantir/atlasdb/backup/BackupTokenPersister.java

@@ -0,0 +1,27 @@
+/*
+ * (c) Copyright 2021 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.backup;
+
+import com.palantir.atlasdb.timelock.api.BackupToken;
+import com.palantir.atlasdb.timelock.api.Namespace;
+import java.util.Optional;
+
+public interface BackupTokenPersister {
+    boolean storeBackupToken(BackupToken backupToken);
+
+    Optional<BackupToken> retrieveBackupToken(Namespace namespace);
+}

atlasdb-backups/src/main/java/com/palantir/atlasdb/backup/InMemoryBackupTokenPersister.java

@@ -0,0 +1,42 @@
+/*
+ * (c) Copyright 2021 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.atlasdb.backup;
+
+import com.palantir.atlasdb.timelock.api.BackupToken;
+import com.palantir.atlasdb.timelock.api.Namespace;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Optional;
+
+public final class InMemoryBackupTokenPersister implements BackupTokenPersister {
+    private final Map<Namespace, BackupToken> storedTokens;
+
+    public InMemoryBackupTokenPersister() {
+        storedTokens = new HashMap<>();
+    }
+
+    @Override
+    public boolean storeBackupToken(BackupToken backupToken) {
+        storedTokens.put(backupToken.getNamespace(), backupToken);
+        return true;
+    }
+
+    @Override
+    public Optional<BackupToken> retrieveBackupToken(Namespace namespace) {
+        return Optional.ofNullable(storedTokens.get(namespace));
+    }
+}

atlasdb-impl-shared/src/main/java/com/palantir/atlasdb/debug/LocalLockTracker.java

@@ -54,7 +54,7 @@ List<TrackedLockEvent> getLocalLockHistory() {
     void logLockResponse(Set<ConjureLockDescriptor> lockDescriptors, ConjureLockResponse response) {
         TrackedLockEvent event = getTimestampedLockEventBuilder()
                 .eventType(EventType.LOCK)
-                .eventDescription(response.accept(new ConjureLockResponse.Visitor<String>() {
+                .eventDescription(response.accept(new ConjureLockResponse.Visitor<>() {
                     @Override
                     public String visitSuccessful(SuccessfulLockResponse value) {
                         return "SUCCESS - locked " + lockDescriptors + "; obtained " + value;

lock-api-objects/src/main/java/com/palantir/lock/v2/TimelockService.java

@@ -46,7 +46,6 @@ default boolean isInitialized() {
 
     TimestampRange getFreshTimestamps(@Safe @QueryParam("number") int numTimestampsRequested);
 
-    // TODO (jkong): Can this be deprecated? Are there users outside of Atlas transactions?
     LockImmutableTimestampResponse lockImmutableTimestamp();
 
     @DoNotDelegate

settings.gradle

@@ -1,6 +1,7 @@
 rootProject.name = 'atlasdb'
 include ":atlasdb-api"
 include ":atlasdb-autobatch"
+include ":atlasdb-backups"
 include ":atlasdb-cassandra"
 include ":atlasdb-cassandra-integration-tests"
 include ":atlasdb-cassandra-multinode-tests"

timelock-agent/src/main/java/com/palantir/timelock/paxos/TimeLockAgent.java

@@ -24,6 +24,7 @@
 import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableSet;
 import com.palantir.atlasdb.AtlasDbConstants;
+import com.palantir.atlasdb.backup.AtlasBackupResource;
 import com.palantir.atlasdb.config.AuxiliaryRemotingParameters;
 import com.palantir.atlasdb.config.ImmutableLeaderConfig;
 import com.palantir.atlasdb.config.ImmutableServerListConfig;
@@ -332,30 +333,34 @@ private void createAndRegisterResources() {
         Function<String, LockService> lockServiceGetter =
                 namespace -> namespaces.get(namespace).getLockService();
 
+        RedirectRetryTargeter redirectRetryTargeter = redirectRetryTargeter();
         if (undertowRegistrar.isPresent()) {
             Consumer<UndertowService> presentUndertowRegistrar = undertowRegistrar.get();
             registerCorruptionHandlerWrappedService(
                     presentUndertowRegistrar,
-                    ConjureTimelockResource.undertow(redirectRetryTargeter(), asyncTimelockServiceGetter));
+                    ConjureTimelockResource.undertow(redirectRetryTargeter, asyncTimelockServiceGetter));
             registerCorruptionHandlerWrappedService(
                     presentUndertowRegistrar,
-                    ConjureLockWatchingResource.undertow(redirectRetryTargeter(), asyncTimelockServiceGetter));
+                    ConjureLockWatchingResource.undertow(redirectRetryTargeter, asyncTimelockServiceGetter));
             registerCorruptionHandlerWrappedService(
-                    presentUndertowRegistrar,
-                    ConjureLockV1Resource.undertow(redirectRetryTargeter(), lockServiceGetter));
+                    presentUndertowRegistrar, ConjureLockV1Resource.undertow(redirectRetryTargeter, lockServiceGetter));
             registerCorruptionHandlerWrappedService(
                     presentUndertowRegistrar,
                     TimeLockPaxosHistoryProviderResource.undertow(corruptionComponents.localHistoryLoader()));
             registerCorruptionHandlerWrappedService(
                     presentUndertowRegistrar,
-                    MultiClientConjureTimelockResource.undertow(redirectRetryTargeter(), asyncTimelockServiceGetter));
+                    MultiClientConjureTimelockResource.undertow(redirectRetryTargeter, asyncTimelockServiceGetter));
+            registerCorruptionHandlerWrappedService(
+                    presentUndertowRegistrar,
+                    AtlasBackupResource.undertow(redirectRetryTargeter, asyncTimelockServiceGetter));
         } else {
-            registrar.accept(ConjureTimelockResource.jersey(redirectRetryTargeter(), asyncTimelockServiceGetter));
-            registrar.accept(ConjureLockWatchingResource.jersey(redirectRetryTargeter(), asyncTimelockServiceGetter));
-            registrar.accept(ConjureLockV1Resource.jersey(redirectRetryTargeter(), lockServiceGetter));
+            registrar.accept(ConjureTimelockResource.jersey(redirectRetryTargeter, asyncTimelockServiceGetter));
+            registrar.accept(ConjureLockWatchingResource.jersey(redirectRetryTargeter, asyncTimelockServiceGetter));
+            registrar.accept(ConjureLockV1Resource.jersey(redirectRetryTargeter, lockServiceGetter));
             registrar.accept(TimeLockPaxosHistoryProviderResource.jersey(corruptionComponents.localHistoryLoader()));
             registrar.accept(
-                    MultiClientConjureTimelockResource.jersey(redirectRetryTargeter(), asyncTimelockServiceGetter));
+                    MultiClientConjureTimelockResource.jersey(redirectRetryTargeter, asyncTimelockServiceGetter));
+            registrar.accept(AtlasBackupResource.jersey(redirectRetryTargeter, asyncTimelockServiceGetter));
         }
     }
 

timelock-api/src/main/conjure/timelock-api.yml

@@ -20,6 +20,10 @@ types:
       base-type: any
       external:
         java: com.palantir.lock.v2.Lease
+    LockToken:
+      base-type: any
+      external:
+        java: com.palantir.lock.v2.LockToken
     LockWatchStateUpdate:
       base-type: any
       external:
@@ -82,6 +86,26 @@ types:
         union:
           successful: SuccessfulLockResponse
           unsuccessful: UnsuccessfulLockResponse
+      # in-progress backup information may be kept in TimeLock/Atlas somewhere.
+      BackupToken: # separate objects for in-progress vs complete backups?
+        fields:
+          namespace: Namespace
+          lockToken: LockToken
+          immutableTimestamp: Long
+          backupStartTimestamp: Long
+          backupEndTimestamp: optional<Long>
+      PrepareBackupRequest:
+        fields:
+          namespaces: set<Namespace>
+      PrepareBackupResponse:
+        fields:
+          successful: set<BackupToken>
+      CompleteBackupRequest:
+        fields:
+          backupTokens: set<BackupToken>
+      CompleteBackupResponse:
+        fields:
+          successfulBackups: set<BackupToken>
       ConjureWaitForLocksResponse:
         fields:
           wasSuccessful: boolean
@@ -117,6 +141,22 @@ types:
           leaderTimes: map<Namespace, LeaderTime>
 
 services:
+  AtlasBackupService:
+    name: Backup service
+    default-auth: header
+    package: com.palantir.atlasdb.backup.api
+    base-path: /backup
+    endpoints:
+      prepareBackup:
+        http: POST /prepare
+        args:
+          request: PrepareBackupRequest
+        returns: PrepareBackupResponse
+      completeBackup:
+        http: POST /complete
+        args:
+          request: CompleteBackupRequest
+        returns: CompleteBackupResponse
   ConjureTimelockService:
     name: Timelock service
     default-auth: header