Add new getLockState() debugging endpoint (#5302)

atlasdb-config/src/main/java/com/palantir/atlasdb/factory/timelock/TimeoutSensitiveLockRpcClient.java

@@ -19,11 +19,13 @@
 import com.palantir.lock.HeldLocksGrant;
 import com.palantir.lock.HeldLocksToken;
 import com.palantir.lock.LockClient;
+import com.palantir.lock.LockDescriptor;
 import com.palantir.lock.LockRefreshToken;
 import com.palantir.lock.LockRequest;
 import com.palantir.lock.LockResponse;
 import com.palantir.lock.LockRpcClient;
 import com.palantir.lock.LockServerOptions;
+import com.palantir.lock.LockState;
 import com.palantir.lock.SimpleHeldLocksToken;
 import java.math.BigInteger;
 import java.util.Optional;
@@ -155,4 +157,9 @@ public void logCurrentState(String namespace) {
         // dump its logs out.
         shortTimeoutProxy.logCurrentState(namespace);
     }
+
+    @Override
+    public LockState getLockState(LockDescriptor lock) {
+        return shortTimeoutProxy.getLockState(lock);
+    }
 }

changelog/@unreleased/pr-5302.v2.yml

@@ -0,0 +1,5 @@
+type: improvement
+improvement:
+  description: Add new getLockState(LockDescriptor) debugging endpoint to the RemoteLockService, which allows for easier, more targeted debugging than would be possible with previous logCurrentState() endpoint. Provide a LockDescriptor, and get the current set of clients holding and waiting for the lock.
+  links:
+  - https://github.com/palantir/atlasdb/pull/5302

lock-api/src/main/java/com/palantir/lock/ForwardingRemoteLockService.java

@@ -60,6 +60,11 @@ public void logCurrentState() {
         delegate().logCurrentState();
     }
 
+    @Override
+    public LockState getLockState(LockDescriptor lock) {
+        return delegate().getLockState(lock);
+    }
+
     @Override
     public void close() throws IOException {
         if (delegate() instanceof Closeable) {

lock-api/src/main/java/com/palantir/lock/LockRpcClient.java

@@ -138,4 +138,8 @@ Set<LockRefreshToken> refreshLockRefreshTokens(
     @POST
     @Path("log-current-state")
     void logCurrentState(@Safe @PathParam("namespace") String namespace);
+
+    @POST
+    @Path("get-debugging-lock-state")
+    LockState getLockState(LockDescriptor lock);
 }

lock-api/src/main/java/com/palantir/lock/LockState.java

@@ -0,0 +1,94 @@
+/*
+ * (c) Copyright 2021 Palantir Technologies Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.palantir.lock;
+
+import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
+import com.fasterxml.jackson.databind.annotation.JsonSerialize;
+import java.util.List;
+import java.util.Optional;
+import org.immutables.value.Value;
+
+@Value.Immutable
+@JsonSerialize(as = ImmutableLockState.class)
+@JsonDeserialize(as = ImmutableLockState.class)
+public interface LockState {
+    boolean isWriteLocked();
+
+    boolean isFrozen();
+
+    List<LockClient> exactCurrentLockHolders();
+
+    List<LockHolder> holders();
+
+    List<LockRequester> requesters();
+
+    @Value.Immutable
+    @JsonSerialize(as = ImmutableLockHolder.class)
+    @JsonDeserialize(as = ImmutableLockHolder.class)
+    interface LockHolder {
+        static LockHolder from(HeldLocksToken lock) {
+            return ImmutableLockHolder.builder()
+                    .client(lock.getClient())
+                    .creationDateMs(lock.getCreationDateMs())
+                    .expirationDateMs(lock.getExpirationDateMs())
+                    .numOtherLocksHeld(lock.getLocks().size() - 1)
+                    .versionId(Optional.ofNullable(lock.getVersionId()))
+                    .requestingThread(lock.getRequestingThread())
+                    .build();
+        }
+
+        LockClient client();
+
+        long creationDateMs();
+
+        long expirationDateMs();
+
+        int numOtherLocksHeld();
+
+        Optional<Long> versionId();
+
+        String requestingThread();
+    }
+
+    @Value.Immutable
+    @JsonSerialize(as = ImmutableLockRequester.class)
+    @JsonDeserialize(as = ImmutableLockRequester.class)
+    interface LockRequester {
+        static ImmutableLockRequester from(LockRequest request, LockClient client) {
+            return ImmutableLockRequester.builder()
+                    .client(client)
+                    .lockGroupBehavior(request.getLockGroupBehavior())
+                    .blockingMode(request.getBlockingMode())
+                    .blockingDuration(Optional.ofNullable(request.getBlockingDuration()))
+                    .versionId(Optional.ofNullable(request.getVersionId()))
+                    .requestingThread(request.getCreatingThreadName())
+                    .build();
+        }
+
+        LockClient client();
+
+        LockGroupBehavior lockGroupBehavior();
+
+        BlockingMode blockingMode();
+
+        Optional<TimeDuration> blockingDuration();
+
+        Optional<Long> versionId();
+
+        String requestingThread();
+    }
+}

lock-api/src/main/java/com/palantir/lock/NamespaceAgnosticLockRpcClient.java

@@ -124,4 +124,8 @@ Optional<HeldLocksToken> lockAndGetHeldLocks(@Safe @PathParam("client") String c
     @POST
     @Path("log-current-state")
     void logCurrentState();
+
+    @POST
+    @Path("get-debugging-lock-state")
+    LockState getLockState(LockDescriptor lock);
 }

lock-api/src/main/java/com/palantir/lock/RemoteLockService.java

@@ -103,4 +103,20 @@ HeldLocksToken lockAndGetHeldLocks(@Safe @PathParam("client") String client, Loc
     @POST
     @Path("log-current-state")
     void logCurrentState();
+
+    /**
+     * Returns the current locking and request state of the specified lock.
+     *
+     * Note that this is a best-effort endpoint, and not all parts of the the returned data
+     * structure is guaranteed to reflect the state of the server at a given point in time.
+     *
+     * Also note that, as this is a debugging endpoint, we do not enforce backcompat guarantees
+     * on this endpoint.
+     */
+    @POST
+    @Idempotent
+    @Produces(MediaType.APPLICATION_JSON)
+    @Consumes(MediaType.APPLICATION_JSON)
+    @Path("get-debugging-lock-state")
+    LockState getLockState(LockDescriptor lock);
 }

lock-api/src/main/java/com/palantir/lock/client/DialogueComposingLockRpcClient.java

@@ -22,11 +22,13 @@
 import com.palantir.lock.HeldLocksGrant;
 import com.palantir.lock.HeldLocksToken;
 import com.palantir.lock.LockClient;
+import com.palantir.lock.LockDescriptor;
 import com.palantir.lock.LockRefreshToken;
 import com.palantir.lock.LockRequest;
 import com.palantir.lock.LockResponse;
 import com.palantir.lock.LockRpcClient;
 import com.palantir.lock.LockServerOptions;
+import com.palantir.lock.LockState;
 import com.palantir.lock.SimpleHeldLocksToken;
 import com.palantir.tokens.auth.AuthHeader;
 import java.math.BigInteger;
@@ -159,4 +161,9 @@ public long currentTimeMillis(String namespace) {
     public void logCurrentState(String namespace) {
         dialogueShimDelegate.logCurrentState(namespace);
     }
+
+    @Override
+    public LockState getLockState(LockDescriptor lock) {
+        return dialogueShimDelegate.getLockState(lock);
+    }
 }

lock-api/src/main/java/com/palantir/lock/client/NamespaceAgnosticLockClientAdaptor.java

@@ -19,11 +19,13 @@
 import com.palantir.lock.HeldLocksGrant;
 import com.palantir.lock.HeldLocksToken;
 import com.palantir.lock.LockClient;
+import com.palantir.lock.LockDescriptor;
 import com.palantir.lock.LockRefreshToken;
 import com.palantir.lock.LockRequest;
 import com.palantir.lock.LockResponse;
 import com.palantir.lock.LockRpcClient;
 import com.palantir.lock.LockServerOptions;
+import com.palantir.lock.LockState;
 import com.palantir.lock.NamespaceAgnosticLockRpcClient;
 import com.palantir.lock.SimpleHeldLocksToken;
 import java.math.BigInteger;
@@ -145,4 +147,9 @@ public long currentTimeMillis() {
     public void logCurrentState() {
         lockRpcClient.logCurrentState(namespace);
     }
+
+    @Override
+    public LockState getLockState(LockDescriptor lock) {
+        return lockRpcClient.getLockState(lock);
+    }
 }

lock-api/src/main/java/com/palantir/lock/client/RemoteLockServiceAdapter.java

@@ -19,12 +19,14 @@
 import com.palantir.lock.HeldLocksGrant;
 import com.palantir.lock.HeldLocksToken;
 import com.palantir.lock.LockClient;
+import com.palantir.lock.LockDescriptor;
 import com.palantir.lock.LockRefreshToken;
 import com.palantir.lock.LockRequest;
 import com.palantir.lock.LockResponse;
 import com.palantir.lock.LockRpcClient;
 import com.palantir.lock.LockServerOptions;
 import com.palantir.lock.LockService;
+import com.palantir.lock.LockState;
 import com.palantir.lock.NamespaceAgnosticLockRpcClient;
 import com.palantir.lock.SimpleHeldLocksToken;
 import java.math.BigInteger;
@@ -157,4 +159,9 @@ public long currentTimeMillis() {
     public void logCurrentState() {
         namespaceAgnosticLockRpcClient.logCurrentState();
     }
+
+    @Override
+    public LockState getLockState(LockDescriptor lock) {
+        return namespaceAgnosticLockRpcClient.getLockState(lock);
+    }
 }

lock-impl/src/main/java/com/palantir/lock/impl/LockServerLock.java

@@ -35,6 +35,10 @@ public LockServerLock(LockDescriptor descriptor, LockClientIndices clients) {
         this.sync = new LockServerSync(clients);
     }
 
+    LockServerSync getSync() {
+        return sync;
+    }
+
     @Override
     public LockDescriptor getDescriptor() {
         return descriptor;

lock-impl/src/main/java/com/palantir/lock/impl/LockServerSync.java

@@ -260,7 +260,7 @@ private synchronized void decrementReadCount(int clientIndex) {
         }
     }
 
-    private synchronized Iterable<Integer> getReadClients() {
+    synchronized Iterable<Integer> getReadClients() {
         if (readLockHolders == null) {
             return ImmutableList.of();
         }

lock-impl/src/main/java/com/palantir/lock/impl/LockServiceImpl.java

@@ -43,13 +43,15 @@
 import com.palantir.common.concurrent.PTExecutors;
 import com.palantir.common.random.SecureRandomPool;
 import com.palantir.common.remoting.ServiceNotAvailableException;
+import com.palantir.common.streams.KeyedStream;
 import com.palantir.lock.BlockingMode;
 import com.palantir.lock.CloseableLockService;
 import com.palantir.lock.CloseableRemoteLockService;
 import com.palantir.lock.ExpiringToken;
 import com.palantir.lock.HeldLocksGrant;
 import com.palantir.lock.HeldLocksToken;
 import com.palantir.lock.HeldLocksTokens;
+import com.palantir.lock.ImmutableLockState;
 import com.palantir.lock.LockClient;
 import com.palantir.lock.LockCollection;
 import com.palantir.lock.LockCollections;
@@ -62,6 +64,9 @@
 import com.palantir.lock.LockServerConfigs;
 import com.palantir.lock.LockServerOptions;
 import com.palantir.lock.LockService;
+import com.palantir.lock.LockState;
+import com.palantir.lock.LockState.LockHolder;
+import com.palantir.lock.LockState.LockRequester;
 import com.palantir.lock.RemoteLockService;
 import com.palantir.lock.SimpleHeldLocksToken;
 import com.palantir.lock.SimpleTimeDuration;
@@ -85,6 +90,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
@@ -131,6 +137,8 @@ public final class LockServiceImpl
     // LegacyTimelockServiceAdapter relies on token ids being convertible to UUIDs; thus this should
     // never be > 127
     public static final int RANDOM_BIT_COUNT = 127;
+    public static final ImmutableLockState EMPTY_LOCK_STATE =
+            ImmutableLockState.builder().isWriteLocked(false).isFrozen(false).build();
 
     @VisibleForTesting
     static final long DEBUG_SLOW_LOG_TRIGGER_MILLIS = 100;
@@ -1128,6 +1136,45 @@ public LockServerOptions getLockServerOptions() {
         return options;
     }
 
+    @Override
+    public LockState getLockState(LockDescriptor descriptor) {
+        LockServerLock readWriteLock = (LockServerLock) descriptorToLockMap.getIfPresent(descriptor);
+        if (readWriteLock == null) {
+            return EMPTY_LOCK_STATE;
+        }
+
+        LockServerSync sync = readWriteLock.getSync();
+        List<LockClient> readHolders;
+        LockClient writeHolders;
+        boolean isFrozen;
+        boolean writeMode;
+        synchronized (sync) {
+            readHolders = ImmutableList.copyOf(Iterables.transform(sync.getReadClients(), clientIndices::fromIndex));
+            writeHolders = sync.getLockHolder();
+            isFrozen = sync.isFrozen();
+            writeMode = readHolders.isEmpty();
+        }
+
+        List<LockClient> lockHolders = getLockHolders(readHolders, writeHolders);
+        ImmutableLockState.Builder lockState = ImmutableLockState.builder()
+                .isWriteLocked(writeMode)
+                .exactCurrentLockHolders(lockHolders)
+                .isFrozen(isFrozen);
+        heldLocksTokenMap.keySet().stream()
+                .filter(token -> token.getLockDescriptors().contains(descriptor))
+                .forEach(lock -> lockState.addHolders(LockHolder.from(lock)));
+        KeyedStream.stream(outstandingLockRequestMultimap)
+                .filterEntries((client, request) -> request.getLockDescriptors().contains(descriptor))
+                .forEach((client, request) -> lockState.addRequesters(LockRequester.from(request, client)));
+        return lockState.build();
+    }
+
+    private List<LockClient> getLockHolders(List<LockClient> readHolders, LockClient writeHolders) {
+        return readHolders.isEmpty()
+                ? Optional.ofNullable(writeHolders).map(ImmutableList::of).orElseGet(ImmutableList::of)
+                : readHolders;
+    }
+
     /**
      * Prints the current state of the lock server to the logs. Useful for
      * debugging.

lock-impl/src/main/java/com/palantir/lock/impl/ThreadPooledLockService.java

@@ -19,11 +19,13 @@
 import com.palantir.lock.HeldLocksGrant;
 import com.palantir.lock.HeldLocksToken;
 import com.palantir.lock.LockClient;
+import com.palantir.lock.LockDescriptor;
 import com.palantir.lock.LockRefreshToken;
 import com.palantir.lock.LockRequest;
 import com.palantir.lock.LockResponse;
 import com.palantir.lock.LockServerOptions;
 import com.palantir.lock.LockService;
+import com.palantir.lock.LockState;
 import com.palantir.lock.SimpleHeldLocksToken;
 import java.io.IOException;
 import java.math.BigInteger;
@@ -150,6 +152,11 @@ public void logCurrentState() {
         delegate.logCurrentState();
     }
 
+    @Override
+    public LockState getLockState(LockDescriptor lock) {
+        return delegate.getLockState(lock);
+    }
+
     @Override
     public void close() throws IOException {
         delegate.close();