diff --git a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/timelock/TimeoutSensitiveLockRpcClient.java b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/timelock/TimeoutSensitiveLockRpcClient.java index 46f1130ce07..02ef92e0fca 100644 --- a/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/timelock/TimeoutSensitiveLockRpcClient.java +++ b/atlasdb-config/src/main/java/com/palantir/atlasdb/factory/timelock/TimeoutSensitiveLockRpcClient.java @@ -19,11 +19,13 @@ import com.palantir.lock.HeldLocksGrant; import com.palantir.lock.HeldLocksToken; import com.palantir.lock.LockClient; +import com.palantir.lock.LockDescriptor; import com.palantir.lock.LockRefreshToken; import com.palantir.lock.LockRequest; import com.palantir.lock.LockResponse; import com.palantir.lock.LockRpcClient; import com.palantir.lock.LockServerOptions; +import com.palantir.lock.LockState; import com.palantir.lock.SimpleHeldLocksToken; import java.math.BigInteger; import java.util.Optional; @@ -155,4 +157,9 @@ public void logCurrentState(String namespace) { // dump its logs out. shortTimeoutProxy.logCurrentState(namespace); } + + @Override + public LockState getLockState(LockDescriptor lock) { + return shortTimeoutProxy.getLockState(lock); + } } diff --git a/changelog/@unreleased/pr-5302.v2.yml b/changelog/@unreleased/pr-5302.v2.yml new file mode 100644 index 00000000000..0dc8f16cbdc --- /dev/null +++ b/changelog/@unreleased/pr-5302.v2.yml @@ -0,0 +1,5 @@ +type: improvement +improvement: + description: Add new getLockState(LockDescriptor) debugging endpoint to the RemoteLockService, which allows for easier, more targeted debugging than would be possible with previous logCurrentState() endpoint. Provide a LockDescriptor, and get the current set of clients holding and waiting for the lock. + links: + - https://github.com/palantir/atlasdb/pull/5302 diff --git a/lock-api/src/main/java/com/palantir/lock/ForwardingRemoteLockService.java b/lock-api/src/main/java/com/palantir/lock/ForwardingRemoteLockService.java index 700ac7b0f24..9867e2953b7 100644 --- a/lock-api/src/main/java/com/palantir/lock/ForwardingRemoteLockService.java +++ b/lock-api/src/main/java/com/palantir/lock/ForwardingRemoteLockService.java @@ -60,6 +60,11 @@ public void logCurrentState() { delegate().logCurrentState(); } + @Override + public LockState getLockState(LockDescriptor lock) { + return delegate().getLockState(lock); + } + @Override public void close() throws IOException { if (delegate() instanceof Closeable) { diff --git a/lock-api/src/main/java/com/palantir/lock/LockRpcClient.java b/lock-api/src/main/java/com/palantir/lock/LockRpcClient.java index 37c518dbfd4..4e90c384e15 100644 --- a/lock-api/src/main/java/com/palantir/lock/LockRpcClient.java +++ b/lock-api/src/main/java/com/palantir/lock/LockRpcClient.java @@ -138,4 +138,8 @@ Set refreshLockRefreshTokens( @POST @Path("log-current-state") void logCurrentState(@Safe @PathParam("namespace") String namespace); + + @POST + @Path("get-debugging-lock-state") + LockState getLockState(LockDescriptor lock); } diff --git a/lock-api/src/main/java/com/palantir/lock/LockState.java b/lock-api/src/main/java/com/palantir/lock/LockState.java new file mode 100644 index 00000000000..82f0143090d --- /dev/null +++ b/lock-api/src/main/java/com/palantir/lock/LockState.java @@ -0,0 +1,94 @@ +/* + * (c) Copyright 2021 Palantir Technologies Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.palantir.lock; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import java.util.List; +import java.util.Optional; +import org.immutables.value.Value; + +@Value.Immutable +@JsonSerialize(as = ImmutableLockState.class) +@JsonDeserialize(as = ImmutableLockState.class) +public interface LockState { + boolean isWriteLocked(); + + boolean isFrozen(); + + List exactCurrentLockHolders(); + + List holders(); + + List requesters(); + + @Value.Immutable + @JsonSerialize(as = ImmutableLockHolder.class) + @JsonDeserialize(as = ImmutableLockHolder.class) + interface LockHolder { + static LockHolder from(HeldLocksToken lock) { + return ImmutableLockHolder.builder() + .client(lock.getClient()) + .creationDateMs(lock.getCreationDateMs()) + .expirationDateMs(lock.getExpirationDateMs()) + .numOtherLocksHeld(lock.getLocks().size() - 1) + .versionId(Optional.ofNullable(lock.getVersionId())) + .requestingThread(lock.getRequestingThread()) + .build(); + } + + LockClient client(); + + long creationDateMs(); + + long expirationDateMs(); + + int numOtherLocksHeld(); + + Optional versionId(); + + String requestingThread(); + } + + @Value.Immutable + @JsonSerialize(as = ImmutableLockRequester.class) + @JsonDeserialize(as = ImmutableLockRequester.class) + interface LockRequester { + static ImmutableLockRequester from(LockRequest request, LockClient client) { + return ImmutableLockRequester.builder() + .client(client) + .lockGroupBehavior(request.getLockGroupBehavior()) + .blockingMode(request.getBlockingMode()) + .blockingDuration(Optional.ofNullable(request.getBlockingDuration())) + .versionId(Optional.ofNullable(request.getVersionId())) + .requestingThread(request.getCreatingThreadName()) + .build(); + } + + LockClient client(); + + LockGroupBehavior lockGroupBehavior(); + + BlockingMode blockingMode(); + + Optional blockingDuration(); + + Optional versionId(); + + String requestingThread(); + } +} diff --git a/lock-api/src/main/java/com/palantir/lock/NamespaceAgnosticLockRpcClient.java b/lock-api/src/main/java/com/palantir/lock/NamespaceAgnosticLockRpcClient.java index 59ead7ad773..8a8db381ee1 100644 --- a/lock-api/src/main/java/com/palantir/lock/NamespaceAgnosticLockRpcClient.java +++ b/lock-api/src/main/java/com/palantir/lock/NamespaceAgnosticLockRpcClient.java @@ -124,4 +124,8 @@ Optional lockAndGetHeldLocks(@Safe @PathParam("client") String c @POST @Path("log-current-state") void logCurrentState(); + + @POST + @Path("get-debugging-lock-state") + LockState getLockState(LockDescriptor lock); } diff --git a/lock-api/src/main/java/com/palantir/lock/RemoteLockService.java b/lock-api/src/main/java/com/palantir/lock/RemoteLockService.java index 4fc21a4ba2f..09032cdae6d 100644 --- a/lock-api/src/main/java/com/palantir/lock/RemoteLockService.java +++ b/lock-api/src/main/java/com/palantir/lock/RemoteLockService.java @@ -103,4 +103,20 @@ HeldLocksToken lockAndGetHeldLocks(@Safe @PathParam("client") String client, Loc @POST @Path("log-current-state") void logCurrentState(); + + /** + * Returns the current locking and request state of the specified lock. + * + * Note that this is a best-effort endpoint, and not all parts of the the returned data + * structure is guaranteed to reflect the state of the server at a given point in time. + * + * Also note that, as this is a debugging endpoint, we do not enforce backcompat guarantees + * on this endpoint. + */ + @POST + @Idempotent + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.APPLICATION_JSON) + @Path("get-debugging-lock-state") + LockState getLockState(LockDescriptor lock); } diff --git a/lock-api/src/main/java/com/palantir/lock/client/DialogueComposingLockRpcClient.java b/lock-api/src/main/java/com/palantir/lock/client/DialogueComposingLockRpcClient.java index 3db9f702a87..fe48c30eca4 100644 --- a/lock-api/src/main/java/com/palantir/lock/client/DialogueComposingLockRpcClient.java +++ b/lock-api/src/main/java/com/palantir/lock/client/DialogueComposingLockRpcClient.java @@ -22,11 +22,13 @@ import com.palantir.lock.HeldLocksGrant; import com.palantir.lock.HeldLocksToken; import com.palantir.lock.LockClient; +import com.palantir.lock.LockDescriptor; import com.palantir.lock.LockRefreshToken; import com.palantir.lock.LockRequest; import com.palantir.lock.LockResponse; import com.palantir.lock.LockRpcClient; import com.palantir.lock.LockServerOptions; +import com.palantir.lock.LockState; import com.palantir.lock.SimpleHeldLocksToken; import com.palantir.tokens.auth.AuthHeader; import java.math.BigInteger; @@ -159,4 +161,9 @@ public long currentTimeMillis(String namespace) { public void logCurrentState(String namespace) { dialogueShimDelegate.logCurrentState(namespace); } + + @Override + public LockState getLockState(LockDescriptor lock) { + return dialogueShimDelegate.getLockState(lock); + } } diff --git a/lock-api/src/main/java/com/palantir/lock/client/NamespaceAgnosticLockClientAdaptor.java b/lock-api/src/main/java/com/palantir/lock/client/NamespaceAgnosticLockClientAdaptor.java index 6f3be14641c..327f33d1272 100644 --- a/lock-api/src/main/java/com/palantir/lock/client/NamespaceAgnosticLockClientAdaptor.java +++ b/lock-api/src/main/java/com/palantir/lock/client/NamespaceAgnosticLockClientAdaptor.java @@ -19,11 +19,13 @@ import com.palantir.lock.HeldLocksGrant; import com.palantir.lock.HeldLocksToken; import com.palantir.lock.LockClient; +import com.palantir.lock.LockDescriptor; import com.palantir.lock.LockRefreshToken; import com.palantir.lock.LockRequest; import com.palantir.lock.LockResponse; import com.palantir.lock.LockRpcClient; import com.palantir.lock.LockServerOptions; +import com.palantir.lock.LockState; import com.palantir.lock.NamespaceAgnosticLockRpcClient; import com.palantir.lock.SimpleHeldLocksToken; import java.math.BigInteger; @@ -145,4 +147,9 @@ public long currentTimeMillis() { public void logCurrentState() { lockRpcClient.logCurrentState(namespace); } + + @Override + public LockState getLockState(LockDescriptor lock) { + return lockRpcClient.getLockState(lock); + } } diff --git a/lock-api/src/main/java/com/palantir/lock/client/RemoteLockServiceAdapter.java b/lock-api/src/main/java/com/palantir/lock/client/RemoteLockServiceAdapter.java index 46e2340b3cf..f97573a1fff 100644 --- a/lock-api/src/main/java/com/palantir/lock/client/RemoteLockServiceAdapter.java +++ b/lock-api/src/main/java/com/palantir/lock/client/RemoteLockServiceAdapter.java @@ -19,12 +19,14 @@ import com.palantir.lock.HeldLocksGrant; import com.palantir.lock.HeldLocksToken; import com.palantir.lock.LockClient; +import com.palantir.lock.LockDescriptor; import com.palantir.lock.LockRefreshToken; import com.palantir.lock.LockRequest; import com.palantir.lock.LockResponse; import com.palantir.lock.LockRpcClient; import com.palantir.lock.LockServerOptions; import com.palantir.lock.LockService; +import com.palantir.lock.LockState; import com.palantir.lock.NamespaceAgnosticLockRpcClient; import com.palantir.lock.SimpleHeldLocksToken; import java.math.BigInteger; @@ -157,4 +159,9 @@ public long currentTimeMillis() { public void logCurrentState() { namespaceAgnosticLockRpcClient.logCurrentState(); } + + @Override + public LockState getLockState(LockDescriptor lock) { + return namespaceAgnosticLockRpcClient.getLockState(lock); + } } diff --git a/lock-impl/src/main/java/com/palantir/lock/impl/LockServerLock.java b/lock-impl/src/main/java/com/palantir/lock/impl/LockServerLock.java index c2242d5045a..5252a105562 100644 --- a/lock-impl/src/main/java/com/palantir/lock/impl/LockServerLock.java +++ b/lock-impl/src/main/java/com/palantir/lock/impl/LockServerLock.java @@ -35,6 +35,10 @@ public LockServerLock(LockDescriptor descriptor, LockClientIndices clients) { this.sync = new LockServerSync(clients); } + LockServerSync getSync() { + return sync; + } + @Override public LockDescriptor getDescriptor() { return descriptor; diff --git a/lock-impl/src/main/java/com/palantir/lock/impl/LockServerSync.java b/lock-impl/src/main/java/com/palantir/lock/impl/LockServerSync.java index 732789bfb24..69ab4914c88 100644 --- a/lock-impl/src/main/java/com/palantir/lock/impl/LockServerSync.java +++ b/lock-impl/src/main/java/com/palantir/lock/impl/LockServerSync.java @@ -260,7 +260,7 @@ private synchronized void decrementReadCount(int clientIndex) { } } - private synchronized Iterable getReadClients() { + synchronized Iterable getReadClients() { if (readLockHolders == null) { return ImmutableList.of(); } diff --git a/lock-impl/src/main/java/com/palantir/lock/impl/LockServiceImpl.java b/lock-impl/src/main/java/com/palantir/lock/impl/LockServiceImpl.java index 0610583cd82..b79befa0747 100644 --- a/lock-impl/src/main/java/com/palantir/lock/impl/LockServiceImpl.java +++ b/lock-impl/src/main/java/com/palantir/lock/impl/LockServiceImpl.java @@ -43,6 +43,7 @@ import com.palantir.common.concurrent.PTExecutors; import com.palantir.common.random.SecureRandomPool; import com.palantir.common.remoting.ServiceNotAvailableException; +import com.palantir.common.streams.KeyedStream; import com.palantir.lock.BlockingMode; import com.palantir.lock.CloseableLockService; import com.palantir.lock.CloseableRemoteLockService; @@ -50,6 +51,7 @@ import com.palantir.lock.HeldLocksGrant; import com.palantir.lock.HeldLocksToken; import com.palantir.lock.HeldLocksTokens; +import com.palantir.lock.ImmutableLockState; import com.palantir.lock.LockClient; import com.palantir.lock.LockCollection; import com.palantir.lock.LockCollections; @@ -62,6 +64,9 @@ import com.palantir.lock.LockServerConfigs; import com.palantir.lock.LockServerOptions; import com.palantir.lock.LockService; +import com.palantir.lock.LockState; +import com.palantir.lock.LockState.LockHolder; +import com.palantir.lock.LockState.LockRequester; import com.palantir.lock.RemoteLockService; import com.palantir.lock.SimpleHeldLocksToken; import com.palantir.lock.SimpleTimeDuration; @@ -85,6 +90,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; @@ -131,6 +137,8 @@ public final class LockServiceImpl // LegacyTimelockServiceAdapter relies on token ids being convertible to UUIDs; thus this should // never be > 127 public static final int RANDOM_BIT_COUNT = 127; + public static final ImmutableLockState EMPTY_LOCK_STATE = + ImmutableLockState.builder().isWriteLocked(false).isFrozen(false).build(); @VisibleForTesting static final long DEBUG_SLOW_LOG_TRIGGER_MILLIS = 100; @@ -1128,6 +1136,45 @@ public LockServerOptions getLockServerOptions() { return options; } + @Override + public LockState getLockState(LockDescriptor descriptor) { + LockServerLock readWriteLock = (LockServerLock) descriptorToLockMap.getIfPresent(descriptor); + if (readWriteLock == null) { + return EMPTY_LOCK_STATE; + } + + LockServerSync sync = readWriteLock.getSync(); + List readHolders; + LockClient writeHolders; + boolean isFrozen; + boolean writeMode; + synchronized (sync) { + readHolders = ImmutableList.copyOf(Iterables.transform(sync.getReadClients(), clientIndices::fromIndex)); + writeHolders = sync.getLockHolder(); + isFrozen = sync.isFrozen(); + writeMode = readHolders.isEmpty(); + } + + List lockHolders = getLockHolders(readHolders, writeHolders); + ImmutableLockState.Builder lockState = ImmutableLockState.builder() + .isWriteLocked(writeMode) + .exactCurrentLockHolders(lockHolders) + .isFrozen(isFrozen); + heldLocksTokenMap.keySet().stream() + .filter(token -> token.getLockDescriptors().contains(descriptor)) + .forEach(lock -> lockState.addHolders(LockHolder.from(lock))); + KeyedStream.stream(outstandingLockRequestMultimap) + .filterEntries((client, request) -> request.getLockDescriptors().contains(descriptor)) + .forEach((client, request) -> lockState.addRequesters(LockRequester.from(request, client))); + return lockState.build(); + } + + private List getLockHolders(List readHolders, LockClient writeHolders) { + return readHolders.isEmpty() + ? Optional.ofNullable(writeHolders).map(ImmutableList::of).orElseGet(ImmutableList::of) + : readHolders; + } + /** * Prints the current state of the lock server to the logs. Useful for * debugging. diff --git a/lock-impl/src/main/java/com/palantir/lock/impl/ThreadPooledLockService.java b/lock-impl/src/main/java/com/palantir/lock/impl/ThreadPooledLockService.java index 9c449953209..ba9c7ce7eb2 100644 --- a/lock-impl/src/main/java/com/palantir/lock/impl/ThreadPooledLockService.java +++ b/lock-impl/src/main/java/com/palantir/lock/impl/ThreadPooledLockService.java @@ -19,11 +19,13 @@ import com.palantir.lock.HeldLocksGrant; import com.palantir.lock.HeldLocksToken; import com.palantir.lock.LockClient; +import com.palantir.lock.LockDescriptor; import com.palantir.lock.LockRefreshToken; import com.palantir.lock.LockRequest; import com.palantir.lock.LockResponse; import com.palantir.lock.LockServerOptions; import com.palantir.lock.LockService; +import com.palantir.lock.LockState; import com.palantir.lock.SimpleHeldLocksToken; import java.io.IOException; import java.math.BigInteger; @@ -150,6 +152,11 @@ public void logCurrentState() { delegate.logCurrentState(); } + @Override + public LockState getLockState(LockDescriptor lock) { + return delegate.getLockState(lock); + } + @Override public void close() throws IOException { delegate.close(); diff --git a/lock-impl/src/test/java/com/palantir/lock/LockServiceTest.java b/lock-impl/src/test/java/com/palantir/lock/LockServiceTest.java index fd64f9fab5d..bd4fc77efa4 100644 --- a/lock-impl/src/test/java/com/palantir/lock/LockServiceTest.java +++ b/lock-impl/src/test/java/com/palantir/lock/LockServiceTest.java @@ -15,9 +15,11 @@ */ package com.palantir.lock; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedMap; +import com.google.common.collect.Iterables; import com.palantir.common.concurrent.InterruptibleFuture; import com.palantir.common.concurrent.PTExecutors; import com.palantir.common.proxy.SimulatingServerProxy; @@ -28,6 +30,7 @@ import java.io.File; import java.util.ArrayList; import java.util.List; +import java.util.Optional; import java.util.SortedMap; import java.util.TreeMap; import java.util.concurrent.Callable; @@ -520,6 +523,45 @@ public void testLogCurrentState() throws Exception { } } + @Test + public void testGetLockState() throws Exception { + LockState state0 = server.getLockState(lock1); + Assert.assertFalse(state0.isWriteLocked()); + Assert.assertEquals(state0.exactCurrentLockHolders(), ImmutableList.of()); + Assert.assertEquals(state0.holders(), ImmutableList.of()); + Assert.assertEquals(state0.requesters(), ImmutableList.of()); + + LockRequest request1 = LockRequest.builder(ImmutableSortedMap.of(lock1, LockMode.READ)) + .doNotBlock() + .build(); + LockResponse response1 = server.lockWithFullLockResponse(LockClient.ANONYMOUS, request1); + Assert.assertTrue(response1.success()); + LockState state1 = server.getLockState(lock1); + Assert.assertFalse(state1.isWriteLocked()); + Assert.assertEquals(state1.exactCurrentLockHolders(), ImmutableList.of(LockClient.ANONYMOUS)); + Assert.assertEquals( + Iterables.getOnlyElement(state1.holders()).requestingThread(), + Thread.currentThread().getName()); + + executor.submit((Callable) () -> { + barrier.await(); + LockRequest request2 = LockRequest.builder(ImmutableSortedMap.of(lock1, LockMode.WRITE)) + .withLockedInVersionId(100L) + .build(); + LockResponse response2 = server.lockWithFullLockResponse(LockClient.ANONYMOUS, request2); + HeldLocksToken validToken = response2.getToken(); + Assert.assertNotNull(validToken); + server.unlock(validToken); + return null; + }); + + barrier.await(); + Thread.sleep(500); + LockState state2 = server.getLockState(lock1); + Assert.assertEquals(state2.requesters().size(), 1); + Assert.assertEquals(Iterables.getOnlyElement(state2.requesters()).versionId(), Optional.of(100L)); + } + /** Tests lock responses */ @Test public void testLockReponse() throws InterruptedException { diff --git a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/lock/BlockingTimeLimitedLockService.java b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/lock/BlockingTimeLimitedLockService.java index e3cd0debbcc..c031ca3f024 100644 --- a/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/lock/BlockingTimeLimitedLockService.java +++ b/timelock-impl/src/main/java/com/palantir/atlasdb/timelock/lock/BlockingTimeLimitedLockService.java @@ -24,10 +24,12 @@ import com.palantir.lock.HeldLocksGrant; import com.palantir.lock.HeldLocksToken; import com.palantir.lock.LockClient; +import com.palantir.lock.LockDescriptor; import com.palantir.lock.LockRefreshToken; import com.palantir.lock.LockRequest; import com.palantir.lock.LockResponse; import com.palantir.lock.LockServerOptions; +import com.palantir.lock.LockState; import com.palantir.lock.SimpleHeldLocksToken; import com.palantir.lock.remoting.BlockingTimeoutException; import com.palantir.logsafe.SafeArg; @@ -183,6 +185,11 @@ public void logCurrentState() { delegate.logCurrentState(); } + @Override + public LockState getLockState(LockDescriptor lock) { + return delegate.getLockState(lock); + } + private T callWithTimeLimit(Callable callable, LockRequestSpecification specification) throws InterruptedException { try {