From 77b8c76f65220a7efab333889f63f39d99ff585e Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Mon, 15 May 2017 09:22:06 -0700 Subject: [PATCH 01/14] Refactor protocol/API to support multiple state machines and per-state machine sessions. --- ...gies.java => CommunicationStrategies.java} | 2 +- ...rategy.java => CommunicationStrategy.java} | 2 +- .../atomix/copycat/client/CopycatClient.java | 243 +--- .../copycat/client/DefaultCopycatClient.java | 397 ------- .../copycat/client/RecoveryStrategies.java | 52 - .../copycat/client/RecoveryStrategy.java | 38 - .../client/impl/CopycatClientState.java | 112 ++ .../client/impl/DefaultCopycatClient.java | 169 +++ .../client/session/ClientSessionManager.java | 366 ------ .../client/session/CopycatSession.java | 213 ++++ .../session/impl/CopycatClientConnection.java | 43 + .../session/impl/CopycatConnection.java | 363 ++++++ .../session/impl/CopycatLeaderConnection.java | 45 + .../impl/CopycatSessionConnection.java | 115 ++ .../CopycatSessionListener.java} | 35 +- .../session/impl/CopycatSessionManager.java | 611 ++++++++++ .../CopycatSessionSequencer.java} | 16 +- .../CopycatSessionState.java} | 170 ++- .../CopycatSessionSubmitter.java} | 94 +- .../DefaultCopycatSession.java} | 124 +- .../copycat/client/util/AddressSelector.java | 40 +- .../client/util/AddressSelectorManager.java | 75 ++ .../copycat/client/util/ClientConnection.java | 355 ------ .../client/util/ClientConnectionManager.java | 84 ++ ....java => CommunicationStrategiesTest.java} | 12 +- .../client/DefaultCopycatClientTest.java | 426 ------- .../client/RecoveryStrategiesTest.java | 51 - .../session/ClientSessionManagerTest.java | 108 -- .../session/ClientSessionStateTest.java | 143 --- .../CopycatSessionSequencerTest.java} | 36 +- .../session/impl/CopycatSessionStateTest.java | 72 ++ .../CopycatSessionSubmitterTest.java} | 175 +-- .../client/util/AddressSelectorTest.java | 14 +- .../io/atomix/copycat/error/CopycatError.java | 26 +- .../copycat/error/UnknownClientException.java | 38 + .../error/UnknownStateMachineException.java | 38 + .../copycat/protocol/ClientRequest.java | 82 ++ .../protocol/ClientRequestTypeResolver.java | 2 + .../copycat/protocol/ClientResponse.java | 34 + .../protocol/ClientResponseTypeResolver.java | 16 +- .../copycat/protocol/CloseSessionRequest.java | 76 ++ .../protocol/CloseSessionResponse.java | 98 ++ .../copycat/protocol/CommandRequest.java | 1 + .../copycat/protocol/ConnectRequest.java | 56 +- .../copycat/protocol/KeepAliveRequest.java | 154 ++- .../copycat/protocol/KeepAliveResponse.java | 4 +- .../copycat/protocol/OpenSessionRequest.java | 151 +++ .../copycat/protocol/OpenSessionResponse.java | 127 +++ .../atomix/copycat/protocol/QueryRequest.java | 1 + .../copycat/protocol/RegisterRequest.java | 1 + .../copycat/protocol/RegisterResponse.java | 28 +- .../atomix/copycat/protocol/ResetRequest.java | 1 + .../copycat/protocol/UnregisterRequest.java | 17 +- .../copycat/protocol/UnregisterResponse.java | 4 +- .../atomix/copycat/server/CopycatServer.java | 50 +- .../server/protocol/AppendRequest.java | 1 + .../server/protocol/ConfigureRequest.java | 1 + .../server/protocol/InstallRequest.java | 1 + .../copycat/server/protocol/JoinRequest.java | 1 + .../copycat/server/protocol/LeaveRequest.java | 1 + .../copycat/server/protocol/PollRequest.java | 1 + .../server/protocol/ReconfigureRequest.java | 1 + .../copycat/server/protocol/VoteRequest.java | 1 + .../server/state/AbstractAppender.java | 6 +- .../copycat/server/state/AbstractState.java | 4 +- .../copycat/server/state/CandidateState.java | 4 +- .../copycat/server/state/ClientContext.java | 180 +++ .../copycat/server/state/ClientManager.java | 66 ++ .../copycat/server/state/ClusterState.java | 8 +- .../copycat/server/state/FollowerState.java | 4 +- .../copycat/server/state/InactiveState.java | 10 + .../copycat/server/state/LeaderState.java | 198 +++- .../copycat/server/state/OperationResult.java | 31 + .../copycat/server/state/PassiveState.java | 80 +- .../copycat/server/state/ReserveState.java | 56 +- .../copycat/server/state/ServerCommit.java | 74 +- .../server/state/ServerCommitPool.java | 81 -- .../copycat/server/state/ServerContext.java | 115 +- .../copycat/server/state/ServerMember.java | 2 +- .../server/state/ServerSessionContext.java | 203 +--- .../server/state/ServerSessionManager.java | 115 +- .../copycat/server/state/ServerState.java | 16 + .../server/state/ServerStateMachine.java | 1004 ----------------- .../state/ServerStateMachineContext.java | 23 +- .../state/ServerStateMachineExecutor.java | 451 ++++++-- .../state/ServerStateMachineManager.java | 649 +++++++++++ .../state/ServerStateMachineSessions.java | 71 ++ .../server/state/StateMachineRegistry.java | 78 ++ .../copycat/server/storage/LogCleaner.java | 40 + .../server/storage/compaction/Compactor.java | 9 +- .../server/storage/entry/ClientEntry.java | 71 ++ .../storage/entry/CloseSessionEntry.java | 37 + .../server/storage/entry/KeepAliveEntry.java | 136 ++- .../storage/entry/OpenSessionEntry.java | 96 ++ .../server/storage/entry/RegisterEntry.java | 26 +- .../server/storage/entry/UnregisterEntry.java | 4 +- .../storage/util/StorageSerialization.java | 14 +- .../server/util/ServerSerialization.java | 34 +- .../server/state/AbstractStateTest.java | 18 +- .../server/state/CandidateStateTest.java | 4 +- .../server/state/ServerContextTest.java | 14 - .../server/state/ServerSessionTest.java | 11 +- .../state/ServerStateMachineManagerTest.java | 162 +++ .../server/state/ServerStateMachineTest.java | 525 --------- .../java/io/atomix/copycat/test/FuzzTest.java | 33 +- .../io/atomix/copycat/test/ClusterTest.java | 137 ++- 106 files changed, 5707 insertions(+), 5007 deletions(-) rename client/src/main/java/io/atomix/copycat/client/{ServerSelectionStrategies.java => CommunicationStrategies.java} (98%) rename client/src/main/java/io/atomix/copycat/client/{ServerSelectionStrategy.java => CommunicationStrategy.java} (98%) delete mode 100644 client/src/main/java/io/atomix/copycat/client/DefaultCopycatClient.java delete mode 100644 client/src/main/java/io/atomix/copycat/client/RecoveryStrategies.java delete mode 100644 client/src/main/java/io/atomix/copycat/client/RecoveryStrategy.java create mode 100644 client/src/main/java/io/atomix/copycat/client/impl/CopycatClientState.java create mode 100644 client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatClient.java delete mode 100644 client/src/main/java/io/atomix/copycat/client/session/ClientSessionManager.java create mode 100644 client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java create mode 100644 client/src/main/java/io/atomix/copycat/client/session/impl/CopycatClientConnection.java create mode 100644 client/src/main/java/io/atomix/copycat/client/session/impl/CopycatConnection.java create mode 100644 client/src/main/java/io/atomix/copycat/client/session/impl/CopycatLeaderConnection.java create mode 100644 client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionConnection.java rename client/src/main/java/io/atomix/copycat/client/session/{ClientSessionListener.java => impl/CopycatSessionListener.java} (77%) create mode 100644 client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java rename client/src/main/java/io/atomix/copycat/client/session/{ClientSequencer.java => impl/CopycatSessionSequencer.java} (96%) rename client/src/main/java/io/atomix/copycat/client/session/{ClientSessionState.java => impl/CopycatSessionState.java} (50%) rename client/src/main/java/io/atomix/copycat/client/session/{ClientSessionSubmitter.java => impl/CopycatSessionSubmitter.java} (83%) rename client/src/main/java/io/atomix/copycat/client/session/{ClientSession.java => impl/DefaultCopycatSession.java} (58%) create mode 100644 client/src/main/java/io/atomix/copycat/client/util/AddressSelectorManager.java delete mode 100644 client/src/main/java/io/atomix/copycat/client/util/ClientConnection.java create mode 100644 client/src/main/java/io/atomix/copycat/client/util/ClientConnectionManager.java rename client/src/test/java/io/atomix/copycat/client/{ServerSelectionStrategiesTest.java => CommunicationStrategiesTest.java} (80%) delete mode 100644 client/src/test/java/io/atomix/copycat/client/DefaultCopycatClientTest.java delete mode 100644 client/src/test/java/io/atomix/copycat/client/RecoveryStrategiesTest.java delete mode 100644 client/src/test/java/io/atomix/copycat/client/session/ClientSessionManagerTest.java delete mode 100644 client/src/test/java/io/atomix/copycat/client/session/ClientSessionStateTest.java rename client/src/test/java/io/atomix/copycat/client/session/{ClientSequencerTest.java => impl/CopycatSessionSequencerTest.java} (84%) create mode 100644 client/src/test/java/io/atomix/copycat/client/session/impl/CopycatSessionStateTest.java rename client/src/test/java/io/atomix/copycat/client/session/{ClientSessionSubmitterTest.java => impl/CopycatSessionSubmitterTest.java} (60%) create mode 100644 protocol/src/main/java/io/atomix/copycat/error/UnknownClientException.java create mode 100644 protocol/src/main/java/io/atomix/copycat/error/UnknownStateMachineException.java create mode 100644 protocol/src/main/java/io/atomix/copycat/protocol/ClientRequest.java create mode 100644 protocol/src/main/java/io/atomix/copycat/protocol/ClientResponse.java create mode 100644 protocol/src/main/java/io/atomix/copycat/protocol/CloseSessionRequest.java create mode 100644 protocol/src/main/java/io/atomix/copycat/protocol/CloseSessionResponse.java create mode 100644 protocol/src/main/java/io/atomix/copycat/protocol/OpenSessionRequest.java create mode 100644 protocol/src/main/java/io/atomix/copycat/protocol/OpenSessionResponse.java create mode 100644 server/src/main/java/io/atomix/copycat/server/state/ClientContext.java create mode 100644 server/src/main/java/io/atomix/copycat/server/state/ClientManager.java create mode 100644 server/src/main/java/io/atomix/copycat/server/state/OperationResult.java delete mode 100644 server/src/main/java/io/atomix/copycat/server/state/ServerCommitPool.java delete mode 100644 server/src/main/java/io/atomix/copycat/server/state/ServerStateMachine.java create mode 100644 server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java create mode 100644 server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineSessions.java create mode 100644 server/src/main/java/io/atomix/copycat/server/state/StateMachineRegistry.java create mode 100644 server/src/main/java/io/atomix/copycat/server/storage/LogCleaner.java create mode 100644 server/src/main/java/io/atomix/copycat/server/storage/entry/ClientEntry.java create mode 100644 server/src/main/java/io/atomix/copycat/server/storage/entry/CloseSessionEntry.java create mode 100644 server/src/main/java/io/atomix/copycat/server/storage/entry/OpenSessionEntry.java create mode 100644 server/src/test/java/io/atomix/copycat/server/state/ServerStateMachineManagerTest.java delete mode 100644 server/src/test/java/io/atomix/copycat/server/state/ServerStateMachineTest.java diff --git a/client/src/main/java/io/atomix/copycat/client/ServerSelectionStrategies.java b/client/src/main/java/io/atomix/copycat/client/CommunicationStrategies.java similarity index 98% rename from client/src/main/java/io/atomix/copycat/client/ServerSelectionStrategies.java rename to client/src/main/java/io/atomix/copycat/client/CommunicationStrategies.java index ac25cc16..da1a6812 100644 --- a/client/src/main/java/io/atomix/copycat/client/ServerSelectionStrategies.java +++ b/client/src/main/java/io/atomix/copycat/client/CommunicationStrategies.java @@ -32,7 +32,7 @@ * * @author Jordan Halterman */ -public interface ServerSelectionStrategy { +public interface CommunicationStrategy { /** * Returns a prioritized list of servers to which the client can connect and submit operations. diff --git a/client/src/main/java/io/atomix/copycat/client/CopycatClient.java b/client/src/main/java/io/atomix/copycat/client/CopycatClient.java index b1231884..99e6ade6 100644 --- a/client/src/main/java/io/atomix/copycat/client/CopycatClient.java +++ b/client/src/main/java/io/atomix/copycat/client/CopycatClient.java @@ -15,8 +15,8 @@ */ package io.atomix.copycat.client; +import io.atomix.catalyst.concurrent.CatalystThreadFactory; import io.atomix.catalyst.concurrent.Listener; -import io.atomix.catalyst.concurrent.SingleThreadContext; import io.atomix.catalyst.concurrent.ThreadContext; import io.atomix.catalyst.serializer.Serializer; import io.atomix.catalyst.transport.Address; @@ -24,8 +24,9 @@ import io.atomix.catalyst.util.Assert; import io.atomix.catalyst.util.ConfigurationException; import io.atomix.copycat.Command; -import io.atomix.copycat.Operation; import io.atomix.copycat.Query; +import io.atomix.copycat.client.impl.DefaultCopycatClient; +import io.atomix.copycat.client.session.CopycatSession; import io.atomix.copycat.protocol.ClientRequestTypeResolver; import io.atomix.copycat.protocol.ClientResponseTypeResolver; import io.atomix.copycat.session.Session; @@ -37,6 +38,8 @@ import java.util.Collections; import java.util.UUID; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.function.Consumer; /** @@ -58,7 +61,7 @@ * greater than the cluster's session timeout. *

* Clients communicate with the distributed state machine by submitting {@link Command commands} and {@link Query queries} to - * the cluster through the {@link #submit(Command)} and {@link #submit(Query)} methods respectively: + * the cluster through the {@link CopycatSession#submit(Command)} and {@link CopycatSession#submit(Query)} methods respectively: *

  *   {@code
  *   client.submit(new PutCommand("foo", "Hello world!")).thenAccept(result -> {
@@ -82,7 +85,7 @@
  * {@link Query.ConsistencyLevel} documentation for more info.
  * 

* Throughout the lifetime of a client, the client may operate on the cluster via multiple sessions according to the configured - * {@link RecoveryStrategy}. In the event that the client's session expires, the client may register a new session and continue + * recovery strategy. In the event that the client's session expires, the client may register a new session and continue * to submit operations under the recovered session. The client will always attempt to ensure commands submitted are eventually * committed to the cluster even across sessions. If a command is submitted under one session but is not completed before the * session is lost and a new session is established, the client will resubmit pending commands from the prior session under @@ -107,9 +110,9 @@ * by the cluster may be lost. *

Session events

* Clients can receive arbitrary event notifications from the cluster by registering an event listener via - * {@link #onEvent(String, Consumer)}. When a command is applied to a state machine, the state machine may publish any number + * {@link CopycatSession#onEvent(String, Consumer)}. When a command is applied to a state machine, the state machine may publish any number * of events to any open session. Events will be sent to the client by the server to which the client is connected as dictated - * by the configured {@link ServerSelectionStrategy}. In the event a client is disconnected from a server, events will be + * by the configured {@link CommunicationStrategy}. In the event a client is disconnected from a server, events will be * retained in memory on all servers until the client reconnects to another server or its session expires. Once a client * reconnects to a new server, the new server will resume sending session events to the client. *
@@ -131,7 +134,7 @@
  * be received by the client in sequential order. This means the {@link CompletableFuture}s returned when submitting
  * operations through the client are guaranteed to be completed in the order in which they were created.
  * 

- * Sequential consistency is also guaranteed for {@link #onEvent(String, Consumer) events} received by a client, and events + * Sequential consistency is also guaranteed for {@link CopycatSession#onEvent(String, Consumer) events} received by a client, and events * are sequenced with command and query responses. If a client submits a command that publishes an event and then immediately * submits a concurrent query, the client will first receive the command response, then the event message, then the query * response. @@ -140,7 +143,7 @@ * thread, it is critical that clients not block the event thread. If clients need to perform blocking actions on response to * an event or response, do so on another thread. *

Serialization

- * All {@link Command commands}, {@link Query queries}, and session {@link #onEvent(String, Consumer) events} must be + * All {@link Command commands}, {@link Query queries}, and session {@link CopycatSession#onEvent(String, Consumer) events} must be * serializable by the {@link Serializer} associated with the client. Serializable types can be registered at any time. * To register a serializable type and serializer, use the {@link Serializer#register(Class) register} methods. *
@@ -194,43 +197,16 @@ enum State {
 
     /**
      * Indicates that the client is connected and its session is open.
-     * 

- * The {@code CONNECTED} state indicates that the client is healthy and operating normally. {@link Command commands} - * and {@link Query queries} submitted and completed while the client is in this state are guaranteed to adhere to - * consistency guarantees. - * {@link Query.ConsistencyLevel levels}. */ CONNECTED, /** * Indicates that the client is suspended and its session may or may not be expired. - *

- * The {@code SUSPENDED} state is indicative of an inability to communicate with the cluster within the context of - * the client's {@link Session}. Operations submitted to or completed by clients in this state should be considered - * unsafe. An operation submitted to a {@link #CONNECTED} client that transitions to the {@code SUSPENDED} state - * prior to the operation's completion may be committed multiple times in the event that the underlying session - * is ultimately {@link Session.State#EXPIRED expired}, thus breaking linearizability. Additionally, state machines - * may see the session expire while the client is in this state. - *

- * If the client is configured with a {@link RecoveryStrategy} that recovers the client's session upon expiration, - * the client will transition back to the {@link #CONNECTED} state once a new session is registered, otherwise the - * client will transition either to the {@link #CONNECTED} or {@link #CLOSED} state based on whether its session - * is expired as determined once it re-establishes communication with the cluster. - *

- * If the client is configured with a {@link RecoveryStrategy} that does not recover the client's session - * upon a session expiration, all guarantees will be maintained by the client even for operations submitted in this - * state. If linearizability guarantees are essential, users should use the {@link RecoveryStrategies#CLOSE} strategy - * and allow the client to fail when its session is lost. */ SUSPENDED, /** * Indicates that the client is closed. - *

- * A client may transition to this state as a result of an expired session or an explicit {@link CopycatClient#close() close} - * by the user. In the event that the client's {@link Session} is lost, if the configured {@link RecoveryStrategy} - * forces the client to close upon failure, the client will immediately be closed. If the {@link RecoveryStrategy} - * attempts to recover the client's session, the client still may close if it is unable to register a new session. */ CLOSED @@ -311,17 +287,6 @@ static Builder builder(Collection

cluster) { */ ThreadContext context(); - /** - * Returns the client transport. - *

- * The transport is the mechanism through which the client communicates with the cluster. The transport cannot - * be used to access client internals, but it serves only as a mechanism for providing users with the same - * transport/protocol used by the client. - * - * @return The client transport. - */ - Transport transport(); - /** * Returns the client serializer. *

@@ -339,115 +304,11 @@ static Builder builder(Collection

cluster) { Serializer serializer(); /** - * Returns the client session. - *

- * The returned {@link Session} instance will remain constant as long as the client maintains its session with the cluster. - * Maintaining the client's session requires that the client be able to communicate with one server that can communicate - * with the leader at any given time. During periods where the cluster is electing a new leader, the client's session will - * not timeout but will resume once a new leader is elected. - * - * @return The client session or {@code null} if no session is register. - */ - Session session(); - - /** - * Submits an operation to the Copycat cluster. - *

- * This method is provided for convenience. The submitted {@link Operation} must be an instance - * of {@link Command} or {@link Query}. - * - * @param operation The operation to submit. - * @param The operation result type. - * @return A completable future to be completed with the operation result. - * @throws IllegalArgumentException If the {@link Operation} is not an instance of {@link Command} or {@link Query}. - * @throws NullPointerException if {@code operation} is null - */ - default CompletableFuture submit(Operation operation) { - Assert.notNull(operation, "operation"); - if (operation instanceof Command) { - return submit((Command) operation); - } else if (operation instanceof Query) { - return submit((Query) operation); - } else { - throw new IllegalArgumentException("unknown operation type"); - } - } - - /** - * Submits a command to the Copycat cluster. - *

- * Commands are used to alter state machine state. All commands will be forwarded to the current cluster leader. - * Once a leader receives the command, it will write the command to its internal {@code Log} and replicate it to a majority - * of the cluster. Once the command has been replicated to a majority of the cluster, it will apply the command to its - * {@code StateMachine} and respond with the result. - *

- * Once the command has been applied to a server state machine, the returned {@link CompletableFuture} - * will be completed with the state machine output. - *

- * Note that all client submissions are guaranteed to be completed in the same order in which they were sent (program order) - * and on the same thread. This does not, however, mean that they'll be applied to the server-side replicated state machine - * in that order. - * - * @param command The command to submit. - * @param The command result type. - * @return A completable future to be completed with the command result. The future is guaranteed to be completed after all - * {@link Command} or {@link Query} submission futures that preceded it. The future will always be completed on the - * @throws NullPointerException if {@code command} is null - */ - CompletableFuture submit(Command command); - - /** - * Submits a query to the Copycat cluster. - *

- * Queries are used to read state machine state. The behavior of query submissions is primarily dependent on the - * query's {@link Query.ConsistencyLevel}. For {@link Query.ConsistencyLevel#LINEARIZABLE} - * and {@link Query.ConsistencyLevel#LINEARIZABLE_LEASE} consistency levels, queries will be forwarded - * to the cluster leader. For lower consistency levels, queries are allowed to read from followers. All queries are executed - * by applying queries to an internal server state machine. - *

- * Once the query has been applied to a server state machine, the returned {@link CompletableFuture} - * will be completed with the state machine output. - * - * @param query The query to submit. - * @param The query result type. - * @return A completable future to be completed with the query result. The future is guaranteed to be completed after all - * {@link Command} or {@link Query} submission futures that preceded it. The future will always be completed on the - * @throws NullPointerException if {@code query} is null - */ - CompletableFuture submit(Query query); - - /** - * Registers a void event listener. - *

- * The registered {@link Runnable} will be {@link Runnable#run() called} when an event is received - * from the Raft cluster for the client. {@link CopycatClient} implementations must guarantee that consumers are - * always called in the same thread for the session. Therefore, no two events will be received concurrently - * by the session. Additionally, events are guaranteed to be received in the order in which they were sent by - * the state machine. - * - * @param event The event to which to listen. - * @param callback The session receive callback. - * @return The listener context. - * @throws NullPointerException if {@code event} or {@code callback} is null - */ - Listener onEvent(String event, Runnable callback); - - /** - * Registers an event listener. - *

- * The registered {@link Consumer} will be {@link Consumer#accept(Object) called} when an event is received - * from the Raft cluster for the session. {@link CopycatClient} implementations must guarantee that consumers are - * always called in the same thread for the session. Therefore, no two events will be received concurrently - * by the session. Additionally, events are guaranteed to be received in the order in which they were sent by - * the state machine. + * Returns a new session builder. * - * @param event The event to which to listen. - * @param callback The session receive callback. - * @param The session event type. - * @return The listener context. - * @throws NullPointerException if {@code event} or {@code callback} is null + * @return A new session builder. */ - Listener onEvent(String event, Consumer callback); + CopycatSession.Builder sessionBuilder(); /** * Connects the client to Copycat cluster via the default server address. @@ -460,13 +321,13 @@ default CompletableFuture submit(Operation operation) { * returned {@link CompletableFuture} will be completed. *

* The client will connect to servers in the cluster according to the pattern specified by the configured - * {@link ServerSelectionStrategy}. + * {@link CommunicationStrategy}. *

* In the event that the client is unable to register a session through any of the servers listed in the provided * {@link Address} list, the client will use the configured {@link ConnectionStrategy} to determine whether and when * to retry the registration attempt. * - * @return A completable future to be completed once the client's {@link #session()} is registered. + * @return A completable future to be completed once the client is registered. */ default CompletableFuture connect() { return connect((Collection

) null); @@ -480,14 +341,14 @@ default CompletableFuture connect() { * returned {@link CompletableFuture} will be completed. *

* The client will connect to servers in the cluster according to the pattern specified by the configured - * {@link ServerSelectionStrategy}. + * {@link CommunicationStrategy}. *

* In the event that the client is unable to register a session through any of the servers listed in the provided * {@link Address} list, the client will use the configured {@link ConnectionStrategy} to determine whether and when * to retry the registration attempt. * * @param members A set of server addresses to which to connect. - * @return A completable future to be completed once the client's {@link #session()} is registered. + * @return A completable future to be completed once the client is registered. */ default CompletableFuture connect(Address... members) { if (members == null || members.length == 0) { @@ -505,29 +366,17 @@ default CompletableFuture connect(Address... members) { * returned {@link CompletableFuture} will be completed. *

* The client will connect to servers in the cluster according to the pattern specified by the configured - * {@link ServerSelectionStrategy}. + * {@link CommunicationStrategy}. *

* In the event that the client is unable to register a session through any of the servers listed in the provided * {@link Address} list, the client will use the configured {@link ConnectionStrategy} to determine whether and when * to retry the registration attempt. * * @param members A set of server addresses to which to connect. - * @return A completable future to be completed once the client's {@link #session()} is registered. + * @return A completable future to be completed once the client is registered. */ CompletableFuture connect(Collection

members); - /** - * Recovers the client session. - *

- * When a client is recovered, the client will create and register a new {@link Session}. Once the session is - * recovered, the client will transition to the {@link State#CONNECTED} state and resubmit pending operations - * from the previous session. Pending operations are guaranteed to be submitted to the new session in the same - * order in which they were submitted to the prior session and prior to submitting any new operations. - * - * @return A completable future to be completed once the client's session is recovered. - */ - CompletableFuture recover(); - /** * Closes the client. *

@@ -559,10 +408,10 @@ final class Builder implements io.atomix.catalyst.util.Builder { private Transport transport; private Serializer serializer; private Duration sessionTimeout = Duration.ZERO; - private Duration unstabilityTimeout = Duration.ZERO; + private Duration unsableTimeout = Duration.ZERO; + private int threadPoolSize = Runtime.getRuntime().availableProcessors(); private ConnectionStrategy connectionStrategy = ConnectionStrategies.ONCE; - private ServerSelectionStrategy serverSelectionStrategy = ServerSelectionStrategies.ANY; - private RecoveryStrategy recoveryStrategy = RecoveryStrategies.CLOSE; + private CommunicationStrategy communicationStrategy = CommunicationStrategies.ANY; private Builder(Collection

cluster) { this.cluster = Assert.notNull(cluster, "cluster"); @@ -610,6 +459,18 @@ public Builder withSerializer(Serializer serializer) { return this; } + /** + * Sets the client thread pool size. + * + * @param threadPoolSize The client thread pool size. + * @return The client builder. + * @throws IllegalArgumentException if the thread pool size is not positive + */ + public Builder withThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = Assert.argNot(threadPoolSize, threadPoolSize <= 0, "threadPoolSize must be positive"); + return this; + } + /** * Sets the client session timeout. * @@ -632,9 +493,8 @@ public Builder withSessionTimeout(Duration sessionTimeout) { * @throws NullPointerException if the unstability timeout is null * @throws IllegalArgumentException if the unstability timeout is not positive */ - public Builder withUnstabilityTimeout(Duration unstabilityTimeout) - { - this.unstabilityTimeout = Assert.arg( + public Builder withUnstableTimeout(Duration unstabilityTimeout) { + this.unsableTimeout = Assert.arg( Assert.notNull(unstabilityTimeout, "unstabilityTimeout"), unstabilityTimeout.toMillis() > 0, "unstability timeout must be positive" @@ -657,22 +517,11 @@ public Builder withConnectionStrategy(ConnectionStrategy connectionStrategy) { /** * Sets the server selection strategy. * - * @param serverSelectionStrategy The server selection strategy. + * @param communicationStrategy The server selection strategy. * @return The client builder. */ - public Builder withServerSelectionStrategy(ServerSelectionStrategy serverSelectionStrategy) { - this.serverSelectionStrategy = Assert.notNull(serverSelectionStrategy, "serverSelectionStrategy"); - return this; - } - - /** - * Sets the client recovery strategy. - * - * @param recoveryStrategy The client recovery strategy. - * @return The client builder. - */ - public Builder withRecoveryStrategy(RecoveryStrategy recoveryStrategy) { - this.recoveryStrategy = Assert.notNull(recoveryStrategy, "recoveryStrategy"); + public Builder withServerSelectionStrategy(CommunicationStrategy communicationStrategy) { + this.communicationStrategy = Assert.notNull(communicationStrategy, "serverSelectionStrategy"); return this; } @@ -696,6 +545,8 @@ public CopycatClient build() { serializer = new Serializer(); } + ScheduledExecutorService executor = Executors.newScheduledThreadPool(threadPoolSize, new CatalystThreadFactory("copycat-client-%d")); + // Add service loader types to the primary serializer. serializer.resolve(new ClientRequestTypeResolver()); serializer.resolve(new ClientResponseTypeResolver()); @@ -704,14 +555,12 @@ public CopycatClient build() { return new DefaultCopycatClient( clientId, cluster, - transport, - new SingleThreadContext("copycat-client-io-%d", serializer.clone()), - new SingleThreadContext("copycat-client-event-%d", serializer.clone()), - serverSelectionStrategy, + transport.client(), + executor, + serializer, connectionStrategy, - recoveryStrategy, sessionTimeout, - unstabilityTimeout + unsableTimeout ); } } diff --git a/client/src/main/java/io/atomix/copycat/client/DefaultCopycatClient.java b/client/src/main/java/io/atomix/copycat/client/DefaultCopycatClient.java deleted file mode 100644 index b938aa69..00000000 --- a/client/src/main/java/io/atomix/copycat/client/DefaultCopycatClient.java +++ /dev/null @@ -1,397 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License - */ -package io.atomix.copycat.client; - -import io.atomix.catalyst.concurrent.BlockingFuture; -import io.atomix.catalyst.concurrent.Futures; -import io.atomix.catalyst.concurrent.Listener; -import io.atomix.catalyst.concurrent.ThreadContext; -import io.atomix.catalyst.serializer.Serializer; -import io.atomix.catalyst.transport.Address; -import io.atomix.catalyst.transport.Transport; -import io.atomix.catalyst.util.Assert; -import io.atomix.copycat.Command; -import io.atomix.copycat.Query; -import io.atomix.copycat.client.session.ClientSession; -import io.atomix.copycat.client.util.AddressSelector; -import io.atomix.copycat.session.ClosedSessionException; -import io.atomix.copycat.session.Session; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Duration; -import java.util.Collection; -import java.util.Collections; -import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CopyOnWriteArraySet; -import java.util.function.Consumer; - -/** - * Default Copycat client implementation. - * - * @author cluster; - private final Transport transport; - private final ThreadContext ioContext; - private final ThreadContext eventContext; - private final AddressSelector selector; - private final Duration sessionTimeout; - private final Duration unstabilityTimeout; - private final ConnectionStrategy connectionStrategy; - private final RecoveryStrategy recoveryStrategy; - private ClientSession session; - private volatile State state = State.CLOSED; - private volatile CompletableFuture openFuture; - private volatile CompletableFuture recoverFuture; - private volatile CompletableFuture closeFuture; - private final Set changeListeners = new CopyOnWriteArraySet<>(); - private final Set> eventListeners = new CopyOnWriteArraySet<>(); - private Listener changeListener; - - DefaultCopycatClient(String clientId, Collection
cluster, Transport transport, ThreadContext ioContext, ThreadContext eventContext, ServerSelectionStrategy selectionStrategy, ConnectionStrategy connectionStrategy, RecoveryStrategy recoveryStrategy, Duration sessionTimeout, Duration unstabilityTimeout) { - this.clientId = Assert.notNull(clientId, "clientId"); - this.cluster = Assert.notNull(cluster, "cluster"); - this.transport = Assert.notNull(transport, "transport"); - this.ioContext = Assert.notNull(ioContext, "ioContext"); - this.eventContext = Assert.notNull(eventContext, "eventContext"); - this.selector = new AddressSelector(selectionStrategy); - this.connectionStrategy = Assert.notNull(connectionStrategy, "connectionStrategy"); - this.recoveryStrategy = Assert.notNull(recoveryStrategy, "recoveryStrategy"); - this.sessionTimeout = Assert.notNull(sessionTimeout, "sessionTimeout"); - this.unstabilityTimeout = Assert.notNull(unstabilityTimeout, "unstabilityTimeout");; - } - - @Override - public State state() { - return state; - } - - /** - * Updates the client state. - */ - private void setState(State state) { - if (this.state != state) { - this.state = state; - LOGGER.debug("State changed: {}", state); - changeListeners.forEach(l -> l.accept(state)); - } - } - - @Override - public Listener onStateChange(Consumer callback) { - return new StateChangeListener(callback); - } - - @Override - public Transport transport() { - return transport; - } - - @Override - public Serializer serializer() { - ThreadContext context = ThreadContext.currentContext(); - return context != null ? context.serializer() : this.eventContext.serializer(); - } - - @Override - public Session session() { - return session; - } - - @Override - public ThreadContext context() { - return eventContext; - } - - /** - * Creates a new child session. - */ - private ClientSession newSession() { - ClientSession session = new ClientSession(clientId, transport.client(), selector, ioContext, connectionStrategy, sessionTimeout, - unstabilityTimeout - ); - - // Update the session change listener. - if (changeListener != null) - changeListener.close(); - changeListener = session.onStateChange(this::onStateChange); - - // Register all event listeners. - eventListeners.forEach(l -> l.register(session)); - return session; - } - - /** - * Handles a session state change. - */ - private void onStateChange(Session.State state) { - switch (state) { - // When the session is opened, transition the state to CONNECTED. - case OPEN: - setState(State.CONNECTED); - break; - // When the session becomes unstable, transition the state to SUSPENDED. - case UNSTABLE: - setState(State.SUSPENDED); - break; - case STALE: - setState(State.SUSPENDED); - this.close(); - break; - // When the session is expired, transition the state to SUSPENDED if necessary. The recovery strategy - // must determine whether to attempt to recover the client. - case EXPIRED: - setState(State.SUSPENDED); - recoveryStrategy.recover(this); - break; - case CLOSED: - setState(State.CLOSED); - break; - default: - break; - } - } - - @Override - public synchronized CompletableFuture connect(Collection
cluster) { - if (state != State.CLOSED) - return CompletableFuture.completedFuture(this); - - if (openFuture == null) { - openFuture = new CompletableFuture<>(); - - // If the provided cluster list is null or empty, use the default list. - if (cluster == null || cluster.isEmpty()) { - cluster = this.cluster; - } - - // If the default list is null or empty, use the default host:port. - if (cluster == null || cluster.isEmpty()) { - cluster = Collections.singletonList(new Address(DEFAULT_HOST, DEFAULT_PORT)); - } - - // Reset the connection list to allow the selection strategy to prioritize connections. - selector.reset(null, cluster); - - // Create and register a new session. - session = newSession(); - session.register().whenCompleteAsync((result, error) -> { - if (error == null) { - openFuture.complete(this); - } else { - openFuture.completeExceptionally(error); - } - }, eventContext.executor()); - } - return openFuture; - } - - @Override - public CompletableFuture submit(Command command) { - ClientSession session = this.session; - if (session == null) - return Futures.exceptionalFuture(new ClosedSessionException("session closed")); - - BlockingFuture future = new BlockingFuture<>(); - session.submit(command).whenComplete((result, error) -> { - if (eventContext.isBlocked()) { - future.accept(result, error); - } else { - eventContext.executor().execute(() -> future.accept(result, error)); - } - }); - return future; - } - - @Override - public CompletableFuture submit(Query query) { - ClientSession session = this.session; - if (session == null) - return Futures.exceptionalFuture(new ClosedSessionException("session closed")); - - BlockingFuture future = new BlockingFuture<>(); - session.submit(query).whenComplete((result, error) -> { - if (eventContext.isBlocked()) { - future.accept(result, error); - } else { - eventContext.executor().execute(() -> future.accept(result, error)); - } - }); - return future; - } - - @Override - public Listener onEvent(String event, Runnable callback) { - return onEvent(event, v -> callback.run()); - } - - @Override - public Listener onEvent(String event, Consumer callback) { - EventListener listener = new EventListener<>(event, callback); - listener.register(session); - return listener; - } - - @Override - public synchronized CompletableFuture recover() { - if (recoverFuture == null) { - LOGGER.debug("Recovering session {}", this.session.id()); - recoverFuture = new CompletableFuture<>(); - session.close().whenCompleteAsync((closeResult, closeError) -> { - session = newSession(); - session.register().whenCompleteAsync((registerResult, registerError) -> { - CompletableFuture recoverFuture = this.recoverFuture; - if (registerError == null) { - recoverFuture.complete(this); - } else { - recoverFuture.completeExceptionally(registerError); - } - this.recoverFuture = null; - }, eventContext.executor()); - }, eventContext.executor()); - } - return recoverFuture; - } - - @Override - public synchronized CompletableFuture close() { - if (state == State.CLOSED) - return CompletableFuture.completedFuture(null); - - if (closeFuture == null) { - // Close the child session and call close listeners once complete. - closeFuture = new CompletableFuture<>(); - session.close().whenCompleteAsync((result, error) -> { - setState(State.CLOSED); - CompletableFuture.runAsync(() -> { - ioContext.close(); - eventContext.close(); - transport.close(); - if (error == null) { - closeFuture.complete(null); - } else { - closeFuture.completeExceptionally(error); - } - }); - }, eventContext.executor()); - } - return closeFuture; - } - - /** - * Kills the client. - * - * @return A completable future to be completed once the client's session has been killed. - */ - public synchronized CompletableFuture kill() { - if (state == State.CLOSED) - return CompletableFuture.completedFuture(null); - - if (closeFuture == null) { - closeFuture = session.kill() - .whenComplete((result, error) -> { - setState(State.CLOSED); - CompletableFuture.runAsync(() -> { - ioContext.close(); - eventContext.close(); - transport.close(); - }); - }); - } - return closeFuture; - } - - @Override - public int hashCode() { - return 23 + 37 * (session != null ? session.hashCode() : 0); - } - - @Override - public boolean equals(Object object) { - return object instanceof DefaultCopycatClient && ((DefaultCopycatClient) object).session() == session; - } - - @Override - public String toString() { - return String.format("%s[session=%s]", getClass().getSimpleName(), session); - } - - /** - * State change listener. - */ - private final class StateChangeListener implements Listener { - private final Consumer callback; - - protected StateChangeListener(Consumer callback) { - this.callback = callback; - changeListeners.add(this); - } - - @Override - public void accept(State state) { - eventContext.executor().execute(() -> callback.accept(state)); - } - - @Override - public void close() { - changeListeners.remove(this); - } - } - - /** - * Event listener wrapper. - */ - private final class EventListener implements Listener { - private final String event; - private final Consumer callback; - private Listener parent; - - private EventListener(String event, Consumer callback) { - this.event = event; - this.callback = callback; - eventListeners.add(this); - } - - /** - * Registers the session event listener. - */ - public void register(ClientSession session) { - parent = session.onEvent(event, this); - } - - @Override - public void accept(T message) { - if (eventContext.isBlocked()) { - callback.accept(message); - } else { - eventContext.executor().execute(() -> callback.accept(message)); - } - } - - @Override - public void close() { - parent.close(); - eventListeners.remove(this); - } - } -} diff --git a/client/src/main/java/io/atomix/copycat/client/RecoveryStrategies.java b/client/src/main/java/io/atomix/copycat/client/RecoveryStrategies.java deleted file mode 100644 index f30eb9d5..00000000 --- a/client/src/main/java/io/atomix/copycat/client/RecoveryStrategies.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License - */ -package io.atomix.copycat.client; - -/** - * Strategies for recovering lost client sessions. - *

- * Client recovery strategies are responsible for recovering a crashed client. When clients fail to contact - * a server for more than their session timeout, the client's session must be closed as linearizability is - * lost. The recovery strategy has the opportunity to recover the crashed client gracefully. - * - * @author - * Client recovery strategies are responsible for recovering a crashed client. When a client is unable - * to communicate with the cluster for some time period, the cluster may expire the client's session. - * In the event that a client reconnects and discovers its session is expired, the client's configured - * recovery strategy will be queried to determine how to handle the failure. Typically, recovery strategies - * can either {@link CopycatClient#recover() recover} or {@link CopycatClient#close() close} the client. - * - * @author l.accept(state)); + return this; + } + + /** + * Registers a state change listener on the session manager. + * + * @param callback The state change listener callback. + * @return The state change listener. + */ + public Listener onStateChange(Consumer callback) { + Listener listener = new Listener() { + @Override + public void accept(CopycatClient.State state) { + callback.accept(state); + } + @Override + public void close() { + changeListeners.remove(this); + } + }; + changeListeners.add(listener); + return listener; + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatClient.java b/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatClient.java new file mode 100644 index 00000000..ac91260c --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatClient.java @@ -0,0 +1,169 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client.impl; + +import io.atomix.catalyst.concurrent.Listener; +import io.atomix.catalyst.concurrent.ThreadContext; +import io.atomix.catalyst.concurrent.ThreadPoolContext; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.catalyst.transport.Address; +import io.atomix.catalyst.transport.Client; +import io.atomix.catalyst.util.Assert; +import io.atomix.copycat.client.ConnectionStrategy; +import io.atomix.copycat.client.CopycatClient; +import io.atomix.copycat.client.session.CopycatSession; +import io.atomix.copycat.client.session.impl.CopycatSessionManager; +import io.atomix.copycat.client.util.ClientConnectionManager; + +import java.time.Duration; +import java.util.Collection; +import java.util.Collections; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.Consumer; + +/** + * Default Copycat client implementation. + * + * @author cluster; + private final CopycatClientState state; + private final ThreadContext threadContext; + private final CopycatSessionManager sessionManager; + private volatile CompletableFuture openFuture; + private volatile CompletableFuture closeFuture; + + public DefaultCopycatClient(String clientId, Collection

cluster, Client client, ScheduledExecutorService threadPoolExecutor, Serializer serializer, ConnectionStrategy connectionStrategy, Duration sessionTimeout, Duration unstableTimeout) { + this.cluster = Assert.notNull(cluster, "cluster"); + this.threadContext = new ThreadPoolContext(threadPoolExecutor, serializer.clone()); + this.state = new CopycatClientState(clientId); + this.sessionManager = new CopycatSessionManager(state, new ClientConnectionManager(client), threadContext, threadPoolExecutor, connectionStrategy, sessionTimeout, unstableTimeout); + } + + @Override + public State state() { + return state.getState(); + } + + @Override + public Listener onStateChange(Consumer callback) { + return state.onStateChange(callback); + } + + @Override + public Serializer serializer() { + return threadContext.serializer(); + } + + @Override + public ThreadContext context() { + return threadContext; + } + + @Override + public synchronized CompletableFuture connect(Collection
cluster) { + if (state.getState() != State.CLOSED) + return CompletableFuture.completedFuture(this); + + if (openFuture == null) { + openFuture = new CompletableFuture<>(); + + // If the provided cluster list is null or empty, use the default list. + if (cluster == null || cluster.isEmpty()) { + cluster = this.cluster; + } + + // If the default list is null or empty, use the default host:port. + if (cluster == null || cluster.isEmpty()) { + cluster = Collections.singletonList(new Address(DEFAULT_HOST, DEFAULT_PORT)); + } + + // Reset the connection list to allow the selection strategy to prioritize connections. + sessionManager.resetConnections(null, cluster); + + // Register the session manager. + sessionManager.open().whenCompleteAsync((result, error) -> { + if (error == null) { + openFuture.complete(this); + } else { + openFuture.completeExceptionally(error); + } + }, threadContext); + } + return openFuture; + } + + @Override + public CopycatSession.Builder sessionBuilder() { + return new SessionBuilder(); + } + + @Override + public synchronized CompletableFuture close() { + if (state.getState() == State.CLOSED) + return CompletableFuture.completedFuture(null); + + if (closeFuture == null) { + closeFuture = sessionManager.close(); + } + return closeFuture; + } + + /** + * Kills the client. + * + * @return A completable future to be completed once the client's session has been killed. + */ + public synchronized CompletableFuture kill() { + if (state.getState() == State.CLOSED) + return CompletableFuture.completedFuture(null); + + if (closeFuture == null) { + closeFuture = sessionManager.kill(); + } + return closeFuture; + } + + @Override + public int hashCode() { + return 23 + 37 * state.getUuid().hashCode(); + } + + @Override + public boolean equals(Object object) { + return object instanceof DefaultCopycatClient && ((DefaultCopycatClient) object).state.getUuid().equals(state.getUuid()); + } + + @Override + public String toString() { + return String.format("%s[id=%d, uuid=%s]", getClass().getSimpleName(), state.getId(), state.getUuid()); + } + + /** + * Default Copycat session builder. + */ + private class SessionBuilder extends CopycatSession.Builder { + @Override + public CopycatSession build() { + return sessionManager.openSession(name, type, communicationStrategy).join(); + } + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/session/ClientSessionManager.java b/client/src/main/java/io/atomix/copycat/client/session/ClientSessionManager.java deleted file mode 100644 index 21aede07..00000000 --- a/client/src/main/java/io/atomix/copycat/client/session/ClientSessionManager.java +++ /dev/null @@ -1,366 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License - */ -package io.atomix.copycat.client.session; - -import io.atomix.catalyst.concurrent.Scheduled; -import io.atomix.catalyst.concurrent.ThreadContext; -import io.atomix.catalyst.util.Assert; -import io.atomix.copycat.client.ConnectionStrategy; -import io.atomix.copycat.client.util.ClientConnection; -import io.atomix.copycat.error.CopycatError; -import io.atomix.copycat.protocol.*; -import io.atomix.copycat.session.ClosedSessionException; -import io.atomix.copycat.session.Session; - -import java.net.ConnectException; -import java.time.Duration; -import java.util.concurrent.CompletableFuture; - -/** - * Client session manager. - * - * @author open() { - CompletableFuture future = new CompletableFuture<>(); - context.executor().execute(() -> register(new RegisterAttempt(1, future))); - return future; - } - - /** - * Expires the manager. - * - * @return A completable future to be completed once the session has been expired. - */ - public CompletableFuture expire() { - CompletableFuture future = new CompletableFuture<>(); - context.executor().execute(() -> { - if (keepAlive != null) - keepAlive.cancel(); - state.setState(Session.State.EXPIRED); - future.complete(null); - }); - return future; - } - - /** - * Registers a session. - */ - private void register(RegisterAttempt attempt) { - state.getLogger().debug("Registering session: attempt {}", attempt.attempt); - - RegisterRequest request = RegisterRequest.builder() - .withClient(state.getClientId()) - .withTimeout(sessionTimeout.toMillis()) - .build(); - - state.getLogger().trace("Sending {}", request); - connection.reset().sendAndReceive(request).whenComplete((response, error) -> { - if (error == null) { - state.getLogger().trace("Received {}", response); - if (response.status() == Response.Status.OK) { - interval = Duration.ofMillis(response.timeout()).dividedBy(2); - connection.reset(response.leader(), response.members()); - state.setSessionId(response.session()) - .setState(Session.State.OPEN); - state.getLogger().info("Registered session {}", response.session()); - attempt.complete(); - keepAlive(); - } else { - strategy.attemptFailed(attempt); - } - } else { - strategy.attemptFailed(attempt); - } - }); - } - - /** - * Sends a keep-alive request to the cluster. - */ - private void keepAlive() { - keepAlive(true); - } - - /** - * Sends a keep-alive request to the cluster. - */ - private void keepAlive(boolean retryOnFailure) { - long sessionId = state.getSessionId(); - - // If the current sessions state is unstable, reset the connection before sending a keep-alive. - if (state.getState() == Session.State.UNSTABLE) - connection.reset(); - - KeepAliveRequest request = KeepAliveRequest.builder() - .withSession(sessionId) - .withCommandSequence(state.getCommandResponse()) - .withEventIndex(state.getEventIndex()) - .build(); - - state.getLogger().trace("{} - Sending {}", sessionId, request); - connection.sendAndReceive(request).whenComplete((response, error) -> { - if (state.getState() != Session.State.CLOSED) { - if (error == null) { - state.getLogger().trace("{} - Received {}", sessionId, response); - // If the request was successful, update the address selector and schedule the next keep-alive. - if (response.status() == Response.Status.OK) { - connection.reset(response.leader(), response.members()); - state.setState(Session.State.OPEN); - scheduleKeepAlive(); - } - // If the session is unknown, immediate expire the session. - else if (response.error() == CopycatError.Type.UNKNOWN_SESSION_ERROR) { - state.setState(Session.State.EXPIRED); - } - // If a leader is still set in the address selector, unset the leader and attempt to send another keep-alive. - // This will ensure that the address selector selects all servers without filtering on the leader. - else if (retryOnFailure && connection.leader() != null) { - connection.reset(null, connection.servers()); - keepAlive(false); - } - // If no leader was set, set the session state to unstable and schedule another keep-alive. - else { - state.setState(Session.State.UNSTABLE); - scheduleKeepAlive(); - } - } - // If a leader is still set in the address selector, unset the leader and attempt to send another keep-alive. - // This will ensure that the address selector selects all servers without filtering on the leader. - else if (retryOnFailure && connection.leader() != null) { - connection.reset(null, connection.servers()); - keepAlive(false); - } - // If no leader was set, set the session state to unstable and schedule another keep-alive. - else { - state.setState(Session.State.UNSTABLE); - scheduleKeepAlive(); - } - } - }); - } - - /** - * Schedules a keep-alive request. - */ - private void scheduleKeepAlive() { - if (keepAlive != null) - keepAlive.cancel(); - keepAlive = context.schedule(interval, () -> { - keepAlive = null; - if (state.getState().active()) { - keepAlive(); - } - }); - } - - /** - * Closes the session manager. - * - * @return A completable future to be completed once the session manager is closed. - */ - public CompletableFuture close() { - if (state.getState() == Session.State.EXPIRED) - return CompletableFuture.completedFuture(null); - - CompletableFuture future = new CompletableFuture<>(); - context.executor().execute(() -> { - if (keepAlive != null) { - keepAlive.cancel(); - keepAlive = null; - } - unregister(future); - }); - return future; - } - - /** - * Unregisters the session. - */ - private void unregister(CompletableFuture future) { - unregister(true, future); - } - - /** - * Unregisters the session. - * - * @param future A completable future to be completed once the session is unregistered. - */ - private void unregister(boolean retryOnFailure, CompletableFuture future) { - long sessionId = state.getSessionId(); - - // If the session is already closed, skip the unregister attempt. - if (state.getState() == Session.State.CLOSED) { - future.complete(null); - return; - } - - state.getLogger().debug("Unregistering session: {}", sessionId); - - // If a keep-alive request is already pending, cancel it. - if (keepAlive != null) { - keepAlive.cancel(); - keepAlive = null; - } - - // If the current sessions state is unstable, reset the connection before sending an unregister request. - if (state.getState() == Session.State.UNSTABLE) { - connection.reset(); - } - - UnregisterRequest request = UnregisterRequest.builder() - .withSession(sessionId) - .build(); - - state.getLogger().trace("{} - Sending {}", sessionId, request); - connection.sendAndReceive(request).whenComplete((response, error) -> { - if (state.getState() != Session.State.CLOSED) { - if (error == null) { - state.getLogger().trace("{} - Received {}", sessionId, response); - // If the request was successful, update the session state and complete the close future. - if (response.status() == Response.Status.OK) { - state.setState(Session.State.CLOSED); - future.complete(null); - } - // If the session is unknown, immediate expire the session and complete the close future. - else if (response.error() == CopycatError.Type.UNKNOWN_SESSION_ERROR) { - state.setState(Session.State.EXPIRED); - future.complete(null); - } - // If a leader is still set in the address selector, unset the leader and send another unregister attempt. - // This will ensure that the address selector selects all servers without filtering on the leader. - else if (retryOnFailure && connection.leader() != null) { - connection.reset(null, connection.servers()); - unregister(false, future); - } - // If no leader was set, set the session state to unstable and fail the unregister attempt. - else { - state.setState(Session.State.UNSTABLE); - future.completeExceptionally(new ClosedSessionException("failed to unregister session")); - } - } - // If a leader is still set in the address selector, unset the leader and send another unregister attempt. - // This will ensure that the address selector selects all servers without filtering on the leader. - else if (retryOnFailure && connection.leader() != null) { - connection.reset(null, connection.servers()); - unregister(false, future); - } - // If no leader was set, set the session state to unstable and schedule another unregister attempt. - else { - state.setState(Session.State.UNSTABLE); - future.completeExceptionally(new ClosedSessionException("failed to unregister session")); - } - } - }); - } - - /** - * Kills the client session manager. - * - * @return A completable future to be completed once the session manager is killed. - */ - public CompletableFuture kill() { - return CompletableFuture.runAsync(() -> { - if (keepAlive != null) - keepAlive.cancel(); - state.setState(Session.State.CLOSED); - }, context.executor()); - } - - @Override - public String toString() { - return String.format("%s[session=%d]", getClass().getSimpleName(), state.getSessionId()); - } - - /** - * Client session connection attempt. - */ - private final class RegisterAttempt implements ConnectionStrategy.Attempt { - private final int attempt; - private final CompletableFuture future; - - private RegisterAttempt(int attempt, CompletableFuture future) { - this.attempt = attempt; - this.future = future; - } - - @Override - public int attempt() { - return attempt; - } - - /** - * Completes the attempt successfully. - */ - public void complete() { - complete(null); - } - - /** - * Completes the attempt successfully. - * - * @param result The attempt result. - */ - public void complete(Void result) { - future.complete(result); - } - - @Override - public void fail() { - future.completeExceptionally(new ConnectException("failed to register session")); - } - - @Override - public void fail(Throwable error) { - future.completeExceptionally(error); - } - - @Override - public void retry() { - state.getLogger().debug("Retrying session register attempt"); - register(new RegisterAttempt(attempt + 1, future)); - } - - @Override - public void retry(Duration after) { - state.getLogger().debug("Retrying session register attempt"); - context.schedule(after, () -> register(new RegisterAttempt(attempt + 1, future))); - } - } - -} diff --git a/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java b/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java new file mode 100644 index 00000000..83576ab6 --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java @@ -0,0 +1,213 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client.session; + +import io.atomix.catalyst.concurrent.Listener; +import io.atomix.catalyst.util.Assert; +import io.atomix.catalyst.util.Builder; +import io.atomix.copycat.Command; +import io.atomix.copycat.Operation; +import io.atomix.copycat.Query; +import io.atomix.copycat.client.CommunicationStrategies; +import io.atomix.copycat.client.CommunicationStrategy; +import io.atomix.copycat.client.CopycatClient; + +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +/** + * Copycat client proxy. + */ +public interface CopycatSession { + + /** + * Returns the unique session identifier. + * + * @return The session identifier. + */ + long id(); + + /** + * Returns the client proxy name. + * + * @return The client proxy name. + */ + String name(); + + /** + * Returns the client proxy type. + * + * @return The client proxy type. + */ + String type(); + + /** + * Submits an operation to the Copycat cluster. + *

+ * This method is provided for convenience. The submitted {@link Operation} must be an instance + * of {@link Command} or {@link Query}. + * + * @param operation The operation to submit. + * @param The operation result type. + * @return A completable future to be completed with the operation result. + * @throws IllegalArgumentException If the {@link Operation} is not an instance of {@link Command} or {@link Query}. + * @throws NullPointerException if {@code operation} is null + */ + default CompletableFuture submit(Operation operation) { + Assert.notNull(operation, "operation"); + if (operation instanceof Command) { + return submit((Command) operation); + } else if (operation instanceof Query) { + return submit((Query) operation); + } else { + throw new IllegalArgumentException("unknown operation type"); + } + } + + /** + * Submits a command to the Copycat cluster. + *

+ * Commands are used to alter state machine state. All commands will be forwarded to the current cluster leader. + * Once a leader receives the command, it will write the command to its internal {@code Log} and replicate it to a majority + * of the cluster. Once the command has been replicated to a majority of the cluster, it will apply the command to its + * {@code StateMachine} and respond with the result. + *

+ * Once the command has been applied to a server state machine, the returned {@link CompletableFuture} + * will be completed with the state machine output. + *

+ * Note that all client submissions are guaranteed to be completed in the same order in which they were sent (program order) + * and on the same thread. This does not, however, mean that they'll be applied to the server-side replicated state machine + * in that order. + * + * @param command The command to submit. + * @param The command result type. + * @return A completable future to be completed with the command result. The future is guaranteed to be completed after all + * {@link Command} or {@link Query} submission futures that preceded it. The future will always be completed on the + * @throws NullPointerException if {@code command} is null + */ + CompletableFuture submit(Command command); + + /** + * Submits a query to the Copycat cluster. + *

+ * Queries are used to read state machine state. The behavior of query submissions is primarily dependent on the + * query's {@link Query.ConsistencyLevel}. For {@link Query.ConsistencyLevel#LINEARIZABLE} + * and {@link Query.ConsistencyLevel#LINEARIZABLE_LEASE} consistency levels, queries will be forwarded + * to the cluster leader. For lower consistency levels, queries are allowed to read from followers. All queries are executed + * by applying queries to an internal server state machine. + *

+ * Once the query has been applied to a server state machine, the returned {@link CompletableFuture} + * will be completed with the state machine output. + * + * @param query The query to submit. + * @param The query result type. + * @return A completable future to be completed with the query result. The future is guaranteed to be completed after all + * {@link Command} or {@link Query} submission futures that preceded it. The future will always be completed on the + * @throws NullPointerException if {@code query} is null + */ + CompletableFuture submit(Query query); + + /** + * Registers a void event listener. + *

+ * The registered {@link Runnable} will be {@link Runnable#run() called} when an event is received + * from the Raft cluster for the client. {@link CopycatClient} implementations must guarantee that consumers are + * always called in the same thread for the session. Therefore, no two events will be received concurrently + * by the session. Additionally, events are guaranteed to be received in the order in which they were sent by + * the state machine. + * + * @param event The event to which to listen. + * @param callback The session receive callback. + * @return The listener context. + * @throws NullPointerException if {@code event} or {@code callback} is null + */ + Listener onEvent(String event, Runnable callback); + + /** + * Registers an event listener. + *

+ * The registered {@link Consumer} will be {@link Consumer#accept(Object) called} when an event is received + * from the Raft cluster for the session. {@link CopycatClient} implementations must guarantee that consumers are + * always called in the same thread for the session. Therefore, no two events will be received concurrently + * by the session. Additionally, events are guaranteed to be received in the order in which they were sent by + * the state machine. + * + * @param event The event to which to listen. + * @param callback The session receive callback. + * @param The session event type. + * @return The listener context. + * @throws NullPointerException if {@code event} or {@code callback} is null + */ + Listener onEvent(String event, Consumer callback); + + /** + * Returns a boolean indicating whether the session is open. + * + * @return Indicates whether the session is open. + */ + boolean isOpen(); + + /** + * Closes the session. + * + * @return A completable future to be completed once the session is closed. + */ + CompletableFuture close(); + + /** + * Copycat session builder. + */ + abstract class Builder implements io.atomix.catalyst.util.Builder { + protected String name; + protected String type; + protected CommunicationStrategy communicationStrategy = CommunicationStrategies.LEADER; + + /** + * Sets the session name. + * + * @param name The session name. + * @return The session builder. + */ + public Builder withName(String name) { + this.name = Assert.notNull(name, "name"); + return this; + } + + /** + * Sets the session type. + * + * @param type The session type. + * @return The session builder. + */ + public Builder withType(String type) { + this.type = Assert.notNull(type, "type"); + return this; + } + + /** + * Sets the session's communication strategy. + * + * @param communicationStrategy The session's communication strategy. + * @return The session builder. + * @throws NullPointerException if the communication strategy is null + */ + public Builder withCommunicationStrategy(CommunicationStrategy communicationStrategy) { + this.communicationStrategy = Assert.notNull(communicationStrategy, "communicationStrategy"); + return this; + } + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatClientConnection.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatClientConnection.java new file mode 100644 index 00000000..7dcdfcfe --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatClientConnection.java @@ -0,0 +1,43 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client.session.impl; + +import io.atomix.copycat.client.util.AddressSelector; +import io.atomix.copycat.client.util.ClientConnectionManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Client connection. + */ +public class CopycatClientConnection extends CopycatConnection { + private static final Logger LOGGER = LoggerFactory.getLogger(CopycatClientConnection.class); + + public CopycatClientConnection(ClientConnectionManager connections, AddressSelector selector) { + super(connections, selector); + } + + @Override + protected String name() { + return "client"; + } + + @Override + protected Logger logger() { + return LOGGER; + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatConnection.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatConnection.java new file mode 100644 index 00000000..d02c7645 --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatConnection.java @@ -0,0 +1,363 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client.session.impl; + +import io.atomix.catalyst.transport.Address; +import io.atomix.catalyst.transport.Connection; +import io.atomix.catalyst.transport.TransportException; +import io.atomix.catalyst.util.Assert; +import io.atomix.copycat.client.util.AddressSelector; +import io.atomix.copycat.client.util.ClientConnectionManager; +import io.atomix.copycat.client.util.OrderedCompletableFuture; +import io.atomix.copycat.error.CopycatError; +import io.atomix.copycat.protocol.Request; +import io.atomix.copycat.protocol.Response; +import org.slf4j.Logger; + +import java.net.ConnectException; +import java.nio.channels.ClosedChannelException; +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeoutException; +import java.util.function.BiFunction; +import java.util.function.Consumer; +import java.util.function.Function; + +/** + * Client connection that recursively connects to servers in the cluster and attempts to submit requests. + * + * @author servers() { + return selector.servers(); + } + + /** + * Resets the client connection. + * + * @return The client connection. + */ + public CopycatConnection reset() { + selector.reset(); + return this; + } + + /** + * Resets the client connection. + * + * @param leader The current cluster leader. + * @param servers The current servers. + * @return The client connection. + */ + public CopycatConnection reset(Address leader, Collection

servers) { + selector.reset(leader, servers); + return this; + } + + /** + * Opens the connection. + * + * @return A completable future to be completed once the connection is opened. + */ + public CompletableFuture open() { + open = true; + return connect().thenApply(c -> null); + } + + /** + * Sends a request to the cluster. + * + * @param type The request type. + * @param request The request to send. + * @return A completable future to be completed with the response. + */ + public CompletableFuture send(String type, Object request) { + CompletableFuture future = new CompletableFuture<>(); + sendRequest((Request) request, (r, c) -> c.send(type, r), future); + return future; + } + + /** + * Sends a request to the cluster and awaits a response. + * + * @param type The request type. + * @param request The request to send. + * @param The request type. + * @param The response type. + * @return A completable future to be completed with the response. + */ + public CompletableFuture sendAndReceive(String type, T request) { + CompletableFuture future = new CompletableFuture<>(); + sendRequest((Request) request, (r, c) -> c.sendAndReceive(type, r), future); + return future; + } + + /** + * Sends the given request attempt to the cluster. + */ + protected void sendRequest(T request, BiFunction> sender, CompletableFuture future) { + if (open) { + connect().whenComplete((c, e) -> sendRequest(request, sender, c, e, future)); + } + } + + /** + * Sends the given request attempt to the cluster via the given connection if connected. + */ + protected void sendRequest(T request, BiFunction> sender, Connection connection, Throwable error, CompletableFuture future) { + if (error == null) { + if (connection != null) { + logger().trace("{} - Sending {}", name(), request); + sender.apply(request, connection).whenComplete((r, e) -> { + if (e != null || r != null) { + handleResponse(request, sender, connection, (Response) r, e, future); + } else { + future.complete(null); + } + }); + } else { + future.completeExceptionally(new ConnectException("Failed to connect to the cluster")); + } + } else { + logger().trace("{} - Resending {}: {}", name(), request, error); + resendRequest(error, request, sender, connection, future); + } + } + + /** + * Resends a request due to a request failure, resetting the connection if necessary. + */ + @SuppressWarnings("unchecked") + protected void resendRequest(Throwable cause, T request, BiFunction sender, Connection connection, CompletableFuture future) { + // If the connection has not changed, reset it and connect to the next server. + if (this.connection == connection) { + logger().trace("{} - Resetting connection. Reason: {}", name(), cause); + this.connection = null; + } + + // Create a new connection and resend the request. This will force retries to piggyback on any existing + // connect attempts. + connect().whenComplete((c, e) -> sendRequest(request, sender, c, e, future)); + } + + /** + * Handles a response from the cluster. + */ + @SuppressWarnings("unchecked") + protected void handleResponse(T request, BiFunction sender, Connection connection, Response response, Throwable error, CompletableFuture future) { + if (error == null) { + if (response.status() == Response.Status.OK + || response.error() == CopycatError.Type.COMMAND_ERROR + || response.error() == CopycatError.Type.QUERY_ERROR + || response.error() == CopycatError.Type.APPLICATION_ERROR + || response.error() == CopycatError.Type.UNKNOWN_CLIENT_ERROR + || response.error() == CopycatError.Type.UNKNOWN_SESSION_ERROR + || response.error() == CopycatError.Type.UNKNOWN_STATE_MACHINE_ERROR + || response.error() == CopycatError.Type.INTERNAL_ERROR) { + logger().trace("{} - Received {}", name(), response); + future.complete(response); + } else { + resendRequest(response.error().createException(), request, sender, connection, future); + } + } else if (error instanceof ConnectException || error instanceof TimeoutException || error instanceof TransportException || error instanceof ClosedChannelException) { + resendRequest(error, request, sender, connection, future); + } else { + logger().debug("{} - {} failed! Reason: {}", name(), request, error); + future.completeExceptionally(error); + } + } + + /** + * Connects to the cluster. + */ + protected CompletableFuture connect() { + // If the address selector has been reset then reset the connection. + if (selector.state() == AddressSelector.State.RESET && connection != null) { + if (connectFuture != null) { + return connectFuture; + } + + CompletableFuture future = new OrderedCompletableFuture<>(); + future.whenComplete((r, e) -> this.connectFuture = null); + this.connectFuture = future; + + this.connection = null; + connect(future); + return future; + } + + // If a connection was already established then use that connection. + if (connection != null) { + return CompletableFuture.completedFuture(connection); + } + + // If a connection is currently being established then piggyback on the connect future. + if (connectFuture != null) { + return connectFuture; + } + + // Create a new connect future and connect to the first server in the cluster. + CompletableFuture future = new OrderedCompletableFuture<>(); + future.whenComplete((r, e) -> this.connectFuture = null); + this.connectFuture = future; + reset().connect(future); + return future; + } + + /** + * Attempts to connect to the cluster. + */ + protected void connect(CompletableFuture future) { + if (!selector.hasNext()) { + logger().debug("{} - Failed to connect to the cluster", name()); + future.complete(null); + } else { + Address address = selector.next(); + logger().debug("{} - Connecting to {}", name(), address); + connections.getConnection(address).whenComplete((c, e) -> handleConnection(address, c, e, future)); + } + } + + /** + * Handles a connection to a server. + */ + protected void handleConnection(Address address, Connection connection, Throwable error, CompletableFuture future) { + if (error == null) { + setupConnection(address, connection, future); + } else { + logger().debug("{} - Failed to connect! Reason: {}", name(), error); + connect(future); + } + } + + /** + * Sets up the given connection. + */ + @SuppressWarnings("unchecked") + protected void setupConnection(Address address, Connection connection, CompletableFuture future) { + logger().debug("{} - Setting up connection to {}", name(), address); + + this.connection = connection; + + connection.onClose(c -> { + if (c.equals(this.connection)) { + logger().debug("{} - Connection closed", name()); + this.connection = null; + connect(); + } + }); + connection.onException(c -> { + if (c.equals(this.connection)) { + logger().debug("{} - Connection lost", name()); + this.connection = null; + connect(); + } + }); + + for (Map.Entry entry : handlers.entrySet()) { + connection.registerHandler(entry.getKey(), entry.getValue()); + } + future.complete(connection); + } + + /** + * Registers a handler for the given message type. + * + * @param type The message type for which to register the handler. + * @param handler The handler to register. + * @param The handler type. + * @return The client connection. + */ + @SuppressWarnings("unchecked") + public CopycatConnection registerHandler(String type, Consumer handler) { + return registerHandler(type, r -> { + handler.accept((T) r); + return null; + }); + } + + /** + * Registers a handler for the given message type. + * + * @param type The message type for which to register the handler. + * @param handler The handler to register. + * @param The handler type. + * @param The response type. + * @return The client connection. + */ + public CopycatConnection registerHandler(String type, Function> handler) { + Assert.notNull(type, "type"); + Assert.notNull(handler, "handler"); + handlers.put(type, handler); + if (connection != null) + connection.registerHandler(type, handler); + return this; + } + + /** + * Closes the connection. + * + * @return A completable future to be completed once the connection is closed. + */ + public CompletableFuture close() { + open = false; + return CompletableFuture.completedFuture(null); + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatLeaderConnection.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatLeaderConnection.java new file mode 100644 index 00000000..e3dec3f7 --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatLeaderConnection.java @@ -0,0 +1,45 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client.session.impl; + +import io.atomix.copycat.client.util.AddressSelector; +import io.atomix.copycat.client.util.ClientConnectionManager; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Leader connection. + */ +public class CopycatLeaderConnection extends CopycatConnection { + private static final Logger LOGGER = LoggerFactory.getLogger(CopycatLeaderConnection.class); + private final String sessionString; + + public CopycatLeaderConnection(CopycatSessionState state, ClientConnectionManager connections, AddressSelector selector) { + super(connections, selector); + this.sessionString = String.valueOf(state.getSessionId()); + } + + @Override + protected String name() { + return sessionString; + } + + @Override + protected Logger logger() { + return LOGGER; + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionConnection.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionConnection.java new file mode 100644 index 00000000..043dd577 --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionConnection.java @@ -0,0 +1,115 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client.session.impl; + +import io.atomix.catalyst.transport.Address; +import io.atomix.catalyst.transport.Connection; +import io.atomix.copycat.client.util.AddressSelector; +import io.atomix.copycat.client.util.ClientConnectionManager; +import io.atomix.copycat.protocol.ConnectRequest; +import io.atomix.copycat.protocol.ConnectResponse; +import io.atomix.copycat.protocol.Response; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.function.Function; + +/** + * Session connection. + */ +public class CopycatSessionConnection extends CopycatConnection { + private static final Logger LOGGER = LoggerFactory.getLogger(CopycatSessionConnection.class); + private final CopycatSessionState state; + private final String sessionString; + + public CopycatSessionConnection(CopycatSessionState state, ClientConnectionManager connections, AddressSelector selector) { + super(connections, selector); + this.state = state; + this.sessionString = String.valueOf(state.getSessionId()); + } + + @Override + protected String name() { + return sessionString; + } + + @Override + protected Logger logger() { + return LOGGER; + } + + @Override + @SuppressWarnings("unchecked") + protected void setupConnection(Address address, Connection connection, CompletableFuture future) { + logger().debug("{} - Setting up connection to {}", name(), address); + + this.connection = connection; + + connection.onClose(c -> { + if (c.equals(this.connection)) { + logger().debug("{} - Connection closed", name()); + this.connection = null; + connect(); + } + }); + connection.onException(c -> { + if (c.equals(this.connection)) { + logger().debug("{} - Connection lost", name()); + this.connection = null; + connect(); + } + }); + + for (Map.Entry entry : handlers.entrySet()) { + connection.registerHandler(entry.getKey(), entry.getValue()); + } + + // When we first connect to a new server, first send a ConnectRequest to the server to establish + // the connection with the server-side state machine. + ConnectRequest request = ConnectRequest.builder() + .withSession(state.getSessionId()) + .withConnection(state.nextConnection()) + .build(); + + logger().trace("{} - Sending {}", name(), request); + connection.sendAndReceive(ConnectRequest.NAME, request).whenComplete((r, e) -> handleConnectResponse(r, e, future)); + } + + /** + * Handles a connect response. + */ + private void handleConnectResponse(ConnectResponse response, Throwable error, CompletableFuture future) { + if (open) { + if (error == null) { + logger().trace("{} - Received {}", name(), response); + // If the connection was successfully created, immediately send a keep-alive request + // to the server to ensure we maintain our session and get an updated list of server addresses. + if (response.status() == Response.Status.OK) { + selector.reset(response.leader(), response.members()); + future.complete(connection); + } else { + connect(future); + } + } else { + logger().debug("{} - Failed to connect! Reason: {}", name(), error); + connect(future); + } + } + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/session/ClientSessionListener.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionListener.java similarity index 77% rename from client/src/main/java/io/atomix/copycat/client/session/ClientSessionListener.java rename to client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionListener.java index 043bd6b3..9877fe2f 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/ClientSessionListener.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionListener.java @@ -1,27 +1,28 @@ /* - * Copyright 2015 the original author or authors. + * Copyright 2017-present Open Networking Laboratory * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License + * limitations under the License. */ -package io.atomix.copycat.client.session; +package io.atomix.copycat.client.session.impl; import io.atomix.catalyst.concurrent.Listener; import io.atomix.catalyst.concurrent.ThreadContext; -import io.atomix.catalyst.transport.Connection; import io.atomix.catalyst.util.Assert; import io.atomix.copycat.protocol.PublishRequest; import io.atomix.copycat.protocol.ResetRequest; import io.atomix.copycat.session.Event; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.Map; import java.util.Set; @@ -35,19 +36,23 @@ * * @author servers) { + selectorManager.resetAll(leader, servers); + } + + /** + * Sets the session manager state. + * + * @param state The session manager state. + */ + private void setState(State state) { + this.state = state; + switch (state) { + case OPEN: + clientState.setState(CopycatClient.State.CONNECTED); + break; + case UNSTABLE: + clientState.setState(CopycatClient.State.SUSPENDED); + if (unstableTime == null) { + unstableTime = System.currentTimeMillis(); + } else if (System.currentTimeMillis() - unstableTime > unstableTimeout.toMillis()) { + setState(State.EXPIRED); + } + break; + case EXPIRED: + clientState.setState(CopycatClient.State.SUSPENDED); + break; + case CLOSED: + clientState.setState(CopycatClient.State.CLOSED); + break; + } + } + + /** + * Opens the session manager. + * + * @return A completable future to be called once the session manager is opened. + */ + public CompletableFuture open() { + CompletableFuture future = new CompletableFuture<>(); + threadContext.execute(() -> connection.open().whenComplete((result, error) -> { + if (error == null) { + registerClient(new RegisterAttempt(1, future)); + } else { + future.completeExceptionally(error); + } + })); + return future; + } + + /** + * Opens a new session. + * + * @param name The session name. + * @param type The session type. + * @param communicationStrategy The strategy with which to communicate with servers. + * @return A completable future to be completed once the session has been opened. + */ + public CompletableFuture openSession(String name, String type, CommunicationStrategy communicationStrategy) { + LOG.trace("Opening session; name: {}, type: {}", name, type); + OpenSessionRequest request = OpenSessionRequest.builder() + .withClient(clientState.getId()) + .withType(type) + .withName(name) + .build(); + + LOG.trace("Sending {}", request); + CompletableFuture future = new CompletableFuture<>(); + ThreadContext threadContext = new ThreadPoolContext(threadPoolExecutor, this.threadContext.serializer().clone()); + Runnable callback = () -> connection.sendAndReceive(OpenSessionRequest.NAME, request).whenCompleteAsync((response, error) -> { + if (error == null) { + if (response.status() == Response.Status.OK) { + CopycatSessionState state = new CopycatSessionState(response.session(), name, type); + sessions.put(state.getSessionId(), state); + CopycatConnection leaderConnection = new CopycatLeaderConnection(state, connectionManager, selectorManager.createSelector(CommunicationStrategies.LEADER)); + CopycatConnection sessionConnection = new CopycatSessionConnection(state, connectionManager, selectorManager.createSelector(communicationStrategy)); + leaderConnection.open().thenCompose(v -> sessionConnection.open()).whenComplete((connectResult, connectError) -> { + if (connectError == null) { + future.complete(new DefaultCopycatSession(state, leaderConnection, sessionConnection, threadContext, this)); + } else { + future.completeExceptionally(connectError); + } + }); + } else { + future.completeExceptionally(response.error().createException()); + } + } else { + future.completeExceptionally(error); + } + }, threadContext); + + if (threadContext.isCurrentContext()) { + callback.run(); + } else { + threadContext.execute(callback); + } + return future; + } + + /** + * Closes a session. + * + * @param sessionId The session identifier. + * @return A completable future to be completed once the session is closed. + */ + public CompletableFuture closeSession(long sessionId) { + CopycatSessionState state = sessions.get(sessionId); + if (state == null) { + return Futures.exceptionalFuture(new UnknownSessionException("Unknown session: " + sessionId)); + } + + LOG.trace("Closing session {}", sessionId); + CloseSessionRequest request = CloseSessionRequest.builder() + .withSession(sessionId) + .build(); + + LOG.trace("Sending {}", request); + CompletableFuture future = new CompletableFuture<>(); + Runnable callback = () -> connection.sendAndReceive(CloseSessionRequest.NAME, request).whenComplete((response, error) -> { + if (error == null) { + if (response.status() == Response.Status.OK) { + sessions.remove(sessionId); + future.complete(null); + } else { + future.completeExceptionally(response.error().createException()); + } + } else { + future.completeExceptionally(error); + } + }); + + if (threadContext.isCurrentContext()) { + callback.run(); + } else { + threadContext.execute(callback); + } + return future; + } + + /** + * Expires the manager. + * + * @return A completable future to be completed once the session has been expired. + */ + CompletableFuture expireSessions() { + CompletableFuture future = new CompletableFuture<>(); + threadContext.execute(() -> { + if (keepAlive != null) + keepAlive.cancel(); + setState(State.EXPIRED); + future.complete(null); + }); + return future; + } + + /** + * Registers a session. + */ + private void registerClient(RegisterAttempt attempt) { + LOG.debug("Registering client: attempt {}", attempt.attempt); + + RegisterRequest request = RegisterRequest.builder() + .withClient(clientState.getUuid()) + .withTimeout(sessionTimeout.toMillis()) + .build(); + + LOG.trace("{} - Sending {}", clientState.getUuid(), request); + selectorManager.resetAll(); + connection.sendAndReceive(RegisterRequest.NAME, request).whenComplete((response, error) -> { + if (error == null) { + LOG.trace("{} - Received {}", clientState.getUuid(), response); + if (response.status() == Response.Status.OK) { + clientState.setId(response.clientId()); + interval = Duration.ofMillis(response.timeout()).dividedBy(2); + selectorManager.resetAll(response.leader(), response.members()); + setState(State.OPEN); + LOG.info("{} - Registered client {}", clientState.getUuid(), clientState.getId()); + attempt.complete(); + keepAliveSessions(); + } else { + strategy.attemptFailed(attempt); + } + } else { + strategy.attemptFailed(attempt); + } + }); + } + + /** + * Resets indexes for the given session. + * + * @param sessionId The session for which to reset indexes. + * @return A completable future to be completed once the session's indexes have been reset. + */ + CompletableFuture resetIndexes(long sessionId) { + CopycatSessionState sessionState = sessions.get(sessionId); + if (sessionState == null) { + return Futures.exceptionalFuture(new IllegalArgumentException("Unknown session: " + sessionId)); + } + + CompletableFuture future = new CompletableFuture<>(); + + KeepAliveRequest request = KeepAliveRequest.builder() + .withClient(clientState.getId()) + .withSessionIds(new long[]{sessionId}) + .withCommandSequences(new long[]{sessionState.getCommandResponse()}) + .withEventIndexes(new long[]{sessionState.getEventIndex()}) + .withConnections(new long[]{sessionState.getConnection()}) + .build(); + + LOG.trace("{} - Sending {}", clientState.getUuid(), request); + connection.sendAndReceive(KeepAliveRequest.NAME, request).whenComplete((response, error) -> { + if (error == null) { + LOG.trace("{} - Received {}", clientState.getUuid(), response); + if (response.status() == Response.Status.OK) { + future.complete(null); + } else { + future.completeExceptionally(response.error().createException()); + } + } else { + future.completeExceptionally(error); + } + }); + return future; + } + + /** + * Sends a keep-alive request to the cluster. + */ + private void keepAliveSessions() { + keepAliveSessions(true); + } + + /** + * Sends a keep-alive request to the cluster. + */ + private void keepAliveSessions(boolean retryOnFailure) { + // If the current sessions state is unstable, reset the connection before sending a keep-alive. + if (state == State.UNSTABLE) { + selectorManager.resetAll(); + } + + Map sessions = new HashMap<>(this.sessions); + long[] sessionIds = new long[sessions.size()]; + long[] commandResponses = new long[sessions.size()]; + long[] eventIndexes = new long[sessions.size()]; + long[] connections = new long[sessions.size()]; + + int i = 0; + for (CopycatSessionState sessionState : sessions.values()) { + sessionIds[i] = sessionState.getSessionId(); + commandResponses[i] = sessionState.getCommandResponse(); + eventIndexes[i] = sessionState.getEventIndex(); + connections[i] = sessionState.getConnection(); + i++; + } + + KeepAliveRequest request = KeepAliveRequest.builder() + .withClient(clientState.getId()) + .withSessionIds(sessionIds) + .withCommandSequences(commandResponses) + .withEventIndexes(eventIndexes) + .withConnections(connections) + .build(); + + LOG.trace("{} - Sending {}", clientState.getUuid(), request); + connection.sendAndReceive(KeepAliveRequest.NAME, request).whenComplete((response, error) -> { + if (state != State.CLOSED) { + if (error == null) { + LOG.trace("{} - Received {}", clientState.getUuid(), response); + // If the request was successful, update the address selector and schedule the next keep-alive. + if (response.status() == Response.Status.OK) { + selectorManager.resetAll(response.leader(), response.members()); + setState(State.OPEN); + scheduleKeepAlive(); + } + // If the session is unknown, immediate expire the session. + else if (response.error() == CopycatError.Type.UNKNOWN_SESSION_ERROR) { + setState(State.EXPIRED); + } + // If a leader is still set in the address selector, unset the leader and attempt to send another keep-alive. + // This will ensure that the address selector selects all servers without filtering on the leader. + else if (retryOnFailure && connection.leader() != null) { + selectorManager.resetAll(null, connection.servers()); + keepAliveSessions(false); + } + // If no leader was set, set the session state to unstable and schedule another keep-alive. + else { + setState(State.UNSTABLE); + scheduleKeepAlive(); + } + } + // If a leader is still set in the address selector, unset the leader and attempt to send another keep-alive. + // This will ensure that the address selector selects all servers without filtering on the leader. + else if (retryOnFailure && connection.leader() != null) { + selectorManager.resetAll(null, connection.servers()); + keepAliveSessions(false); + } + // If no leader was set, set the session state to unstable and schedule another keep-alive. + else { + setState(State.UNSTABLE); + scheduleKeepAlive(); + } + } + }); + } + + /** + * Schedules a keep-alive request. + */ + private void scheduleKeepAlive() { + if (keepAlive != null) + keepAlive.cancel(); + keepAlive = threadContext.schedule(interval, () -> { + keepAlive = null; + if (state.isActive()) { + keepAliveSessions(); + } + }); + } + + /** + * Closes the session manager. + * + * @return A completable future to be completed once the session manager is closed. + */ + public CompletableFuture close() { + if (state == State.EXPIRED) + return CompletableFuture.completedFuture(null); + + CompletableFuture future = new CompletableFuture<>(); + threadContext.execute(() -> { + if (keepAlive != null) { + keepAlive.cancel(); + keepAlive = null; + } + unregister(future); + }); + return future; + } + + /** + * Unregisters the session. + */ + private void unregister(CompletableFuture future) { + unregister(true, future); + } + + /** + * Unregisters the session. + * + * @param future A completable future to be completed once the session is unregistered. + */ + private void unregister(boolean retryOnFailure, CompletableFuture future) { + // If the session is already closed, skip the unregister attempt. + if (state == State.CLOSED) { + future.complete(null); + return; + } + + LOG.debug("{} - Unregistering client: {}", clientState.getUuid(), clientState.getId()); + + // If a keep-alive request is already pending, cancel it. + if (keepAlive != null) { + keepAlive.cancel(); + keepAlive = null; + } + + // If the current sessions state is unstable, reset the connection before sending an unregister request. + if (state == State.UNSTABLE) { + selectorManager.resetAll(); + } + + UnregisterRequest request = UnregisterRequest.builder() + .withClient(clientState.getId()) + .build(); + + LOG.trace("{} - Sending {}", clientState, request); + connection.sendAndReceive(UnregisterRequest.NAME, request).whenComplete((response, error) -> { + if (state != State.CLOSED) { + if (error == null) { + LOG.trace("{} - Received {}", clientState.getUuid(), response); + // If the request was successful, update the session state and complete the close future. + if (response.status() == Response.Status.OK) { + setState(State.CLOSED); + future.complete(null); + } + // If the session is unknown, immediate expire the session and complete the close future. + else if (response.error() == CopycatError.Type.UNKNOWN_SESSION_ERROR) { + setState(State.EXPIRED); + future.complete(null); + } + // If a leader is still set in the address selector, unset the leader and send another unregister attempt. + // This will ensure that the address selector selects all servers without filtering on the leader. + else if (retryOnFailure && connection.leader() != null) { + selectorManager.resetAll(null, connection.servers()); + unregister(false, future); + } + // If no leader was set, set the session state to unstable and fail the unregister attempt. + else { + setState(State.UNSTABLE); + future.completeExceptionally(new ClosedSessionException("failed to unregister session")); + } + } + // If a leader is still set in the address selector, unset the leader and send another unregister attempt. + // This will ensure that the address selector selects all servers without filtering on the leader. + else if (retryOnFailure && connection.leader() != null) { + selectorManager.resetAll(null, connection.servers()); + unregister(false, future); + } + // If no leader was set, set the session state to unstable and schedule another unregister attempt. + else { + setState(State.UNSTABLE); + future.completeExceptionally(new ClosedSessionException("failed to unregister session")); + } + } + }); + } + + /** + * Kills the client session manager. + * + * @return A completable future to be completed once the session manager is killed. + */ + public CompletableFuture kill() { + return CompletableFuture.runAsync(() -> { + if (keepAlive != null) + keepAlive.cancel(); + setState(State.CLOSED); + }, threadContext); + } + + @Override + public String toString() { + return String.format("%s[client=%s]", getClass().getSimpleName(), clientState.getUuid()); + } + + /** + * Client session connection attempt. + */ + private final class RegisterAttempt implements ConnectionStrategy.Attempt { + private final int attempt; + private final CompletableFuture future; + + private RegisterAttempt(int attempt, CompletableFuture future) { + this.attempt = attempt; + this.future = future; + } + + @Override + public int attempt() { + return attempt; + } + + /** + * Completes the attempt successfully. + */ + public void complete() { + complete(null); + } + + /** + * Completes the attempt successfully. + * + * @param result The attempt result. + */ + public void complete(Void result) { + future.complete(result); + } + + @Override + public void fail() { + future.completeExceptionally(new ConnectException("failed to register session")); + } + + @Override + public void fail(Throwable error) { + future.completeExceptionally(error); + } + + @Override + public void retry() { + LOG.debug("Retrying session register attempt"); + registerClient(new RegisterAttempt(attempt + 1, future)); + } + + @Override + public void retry(Duration after) { + LOG.debug("Retrying session register attempt"); + threadContext.schedule(after, () -> registerClient(new RegisterAttempt(attempt + 1, future))); + } + } + + /** + * Session manager state. + */ + private enum State { + OPEN(true), + UNSTABLE(true), + EXPIRED(false), + CLOSED(false); + + private final boolean active; + + State(boolean active) { + this.active = active; + } + + /** + * Returns whether the state is active, requiring keep-alives. + * + * @return Whether the state is active. + */ + public boolean isActive() { + return active; + } + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/session/ClientSequencer.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionSequencer.java similarity index 96% rename from client/src/main/java/io/atomix/copycat/client/session/ClientSequencer.java rename to client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionSequencer.java index 23c1232b..efae4ff1 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/ClientSequencer.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionSequencer.java @@ -1,19 +1,19 @@ /* - * Copyright 2016 the original author or authors. + * Copyright 2017-present Open Networking Laboratory * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License + * limitations under the License. */ -package io.atomix.copycat.client.session; +package io.atomix.copycat.client.session.impl; import io.atomix.copycat.protocol.OperationResponse; import io.atomix.copycat.protocol.PublishRequest; @@ -54,17 +54,17 @@ * * @author 0 && (System.currentTimeMillis() - unstableSince) > unstabilityTimeout) { - return setStateAndCallListeners(Session.State.STALE); - } - } else if (this.state != Session.State.STALE) { - unstableSince = System.currentTimeMillis(); - return setStateAndCallListeners(state); - } - } - - return this; + public String getSessionType() { + return type; } - private ClientSessionState setStateAndCallListeners(Session.State state) { - this.state = state; - changeListeners.forEach(l -> l.accept(state)); - return this; + /** + * Returns a boolean indicating whether the session is open. + * + * @return Whether the session is open. + */ + public boolean isOpen() { + return open.get(); } /** - * Registers a state change listener on the session manager. - * - * @param callback The state change listener callback. - * @return The state change listener. + * Closes the session. */ - public Listener onStateChange(Consumer callback) { - Listener listener = new Listener() { - @Override - public void accept(Session.State state) { - callback.accept(state); - } - @Override - public void close() { - changeListeners.remove(this); - } - }; - changeListeners.add(listener); - return listener; + void close() { + open.set(false); } /** @@ -160,7 +91,7 @@ public void close() { * @param commandRequest The last command request sequence number. * @return The client session state. */ - public ClientSessionState setCommandRequest(long commandRequest) { + public CopycatSessionState setCommandRequest(long commandRequest) { this.commandRequest = commandRequest; return this; } @@ -189,7 +120,7 @@ public long nextCommandRequest() { * @param commandResponse The last command sequence number for which a response has been received. * @return The client session state. */ - public ClientSessionState setCommandResponse(long commandResponse) { + public CopycatSessionState setCommandResponse(long commandResponse) { this.commandResponse = commandResponse; return this; } @@ -209,7 +140,7 @@ public long getCommandResponse() { * @param responseIndex The highest index for which a command or query response has been received. * @return The client session state. */ - public ClientSessionState setResponseIndex(long responseIndex) { + public CopycatSessionState setResponseIndex(long responseIndex) { this.responseIndex = Math.max(this.responseIndex, responseIndex); return this; } @@ -229,7 +160,7 @@ public long getResponseIndex() { * @param eventIndex The highest index for which an event has been received in sequence. * @return The client session state. */ - public ClientSessionState setEventIndex(long eventIndex) { + public CopycatSessionState setEventIndex(long eventIndex) { this.eventIndex = eventIndex; return this; } @@ -243,4 +174,33 @@ public long getEventIndex() { return eventIndex; } + /** + * Sets the session's current connection. + * + * @param connection The session's current connection. + * @return The client session state. + */ + public CopycatSessionState setConnection(int connection) { + this.connection = connection; + return this; + } + + /** + * Returns the session's current connection. + * + * @return The session's current connection. + */ + public long getConnection() { + return connection; + } + + /** + * Returns the session's next connection ID. + * + * @return The session's next connection ID. + */ + public long nextConnection() { + return ++connection; + } + } diff --git a/client/src/main/java/io/atomix/copycat/client/session/ClientSessionSubmitter.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionSubmitter.java similarity index 83% rename from client/src/main/java/io/atomix/copycat/client/session/ClientSessionSubmitter.java rename to client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionSubmitter.java index cfa12841..148437cc 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/ClientSessionSubmitter.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionSubmitter.java @@ -1,22 +1,21 @@ /* - * Copyright 2015 the original author or authors. + * Copyright 2017-present Open Networking Laboratory * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License + * limitations under the License. */ -package io.atomix.copycat.client.session; +package io.atomix.copycat.client.session.impl; import io.atomix.catalyst.concurrent.ThreadContext; -import io.atomix.catalyst.transport.Connection; import io.atomix.catalyst.transport.TransportException; import io.atomix.catalyst.util.Assert; import io.atomix.copycat.Command; @@ -28,7 +27,8 @@ import io.atomix.copycat.error.UnknownSessionException; import io.atomix.copycat.protocol.*; import io.atomix.copycat.session.ClosedSessionException; -import io.atomix.copycat.session.Session; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.net.ConnectException; import java.nio.channels.ClosedChannelException; @@ -48,24 +48,32 @@ * * @author CompletableFuture submit(Command command) { CompletableFuture future = new CompletableFuture<>(); - context.executor().execute(() -> submitCommand(command, future)); + context.execute(() -> submitCommand(command, future)); return future; } @@ -110,7 +118,7 @@ private void submitCommand(CommandRequest request, CompletableFuture futu */ public CompletableFuture submit(Query query) { CompletableFuture future = new CompletableFuture<>(); - context.executor().execute(() -> submitQuery(query, future)); + context.execute(() -> submitQuery(query, future)); return future; } @@ -140,12 +148,12 @@ private void submitQuery(QueryRequest request, CompletableFuture future) * @param attempt The attempt to submit. */ private void submit(OperationAttempt attempt) { - if (state.getState() == Session.State.CLOSED || state.getState() == Session.State.EXPIRED) { + if (!state.isOpen()) { attempt.fail(new ClosedSessionException("session closed")); } else { - state.getLogger().trace("{} - Sending {}", state.getSessionId(), attempt.request); + LOG.trace("{} - Sending {}", state.getSessionId(), attempt.request); attempts.put(attempt.sequence, attempt); - connection.sendAndReceive(attempt.request).whenComplete(attempt); + attempt.send(); attempt.future.whenComplete((r, e) -> attempts.remove(attempt.sequence)); } } @@ -168,28 +176,13 @@ private void resubmit(long commandSequence, OperationAttempt attempt) { long responseSequence = state.getCommandResponse(); if (commandSequence < responseSequence && keepAliveIndex.get() != responseSequence) { keepAliveIndex.set(responseSequence); - KeepAliveRequest request = KeepAliveRequest.builder() - .withSession(state.getSessionId()) - .withCommandSequence(state.getCommandResponse()) - .withEventIndex(state.getEventIndex()) - .build(); - state.getLogger().trace("{} - Sending {}", state.getSessionId(), request); - connection.sendAndReceive(request).whenComplete((response, error) -> { + manager.resetIndexes(state.getSessionId()).whenCompleteAsync((result, error) -> { if (error == null) { - state.getLogger().trace("{} - Received {}", state.getSessionId(), response); - - // If the keep-alive is successful, recursively resubmit operations starting - // at the submitted response sequence number rather than the command sequence. - if (response.status() == Response.Status.OK) { - resubmit(responseSequence, attempt); - } else { - attempt.retry(Duration.ofSeconds(FIBONACCI[Math.min(attempt.attempt-1, FIBONACCI.length-1)])); - } + resubmit(responseSequence, attempt); } else { - keepAliveIndex.set(0); attempt.retry(Duration.ofSeconds(FIBONACCI[Math.min(attempt.attempt-1, FIBONACCI.length-1)])); } - }); + }, context); } else { for (Map.Entry entry : attempts.entrySet()) { OperationAttempt operation = entry.getValue(); @@ -229,6 +222,11 @@ protected OperationAttempt(long sequence, int attempt, T request, CompletableFut this.future = future; } + /** + * Sends the attempt. + */ + protected abstract void send(); + /** * Returns the next instance of the attempt. * @@ -256,10 +254,6 @@ protected OperationAttempt(long sequence, int attempt, T request, CompletableFut * @param error The completion exception. */ protected void complete(Throwable error) { - // If the exception is an UnknownSessionException, expire the session. - if (error instanceof UnknownSessionException) { - state.setState(Session.State.EXPIRED); - } sequence(null, () -> future.completeExceptionally(error)); } @@ -293,7 +287,7 @@ public void fail(Throwable t) { * Immediately retries the attempt. */ public void retry() { - context.executor().execute(() -> submit(next())); + context.execute(() -> submit(next())); } /** @@ -319,6 +313,11 @@ public CommandAttempt(long sequence, int attempt, CommandRequest request, Comple super(sequence, attempt, request, future); } + @Override + protected void send() { + leaderConnection.sendAndReceive(CommandRequest.NAME, request).whenComplete(this); + } + @Override protected OperationAttempt next() { return new CommandAttempt<>(sequence, this.attempt + 1, request, future); @@ -332,7 +331,7 @@ protected Throwable defaultException() { @Override public void accept(CommandResponse response, Throwable error) { if (error == null) { - state.getLogger().trace("{} - Received {}", state.getSessionId(), response); + LOG.trace("{} - Received {}", state.getSessionId(), response); if (response.status() == Response.Status.OK) { complete(response); } @@ -343,7 +342,9 @@ else if (response.error() == CopycatError.Type.COMMAND_ERROR) { } // The following exceptions need to be handled at a higher level by the client or the user. else if (response.error() == CopycatError.Type.APPLICATION_ERROR + || response.error() == CopycatError.Type.UNKNOWN_CLIENT_ERROR || response.error() == CopycatError.Type.UNKNOWN_SESSION_ERROR + || response.error() == CopycatError.Type.UNKNOWN_STATE_MACHINE_ERROR || response.error() == CopycatError.Type.INTERNAL_ERROR) { complete(response.error().createException()); } @@ -361,13 +362,13 @@ else if (response.error() == CopycatError.Type.APPLICATION_ERROR @Override public void fail(Throwable cause) { super.fail(cause); - if (!(cause instanceof UnknownSessionException)) { + if (!CLOSED_PREDICATE.test(cause)) { CommandRequest request = CommandRequest.builder() .withSession(this.request.session()) .withSequence(this.request.sequence()) .withCommand(new NoOpCommand()) .build(); - context.executor().execute(() -> submit(new CommandAttempt<>(sequence, this.attempt + 1, request, future))); + context.execute(() -> submit(new CommandAttempt<>(sequence, this.attempt + 1, request, future))); } } @@ -394,6 +395,11 @@ public QueryAttempt(long sequence, int attempt, QueryRequest request, Completabl super(sequence, attempt, request, future); } + @Override + protected void send() { + sessionConnection.sendAndReceive(QueryRequest.NAME, request).whenComplete(this); + } + @Override protected OperationAttempt next() { return new QueryAttempt<>(sequence, this.attempt + 1, request, future); @@ -407,7 +413,7 @@ protected Throwable defaultException() { @Override public void accept(QueryResponse response, Throwable error) { if (error == null) { - state.getLogger().trace("{} - Received {}", state.getSessionId(), response); + LOG.trace("{} - Received {}", state.getSessionId(), response); if (response.status() == Response.Status.OK) { complete(response); } else { diff --git a/client/src/main/java/io/atomix/copycat/client/session/ClientSession.java b/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java similarity index 58% rename from client/src/main/java/io/atomix/copycat/client/session/ClientSession.java rename to client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java index ba5860cc..232459a3 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/ClientSession.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java @@ -1,35 +1,29 @@ /* - * Copyright 2015 the original author or authors. + * Copyright 2017-present Open Networking Laboratory * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and - * limitations under the License + * limitations under the License. */ -package io.atomix.copycat.client.session; +package io.atomix.copycat.client.session.impl; -import io.atomix.catalyst.concurrent.Futures; import io.atomix.catalyst.concurrent.Listener; import io.atomix.catalyst.concurrent.ThreadContext; -import io.atomix.catalyst.transport.Client; import io.atomix.catalyst.util.Assert; import io.atomix.copycat.Command; import io.atomix.copycat.Operation; import io.atomix.copycat.Query; -import io.atomix.copycat.client.ConnectionStrategy; -import io.atomix.copycat.client.util.AddressSelector; -import io.atomix.copycat.client.util.ClientConnection; -import io.atomix.copycat.session.ClosedSessionException; +import io.atomix.copycat.client.session.CopycatSession; import io.atomix.copycat.session.Session; -import java.time.Duration; import java.util.concurrent.CompletableFuture; import java.util.function.Consumer; @@ -52,24 +46,18 @@ * * @author onStateChange(Consumer callback) { - return state.onStateChange(callback); + public String type() { + return state.getSessionType(); } /** @@ -112,11 +100,7 @@ public CompletableFuture submit(Operation operation) { * @return A completable future to be completed with the command result. */ public CompletableFuture submit(Command command) { - State state = state(); - if (state == State.CLOSED || state == State.EXPIRED) { - return Futures.exceptionalFuture(new ClosedSessionException("session closed")); - } - return submitter.submit(command); + return sessionSubmitter.submit(command); } /** @@ -127,20 +111,7 @@ public CompletableFuture submit(Command command) { * @return A completable future to be completed with the query result. */ public CompletableFuture submit(Query query) { - State state = state(); - if (state == State.CLOSED || state == State.EXPIRED) { - return Futures.exceptionalFuture(new ClosedSessionException("session closed")); - } - return submitter.submit(query); - } - - /** - * Opens the session. - * - * @return A completable future to be completed once the session is opened. - */ - public CompletableFuture register() { - return manager.open().thenApply(v -> this); + return sessionSubmitter.submit(query); } /** @@ -158,7 +129,7 @@ public CompletableFuture register() { * @throws NullPointerException if {@code event} or {@code callback} is null */ public Listener onEvent(String event, Runnable callback) { - return listener.onEvent(event, callback); + return sessionListener.onEvent(event, callback); } /** @@ -177,52 +148,17 @@ public Listener onEvent(String event, Runnable callback) { * @throws NullPointerException if {@code event} or {@code callback} is null */ public Listener onEvent(String event, Consumer callback) { - return listener.onEvent(event, callback); + return sessionListener.onEvent(event, callback); } - /** - * Closes the session. - * - * @return A completable future to be completed once the session is closed. - */ - public CompletableFuture close() { - CompletableFuture future = new CompletableFuture<>(); - submitter.close() - .thenCompose(v -> listener.close()) - .thenCompose(v -> manager.close()) - .whenComplete((managerResult, managerError) -> { - connection.close().whenComplete((connectionResult, connectionError) -> { - if (managerError != null) { - future.completeExceptionally(managerError); - } else if (connectionError != null) { - future.completeExceptionally(connectionError); - } else { - future.complete(null); - } - }); - }); - return future; - } - - /** - * Expires the session. - * - * @return A completable future to be completed once the session has been expired. - */ - public CompletableFuture expire() { - return manager.expire(); + @Override + public boolean isOpen() { + return state.isOpen(); } - /** - * Kills the session. - * - * @return A completable future to be completed once the session has been killed. - */ - public CompletableFuture kill() { - return submitter.close() - .thenCompose(v -> listener.close()) - .thenCompose(v -> manager.kill()) - .thenCompose(v -> connection.close()); + @Override + public CompletableFuture close() { + return sessionManager.closeSession(state.getSessionId()).whenComplete((result, error) -> state.close()); } @Override @@ -235,7 +171,7 @@ public int hashCode() { @Override public boolean equals(Object object) { - return object instanceof ClientSession && ((ClientSession) object).id() == id(); + return object instanceof DefaultCopycatSession && ((DefaultCopycatSession) object).id() == id(); } @Override diff --git a/client/src/main/java/io/atomix/copycat/client/util/AddressSelector.java b/client/src/main/java/io/atomix/copycat/client/util/AddressSelector.java index bd27cde8..047771be 100644 --- a/client/src/main/java/io/atomix/copycat/client/util/AddressSelector.java +++ b/client/src/main/java/io/atomix/copycat/client/util/AddressSelector.java @@ -17,18 +17,19 @@ import io.atomix.catalyst.transport.Address; import io.atomix.catalyst.util.Assert; -import io.atomix.copycat.client.ServerSelectionStrategy; +import io.atomix.copycat.client.CommunicationStrategy; import java.util.ArrayList; import java.util.Collection; import java.util.Iterator; +import java.util.LinkedList; /** * Client address selector. * * @author (servers)); } /** @@ -77,6 +84,15 @@ public State state() { } } + /** + * Returns the current address selection. + * + * @return The current address selection. + */ + public Address current() { + return selection; + } + /** * Returns the current selector leader. * @@ -166,9 +182,17 @@ public boolean hasNext() { @Override public Address next() { - if (selectionsIterator == null) + if (selectionsIterator == null) { selectionsIterator = selections.iterator(); - return selectionsIterator.next(); + } + Address selection = selectionsIterator.next(); + this.selection = selection; + return selection; + } + + @Override + public void close() { + selectors.remove(this); } @Override diff --git a/client/src/main/java/io/atomix/copycat/client/util/AddressSelectorManager.java b/client/src/main/java/io/atomix/copycat/client/util/AddressSelectorManager.java new file mode 100644 index 00000000..148fea91 --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/util/AddressSelectorManager.java @@ -0,0 +1,75 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client.util; + +import io.atomix.catalyst.transport.Address; +import io.atomix.copycat.client.CommunicationStrategy; + +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; + +/** + * Address selectors. + */ +public final class AddressSelectorManager { + private final Set selectors = new CopyOnWriteArraySet<>(); + private volatile Address leader; + private volatile Collection
servers = Collections.emptyList(); + + /** + * Creates a new address selector. + * + * @param selectionStrategy The server selection strategy. + * @return A new address selector. + */ + public AddressSelector createSelector(CommunicationStrategy selectionStrategy) { + AddressSelector selector = new AddressSelector(leader, servers, selectionStrategy, this); + selectors.add(selector); + return selector; + } + + /** + * Resets all child selectors. + */ + public void resetAll() { + selectors.forEach(AddressSelector::reset); + } + + /** + * Resets all child selectors. + * + * @param leader The current cluster leader. + * @param servers The collection of all active servers. + */ + public void resetAll(Address leader, Collection
servers) { + this.leader = leader; + this.servers = new LinkedList<>(servers); + selectors.forEach(s -> s.reset(leader, servers)); + } + + /** + * Removes the given selector. + * + * @param selector The address selector to remove. + */ + void remove(AddressSelector selector) { + selectors.remove(selector); + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/util/ClientConnection.java b/client/src/main/java/io/atomix/copycat/client/util/ClientConnection.java deleted file mode 100644 index 9966fb43..00000000 --- a/client/src/main/java/io/atomix/copycat/client/util/ClientConnection.java +++ /dev/null @@ -1,355 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License - */ -package io.atomix.copycat.client.util; - -import io.atomix.catalyst.concurrent.Listener; -import io.atomix.catalyst.transport.Address; -import io.atomix.catalyst.transport.Client; -import io.atomix.catalyst.transport.Connection; -import io.atomix.catalyst.transport.TransportException; -import io.atomix.catalyst.util.Assert; -import io.atomix.copycat.error.CopycatError; -import io.atomix.copycat.protocol.ConnectRequest; -import io.atomix.copycat.protocol.ConnectResponse; -import io.atomix.copycat.protocol.Request; -import io.atomix.copycat.protocol.Response; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.net.ConnectException; -import java.nio.channels.ClosedChannelException; -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeoutException; -import java.util.function.BiFunction; -import java.util.function.Consumer; -import java.util.function.Function; - -/** - * Client connection that recursively connects to servers in the cluster and attempts to submit requests. - * - * @author servers() { - return selector.servers(); - } - - /** - * Resets the client connection. - * - * @return The client connection. - */ - public ClientConnection reset() { - selector.reset(); - return this; - } - - /** - * Resets the client connection. - * - * @param leader The current cluster leader. - * @param servers The current servers. - * @return The client connection. - */ - public ClientConnection reset(Address leader, Collection
servers) { - selector.reset(leader, servers); - return this; - } - - @Override - public CompletableFuture send(Object request) { - CompletableFuture future = new CompletableFuture<>(); - sendRequest((Request) request, (r, c) -> c.send(r), future); - return future; - } - - @Override - public CompletableFuture sendAndReceive(T request) { - CompletableFuture future = new CompletableFuture<>(); - sendRequest((Request) request, (r, c) -> c.sendAndReceive(r), future); - return future; - } - - /** - * Sends the given request attempt to the cluster. - */ - private void sendRequest(T request, BiFunction> sender, CompletableFuture future) { - if (open) { - connect().whenComplete((c, e) -> sendRequest(request, sender, c, e, future)); - } - } - - /** - * Sends the given request attempt to the cluster via the given connection if connected. - */ - private void sendRequest(T request, BiFunction> sender, Connection connection, Throwable error, CompletableFuture future) { - if (open) { - if (error == null) { - if (connection != null) { - LOGGER.trace("{} - Sending {}", id, request); - sender.apply(request, connection).whenComplete((r, e) -> { - if (e != null || r != null) { - handleResponse(request, sender, connection, (Response) r, e, future); - } else { - future.complete(null); - } - }); - } else { - future.completeExceptionally(new ConnectException("Failed to connect to the cluster")); - } - } else { - LOGGER.trace("{} - Resending {}: {}", id, request, error); - resendRequest(error, request, sender, connection, future); - } - } - } - - /** - * Resends a request due to a request failure, resetting the connection if necessary. - */ - @SuppressWarnings("unchecked") - private void resendRequest(Throwable cause, T request, BiFunction sender, Connection connection, CompletableFuture future) { - // If the connection has not changed, reset it and connect to the next server. - if (this.connection == connection) { - LOGGER.trace("{} - Resetting connection. Reason: {}", id, cause); - this.connection = null; - connection.close(); - } - - // Create a new connection and resend the request. This will force retries to piggyback on any existing - // connect attempts. - connect().whenComplete((c, e) -> sendRequest(request, sender, c, e, future)); - } - - /** - * Handles a response from the cluster. - */ - @SuppressWarnings("unchecked") - private void handleResponse(T request, BiFunction sender, Connection connection, Response response, Throwable error, CompletableFuture future) { - if (open) { - if (error == null) { - if (response.status() == Response.Status.OK - || response.error() == CopycatError.Type.COMMAND_ERROR - || response.error() == CopycatError.Type.QUERY_ERROR - || response.error() == CopycatError.Type.APPLICATION_ERROR - || response.error() == CopycatError.Type.UNKNOWN_SESSION_ERROR - || response.error() == CopycatError.Type.INTERNAL_ERROR) { - LOGGER.trace("{} - Received {}", id, response); - future.complete(response); - } else { - resendRequest(response.error().createException(), request, sender, connection, future); - } - } else if (error instanceof ConnectException || error instanceof TimeoutException || error instanceof TransportException || error instanceof ClosedChannelException) { - resendRequest(error, request, sender, connection, future); - } else { - LOGGER.debug("{} - {} failed! Reason: {}", id, request, error); - future.completeExceptionally(error); - } - } - } - - /** - * Connects to the cluster. - */ - private CompletableFuture connect() { - // If the address selector has been reset then reset the connection. - if (selector.state() == AddressSelector.State.RESET && connection != null) { - if (connectFuture != null) { - return connectFuture; - } - - CompletableFuture future = new OrderedCompletableFuture<>(); - future.whenComplete((r, e) -> this.connectFuture = null); - this.connectFuture = future; - - Connection oldConnection = this.connection; - this.connection = null; - oldConnection.close(); - connect(future); - return future; - } - - // If a connection was already established then use that connection. - if (connection != null) { - return CompletableFuture.completedFuture(connection); - } - - // If a connection is currently being established then piggyback on the connect future. - if (connectFuture != null) { - return connectFuture; - } - - // Create a new connect future and connect to the first server in the cluster. - CompletableFuture future = new OrderedCompletableFuture<>(); - future.whenComplete((r, e) -> this.connectFuture = null); - this.connectFuture = future; - reset().connect(future); - return future; - } - - /** - * Attempts to connect to the cluster. - */ - private void connect(CompletableFuture future) { - if (!selector.hasNext()) { - LOGGER.debug("{} - Failed to connect to the cluster", id); - future.complete(null); - } else { - Address address = selector.next(); - LOGGER.debug("{} - Connecting to {}", id, address); - client.connect(address).whenComplete((c, e) -> handleConnection(address, c, e, future)); - } - } - - /** - * Handles a connection to a server. - */ - private void handleConnection(Address address, Connection connection, Throwable error, CompletableFuture future) { - if (open) { - if (error == null) { - setupConnection(address, connection, future); - } else { - LOGGER.debug("{} - Failed to connect! Reason: {}", id, error); - connect(future); - } - } - } - - /** - * Sets up the given connection. - */ - @SuppressWarnings("unchecked") - private void setupConnection(Address address, Connection connection, CompletableFuture future) { - LOGGER.debug("{} - Setting up connection to {}", id, address); - - this.connection = connection; - - connection.onClose(c -> { - if (c.equals(this.connection)) { - LOGGER.debug("{} - Connection closed", id); - this.connection = null; - } - }); - connection.onException(c -> { - if (c.equals(this.connection)) { - LOGGER.debug("{} - Connection lost", id); - this.connection = null; - } - }); - - for (Map.Entry, Function> entry : handlers.entrySet()) { - connection.handler(entry.getKey(), entry.getValue()); - } - - // When we first connect to a new server, first send a ConnectRequest to the server to establish - // the connection with the server-side state machine. - ConnectRequest request = ConnectRequest.builder() - .withClientId(id) - .build(); - - LOGGER.trace("{} - Sending {}", id, request); - connection.sendAndReceive(request).whenComplete((r, e) -> handleConnectResponse(r, e, future)); - } - - /** - * Handles a connect response. - */ - private void handleConnectResponse(ConnectResponse response, Throwable error, CompletableFuture future) { - if (open) { - if (error == null) { - LOGGER.trace("{} - Received {}", id, response); - // If the connection was successfully created, immediately send a keep-alive request - // to the server to ensure we maintain our session and get an updated list of server addresses. - if (response.status() == Response.Status.OK) { - selector.reset(response.leader(), response.members()); - future.complete(connection); - } else { - connect(future); - } - } else { - LOGGER.debug("{} - Failed to connect! Reason: {}", id, error); - connect(future); - } - } - } - - @Override - public Connection handler(Class type, Consumer handler) { - return handler(type, r -> { - handler.accept(r); - return null; - }); - } - - @Override - public Connection handler(Class type, Function> handler) { - Assert.notNull(type, "type"); - Assert.notNull(handler, "handler"); - handlers.put(type, handler); - if (connection != null) - connection.handler(type, handler); - return this; - } - - @Override - public Listener onException(Consumer listener) { - throw new UnsupportedOperationException(); - } - - @Override - public Listener onClose(Consumer listener) { - throw new UnsupportedOperationException(); - } - - @Override - public CompletableFuture close() { - open = false; - return CompletableFuture.completedFuture(null); - } - -} diff --git a/client/src/main/java/io/atomix/copycat/client/util/ClientConnectionManager.java b/client/src/main/java/io/atomix/copycat/client/util/ClientConnectionManager.java new file mode 100644 index 00000000..b927e03a --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/util/ClientConnectionManager.java @@ -0,0 +1,84 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client.util; + +import io.atomix.catalyst.transport.Address; +import io.atomix.catalyst.transport.Client; +import io.atomix.catalyst.transport.Connection; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +/** + * Connection manager. + * + * @author Jordan Halterman + */ +public final class ClientConnectionManager { + private final Client client; + private final Map connections = new HashMap<>(); + + public ClientConnectionManager(Client client) { + this.client = client; + } + + /** + * Returns the connection for the given member. + * + * @param address The member for which to get the connection. + * @return A completable future to be called once the connection is received. + */ + public CompletableFuture getConnection(Address address) { + Connection connection = connections.get(address); + return connection == null ? createConnection(address) : CompletableFuture.completedFuture(connection); + } + + /** + * Creates a connection for the given member. + * + * @param address The member for which to create the connection. + * @return A completable future to be called once the connection has been created. + */ + private CompletableFuture createConnection(Address address) { + return client.connect(address).thenApply(connection -> { + connection.onClose(c -> { + if (connections.get(address) == c) { + connections.remove(address); + } + }); + connections.put(address, connection); + return connection; + }); + } + + /** + * Closes the connection manager. + * + * @return A completable future to be completed once the connection manager is closed. + */ + public CompletableFuture close() { + CompletableFuture[] futures = new CompletableFuture[connections.size()]; + + int i = 0; + for (Connection connection : connections.values()) { + futures[i++] = connection.close(); + } + + return CompletableFuture.allOf(futures); + } + +} diff --git a/client/src/test/java/io/atomix/copycat/client/ServerSelectionStrategiesTest.java b/client/src/test/java/io/atomix/copycat/client/CommunicationStrategiesTest.java similarity index 80% rename from client/src/test/java/io/atomix/copycat/client/ServerSelectionStrategiesTest.java rename to client/src/test/java/io/atomix/copycat/client/CommunicationStrategiesTest.java index dca28367..588eb916 100644 --- a/client/src/test/java/io/atomix/copycat/client/ServerSelectionStrategiesTest.java +++ b/client/src/test/java/io/atomix/copycat/client/CommunicationStrategiesTest.java @@ -32,7 +32,7 @@ * @author results = (List
) ServerSelectionStrategies.ANY.selectConnections(null, servers); + List
results = (List
) CommunicationStrategies.ANY.selectConnections(null, servers); assertTrue(listsEqual(results, servers)); } @@ -51,7 +51,7 @@ public void testAnySelectionStrategy() throws Throwable { * Tests the LEADER server selection strategy. */ public void testLeaderSelectionStrategy() throws Throwable { - List
results = (List
) ServerSelectionStrategies.LEADER.selectConnections(new Address("localhost", 5000), servers); + List
results = (List
) CommunicationStrategies.LEADER.selectConnections(new Address("localhost", 5000), servers); assertEquals(results.size(), 1); assertEquals(results.get(0), new Address("localhost", 5000)); } @@ -60,7 +60,7 @@ public void testLeaderSelectionStrategy() throws Throwable { * Tests the LEADER server selection strategy. */ public void testNoLeaderSelectionStrategy() throws Throwable { - List
results = (List
) ServerSelectionStrategies.LEADER.selectConnections(null, servers); + List
results = (List
) CommunicationStrategies.LEADER.selectConnections(null, servers); assertTrue(listsEqual(results, servers)); } @@ -68,7 +68,7 @@ public void testNoLeaderSelectionStrategy() throws Throwable { * Tests the FOLLOWERS server selection strategy. */ public void testFollowersNoLeaderSelectionStrategy() throws Throwable { - List
results = (List
) ServerSelectionStrategies.FOLLOWERS.selectConnections(null, servers); + List
results = (List
) CommunicationStrategies.FOLLOWERS.selectConnections(null, servers); assertTrue(listsEqual(results, servers)); } @@ -76,7 +76,7 @@ public void testFollowersNoLeaderSelectionStrategy() throws Throwable { * Tests the FOLLOWERS server selection strategy. */ public void testFollowersSelectionStrategy() throws Throwable { - List
results = (List
) ServerSelectionStrategies.FOLLOWERS.selectConnections(new Address("localhost", 5000), servers); + List
results = (List
) CommunicationStrategies.FOLLOWERS.selectConnections(new Address("localhost", 5000), servers); assertEquals(results.size(), servers.size() - 1); assertFalse(results.contains(new Address("localhost", 5000))); } diff --git a/client/src/test/java/io/atomix/copycat/client/DefaultCopycatClientTest.java b/client/src/test/java/io/atomix/copycat/client/DefaultCopycatClientTest.java deleted file mode 100644 index 1424c1b8..00000000 --- a/client/src/test/java/io/atomix/copycat/client/DefaultCopycatClientTest.java +++ /dev/null @@ -1,426 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License - */ -package io.atomix.copycat.client; - -import io.atomix.catalyst.transport.Address; -import io.atomix.catalyst.transport.Client; -import io.atomix.catalyst.transport.Connection; -import io.atomix.catalyst.transport.Transport; -import io.atomix.copycat.Command; -import io.atomix.copycat.Query; -import io.atomix.copycat.error.CopycatError; -import io.atomix.copycat.protocol.*; -import io.atomix.copycat.session.ClosedSessionException; -import org.mockito.Mockito; -import org.testng.annotations.Test; - -import java.util.Arrays; -import java.util.Collection; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionException; -import java.util.concurrent.CountDownLatch; - -import static org.mockito.Matchers.any; -import static org.mockito.Matchers.isA; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.testng.Assert.assertEquals; -import static org.testng.Assert.fail; - -/** - * Copycat client test. - * - * @author MEMBERS = Arrays.asList( - new Address("localhost", 5000), - new Address("localhost", 5001), - new Address("localhost", 5002) - ); - - /** - * Tests calling the recovery strategy when a command fails due to UnknownSessionException. - */ - public void testCommandRetryOnLeaderFailure() throws Throwable { - Connection connection = mock(Connection.class); - when(connection.close()).thenReturn(CompletableFuture.completedFuture(null)); - - Client client = mock(Client.class); - when(client.connect(any())).thenReturn(CompletableFuture.completedFuture(connection)); - - Transport transport = mock(Transport.class); - when(transport.client()).thenReturn(client); - - // Handle connect requests. - when(connection.sendAndReceive(isA(ConnectRequest.class))) - .thenReturn(CompletableFuture.completedFuture(ConnectResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle register requests. - when(connection.sendAndReceive(isA(RegisterRequest.class))) - .thenReturn(CompletableFuture.completedFuture(RegisterResponse.builder() - .withStatus(Response.Status.OK) - .withSession(1) - .withTimeout(5000) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle keep-alive requests. - Mockito.when(connection.sendAndReceive(isA(KeepAliveRequest.class))) - .thenReturn(CompletableFuture.completedFuture(KeepAliveResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Fail the first request and succeed on the second. - Mockito.when(connection.sendAndReceive(isA(CommandRequest.class))) - .thenReturn(CompletableFuture.completedFuture(CommandResponse.builder() - .withStatus(Response.Status.ERROR) - .withError(CopycatError.Type.NO_LEADER_ERROR) - .build())) - .thenReturn(CompletableFuture.completedFuture(CommandResponse.builder() - .withStatus(Response.Status.OK) - .withIndex(1) - .withEventIndex(0) - .withResult("Hello world!") - .build())); - - CopycatClient copycatClient = CopycatClient.builder() - .withTransport(transport) - .build(); - - copycatClient.connect(MEMBERS).join(); - - assertEquals(copycatClient.submit(new TestCommand()).join(), "Hello world!"); - } - - /** - * Tests calling the recovery strategy when a command fails due to UnknownSessionException. - */ - public void testCommandSessionExpiration() throws Throwable { - Connection connection = mock(Connection.class); - when(connection.close()).thenReturn(CompletableFuture.completedFuture(null)); - - Client client = mock(Client.class); - when(client.connect(any())).thenReturn(CompletableFuture.completedFuture(connection)); - - Transport transport = mock(Transport.class); - when(transport.client()).thenReturn(client); - - // Handle connect requests. - when(connection.sendAndReceive(isA(ConnectRequest.class))) - .thenReturn(CompletableFuture.completedFuture(ConnectResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle register requests. - when(connection.sendAndReceive(isA(RegisterRequest.class))) - .thenReturn(CompletableFuture.completedFuture(RegisterResponse.builder() - .withStatus(Response.Status.OK) - .withSession(1) - .withTimeout(5000) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle keep-alive requests. - Mockito.when(connection.sendAndReceive(isA(KeepAliveRequest.class))) - .thenReturn(CompletableFuture.completedFuture(KeepAliveResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Fail command requests. - Mockito.when(connection.sendAndReceive(isA(CommandRequest.class))) - .thenReturn(CompletableFuture.completedFuture(CommandResponse.builder() - .withStatus(Response.Status.ERROR) - .withError(CopycatError.Type.UNKNOWN_SESSION_ERROR) - .build())); - - final CountDownLatch latch = new CountDownLatch(1); - CopycatClient copycatClient = CopycatClient.builder() - .withTransport(transport) - .withRecoveryStrategy(c -> { - latch.countDown(); - }) - .build(); - - copycatClient.connect(MEMBERS).join(); - - copycatClient.submit(new TestCommand()); - - latch.await(); - } - - /** - * Tests calling the recovery strategy when a query fails due to UnknownSessionException. - */ - public void testQuerySessionExpiration() throws Throwable { - Connection connection = mock(Connection.class); - when(connection.close()).thenReturn(CompletableFuture.completedFuture(null)); - - Client client = mock(Client.class); - when(client.connect(any())).thenReturn(CompletableFuture.completedFuture(connection)); - - Transport transport = mock(Transport.class); - when(transport.client()).thenReturn(client); - - // Handle connect requests. - when(connection.sendAndReceive(isA(ConnectRequest.class))) - .thenReturn(CompletableFuture.completedFuture(ConnectResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle register requests. - when(connection.sendAndReceive(isA(RegisterRequest.class))) - .thenReturn(CompletableFuture.completedFuture(RegisterResponse.builder() - .withStatus(Response.Status.OK) - .withSession(1) - .withTimeout(5000) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle keep-alive requests. - Mockito.when(connection.sendAndReceive(isA(KeepAliveRequest.class))) - .thenReturn(CompletableFuture.completedFuture(KeepAliveResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Fail query requests. - Mockito.when(connection.sendAndReceive(isA(QueryRequest.class))) - .thenReturn(CompletableFuture.completedFuture(QueryResponse.builder() - .withStatus(Response.Status.ERROR) - .withError(CopycatError.Type.UNKNOWN_SESSION_ERROR) - .build())); - - final CountDownLatch latch = new CountDownLatch(1); - CopycatClient copycatClient = CopycatClient.builder() - .withTransport(transport) - .withRecoveryStrategy(c -> { - latch.countDown(); - }) - .build(); - - copycatClient.connect(MEMBERS).join(); - - copycatClient.submit(new TestQuery()); - - latch.await(); - } - - /** - * Tests calling the recovery strategy when a command fails due to UnknownSessionException. - */ - public void testCommandSessionRecovery() throws Throwable { - Connection connection = mock(Connection.class); - when(connection.close()).thenReturn(CompletableFuture.completedFuture(null)); - - Client client = mock(Client.class); - when(client.connect(any())).thenReturn(CompletableFuture.completedFuture(connection)); - - Transport transport = mock(Transport.class); - when(transport.client()).thenReturn(client); - - // Handle connect requests. - when(connection.sendAndReceive(isA(ConnectRequest.class))) - .thenReturn(CompletableFuture.completedFuture(ConnectResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle register requests. - when(connection.sendAndReceive(isA(RegisterRequest.class))) - .thenReturn(CompletableFuture.completedFuture(RegisterResponse.builder() - .withStatus(Response.Status.OK) - .withSession(1) - .withTimeout(5000) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())) - .thenReturn(CompletableFuture.completedFuture(RegisterResponse.builder() - .withStatus(Response.Status.OK) - .withSession(2) - .withTimeout(5000) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle keep-alive requests. - Mockito.when(connection.sendAndReceive(isA(KeepAliveRequest.class))) - .thenReturn(CompletableFuture.completedFuture(KeepAliveResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Fail command requests. - Mockito.when(connection.sendAndReceive(isA(CommandRequest.class))) - .thenReturn(CompletableFuture.completedFuture(CommandResponse.builder() - .withStatus(Response.Status.ERROR) - .withError(CopycatError.Type.UNKNOWN_SESSION_ERROR) - .build())) - .thenReturn(CompletableFuture.completedFuture(CommandResponse.builder() - .withStatus(Response.Status.OK) - .withIndex(1) - .withEventIndex(0) - .withResult("Hello world!") - .build())); - - CopycatClient copycatClient = CopycatClient.builder() - .withTransport(transport) - .withRecoveryStrategy(RecoveryStrategies.RECOVER) - .build(); - - copycatClient.connect(MEMBERS).join(); - - try { - copycatClient.submit(new TestCommand()).join(); - fail(); - } catch (CompletionException e) { - if (!(e.getCause() instanceof ClosedSessionException)) { - fail(); - } - } - - CountDownLatch latch = new CountDownLatch(1); - copycatClient.onStateChange(state -> { - if (state == CopycatClient.State.CONNECTED) { - assertEquals(2, copycatClient.session().id()); - latch.countDown(); - } - }); - latch.await(); - - assertEquals(copycatClient.submit(new TestCommand()).join(), "Hello world!"); - } - - /** - * Tests calling the recovery strategy when a command fails due to UnknownSessionException. - */ - public void testQuerySessionRecovery() throws Throwable { - Connection connection = mock(Connection.class); - when(connection.close()).thenReturn(CompletableFuture.completedFuture(null)); - - Client client = mock(Client.class); - when(client.connect(any())).thenReturn(CompletableFuture.completedFuture(connection)); - - Transport transport = mock(Transport.class); - when(transport.client()).thenReturn(client); - - // Handle connect requests. - when(connection.sendAndReceive(isA(ConnectRequest.class))) - .thenReturn(CompletableFuture.completedFuture(ConnectResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle register requests. - when(connection.sendAndReceive(isA(RegisterRequest.class))) - .thenReturn(CompletableFuture.completedFuture(RegisterResponse.builder() - .withStatus(Response.Status.OK) - .withSession(1) - .withTimeout(5000) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())) - .thenReturn(CompletableFuture.completedFuture(RegisterResponse.builder() - .withStatus(Response.Status.OK) - .withSession(2) - .withTimeout(5000) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Handle keep-alive requests. - Mockito.when(connection.sendAndReceive(isA(KeepAliveRequest.class))) - .thenReturn(CompletableFuture.completedFuture(KeepAliveResponse.builder() - .withStatus(Response.Status.OK) - .withLeader(LEADER) - .withMembers(MEMBERS) - .build())); - - // Fail command requests. - Mockito.when(connection.sendAndReceive(isA(QueryRequest.class))) - .thenReturn(CompletableFuture.completedFuture(QueryResponse.builder() - .withStatus(Response.Status.ERROR) - .withError(CopycatError.Type.UNKNOWN_SESSION_ERROR) - .build())) - .thenReturn(CompletableFuture.completedFuture(QueryResponse.builder() - .withStatus(Response.Status.OK) - .withIndex(1) - .withEventIndex(0) - .withResult("Hello world!") - .build())); - - CopycatClient copycatClient = CopycatClient.builder() - .withTransport(transport) - .withRecoveryStrategy(RecoveryStrategies.RECOVER) - .build(); - - copycatClient.connect(MEMBERS).join(); - - try { - copycatClient.submit(new TestQuery()).join(); - fail(); - } catch (CompletionException e) { - if (!(e.getCause() instanceof ClosedSessionException)) { - fail(); - } - } - - CountDownLatch latch = new CountDownLatch(1); - copycatClient.onStateChange(state -> { - if (state == CopycatClient.State.CONNECTED) { - assertEquals(2, copycatClient.session().id()); - latch.countDown(); - } - }); - latch.await(); - - assertEquals(copycatClient.submit(new TestQuery()).join(), "Hello world!"); - } - - /** - * Test command. - */ - private static class TestCommand implements Command { - } - - /** - * Test query. - */ - private static class TestQuery implements Query { - } - -} diff --git a/client/src/test/java/io/atomix/copycat/client/RecoveryStrategiesTest.java b/client/src/test/java/io/atomix/copycat/client/RecoveryStrategiesTest.java deleted file mode 100644 index 38fd861c..00000000 --- a/client/src/test/java/io/atomix/copycat/client/RecoveryStrategiesTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License - */ -package io.atomix.copycat.client; - -import org.testng.annotations.Test; - -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; - -/** - * Session recovery strategies test. - * - * @author future1 = new CompletableFuture<>(); CompletableFuture future2 = new CompletableFuture<>(); - Connection connection = mock(Connection.class); - Mockito.>when(connection.sendAndReceive(any(CommandRequest.class))) + CopycatConnection connection = mock(CopycatLeaderConnection.class); + Mockito.>when(connection.sendAndReceive(CommandRequest.NAME, any(CommandRequest.class))) .thenReturn(future1) .thenReturn(future2); - ClientSessionState state = new ClientSessionState(UUID.randomUUID().toString()) - .setSessionId(1) - .setState(Session.State.OPEN); - - Executor executor = new MockExecutor(); - ThreadContext context = mock(ThreadContext.class); - when(context.executor()).thenReturn(executor); + CopycatSessionState state = new CopycatSessionState(1, UUID.randomUUID().toString(), "test"); + CopycatSessionManager manager = mock(CopycatSessionManager.class); + ThreadContext threadContext = new TestContext(); - ClientSessionSubmitter submitter = new ClientSessionSubmitter(connection, state, new ClientSequencer(state), context); + CopycatSessionSubmitter submitter = new CopycatSessionSubmitter(connection, mock(CopycatSessionConnection.class), state, new CopycatSessionSequencer(state), manager, threadContext); CompletableFuture result1 = submitter.submit(new TestCommand()); CompletableFuture result2 = submitter.submit(new TestCommand()); @@ -126,23 +119,19 @@ public void testResequenceCommand() throws Throwable { * Tests submitting a query to the cluster. */ public void testSubmitQuery() throws Throwable { - Connection connection = mock(Connection.class); - when(connection.sendAndReceive(any(QueryRequest.class))) + CopycatConnection connection = mock(CopycatSessionConnection.class); + when(connection.sendAndReceive(QueryRequest.NAME, any(QueryRequest.class))) .thenReturn(CompletableFuture.completedFuture(QueryResponse.builder() .withStatus(Response.Status.OK) .withIndex(10) .withResult("Hello world!") .build())); - ClientSessionState state = new ClientSessionState(UUID.randomUUID().toString()) - .setSessionId(1) - .setState(Session.State.OPEN); - - Executor executor = new MockExecutor(); - ThreadContext context = mock(ThreadContext.class); - when(context.executor()).thenReturn(executor); + CopycatSessionState state = new CopycatSessionState(1, UUID.randomUUID().toString(), "test"); + CopycatSessionManager manager = mock(CopycatSessionManager.class); + ThreadContext threadContext = new TestContext(); - ClientSessionSubmitter submitter = new ClientSessionSubmitter(connection, state, new ClientSequencer(state), context); + CopycatSessionSubmitter submitter = new CopycatSessionSubmitter(mock(CopycatLeaderConnection.class), connection, state, new CopycatSessionSequencer(state), manager, threadContext); assertEquals(submitter.submit(new TestQuery()).get(), "Hello world!"); assertEquals(state.getResponseIndex(), 10); } @@ -154,20 +143,16 @@ public void testResequenceQuery() throws Throwable { CompletableFuture future1 = new CompletableFuture<>(); CompletableFuture future2 = new CompletableFuture<>(); - Connection connection = mock(Connection.class); - Mockito.>when(connection.sendAndReceive(any(QueryRequest.class))) + CopycatConnection connection = mock(CopycatSessionConnection.class); + Mockito.>when(connection.sendAndReceive(QueryRequest.NAME, any(QueryRequest.class))) .thenReturn(future1) .thenReturn(future2); - ClientSessionState state = new ClientSessionState(UUID.randomUUID().toString()) - .setSessionId(1) - .setState(Session.State.OPEN); + CopycatSessionState state = new CopycatSessionState(1, UUID.randomUUID().toString(), "test"); + CopycatSessionManager manager = mock(CopycatSessionManager.class); + ThreadContext threadContext = new TestContext(); - Executor executor = new MockExecutor(); - ThreadContext context = mock(ThreadContext.class); - when(context.executor()).thenReturn(executor); - - ClientSessionSubmitter submitter = new ClientSessionSubmitter(connection, state, new ClientSequencer(state), context); + CopycatSessionSubmitter submitter = new CopycatSessionSubmitter(mock(CopycatLeaderConnection.class), connection, state, new CopycatSessionSequencer(state), manager, threadContext); CompletableFuture result1 = submitter.submit(new TestQuery()); CompletableFuture result2 = submitter.submit(new TestQuery()); @@ -204,20 +189,16 @@ public void testSkippingOverFailedQuery() throws Throwable { CompletableFuture future1 = new CompletableFuture<>(); CompletableFuture future2 = new CompletableFuture<>(); - Connection connection = mock(Connection.class); - Mockito.>when(connection.sendAndReceive(any(QueryRequest.class))) + CopycatConnection connection = mock(CopycatSessionConnection.class); + Mockito.>when(connection.sendAndReceive(QueryRequest.NAME, any(QueryRequest.class))) .thenReturn(future1) .thenReturn(future2); - ClientSessionState state = new ClientSessionState(UUID.randomUUID().toString()) - .setSessionId(1) - .setState(Session.State.OPEN); - - Executor executor = new MockExecutor(); - ThreadContext context = mock(ThreadContext.class); - when(context.executor()).thenReturn(executor); + CopycatSessionState state = new CopycatSessionState(1, UUID.randomUUID().toString(), "test"); + CopycatSessionManager manager = mock(CopycatSessionManager.class); + ThreadContext threadContext = new TestContext(); - ClientSessionSubmitter submitter = new ClientSessionSubmitter(connection, state, new ClientSequencer(state), context); + CopycatSessionSubmitter submitter = new CopycatSessionSubmitter(mock(CopycatLeaderConnection.class), connection, state, new CopycatSessionSequencer(state), manager, threadContext); CompletableFuture result1 = submitter.submit(new TestQuery()); CompletableFuture result2 = submitter.submit(new TestQuery()); @@ -247,19 +228,15 @@ public void testSkippingOverFailedQuery() throws Throwable { public void testExpireSessionOnCommandFailure() throws Throwable { CompletableFuture future = new CompletableFuture<>(); - Connection connection = mock(Connection.class); - Mockito.>when(connection.sendAndReceive(any(QueryRequest.class))) + CopycatConnection connection = mock(CopycatLeaderConnection.class); + Mockito.>when(connection.sendAndReceive(QueryRequest.NAME, any(QueryRequest.class))) .thenReturn(future); - ClientSessionState state = new ClientSessionState(UUID.randomUUID().toString()) - .setSessionId(1) - .setState(Session.State.OPEN); + CopycatSessionState state = new CopycatSessionState(1, UUID.randomUUID().toString(), "test"); + CopycatSessionManager manager = mock(CopycatSessionManager.class); + ThreadContext threadContext = new TestContext(); - Executor executor = new MockExecutor(); - ThreadContext context = mock(ThreadContext.class); - when(context.executor()).thenReturn(executor); - - ClientSessionSubmitter submitter = new ClientSessionSubmitter(connection, state, new ClientSequencer(state), context); + CopycatSessionSubmitter submitter = new CopycatSessionSubmitter(connection, mock(CopycatSessionConnection.class), state, new CopycatSessionSequencer(state), manager, threadContext); CompletableFuture result = submitter.submit(new TestCommand()); @@ -270,7 +247,6 @@ public void testExpireSessionOnCommandFailure() throws Throwable { future.completeExceptionally(new UnknownSessionException("unknown session")); assertTrue(result.isCompletedExceptionally()); - assertEquals(ClientSession.State.EXPIRED, state.getState()); } /** @@ -279,19 +255,15 @@ public void testExpireSessionOnCommandFailure() throws Throwable { public void testExpireSessionOnQueryFailure() throws Throwable { CompletableFuture future = new CompletableFuture<>(); - Connection connection = mock(Connection.class); - Mockito.>when(connection.sendAndReceive(any(QueryRequest.class))) + CopycatConnection connection = mock(CopycatSessionConnection.class); + Mockito.>when(connection.sendAndReceive(QueryRequest.NAME, any(QueryRequest.class))) .thenReturn(future); - ClientSessionState state = new ClientSessionState(UUID.randomUUID().toString()) - .setSessionId(1) - .setState(Session.State.OPEN); - - Executor executor = new MockExecutor(); - ThreadContext context = mock(ThreadContext.class); - when(context.executor()).thenReturn(executor); + CopycatSessionState state = new CopycatSessionState(1, UUID.randomUUID().toString(), "test"); + CopycatSessionManager manager = mock(CopycatSessionManager.class); + ThreadContext threadContext = new TestContext(); - ClientSessionSubmitter submitter = new ClientSessionSubmitter(connection, state, new ClientSequencer(state), context); + CopycatSessionSubmitter submitter = new CopycatSessionSubmitter(mock(CopycatLeaderConnection.class), connection, state, new CopycatSessionSequencer(state), manager, threadContext); CompletableFuture result = submitter.submit(new TestQuery()); @@ -302,7 +274,6 @@ public void testExpireSessionOnQueryFailure() throws Throwable { future.completeExceptionally(new UnknownSessionException("unknown session")); assertTrue(result.isCompletedExceptionally()); - assertEquals(ClientSession.State.EXPIRED, state.getState()); } /** @@ -318,9 +289,49 @@ private static class TestQuery implements Query { } /** - * Mock executor. + * Test thread context. */ - private static class MockExecutor implements Executor { + private static class TestContext implements ThreadContext { + @Override + public Logger logger() { + return null; + } + + @Override + public Serializer serializer() { + return null; + } + + @Override + public boolean isBlocked() { + return false; + } + + @Override + public void block() { + + } + + @Override + public void unblock() { + + } + + @Override + public Scheduled schedule(Duration delay, Runnable callback) { + return null; + } + + @Override + public Scheduled schedule(Duration initialDelay, Duration interval, Runnable callback) { + return null; + } + + @Override + public void close() { + + } + @Override public void execute(Runnable command) { command.run(); diff --git a/client/src/test/java/io/atomix/copycat/client/util/AddressSelectorTest.java b/client/src/test/java/io/atomix/copycat/client/util/AddressSelectorTest.java index df05619e..98a8a8d4 100644 --- a/client/src/test/java/io/atomix/copycat/client/util/AddressSelectorTest.java +++ b/client/src/test/java/io/atomix/copycat/client/util/AddressSelectorTest.java @@ -16,7 +16,7 @@ package io.atomix.copycat.client.util; import io.atomix.catalyst.transport.Address; -import io.atomix.copycat.client.ServerSelectionStrategies; +import io.atomix.copycat.client.CommunicationStrategies; import org.testng.annotations.Test; import java.util.Arrays; @@ -36,13 +36,15 @@ public class AddressSelectorTest { * Tests iterating an address selector. */ public void testIterate() throws Throwable { + AddressSelectorManager selectors = new AddressSelectorManager(); + Collection
servers = Arrays.asList( new Address("localhost", 5000), new Address("localhost", 5001), new Address("localhost", 5002) ); - AddressSelector selector = new AddressSelector(ServerSelectionStrategies.ANY); + AddressSelector selector = selectors.createSelector(CommunicationStrategies.ANY); selector.reset(null, servers); assertNull(selector.leader()); assertEquals(selector.servers(), servers); @@ -62,13 +64,15 @@ public void testIterate() throws Throwable { * Tests resetting an address selector. */ public void testReset() throws Throwable { + AddressSelectorManager selectors = new AddressSelectorManager(); + Collection
servers = Arrays.asList( new Address("localhost", 5000), new Address("localhost", 5001), new Address("localhost", 5002) ); - AddressSelector selector = new AddressSelector(ServerSelectionStrategies.ANY); + AddressSelector selector = selectors.createSelector(CommunicationStrategies.ANY); selector.reset(null, servers); selector.next(); selector.next(); @@ -85,13 +89,15 @@ public void testReset() throws Throwable { * Tests updating the members in a selector. */ public void testUpdate() throws Throwable { + AddressSelectorManager selectors = new AddressSelectorManager(); + Collection
servers = Arrays.asList( new Address("localhost", 5000), new Address("localhost", 5001), new Address("localhost", 5002) ); - AddressSelector selector = new AddressSelector(ServerSelectionStrategies.ANY); + AddressSelector selector = selectors.createSelector(CommunicationStrategies.ANY); selector.reset(null, servers); assertNull(selector.leader()); assertEquals(selector.servers(), servers); diff --git a/protocol/src/main/java/io/atomix/copycat/error/CopycatError.java b/protocol/src/main/java/io/atomix/copycat/error/CopycatError.java index 6208508d..74f4a24a 100644 --- a/protocol/src/main/java/io/atomix/copycat/error/CopycatError.java +++ b/protocol/src/main/java/io/atomix/copycat/error/CopycatError.java @@ -124,20 +124,40 @@ public CopycatException createException() { } }, + /** + * Unknown client error. + */ + UNKNOWN_CLIENT_ERROR(6) { + @Override + public CopycatException createException() { + return new UnknownClientException("Unknown client"); + } + }, + /** * Unknown session error. */ - UNKNOWN_SESSION_ERROR(6) { + UNKNOWN_SESSION_ERROR(7) { @Override public CopycatException createException() { return new UnknownSessionException("unknown member session"); } }, + /** + * Unknown state machine error. + */ + UNKNOWN_STATE_MACHINE_ERROR(8) { + @Override + public CopycatException createException() { + return new UnknownStateMachineException("Unknown state machine"); + } + }, + /** * Internal error. */ - INTERNAL_ERROR(7) { + INTERNAL_ERROR(9) { @Override public CopycatException createException() { return new InternalException("internal Raft error"); @@ -147,7 +167,7 @@ public CopycatException createException() { /** * Configuration error. */ - CONFIGURATION_ERROR(8) { + CONFIGURATION_ERROR(10) { @Override public CopycatException createException() { return new ConfigurationException("configuration failed"); diff --git a/protocol/src/main/java/io/atomix/copycat/error/UnknownClientException.java b/protocol/src/main/java/io/atomix/copycat/error/UnknownClientException.java new file mode 100644 index 00000000..f3d7fc0c --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/error/UnknownClientException.java @@ -0,0 +1,38 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.error; + +/** + * Indicates that an operation or other request from an unknown client was received. + * + * @author Jordan Halterman + */ +public class UnknownClientException extends CopycatException { + private static final CopycatError.Type TYPE = CopycatError.Type.UNKNOWN_CLIENT_ERROR; + + public UnknownClientException(String message, Object... args) { + super(TYPE, message, args); + } + + public UnknownClientException(Throwable cause, String message, Object... args) { + super(TYPE, cause, message, args); + } + + public UnknownClientException(Throwable cause) { + super(TYPE, cause); + } + +} diff --git a/protocol/src/main/java/io/atomix/copycat/error/UnknownStateMachineException.java b/protocol/src/main/java/io/atomix/copycat/error/UnknownStateMachineException.java new file mode 100644 index 00000000..52c2d7a4 --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/error/UnknownStateMachineException.java @@ -0,0 +1,38 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.error; + +/** + * Indicates that an operation or other request for an unknown state machine was received. + * + * @author Jordan Halterman + */ +public class UnknownStateMachineException extends CopycatException { + private static final CopycatError.Type TYPE = CopycatError.Type.UNKNOWN_STATE_MACHINE_ERROR; + + public UnknownStateMachineException(String message, Object... args) { + super(TYPE, message, args); + } + + public UnknownStateMachineException(Throwable cause, String message, Object... args) { + super(TYPE, cause, message, args); + } + + public UnknownStateMachineException(Throwable cause) { + super(TYPE, cause); + } + +} diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequest.java new file mode 100644 index 00000000..b212cdda --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequest.java @@ -0,0 +1,82 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.protocol; + +import io.atomix.catalyst.buffer.BufferInput; +import io.atomix.catalyst.buffer.BufferOutput; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.catalyst.util.Assert; + +/** + * Base client request. + *

+ * This is the base request for client-related requests. Many client requests are handled within the + * context of a {@link #client()} identifier. + * + * @author Jordan Halterman + */ +public abstract class ClientRequest extends AbstractRequest { + protected long client; + + /** + * Returns the client ID. + * + * @return The client ID. + */ + public long client() { + return client; + } + + @Override + public void readObject(BufferInput buffer, Serializer serializer) { + client = buffer.readLong(); + } + + @Override + public void writeObject(BufferOutput buffer, Serializer serializer) { + buffer.writeLong(client); + } + + /** + * Session request builder. + */ + public static abstract class Builder, U extends ClientRequest> extends AbstractRequest.Builder { + protected Builder(U request) { + super(request); + } + + /** + * Sets the client ID. + * + * @param client The client ID. + * @return The request builder. + * @throws NullPointerException if {@code client} is null + */ + @SuppressWarnings("unchecked") + public T withClient(long client) { + request.client = Assert.argNot(client, client <= 0, "client must be positive"); + return (T) this; + } + + @Override + public U build() { + super.build(); + Assert.argNot(request.client <= 0, "client must be positive"); + return request; + } + } + +} diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequestTypeResolver.java b/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequestTypeResolver.java index c6ba5f2c..f301097e 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequestTypeResolver.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequestTypeResolver.java @@ -36,6 +36,8 @@ public final class ClientRequestTypeResolver implements SerializableTypeResolver put(QueryRequest.class, -7); put(RegisterRequest.class, -8); put(UnregisterRequest.class, -9); + put(OpenSessionRequest.class, -10); + put(CloseSessionRequest.class, -11); }}; @Override diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponse.java b/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponse.java new file mode 100644 index 00000000..0d6822b9 --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponse.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.protocol; + +/** + * Base client response. + * + * @author Jordan Halterman + */ +public abstract class ClientResponse extends AbstractResponse { + + /** + * Client response builder. + */ + public static abstract class Builder, U extends ClientResponse> extends AbstractResponse.Builder { + protected Builder(U response) { + super(response); + } + } + +} diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponseTypeResolver.java b/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponseTypeResolver.java index 07a0f4ef..67fd2a1d 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponseTypeResolver.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponseTypeResolver.java @@ -29,13 +29,15 @@ public final class ClientResponseTypeResolver implements SerializableTypeResolver { @SuppressWarnings("unchecked") private static final Map, Integer> TYPES = new HashMap() {{ - put(CommandResponse.class, -10); - put(ConnectResponse.class, -11); - put(KeepAliveResponse.class, -12); - put(ResetRequest.class, -13); - put(QueryResponse.class, -14); - put(RegisterResponse.class, -15); - put(UnregisterResponse.class, -16); + put(CommandResponse.class, -12); + put(ConnectResponse.class, -13); + put(KeepAliveResponse.class, -14); + put(ResetRequest.class, -15); + put(QueryResponse.class, -16); + put(RegisterResponse.class, -17); + put(UnregisterResponse.class, -18); + put(OpenSessionResponse.class, -19); + put(CloseSessionResponse.class, -20); }}; @Override diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/CloseSessionRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/CloseSessionRequest.java new file mode 100644 index 00000000..6e1965cb --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/protocol/CloseSessionRequest.java @@ -0,0 +1,76 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.protocol; + +import java.util.Objects; + +/** + * Close session request. + * + * @author Jordan Halterman + */ +public class CloseSessionRequest extends SessionRequest { + public static final String NAME = "close-session"; + + /** + * Returns a new unregister request builder. + * + * @return A new unregister request builder. + */ + public static Builder builder() { + return new Builder(new CloseSessionRequest()); + } + + /** + * Returns a unregister request builder for an existing request. + * + * @param request The request to build. + * @return The unregister request builder. + * @throws NullPointerException if {@code request} is null + */ + public static Builder builder(CloseSessionRequest request) { + return new Builder(request); + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), session); + } + + @Override + public boolean equals(Object object) { + if (object instanceof CloseSessionRequest) { + CloseSessionRequest request = (CloseSessionRequest) object; + return request.session == session; + } + return false; + } + + @Override + public String toString() { + return String.format("%s[session=%d]", getClass().getSimpleName(), session); + } + + /** + * Unregister request builder. + */ + public static class Builder extends SessionRequest.Builder { + protected Builder(CloseSessionRequest request) { + super(request); + } + } + +} diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/CloseSessionResponse.java b/protocol/src/main/java/io/atomix/copycat/protocol/CloseSessionResponse.java new file mode 100644 index 00000000..7fc623e8 --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/protocol/CloseSessionResponse.java @@ -0,0 +1,98 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.protocol; + +import io.atomix.catalyst.buffer.BufferInput; +import io.atomix.catalyst.buffer.BufferOutput; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.copycat.error.CopycatError; + +import java.util.Objects; + +/** + * Close session response. + * + * @author Jordan Halterman + */ +public class CloseSessionResponse extends SessionResponse { + + /** + * Returns a new keep alive response builder. + * + * @return A new keep alive response builder. + */ + public static Builder builder() { + return new Builder(new CloseSessionResponse()); + } + + /** + * Returns a keep alive response builder for an existing response. + * + * @param response The response to build. + * @return The keep alive response builder. + * @throws NullPointerException if {@code response} is null + */ + public static Builder builder(CloseSessionResponse response) { + return new Builder(response); + } + + @Override + public void readObject(BufferInput buffer, Serializer serializer) { + status = Status.forId(buffer.readByte()); + if (status == Status.OK) { + error = null; + } else { + error = CopycatError.forId(buffer.readByte()); + } + } + + @Override + public void writeObject(BufferOutput buffer, Serializer serializer) { + buffer.writeByte(status.id()); + if (status == Status.ERROR) { + buffer.writeByte(error.id()); + } + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), status); + } + + @Override + public boolean equals(Object object) { + if (object instanceof CloseSessionResponse) { + CloseSessionResponse response = (CloseSessionResponse) object; + return response.status == status; + } + return false; + } + + @Override + public String toString() { + return String.format("%s[status=%s, error=%s]", getClass().getSimpleName(), status, error); + } + + /** + * Status response builder. + */ + public static class Builder extends SessionResponse.Builder { + protected Builder(CloseSessionResponse response) { + super(response); + } + } + +} diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/CommandRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/CommandRequest.java index 5e209e4d..fec7094a 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/CommandRequest.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/CommandRequest.java @@ -41,6 +41,7 @@ * @author Jordan Halterman */ public class CommandRequest extends OperationRequest { + public static final String NAME = "command"; /** * Returns a new submit request builder. diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/ConnectRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/ConnectRequest.java index 7fa93848..400e5566 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/ConnectRequest.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/ConnectRequest.java @@ -18,7 +18,6 @@ import io.atomix.catalyst.buffer.BufferInput; import io.atomix.catalyst.buffer.BufferOutput; import io.atomix.catalyst.serializer.Serializer; -import io.atomix.catalyst.util.Assert; import java.util.Objects; @@ -33,6 +32,7 @@ * @author Jordan Halterman */ public class ConnectRequest extends AbstractRequest { + public static final String NAME = "connect"; /** * Returns a new connect client request builder. @@ -54,42 +54,54 @@ public static Builder builder(ConnectRequest request) { return new Builder(request); } - private String client; + private long session; + private long connection; /** - * Returns the connecting client ID. + * Returns the connecting session ID. * - * @return The connecting client ID. + * @return The connecting session ID. */ - public String client() { - return client; + public long session() { + return session; + } + + /** + * Returns the connection ID. + * + * @return The connection ID. + */ + public long connection() { + return connection; } @Override public void writeObject(BufferOutput buffer, Serializer serializer) { super.writeObject(buffer, serializer); - buffer.writeString(client); + buffer.writeLong(session); + buffer.writeLong(connection); } @Override public void readObject(BufferInput buffer, Serializer serializer) { super.readObject(buffer, serializer); - client = buffer.readString(); + session = buffer.readLong(); + connection = buffer.readLong(); } @Override public int hashCode() { - return Objects.hash(getClass(), client); + return Objects.hash(getClass(), session); } @Override public boolean equals(Object object) { - return object instanceof ConnectRequest && ((ConnectRequest) object).client.equals(client); + return object instanceof ConnectRequest && ((ConnectRequest) object).session == session; } @Override public String toString() { - return String.format("%s[client=%s]", getClass().getSimpleName(), client); + return String.format("%s[session=%d, connection=%d]", getClass().getSimpleName(), session, connection); } /** @@ -101,21 +113,25 @@ protected Builder(ConnectRequest request) { } /** - * Sets the connecting client ID. + * Sets the connecting session ID. * - * @param clientId The connecting client ID. + * @param session The connecting session ID. * @return The connect request builder. */ - public Builder withClientId(String clientId) { - request.client = Assert.notNull(clientId, "clientId"); + public Builder withSession(long session) { + request.session = session; return this; } - @Override - public ConnectRequest build() { - super.build(); - Assert.stateNot(request.client == null, "client cannot be null"); - return request; + /** + * Sets the connection ID. + * + * @param connection The connection ID. + * @return The connect request builder. + */ + public Builder withConnection(long connection) { + request.connection = connection; + return this; } } diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/KeepAliveRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/KeepAliveRequest.java index 74a06633..82efa196 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/KeepAliveRequest.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/KeepAliveRequest.java @@ -20,6 +20,7 @@ import io.atomix.catalyst.serializer.Serializer; import io.atomix.catalyst.util.Assert; +import java.util.Arrays; import java.util.Objects; /** @@ -29,13 +30,14 @@ * a {@link RegisterRequest}. Once a session has been registered, clients are responsible for sending * keep alive requests to the cluster at a rate less than the provided {@link RegisterResponse#timeout()}. * Keep alive requests also server to acknowledge the receipt of responses and events by the client. - * The {@link #commandSequence()} number indicates the highest command sequence number for which the client - * has received a response, and the {@link #eventIndex()} number indicates the highest index for which the + * The {@link #commandSequences()} number indicates the highest command sequence number for which the client + * has received a response, and the {@link #eventIndexes()} numbers indicate the highest index for which the * client has received an event in proper sequence. * * @author Jordan Halterman */ -public class KeepAliveRequest extends SessionRequest { +public class KeepAliveRequest extends ClientRequest { + public static final String NAME = "keep-alive"; /** * Returns a new keep alive request builder. @@ -57,101 +59,183 @@ public static Builder builder(KeepAliveRequest request) { return new Builder(request); } - private long commandSequence; - private long eventIndex; + private long[] sessionIds; + private long[] commandSequences; + private long[] eventIndexes; + private long[] connections; /** - * Returns the command sequence number. + * Returns the session identifiers. * - * @return The command sequence number. + * @return The session identifiers. */ - public long commandSequence() { - return commandSequence; + public long[] sessionIds() { + return sessionIds; } /** - * Returns the event index. + * Returns the command sequence numbers. * - * @return The event index. + * @return The command sequence numbers. */ - public long eventIndex() { - return eventIndex; + public long[] commandSequences() { + return commandSequences; + } + + /** + * Returns the event indexes. + * + * @return The event indexes. + */ + public long[] eventIndexes() { + return eventIndexes; + } + + /** + * Returns the session connections. + * + * @return The session connections. + */ + public long[] connections() { + return connections; } @Override public void readObject(BufferInput buffer, Serializer serializer) { super.readObject(buffer, serializer); - commandSequence = buffer.readLong(); - eventIndex = buffer.readLong(); + int sessionsLength = buffer.readInt(); + sessionIds = new long[sessionsLength]; + for (int i = 0; i < sessionsLength; i++) { + sessionIds[i] = buffer.readLong(); + } + + int commandSequencesLength = buffer.readInt(); + commandSequences = new long[commandSequencesLength]; + for (int i = 0; i < commandSequencesLength; i++) { + commandSequences[i] = buffer.readLong(); + } + + int eventIndexesLength = buffer.readInt(); + eventIndexes = new long[eventIndexesLength]; + for (int i = 0; i < eventIndexesLength; i++) { + eventIndexes[i] = buffer.readLong(); + } + + int connectionsLength = buffer.readInt(); + connections = new long[connectionsLength]; + for (int i = 0; i < connectionsLength; i++) { + connections[i] = buffer.readLong(); + } } @Override public void writeObject(BufferOutput buffer, Serializer serializer) { super.writeObject(buffer, serializer); - buffer.writeLong(commandSequence); - buffer.writeLong(eventIndex); + buffer.writeInt(sessionIds.length); + for (long sessionId : sessionIds) { + buffer.writeLong(sessionId); + } + + buffer.writeInt(commandSequences.length); + for (long commandSequence : commandSequences) { + buffer.writeLong(commandSequence); + } + + buffer.writeInt(eventIndexes.length); + for (long eventIndex : eventIndexes) { + buffer.writeLong(eventIndex); + } + + buffer.writeInt(connections.length); + for (long connection : connections) { + buffer.writeLong(connection); + } } @Override public int hashCode() { - return Objects.hash(getClass(), session, commandSequence); + return Objects.hash(getClass(), client, commandSequences, eventIndexes); } @Override public boolean equals(Object object) { if (object instanceof KeepAliveRequest) { KeepAliveRequest request = (KeepAliveRequest) object; - return request.session == session - && request.commandSequence == commandSequence - && request.eventIndex == eventIndex; + return request.client == client + && Arrays.equals(request.commandSequences, commandSequences) + && Arrays.equals(request.eventIndexes, eventIndexes); } return false; } @Override public String toString() { - return String.format("%s[session=%d, commandSequence=%d, eventIndex=%d]", getClass().getSimpleName(), session, commandSequence, eventIndex); + return String.format("%s[client=%s, sessionIds=%s, commandSequences=%s, eventIndexes=%s, connections=%s]", getClass().getSimpleName(), client, Arrays.toString(sessionIds), Arrays.toString(commandSequences), Arrays.toString(eventIndexes), Arrays.toString(connections)); } /** * Keep alive request builder. */ - public static class Builder extends SessionRequest.Builder { + public static class Builder extends ClientRequest.Builder { protected Builder(KeepAliveRequest request) { super(request); } /** - * Sets the command sequence number. + * Sets the session identifiers. * - * @param commandSequence The command sequence number. + * @param sessionIds The session identifiers. + * @return The request builders. + * @throws NullPointerException if {@code sessionIds} is {@code null} + */ + public Builder withSessionIds(long[] sessionIds) { + request.sessionIds = Assert.notNull(sessionIds, "sessionIds"); + return this; + } + + /** + * Sets the command sequence numbers. + * + * @param commandSequences The command sequence numbers. * @return The request builder. - * @throws IllegalArgumentException if {@code commandSequence} is less than 0 + * @throws NullPointerException if {@code commandSequences} is {@code null} */ - public Builder withCommandSequence(long commandSequence) { - request.commandSequence = Assert.argNot(commandSequence, commandSequence < 0, "commandSequence cannot be negative"); + public Builder withCommandSequences(long[] commandSequences) { + request.commandSequences = Assert.notNull(commandSequences, "commandSequences"); return this; } /** - * Sets the event index. + * Sets the event indexes. * - * @param eventIndex The event index. + * @param eventIndexes The event indexes. * @return The request builder. - * @throws IllegalArgumentException if {@code eventIndex} is less than 0 + * @throws NullPointerException if {@code eventIndexes} is {@code null} */ - public Builder withEventIndex(long eventIndex) { - request.eventIndex = Assert.argNot(eventIndex, eventIndex < 0, "eventIndex cannot be negative"); + public Builder withEventIndexes(long[] eventIndexes) { + request.eventIndexes = Assert.notNull(eventIndexes, "eventIndexes"); return this; } /** - * @throws IllegalStateException is session is not positive + * Sets the client connections. + * + * @param connections The client connections. + * @return The request builder. + * @throws NullPointerException if {@code connections} is {@code null} */ + public Builder withConnections(long[] connections) { + request.connections = Assert.notNull(connections, "connections"); + return this; + } + @Override public KeepAliveRequest build() { super.build(); - Assert.state(request.session > 0, "session must be positive"); + Assert.notNull(request.sessionIds, "sessionIds"); + Assert.notNull(request.commandSequences, "commandSequences"); + Assert.notNull(request.eventIndexes, "eventIndexes"); + Assert.notNull(request.connections, "connections"); return request; } } diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/KeepAliveResponse.java b/protocol/src/main/java/io/atomix/copycat/protocol/KeepAliveResponse.java index 44b9340a..3b59faf4 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/KeepAliveResponse.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/KeepAliveResponse.java @@ -35,7 +35,7 @@ * * @author Jordan Halterman */ -public class KeepAliveResponse extends SessionResponse { +public class KeepAliveResponse extends ClientResponse { /** * Returns a new keep alive response builder. @@ -129,7 +129,7 @@ public String toString() { /** * Status response builder. */ - public static class Builder extends SessionResponse.Builder { + public static class Builder extends ClientResponse.Builder { protected Builder(KeepAliveResponse response) { super(response); diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/OpenSessionRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/OpenSessionRequest.java new file mode 100644 index 00000000..dbd501a9 --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/protocol/OpenSessionRequest.java @@ -0,0 +1,151 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.protocol; + +import io.atomix.catalyst.buffer.BufferInput; +import io.atomix.catalyst.buffer.BufferOutput; +import io.atomix.catalyst.serializer.SerializationException; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.catalyst.util.Assert; + +import java.util.Objects; + +/** + * Open session request. + */ +public class OpenSessionRequest extends ClientRequest { + public static final String NAME = "open-session"; + + /** + * Returns a new open session request builder. + * + * @return A new open session request builder. + */ + public static Builder builder() { + return new Builder(new OpenSessionRequest()); + } + + /** + * Returns an open session request builder for an existing request. + * + * @param request The request to build. + * @return The open session request builder. + * @throws NullPointerException if {@code request} is null + */ + public static Builder builder(OpenSessionRequest request) { + return new Builder(request); + } + + private String name; + private String type; + + /** + * Returns the state machine name. + * + * @return The state machine name. + */ + public String name() { + return name; + } + + /** + * Returns the state machine type; + * + * @return The state machine type. + */ + public String type() { + return type; + } + + @Override + public void readObject(BufferInput buffer, Serializer serializer) { + super.readObject(buffer, serializer); + name = buffer.readString(); + type = buffer.readString(); + } + + @Override + public void writeObject(BufferOutput buffer, Serializer serializer) { + super.writeObject(buffer, serializer); + buffer.writeString(name); + buffer.writeString(type); + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), name, type); + } + + @Override + public boolean equals(Object object) { + if (object instanceof OpenSessionRequest) { + OpenSessionRequest request = (OpenSessionRequest) object; + return request.client == client + && request.name.equals(name) + && request.type.equals(type); + } + return false; + } + + @Override + public String toString() { + return String.format("%s[client=%s, name=%s, type=%s]", getClass().getSimpleName(), client, name, type); + } + + /** + * Open session request builder. + */ + public static class Builder extends ClientRequest.Builder { + protected Builder(OpenSessionRequest request) { + super(request); + } + + /** + * Sets the state machine name. + * + * @param name The state machine name. + * @return The open session request builder. + * @throws NullPointerException if {@code name} is {@code null} + */ + public Builder withName(String name) { + request.name = Assert.notNull(name, "name"); + return this; + } + + /** + * Sets the state machine type. + * + * @param type The state machine type. + * @return The open session request builder. + * @throws NullPointerException if {@code type} is {@code null} + */ + public Builder withType(String type) { + request.type = Assert.notNull(type, "type"); + return this; + } + + /** + * @throws IllegalStateException is session is not positive + */ + @Override + public OpenSessionRequest build() { + super.build(); + Assert.notNull(request.name, "name"); + Assert.notNull(request.type, "type"); + return request; + } + } +} diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/OpenSessionResponse.java b/protocol/src/main/java/io/atomix/copycat/protocol/OpenSessionResponse.java new file mode 100644 index 00000000..f89007b7 --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/protocol/OpenSessionResponse.java @@ -0,0 +1,127 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.protocol; + +import io.atomix.catalyst.buffer.BufferInput; +import io.atomix.catalyst.buffer.BufferOutput; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.catalyst.util.Assert; +import io.atomix.copycat.error.CopycatError; + +import java.util.Objects; + +/** + * Open session response. + * + * @author Jordan Halterman + */ +public class OpenSessionResponse extends ClientResponse { + + /** + * Returns a new register client response builder. + * + * @return A new register client response builder. + */ + public static Builder builder() { + return new Builder(new OpenSessionResponse()); + } + + /** + * Returns a register client response builder for an existing response. + * + * @param response The response to build. + * @return The register client response builder. + * @throws NullPointerException if {@code response} is null + */ + public static Builder builder(OpenSessionResponse response) { + return new Builder(response); + } + + private long session; + + /** + * Returns the registered session ID. + * + * @return The registered session ID. + */ + public long session() { + return session; + } + + @Override + public void readObject(BufferInput buffer, Serializer serializer) { + status = Status.forId(buffer.readByte()); + if (status == Status.OK) { + error = null; + session = buffer.readLong(); + } else { + error = CopycatError.forId(buffer.readByte()); + session = 0; + } + } + + @Override + public void writeObject(BufferOutput buffer, Serializer serializer) { + buffer.writeByte(status.id()); + if (status == Status.OK) { + buffer.writeLong(session); + } else { + buffer.writeByte(error.id()); + } + } + + @Override + public int hashCode() { + return Objects.hash(getClass(), status, session); + } + + @Override + public boolean equals(Object object) { + if (object instanceof OpenSessionResponse) { + OpenSessionResponse response = (OpenSessionResponse) object; + return response.status == status + && response.session == session; + } + return false; + } + + @Override + public String toString() { + return String.format("%s[status=%s, error=%s, session=%d]", getClass().getSimpleName(), status, error, session); + } + + /** + * Register response builder. + */ + public static class Builder extends ClientResponse.Builder { + protected Builder(OpenSessionResponse response) { + super(response); + } + + /** + * Sets the response session ID. + * + * @param session The session ID. + * @return The register response builder. + * @throws IllegalArgumentException if {@code session} is less than 1 + */ + public Builder withSession(long session) { + response.session = Assert.argNot(session, session < 1, "session must be positive"); + return this; + } + } + +} diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/QueryRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/QueryRequest.java index 5e79bdce..9398bfa6 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/QueryRequest.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/QueryRequest.java @@ -44,6 +44,7 @@ * @author Jordan Halterman */ public class QueryRequest extends OperationRequest { + public static final String NAME = "query"; /** * Returns a new query request builder. diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/RegisterRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/RegisterRequest.java index 0b783d37..68725dde 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/RegisterRequest.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/RegisterRequest.java @@ -34,6 +34,7 @@ * @author Jordan Halterman */ public class RegisterRequest extends AbstractRequest { + public static final String NAME = "register"; /** * Returns a new register client request builder. diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/RegisterResponse.java b/protocol/src/main/java/io/atomix/copycat/protocol/RegisterResponse.java index 7fa48e7c..6e0c5173 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/RegisterResponse.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/RegisterResponse.java @@ -30,7 +30,7 @@ *

* Session register responses are sent in response to {@link RegisterRequest}s * sent by a client. Upon the successful registration of a session, the register response will contain the - * registered {@link #session()} identifier, the session {@link #timeout()}, and the current cluster + * registered {@link #clientId()} identifier, the session {@link #timeout()}, and the current cluster * {@link #leader()} and {@link #members()} to allow the client to make intelligent decisions about * connecting to and communicating with the cluster. * @@ -58,7 +58,7 @@ public static Builder builder(RegisterResponse response) { return new Builder(response); } - private long session; + private long clientId; private Address leader; private Collection

members; private long timeout; @@ -68,8 +68,8 @@ public static Builder builder(RegisterResponse response) { * * @return The registered session ID. */ - public long session() { - return session; + public long clientId() { + return clientId; } /** @@ -104,13 +104,13 @@ public void readObject(BufferInput buffer, Serializer serializer) { status = Status.forId(buffer.readByte()); if (status == Status.OK) { error = null; - session = buffer.readLong(); + clientId = buffer.readLong(); timeout = buffer.readLong(); leader = serializer.readObject(buffer); members = serializer.readObject(buffer); } else { error = CopycatError.forId(buffer.readByte()); - session = 0; + clientId = 0; members = null; } } @@ -119,7 +119,7 @@ public void readObject(BufferInput buffer, Serializer serializer) { public void writeObject(BufferOutput buffer, Serializer serializer) { buffer.writeByte(status.id()); if (status == Status.OK) { - buffer.writeLong(session); + buffer.writeLong(clientId); buffer.writeLong(timeout); serializer.writeObject(leader, buffer); serializer.writeObject(members, buffer); @@ -130,7 +130,7 @@ public void writeObject(BufferOutput buffer, Serializer serializer) { @Override public int hashCode() { - return Objects.hash(getClass(), status, session, leader, members); + return Objects.hash(getClass(), status, clientId, leader, members); } @Override @@ -138,7 +138,7 @@ public boolean equals(Object object) { if (object instanceof RegisterResponse) { RegisterResponse response = (RegisterResponse) object; return response.status == status - && response.session == session + && response.clientId == clientId && ((response.leader == null && leader == null) || (response.leader != null && leader != null && response.leader.equals(leader))) && ((response.members == null && members == null) @@ -150,7 +150,7 @@ public boolean equals(Object object) { @Override public String toString() { - return String.format("%s[status=%s, error=%s, session=%d, leader=%s, members=%s]", getClass().getSimpleName(), status, error, session, leader, members); + return String.format("%s[status=%s, error=%s, clientId=%d, leader=%s, members=%s]", getClass().getSimpleName(), status, error, clientId, leader, members); } /** @@ -162,14 +162,14 @@ protected Builder(RegisterResponse response) { } /** - * Sets the response session ID. + * Sets the response client ID. * - * @param session The session ID. + * @param clientId The client ID. * @return The register response builder. * @throws IllegalArgumentException if {@code session} is less than 1 */ - public Builder withSession(long session) { - response.session = Assert.argNot(session, session < 1, "session must be positive"); + public Builder withClientId(long clientId) { + response.clientId = Assert.argNot(clientId, clientId < 1, "clientId must be positive"); return this; } diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/ResetRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/ResetRequest.java index ac9d7cf8..1d40c75e 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/ResetRequest.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/ResetRequest.java @@ -31,6 +31,7 @@ * @author Jordan Halterman */ public class ResetRequest extends SessionRequest { + public static final String NAME = "reset"; /** * Returns a new publish response builder. diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/UnregisterRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/UnregisterRequest.java index 27c16e29..8680b3ac 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/UnregisterRequest.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/UnregisterRequest.java @@ -18,15 +18,12 @@ import java.util.Objects; /** - * Session unregister request. - *

- * The unregister request is sent by a client with an open session to the cluster to explicitly - * unregister its session. Note that if a client does not send an unregister request, its session will - * eventually expire. The unregister request simply provides a more orderly method for closing client sessions. + * Client unregister request. * * @author Jordan Halterman */ -public class UnregisterRequest extends SessionRequest { +public class UnregisterRequest extends ClientRequest { + public static final String NAME = "unregister"; /** * Returns a new unregister request builder. @@ -50,27 +47,27 @@ public static Builder builder(UnregisterRequest request) { @Override public int hashCode() { - return Objects.hash(getClass(), session); + return Objects.hash(getClass(), client); } @Override public boolean equals(Object object) { if (object instanceof UnregisterRequest) { UnregisterRequest request = (UnregisterRequest) object; - return request.session == session; + return request.client == client; } return false; } @Override public String toString() { - return String.format("%s[session=%d]", getClass().getSimpleName(), session); + return String.format("%s[client=%s]", getClass().getSimpleName(), client); } /** * Unregister request builder. */ - public static class Builder extends SessionRequest.Builder { + public static class Builder extends ClientRequest.Builder { protected Builder(UnregisterRequest request) { super(request); } diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/UnregisterResponse.java b/protocol/src/main/java/io/atomix/copycat/protocol/UnregisterResponse.java index 26d5d628..9a2c4662 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/UnregisterResponse.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/UnregisterResponse.java @@ -32,7 +32,7 @@ * * @author Jordan Halterman */ -public class UnregisterResponse extends SessionResponse { +public class UnregisterResponse extends ClientResponse { /** * Returns a new keep alive response builder. @@ -94,7 +94,7 @@ public String toString() { /** * Status response builder. */ - public static class Builder extends SessionResponse.Builder { + public static class Builder extends ClientResponse.Builder { protected Builder(UnregisterResponse response) { super(response); } diff --git a/server/src/main/java/io/atomix/copycat/server/CopycatServer.java b/server/src/main/java/io/atomix/copycat/server/CopycatServer.java index 990f9095..e1e2cd2e 100644 --- a/server/src/main/java/io/atomix/copycat/server/CopycatServer.java +++ b/server/src/main/java/io/atomix/copycat/server/CopycatServer.java @@ -16,10 +16,7 @@ package io.atomix.copycat.server; import io.atomix.catalyst.buffer.PooledHeapAllocator; -import io.atomix.catalyst.concurrent.Futures; -import io.atomix.catalyst.concurrent.Listener; -import io.atomix.catalyst.concurrent.SingleThreadContext; -import io.atomix.catalyst.concurrent.ThreadContext; +import io.atomix.catalyst.concurrent.*; import io.atomix.catalyst.serializer.Serializer; import io.atomix.catalyst.transport.Address; import io.atomix.catalyst.transport.Server; @@ -34,6 +31,7 @@ import io.atomix.copycat.server.cluster.Member; import io.atomix.copycat.server.state.ConnectionManager; import io.atomix.copycat.server.state.ServerContext; +import io.atomix.copycat.server.state.StateMachineRegistry; import io.atomix.copycat.server.storage.Log; import io.atomix.copycat.server.storage.Storage; import io.atomix.copycat.server.storage.StorageLevel; @@ -44,11 +42,11 @@ import org.slf4j.LoggerFactory; import java.time.Duration; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; +import java.util.*; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -693,7 +691,7 @@ private CompletableFuture start(Supplier> */ private CompletableFuture listen() { CompletableFuture future = new CompletableFuture<>(); - context.getThreadContext().executor().execute(() -> { + context.getThreadContext().execute(() -> { internalServer.listen(cluster().member().serverAddress(), context::connectServer).whenComplete((internalResult, internalError) -> { if (internalError == null) { // If the client address is different than the server address, start a separate client server. @@ -736,7 +734,7 @@ public CompletableFuture shutdown() { return Futures.exceptionalFuture(new IllegalStateException("context not open")); CompletableFuture future = new CompletableFuture<>(); - context.getThreadContext().executor().execute(() -> { + context.getThreadContext().execute(() -> { started = false; if (clientServer != null) { clientServer.close().whenCompleteAsync((clientResult, clientError) -> { @@ -748,8 +746,8 @@ public CompletableFuture shutdown() { } else { future.complete(null); } - }, context.getThreadContext().executor()); - }, context.getThreadContext().executor()); + }, context.getThreadContext()); + }, context.getThreadContext()); } else { internalServer.close().whenCompleteAsync((internalResult, internalError) -> { if (internalError != null) { @@ -757,7 +755,7 @@ public CompletableFuture shutdown() { } else { future.complete(null); } - }, context.getThreadContext().executor()); + }, context.getThreadContext()); } context.transition(CopycatServer.State.INACTIVE); @@ -857,6 +855,7 @@ public static class Builder implements io.atomix.catalyst.util.Builder stateMachineFactory; private Address clientAddress; private Address serverAddress; private Duration electionTimeout = DEFAULT_ELECTION_TIMEOUT; private Duration heartbeatInterval = DEFAULT_HEARTBEAT_INTERVAL; private Duration sessionTimeout = DEFAULT_SESSION_TIMEOUT; private Duration globalSuspendTimeout = DEFAULT_GLOBAL_SUSPEND_TIMEOUT; + private final StateMachineRegistry stateMachineRegistry = new StateMachineRegistry(); + private int threadPoolSize = DEFAULT_THREAD_POOL_SIZE; private Builder(Address clientAddress, Address serverAddress) { this.clientAddress = Assert.notNull(clientAddress, "clientAddress"); @@ -966,12 +966,13 @@ public Builder withStorage(Storage storage) { /** * Sets the Raft state machine factory. * + * @param type The state machine type name. * @param factory The Raft state machine factory. * @return The server builder. * @throws NullPointerException if the {@code factory} is {@code null} */ - public Builder withStateMachine(Supplier factory) { - this.stateMachineFactory = Assert.notNull(factory, "factory"); + public Builder addStateMachine(String type, Supplier factory) { + stateMachineRegistry.register(type, factory); return this; } @@ -1033,13 +1034,25 @@ public Builder withGlobalSuspendTimeout(Duration globalSuspendTimeout) { return this; } + /** + * Sets the server thread pool size. + * + * @param threadPoolSize The server thread pool size. + * @return The server builder. + */ + public Builder withThreadPoolSize(int threadPoolSize) { + this.threadPoolSize = Assert.arg(threadPoolSize, threadPoolSize > 0, "threadPoolSize must be positive"); + return this; + } + /** * @throws ConfigurationException if a state machine, members or transport are not configured */ @Override public CopycatServer build() { - if (stateMachineFactory == null) - throw new ConfigurationException("state machine not configured"); + if (stateMachineRegistry.size() == 0) { + throw new ConfigurationException("No state machines registered"); + } // If the transport is not configured, attempt to use the default Netty transport. if (serverTransport == null) { @@ -1074,8 +1087,9 @@ public CopycatServer build() { ConnectionManager connections = new ConnectionManager(serverTransport.client()); ThreadContext threadContext = new SingleThreadContext(String.format("copycat-server-%s-%s", serverAddress, name), serializer); + ScheduledExecutorService threadPool = Executors.newScheduledThreadPool(threadPoolSize, new CatalystThreadFactory("copycat-" + name + "-state-%d")); - ServerContext context = new ServerContext(name, type, serverAddress, clientAddress, storage, serializer, stateMachineFactory, connections, threadContext); + ServerContext context = new ServerContext(name, type, serverAddress, clientAddress, storage, serializer, stateMachineRegistry, connections, threadPool, threadContext); context.setElectionTimeout(electionTimeout) .setHeartbeatInterval(heartbeatInterval) .setSessionTimeout(sessionTimeout) diff --git a/server/src/main/java/io/atomix/copycat/server/protocol/AppendRequest.java b/server/src/main/java/io/atomix/copycat/server/protocol/AppendRequest.java index fc258e0c..cb978881 100644 --- a/server/src/main/java/io/atomix/copycat/server/protocol/AppendRequest.java +++ b/server/src/main/java/io/atomix/copycat/server/protocol/AppendRequest.java @@ -37,6 +37,7 @@ * @author Jordan Halterman */ public class AppendRequest extends AbstractRequest { + public static final String NAME = "append"; /** * Returns a new append request builder. diff --git a/server/src/main/java/io/atomix/copycat/server/protocol/ConfigureRequest.java b/server/src/main/java/io/atomix/copycat/server/protocol/ConfigureRequest.java index b8e46975..add3fb1e 100644 --- a/server/src/main/java/io/atomix/copycat/server/protocol/ConfigureRequest.java +++ b/server/src/main/java/io/atomix/copycat/server/protocol/ConfigureRequest.java @@ -36,6 +36,7 @@ * @author Jordan Halterman */ public class ConfigureRequest extends AbstractRequest { + public static final String NAME = "configure"; /** * Returns a new configuration request builder. diff --git a/server/src/main/java/io/atomix/copycat/server/protocol/InstallRequest.java b/server/src/main/java/io/atomix/copycat/server/protocol/InstallRequest.java index 514a8f01..ea75f8c3 100644 --- a/server/src/main/java/io/atomix/copycat/server/protocol/InstallRequest.java +++ b/server/src/main/java/io/atomix/copycat/server/protocol/InstallRequest.java @@ -37,6 +37,7 @@ * @author Jordan Halterman */ public class InstallRequest extends AbstractRequest { + public static final String NAME = "install"; /** * Returns a new install request builder. diff --git a/server/src/main/java/io/atomix/copycat/server/protocol/JoinRequest.java b/server/src/main/java/io/atomix/copycat/server/protocol/JoinRequest.java index 0c45885e..653d8008 100644 --- a/server/src/main/java/io/atomix/copycat/server/protocol/JoinRequest.java +++ b/server/src/main/java/io/atomix/copycat/server/protocol/JoinRequest.java @@ -25,6 +25,7 @@ * @author Jordan Halterman */ public class JoinRequest extends ConfigurationRequest { + public static final String NAME = "join"; /** * Returns a new join request builder. diff --git a/server/src/main/java/io/atomix/copycat/server/protocol/LeaveRequest.java b/server/src/main/java/io/atomix/copycat/server/protocol/LeaveRequest.java index 0d65f772..cacbd65b 100644 --- a/server/src/main/java/io/atomix/copycat/server/protocol/LeaveRequest.java +++ b/server/src/main/java/io/atomix/copycat/server/protocol/LeaveRequest.java @@ -25,6 +25,7 @@ * @author Jordan Halterman */ public class LeaveRequest extends ConfigurationRequest { + public static final String NAME = "leave"; /** * Returns a new leave request builder. diff --git a/server/src/main/java/io/atomix/copycat/server/protocol/PollRequest.java b/server/src/main/java/io/atomix/copycat/server/protocol/PollRequest.java index 0e724b36..feda477c 100644 --- a/server/src/main/java/io/atomix/copycat/server/protocol/PollRequest.java +++ b/server/src/main/java/io/atomix/copycat/server/protocol/PollRequest.java @@ -33,6 +33,7 @@ * @author Jordan Halterman */ public class PollRequest extends AbstractRequest { + public static final String NAME = "poll"; /** * Returns a new poll request builder. diff --git a/server/src/main/java/io/atomix/copycat/server/protocol/ReconfigureRequest.java b/server/src/main/java/io/atomix/copycat/server/protocol/ReconfigureRequest.java index 3f46bd72..56e0c0b5 100644 --- a/server/src/main/java/io/atomix/copycat/server/protocol/ReconfigureRequest.java +++ b/server/src/main/java/io/atomix/copycat/server/protocol/ReconfigureRequest.java @@ -28,6 +28,7 @@ * @author Jordan Halterman */ public class VoteRequest extends AbstractRequest { + public static final String NAME = "vote"; /** * Returns a new vote request builder. diff --git a/server/src/main/java/io/atomix/copycat/server/state/AbstractAppender.java b/server/src/main/java/io/atomix/copycat/server/state/AbstractAppender.java index a8f663d0..0f8111fa 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/AbstractAppender.java +++ b/server/src/main/java/io/atomix/copycat/server/state/AbstractAppender.java @@ -193,7 +193,7 @@ protected void sendAppendRequest(Connection connection, MemberState member, Appe long timestamp = System.nanoTime(); logger.trace("{} - Sending {} to {}", context.getCluster().member().address(), request, member.getMember().address()); - connection.sendAndReceive(request).whenComplete((response, error) -> { + connection.sendAndReceive(AppendRequest.NAME, request).whenComplete((response, error) -> { context.checkThread(); // Complete the append to the member. @@ -405,7 +405,7 @@ protected void sendConfigureRequest(MemberState member, ConfigureRequest request */ protected void sendConfigureRequest(Connection connection, MemberState member, ConfigureRequest request) { logger.trace("{} - Sending {} to {}", context.getCluster().member().address(), request, member.getMember().serverAddress()); - connection.sendAndReceive(request).whenComplete((response, error) -> { + connection.sendAndReceive(ConfigureRequest.NAME, request).whenComplete((response, error) -> { context.checkThread(); // Complete the configure to the member. @@ -538,7 +538,7 @@ protected void sendInstallRequest(MemberState member, InstallRequest request) { */ protected void sendInstallRequest(Connection connection, MemberState member, InstallRequest request) { logger.trace("{} - Sending {} to {}", context.getCluster().member().address(), request, member.getMember().serverAddress()); - connection.sendAndReceive(request).whenComplete((response, error) -> { + connection.sendAndReceive(InstallRequest.NAME, request).whenComplete((response, error) -> { context.checkThread(); // Complete the install to the member. diff --git a/server/src/main/java/io/atomix/copycat/server/state/AbstractState.java b/server/src/main/java/io/atomix/copycat/server/state/AbstractState.java index 6501b6c7..fa45f3c9 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/AbstractState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/AbstractState.java @@ -75,11 +75,11 @@ public boolean isOpen() { /** * Forwards the given request to the leader if possible. */ - protected CompletableFuture forward(T request) { + protected CompletableFuture forward(String type, T request) { CompletableFuture future = new CompletableFuture<>(); context.getConnections().getConnection(context.getLeader().serverAddress()).whenComplete((connection, connectError) -> { if (connectError == null) { - connection.sendAndReceive(request).whenComplete((response, responseError) -> { + connection.sendAndReceive(type, request).whenComplete((response, responseError) -> { if (responseError == null) { future.complete(response); } else { diff --git a/server/src/main/java/io/atomix/copycat/server/state/CandidateState.java b/server/src/main/java/io/atomix/copycat/server/state/CandidateState.java index 4750b490..41d556b3 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/CandidateState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/CandidateState.java @@ -147,7 +147,7 @@ private void sendVoteRequests() { .build(); context.getConnections().getConnection(member.serverAddress()).thenAccept(connection -> { - connection.sendAndReceive(request).whenCompleteAsync((response, error) -> { + connection.sendAndReceive(VoteRequest.NAME, request).whenCompleteAsync((response, error) -> { context.checkThread(); if (isOpen() && !complete.get()) { if (error != null) { @@ -171,7 +171,7 @@ private void sendVoteRequests() { } } } - }, context.getThreadContext().executor()); + }, context.getThreadContext()); }); } } diff --git a/server/src/main/java/io/atomix/copycat/server/state/ClientContext.java b/server/src/main/java/io/atomix/copycat/server/state/ClientContext.java new file mode 100644 index 00000000..55ae54a7 --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/state/ClientContext.java @@ -0,0 +1,180 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.state; + +import io.atomix.copycat.server.storage.LogCleaner; + +/** + * Client context. + */ +public class ClientContext { + private final long id; + private final long timeout; + private final LogCleaner cleaner; + private State state = State.OPEN; + private long timestamp; + private long keepAliveIndex; + private volatile boolean unregistering = false; + + public ClientContext(long id, long timeout, LogCleaner cleaner) { + this.id = id; + this.timeout = timeout; + this.cleaner = cleaner; + } + + /** + * Returns the client ID. + * + * @return The client ID. + */ + public long id() { + return id; + } + + /** + * Returns the client timeout. + * + * @return The client timeout. + */ + public long timeout() { + return timeout; + } + + /** + * Returns the client state. + * + * @return The client state. + */ + public State state() { + return state; + } + + /** + * Updates the client state. + * + * @param state The updated client state. + */ + private void setState(State state) { + if (this.state != state) { + this.state = state; + } + } + + /** + * Returns the session timestamp. + * + * @return The session timestamp. + */ + long getTimestamp() { + return timestamp; + } + + /** + * Sets the session timestamp. + * + * @param timestamp The session timestamp. + */ + private void updateTimestamp(long timestamp) { + this.timestamp = Math.max(this.timestamp, timestamp); + } + + /** + * Registers the client's session. + * + * @param timestamp The timestamp of the register. + */ + void open(long timestamp) { + // Update the client's timestamp. + updateTimestamp(timestamp); + + // Set the client's state to OPEN. + setState(State.OPEN); + } + + /** + * Keeps the client's session alive. + * + * @param index The index of the keep-alive. + * @param timestamp The keep-alive timestamp. + */ + void keepAlive(long index, long timestamp) { + // Update the client's timestamp. + updateTimestamp(timestamp); + + // Set the client's state to OPEN. + setState(State.OPEN); + + // If an old keep-alive was applied, clean it from the log. + long oldKeepAliveIndex = this.keepAliveIndex; + if (oldKeepAliveIndex > 0) { + cleaner.clean(oldKeepAliveIndex); + } + this.keepAliveIndex = index; + } + + /** + * Sets the session as suspect. + * + * @param timestamp The suspicion timestamp. + */ + void suspect(long timestamp) { + // Update the client's timestamp. + updateTimestamp(timestamp); + + // Set the client's state to SUSPICIOUS. + setState(State.SUSPICIOUS); + } + + /** + * Sets the session as being unregistered. + */ + void unregister() { + unregistering = true; + } + + /** + * Indicates whether the session is being unregistered. + */ + boolean isUnregistering() { + return unregistering; + } + + /** + * Closes the session. + * + * @param index The index at which to close the session. + */ + void close(long index) { + // Set the client's state to CLOSED. + setState(State.CLOSED); + + // Release the client's RegisterEntry from the log. + cleaner.clean(id); + + // Release the UnregisterEntry from the log. + cleaner.clean(index); + } + + /** + * Client state. + */ + public enum State { + OPEN, + SUSPICIOUS, + CLOSED, + } + +} diff --git a/server/src/main/java/io/atomix/copycat/server/state/ClientManager.java b/server/src/main/java/io/atomix/copycat/server/state/ClientManager.java new file mode 100644 index 00000000..305b46c1 --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/state/ClientManager.java @@ -0,0 +1,66 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.state; + +import java.util.Collection; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * Client manager. + */ +public class ClientManager { + private final Map clients = new ConcurrentHashMap<>(); + + /** + * Returns a client by ID. + * + * @param client The client ID. + * @return The client context. + */ + public ClientContext getClient(long client) { + return clients.get(client); + } + + /** + * Returns the collection of registered clients. + * + * @return The collection of registered clients. + */ + public Collection getClients() { + return clients.values(); + } + + /** + * Registers a client. + * + * @param client The client to register. + */ + void registerClient(ClientContext client) { + clients.put(client.id(), client); + } + + /** + * Unregisters a client. + * + * @param client The client ID. + * @return The unregistered client context. + */ + ClientContext unregisterClient(long client) { + return clients.remove(client); + } + +} diff --git a/server/src/main/java/io/atomix/copycat/server/state/ClusterState.java b/server/src/main/java/io/atomix/copycat/server/state/ClusterState.java index b8adf5fc..f41481c4 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ClusterState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ClusterState.java @@ -350,7 +350,7 @@ public synchronized CompletableFuture join(Collection

cluster) { private synchronized CompletableFuture join() { joinFuture = new CompletableFuture<>(); - context.getThreadContext().executor().execute(() -> { + context.getThreadContext().execute(() -> { // Transition the server to the appropriate state for the local member type. context.transition(member.type()); @@ -384,7 +384,7 @@ private void join(Iterator iterator) { JoinRequest request = JoinRequest.builder() .withMember(new ServerMember(member().type(), member().serverAddress(), member().clientAddress(), member().updated())) .build(); - return connection.sendAndReceive(request); + return connection.sendAndReceive(JoinRequest.NAME, request); }).whenComplete((response, error) -> { // Cancel the join timer. cancelJoinTimer(); @@ -456,7 +456,7 @@ CompletableFuture identify() { .withMember(member()) .build(); LOGGER.trace("{} - Sending {} to {}", member.address(), request, leader.address()); - return connection.sendAndReceive(request); + return connection.sendAndReceive(ReconfigureRequest.NAME, request); }).whenComplete((response, error) -> { cancelJoinTimer(); if (error == null) { @@ -510,7 +510,7 @@ public synchronized CompletableFuture leave() { leaveFuture = new CompletableFuture<>(); - context.getThreadContext().executor().execute(() -> { + context.getThreadContext().execute(() -> { // If a join attempt is still underway, cancel the join and complete the join future exceptionally. // The join future will be set to null once completed. cancelJoinTimer(); diff --git a/server/src/main/java/io/atomix/copycat/server/state/FollowerState.java b/server/src/main/java/io/atomix/copycat/server/state/FollowerState.java index 4e00fde1..290f5309 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/FollowerState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/FollowerState.java @@ -144,7 +144,7 @@ private void sendPollRequests() { .withLogTerm(lastTerm) .build(); context.getConnections().getConnection(member.serverAddress()).thenAccept(connection -> { - connection.sendAndReceive(request).whenCompleteAsync((response, error) -> { + connection.sendAndReceive(PollRequest.NAME, request).whenCompleteAsync((response, error) -> { context.checkThread(); if (isOpen() && !complete.get()) { if (error != null) { @@ -167,7 +167,7 @@ private void sendPollRequests() { } } } - }, context.getThreadContext().executor()); + }, context.getThreadContext()); }); } } diff --git a/server/src/main/java/io/atomix/copycat/server/state/InactiveState.java b/server/src/main/java/io/atomix/copycat/server/state/InactiveState.java index 4ad086f6..77adba7d 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/InactiveState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/InactiveState.java @@ -84,6 +84,16 @@ public CompletableFuture unregister(UnregisterRequest reques return Futures.exceptionalFuture(new IllegalStateException("inactive state")); } + @Override + public CompletableFuture openSession(OpenSessionRequest request) { + return Futures.exceptionalFuture(new IllegalStateException("inactive state")); + } + + @Override + public CompletableFuture closeSession(CloseSessionRequest request) { + return Futures.exceptionalFuture(new IllegalStateException("inactive state")); + } + @Override public void reset(ResetRequest request) { } diff --git a/server/src/main/java/io/atomix/copycat/server/state/LeaderState.java b/server/src/main/java/io/atomix/copycat/server/state/LeaderState.java index b0fadb1d..697a5494 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/LeaderState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/LeaderState.java @@ -28,7 +28,6 @@ import io.atomix.copycat.server.protocol.*; import io.atomix.copycat.server.storage.entry.*; import io.atomix.copycat.server.storage.system.Configuration; -import io.atomix.copycat.session.Session; import java.time.Duration; import java.time.Instant; @@ -158,18 +157,18 @@ private void checkSessions() { long term = context.getTerm(); // Iterate through all currently registered sessions. - for (ServerSessionContext session : context.getStateMachine().executor().context().sessions().sessions.values()) { + for (ClientContext client : context.getStateMachine().getClients().getClients()) { // If the session isn't already being unregistered by this leader and a keep-alive entry hasn't // been committed for the session in some time, log and commit a new UnregisterEntry. - if (session.state() == Session.State.UNSTABLE && !session.isUnregistering()) { - LOGGER.debug("{} - Detected expired session: {}", context.getCluster().member().address(), session.id()); + if (client.state() == ClientContext.State.SUSPICIOUS && !client.isUnregistering()) { + LOGGER.debug("{} - Detected expired client: {}", context.getCluster().member().address(), client.id()); // Log the unregister entry, indicating that the session was explicitly unregistered by the leader. // This will result in state machine expire() methods being called when the entry is applied. final long index; try (UnregisterEntry entry = context.getLog().create(UnregisterEntry.class)) { entry.setTerm(term) - .setSession(session.id()) + .setClient(client.id()) .setExpired(true) .setTimestamp(System.currentTimeMillis()); index = context.getLog().append(entry); @@ -185,7 +184,7 @@ private void checkSessions() { // Mark the session as being unregistered in order to ensure this leader doesn't attempt // to unregister it again. - session.unregister(); + client.unregister(); } } } @@ -480,7 +479,7 @@ public CompletableFuture command(final CommandRequest request) logRequest(request); // Get the client's server session. If the session doesn't exist, return an unknown session error. - ServerSessionContext session = context.getStateMachine().executor().context().sessions().getSession(request.session()); + ServerSessionContext session = context.getStateMachine().getSessions().getSession(request.session()); if (session == null) { return CompletableFuture.completedFuture(logResponse(CommandResponse.builder() .withStatus(Response.Status.ERROR) @@ -525,7 +524,7 @@ public CompletableFuture command(final CommandRequest request) if (isOpen()) { // If the command was successfully committed, apply it to the state machine. if (commitError == null) { - context.getStateMachine().apply(index).whenComplete((result, error) -> { + context.getStateMachine().apply(index).whenComplete((result, error) -> { if (isOpen()) { completeOperation(result, CommandResponse.builder(), error, future); } @@ -588,7 +587,7 @@ private CompletableFuture query(QueryEntry entry) { * since the leader will step down in the event it fails to contact a majority of the cluster. */ private CompletableFuture queryBoundedLinearizable(QueryEntry entry) { - return sequenceAndApply(entry); + return applyQuery(entry); } /** @@ -598,7 +597,7 @@ private CompletableFuture queryBoundedLinearizable(QueryEntry ent * applied, we verify the node's leadership prior to responding successfully to the query. */ private CompletableFuture queryLinearizable(QueryEntry entry) { - return sequenceAndApply(entry) + return applyQuery(entry) .thenCompose(response -> appender.appendEntries() .thenApply(index -> response) .exceptionally(error -> QueryResponse.builder() @@ -607,33 +606,6 @@ private CompletableFuture queryLinearizable(QueryEntry entry) { .build())); } - /** - * Sequences and applies the given query entry. - */ - private CompletableFuture sequenceAndApply(QueryEntry entry) { - // Get the client's server session. If the session doesn't exist, return an unknown session error. - ServerSessionContext session = context.getStateMachine().executor().context().sessions().getSession(entry.getSession()); - if (session == null) { - return CompletableFuture.completedFuture(logResponse(QueryResponse.builder() - .withStatus(Response.Status.ERROR) - .withError(CopycatError.Type.UNKNOWN_SESSION_ERROR) - .build())); - } - - CompletableFuture future = new CompletableFuture<>(); - - // If the query's sequence number is less than the session's current request sequence number but greater than the - // session's current applied sequence number, queue the request for handling once the state machine is caught up. - if (entry.getSequence() > session.getCommandSequence()) { - session.registerSequenceQuery(entry.getSequence(), () -> applyQuery(entry, future)); - } - // If the query is already in sequence then just apply it. - else { - applyQuery(entry, future); - } - return future; - } - @Override public CompletableFuture register(RegisterRequest request) { final long timestamp = System.currentTimeMillis(); @@ -655,7 +627,6 @@ public CompletableFuture register(RegisterRequest request) { try (RegisterEntry entry = context.getLog().create(RegisterEntry.class)) { entry.setTerm(context.getTerm()) .setTimestamp(timestamp) - .setClient(request.client()) .setTimeout(timeout); index = context.getLog().append(entry); LOGGER.trace("{} - Appended {}", context.getCluster().member().address(), entry); @@ -666,12 +637,12 @@ public CompletableFuture register(RegisterRequest request) { context.checkThread(); if (isOpen()) { if (commitError == null) { - context.getStateMachine().apply(index).whenComplete((sessionId, sessionError) -> { + context.getStateMachine().apply(index).whenComplete((clientId, sessionError) -> { if (isOpen()) { if (sessionError == null) { future.complete(logResponse(RegisterResponse.builder() .withStatus(Response.Status.OK) - .withSession((Long) sessionId) + .withClientId((Long) clientId) .withTimeout(timeout) .withLeader(context.getCluster().member().clientAddress()) .withMembers(context.getCluster().members().stream() @@ -715,7 +686,7 @@ public CompletableFuture connect(ConnectRequest request, Connec logRequest(request); // Associate the connection with the appropriate client. - context.getStateMachine().executor().context().sessions().registerConnection(request.client(), connection); + context.getStateMachine().getSessions().registerConnection(request.session(), request.connection(), connection); return CompletableFuture.completedFuture(ConnectResponse.builder() .withStatus(Response.Status.OK) @@ -738,9 +709,11 @@ public CompletableFuture keepAlive(KeepAliveRequest request) try (KeepAliveEntry entry = context.getLog().create(KeepAliveEntry.class)) { entry.setTerm(context.getTerm()) - .setSession(request.session()) - .setCommandSequence(request.commandSequence()) - .setEventIndex(request.eventIndex()) + .setClient(request.client()) + .setSessionIds(request.sessionIds()) + .setCommandSequences(request.commandSequences()) + .setEventIndexes(request.eventIndexes()) + .setConnections(request.connections()) .setTimestamp(timestamp); index = context.getLog().append(entry); LOGGER.trace("{} - Appended {}", context.getCluster().member().address(), entry); @@ -801,12 +774,20 @@ public CompletableFuture unregister(UnregisterRequest reques final long timestamp = System.currentTimeMillis(); final long index; + ClientContext client = context.getStateMachine().getClients().getClient(request.client()); + if (client == null) { + return CompletableFuture.completedFuture(logResponse(UnregisterResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.UNKNOWN_CLIENT_ERROR) + .build())); + } + context.checkThread(); logRequest(request); try (UnregisterEntry entry = context.getLog().create(UnregisterEntry.class)) { entry.setTerm(context.getTerm()) - .setSession(request.session()) + .setClient(client.id()) .setExpired(false) .setTimestamp(timestamp); index = context.getLog().append(entry); @@ -855,6 +836,133 @@ public CompletableFuture unregister(UnregisterRequest reques return future; } + @Override + public CompletableFuture openSession(OpenSessionRequest request) { + final long timestamp = System.currentTimeMillis(); + final long index; + + ClientContext client = context.getStateMachine().getClients().getClient(request.client()); + if (client == null) { + return CompletableFuture.completedFuture(logResponse(OpenSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.UNKNOWN_CLIENT_ERROR) + .build())); + } + + context.checkThread(); + logRequest(request); + + try (OpenSessionEntry entry = context.getLog().create(OpenSessionEntry.class)) { + entry.setTerm(context.getTerm()) + .setClient(client.id()) + .setType(request.type()) + .setName(request.name()) + .setTimestamp(timestamp); + index = context.getLog().append(entry); + LOGGER.trace("{} - Appended {}", context.getCluster().member().address(), entry); + } + + CompletableFuture future = new CompletableFuture<>(); + appender.appendEntries(index).whenComplete((commitIndex, commitError) -> { + context.checkThread(); + if (isOpen()) { + if (commitError == null) { + context.getStateMachine().apply(index).whenComplete((sessionId, sessionError) -> { + if (isOpen()) { + if (sessionError == null) { + future.complete(logResponse(OpenSessionResponse.builder() + .withStatus(Response.Status.OK) + .withSession(sessionId) + .build())); + } else if (sessionError instanceof CompletionException && sessionError.getCause() instanceof CopycatException) { + future.complete(logResponse(OpenSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(((CopycatException) sessionError.getCause()).getType()) + .build())); + } else if (sessionError instanceof CopycatException) { + future.complete(logResponse(OpenSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(((CopycatException) sessionError).getType()) + .build())); + } else { + future.complete(logResponse(OpenSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.INTERNAL_ERROR) + .build())); + } + checkSessions(); + } + }); + } else { + future.complete(logResponse(OpenSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.INTERNAL_ERROR) + .build())); + } + } + }); + + return future; + } + + @Override + public CompletableFuture closeSession(CloseSessionRequest request) { + final long timestamp = System.currentTimeMillis(); + final long index; + + context.checkThread(); + logRequest(request); + + try (CloseSessionEntry entry = context.getLog().create(CloseSessionEntry.class)) { + entry.setTerm(context.getTerm()) + .setSession(request.session()) + .setTimestamp(timestamp); + index = context.getLog().append(entry); + LOGGER.trace("{} - Appended {}", context.getCluster().member().address(), entry); + } + + CompletableFuture future = new CompletableFuture<>(); + appender.appendEntries(index).whenComplete((commitIndex, commitError) -> { + context.checkThread(); + if (isOpen()) { + if (commitError == null) { + context.getStateMachine().apply(index).whenComplete((closeResult, closeError) -> { + if (isOpen()) { + if (closeError == null) { + future.complete(logResponse(CloseSessionResponse.builder() + .withStatus(Response.Status.OK) + .build())); + } else if (closeError instanceof CompletionException && closeError.getCause() instanceof CopycatException) { + future.complete(logResponse(CloseSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(((CopycatException) closeError.getCause()).getType()) + .build())); + } else if (closeError instanceof CopycatException) { + future.complete(logResponse(CloseSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(((CopycatException) closeError).getType()) + .build())); + } else { + future.complete(logResponse(CloseSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.INTERNAL_ERROR) + .build())); + } + checkSessions(); + } + }); + } else { + future.complete(logResponse(CloseSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.INTERNAL_ERROR) + .build())); + } + } + }); + + return future; + } + /** * Cancels the append timer. */ diff --git a/server/src/main/java/io/atomix/copycat/server/state/OperationResult.java b/server/src/main/java/io/atomix/copycat/server/state/OperationResult.java new file mode 100644 index 00000000..d4fb9cb2 --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/state/OperationResult.java @@ -0,0 +1,31 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.state; + +/** + * Operation result. + */ +final class OperationResult { + final long index; + final long eventIndex; + final Object result; + + OperationResult(long index, long eventIndex, Object result) { + this.index = index; + this.eventIndex = eventIndex; + this.result = result; + } +} diff --git a/server/src/main/java/io/atomix/copycat/server/state/PassiveState.java b/server/src/main/java/io/atomix/copycat/server/state/PassiveState.java index bfe3c291..004c0a76 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/PassiveState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/PassiveState.java @@ -26,7 +26,6 @@ import io.atomix.copycat.server.protocol.AppendResponse; import io.atomix.copycat.server.protocol.InstallRequest; import io.atomix.copycat.server.protocol.InstallResponse; -import io.atomix.copycat.server.session.ServerSession; import io.atomix.copycat.server.storage.entry.Entry; import io.atomix.copycat.server.storage.entry.QueryEntry; import io.atomix.copycat.server.storage.snapshot.Snapshot; @@ -72,9 +71,9 @@ private void truncateUncommittedEntries() { @Override public void reset(ResetRequest request) { - ServerSessionContext session = context.getStateMachine().executor().context().sessions().getSession(request.session()); + ServerSessionContext session = context.getStateMachine().getSessions().getSession(request.session()); if (session != null) { - context.getStateMachine().executor().executor().execute(() -> session.resendEvents(request.index())); + session.getStateMachineExecutor().reset(request.index(), session); } } @@ -91,7 +90,7 @@ public CompletableFuture connect(ConnectRequest request, Connec .build())); } else { // Associate the connection with the appropriate session. - context.getStateMachine().executor().context().sessions().registerConnection(request.client(), connection); + context.getStateMachine().getSessions().registerConnection(request.session(), request.connection(), connection); return CompletableFuture.completedFuture(ConnectResponse.builder() .withStatus(Response.Status.OK) @@ -258,7 +257,7 @@ public CompletableFuture query(QueryRequest request) { .setSequence(request.sequence()) .setQuery(request.query()); - return queryLocal(entry).thenApply(this::logResponse); + return applyQuery(entry).thenApply(this::logResponse); } else { return queryForward(request); } @@ -276,7 +275,7 @@ private CompletableFuture queryForward(QueryRequest request) { } LOGGER.trace("{} - Forwarding {}", context.getCluster().member().address(), request); - return this.forward(request) + return this.forward(QueryRequest.NAME, request) .exceptionally(error -> QueryResponse.builder() .withStatus(Response.Status.ERROR) .withError(CopycatError.Type.NO_LEADER_ERROR) @@ -288,76 +287,17 @@ private CompletableFuture queryForward(QueryRequest request) { * Performs a local query. */ protected CompletableFuture queryLocal(QueryEntry entry) { - CompletableFuture future = new CompletableFuture<>(); - sequenceQuery(entry, future); - return future; - } - - /** - * Sequences the given query. - */ - private void sequenceQuery(QueryEntry entry, CompletableFuture future) { - // Get the client's server session. If the session doesn't exist, return an unknown session error. - ServerSessionContext session = context.getStateMachine().executor().context().sessions().getSession(entry.getSession()); - if (session == null) { - future.complete(logResponse(QueryResponse.builder() - .withStatus(Response.Status.ERROR) - .withError(CopycatError.Type.UNKNOWN_SESSION_ERROR) - .build())); - } else { - sequenceQuery(entry, session, future); - } - } - - /** - * Sequences the given query. - */ - private void sequenceQuery(QueryEntry entry, ServerSessionContext session, CompletableFuture future) { - // If the query's sequence number is greater than the session's current sequence number, queue the request for - // handling once the state machine is caught up. - if (entry.getSequence() > session.getCommandSequence()) { - session.registerSequenceQuery(entry.getSequence(), () -> indexQuery(entry, future)); - } else { - indexQuery(entry, future); - } - } - - /** - * Ensures the given query is applied after the appropriate index. - */ - private void indexQuery(QueryEntry entry, CompletableFuture future) { - // Get the client's server session. If the session doesn't exist, return an unknown session error. - ServerSessionContext session = context.getStateMachine().executor().context().sessions().getSession(entry.getSession()); - if (session == null) { - future.complete(logResponse(QueryResponse.builder() - .withStatus(Response.Status.ERROR) - .withError(CopycatError.Type.UNKNOWN_SESSION_ERROR) - .build())); - } else { - indexQuery(entry, session, future); - } - } - - /** - * Ensures the given query is applied after the appropriate index. - */ - private void indexQuery(QueryEntry entry, ServerSessionContext session, CompletableFuture future) { - // If the query index is greater than the session's last applied index, queue the request for handling once the - // state machine is caught up. - if (entry.getIndex() > session.getLastApplied()) { - session.registerIndexQuery(entry.getIndex(), () -> applyQuery(entry, future)); - } else { - applyQuery(entry, future); - } + return applyQuery(entry); } /** * Applies a query to the state machine. */ - protected CompletableFuture applyQuery(QueryEntry entry, CompletableFuture future) { + protected CompletableFuture applyQuery(QueryEntry entry) { // In the case of the leader, the state machine is always up to date, so no queries will be queued and all query // indexes will be the last applied index. - context.getStateMachine().apply(entry).whenComplete((result, error) -> { + CompletableFuture future = new CompletableFuture<>(); + context.getStateMachine().apply(entry).whenComplete((result, error) -> { completeOperation(result, QueryResponse.builder(), error, future); entry.release(); }); @@ -367,7 +307,7 @@ protected CompletableFuture applyQuery(QueryEntry entry, Completa /** * Completes an operation. */ - protected void completeOperation(ServerStateMachine.Result result, OperationResponse.Builder builder, Throwable error, CompletableFuture future) { + protected void completeOperation(OperationResult result, OperationResponse.Builder builder, Throwable error, CompletableFuture future) { if (isOpen()) { if (result != null) { builder.withIndex(result.index); diff --git a/server/src/main/java/io/atomix/copycat/server/state/ReserveState.java b/server/src/main/java/io/atomix/copycat/server/state/ReserveState.java index 58dda314..04b6932a 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ReserveState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ReserveState.java @@ -101,7 +101,7 @@ public CompletableFuture command(CommandRequest request) { .withError(CopycatError.Type.NO_LEADER_ERROR) .build())); } else { - return this.forward(request) + return this.forward(CommandRequest.NAME, request) .exceptionally(error -> CommandResponse.builder() .withStatus(Response.Status.ERROR) .withError(CopycatError.Type.NO_LEADER_ERROR) @@ -121,7 +121,7 @@ public CompletableFuture query(QueryRequest request) { .withError(CopycatError.Type.NO_LEADER_ERROR) .build())); } else { - return this.forward(request) + return this.forward(QueryRequest.NAME, request) .exceptionally(error -> QueryResponse.builder() .withStatus(Response.Status.ERROR) .withError(CopycatError.Type.NO_LEADER_ERROR) @@ -141,7 +141,7 @@ public CompletableFuture register(RegisterRequest request) { .withError(CopycatError.Type.NO_LEADER_ERROR) .build())); } else { - return this.forward(request) + return this.forward(RegisterRequest.NAME, request) .exceptionally(error -> RegisterResponse.builder() .withStatus(Response.Status.ERROR) .withError(CopycatError.Type.NO_LEADER_ERROR) @@ -171,7 +171,7 @@ public CompletableFuture keepAlive(KeepAliveRequest request) .withError(CopycatError.Type.NO_LEADER_ERROR) .build())); } else { - return this.forward(request) + return this.forward(KeepAliveRequest.NAME, request) .exceptionally(error -> KeepAliveResponse.builder() .withStatus(Response.Status.ERROR) .withError(CopycatError.Type.NO_LEADER_ERROR) @@ -191,7 +191,7 @@ public CompletableFuture unregister(UnregisterRequest reques .withError(CopycatError.Type.NO_LEADER_ERROR) .build())); } else { - return this.forward(request) + return this.forward(UnregisterRequest.NAME, request) .exceptionally(error -> UnregisterResponse.builder() .withStatus(Response.Status.ERROR) .withError(CopycatError.Type.NO_LEADER_ERROR) @@ -200,6 +200,46 @@ public CompletableFuture unregister(UnregisterRequest reques } } + @Override + public CompletableFuture openSession(OpenSessionRequest request) { + context.checkThread(); + logRequest(request); + + if (context.getLeader() == null) { + return CompletableFuture.completedFuture(logResponse(OpenSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.NO_LEADER_ERROR) + .build())); + } else { + return this.forward(OpenSessionRequest.NAME, request) + .exceptionally(error -> OpenSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.NO_LEADER_ERROR) + .build()) + .thenApply(this::logResponse); + } + } + + @Override + public CompletableFuture closeSession(CloseSessionRequest request) { + context.checkThread(); + logRequest(request); + + if (context.getLeader() == null) { + return CompletableFuture.completedFuture(logResponse(CloseSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.NO_LEADER_ERROR) + .build())); + } else { + return this.forward(CloseSessionRequest.NAME, request) + .exceptionally(error -> CloseSessionResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.NO_LEADER_ERROR) + .build()) + .thenApply(this::logResponse); + } + } + @Override public CompletableFuture join(JoinRequest request) { context.checkThread(); @@ -211,7 +251,7 @@ public CompletableFuture join(JoinRequest request) { .withError(CopycatError.Type.NO_LEADER_ERROR) .build())); } else { - return this.forward(request) + return this.forward(JoinRequest.NAME, request) .exceptionally(error -> JoinResponse.builder() .withStatus(Response.Status.ERROR) .withError(CopycatError.Type.NO_LEADER_ERROR) @@ -231,7 +271,7 @@ public CompletableFuture reconfigure(ReconfigureRequest req .withError(CopycatError.Type.NO_LEADER_ERROR) .build())); } else { - return this.forward(request) + return this.forward(ReconfigureRequest.NAME, request) .exceptionally(error -> ReconfigureResponse.builder() .withStatus(Response.Status.ERROR) .withError(CopycatError.Type.NO_LEADER_ERROR) @@ -251,7 +291,7 @@ public CompletableFuture leave(LeaveRequest request) { .withError(CopycatError.Type.NO_LEADER_ERROR) .build())); } else { - return this.forward(request) + return this.forward(LeaveRequest.NAME, request) .exceptionally(error -> LeaveResponse.builder() .withStatus(Response.Status.ERROR) .withError(CopycatError.Type.NO_LEADER_ERROR) diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerCommit.java b/server/src/main/java/io/atomix/copycat/server/state/ServerCommit.java index 8d7abde2..a907eafe 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerCommit.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerCommit.java @@ -20,8 +20,9 @@ import io.atomix.copycat.Operation; import io.atomix.copycat.server.Commit; import io.atomix.copycat.server.session.ServerSession; -import io.atomix.copycat.server.storage.Log; -import io.atomix.copycat.server.storage.entry.OperationEntry; +import io.atomix.copycat.server.storage.LogCleaner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.time.Instant; import java.util.concurrent.atomic.AtomicInteger; @@ -32,35 +33,20 @@ * @author Jordan Halterman */ final class ServerCommit implements Commit> { - private final ServerCommitPool pool; - private final Log log; - private final AtomicInteger references = new AtomicInteger(); - private volatile long index; - private volatile ServerSessionContext session; - private volatile Instant instant; - private volatile Operation operation; - - public ServerCommit(ServerCommitPool pool, Log log) { - this.pool = pool; - this.log = log; - } - - /** - * Resets the commit. - * - * @param entry The entry. - */ - void reset(OperationEntry entry, ServerSessionContext session, long timestamp) { - if (references.compareAndSet(0, 1)) { - this.index = entry.getIndex(); - this.session = session; - this.instant = Instant.ofEpochMilli(timestamp); - this.operation = entry.getOperation(); - session.acquire(); - references.set(1); - } else { - throw new IllegalStateException("Cannot recycle commit with " + references.get() + " references"); - } + private static final Logger LOGGER = LoggerFactory.getLogger(ServerCommit.class); + private final LogCleaner cleaner; + private final AtomicInteger references = new AtomicInteger(1); + private final long index; + private final ServerSessionContext session; + private final Instant instant; + private final Operation operation; + + public ServerCommit(long index, Operation operation, ServerSessionContext session, long timestamp, LogCleaner cleaner) { + this.index = index; + this.session = session; + this.instant = Instant.ofEpochMilli(timestamp); + this.operation = operation; + this.cleaner = cleaner; } /** @@ -133,36 +119,22 @@ public void close() { * Cleans up the commit. */ private void cleanup() { - if (operation instanceof Command && log.isOpen()) { - try { - log.release(index); - } catch (IllegalStateException e) { - } + if (operation instanceof Command) { + cleaner.clean(index); } - - session.release(); - - index = 0; - session = null; - instant = null; - operation = null; - - pool.release(this); } @Override protected void finalize() throws Throwable { - pool.warn(this); + if (references.get() > 0) { + LOGGER.warn("An unreleased commit was garbage collected"); + } super.finalize(); } @Override public String toString() { - if (references() > 0) { - return String.format("%s[index=%d, session=%s, time=%s, operation=%s]", getClass().getSimpleName(), index(), session(), time(), operation()); - } else { - return String.format("%s[index=unknown]", getClass().getSimpleName()); - } + return String.format("%s[index=%d, session=%s, time=%s, operation=%s]", getClass().getSimpleName(), index(), session(), time(), operation()); } } diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerCommitPool.java b/server/src/main/java/io/atomix/copycat/server/state/ServerCommitPool.java deleted file mode 100644 index 59569c9d..00000000 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerCommitPool.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.atomix.copycat.server.state; - -import io.atomix.catalyst.util.Assert; -import io.atomix.copycat.server.storage.Log; -import io.atomix.copycat.server.storage.entry.OperationEntry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.Queue; -import java.util.concurrent.ConcurrentLinkedQueue; - -/** - * Server commit pool. - * - * @author Jordan Halterman - */ -final class ServerCommitPool implements AutoCloseable { - private static final Logger LOGGER = LoggerFactory.getLogger(ServerCommitPool.class); - private final Log log; - private final ServerSessionManager sessions; - private final Queue pool = new ConcurrentLinkedQueue<>(); - - public ServerCommitPool(Log log, ServerSessionManager sessions) { - this.log = Assert.notNull(log, "log"); - this.sessions = Assert.notNull(sessions, "sessions"); - } - - /** - * Acquires a commit from the pool. - * - * @param entry The entry for which to acquire the commit. - * @return The commit to acquire. - */ - public ServerCommit acquire(OperationEntry entry, ServerSessionContext session, long timestamp) { - ServerCommit commit = pool.poll(); - if (commit == null) { - commit = new ServerCommit(this, log); - } - commit.reset(entry, session, timestamp); - return commit; - } - - /** - * Releases a commit back to the pool. - * - * @param commit The commit to release. - */ - public void release(ServerCommit commit) { - pool.add(commit); - } - - /** - * Issues a warning that the given commit was garbage collected. - * - * @param commit The commit that was garbage collected. - */ - public void warn(ServerCommit commit) { - LOGGER.trace("Server commit " + commit + " was garbage collected!\nCommit log is dirty!"); - } - - @Override - public void close() { - pool.clear(); - } - -} diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerContext.java b/server/src/main/java/io/atomix/copycat/server/state/ServerContext.java index 20a2997e..eaee49a2 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerContext.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerContext.java @@ -25,22 +25,23 @@ import io.atomix.catalyst.util.Assert; import io.atomix.copycat.protocol.*; import io.atomix.copycat.server.CopycatServer; -import io.atomix.copycat.server.Snapshottable; import io.atomix.copycat.server.StateMachine; import io.atomix.copycat.server.cluster.Cluster; import io.atomix.copycat.server.cluster.Member; import io.atomix.copycat.server.protocol.*; import io.atomix.copycat.server.storage.Log; import io.atomix.copycat.server.storage.Storage; -import io.atomix.copycat.server.storage.compaction.Compaction; import io.atomix.copycat.server.storage.snapshot.SnapshotStore; import io.atomix.copycat.server.storage.system.MetaStore; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.time.Duration; +import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledExecutorService; import java.util.function.Consumer; import java.util.function.Function; import java.util.function.Supplier; @@ -59,14 +60,15 @@ public class ServerContext implements AutoCloseable { private final Listeners electionListeners = new Listeners<>(); protected final String name; protected final ThreadContext threadContext; - protected final Supplier stateMachineFactory; + protected final StateMachineRegistry registry; protected final ClusterState cluster; protected final Storage storage; protected final Serializer serializer; private MetaStore meta; private Log log; private SnapshotStore snapshot; - private ServerStateMachine stateMachine; + private ServerStateMachineManager stateMachine; + protected final ScheduledExecutorService threadPool; protected final ThreadContext stateContext; protected final ConnectionManager connections; protected ServerState state = new InactiveState(this); @@ -81,24 +83,43 @@ public class ServerContext implements AutoCloseable { private long globalIndex; @SuppressWarnings("unchecked") - public ServerContext(String name, Member.Type type, Address serverAddress, Address clientAddress, Storage storage, Serializer serializer, Supplier stateMachineFactory, ConnectionManager connections, ThreadContext threadContext) { + public ServerContext(String name, Member.Type type, Address serverAddress, Address clientAddress, Storage storage, Serializer serializer, StateMachineRegistry registry, ConnectionManager connections, ScheduledExecutorService threadPool, ThreadContext threadContext) { this.name = Assert.notNull(name, "name"); this.storage = Assert.notNull(storage, "storage"); this.serializer = Assert.notNull(serializer, "serializer"); this.threadContext = Assert.notNull(threadContext, "threadContext"); this.connections = Assert.notNull(connections, "connections"); - this.stateMachineFactory = Assert.notNull(stateMachineFactory, "stateMachineFactory"); + this.registry = Assert.notNull(registry, "registry"); this.stateContext = new SingleThreadContext(String.format("copycat-server-%s-%s-state", serverAddress, name), threadContext.serializer().clone()); + this.threadPool = Assert.notNull(threadPool, "threadPool"); // Open the meta store. - threadContext.execute(() -> this.meta = storage.openMetaStore(name)).join(); + CountDownLatch metaLatch = new CountDownLatch(1); + threadContext.execute(() -> { + this.meta = storage.openMetaStore(name); + metaLatch.countDown(); + }); + + try { + metaLatch.await(); + } catch (InterruptedException e) { + } // Load the current term and last vote from disk. this.term = meta.loadTerm(); this.lastVotedFor = meta.loadVote(); // Reset the state machine. - threadContext.execute(this::reset).join(); + CountDownLatch resetLatch = new CountDownLatch(1); + threadContext.execute(() -> { + reset(); + resetLatch.countDown(); + }); + + try { + resetLatch.await(); + } catch (InterruptedException e) { + } this.cluster = new ClusterState(type, serverAddress, clientAddress, this); } @@ -414,10 +435,19 @@ long getGlobalIndex() { * * @return The server state machine. */ - public ServerStateMachine getStateMachine() { + public ServerStateMachineManager getStateMachine() { return stateMachine; } + /** + * Returns the server state machine registry. + * + * @return The server state machine registry. + */ + public StateMachineRegistry getStateMachineRegistry() { + return registry; + } + /** * Returns the current state. * @@ -478,19 +508,8 @@ ServerContext reset() { // Open the snapshot store. snapshot = storage.openSnapshotStore(name); - // Create a new user state machine. - StateMachine stateMachine = stateMachineFactory.get(); - - // Configure the log compaction mode. If the state machine supports snapshotting, the default - // compaction mode is SNAPSHOT, otherwise the default is SEQUENTIAL. - if (stateMachine instanceof Snapshottable) { - log.compactor().withDefaultCompactionMode(Compaction.Mode.SNAPSHOT); - } else { - log.compactor().withDefaultCompactionMode(Compaction.Mode.SEQUENTIAL); - } - // Create a new internal server state machine. - this.stateMachine = new ServerStateMachine(stateMachine, this, stateContext); + this.stateMachine = new ServerStateMachineManager(this, threadPool, stateContext); return this; } @@ -518,15 +537,17 @@ public void connectClient(Connection connection) { // Note we do not use method references here because the "state" variable changes over time. // We have to use lambdas to ensure the request handler points to the current state. - connection.handler(RegisterRequest.class, (Function>) request -> state.register(request)); - connection.handler(ConnectRequest.class, (Function>) request -> state.connect(request, connection)); - connection.handler(KeepAliveRequest.class, (Function>) request -> state.keepAlive(request)); - connection.handler(UnregisterRequest.class, (Function>) request -> state.unregister(request)); - connection.handler(ResetRequest.class, (Consumer) request -> state.reset(request)); - connection.handler(CommandRequest.class, (Function>) request -> state.command(request)); - connection.handler(QueryRequest.class, (Function>) request -> state.query(request)); + connection.registerHandler(RegisterRequest.NAME, (Function>) request -> state.register(request)); + connection.registerHandler(ConnectRequest.NAME, (Function>) request -> state.connect(request, connection)); + connection.registerHandler(KeepAliveRequest.NAME, (Function>) request -> state.keepAlive(request)); + connection.registerHandler(UnregisterRequest.NAME, (Function>) request -> state.unregister(request)); + connection.registerHandler(OpenSessionRequest.NAME, (Function>) request -> state.openSession(request)); + connection.registerHandler(CloseSessionRequest.NAME, (Function>) request -> state.closeSession(request)); + connection.registerHandler(ResetRequest.NAME, (Consumer) request -> state.reset(request)); + connection.registerHandler(CommandRequest.NAME, (Function>) request -> state.command(request)); + connection.registerHandler(QueryRequest.NAME, (Function>) request -> state.query(request)); - connection.onClose(stateMachine.executor().context().sessions()::unregisterConnection); + connection.onClose(stateMachine.getSessions()::unregisterConnection); } /** @@ -538,23 +559,25 @@ public void connectServer(Connection connection) { // Handlers for all request types are registered since requests can be proxied between servers. // Note we do not use method references here because the "state" variable changes over time. // We have to use lambdas to ensure the request handler points to the current state. - connection.handler(RegisterRequest.class, (Function>) request -> state.register(request)); - connection.handler(ConnectRequest.class, (Function>) request -> state.connect(request, connection)); - connection.handler(KeepAliveRequest.class, (Function>) request -> state.keepAlive(request)); - connection.handler(UnregisterRequest.class, (Function>) request -> state.unregister(request)); - connection.handler(ResetRequest.class, (Consumer) request -> state.reset(request)); - connection.handler(ConfigureRequest.class, (Function>) request -> state.configure(request)); - connection.handler(InstallRequest.class, (Function>) request -> state.install(request)); - connection.handler(JoinRequest.class, (Function>) request -> state.join(request)); - connection.handler(ReconfigureRequest.class, (Function>) request -> state.reconfigure(request)); - connection.handler(LeaveRequest.class, (Function>) request -> state.leave(request)); - connection.handler(AppendRequest.class, (Function>) request -> state.append(request)); - connection.handler(PollRequest.class, (Function>) request -> state.poll(request)); - connection.handler(VoteRequest.class, (Function>) request -> state.vote(request)); - connection.handler(CommandRequest.class, (Function>) request -> state.command(request)); - connection.handler(QueryRequest.class, (Function>) request -> state.query(request)); - - connection.onClose(stateMachine.executor().context().sessions()::unregisterConnection); + connection.registerHandler(RegisterRequest.NAME, (Function>) request -> state.register(request)); + connection.registerHandler(ConnectRequest.NAME, (Function>) request -> state.connect(request, connection)); + connection.registerHandler(KeepAliveRequest.NAME, (Function>) request -> state.keepAlive(request)); + connection.registerHandler(UnregisterRequest.NAME, (Function>) request -> state.unregister(request)); + connection.registerHandler(OpenSessionRequest.NAME, (Function>) request -> state.openSession(request)); + connection.registerHandler(CloseSessionRequest.NAME, (Function>) request -> state.closeSession(request)); + connection.registerHandler(ResetRequest.NAME, (Consumer) request -> state.reset(request)); + connection.registerHandler(ConfigureRequest.NAME, (Function>) request -> state.configure(request)); + connection.registerHandler(InstallRequest.NAME, (Function>) request -> state.install(request)); + connection.registerHandler(JoinRequest.NAME, (Function>) request -> state.join(request)); + connection.registerHandler(ReconfigureRequest.NAME, (Function>) request -> state.reconfigure(request)); + connection.registerHandler(LeaveRequest.NAME, (Function>) request -> state.leave(request)); + connection.registerHandler(AppendRequest.NAME, (Function>) request -> state.append(request)); + connection.registerHandler(PollRequest.NAME, (Function>) request -> state.poll(request)); + connection.registerHandler(VoteRequest.NAME, (Function>) request -> state.vote(request)); + connection.registerHandler(CommandRequest.NAME, (Function>) request -> state.command(request)); + connection.registerHandler(QueryRequest.NAME, (Function>) request -> state.query(request)); + + connection.onClose(stateMachine.getSessions()::unregisterConnection); } /** diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerMember.java b/server/src/main/java/io/atomix/copycat/server/state/ServerMember.java index 6f353f8c..921b7a85 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerMember.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerMember.java @@ -211,7 +211,7 @@ ServerMember update(Address clientAddress, Instant time) { */ CompletableFuture configure(Member.Type type) { CompletableFuture future = new CompletableFuture<>(); - cluster.getContext().getThreadContext().executor().execute(() -> configure(type, future)); + cluster.getContext().getThreadContext().execute(() -> configure(type, future)); return future; } diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java b/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java index 98ea8f4e..ed01caa5 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java @@ -38,15 +38,13 @@ class ServerSessionContext implements ServerSession { private static final Logger LOGGER = LoggerFactory.getLogger(ServerSessionContext.class); private final long id; - private final String client; + private final long client; private final Log log; - private final ServerStateMachineContext context; - private boolean open; + private final ServerStateMachineExecutor executor; private volatile State state = State.OPEN; - private final long timeout; private Connection connection; + private final String messageType; private volatile long references; - private long keepAliveIndex; private long requestSequence; private long commandSequence; private long lastApplied; @@ -54,24 +52,23 @@ class ServerSessionContext implements ServerSession { private long eventIndex; private long completeIndex; private long closeIndex; - private long timestamp; private final Map> sequenceQueries = new HashMap<>(); private final Map> indexQueries = new HashMap<>(); - private final Map results = new HashMap<>(); + private final Map results = new HashMap<>(); private final Queue events = new LinkedList<>(); private EventHolder event; private boolean unregistering; private final Listeners changeListeners = new Listeners<>(); - ServerSessionContext(long id, String client, Log log, ServerStateMachineContext context, long timeout) { + ServerSessionContext(long id, long client, Log log, ServerStateMachineExecutor executor) { this.id = id; - this.client = Assert.notNull(client, "client"); + this.client = client; this.log = Assert.notNull(log, "log"); this.eventIndex = id; this.completeIndex = id; - this.lastApplied = id - 1; - this.context = context; - this.timeout = timeout; + this.lastApplied = id; + this.executor = executor; + this.messageType = String.valueOf(id); } @Override @@ -84,15 +81,17 @@ public long id() { * * @return The session client ID. */ - public String client() { + public long client() { return client; } /** - * Opens the session. + * Returns the state machine executor associated with the session. + * + * @return The state machine executor associated with the session. */ - void open() { - open = true; + ServerStateMachineExecutor getStateMachineExecutor() { + return executor; } @Override @@ -118,89 +117,6 @@ public Listener onStateChange(Consumer callback) { return changeListeners.add(callback); } - /** - * Acquires a reference to the session. - */ - void acquire() { - references++; - } - - /** - * Releases a reference to the session. - */ - void release() { - long references = --this.references; - if (!state.active() && references == 0) { - context.sessions().unregisterSession(id); - log.release(id); - if (closeIndex > 0) { - log.release(closeIndex); - } - } - } - - /** - * Returns the number of open command references for the session. - * - * @return The number of open command references for the session. - */ - long references() { - return references; - } - - /** - * Returns the session timeout. - * - * @return The session timeout. - */ - long timeout() { - return timeout; - } - - /** - * Returns the session timestamp. - * - * @return The session timestamp. - */ - long getTimestamp() { - return timestamp; - } - - /** - * Sets the session timestamp. - * - * @param timestamp The session timestamp. - * @return The server session. - */ - ServerSessionContext setTimestamp(long timestamp) { - this.timestamp = Math.max(this.timestamp, timestamp); - return this; - } - - /** - * Returns the current session keep alive index. - * - * @return The current session keep alive index. - */ - long getKeepAliveIndex() { - return keepAliveIndex; - } - - /** - * Sets the current session keep alive index. - * - * @param keepAliveIndex The current session keep alive index. - * @return The server session. - */ - ServerSessionContext setKeepAliveIndex(long keepAliveIndex) { - long previousKeepAliveIndex = this.keepAliveIndex; - this.keepAliveIndex = keepAliveIndex; - if (previousKeepAliveIndex > 0) { - log.release(previousKeepAliveIndex); - } - return this; - } - /** * Returns the session request number. * @@ -351,7 +267,7 @@ ServerSessionContext registerIndexQuery(long index, Runnable query) { * @param result The result. * @return The server session. */ - ServerSessionContext registerResult(long sequence, ServerStateMachine.Result result) { + ServerSessionContext registerResult(long sequence, OperationResult result) { results.put(sequence, result); return this; } @@ -382,7 +298,7 @@ ServerSessionContext clearResults(long sequence) { * @param sequence The response sequence. * @return The response. */ - ServerStateMachine.Result getResult(long sequence) { + OperationResult getResult(long sequence) { return results.get(sequence); } @@ -419,20 +335,22 @@ public Session publish(String event) { @Override public Session publish(String event, Object message) { - Assert.state(open, "cannot publish events during session registration"); + // Store volatile state in a local variable. + State state = this.state; Assert.stateNot(state == State.CLOSED, "session is closed"); Assert.stateNot(state == State.EXPIRED, "session is expired"); - Assert.state(context.type() == ServerStateMachineContext.Type.COMMAND, "session events can only be published during command execution"); + Assert.state(executor.context().type() == ServerStateMachineContext.Type.COMMAND, "session events can only be published during command execution"); // If the client acked an index greater than the current event sequence number since we know the // client must have received it from another server. - if (completeIndex > context.index()) + if (completeIndex > executor.context().index()) { return this; + } // If no event has been published for this index yet, create a new event holder. - if (this.event == null || this.event.eventIndex != context.index()) { + if (this.event == null || this.event.eventIndex != executor.context().index()) { long previousIndex = eventIndex; - eventIndex = context.index(); + eventIndex = executor.context().index(); this.event = new EventHolder(eventIndex, previousIndex); } @@ -450,6 +368,7 @@ void commit(long index) { events.add(event); sendEvent(event); } + setLastApplied(index); } /** @@ -521,77 +440,7 @@ private void sendEvent(EventHolder event, Connection connection) { .build(); LOGGER.trace("{} - Sending {}", id, request); - connection.send(request); - } - - /** - * Sets the session as suspect. - */ - void suspect() { - setState(State.UNSTABLE); - } - - /** - * Sets the session as trusted. - */ - void trust() { - setState(State.OPEN); - } - - /** - * Sets the session as being unregistered. - */ - void unregister() { - unregistering = true; - } - - /** - * Indicates whether the session is being unregistered. - */ - boolean isUnregistering() { - return unregistering; - } - - /** - * Expires the session. - * - * @param index The index at which to expire the session. - */ - void expire(long index) { - setState(State.EXPIRED); - cleanState(index); - } - - /** - * Closes the session. - * - * @param index The index at which to close the session. - */ - void close(long index) { - setState(State.CLOSED); - cleanState(index); - } - - /** - * Cleans session entries on close. - */ - private void cleanState(long index) { - // If the keep alive index is set, release the entry. - if (keepAliveIndex > 0) { - log.release(keepAliveIndex); - } - - context.sessions().unregisterSession(id); - - // If no references to session commands are open, release session-related entries. - if (references == 0) { - log.release(id); - if (index > 0) { - log.release(index); - } - } else { - this.closeIndex = index; - } + connection.send(messageType, request); } @Override diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerSessionManager.java b/server/src/main/java/io/atomix/copycat/server/state/ServerSessionManager.java index b68d2063..83576018 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerSessionManager.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerSessionManager.java @@ -16,15 +16,10 @@ package io.atomix.copycat.server.state; import io.atomix.catalyst.transport.Connection; -import io.atomix.catalyst.util.Assert; -import io.atomix.copycat.server.session.ServerSession; -import io.atomix.copycat.server.session.SessionListener; -import io.atomix.copycat.server.session.Sessions; -import java.util.HashSet; +import java.util.Collection; import java.util.Iterator; import java.util.Map; -import java.util.Set; import java.util.concurrent.ConcurrentHashMap; /** @@ -32,88 +27,68 @@ * * @author Jordan Halterman */ -class ServerSessionManager implements Sessions { - private final Map connections = new ConcurrentHashMap<>(); - final Map sessions = new ConcurrentHashMap<>(); - final Map clients = new ConcurrentHashMap<>(); - final Set listeners = new HashSet<>(); - private final ServerContext context; +class ServerSessionManager { + private final Map sessions = new ConcurrentHashMap<>(); + private final Map connections = new ConcurrentHashMap<>(); - public ServerSessionManager(ServerContext context) { - this.context = Assert.notNull(context, "context"); - } - - @Override - public ServerSession session(long sessionId) { - return sessions.get(sessionId); - } - - @Override - public Sessions addListener(SessionListener listener) { - listeners.add(Assert.notNull(listener, "listener")); - return this; - } - - @Override - public Sessions removeListener(SessionListener listener) { - listeners.remove(Assert.notNull(listener, "listener")); - return this; + /** + * Registers a connection. + */ + void registerConnection(long sessionId, long connectionId, Connection connection) { + ServerSessionContext session = sessions.get(sessionId); + TimestampedConnection existingConnection = connections.get(sessionId); + if (existingConnection == null || existingConnection.id < connectionId) { + connections.put(sessionId, new TimestampedConnection(connectionId, connection)); + if (session != null) { + session.setConnection(connection); + } + } } /** * Registers a connection. */ - ServerSessionManager registerConnection(String client, Connection connection) { - ServerSessionContext session = clients.get(client); - if (session != null) { - session.setConnection(connection); + void registerConnection(long sessionId, long connectionId) { + TimestampedConnection connection = connections.get(sessionId); + if (connection != null && connection.id < connectionId) { + connections.remove(sessionId, connection); } - connections.put(client, connection); - return this; } /** * Unregisters a connection. */ - ServerSessionManager unregisterConnection(Connection connection) { - Iterator> iterator = connections.entrySet().iterator(); + void unregisterConnection(Connection connection) { + Iterator> iterator = connections.entrySet().iterator(); while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - if (entry.getValue().equals(connection)) { - ServerSessionContext session = clients.get(entry.getKey()); + Map.Entry entry = iterator.next(); + if (entry.getValue().connection.equals(connection)) { + ServerSessionContext session = sessions.get(entry.getKey()); if (session != null) { session.setConnection(null); } iterator.remove(); } } - return this; } /** * Registers a session. */ - ServerSessionContext registerSession(ServerSessionContext session) { - ServerSessionContext oldSession = clients.remove(session.client()); - if (oldSession != null) { - sessions.remove(oldSession.id()); - } - session.setConnection(connections.get(session.client())); + void registerSession(ServerSessionContext session) { sessions.put(session.id(), session); - clients.put(session.client(), session); - return oldSession; + TimestampedConnection connection = connections.get(session.id()); + if (connection != null) { + session.setConnection(connection.connection); + } } /** * Unregisters a session. */ - ServerSessionContext unregisterSession(long sessionId) { - ServerSessionContext session = sessions.remove(sessionId); - if (session != null) { - clients.remove(session.client(), session); - connections.remove(session.client(), session.getConnection()); - } - return session; + void unregisterSession(long sessionId) { + sessions.remove(sessionId); + connections.remove(sessionId); } /** @@ -127,19 +102,25 @@ ServerSessionContext getSession(long sessionId) { } /** - * Gets a session by client ID. + * Returns the collection of registered sessions. * - * @param clientId The client ID. - * @return The session or {@code null} if the session doesn't exist. + * @return The collection of registered sessions. */ - ServerSessionContext getSession(String clientId) { - return clients.get(clientId); + Collection getSessions() { + return sessions.values(); } - @Override - @SuppressWarnings("unchecked") - public Iterator iterator() { - return (Iterator) sessions.values().iterator(); + /** + * Connection ID/connection holder. + */ + private static class TimestampedConnection { + private final long id; + private final Connection connection; + + TimestampedConnection(long id, Connection connection) { + this.id = id; + this.connection = connection; + } } } diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerState.java b/server/src/main/java/io/atomix/copycat/server/state/ServerState.java index f4901307..573c03cd 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerState.java @@ -69,6 +69,22 @@ public interface ServerState extends Managed { */ CompletableFuture unregister(UnregisterRequest request); + /** + * Handles an open session request. + * + * @param request The request to handle. + * @return A completable future to be completed with the request response. + */ + CompletableFuture openSession(OpenSessionRequest request); + + /** + * Handles a close session request. + * + * @param request The request to handle. + * @return A completable future to be completed with the request response. + */ + CompletableFuture closeSession(CloseSessionRequest request); + /** * Handles a reset request. * diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachine.java b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachine.java deleted file mode 100644 index 980574fd..00000000 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachine.java +++ /dev/null @@ -1,1004 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.atomix.copycat.server.state; - -import io.atomix.catalyst.concurrent.ComposableFuture; -import io.atomix.catalyst.concurrent.Futures; -import io.atomix.catalyst.concurrent.ThreadContext; -import io.atomix.catalyst.util.Assert; -import io.atomix.copycat.error.InternalException; -import io.atomix.copycat.error.UnknownSessionException; -import io.atomix.copycat.server.Snapshottable; -import io.atomix.copycat.server.StateMachine; -import io.atomix.copycat.server.session.SessionListener; -import io.atomix.copycat.server.storage.Log; -import io.atomix.copycat.server.storage.entry.*; -import io.atomix.copycat.server.storage.snapshot.Snapshot; -import io.atomix.copycat.server.storage.snapshot.SnapshotReader; -import io.atomix.copycat.server.storage.snapshot.SnapshotWriter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.time.Instant; -import java.util.concurrent.CompletableFuture; - -/** - * Internal server state machine. - *

- * The internal state machine handles application of commands to the user provided {@link StateMachine} - * and keeps track of internal state like sessions and the various indexes relevant to log compaction. - * - * @author - * Snapshots of the state machine are taken only once the log becomes compactable. This means snapshots - * are largely dependent on the storage configuration and ensures that snapshots are not taken more - * frequently than will benefit log compaction. - */ - private void takeSnapshot() { - state.checkThread(); - - // If no snapshot has been taken, take a snapshot and hold it in memory until the complete - // index has met the snapshot index. Note that this will be executed in the state machine thread. - // Snapshots are only taken of the state machine when the log becomes compactable. If the log compactor's - // compactIndex is greater than the last snapshot index and the lastApplied index is greater than the - // last snapshot index, take the snapshot. - Snapshot currentSnapshot = state.getSnapshotStore().currentSnapshot(); - if (pendingSnapshot == null && stateMachine instanceof Snapshottable - && (currentSnapshot == null || (log.compactor().compactIndex() > currentSnapshot.index() && lastApplied > currentSnapshot.index()))) { - pendingSnapshot = state.getSnapshotStore().createSnapshot(lastApplied); - - // Write the snapshot data. Note that we don't complete the snapshot here since the completion - // of a snapshot is predicated on session events being received by clients up to the snapshot index. - LOGGER.info("{} - Taking snapshot {}", state.getCluster().member().address(), pendingSnapshot.index()); - executor.executor().execute(() -> { - synchronized (pendingSnapshot) { - try (SnapshotWriter writer = pendingSnapshot.writer()) { - ((Snapshottable) stateMachine).snapshot(writer); - } - } - }); - } - } - - /** - * Installs a snapshot of the state machine state if necessary. - *

- * Snapshots are installed only if there's a local snapshot stored with a version equal to the - * last applied index. - */ - private void installSnapshot() { - state.checkThread(); - - // If the last stored snapshot has not yet been installed and its index matches the last applied state - // machine index, install the snapshot. This requires that the state machine see all indexes sequentially - // even for entries that have been compacted from the log. - Snapshot currentSnapshot = state.getSnapshotStore().currentSnapshot(); - if (currentSnapshot != null && currentSnapshot.index() > log.compactor().snapshotIndex() && currentSnapshot.index() == lastApplied && stateMachine instanceof Snapshottable) { - - // Install the snapshot in the state machine thread. Multiple threads can access snapshots, so we - // synchronize on the snapshot object. In practice, this probably isn't even necessary and could prove - // to be an expensive operation. Snapshots can be read concurrently with separate SnapshotReaders since - // memory snapshots are copied to the reader and file snapshots open a separate FileBuffer for each reader. - LOGGER.info("{} - Installing snapshot {}", state.getCluster().member().address(), currentSnapshot.index()); - executor.executor().execute(() -> { - synchronized (currentSnapshot) { - try (SnapshotReader reader = currentSnapshot.reader()) { - ((Snapshottable) stateMachine).install(reader); - } - } - }); - - // Once a snapshot has been applied, snapshot dependent entries can be cleaned from the log. - log.compactor().snapshotIndex(currentSnapshot.index()); - } - } - - /** - * Completes a snapshot of the state machine state. - *

- * When a snapshot of the state machine is taken, the snapshot isn't immediately made available for - * recovery or replication. Session events are dependent on original commands being retained in the log - * for fault tolerance. Thus, a snapshot cannot be completed until all prior events have been received - * by clients. So, we take a snapshot of the state machine state and complete the snapshot only after - * prior events have been received. - */ - private void completeSnapshot() { - state.checkThread(); - - // If a snapshot is pending to be persisted and the last completed index is greater than the - // waiting snapshot index and no current or newer snapshot exists, - // persist the snapshot and update the last snapshot index. - if (pendingSnapshot != null && lastCompleted > pendingSnapshot.index()) { - long snapshotIndex = pendingSnapshot.index(); - LOGGER.debug("{} - Completing snapshot {}", state.getCluster().member().address(), snapshotIndex); - synchronized (pendingSnapshot) { - Snapshot currentSnapshot = state.getSnapshotStore().currentSnapshot(); - if (currentSnapshot == null || snapshotIndex > currentSnapshot.index()) { - pendingSnapshot.complete(); - } else { - LOGGER.debug("{} - Discarding pending snapshot at index {} since the current snapshot is at index {}", state.getCluster().member().address(), pendingSnapshot.index(), currentSnapshot.index()); - } - pendingSnapshot = null; - } - - // Once the snapshot has been completed, snapshot dependent entries can be cleaned from the log. - log.compactor().snapshotIndex(snapshotIndex); - log.compactor().compact(); - } - } - - /** - * Returns the server state machine executor. - * - * @return The server state machine executor. - */ - ServerStateMachineExecutor executor() { - return executor; - } - - /** - * Returns the last applied index. - * - * @return The last applied index. - */ - long getLastApplied() { - return lastApplied; - } - - /** - * Sets the last applied index. - *

- * The last applied index is updated *after* each time a non-query entry is applied to the state machine. - * - * @param lastApplied The last applied index. - */ - private void setLastApplied(long lastApplied) { - // lastApplied should be either equal to or one greater than this.lastApplied. - if (lastApplied > this.lastApplied) { - Assert.arg(lastApplied == this.lastApplied + 1, "lastApplied must be sequential"); - - this.lastApplied = lastApplied; - - // Update the index for each session. This will be used to trigger queries that are awaiting the - // application of specific indexes to the state machine. Setting the session index may cause query - // callbacks to be called and queries to be evaluated. - for (ServerSessionContext session : executor.context().sessions().sessions.values()) { - session.setLastApplied(lastApplied); - } - - // Take a state machine snapshot if necessary. - takeSnapshot(); - - // Install a state machine snapshot if necessary. - installSnapshot(); - } else { - Assert.arg(lastApplied == this.lastApplied, "lastApplied cannot be decreased"); - } - } - - /** - * Returns the highest index completed for all sessions. - *

- * The lastCompleted index is representative of the highest index for which related events have been - * received by *all* clients. In other words, no events lower than the given index should remain in - * memory. - * - * @return The highest index completed for all sessions. - */ - long getLastCompleted() { - return lastCompleted > 0 ? lastCompleted : lastApplied; - } - - /** - * Calculates the last completed session event index. - */ - private long calculateLastCompleted(long index) { - // Calculate the last completed index as the lowest index acknowledged by all clients. - long lastCompleted = index; - for (ServerSessionContext session : executor.context().sessions().sessions.values()) { - lastCompleted = Math.min(lastCompleted, session.getLastCompleted()); - } - return lastCompleted; - } - - /** - * Updates the last completed event index based on a commit at the given index. - */ - private void setLastCompleted(long lastCompleted) { - if (!log.isOpen()) - return; - - this.lastCompleted = Math.max(this.lastCompleted, lastCompleted); - - // Update the log compaction minor index. - log.compactor().minorIndex(this.lastCompleted); - - completeSnapshot(); - } - - /** - * Applies all commits up to the given index. - *

- * Calls to this method are assumed not to expect a result. This allows some optimizations to be - * made internally since linearizable events don't have to be waited to complete the command. - * - * @param index The index up to which to apply commits. - */ - public void applyAll(long index) { - if (!log.isOpen()) - return; - - // If the effective commit index is greater than the last index applied to the state machine then apply remaining entries. - long lastIndex = Math.min(index, log.lastIndex()); - if (lastIndex > lastApplied) { - for (long i = lastApplied + 1; i <= lastIndex; i++) { - Entry entry = log.get(i); - if (entry != null) { - apply(entry).whenComplete((result, error) -> entry.release()); - } - setLastApplied(i); - } - } - } - - /** - * Applies the entry at the given index to the state machine. - *

- * Calls to this method are assumed to expect a result. This means linearizable session events - * triggered by the application of the command at the given index will be awaited before completing - * the returned future. - * - * @param index The index to apply. - * @return A completable future to be completed once the commit has been applied. - */ - public CompletableFuture apply(long index) { - try { - // If entries remain to be applied prior to this entry then synchronously apply them. - if (index > lastApplied + 1) { - applyAll(index - 1); - } - - // Read the entry from the log. If the entry is non-null them apply the entry, otherwise - // simply update the last applied index and return a null result. - try (Entry entry = log.get(index)) { - if (entry != null) { - return apply(entry); - } else { - return CompletableFuture.completedFuture(null); - } - } - } catch (Exception e) { - e.printStackTrace(); - return Futures.exceptionalFuture(e); - } finally { - setLastApplied(index); - } - } - - /** - * Applies an entry to the state machine. - *

- * Calls to this method are assumed to expect a result. This means linearizable session events - * triggered by the application of the given entry will be awaited before completing the returned future. - * - * @param entry The entry to apply. - * @return A completable future to be completed with the result. - */ - @SuppressWarnings("unchecked") - public CompletableFuture apply(Entry entry) { - LOGGER.trace("{} - Applying {}", state.getCluster().member().address(), entry); - if (entry instanceof QueryEntry) { - return (CompletableFuture) apply((QueryEntry) entry); - } else if (entry instanceof CommandEntry) { - return (CompletableFuture) apply((CommandEntry) entry); - } else if (entry instanceof RegisterEntry) { - return (CompletableFuture) apply((RegisterEntry) entry); - } else if (entry instanceof KeepAliveEntry) { - return (CompletableFuture) apply((KeepAliveEntry) entry); - } else if (entry instanceof UnregisterEntry) { - return (CompletableFuture) apply((UnregisterEntry) entry); - } else if (entry instanceof InitializeEntry) { - return (CompletableFuture) apply((InitializeEntry) entry); - } else if (entry instanceof ConfigurationEntry) { - return (CompletableFuture) apply((ConfigurationEntry) entry); - } - return Futures.exceptionalFuture(new InternalException("unknown state machine operation")); - } - - /** - * Applies a configuration entry to the internal state machine. - *

- * Configuration entries are applied to internal server state when written to the log. Thus, no significant - * logic needs to take place in the handling of configuration entries. We simply release the previous configuration - * entry since it was overwritten by a more recent committed configuration entry. - */ - private CompletableFuture apply(ConfigurationEntry entry) { - // Clean the configuration entry from the log. The entry will be retained until it has been stored - // on all servers. - log.release(entry.getIndex()); - return CompletableFuture.completedFuture(null); - } - - /** - * Applies register session entry to the state machine. - *

- * Register entries are applied to the state machine to create a new session. The resulting session ID is the - * index of the RegisterEntry. Once a new session is registered, we call register() on the state machine. - * In the event that the {@code synchronous} flag is set, that indicates that the registration command expects a - * response, i.e. it was applied by a leader. In that case, any events published during the execution of the - * state machine's register() method must be completed synchronously prior to the completion of the returned future. - */ - private CompletableFuture apply(RegisterEntry entry) { - // Allow the executor to execute any scheduled events. - long timestamp = executor.timestamp(entry.getTimestamp()); - - long sessionId = entry.getIndex(); - ServerSessionContext session = new ServerSessionContext(sessionId, entry.getClient(), log, executor.context(), entry.getTimeout()); - - ServerSessionContext oldSession = executor.context().sessions().registerSession(session); - - // Update the session timestamp *after* executing any scheduled operations. The executor's timestamp - // is guaranteed to be monotonically increasing, whereas the RegisterEntry may have an earlier timestamp - // if, e.g., it was written shortly after a leader change. - session.setTimestamp(timestamp); - - // Determine whether any sessions appear to be expired. This won't immediately expire the session(s), - // but it will make them available to be unregistered by the leader. - suspectSessions(0, timestamp); - - ThreadContext context = ThreadContext.currentContextOrThrow(); - long index = entry.getIndex(); - - // Call the register() method on the user-provided state machine to allow the state machine to react to - // a new session being registered. User state machine methods are always called in the state machine thread. - CompletableFuture future = new ComposableFuture<>(); - executor.executor().execute(() -> registerSession(index, timestamp, session, oldSession, future, context)); - return future; - } - - /** - * Registers a session. - */ - private void registerSession(long index, long timestamp, ServerSessionContext session, ServerSessionContext oldSession, CompletableFuture future, ThreadContext context) { - if (!log.isOpen()) { - context.executor().execute(() -> future.completeExceptionally(new IllegalStateException("log closed"))); - return; - } - - // Trigger scheduled callbacks in the state machine. - executor.tick(index, timestamp); - - // Update the state machine context with the register entry's index. This ensures that events published - // within the register method will be properly associated with the unregister entry's index. All events - // published during registration of a session are linearizable to ensure that clients receive related events - // before the registration is completed. - executor.init(index, Instant.ofEpochMilli(timestamp), ServerStateMachineContext.Type.COMMAND); - - // If the session overrides a previous session, expire the old session. - if (oldSession != null) { - oldSession.expire(0); - } - - // Register the session and then open it. This ensures that state machines cannot publish events to this - // session before the client has learned of the session ID. - for (SessionListener listener : executor.context().sessions().listeners) { - if (oldSession != null) { - listener.expire(oldSession); - listener.close(oldSession); - } - listener.register(session); - } - session.open(); - - // Calculate the last completed index. - long lastCompleted = calculateLastCompleted(index); - - // Once register callbacks have been completed, ensure that events published during the callbacks are - // received by clients. The state machine context will generate an event future for all published events - // to all sessions. - executor.commit(); - context.executor().execute(() -> { - setLastCompleted(lastCompleted); - future.complete(index); - }); - } - - /** - * Applies a session keep alive entry to the state machine. - *

- * Keep alive entries are applied to the internal state machine to reset the timeout for a specific session. - * If the session indicated by the KeepAliveEntry is still held in memory, we mark the session as trusted, - * indicating that the client has committed a keep alive within the required timeout. Additionally, we check - * all other sessions for expiration based on the timestamp provided by this KeepAliveEntry. Note that sessions - * are never completely expired via this method. Leaders must explicitly commit an UnregisterEntry to expire - * a session. - *

- * When a KeepAliveEntry is committed to the internal state machine, two specific fields provided in the entry - * are used to update server-side session state. The {@code commandSequence} indicates the highest command for - * which the session has received a successful response in the proper sequence. By applying the {@code commandSequence} - * to the server session, we clear command output held in memory up to that point. The {@code eventVersion} indicates - * the index up to which the client has received event messages in sequence for the session. Applying the - * {@code eventVersion} to the server-side session results in events up to that index being removed from memory - * as they were acknowledged by the client. It's essential that both of these fields be applied via entries committed - * to the Raft log to ensure they're applied on all servers in sequential order. - *

- * Keep alive entries are retained in the log until the next time the client sends a keep alive entry or until the - * client's session is expired. This ensures for sessions that have long timeouts, keep alive entries cannot be cleaned - * from the log before they're replicated to some servers. - */ - private CompletableFuture apply(KeepAliveEntry entry) { - ServerSessionContext session = executor.context().sessions().getSession(entry.getSession()); - - // Update the deterministic executor time and allow the executor to execute any scheduled events. - long timestamp = executor.timestamp(entry.getTimestamp()); - - // Determine whether any sessions appear to be expired. This won't immediately expire the session(s), - // but it will make them available to be unregistered by the leader. Note that it's safe to trigger - // scheduled executor callbacks even if the keep-alive entry is for an unknown session since the - // leader still committed the entry with its time and so time will still progress deterministically. - suspectSessions(entry.getSession(), timestamp); - - CompletableFuture future; - - // If the server session is null, the session either never existed or already expired. - if (session == null) { - log.release(entry.getIndex()); - future = Futures.exceptionalFuture(new UnknownSessionException("unknown session: " + entry.getSession())); - } - // If the session is in an inactive state, return an UnknownSessionException. - else if (!session.state().active()) { - log.release(entry.getIndex()); - future = Futures.exceptionalFuture(new UnknownSessionException("inactive session: " + entry.getSession())); - } - // If the session exists, don't allow it to expire even if its expiration has passed since we still - // managed to receive a keep alive request from the client before it was removed. This allows the - // client some arbitrary leeway in keeping its session alive. It's up to the leader to explicitly - // expire a session by committing an UnregisterEntry in order to ensure sessions can't be expired - // during leadership changes. - else { - ThreadContext context = ThreadContext.currentContextOrThrow(); - - long index = entry.getIndex(); - - // Set the session as trusted. This will prevent the leader from explicitly unregistering the - // session if it hasn't done so already. - session.trust(); - - // Update the session's timestamp with the current state machine time. - session.setTimestamp(timestamp); - - // Store the command/event sequence and event index instead of acquiring a reference to the entry. - long commandSequence = entry.getCommandSequence(); - long eventIndex = entry.getEventIndex(); - - future = new CompletableFuture<>(); - - // The keep-alive entry also serves to clear cached command responses and events from memory. - // Remove responses and clear/resend events in the state machine thread to prevent thread safety issues. - executor.executor().execute(() -> keepAliveSession(index, timestamp, commandSequence, eventIndex, session, future, context)); - - // Update the session keep alive index for log cleaning. - session.setKeepAliveIndex(entry.getIndex()); - - // Update the session's request sequence number. The command sequence number will be applied - // iff the existing request sequence number is less than the command sequence number. This must - // be applied to ensure that request sequence numbers are reset after a leader change since leaders - // track request sequence numbers in local memory. - session.resetRequestSequence(commandSequence); - - // Update the sessions' command sequence number. The command sequence number will be applied - // iff the existing sequence number is less than the keep-alive command sequence number. This should - // not be the case under normal operation since the command sequence number in keep-alive requests - // represents the highest sequence for which a client has received a response (the command has already - // been completed), but since the log compaction algorithm can exclude individual entries from replication, - // the command sequence number must be applied for keep-alive requests to reset the sequence number in - // the event the last command for the session was cleaned/compacted from the log. - session.setCommandSequence(commandSequence); - } - - return future; - } - - /** - * Applies a keep alive for a session. - */ - private void keepAliveSession(long index, long timestamp, long commandSequence, long eventIndex, ServerSessionContext session, CompletableFuture future, ThreadContext context) { - if (!log.isOpen()) { - context.executor().execute(() -> future.completeExceptionally(new IllegalStateException("log closed"))); - return; - } - - // If the session is already in an inactive state, complete the future exceptionally. - if (!session.state().active()) { - context.executor().execute(() -> future.completeExceptionally(new UnknownSessionException("inactive session: " + session.id()))); - return; - } - - // Trigger scheduled callbacks in the state machine. - executor.tick(index, timestamp); - - // Update the state machine context with the keep-alive entry's index. This ensures that events published - // as a result of asynchronous callbacks will be executed at the proper index with SEQUENTIAL consistency. - executor.init(index, Instant.ofEpochMilli(timestamp), ServerStateMachineContext.Type.COMMAND); - - session.clearResults(commandSequence).resendEvents(eventIndex); - - // Calculate the last completed index. - long lastCompleted = calculateLastCompleted(index); - - // Callbacks in the state machine may have been triggered by the execution of the keep-alive. - // Get any futures for scheduled tasks and await their completion, then update the highest - // index completed for all sessions to allow log compaction to progress. - executor.commit(); - context.executor().execute(() -> { - setLastCompleted(lastCompleted); - future.complete(null); - }); - } - - /** - * Applies an unregister session entry to the state machine. - *

- * Unregister entries may either be committed by clients or by the cluster's leader. Clients will commit - * an unregister entry when closing their session normally. Leaders will commit an unregister entry when - * an expired session is detected. This ensures that sessions are never expired purely on gaps in the log - * which may result from normal log cleaning or lengthy leadership changes. - *

- * If the session was unregistered by the client, the isExpired flag will be false. Sessions expired by - * the client are only close()ed on the state machine but not expire()d. Alternatively, entries where - * isExpired is true were committed by a leader. For expired sessions, the state machine's expire() method - * is called before close(). - *

- * State machines may publish events during the handling of session expired or closed events. If the - * {@code synchronous} flag passed to this method is true, events published during the commitment of the - * UnregisterEntry must be synchronously completed prior to the completion of the returned future. This - * ensures that state changes resulting from the expiration or closing of a session are completed before - * the session close itself is completed. - */ - private CompletableFuture apply(UnregisterEntry entry) { - // Get the session from the context sessions. Note that we do not unregister the session here. Sessions - // can only be unregistered once all references to session commands have been released by the state machine. - ServerSessionContext session = executor.context().sessions().getSession(entry.getSession()); - - // Update the deterministic executor time and allow the executor to execute any scheduled events. - long timestamp = executor.timestamp(entry.getTimestamp()); - - // Determine whether any sessions appear to be expired. This won't immediately expire the session(s), - // but it will make them available to be unregistered by the leader. Note that it's safe to trigger - // scheduled executor callbacks even if the keep-alive entry is for an unknown session since the - // leader still committed the entry with its time and so time will still progress deterministically. - suspectSessions(entry.getSession(), timestamp); - - CompletableFuture future; - - // If the server session is null, the session either never existed or already expired. - if (session == null) { - log.release(entry.getIndex()); - future = Futures.exceptionalFuture(new UnknownSessionException("unknown session: " + entry.getSession())); - } - // If the session is not in an active state, return an UnknownSessionException. - else if (!session.state().active()) { - log.release(entry.getIndex()); - future = Futures.exceptionalFuture(new UnknownSessionException("inactive session: " + entry.getSession())); - } - // If the session exists, don't allow it to expire even if its expiration has passed since we still - // managed to receive a keep alive request from the client before it was removed. - else { - ThreadContext context = ThreadContext.currentContextOrThrow(); - future = new CompletableFuture<>(); - - long index = entry.getIndex(); - - // If the entry was marked expired, that indicates that the leader explicitly expired the session due to - // the session not being kept alive by the client. In all other cases, we close the session normally. - if (entry.isExpired()) { - executor.executor().execute(() -> expireSession(index, timestamp, session, future, context)); - } - // If the unregister entry is not indicated as expired, a client must have submitted a request to unregister - // the session. In that case, we simply close the session without expiring it. - else { - executor.executor().execute(() -> closeSession(index, timestamp, session, future, context)); - } - } - - return future; - } - - /** - * Expires the given session. - */ - private void expireSession(long index, long timestamp, ServerSessionContext session, CompletableFuture future, ThreadContext context) { - if (!log.isOpen()) { - context.executor().execute(() -> future.completeExceptionally(new IllegalStateException("log closed"))); - return; - } - - // If the session is already in an inactive state, complete the future exceptionally. - if (!session.state().active()) { - context.executor().execute(() -> future.completeExceptionally(new UnknownSessionException("inactive session: " + session.id()))); - return; - } - - // Trigger scheduled callbacks in the state machine. - executor.tick(index, timestamp); - - // Update the state machine context with the unregister entry's index. This ensures that events published - // within the expire or close methods will be properly associated with the unregister entry's index. - // All events published during expiration or closing of a session are linearizable to ensure that clients - // receive related events before the expiration is completed. - executor.init(index, Instant.ofEpochMilli(timestamp), ServerStateMachineContext.Type.COMMAND); - - // Expire the session and call state machine callbacks. - session.expire(index); - for (SessionListener listener : executor.context().sessions().listeners) { - listener.expire(session); - listener.close(session); - } - - // Calculate the last completed index. - long lastCompleted = calculateLastCompleted(index); - - // Once expiration callbacks have been completed, ensure that events published during the callbacks - // are published in batch. The state machine context will generate an event future for all published events - // to all sessions. If the event future is non-null, that indicates events are pending which were published - // during the call to expire(). Wait for the events to be received by the client before completing the future. - executor.commit(); - context.executor().execute(() -> { - setLastCompleted(lastCompleted); - future.complete(null); - }); - } - - /** - * Closes the given session. - */ - private void closeSession(long index, long timestamp, ServerSessionContext session, CompletableFuture future, ThreadContext context) { - if (!log.isOpen()) { - context.executor().execute(() -> future.completeExceptionally(new IllegalStateException("log closed"))); - return; - } - - // If the session is already in an inactive state, complete the future exceptionally. - if (!session.state().active()) { - context.executor().execute(() -> future.completeExceptionally(new UnknownSessionException("inactive session: " + session.id()))); - return; - } - - // Trigger scheduled callbacks in the state machine. - executor.tick(index, timestamp); - - // Update the state machine context with the unregister entry's index. This ensures that events published - // within the close method will be properly associated with the unregister entry's index. All events published - // during expiration or closing of a session are linearizable to ensure that clients receive related events - // before the expiration is completed. - executor.init(index, Instant.ofEpochMilli(timestamp), ServerStateMachineContext.Type.COMMAND); - - // Close the session and call state machine callbacks. - session.close(index); - for (SessionListener listener : executor.context().sessions().listeners) { - listener.unregister(session); - listener.close(session); - } - - // Calculate the last completed index. - long lastCompleted = calculateLastCompleted(index); - - // Once close callbacks have been completed, ensure that events published during the callbacks - // are published in batch. The state machine context will generate an event future for all published events - // to all sessions. If the event future is non-null, that indicates events are pending which were published - // during the call to expire(). Wait for the events to be received by the client before completing the future. - executor.commit(); - context.executor().execute(() -> { - setLastCompleted(lastCompleted); - future.complete(null); - }); - } - - /** - * Applies a command entry to the state machine. - *

- * Command entries result in commands being executed on the user provided {@link StateMachine} and a - * response being sent back to the client by completing the returned future. All command responses are - * cached in the command's {@link ServerSessionContext} for fault tolerance. In the event that the same command - * is applied to the state machine more than once, the original response will be returned. - *

- * Command entries are written with a sequence number. The sequence number is used to ensure that - * commands are applied to the state machine in sequential order. If a command entry has a sequence - * number that is less than the next sequence number for the session, that indicates that it is a - * duplicate of a command that was already applied. Otherwise, commands are assumed to have been - * received in sequential order. The reason for this assumption is because leaders always sequence - * commands as they're written to the log, so no sequence number will be skipped. - */ - private CompletableFuture apply(CommandEntry entry) { - final CompletableFuture future = new CompletableFuture<>(); - final ThreadContext context = ThreadContext.currentContextOrThrow(); - - // First check to ensure that the session exists. - ServerSessionContext session = executor.context().sessions().getSession(entry.getSession()); - - // If the session is null, return an UnknownSessionException. Commands applied to the state machine must - // have a session. We ensure that session register/unregister entries are not compacted from the log - // until all associated commands have been cleaned. - if (session == null) { - log.release(entry.getIndex()); - return Futures.exceptionalFuture(new UnknownSessionException("unknown session: " + entry.getSession())); - } - // If the session is not in an active state, return an UnknownSessionException. Sessions are retained in the - // session registry until all prior commands have been released by the state machine, but new commands can - // only be applied for sessions in an active state. - else if (!session.state().active()) { - log.release(entry.getIndex()); - return Futures.exceptionalFuture(new UnknownSessionException("inactive session: " + entry.getSession())); - } - // If the command's sequence number is less than the next session sequence number then that indicates that - // we've received a command that was previously applied to the state machine. Ensure linearizability by - // returning the cached response instead of applying it to the user defined state machine. - else if (entry.getSequence() > 0 && entry.getSequence() < session.nextCommandSequence()) { - // Ensure the response check is executed in the state machine thread in order to ensure the - // command was applied, otherwise there will be a race condition and concurrent modification issues. - long sequence = entry.getSequence(); - - // Switch to the state machine thread and get the existing response. - executor.executor().execute(() -> sequenceCommand(sequence, session, future, context)); - return future; - } - // If we've made it this far, the command must have been applied in the proper order as sequenced by the - // session. This should be the case for most commands applied to the state machine. - else { - // Allow the executor to execute any scheduled events. - long index = entry.getIndex(); - long sequence = entry.getSequence(); - - // Calculate the updated timestamp for the command. - long timestamp = executor.timestamp(entry.getTimestamp()); - - // Execute the command in the state machine thread. Once complete, the CompletableFuture callback will be completed - // in the state machine thread. Register the result in that thread and then complete the future in the caller's thread. - ServerCommit commit = commits.acquire(entry, session, timestamp); - executor.executor().execute(() -> executeCommand(index, sequence, timestamp, commit, session, future, context)); - - // Update the last applied index prior to the command sequence number. This is necessary to ensure queries sequenced - // at this index receive the index of the command. - setLastApplied(index); - - // Update the session timestamp and command sequence number. This is done in the caller's thread since all - // timestamp/index/sequence checks are done in this thread prior to executing operations on the state machine thread. - session.setTimestamp(timestamp).setCommandSequence(sequence); - return future; - } - } - - /** - * Sequences a command according to the command sequence number. - */ - private void sequenceCommand(long sequence, ServerSessionContext session, CompletableFuture future, ThreadContext context) { - if (!log.isOpen()) { - context.executor().execute(() -> future.completeExceptionally(new IllegalStateException("log closed"))); - return; - } - - Result result = session.getResult(sequence); - if (result == null) { - LOGGER.debug("Missing command result for {}:{}", session.id(), sequence); - } - context.executor().execute(() -> future.complete(result)); - } - - /** - * Executes a state machine command. - */ - private void executeCommand(long index, long sequence, long timestamp, ServerCommit commit, ServerSessionContext session, CompletableFuture future, ThreadContext context) { - if (!log.isOpen()) { - context.executor().execute(() -> future.completeExceptionally(new IllegalStateException("log closed"))); - return; - } - - // If the session is already in an inactive state, complete the future exceptionally. - if (!session.state().active()) { - context.executor().execute(() -> future.completeExceptionally(new UnknownSessionException("inactive session: " + session.id()))); - return; - } - - // Trigger scheduled callbacks in the state machine. - executor.tick(index, timestamp); - - // Update the state machine context with the commit index and local server context. The synchronous flag - // indicates whether the server expects linearizable completion of published events. Events will be published - // based on the configured consistency level for the context. - executor.init(commit.index(), commit.time(), ServerStateMachineContext.Type.COMMAND); - - // Store the event index to return in the command response. - long eventIndex = session.getEventIndex(); - - try { - // Execute the state machine operation and get the result. - Object output = executor.executeOperation(commit); - - // Once the operation has been applied to the state machine, commit events published by the command. - // The state machine context will build a composite future for events published to all sessions. - executor.commit(); - - // Store the result for linearizability and complete the command. - Result result = new Result(index, eventIndex, output); - session.registerResult(sequence, result); - context.executor().execute(() -> future.complete(result)); - } catch (Exception e) { - // If an exception occurs during execution of the command, store the exception. - Result result = new Result(index, eventIndex, e); - session.registerResult(sequence, result); - context.executor().execute(() -> future.complete(result)); - } - } - - /** - * Applies a query entry to the state machine. - *

- * Query entries are applied to the user {@link StateMachine} for read-only operations. - * Because queries are read-only, they may only be applied on a single server in the cluster, - * and query entries do not go through the Raft log. Thus, it is critical that measures be taken - * to ensure clients see a consistent view of the cluster event when switching servers. To do so, - * clients provide a sequence and version number for each query. The sequence number is the order - * in which the query was sent by the client. Sequence numbers are shared across both commands and - * queries. The version number indicates the last index for which the client saw a command or query - * response. In the event that the lastApplied index of this state machine does not meet the provided - * version number, we wait for the state machine to catch up before applying the query. This ensures - * clients see state progress monotonically even when switching servers. - *

- * Because queries may only be applied on a single server in the cluster they cannot result in the - * publishing of session events. Events require commands to be written to the Raft log to ensure - * fault-tolerance and consistency across the cluster. - */ - private CompletableFuture apply(QueryEntry entry) { - ServerSessionContext session = executor.context().sessions().getSession(entry.getSession()); - - // If the session is null then that indicates that the session already timed out or it never existed. - // Return with an UnknownSessionException. - if (session == null) { - return Futures.exceptionalFuture(new UnknownSessionException("unknown session " + entry.getSession())); - } - // If the session is not in an active state, return an UnknownSessionException. Sessions are retained in the - // session registry until all prior commands have been released by the state machine, but new operations can - // only be applied for sessions in an active state. - else if (!session.state().active()) { - return Futures.exceptionalFuture(new UnknownSessionException("inactive session: " + entry.getSession())); - } else { - CompletableFuture future = new CompletableFuture<>(); - ThreadContext context = ThreadContext.currentContextOrThrow(); - ServerCommit commit = commits.acquire(entry.setIndex(lastApplied), session, executor.timestamp()); - executor.executor().execute(() -> executeQuery(commit, session, future, context)); - return future; - } - } - - /** - * Executes a state machine query. - */ - private void executeQuery(ServerCommit commit, ServerSessionContext session, CompletableFuture future, ThreadContext context) { - if (!log.isOpen()) { - context.executor().execute(() -> future.completeExceptionally(new IllegalStateException("log closed"))); - return; - } - - // If the session is already in an inactive state, complete the future exceptionally. - if (!session.state().active()) { - context.executor().execute(() -> future.completeExceptionally(new UnknownSessionException("inactive session: " + session.id()))); - return; - } - - long index = commit.index(); - long eventIndex = session.getEventIndex(); - - // Update the state machine context with the query entry's index. We set a null consistency - // level to indicate that events cannot be published in this context. Publishing events in - // response to state machine queries is non-deterministic as queries are not replicated. - executor.init(index, commit.time(), ServerStateMachineContext.Type.QUERY); - - try { - Object result = executor.executeOperation(commit); - context.executor().execute(() -> future.complete(new Result(index, eventIndex, result))); - } catch (Exception e) { - context.executor().execute(() -> future.complete(new Result(index, eventIndex, e))); - } - } - - /** - * Applies an initialize entry to the state machine. - *

- * Initialize entries are committed by leaders at the start of their term. Typically, no-op entries - * serve as a mechanism to allow leaders to commit entries from prior terms. However, we extend - * the functionality of the no-op entry to use it as an indicator that a leadership change occurred. - * In order to ensure timeouts do not expire during lengthy leadership changes, we use no-op entries - * to reset timeouts for client sessions and server heartbeats. - */ - private CompletableFuture apply(InitializeEntry entry) { - // Iterate through all the server sessions and reset timestamps. This ensures that sessions do not - // timeout during leadership changes or shortly thereafter. - long timestamp = executor.timestamp(entry.getTimestamp()); - for (ServerSessionContext session : executor.context().sessions().sessions.values()) { - session.setTimestamp(timestamp); - } - log.release(entry.getIndex()); - return Futures.completedFutureAsync(entry.getIndex(), ThreadContext.currentContextOrThrow().executor()); - } - - /** - * Marked as suspicious any sessions that have timed out according to the given timestamp. - *

- * Sessions are marked suspicious instead of being expired since log cleaning can result in large - * gaps in time between entries in the log. Thus, once log compaction has occurred, it's possible - * that a session could be marked expired when in fact its keep alive entries were simply compacted - * from the log. Forcing the leader to expire sessions ensures that keep alives are not missed with - * regard to session expiration. - */ - private void suspectSessions(long exclude, long timestamp) { - for (ServerSessionContext session : executor.context().sessions().sessions.values()) { - if (session.id() != exclude && timestamp - session.timeout() > session.getTimestamp()) { - session.suspect(); - } - } - } - - @Override - public void close() { - executor.close(); - } - - /** - * State machine result. - */ - static final class Result { - final long index; - final long eventIndex; - final Object result; - - Result(long index, long eventIndex, Object result) { - this.index = index; - this.eventIndex = eventIndex; - this.result = result; - } - } - -} diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineContext.java b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineContext.java index 55550dd5..d0399eed 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineContext.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineContext.java @@ -1,11 +1,11 @@ /* - * Copyright 2015 the original author or authors. + * Copyright 2017-present Open Networking Laboratory * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,10 +13,10 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - package io.atomix.copycat.server.state; import io.atomix.copycat.server.StateMachineContext; +import io.atomix.copycat.server.session.Sessions; import java.time.Clock; import java.time.Instant; @@ -37,13 +37,11 @@ enum Type { } private final ServerClock clock = new ServerClock(); - private final ConnectionManager connections; - private final ServerSessionManager sessions; + private final ServerStateMachineSessions sessions; private Type type; private long index; - public ServerStateMachineContext(ConnectionManager connections, ServerSessionManager sessions) { - this.connections = connections; + public ServerStateMachineContext(ServerStateMachineSessions sessions) { this.sessions = sessions; } @@ -84,20 +82,13 @@ public Clock clock() { } @Override - public ServerSessionManager sessions() { + public Sessions sessions() { return sessions; } - /** - * Returns the server connections. - */ - ConnectionManager connections() { - return connections; - } - @Override public String toString() { return String.format("%s[index=%d, time=%s]", getClass().getSimpleName(), index, clock); } -} +} \ No newline at end of file diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineExecutor.java b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineExecutor.java index f8bca80c..a7ad576c 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineExecutor.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineExecutor.java @@ -16,16 +16,22 @@ package io.atomix.copycat.server.state; -import io.atomix.catalyst.serializer.Serializer; -import io.atomix.catalyst.util.Assert; -import io.atomix.catalyst.concurrent.NonBlockingFuture; import io.atomix.catalyst.concurrent.Scheduled; import io.atomix.catalyst.concurrent.ThreadContext; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.catalyst.transport.Address; +import io.atomix.catalyst.util.Assert; +import io.atomix.copycat.Command; import io.atomix.copycat.NoOpCommand; import io.atomix.copycat.Operation; +import io.atomix.copycat.Query; import io.atomix.copycat.error.ApplicationException; import io.atomix.copycat.server.Commit; +import io.atomix.copycat.server.StateMachine; import io.atomix.copycat.server.StateMachineExecutor; +import io.atomix.copycat.server.session.SessionListener; +import io.atomix.copycat.server.storage.Log; +import io.atomix.copycat.server.storage.LogCleaner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,10 +39,8 @@ import java.time.Instant; import java.util.*; import java.util.concurrent.CompletableFuture; -import java.util.concurrent.Executor; import java.util.function.Consumer; import java.util.function.Function; -import java.util.function.Supplier; /** * Raft server state machine executor. @@ -45,36 +49,31 @@ */ class ServerStateMachineExecutor implements StateMachineExecutor { private static final Logger LOGGER = LoggerFactory.getLogger(ServerStateMachineExecutor.class); + private final StateMachine stateMachine; + private final ServerContext server; + private final ServerStateMachineSessions sessions = new ServerStateMachineSessions(); + private final ServerStateMachineContext context = new ServerStateMachineContext(sessions); private final ThreadContext executor; - private final ServerStateMachineContext context; - private final Queue tasks = new ArrayDeque<>(); + private final Queue tasks = new ArrayDeque<>(); private final List scheduledTasks = new ArrayList<>(); private final List complete = new ArrayList<>(); private final Map operations = new HashMap<>(); - private long timestamp; - ServerStateMachineExecutor(ServerStateMachineContext context, ThreadContext executor) { + ServerStateMachineExecutor(StateMachine stateMachine, ServerContext server, ThreadContext executor) { + this.stateMachine = stateMachine; + this.server = server; this.executor = executor; - this.context = context; - } - - /** - * Returns the current executor timestamp. - * - * @return The current executor timestamp. - */ - long timestamp() { - return timestamp; + init(); } /** - * Returns an updated executor timestamp. - * - * @return The updated executor timestamp. + * Initializes the state machine. */ - long timestamp(long timestamp) { - this.timestamp = Math.max(this.timestamp, timestamp); - return this.timestamp; + private void init() { + if (stateMachine instanceof SessionListener) { + sessions.addListener((SessionListener) stateMachine); + } + stateMachine.init(this); } @Override @@ -92,23 +91,340 @@ public Serializer serializer() { return executor.serializer(); } - @Override - public Executor executor() { - return executor.executor(); + /** + * Executes scheduled callbacks based on the provided time. + */ + void tick(long index, long timestamp) { + executor.execute(() -> { + // Only create an iterator if there are actually tasks scheduled. + if (!scheduledTasks.isEmpty()) { + + // Iterate through scheduled tasks until we reach a task that has not met its scheduled time. + // The tasks list is sorted by time on insertion. + Iterator iterator = scheduledTasks.iterator(); + while (iterator.hasNext()) { + ServerScheduledTask task = iterator.next(); + if (task.complete(timestamp)) { + context.update(index, Instant.ofEpochMilli(task.time), ServerStateMachineContext.Type.COMMAND); + task.execute(); + complete.add(task); + iterator.remove(); + } else { + break; + } + } + + // Iterate through tasks that were completed and reschedule them. + for (ServerScheduledTask task : complete) { + task.reschedule(timestamp); + } + complete.clear(); + } + }); + } + + /** + * Registers the given session. + * + * @param index The index of the registration. + * @param timestamp The timestamp of the registration. + * @param session The session to register. + */ + CompletableFuture register(long index, long timestamp, ServerSessionContext session) { + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> { + // Update the state machine context with the keep-alive entry's index. This ensures that events published + // as a result of asynchronous callbacks will be executed at the proper index with SEQUENTIAL consistency. + context.update(index, Instant.ofEpochMilli(timestamp), ServerStateMachineContext.Type.COMMAND); + + // Add the session to the sessions list. + sessions.add(session); + + // Iterate through and invoke session listeners. + for (SessionListener listener : sessions.listeners) { + listener.register(session); + } + + // Complete the future. + future.complete(null); + }); + return future; } /** - * Initializes the execution of a task. + * Keeps the given session alive. + * + * @param index The index of the keep-alive. + * @param timestamp The timestamp of the keep-alive. + * @param session The session to keep-alive. + * @param commandSequence The session command sequence number. + * @param eventIndex The session event index. */ - void init(long index, Instant instant, ServerStateMachineContext.Type type) { - context.update(index, instant, type); + CompletableFuture keepAlive(long index, long timestamp, ServerSessionContext session, long commandSequence, long eventIndex) { + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> { + // Update the state machine context with the keep-alive entry's index. This ensures that events published + // as a result of asynchronous callbacks will be executed at the proper index with SEQUENTIAL consistency. + context.update(index, Instant.ofEpochMilli(timestamp), ServerStateMachineContext.Type.COMMAND); + + // Clear results cached in the session. + session.clearResults(commandSequence); + + // Resend missing events starting from the last received event index. + session.resendEvents(eventIndex); + + // Update the session's request sequence number. The command sequence number will be applied + // iff the existing request sequence number is less than the command sequence number. This must + // be applied to ensure that request sequence numbers are reset after a leader change since leaders + // track request sequence numbers in local memory. + session.resetRequestSequence(commandSequence); + + // Update the sessions' command sequence number. The command sequence number will be applied + // iff the existing sequence number is less than the keep-alive command sequence number. This should + // not be the case under normal operation since the command sequence number in keep-alive requests + // represents the highest sequence for which a client has received a response (the command has already + // been completed), but since the log compaction algorithm can exclude individual entries from replication, + // the command sequence number must be applied for keep-alive requests to reset the sequence number in + // the event the last command for the session was cleaned/compacted from the log. + session.setCommandSequence(commandSequence); + + // Set the last applied index for the session. This will cause queries to be triggered if enqueued. + session.setLastApplied(index); + + // Complete the future. + future.complete(null); + }); + return future; + } + + /** + * Expires the given session. + * + * @param index The index of the expiration. + * @param timestamp The timestamp of the expiration. + * @param session The session to expire. + */ + CompletableFuture expire(long index, long timestamp, ServerSessionContext session) { + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> { + + // Remove the session from the sessions list. + sessions.remove(session); + + // Update the state machine context with the keep-alive entry's index. This ensures that events published + // as a result of asynchronous callbacks will be executed at the proper index with SEQUENTIAL consistency. + context.update(index, Instant.ofEpochMilli(timestamp), ServerStateMachineContext.Type.COMMAND); + + // Iterate through and invoke session listeners. + for (SessionListener listener : sessions.listeners) { + listener.expire(session); + listener.close(session); + } + + // Commit the index, causing event messages to be sent. + commit(); + + // Complete the future. + future.complete(null); + }); + return future; + } + + /** + * Unregister the given session. + * + * @param index The index of the unregister. + * @param timestamp The timestamp of the unregister. + * @param session The session to unregister. + */ + CompletableFuture unregister(long index, long timestamp, ServerSessionContext session) { + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> { + // Update the state machine context with the keep-alive entry's index. This ensures that events published + // as a result of asynchronous callbacks will be executed at the proper index with SEQUENTIAL consistency. + context.update(index, Instant.ofEpochMilli(timestamp), ServerStateMachineContext.Type.COMMAND); + + // Iterate through and invoke session listeners. + for (SessionListener listener : sessions.listeners) { + listener.unregister(session); + listener.close(session); + } + + // Remove the session from the sessions list. + sessions.remove(session); + + // Commit the index, causing event messages to be sent. + commit(); + + // Complete the future. + future.complete(null); + }); + return future; + } + + /** + * Resends events for the given session. + * + * @param index The index from which to resend events. + * @param session The session for which to resend events. + */ + void reset(long index, ServerSessionContext session) { + executor.execute(() -> session.resendEvents(index)); + } + + /** + * Executes the given command on the state machine. + * + * @param index The index of the command. + * @param timestamp The timestamp of the command. + * @param sequence The command sequence number. + * @param session The session that submitted the command. + * @param command The command to execute. + * @return A future to be completed with the command result. + */ + CompletableFuture executeCommand(long index, long timestamp, long sequence, ServerSessionContext session, Command command) { + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> executeCommand(index, timestamp, sequence, session, command, future)); + return future; + } + + /** + * Executes a command on the state machine thread. + */ + private void executeCommand(long index, long timestamp, long sequence, ServerSessionContext session, Command command, CompletableFuture future) { + // If the command's sequence number is less than the next session sequence number then that indicates that + // we've received a command that was previously applied to the state machine. Ensure linearizability by + // returning the cached response instead of applying it to the user defined state machine. + if (sequence > 0 && sequence < session.nextCommandSequence()) { + sequenceCommand(sequence, session, future); + } + // If we've made it this far, the command must have been applied in the proper order as sequenced by the + // session. This should be the case for most commands applied to the state machine. + else { + // Execute the command in the state machine thread. Once complete, the CompletableFuture callback will be completed + // in the state machine thread. Register the result in that thread and then complete the future in the caller's thread. + applyCommand(index, sequence, timestamp, command, session, future); + + // Update the session timestamp and command sequence number. This is done in the caller's thread since all + // timestamp/index/sequence checks are done in this thread prior to executing operations on the state machine thread. + session.setCommandSequence(sequence); + } + } + + /** + * Loads and returns a cached command result according to the sequence number. + */ + private void sequenceCommand(long sequence, ServerSessionContext session, CompletableFuture future) { + OperationResult result = session.getResult(sequence); + if (result == null) { + LOGGER.debug("Missing command result for {}:{}", session.id(), sequence); + } + future.complete(result); + } + + /** + * Applies the given commit to the state machine. + */ + private void applyCommand(long index, long sequence, long timestamp, Command command, ServerSessionContext session, CompletableFuture future) { + ServerCommit commit = new ServerCommit(index, command, session, timestamp, new LogCleaner(server.getLog())); + context.update(commit.index(), commit.time(), ServerStateMachineContext.Type.COMMAND); + + long eventIndex = session.getEventIndex(); + + OperationResult result; + try { + // Execute the state machine operation and get the result. + Object output = applyCommit(commit); + + // Once the operation has been applied to the state machine, commit events published by the command. + // The state machine context will build a composite future for events published to all sessions. + commit(); + + // Store the result for linearizability and complete the command. + result = new OperationResult(index, eventIndex, output); + session.registerResult(sequence, result); + future.complete(result); + } catch (Exception e) { + // If an exception occurs during execution of the command, store the exception. + result = new OperationResult(index, eventIndex, e); + } + + session.registerResult(sequence, result); + future.complete(result); + } + + /** + * Executes the given query on the state machine. + * + * @param index The index of the query. + * @param sequence The query sequence number. + * @param timestamp The timestamp of the query. + * @param session The session that submitted the query. + * @param query The query to execute. + * @return A future to be completed with the query result. + */ + CompletableFuture executeQuery(long index, long sequence, long timestamp, ServerSessionContext session, Query query) { + CompletableFuture future = new CompletableFuture<>(); + executor.execute(() -> executeQuery(index, sequence, timestamp, session, query, future)); + return future; + } + + /** + * Executes a query on the state machine thread. + */ + private void executeQuery(long index, long sequence, long timestamp, ServerSessionContext session, Query query, CompletableFuture future) { + sequenceQuery(index, sequence, timestamp, session, query, future); + } + + /** + * Sequences the given query. + */ + private void sequenceQuery(long index, long sequence, long timestamp, ServerSessionContext session, Query query, CompletableFuture future) { + // If the query's sequence number is greater than the session's current sequence number, queue the request for + // handling once the state machine is caught up. + if (sequence > session.getCommandSequence()) { + session.registerSequenceQuery(sequence, () -> indexQuery(index, timestamp, session, query, future)); + } else { + indexQuery(index, timestamp, session, query, future); + } + } + + /** + * Ensures the given query is applied after the appropriate index. + */ + private void indexQuery(long index, long timestamp, ServerSessionContext session, Query query, CompletableFuture future) { + // If the query index is greater than the session's last applied index, queue the request for handling once the + // state machine is caught up. + if (index > session.getLastApplied()) { + session.registerIndexQuery(index, () -> applyQuery(index, timestamp, session, query, future)); + } else { + applyQuery(index, timestamp, session, query, future); + } + } + + /** + * Applies a query to the state machine. + */ + private void applyQuery(long index, long timestamp, ServerSessionContext session, Query query, CompletableFuture future) { + ServerCommit commit = new ServerCommit(index, query, session, timestamp, new LogCleaner(server.getLog())); + context.update(commit.index(), commit.time(), ServerStateMachineContext.Type.QUERY); + + long eventIndex = session.getEventIndex(); + + OperationResult result; + try { + result = new OperationResult(index, eventIndex, applyCommit(commit)); + } catch (Exception e) { + result = new OperationResult(index, eventIndex, e); + } + future.complete(result); } /** * Executes an operation. */ @SuppressWarnings("unchecked") - , U> U executeOperation(Commit commit) { + private , U> U applyCommit(Commit commit) { // If the commit operation is a no-op command, complete the operation. if (commit.operation() instanceof NoOpCommand) { commit.close(); @@ -153,67 +469,22 @@ , U> U executeOperation(Commit commit) { * Commits the application of a command to the state machine. */ @SuppressWarnings("unchecked") - void commit() { + private void commit() { // Execute any tasks that were queue during execution of the command. if (!tasks.isEmpty()) { - for (ServerTask task : tasks) { + for (Runnable callback : tasks) { context.update(context.index(), context.clock().instant(), ServerStateMachineContext.Type.COMMAND); - try { - task.future.complete(task.callback.get()); - } catch (Exception e) { - task.future.completeExceptionally(e); - } + callback.run(); } tasks.clear(); } context.commit(); } - /** - * Executes scheduled callbacks based on the provided time. - */ - void tick(long index, long timestamp) { - // Only create an iterator if there are actually tasks scheduled. - if (!scheduledTasks.isEmpty()) { - - // Iterate through scheduled tasks until we reach a task that has not met its scheduled time. - // The tasks list is sorted by time on insertion. - Iterator iterator = scheduledTasks.iterator(); - while (iterator.hasNext()) { - ServerScheduledTask task = iterator.next(); - if (task.complete(timestamp)) { - context.update(index, Instant.ofEpochMilli(task.time), ServerStateMachineContext.Type.COMMAND); - task.execute(); - complete.add(task); - iterator.remove(); - } else { - break; - } - } - - // Iterate through tasks that were completed and reschedule them. - for (ServerScheduledTask task : complete) { - task.reschedule(); - } - complete.clear(); - } - } - - @Override - @SuppressWarnings("unchecked") - public CompletableFuture execute(Runnable callback) { - return execute((Supplier) () -> { - callback.run(); - return null; - }); - } - @Override - public CompletableFuture execute(Supplier callback) { + public void execute(Runnable callback) { Assert.state(context.type() == ServerStateMachineContext.Type.COMMAND, "callbacks can only be scheduled during command execution"); - CompletableFuture future = new NonBlockingFuture<>(); - tasks.add(new ServerTask(callback, future)); - return future; + tasks.add(callback); } @Override @@ -256,19 +527,6 @@ public void close() { executor.close(); } - /** - * Server task. - */ - private static class ServerTask { - private final Supplier callback; - private final CompletableFuture future; - - private ServerTask(Supplier callback, CompletableFuture future) { - this.callback = callback; - this.future = future; - } - } - /** * Scheduled task. */ @@ -327,7 +585,7 @@ private Scheduled schedule() { /** * Reschedules the task. */ - private void reschedule() { + private void reschedule(long timestamp) { if (interval > 0) { time = timestamp + interval; schedule(); @@ -353,5 +611,4 @@ public synchronized void cancel() { scheduledTasks.remove(this); } } - } diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java new file mode 100644 index 00000000..5cbc8ae6 --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java @@ -0,0 +1,649 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.state; + +import io.atomix.catalyst.concurrent.ComposableFuture; +import io.atomix.catalyst.concurrent.Futures; +import io.atomix.catalyst.concurrent.ThreadContext; +import io.atomix.catalyst.concurrent.ThreadPoolContext; +import io.atomix.catalyst.util.Assert; +import io.atomix.copycat.Command; +import io.atomix.copycat.Query; +import io.atomix.copycat.error.InternalException; +import io.atomix.copycat.error.UnknownClientException; +import io.atomix.copycat.error.UnknownSessionException; +import io.atomix.copycat.error.UnknownStateMachineException; +import io.atomix.copycat.server.StateMachine; +import io.atomix.copycat.server.storage.Log; +import io.atomix.copycat.server.storage.LogCleaner; +import io.atomix.copycat.server.storage.entry.*; +import io.atomix.copycat.server.storage.snapshot.Snapshot; +import io.atomix.copycat.session.Session; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ScheduledExecutorService; +import java.util.function.BiConsumer; +import java.util.function.Supplier; + +/** + * Internal server state machine. + *

+ * The internal state machine handles application of commands to the user provided {@link StateMachine} + * and keeps track of internal state like sessions and the various indexes relevant to log compaction. + * + * @author + * The last applied index is updated *after* each time a non-query entry is applied to the state machine. + * + * @param lastApplied The last applied index. + */ + private void setLastApplied(long lastApplied) { + // lastApplied should be either equal to or one greater than this.lastApplied. + if (lastApplied > this.lastApplied) { + Assert.arg(lastApplied == this.lastApplied + 1, "lastApplied must be sequential"); + this.lastApplied = lastApplied; + } else { + Assert.arg(lastApplied == this.lastApplied, "lastApplied cannot be decreased"); + } + } + + /** + * Updates and returns the current logical timestamp. + * + * @param entry The entry with which to update the timestamp. + * @return The updated timestamp. + */ + private long updateTimestamp(TimestampedEntry entry) { + timestamp = Math.max(this.timestamp, entry.getTimestamp()); + for (ServerStateMachineExecutor executor : stateMachines.values()) { + executor.tick(entry.getIndex(), timestamp); + } + return timestamp; + } + + /** + * Updates the last completed event index based on a commit at the given index. + */ + private void updateLastCompleted(long index) { + if (!log.isOpen()) + return; + + // Calculate the last completed index as the lowest index acknowledged by all clients. + long lastCompleted = index; + for (ServerSessionContext session : sessionManager.getSessions()) { + lastCompleted = Math.min(lastCompleted, session.getLastCompleted()); + } + + this.lastCompleted = Math.max(this.lastCompleted, lastCompleted); + + // Update the log compaction minor index. + log.compactor().minorIndex(this.lastCompleted); + } + + /** + * Applies all commits up to the given index. + *

+ * Calls to this method are assumed not to expect a result. This allows some optimizations to be + * made internally since linearizable events don't have to be waited to complete the command. + * + * @param index The index up to which to apply commits. + */ + public void applyAll(long index) { + if (!log.isOpen()) + return; + + // If the effective commit index is greater than the last index applied to the state machine then apply remaining entries. + long lastIndex = Math.min(index, log.lastIndex()); + if (lastIndex > lastApplied) { + for (long i = lastApplied + 1; i <= lastIndex; i++) { + Entry entry = log.get(i); + if (entry != null) { + apply(entry).whenComplete((result, error) -> entry.release()); + } + setLastApplied(i); + } + } + } + + /** + * Applies the entry at the given index to the state machine. + *

+ * Calls to this method are assumed to expect a result. This means linearizable session events + * triggered by the application of the command at the given index will be awaited before completing + * the returned future. + * + * @param index The index to apply. + * @return A completable future to be completed once the commit has been applied. + */ + public CompletableFuture apply(long index) { + try { + // If entries remain to be applied prior to this entry then synchronously apply them. + if (index > lastApplied + 1) { + applyAll(index - 1); + } + + // Read the entry from the log. If the entry is non-null them apply the entry, otherwise + // simply update the last applied index and return a null result. + try (Entry entry = log.get(index)) { + if (entry != null) { + return apply(entry); + } else { + return CompletableFuture.completedFuture(null); + } + } + } catch (Exception e) { + e.printStackTrace(); + return Futures.exceptionalFuture(e); + } finally { + setLastApplied(index); + } + } + + /** + * Applies an entry to the state machine. + *

+ * Calls to this method are assumed to expect a result. This means linearizable session events + * triggered by the application of the given entry will be awaited before completing the returned future. + * + * @param entry The entry to apply. + * @return A completable future to be completed with the result. + */ + @SuppressWarnings("unchecked") + public CompletableFuture apply(Entry entry) { + LOGGER.trace("{} - Applying {}", state.getCluster().member().address(), entry); + ThreadContext context = ThreadContext.currentContextOrThrow(); + ComposableFuture future = new ComposableFuture(); + BiConsumer callback = (result, error) -> { + if (error == null) { + context.execute(() -> future.complete(result)); + } else { + context.execute(() -> future.completeExceptionally(error)); + } + }; + + threadContext.execute(() -> { + if (entry instanceof QueryEntry) { + ((CompletableFuture) apply((QueryEntry) entry)).whenComplete(callback); + } else if (entry instanceof CommandEntry) { + ((CompletableFuture) apply((CommandEntry) entry)).whenComplete(callback); + } else if (entry instanceof OpenSessionEntry) { + ((CompletableFuture) apply((OpenSessionEntry) entry)).whenComplete(callback); + } else if (entry instanceof CloseSessionEntry) { + ((CompletableFuture) apply((CloseSessionEntry) entry)).whenComplete(callback); + } else if (entry instanceof RegisterEntry) { + ((CompletableFuture) apply((RegisterEntry) entry)).whenComplete(callback); + } else if (entry instanceof KeepAliveEntry) { + ((CompletableFuture) apply((KeepAliveEntry) entry)).whenComplete(callback); + } else if (entry instanceof UnregisterEntry) { + ((CompletableFuture) apply((UnregisterEntry) entry)).whenComplete(callback); + } else if (entry instanceof InitializeEntry) { + ((CompletableFuture) apply((InitializeEntry) entry)).whenComplete(callback); + } else if (entry instanceof ConfigurationEntry) { + ((CompletableFuture) apply((ConfigurationEntry) entry)).whenComplete(callback); + } else { + future.completeExceptionally(new InternalException("Unknown entry type")); + } + }); + return future; + } + + /** + * Applies a configuration entry to the internal state machine. + *

+ * Configuration entries are applied to internal server state when written to the log. Thus, no significant + * logic needs to take place in the handling of configuration entries. We simply release the previous configuration + * entry since it was overwritten by a more recent committed configuration entry. + */ + private CompletableFuture apply(ConfigurationEntry entry) { + // Clean the configuration entry from the log. The entry will be retained until it has been stored + // on all servers. + log.release(entry.getIndex()); + return CompletableFuture.completedFuture(null); + } + + /** + * Applies register session entry to the state machine. + *

+ * Register entries are applied to the state machine to create a new session. The resulting session ID is the + * index of the RegisterEntry. Once a new session is registered, we call register() on the state machine. + * In the event that the {@code synchronous} flag is set, that indicates that the registration command expects a + * response, i.e. it was applied by a leader. In that case, any events published during the execution of the + * state machine's register() method must be completed synchronously prior to the completion of the returned future. + */ + private CompletableFuture apply(RegisterEntry entry) { + long index = entry.getIndex(); + long timestamp = updateTimestamp(entry); + + ClientContext client = new ClientContext(entry.getIndex(), entry.getTimeout(), new LogCleaner(log)); + clientManager.registerClient(client); + + // Update the session timestamp *after* executing any scheduled operations. The executor's timestamp + // is guaranteed to be monotonically increasing, whereas the RegisterEntry may have an earlier timestamp + // if, e.g., it was written shortly after a leader change. + client.open(timestamp); + + // Determine whether any sessions appear to be expired. This won't immediately expire the session(s), + // but it will make them available to be unregistered by the leader. + suspectClients(client.id(), timestamp); + + // Update the last completed index to allow event entries to be compacted. + updateLastCompleted(index); + + return CompletableFuture.completedFuture(client.id()); + } + + /** + * Applies a session keep alive entry to the state machine. + *

+ * Keep alive entries are applied to the internal state machine to reset the timeout for a specific session. + * If the session indicated by the KeepAliveEntry is still held in memory, we mark the session as trusted, + * indicating that the client has committed a keep alive within the required timeout. Additionally, we check + * all other sessions for expiration based on the timestamp provided by this KeepAliveEntry. Note that sessions + * are never completely expired via this method. Leaders must explicitly commit an UnregisterEntry to expire + * a session. + *

+ * When a KeepAliveEntry is committed to the internal state machine, two specific fields provided in the entry + * are used to update server-side session state. The {@code commandSequence} indicates the highest command for + * which the session has received a successful response in the proper sequence. By applying the {@code commandSequence} + * to the server session, we clear command output held in memory up to that point. The {@code eventVersion} indicates + * the index up to which the client has received event messages in sequence for the session. Applying the + * {@code eventVersion} to the server-side session results in events up to that index being removed from memory + * as they were acknowledged by the client. It's essential that both of these fields be applied via entries committed + * to the Raft log to ensure they're applied on all servers in sequential order. + *

+ * Keep alive entries are retained in the log until the next time the client sends a keep alive entry or until the + * client's session is expired. This ensures for sessions that have long timeouts, keep alive entries cannot be cleaned + * from the log before they're replicated to some servers. + */ + private CompletableFuture apply(KeepAliveEntry entry) { + ClientContext client = clientManager.getClient(entry.getClient()); + + // Update the deterministic executor time and allow the executor to execute any scheduled events. + long timestamp = updateTimestamp(entry); + + // Determine whether any sessions appear to be expired. This won't immediately expire the session(s), + // but it will make them available to be unregistered by the leader. Note that it's safe to trigger + // scheduled executor callbacks even if the keep-alive entry is for an unknown session since the + // leader still committed the entry with its time and so time will still progress deterministically. + suspectClients(entry.getClient(), timestamp); + + // If the server session is null, the session either never existed or already expired. + if (client == null) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownSessionException("Unknown client: " + entry.getClient())); + } + // If the session is in an inactive state, return an UnknownSessionException. + else if (client.state() == ClientContext.State.CLOSED) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownSessionException("Inactive client: " + entry.getClient())); + } + + // If the session exists, don't allow it to expire even if its expiration has passed since we still + // managed to receive a keep alive request from the client before it was removed. This allows the + // client some arbitrary leeway in keeping its session alive. It's up to the leader to explicitly + // expire a session by committing an UnregisterEntry in order to ensure sessions can't be expired + // during leadership changes. + long index = entry.getIndex(); + + // Set the session as trusted. This will prevent the leader from explicitly unregistering the + // session if it hasn't done so already. + client.keepAlive(index, timestamp); + + // Store the session/command/event sequence and event index instead of acquiring a reference to the entry. + long[] sessionIds = entry.getSessionIds(); + long[] commandSequences = entry.getCommandSequences(); + long[] eventIndexes = entry.getEventIndexes(); + long[] connections = entry.getConnections(); + + for (int i = 0; i < sessionIds.length; i++) { + long sessionId = sessionIds[i]; + long commandSequence = commandSequences[i]; + long eventIndex = eventIndexes[i]; + long connection = connections[i]; + + // Register the connection with the session manager. This will cause the session manager to remove connections + // from sessions if the session isn't currently connected to this server. + sessionManager.registerConnection(sessionId, connection); + + ServerSessionContext session = sessionManager.getSession(sessionId); + if (session != null) { + session.getStateMachineExecutor().keepAlive(index, timestamp, session, commandSequence, eventIndex); + } + } + + // Update the last completed index to allow event entries to be compacted. + updateLastCompleted(index); + + return CompletableFuture.completedFuture(null); + } + + /** + * Applies an unregister session entry to the state machine. + *

+ * Unregister entries may either be committed by clients or by the cluster's leader. Clients will commit + * an unregister entry when closing their session normally. Leaders will commit an unregister entry when + * an expired session is detected. This ensures that sessions are never expired purely on gaps in the log + * which may result from normal log cleaning or lengthy leadership changes. + *

+ * If the session was unregistered by the client, the isExpired flag will be false. Sessions expired by + * the client are only close()ed on the state machine but not expire()d. Alternatively, entries where + * isExpired is true were committed by a leader. For expired sessions, the state machine's expire() method + * is called before close(). + *

+ * State machines may publish events during the handling of session expired or closed events. If the + * {@code synchronous} flag passed to this method is true, events published during the commitment of the + * UnregisterEntry must be synchronously completed prior to the completion of the returned future. This + * ensures that state changes resulting from the expiration or closing of a session are completed before + * the session close itself is completed. + */ + private CompletableFuture apply(UnregisterEntry entry) { + // Get the session from the context sessions. Note that we do not unregister the session here. Sessions + // can only be unregistered once all references to session commands have been released by the state machine. + ClientContext client = clientManager.unregisterClient(entry.getClient()); + + // Update the deterministic executor time and allow the executor to execute any scheduled events. + long timestamp = updateTimestamp(entry); + + // Determine whether any sessions appear to be expired. This won't immediately expire the session(s), + // but it will make them available to be unregistered by the leader. Note that it's safe to trigger + // scheduled executor callbacks even if the keep-alive entry is for an unknown session since the + // leader still committed the entry with its time and so time will still progress deterministically. + suspectClients(entry.getClient(), timestamp); + + // If the server session is null, the session either never existed or already expired. + if (client == null) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownSessionException("Unknown client: " + entry.getClient())); + } + // If the session is not in an active state, return an UnknownSessionException. + else if (client.state() == ClientContext.State.CLOSED) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownSessionException("Inactive client: " + entry.getClient())); + } + + long index = entry.getIndex(); + + // If the entry was marked expired, that indicates that the leader explicitly expired the session due to + // the session not being kept alive by the client. In all other cases, we close the session normally. + if (entry.isExpired()) { + sessionManager.getSessions().stream() + .filter(s -> s.client() == client.id()) + .forEach(s -> s.getStateMachineExecutor().expire(index, timestamp, s)); + } + // If the unregister entry is not indicated as expired, a client must have submitted a request to unregister + // the session. In that case, we simply close the session without expiring it. + else { + sessionManager.getSessions().stream() + .filter(s -> s.client() == client.id()) + .forEach(s -> s.getStateMachineExecutor().unregister(index, timestamp, s)); + } + + // Close the client. + client.close(index); + + // Update the last completed index to allow event entries to be compacted. + updateLastCompleted(index); + + return CompletableFuture.completedFuture(null); + } + + /** + * Applies an open session entry to the state machine. + */ + private CompletableFuture apply(OpenSessionEntry entry) { + ClientContext client = clientManager.getClient(entry.getClient()); + + long index = entry.getIndex(); + long timestamp = updateTimestamp(entry); + + // If the server session is null, the session either never existed or already expired. + if (client == null) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownClientException("Unknown client: " + entry.getClient())); + } + // If the session is not in an active state, return an UnknownSessionException. + else if (client.state() == ClientContext.State.CLOSED) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownClientException("Inactive client: " + entry.getClient())); + } + + // Get the state machine executor or create one if it doesn't already exist. + ServerStateMachineExecutor stateMachineExecutor = stateMachines.get(entry.getName()); + if (stateMachineExecutor == null) { + Supplier stateMachineSupplier = state.getStateMachineRegistry().getFactory(entry.getType()); + if (stateMachineSupplier == null) { + return Futures.exceptionalFuture(new UnknownStateMachineException("Unknown state machine type " + entry.getType())); + } + stateMachineExecutor = new ServerStateMachineExecutor(stateMachineSupplier.get(), state, new ThreadPoolContext(threadPool, threadContext.serializer().clone())); + stateMachines.put(entry.getName(), stateMachineExecutor); + } + + // Create and register the session. + ServerSessionContext session = new ServerSessionContext(entry.getIndex(), entry.getClient(), log, stateMachineExecutor); + sessionManager.registerSession(session); + + // Return the session ID for commands. + return session.getStateMachineExecutor().register(index, timestamp, session).thenApplyAsync(v -> session.id(), threadContext); + } + + /** + * Applies a close session entry to the state machine. + */ + private CompletableFuture apply(CloseSessionEntry entry) { + ServerSessionContext session = sessionManager.getSession(entry.getSession()); + + long index = entry.getIndex(); + long timestamp = updateTimestamp(entry); + + // If the server session is null, the session either never existed or already expired. + if (session == null) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownSessionException("Unknown session: " + entry.getSession())); + } + // If the session is not in an active state, return an UnknownSessionException. + else if (session.state() == Session.State.CLOSED) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownSessionException("Inactive session: " + entry.getSession())); + } + + // Get the state machine executor associated with the session and unregister the session. + ServerStateMachineExecutor stateMachineExecutor = session.getStateMachineExecutor(); + return stateMachineExecutor.unregister(index, timestamp, session).thenApplyAsync(v -> v, threadContext); + } + + /** + * Applies a command entry to the state machine. + *

+ * Command entries result in commands being executed on the user provided {@link StateMachine} and a + * response being sent back to the client by completing the returned future. All command responses are + * cached in the command's {@link ServerSessionContext} for fault tolerance. In the event that the same command + * is applied to the state machine more than once, the original response will be returned. + *

+ * Command entries are written with a sequence number. The sequence number is used to ensure that + * commands are applied to the state machine in sequential order. If a command entry has a sequence + * number that is less than the next sequence number for the session, that indicates that it is a + * duplicate of a command that was already applied. Otherwise, commands are assumed to have been + * received in sequential order. The reason for this assumption is because leaders always sequence + * commands as they're written to the log, so no sequence number will be skipped. + */ + private CompletableFuture apply(CommandEntry entry) { + long index = entry.getIndex(); + long timestamp = updateTimestamp(entry); + long sequenceNumber = entry.getSequence(); + Command command = entry.getCommand(); + + // First check to ensure that the session exists. + ServerSessionContext session = sessionManager.getSession(entry.getSession()); + + // If the session is null, return an UnknownSessionException. Commands applied to the state machine must + // have a session. We ensure that session register/unregister entries are not compacted from the log + // until all associated commands have been cleaned. + if (session == null) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownSessionException("unknown session: " + entry.getSession())); + } + // If the session is not in an active state, return an UnknownSessionException. Sessions are retained in the + // session registry until all prior commands have been released by the state machine, but new commands can + // only be applied for sessions in an active state. + else if (!session.state().active()) { + log.release(entry.getIndex()); + return Futures.exceptionalFuture(new UnknownSessionException("inactive session: " + entry.getSession())); + } + + // Execute the command using the state machine associated with the session. + return session.getStateMachineExecutor().executeCommand(index, timestamp, sequenceNumber, session, command); + } + + /** + * Applies a query entry to the state machine. + *

+ * Query entries are applied to the user {@link StateMachine} for read-only operations. + * Because queries are read-only, they may only be applied on a single server in the cluster, + * and query entries do not go through the Raft log. Thus, it is critical that measures be taken + * to ensure clients see a consistent view of the cluster event when switching servers. To do so, + * clients provide a sequence and version number for each query. The sequence number is the order + * in which the query was sent by the client. Sequence numbers are shared across both commands and + * queries. The version number indicates the last index for which the client saw a command or query + * response. In the event that the lastApplied index of this state machine does not meet the provided + * version number, we wait for the state machine to catch up before applying the query. This ensures + * clients see state progress monotonically even when switching servers. + *

+ * Because queries may only be applied on a single server in the cluster they cannot result in the + * publishing of session events. Events require commands to be written to the Raft log to ensure + * fault-tolerance and consistency across the cluster. + */ + private CompletableFuture apply(QueryEntry entry) { + ServerSessionContext session = sessionManager.getSession(entry.getSession()); + + long index = entry.getIndex(); + long sequence = entry.getSequence(); + long timestamp = updateTimestamp(entry); + Query query = entry.getQuery(); + + // If the session is null then that indicates that the session already timed out or it never existed. + // Return with an UnknownSessionException. + if (session == null) { + return Futures.exceptionalFuture(new UnknownSessionException("unknown session " + entry.getSession())); + } + // If the session is not in an active state, return an UnknownSessionException. Sessions are retained in the + // session registry until all prior commands have been released by the state machine, but new operations can + // only be applied for sessions in an active state. + else if (!session.state().active()) { + return Futures.exceptionalFuture(new UnknownSessionException("inactive session: " + entry.getSession())); + } + + // Execute the query using the state machine associated with the session. + return session.getStateMachineExecutor().executeQuery(index, sequence, timestamp, session, query); + } + + /** + * Applies an initialize entry to the state machine. + *

+ * Initialize entries are committed by leaders at the start of their term. Typically, no-op entries + * serve as a mechanism to allow leaders to commit entries from prior terms. However, we extend + * the functionality of the no-op entry to use it as an indicator that a leadership change occurred. + * In order to ensure timeouts do not expire during lengthy leadership changes, we use no-op entries + * to reset timeouts for client sessions and server heartbeats. + */ + private CompletableFuture apply(InitializeEntry entry) { + // Iterate through all the server sessions and reset timestamps. This ensures that sessions do not + // timeout during leadership changes or shortly thereafter. + long index = entry.getIndex(); + long timestamp = updateTimestamp(entry); + for (ClientContext clientContext : clientManager.getClients()) { + clientContext.keepAlive(index, timestamp); + } + log.release(entry.getIndex()); + return Futures.completedFutureAsync(entry.getIndex(), ThreadContext.currentContextOrThrow()); + } + + /** + * Marked as suspicious any sessions that have timed out according to the given timestamp. + *

+ * Sessions are marked suspicious instead of being expired since log cleaning can result in large + * gaps in time between entries in the log. Thus, once log compaction has occurred, it's possible + * that a session could be marked expired when in fact its keep alive entries were simply compacted + * from the log. Forcing the leader to expire sessions ensures that keep alives are not missed with + * regard to session expiration. + */ + private void suspectClients(long exclude, long timestamp) { + for (ClientContext clientContext : clientManager.getClients()) { + if (clientContext.id() != exclude && timestamp - clientContext.timeout() > clientContext.getTimestamp()) { + clientContext.suspect(timestamp); + } + } + } + + @Override + public void close() { + threadContext.close(); + } +} diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineSessions.java b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineSessions.java new file mode 100644 index 00000000..681ee42f --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineSessions.java @@ -0,0 +1,71 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.state; + +import io.atomix.copycat.server.session.ServerSession; +import io.atomix.copycat.server.session.SessionListener; +import io.atomix.copycat.server.session.Sessions; + +import java.util.*; + +/** + * State machine sessions. + */ +class ServerStateMachineSessions implements Sessions { + final Map sessions = new HashMap<>(); + final Set listeners = new HashSet<>(); + + /** + * Adds a session to the sessions list. + * + * @param session The session to add. + */ + void add(ServerSessionContext session) { + sessions.put(session.id(), session); + } + + /** + * Removes a session from the sessions list. + * + * @param session The session to remove. + */ + void remove(ServerSessionContext session) { + sessions.remove(session.id()); + } + + @Override + public ServerSession session(long sessionId) { + return sessions.get(sessionId); + } + + @Override + public Sessions addListener(SessionListener listener) { + listeners.add(listener); + return this; + } + + @Override + public Sessions removeListener(SessionListener listener) { + listeners.remove(listener); + return this; + } + + @Override + @SuppressWarnings("unchecked") + public Iterator iterator() { + return (Iterator) sessions.values().iterator(); + } +} \ No newline at end of file diff --git a/server/src/main/java/io/atomix/copycat/server/state/StateMachineRegistry.java b/server/src/main/java/io/atomix/copycat/server/state/StateMachineRegistry.java new file mode 100644 index 00000000..e7580210 --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/state/StateMachineRegistry.java @@ -0,0 +1,78 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.state; + +import io.atomix.catalyst.util.Assert; +import io.atomix.copycat.server.StateMachine; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Supplier; + +/** + * State machine registry. + */ +public class StateMachineRegistry { + private final Map> stateMachines = new ConcurrentHashMap<>(); + + /** + * Returns the number of registered state machines. + * + * @return The number of registered state machines. + */ + public int size() { + return stateMachines.size(); + } + + /** + * Registers a new state machine type. + * + * @param type The state machine type to register. + * @param factory The state machine factory. + * @return The state machine registry. + */ + public StateMachineRegistry register(String type, Supplier factory) { + stateMachines.put(Assert.notNull(type, "type"), Assert.notNull(factory, "factory")); + return this; + } + + /** + * Unregisters the given state machine type. + * + * @param type The state machine type to unregister. + * @return The state machine registry. + */ + public StateMachineRegistry unregister(String type) { + stateMachines.remove(type); + return this; + } + + /** + * Returns the factory for the given state machine type. + * + * @param type The state machine type for which to return the factory. + * @return The factory for the given state machine type or {@code null} if the type is not registered. + */ + public Supplier getFactory(String type) { + return stateMachines.get(type); + } + + @Override + public String toString() { + return String.format("%s[stateMachines=%s]", getClass().getSimpleName(), stateMachines); + } + +} diff --git a/server/src/main/java/io/atomix/copycat/server/storage/LogCleaner.java b/server/src/main/java/io/atomix/copycat/server/storage/LogCleaner.java new file mode 100644 index 00000000..18e3bedb --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/storage/LogCleaner.java @@ -0,0 +1,40 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.storage; + +/** + * Log cleaner. + */ +public class LogCleaner { + private final Log log; + + public LogCleaner(Log log) { + this.log = log; + } + + /** + * Cleans the entry at the given index. + * + * @param index The index at which to clean the entry. + */ + public void clean(long index) { + try { + log.release(index); + } catch (IllegalStateException e) { + // Do nothing. + } + } +} diff --git a/server/src/main/java/io/atomix/copycat/server/storage/compaction/Compactor.java b/server/src/main/java/io/atomix/copycat/server/storage/compaction/Compactor.java index d14b84ff..9d3c646f 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/compaction/Compactor.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/compaction/Compactor.java @@ -209,11 +209,16 @@ private synchronized CompletableFuture compact(Compaction compaction, Comp for (CompactionTask task : tasks) { LOGGER.debug("Executing {}", task); ThreadContext taskThread = new ThreadPoolContext(executor, segments.serializer()); - taskThread.execute(task).whenComplete((result, error) -> { + CompletableFuture taskFuture = new CompletableFuture<>(); + taskThread.execute(() -> { + task.run(); + taskFuture.complete(null); + }); + taskFuture.whenComplete((result, error) -> { LOGGER.debug("{} complete", task); if (counter.incrementAndGet() == tasks.size()) { if (context != null) { - context.executor().execute(() -> future.complete(null)); + context.execute(() -> future.complete(null)); } else { future.complete(null); } diff --git a/server/src/main/java/io/atomix/copycat/server/storage/entry/ClientEntry.java b/server/src/main/java/io/atomix/copycat/server/storage/entry/ClientEntry.java new file mode 100644 index 00000000..d5b06f7b --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/storage/entry/ClientEntry.java @@ -0,0 +1,71 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.storage.entry; + +import io.atomix.catalyst.buffer.BufferInput; +import io.atomix.catalyst.buffer.BufferOutput; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.catalyst.util.reference.ReferenceManager; + +/** + * Base class for client-related entries. + * + * @author Jordan Halterman + */ +public abstract class ClientEntry> extends TimestampedEntry { + private long client; + + protected ClientEntry() { + } + + protected ClientEntry(ReferenceManager> referenceManager) { + super(referenceManager); + } + + /** + * Sets the client ID. + * + * @param client The client ID. + * @return The client entry. + */ + @SuppressWarnings("unchecked") + public T setClient(long client) { + this.client = client; + return (T) this; + } + + /** + * Returns the client ID. + * + * @return The client ID. + */ + public long getClient() { + return client; + } + + @Override + public void writeObject(BufferOutput buffer, Serializer serializer) { + super.writeObject(buffer, serializer); + buffer.writeLong(client); + } + + @Override + public void readObject(BufferInput buffer, Serializer serializer) { + super.readObject(buffer, serializer); + client = buffer.readLong(); + } + +} diff --git a/server/src/main/java/io/atomix/copycat/server/storage/entry/CloseSessionEntry.java b/server/src/main/java/io/atomix/copycat/server/storage/entry/CloseSessionEntry.java new file mode 100644 index 00000000..869627ce --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/storage/entry/CloseSessionEntry.java @@ -0,0 +1,37 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.storage.entry; + +import io.atomix.catalyst.util.reference.ReferenceManager; + +/** + * Close session entry. + */ +public class CloseSessionEntry extends SessionEntry { + + public CloseSessionEntry() { + } + + public CloseSessionEntry(ReferenceManager> referenceManager) { + super(referenceManager); + } + + @Override + public String toString() { + return String.format("%s[index=%d, term=%d, session=%d, timestamp=%d]", getClass().getSimpleName(), getIndex(), getTerm(), getSession(), getTimestamp()); + } + +} diff --git a/server/src/main/java/io/atomix/copycat/server/storage/entry/KeepAliveEntry.java b/server/src/main/java/io/atomix/copycat/server/storage/entry/KeepAliveEntry.java index e7b0a85a..719dd8d0 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/entry/KeepAliveEntry.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/entry/KeepAliveEntry.java @@ -22,11 +22,13 @@ import io.atomix.copycat.protocol.KeepAliveRequest; import io.atomix.copycat.session.Session; +import java.util.Arrays; + /** * Stores a client keep-alive request. *

* The {@code KeepAliveEntry} is logged and replicated to the cluster to indicate that a client - * has kept its {@link #getSession() session} alive. Each client must periodically submit a + * has kept its {@link #getClient() client} alive. Each client must periodically submit a * {@link KeepAliveRequest} which results in a keep-alive entry * being written to the Raft log. When a keep-alive is committed to the internal Raft state machine, * the session timeout for the associated {@link Session} will be @@ -34,9 +36,11 @@ * * @author Jordan Halterman */ -public class KeepAliveEntry extends SessionEntry { - private long commandSequence; - private long eventIndex; +public class KeepAliveEntry extends ClientEntry { + private long[] sessionIds; + private long[] commandSequences; + private long[] eventIndexes; + private long[] connections; public KeepAliveEntry() { } @@ -46,62 +50,140 @@ public KeepAliveEntry(ReferenceManager> referenceManager) { } /** - * Returns the command sequence number. + * Returns the session identifiers. + * + * @return The session identifiers. + */ + public long[] getSessionIds() { + return sessionIds; + } + + /** + * Sets the session identifiers. + * + * @param sessionIds The session identifiers. + * @return The keep alive entry. + */ + public KeepAliveEntry setSessionIds(long[] sessionIds) { + this.sessionIds = sessionIds; + return this; + } + + /** + * Returns the command sequence numbers. * - * @return The command sequence number. + * @return The command sequence numbers. */ - public long getCommandSequence() { - return commandSequence; + public long[] getCommandSequences() { + return commandSequences; } /** - * Sets the command sequence number. + * Sets the command sequence numbers. * - * @param commandSequence The command sequence number. + * @param commandSequence The command sequence numbers. * @return The keep alive entry. */ - public KeepAliveEntry setCommandSequence(long commandSequence) { - this.commandSequence = commandSequence; + public KeepAliveEntry setCommandSequences(long[] commandSequence) { + this.commandSequences = commandSequence; return this; } /** - * Returns the event index. + * Returns the event indexes. * - * @return The event index. + * @return The event indexes. */ - public long getEventIndex() { - return eventIndex; + public long[] getEventIndexes() { + return eventIndexes; } /** - * Sets the event index. + * Sets the event indexes. * - * @param eventIndex The event index. + * @param eventIndexes The event indexes. * @return The keep alive entry. */ - public KeepAliveEntry setEventIndex(long eventIndex) { - this.eventIndex = eventIndex; + public KeepAliveEntry setEventIndexes(long[] eventIndexes) { + this.eventIndexes = eventIndexes; + return this; + } + + /** + * Returns the connection IDs. + * + * @return The connection IDs. + */ + public long[] getConnections() { + return connections; + } + + /** + * Sets the connection IDs. + * + * @param connections The connection IDs. + * @return The keep alive entry. + */ + public KeepAliveEntry setConnections(long[] connections) { + this.connections = connections; return this; } @Override - public void readObject(BufferInput buffer, Serializer serializer) { + public void readObject(BufferInput buffer, Serializer serializer) { super.readObject(buffer, serializer); - commandSequence = buffer.readLong(); - eventIndex = buffer.readLong(); + int sessionsLength = buffer.readInt(); + sessionIds = new long[sessionsLength]; + for (int i = 0; i < sessionsLength; i++) { + sessionIds[i] = buffer.readLong(); + } + + int commandSequencesLength = buffer.readInt(); + commandSequences = new long[commandSequencesLength]; + for (int i = 0; i < commandSequencesLength; i++) { + commandSequences[i] = buffer.readLong(); + } + + int eventIndexesLength = buffer.readInt(); + eventIndexes = new long[eventIndexesLength]; + for (int i = 0; i < eventIndexesLength; i++) { + eventIndexes[i] = buffer.readLong(); + } + + int connectionsLength = buffer.readInt(); + connections = new long[connectionsLength]; + for (int i = 0; i < connectionsLength; i++) { + connections[i] = buffer.readLong(); + } } @Override - public void writeObject(BufferOutput buffer, Serializer serializer) { + public void writeObject(BufferOutput buffer, Serializer serializer) { super.writeObject(buffer, serializer); - buffer.writeLong(commandSequence); - buffer.writeLong(eventIndex); + buffer.writeInt(sessionIds.length); + for (long sessionId : sessionIds) { + buffer.writeLong(sessionId); + } + + buffer.writeInt(commandSequences.length); + for (long commandSequence : commandSequences) { + buffer.writeLong(commandSequence); + } + + buffer.writeInt(eventIndexes.length); + for (long eventIndex : eventIndexes) { + buffer.writeLong(eventIndex); + } + + buffer.writeInt(connections.length); + for (long connection : connections) { + buffer.writeLong(connection); + } } @Override public String toString() { - return String.format("%s[index=%d, term=%d, session=%d, commandSequence=%d, eventIndex=%d, timestamp=%d]", getClass().getSimpleName(), getIndex(), getTerm(), getSession(), getCommandSequence(), getEventIndex(), getTimestamp()); + return String.format("%s[index=%d, term=%d, client=%s, sessions=%s, commandSequences=%s, eventIndexes=%s, connections=%s, timestamp=%d]", getClass().getSimpleName(), getIndex(), getTerm(), getClient(), Arrays.toString(getSessionIds()), Arrays.toString(getCommandSequences()), Arrays.toString(getEventIndexes()), Arrays.toString(getConnections()), getTimestamp()); } } diff --git a/server/src/main/java/io/atomix/copycat/server/storage/entry/OpenSessionEntry.java b/server/src/main/java/io/atomix/copycat/server/storage/entry/OpenSessionEntry.java new file mode 100644 index 00000000..ac5a790d --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/storage/entry/OpenSessionEntry.java @@ -0,0 +1,96 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.storage.entry; + +import io.atomix.catalyst.buffer.BufferInput; +import io.atomix.catalyst.buffer.BufferOutput; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.catalyst.util.reference.ReferenceManager; + +/** + * Open session entry. + */ +public class OpenSessionEntry extends ClientEntry { + private String name; + private String type; + + public OpenSessionEntry() { + } + + public OpenSessionEntry(ReferenceManager> referenceManager) { + super(referenceManager); + } + + /** + * Returns the session state machine name. + * + * @return The session's state machine name. + */ + public String getName() { + return name; + } + + /** + * Sets the session's state machine name. + * + * @param name The session's state machine name. + * @return The open session entry. + */ + public OpenSessionEntry setName(String name) { + this.name = name; + return this; + } + + /** + * Returns the session state machine type. + * + * @return The session's state machine type. + */ + public String getType() { + return type; + } + + /** + * Sets the session's state machine type. + * + * @param type The session's state machine type. + * @return The open session entry. + */ + public OpenSessionEntry setType(String type) { + this.type = type; + return this; + } + + @Override + public void writeObject(BufferOutput buffer, Serializer serializer) { + super.writeObject(buffer, serializer); + buffer.writeString(name); + buffer.writeString(type); + } + + @Override + public void readObject(BufferInput buffer, Serializer serializer) { + super.readObject(buffer, serializer); + name = buffer.readString(); + type = buffer.readString(); + } + + @Override + public String toString() { + return String.format("%s[index=%d, term=%d, client=%s, name=%s, type=%s, timestamp=%d]", getClass().getSimpleName(), getIndex(), getTerm(), getClient(), getName(), getType(), getTimestamp()); + } + +} diff --git a/server/src/main/java/io/atomix/copycat/server/storage/entry/RegisterEntry.java b/server/src/main/java/io/atomix/copycat/server/storage/entry/RegisterEntry.java index 79b5d439..801a7838 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/entry/RegisterEntry.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/entry/RegisterEntry.java @@ -34,7 +34,6 @@ * @author Jordan Halterman */ public class RegisterEntry extends TimestampedEntry { - private String client; private long timeout; public RegisterEntry() { @@ -44,27 +43,6 @@ public RegisterEntry(ReferenceManager> referenceManager) { super(referenceManager); } - /** - * Returns the entry client ID. - * - * @return The entry client ID. - */ - public String getClient() { - return client; - } - - /** - * Sets the entry client ID. - * - * @param client The entry client ID. - * @return The register entry. - * @throws NullPointerException if {@code client} is null - */ - public RegisterEntry setClient(String client) { - this.client = Assert.notNull(client, "client"); - return this; - } - /** * Returns the session timeout. * @@ -88,20 +66,18 @@ public RegisterEntry setTimeout(long timeout) { @Override public void writeObject(BufferOutput buffer, Serializer serializer) { super.writeObject(buffer, serializer); - buffer.writeString(client); buffer.writeLong(timeout); } @Override public void readObject(BufferInput buffer, Serializer serializer) { super.readObject(buffer, serializer); - client = buffer.readString(); timeout = buffer.readLong(); } @Override public String toString() { - return String.format("%s[index=%d, term=%d, client=%s, timeout=%d]", getClass().getSimpleName(), getIndex(), getTerm(), getClient(), getTimestamp()); + return String.format("%s[index=%d, term=%d, timeout=%d]", getClass().getSimpleName(), getIndex(), getTerm(), getTimeout()); } } diff --git a/server/src/main/java/io/atomix/copycat/server/storage/entry/UnregisterEntry.java b/server/src/main/java/io/atomix/copycat/server/storage/entry/UnregisterEntry.java index 445b6900..a1d93677 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/entry/UnregisterEntry.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/entry/UnregisterEntry.java @@ -34,7 +34,7 @@ * * @author Jordan Halterman */ -public class UnregisterEntry extends SessionEntry { +public class UnregisterEntry extends ClientEntry { private boolean expired; public UnregisterEntry() { @@ -83,7 +83,7 @@ public void readObject(BufferInput buffer, Serializer serializer) { @Override public String toString() { - return String.format("%s[index=%d, term=%d, session=%d, expired=%b, timestamp=%d]", getClass().getSimpleName(), getIndex(), getTerm(), getSession(), isExpired(), getTimestamp()); + return String.format("%s[index=%d, term=%d, client=%s, expired=%b, timestamp=%d]", getClass().getSimpleName(), getIndex(), getTerm(), getClient(), isExpired(), getTimestamp()); } } diff --git a/server/src/main/java/io/atomix/copycat/server/storage/util/StorageSerialization.java b/server/src/main/java/io/atomix/copycat/server/storage/util/StorageSerialization.java index 7003137e..9ada4e6a 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/util/StorageSerialization.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/util/StorageSerialization.java @@ -31,13 +31,13 @@ public final class StorageSerialization implements SerializableTypeResolver { @SuppressWarnings("unchecked") private static final Map, Integer> TYPES = new HashMap() {{ - put(CommandEntry.class, -36); - put(ConfigurationEntry.class, -37); - put(KeepAliveEntry.class, -38); - put(InitializeEntry.class, -39); - put(QueryEntry.class, -40); - put(RegisterEntry.class, -41); - put(UnregisterEntry.class, -43); + put(CommandEntry.class, -38); + put(ConfigurationEntry.class, -39); + put(KeepAliveEntry.class, -40); + put(InitializeEntry.class, -41); + put(QueryEntry.class, -42); + put(RegisterEntry.class, -43); + put(UnregisterEntry.class, -44); }}; @Override diff --git a/server/src/main/java/io/atomix/copycat/server/util/ServerSerialization.java b/server/src/main/java/io/atomix/copycat/server/util/ServerSerialization.java index ebcc75c6..292eba0d 100644 --- a/server/src/main/java/io/atomix/copycat/server/util/ServerSerialization.java +++ b/server/src/main/java/io/atomix/copycat/server/util/ServerSerialization.java @@ -32,23 +32,23 @@ public final class ServerSerialization implements SerializableTypeResolver { @SuppressWarnings("unchecked") private static final Map, Integer> TYPES = new HashMap() {{ - put(AppendRequest.class, -18); - put(ConfigureRequest.class, -19); - put(InstallRequest.class, -20); - put(JoinRequest.class, -21); - put(LeaveRequest.class, -22); - put(PollRequest.class, -23); - put(ReconfigureRequest.class, -24); - put(VoteRequest.class, -25); - put(AppendResponse.class, -27); - put(ConfigureResponse.class, -28); - put(InstallResponse.class, -29); - put(JoinResponse.class, -30); - put(LeaveResponse.class, -31); - put(PollResponse.class, -32); - put(ReconfigureResponse.class, -33); - put(VoteResponse.class, -34); - put(ServerMember.class, -35); + put(AppendRequest.class, -21); + put(ConfigureRequest.class, -22); + put(InstallRequest.class, -23); + put(JoinRequest.class, -24); + put(LeaveRequest.class, -25); + put(PollRequest.class, -26); + put(ReconfigureRequest.class, -27); + put(VoteRequest.class, -28); + put(AppendResponse.class, -29); + put(ConfigureResponse.class, -30); + put(InstallResponse.class, -31); + put(JoinResponse.class, -32); + put(LeaveResponse.class, -33); + put(PollResponse.class, -34); + put(ReconfigureResponse.class, -35); + put(VoteResponse.class, -36); + put(ServerMember.class, -37); }}; @Override diff --git a/server/src/test/java/io/atomix/copycat/server/state/AbstractStateTest.java b/server/src/test/java/io/atomix/copycat/server/state/AbstractStateTest.java index 44708957..1dfdc21f 100644 --- a/server/src/test/java/io/atomix/copycat/server/state/AbstractStateTest.java +++ b/server/src/test/java/io/atomix/copycat/server/state/AbstractStateTest.java @@ -15,6 +15,7 @@ */ package io.atomix.copycat.server.state; +import io.atomix.catalyst.concurrent.CatalystThreadFactory; import io.atomix.catalyst.serializer.Serializer; import io.atomix.catalyst.transport.Address; import io.atomix.catalyst.transport.local.LocalServerRegistry; @@ -45,6 +46,7 @@ import java.time.Instant; import java.util.ArrayList; import java.util.List; +import java.util.concurrent.Executors; /** * Abstract state test. @@ -79,9 +81,19 @@ void beforeMethod() throws Throwable { transport = new LocalTransport(new LocalServerRegistry()); serverCtx = new SingleThreadContext("test-server", serializer); - new SingleThreadContext("test", serializer.clone()).executor().execute(() -> { - serverContext = new ServerContext("test", members.get(0).type(), members.get(0).serverAddress(), members.get(0).clientAddress(), storage, serializer, TestStateMachine::new, new ConnectionManager(transport.client()), serverCtx); - serverContext.getThreadContext().executor().execute(() -> { + new SingleThreadContext("test", serializer.clone()).execute(() -> { + serverContext = new ServerContext( + "test", + members.get(0).type(), + members.get(0).serverAddress(), + members.get(0).clientAddress(), + storage, + serializer, + new StateMachineRegistry().register("test", TestStateMachine::new), + new ConnectionManager(transport.client()), + Executors.newScheduledThreadPool(2, new CatalystThreadFactory("test-%d")), + serverCtx); + serverContext.getThreadContext().execute(() -> { serverContext.getClusterState().configure(new Configuration(0, 0, Instant.now().toEpochMilli(), members)); resume(); }); diff --git a/server/src/test/java/io/atomix/copycat/server/state/CandidateStateTest.java b/server/src/test/java/io/atomix/copycat/server/state/CandidateStateTest.java index 061a4367..8e32e32c 100644 --- a/server/src/test/java/io/atomix/copycat/server/state/CandidateStateTest.java +++ b/server/src/test/java/io/atomix/copycat/server/state/CandidateStateTest.java @@ -173,7 +173,7 @@ public void testCandidateTransitionsToLeaderOnElection() throws Throwable { for (MemberState member : serverContext.getClusterState().getRemoteMemberStates()) { Server server = transport.server(); server.listen(member.getMember().serverAddress(), c -> { - c.handler(VoteRequest.class, (Function) request -> CompletableFuture.completedFuture(VoteResponse.builder() + c.registerHandler(VoteRequest.NAME, (Function) request -> CompletableFuture.completedFuture(VoteResponse.builder() .withTerm(2) .withVoted(true) .build())); @@ -204,7 +204,7 @@ public void testCandidateTransitionsToFollowerOnRejection() throws Throwable { for (MemberState member : serverContext.getClusterState().getRemoteMemberStates()) { Server server = transport.server(); server.listen(member.getMember().serverAddress(), c -> { - c.handler(VoteRequest.class, (Function) request -> CompletableFuture.completedFuture(VoteResponse.builder() + c.registerHandler(VoteRequest.NAME, (Function) request -> CompletableFuture.completedFuture(VoteResponse.builder() .withTerm(2) .withVoted(false) .build())); diff --git a/server/src/test/java/io/atomix/copycat/server/state/ServerContextTest.java b/server/src/test/java/io/atomix/copycat/server/state/ServerContextTest.java index 8837f8a2..8c858887 100644 --- a/server/src/test/java/io/atomix/copycat/server/state/ServerContextTest.java +++ b/server/src/test/java/io/atomix/copycat/server/state/ServerContextTest.java @@ -93,18 +93,4 @@ void afterMethod() throws Throwable { super.afterMethod(); } - /** - * Tests a server response. - */ - private void test(T request, Consumer callback) throws Throwable { - clientCtx.execute(() -> { - connection.sendAndReceive(request).whenComplete((response, error) -> { - threadAssertNull(error); - callback.accept(response); - resume(); - }); - }); - await(); - } - } diff --git a/server/src/test/java/io/atomix/copycat/server/state/ServerSessionTest.java b/server/src/test/java/io/atomix/copycat/server/state/ServerSessionTest.java index afa24004..a769d2c7 100644 --- a/server/src/test/java/io/atomix/copycat/server/state/ServerSessionTest.java +++ b/server/src/test/java/io/atomix/copycat/server/state/ServerSessionTest.java @@ -18,7 +18,6 @@ import io.atomix.copycat.server.storage.Log; import org.testng.annotations.Test; -import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import static org.mockito.Mockito.mock; @@ -37,7 +36,7 @@ public class ServerSessionTest { */ public void testInitializeSession() throws Throwable { ServerStateMachineContext context = mock(ServerStateMachineContext.class); - ServerSessionContext session = new ServerSessionContext(10, UUID.randomUUID().toString(), mock(Log.class), context, 1000); + ServerSessionContext session = new ServerSessionContext(10, 1, mock(Log.class), mock(ServerStateMachineExecutor.class)); assertEquals(session.id(), 10); assertEquals(session.getLastCompleted(), 9); assertEquals(session.getLastApplied(), 9); @@ -48,7 +47,7 @@ public void testInitializeSession() throws Throwable { */ public void testSequenceIndexQuery() throws Throwable { ServerStateMachineContext context = mock(ServerStateMachineContext.class); - ServerSessionContext session = new ServerSessionContext(10, UUID.randomUUID().toString(), mock(Log.class), context, 1000); + ServerSessionContext session = new ServerSessionContext(10, 1, mock(Log.class), mock(ServerStateMachineExecutor.class)); AtomicBoolean complete = new AtomicBoolean(); session.registerIndexQuery(10, () -> complete.set(true)); assertFalse(complete.get()); @@ -63,7 +62,7 @@ public void testSequenceIndexQuery() throws Throwable { */ public void testSequenceSequenceQuery() throws Throwable { ServerStateMachineContext context = mock(ServerStateMachineContext.class); - ServerSessionContext session = new ServerSessionContext(10, UUID.randomUUID().toString(), mock(Log.class), context, 1000); + ServerSessionContext session = new ServerSessionContext(10, 1, mock(Log.class), mock(ServerStateMachineExecutor.class)); AtomicBoolean complete = new AtomicBoolean(); session.registerSequenceQuery(10, () -> complete.set(true)); assertFalse(complete.get()); @@ -78,8 +77,8 @@ public void testSequenceSequenceQuery() throws Throwable { */ public void testCacheResponse() throws Throwable { ServerStateMachineContext context = mock(ServerStateMachineContext.class); - ServerSessionContext session = new ServerSessionContext(10, UUID.randomUUID().toString(), mock(Log.class), context, 1000); - session.registerResult(2, new ServerStateMachine.Result(2, 2, "Hello world!")); + ServerSessionContext session = new ServerSessionContext(10, 1, mock(Log.class), mock(ServerStateMachineExecutor.class)); + session.registerResult(2, new OperationResult(2, 2, "Hello world!")); assertEquals(session.getResult(2).result, "Hello world!"); session.clearResults(3); assertNull(session.getResult(2)); diff --git a/server/src/test/java/io/atomix/copycat/server/state/ServerStateMachineManagerTest.java b/server/src/test/java/io/atomix/copycat/server/state/ServerStateMachineManagerTest.java new file mode 100644 index 00000000..9441395c --- /dev/null +++ b/server/src/test/java/io/atomix/copycat/server/state/ServerStateMachineManagerTest.java @@ -0,0 +1,162 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.state; + +import io.atomix.catalyst.concurrent.CatalystThreadFactory; +import io.atomix.catalyst.concurrent.SingleThreadContext; +import io.atomix.catalyst.concurrent.ThreadContext; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.catalyst.transport.Address; +import io.atomix.catalyst.transport.Transport; +import io.atomix.catalyst.transport.local.LocalServerRegistry; +import io.atomix.catalyst.transport.local.LocalTransport; +import io.atomix.copycat.Command; +import io.atomix.copycat.Query; +import io.atomix.copycat.protocol.ClientRequestTypeResolver; +import io.atomix.copycat.protocol.ClientResponseTypeResolver; +import io.atomix.copycat.server.Commit; +import io.atomix.copycat.server.StateMachine; +import io.atomix.copycat.server.StateMachineExecutor; +import io.atomix.copycat.server.cluster.Member; +import io.atomix.copycat.server.storage.Storage; +import io.atomix.copycat.server.storage.StorageLevel; +import io.atomix.copycat.server.storage.util.StorageSerialization; +import io.atomix.copycat.server.util.ServerSerialization; +import io.atomix.copycat.util.ProtocolSerialization; +import net.jodah.concurrentunit.ConcurrentTestCase; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.time.Instant; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +/** + * Server state machine test. + * + * @author { + state = new ServerContext( + "test", + member.type(), + member.serverAddress(), + member.clientAddress(), + storage, + serializer, + new StateMachineRegistry().register("test", TestStateMachine::new), + new ConnectionManager(new LocalTransport(registry).client()), + Executors.newScheduledThreadPool(2, new CatalystThreadFactory("test-%d")), + callerContext); + resume(); + }); + await(1000); + timestamp = System.currentTimeMillis(); + sequence = new AtomicLong(); + } + + // TODO: Add tests! + + @AfterMethod + public void closeStateMachine() { + state.close(); + callerContext.close(); + } + + /** + * Test state machine. + */ + private class TestStateMachine extends StateMachine { + @Override + public void configure(StateMachineExecutor executor) { + executor.register(TestCommand.class, this::testCommand); + executor.register(TestQuery.class, this::testQuery); + executor.register(EventCommand.class, this::eventCommand); + executor.register(TestExecute.class, this::testExecute); + } + + private long testCommand(Commit commit) { + return sequence.incrementAndGet(); + } + + private void eventCommand(Commit commit) { + commit.session().publish("hello", "world!"); + } + + private long testQuery(Commit commit) { + return sequence.incrementAndGet(); + } + + private void testExecute(Commit commit) { + executor.execute((Runnable) () -> { + threadAssertEquals(context.index(), 2L); + resume(); + }); + } + } + + /** + * Test command. + */ + private static class TestCommand implements Command { + } + + /** + * Event command. + */ + private static class EventCommand implements Command { + } + + /** + * Test query. + */ + private static class TestQuery implements Query { + } + + /** + * Test execute. + */ + private static class TestExecute implements Command { + } + +} diff --git a/server/src/test/java/io/atomix/copycat/server/state/ServerStateMachineTest.java b/server/src/test/java/io/atomix/copycat/server/state/ServerStateMachineTest.java deleted file mode 100644 index 47a9b373..00000000 --- a/server/src/test/java/io/atomix/copycat/server/state/ServerStateMachineTest.java +++ /dev/null @@ -1,525 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.atomix.copycat.server.state; - -import io.atomix.catalyst.concurrent.SingleThreadContext; -import io.atomix.catalyst.concurrent.ThreadContext; -import io.atomix.catalyst.serializer.Serializer; -import io.atomix.catalyst.transport.Address; -import io.atomix.catalyst.transport.Transport; -import io.atomix.catalyst.transport.local.LocalServerRegistry; -import io.atomix.catalyst.transport.local.LocalTransport; -import io.atomix.copycat.Command; -import io.atomix.copycat.Query; -import io.atomix.copycat.protocol.ClientRequestTypeResolver; -import io.atomix.copycat.protocol.ClientResponseTypeResolver; -import io.atomix.copycat.server.Commit; -import io.atomix.copycat.server.StateMachine; -import io.atomix.copycat.server.StateMachineExecutor; -import io.atomix.copycat.server.cluster.Member; -import io.atomix.copycat.server.storage.Storage; -import io.atomix.copycat.server.storage.StorageLevel; -import io.atomix.copycat.server.storage.entry.*; -import io.atomix.copycat.server.storage.util.StorageSerialization; -import io.atomix.copycat.server.util.ServerSerialization; -import io.atomix.copycat.session.Session; -import io.atomix.copycat.util.ProtocolSerialization; -import net.jodah.concurrentunit.ConcurrentTestCase; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.time.Instant; -import java.util.UUID; -import java.util.concurrent.atomic.AtomicLong; - -import static org.testng.Assert.*; - -/** - * Server state machine test. - * - * @author { - state = new ServerContext("test", member.type(), member.serverAddress(), member.clientAddress(), storage, serializer, TestStateMachine::new, new ConnectionManager(new LocalTransport(registry).client()), callerContext); - resume(); - }); - await(1000); - timestamp = System.currentTimeMillis(); - sequence = new AtomicLong(); - } - - /** - * Tests registering a session. - */ - public void testSessionRegisterKeepAlive() throws Throwable { - callerContext.execute(() -> { - - long index; - try (RegisterEntry entry = state.getLog().create(RegisterEntry.class)) { - entry.setTerm(1) - .setTimestamp(timestamp) - .setTimeout(500) - .setClient(UUID.randomUUID().toString()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNull(error); - resume(); - }); - }); - - await(); - - ServerSessionContext session = state.getStateMachine().executor().context().sessions().getSession(1); - assertNotNull(session); - assertEquals(session.id(), 1); - assertEquals(session.getTimestamp(), timestamp); - - callerContext.execute(() -> { - - long index; - try (KeepAliveEntry entry = state.getLog().create(KeepAliveEntry.class)) { - entry.setTerm(1) - .setSession(1) - .setTimestamp(timestamp + 1000) - .setCommandSequence(0) - .setEventIndex(0); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNull(error); - resume(); - }); - }); - - await(); - - assertEquals(session.getTimestamp(), timestamp + 1000); - } - - /** - * Tests resetting session timeouts when a new leader is elected. - */ - public void testSessionLeaderReset() throws Throwable { - callerContext.execute(() -> { - - long index; - try (RegisterEntry entry = state.getLog().create(RegisterEntry.class)) { - entry.setTerm(1) - .setTimestamp(timestamp) - .setTimeout(500) - .setClient(UUID.randomUUID().toString()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNull(error); - resume(); - }); - }); - - await(); - - ServerSessionContext session = state.getStateMachine().executor().context().sessions().getSession(1); - assertNotNull(session); - assertEquals(session.id(), 1); - assertEquals(session.getTimestamp(), timestamp); - - callerContext.execute(() -> { - - long index; - try (InitializeEntry entry = state.getLog().create(InitializeEntry.class)) { - entry.setTerm(1) - .setTimestamp(timestamp + 100); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNull(error); - resume(); - }); - }); - - await(); - - assertEquals(session.getTimestamp(), timestamp + 100); - } - - /** - * Tests expiring a session. - */ - public void testSessionSuspect() throws Throwable { - callerContext.execute(() -> { - - long index; - try (RegisterEntry entry = state.getLog().create(RegisterEntry.class)) { - entry.setTerm(1) - .setTimestamp(timestamp) - .setTimeout(500) - .setClient(UUID.randomUUID().toString()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNull(error); - resume(); - }); - }); - - await(); - - ServerSessionContext session = state.getStateMachine().executor().context().sessions().getSession(1); - assertNotNull(session); - assertEquals(session.id(), 1); - assertEquals(session.getTimestamp(), timestamp); - - callerContext.execute(() -> { - - long index; - try (KeepAliveEntry entry = state.getLog().create(KeepAliveEntry.class)) { - entry.setTerm(1) - .setSession(2) - .setTimestamp(timestamp + 1000) - .setCommandSequence(0) - .setEventIndex(0); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNotNull(error); - resume(); - }); - }); - - await(); - - assertTrue(session.state() == Session.State.UNSTABLE); - } - - /** - * Tests executing an asynchronous callback in the state machine. - */ - public void testExecute() throws Throwable { - callerContext.execute(() -> { - - long index; - try (RegisterEntry entry = state.getLog().create(RegisterEntry.class)) { - entry.setTerm(1) - .setTimestamp(timestamp) - .setTimeout(500) - .setClient(UUID.randomUUID().toString()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNull(error); - resume(); - }); - }); - - await(); - - ServerSessionContext session = state.getStateMachine().executor().context().sessions().getSession(1); - assertNotNull(session); - assertEquals(session.id(), 1); - assertEquals(session.getTimestamp(), timestamp); - assertEquals(session.getCommandSequence(), 0); - - callerContext.execute(() -> { - - long index; - try (CommandEntry entry = state.getLog().create(CommandEntry.class)) { - entry.setTerm(1) - .setSession(1) - .setSequence(1) - .setTimestamp(timestamp + 100) - .setCommand(new TestExecute()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNull(result.result); - threadAssertNull(error); - resume(); - }); - - }); - - await(1000, 2); - } - - /** - * Tests command sequencing. - */ - public void testCommandSequence() throws Throwable { - callerContext.execute(() -> { - - long index; - try (RegisterEntry entry = state.getLog().create(RegisterEntry.class)) { - entry.setTerm(1) - .setTimestamp(timestamp) - .setTimeout(500) - .setClient(UUID.randomUUID().toString()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNull(error); - resume(); - }); - }); - - await(); - - ServerSessionContext session = state.getStateMachine().executor().context().sessions().getSession(1); - assertNotNull(session); - assertEquals(session.id(), 1); - assertEquals(session.getTimestamp(), timestamp); - assertEquals(session.getCommandSequence(), 0); - - callerContext.execute(() -> { - - long index; - try (CommandEntry entry = state.getLog().create(CommandEntry.class)) { - entry.setTerm(1) - .setSession(1) - .setSequence(1) - .setTimestamp(timestamp + 100) - .setCommand(new TestCommand()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertEquals(result.result, 1L); - resume(); - }); - - }); - - await(); - - assertEquals(session.getCommandSequence(), 1); - assertEquals(session.getTimestamp(), timestamp + 100); - - callerContext.execute(() -> { - - long index; - try (CommandEntry entry = state.getLog().create(CommandEntry.class)) { - entry.setTerm(1) - .setSession(1) - .setSequence(2) - .setTimestamp(timestamp + 200) - .setCommand(new TestCommand()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertEquals(result.result, 2L); - resume(); - }); - - }); - - callerContext.execute(() -> { - - long index; - try (CommandEntry entry = state.getLog().create(CommandEntry.class)) { - entry.setTerm(1) - .setSession(1) - .setSequence(3) - .setTimestamp(timestamp + 300) - .setCommand(new TestCommand()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertEquals(result.result, 3L); - resume(); - }); - - }); - - await(1000, 2); - - assertEquals(session.getCommandSequence(), 3); - assertEquals(session.getTimestamp(), timestamp + 300); - } - - /** - * Tests serializing queries. - */ - public void testQuerySerialize() throws Throwable { - callerContext.execute(() -> { - - long index; - try (RegisterEntry entry = state.getLog().create(RegisterEntry.class)) { - entry.setTerm(1) - .setTimestamp(timestamp) - .setTimeout(500) - .setClient(UUID.randomUUID().toString()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertNull(error); - resume(); - }); - - threadAssertEquals(state.getStateMachine().getLastApplied(), index); - }); - - await(); - - ServerSessionContext session = state.getStateMachine().executor().context().sessions().getSession(1); - assertNotNull(session); - assertEquals(session.id(), 1); - assertEquals(session.getTimestamp(), timestamp); - assertEquals(session.getCommandSequence(), 0); - - callerContext.execute(() -> { - - QueryEntry entry = state.getLog().create(QueryEntry.class); - entry.setIndex(1) - .setTerm(1) - .setSession(1) - .setTimestamp(timestamp + 200) - .setSequence(0) - .setQuery(new TestQuery()); - - state.getStateMachine().apply(entry).whenComplete((result, error) -> { - threadAssertEquals(result.result, 1L); - resume(); - }); - }); - - callerContext.execute(() -> { - - long index; - try (CommandEntry entry = state.getLog().create(CommandEntry.class)) { - entry.setTerm(1) - .setSession(1) - .setSequence(1) - .setTimestamp(timestamp + 100) - .setCommand(new TestCommand()); - index = state.getLog().append(entry); - } - - state.getStateMachine().apply(index).whenComplete((result, error) -> { - threadAssertEquals(result.result, 2L); - resume(); - }); - - threadAssertEquals(state.getStateMachine().getLastApplied(), index); - }); - - await(1000, 2); - - assertEquals(session.getCommandSequence(), 1); - assertEquals(session.getTimestamp(), timestamp + 100); - } - - @AfterMethod - public void closeStateMachine() { - state.close(); - callerContext.close(); - } - - /** - * Test state machine. - */ - private class TestStateMachine extends StateMachine { - @Override - public void configure(StateMachineExecutor executor) { - executor.register(TestCommand.class, this::testCommand); - executor.register(TestQuery.class, this::testQuery); - executor.register(EventCommand.class, this::eventCommand); - executor.register(TestExecute.class, this::testExecute); - } - - private long testCommand(Commit commit) { - return sequence.incrementAndGet(); - } - - private void eventCommand(Commit commit) { - commit.session().publish("hello", "world!"); - } - - private long testQuery(Commit commit) { - return sequence.incrementAndGet(); - } - - private void testExecute(Commit commit) { - executor.execute((Runnable) () -> { - threadAssertEquals(context.index(), 2L); - resume(); - }); - } - } - - /** - * Test command. - */ - private static class TestCommand implements Command { - } - - /** - * Event command. - */ - private static class EventCommand implements Command { - } - - /** - * Test query. - */ - private static class TestQuery implements Query { - } - - /** - * Test execute. - */ - private static class TestExecute implements Command { - } - -} diff --git a/test/src/main/java/io/atomix/copycat/test/FuzzTest.java b/test/src/main/java/io/atomix/copycat/test/FuzzTest.java index 1531ad37..6c5df2ed 100644 --- a/test/src/main/java/io/atomix/copycat/test/FuzzTest.java +++ b/test/src/main/java/io/atomix/copycat/test/FuzzTest.java @@ -22,7 +22,10 @@ import io.atomix.catalyst.transport.netty.NettyTransport; import io.atomix.copycat.Command; import io.atomix.copycat.Query; -import io.atomix.copycat.client.*; +import io.atomix.copycat.client.CommunicationStrategies; +import io.atomix.copycat.client.ConnectionStrategies; +import io.atomix.copycat.client.CopycatClient; +import io.atomix.copycat.client.session.CopycatSession; import io.atomix.copycat.server.Commit; import io.atomix.copycat.server.CopycatServer; import io.atomix.copycat.server.Snapshottable; @@ -149,7 +152,8 @@ private void runFuzzTest() throws Exception { int clients = randomNumber(100) + 1; for (int i = 0; i < clients; i++) { - CopycatClient client = createClient(RecoveryStrategies.RECOVER); + CopycatClient client = createClient(); + CopycatSession session = createSession(client); final int clientId = i; client.context().schedule(Duration.ofMillis((100 * clients) + (randomNumber(50) - 25)), Duration.ofMillis((100 * clients) + (randomNumber(50) - 25)), () -> { @@ -157,7 +161,7 @@ private void runFuzzTest() throws Exception { int type = randomNumber(4); switch (type) { case 0: - client.submit(new Put(randomKey(), randomString(1024 * 16))).thenAccept(result -> { + session.submit(new Put(randomKey(), randomString(1024 * 16))).thenAccept(result -> { synchronized (lock) { if (result < lastLinearizableIndex) { System.out.println(result + " is less than last linearizable index " + lastLinearizableIndex); @@ -179,10 +183,10 @@ private void runFuzzTest() throws Exception { }); break; case 1: - client.submit(new Get(randomKey(), randomConsistency())); + session.submit(new Get(randomKey(), randomConsistency())); break; case 2: - client.submit(new Remove(randomKey())).thenAccept(result -> { + session.submit(new Remove(randomKey())).thenAccept(result -> { synchronized (lock) { if (result < lastLinearizableIndex) { System.out.println(result + " is less than last linearizable index " + lastLinearizableIndex); @@ -205,7 +209,7 @@ private void runFuzzTest() throws Exception { break; case 3: Query.ConsistencyLevel consistency = randomConsistency(); - client.submit(new Index(consistency)).thenAccept(result -> { + session.submit(new Index(consistency)).thenAccept(result -> { synchronized (lock) { switch (consistency) { case LINEARIZABLE: @@ -392,7 +396,7 @@ private CopycatServer createServer(Member member) { .withMinorCompactionInterval(Duration.ofSeconds(randomNumber(30) + 15)) .withMajorCompactionInterval(Duration.ofSeconds(randomNumber(60) + 60)) .build()) - .withStateMachine(FuzzStateMachine::new); + .addStateMachine("test", FuzzStateMachine::new); CopycatServer server = builder.build(); server.serializer().disableWhitelist(); @@ -403,12 +407,11 @@ private CopycatServer createServer(Member member) { /** * Creates a Copycat client. */ - private CopycatClient createClient(RecoveryStrategy strategy) throws Exception { + private CopycatClient createClient() throws Exception { CopycatClient client = CopycatClient.builder() .withTransport(new NettyTransport()) .withConnectionStrategy(ConnectionStrategies.FIBONACCI_BACKOFF) - .withRecoveryStrategy(strategy) - .withServerSelectionStrategy(ServerSelectionStrategies.values()[randomNumber(ServerSelectionStrategies.values().length)]) + .withServerSelectionStrategy(CommunicationStrategies.values()[randomNumber(CommunicationStrategies.values().length)]) .build(); client.serializer().disableWhitelist(); CountDownLatch latch = new CountDownLatch(1); @@ -418,6 +421,16 @@ private CopycatClient createClient(RecoveryStrategy strategy) throws Exception { return client; } + /** + * Creates a test session. + */ + private CopycatSession createSession(CopycatClient client) { + return client.sessionBuilder() + .withName("test") + .withType("test") + .build(); + } + /** * Fuzz test state machine. */ diff --git a/test/src/test/java/io/atomix/copycat/test/ClusterTest.java b/test/src/test/java/io/atomix/copycat/test/ClusterTest.java index 03e5f080..cfc19635 100644 --- a/test/src/test/java/io/atomix/copycat/test/ClusterTest.java +++ b/test/src/test/java/io/atomix/copycat/test/ClusterTest.java @@ -15,18 +15,16 @@ */ package io.atomix.copycat.test; +import io.atomix.catalyst.concurrent.Listener; import io.atomix.catalyst.transport.Address; import io.atomix.catalyst.transport.local.LocalServerRegistry; import io.atomix.catalyst.transport.local.LocalTransport; -import io.atomix.catalyst.concurrent.Listener; import io.atomix.copycat.Command; import io.atomix.copycat.Query; import io.atomix.copycat.client.ConnectionStrategies; import io.atomix.copycat.client.CopycatClient; -import io.atomix.copycat.client.DefaultCopycatClient; -import io.atomix.copycat.client.RecoveryStrategies; -import io.atomix.copycat.client.RecoveryStrategy; -import io.atomix.copycat.client.session.ClientSession; +import io.atomix.copycat.client.impl.DefaultCopycatClient; +import io.atomix.copycat.client.session.CopycatSession; import io.atomix.copycat.server.Commit; import io.atomix.copycat.server.CopycatServer; import io.atomix.copycat.server.Snapshottable; @@ -113,7 +111,8 @@ public void testReserveJoinLate() throws Throwable { private void testServerJoinLate(Member.Type type, CopycatServer.State state) throws Throwable { createServers(3); CopycatClient client = createClient(); - submit(client, 0, 1000); + CopycatSession session = createSession(client); + submit(session, 0, 1000); await(30000); CopycatServer joiner = createServer(nextMember(type)); joiner.onStateChange(s -> { @@ -127,11 +126,11 @@ private void testServerJoinLate(Member.Type type, CopycatServer.State state) thr /** * Submits a bunch of commands recursively. */ - private void submit(CopycatClient client, int count, int total) { + private void submit(CopycatSession session, int count, int total) { if (count < total) { - client.submit(new TestCommand()).whenComplete((result, error) -> { + session.submit(new TestCommand()).whenComplete((result, error) -> { threadAssertNull(error); - submit(client, count + 1, total); + submit(session, count + 1, total); }); } else { resume(); @@ -144,13 +143,14 @@ private void submit(CopycatClient client, int count, int total) { public void testCrashRecover() throws Throwable { List servers = createServers(3); CopycatClient client = createClient(); - submit(client, 0, 1000); + CopycatSession session = createSession(client); + submit(session, 0, 1000); await(30000); servers.get(0).shutdown().get(10, TimeUnit.SECONDS); CopycatServer server = createServer(members.get(0)); server.join(members.stream().map(Member::serverAddress).collect(Collectors.toList())).thenRun(this::resume); await(30000); - submit(client, 0, 1000); + submit(session, 0, 1000); await(30000); } @@ -431,7 +431,8 @@ private void testSubmitCommand(int nodes) throws Throwable { createServers(nodes); CopycatClient client = createClient(); - client.submit(new TestCommand()).thenAccept(result -> { + CopycatSession session = createSession(client); + session.submit(new TestCommand()).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -467,7 +468,8 @@ private void testSubmitCommand(int live, int total) throws Throwable { createServers(live, total); CopycatClient client = createClient(); - client.submit(new TestCommand()).thenAccept(result -> { + CopycatSession session = createSession(client); + session.submit(new TestCommand()).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -587,7 +589,8 @@ private void testSubmitQuery(int nodes, Query.ConsistencyLevel consistency) thro createServers(nodes); CopycatClient client = createClient(); - client.submit(new TestQuery(consistency)).thenAccept(result -> { + CopycatSession session = createSession(client); + session.submit(new TestQuery(consistency)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -640,13 +643,14 @@ private void testSequentialEvent(int nodes) throws Throwable { AtomicLong index = new AtomicLong(); CopycatClient client = createClient(); - client.onEvent("test", message -> { + CopycatSession session = createSession(client); + session.onEvent("test", message -> { threadAssertEquals(count.incrementAndGet(), 2L); threadAssertEquals(index.get(), message); resume(); }); - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); threadAssertEquals(count.incrementAndGet(), 1L); index.set(result); @@ -698,20 +702,21 @@ private void testEvents(int nodes) throws Throwable { createServers(nodes); CopycatClient client = createClient(); - client.onEvent("test", message -> { + CopycatSession session = createSession(client); + session.onEvent("test", message -> { threadAssertNotNull(message); resume(); }); - createClient().onEvent("test", message -> { + createSession(createClient()).onEvent("test", message -> { threadAssertNotNull(message); resume(); }); - createClient().onEvent("test", message -> { + createSession(createClient()).onEvent("test", message -> { threadAssertNotNull(message); resume(); }); - client.submit(new TestEvent(false)).thenAccept(result -> { + session.submit(new TestEvent(false)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -750,21 +755,22 @@ private void testSequenceOperations(int nodes, Query.ConsistencyLevel consistenc AtomicLong index = new AtomicLong(); CopycatClient client = createClient(); - client.onEvent("test", message -> { + CopycatSession session = createSession(client); + session.onEvent("test", message -> { threadAssertEquals(counter.incrementAndGet(), 3); threadAssertTrue(message >= index.get()); index.set(message); resume(); }); - client.submit(new TestCommand()).thenAccept(result -> { + session.submit(new TestCommand()).thenAccept(result -> { threadAssertNotNull(result); threadAssertEquals(counter.incrementAndGet(), 1); threadAssertTrue(index.compareAndSet(0, result)); resume(); }); - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); threadAssertEquals(counter.incrementAndGet(), 2); threadAssertTrue(result > index.get()); @@ -772,7 +778,7 @@ private void testSequenceOperations(int nodes, Query.ConsistencyLevel consistenc resume(); }); - client.submit(new TestQuery(consistency)).thenAccept(result -> { + session.submit(new TestQuery(consistency)).thenAccept(result -> { threadAssertNotNull(result); threadAssertEquals(counter.incrementAndGet(), 4); long i = index.get(); @@ -792,18 +798,19 @@ public void testBlockOnEvent() throws Throwable { AtomicLong index = new AtomicLong(); CopycatClient client = createClient(); + CopycatSession session = createSession(client); - client.onEvent("test", event -> { + session.onEvent("test", event -> { threadAssertEquals(index.get(), event); try { - threadAssertTrue(index.get() <= client.submit(new TestQuery(Query.ConsistencyLevel.LINEARIZABLE)).get(10, TimeUnit.SECONDS)); + threadAssertTrue(index.get() <= session.submit(new TestQuery(Query.ConsistencyLevel.LINEARIZABLE)).get(5, TimeUnit.SECONDS)); } catch (InterruptedException | TimeoutException | ExecutionException e) { threadFail(e); } resume(); }); - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); index.compareAndSet(0, result); resume(); @@ -826,13 +833,14 @@ private void testManyEvents(int nodes) throws Throwable { createServers(nodes); CopycatClient client = createClient(); - client.onEvent("test", message -> { + CopycatSession session = createSession(client); + session.onEvent("test", message -> { threadAssertNotNull(message); resume(); }); for (int i = 0 ; i < 10; i++) { - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -862,13 +870,14 @@ private void testManyEventsAfterLeaderShutdown(int nodes) throws Throwable { List servers = createServers(nodes); CopycatClient client = createClient(); - client.onEvent("test", message -> { + CopycatSession session = createSession(client); + session.onEvent("test", message -> { threadAssertNotNull(message); resume(); }); for (int i = 0; i < 10; i++) { - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -880,7 +889,7 @@ private void testManyEventsAfterLeaderShutdown(int nodes) throws Throwable { leader.shutdown().get(10, TimeUnit.SECONDS); for (int i = 0; i < 10; i++) { - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -910,13 +919,14 @@ private void testEventsAfterFollowerKill(int nodes) throws Throwable { List servers = createServers(nodes); CopycatClient client = createClient(); - client.onEvent("test", message -> { + CopycatSession session = createSession(client); + session.onEvent("test", message -> { threadAssertNotNull(message); resume(); }); for (int i = 0 ; i < 10; i++) { - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -924,7 +934,7 @@ private void testEventsAfterFollowerKill(int nodes) throws Throwable { await(30000, 2); } - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -935,7 +945,7 @@ private void testEventsAfterFollowerKill(int nodes) throws Throwable { await(30000, 2); for (int i = 0 ; i < 10; i++) { - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -958,13 +968,14 @@ private void testEventsAfterLeaderKill(int nodes) throws Throwable { List servers = createServers(nodes); CopycatClient client = createClient(); - client.onEvent("test", message -> { + CopycatSession session = createSession(client); + session.onEvent("test", message -> { threadAssertNotNull(message); resume(); }); for (int i = 0 ; i < 10; i++) { - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -972,7 +983,7 @@ private void testEventsAfterLeaderKill(int nodes) throws Throwable { await(30000, 2); } - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -983,7 +994,7 @@ private void testEventsAfterLeaderKill(int nodes) throws Throwable { await(30000, 2); for (int i = 0 ; i < 10; i++) { - client.submit(new TestEvent(true)).thenAccept(result -> { + session.submit(new TestEvent(true)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -1006,23 +1017,24 @@ private void testManySessionsManyEvents(int nodes) throws Throwable { createServers(nodes); CopycatClient client = createClient(); - client.onEvent("test", message -> { + CopycatSession session = createSession(client); + session.onEvent("test", message -> { threadAssertNotNull(message); resume(); }); - createClient().onEvent("test", message -> { + createSession(createClient()).onEvent("test", message -> { threadAssertNotNull(message); resume(); }); - createClient().onEvent("test", message -> { + createSession(createClient()).onEvent("test", message -> { threadAssertNotNull(message); resume(); }); for (int i = 0; i < 10; i++) { - client.submit(new TestEvent(false)).thenAccept(result -> { + session.submit(new TestEvent(false)).thenAccept(result -> { threadAssertNotNull(result); resume(); }); @@ -1057,7 +1069,7 @@ public void testFiveNodeExpireEvent() throws Throwable { */ public void testStateTransitionWithRecovery() throws Throwable { createServers(3); - final CopycatClient client = createClient(RecoveryStrategies.RECOVER); + final CopycatClient client = createClient(); final AtomicReference prev = new AtomicReference<>(CopycatClient.State.CONNECTED); Listener stateListener = client.onStateChange(s -> { @@ -1076,7 +1088,7 @@ public void testStateTransitionWithRecovery() throws Throwable { threadFail("State not allowed"); } }); - ((ClientSession) client.session()).expire().thenAccept(v -> resume()); + ((DefaultCopycatClient) client).kill().thenAccept(v -> resume()); await(5000, 3); stateListener.close(); } @@ -1088,9 +1100,11 @@ private void testSessionExpire(int nodes) throws Throwable { createServers(nodes); CopycatClient client1 = createClient(); + CopycatSession session1 = createSession(client1); CopycatClient client2 = createClient(); - client1.onEvent("expired", this::resume); - client1.submit(new TestExpire()).thenRun(this::resume); + createSession(client2); + session1.onEvent("expired", this::resume); + session1.submit(new TestExpire()).thenRun(this::resume); ((DefaultCopycatClient) client2).kill().thenRun(this::resume); await(Duration.ofSeconds(10).toMillis(), 3); } @@ -1123,11 +1137,12 @@ private void testSessionClose(int nodes) throws Throwable { createServers(nodes); CopycatClient client1 = createClient(); + CopycatSession session1 = createSession(client1); CopycatClient client2 = createClient(); - client1.submit(new TestClose()).thenRun(this::resume); + session1.submit(new TestClose()).thenRun(this::resume); await(Duration.ofSeconds(10).toMillis(), 1); - client1.onEvent("closed", this::resume); - client2.close().thenRun(this::resume); + session1.onEvent("closed", this::resume); + createSession(client2).close().thenRun(this::resume); await(Duration.ofSeconds(10).toMillis(), 2); } @@ -1195,7 +1210,7 @@ private CopycatServer createServer(Member member) { .withMaxSegmentSize(1024 * 1024) .withCompactionThreads(1) .build()) - .withStateMachine(TestStateMachine::new); + .addStateMachine("test", TestStateMachine::new); CopycatServer server = builder.build(); server.serializer().disableWhitelist(); @@ -1207,17 +1222,9 @@ private CopycatServer createServer(Member member) { * Creates a Copycat client. */ private CopycatClient createClient() throws Throwable { - return createClient(RecoveryStrategies.CLOSE); - } - - /** - * Creates a Copycat client. - */ - private CopycatClient createClient(RecoveryStrategy strategy) throws Throwable { CopycatClient client = CopycatClient.builder() .withTransport(new LocalTransport(registry)) .withConnectionStrategy(ConnectionStrategies.FIBONACCI_BACKOFF) - .withRecoveryStrategy(strategy) .build(); client.serializer().disableWhitelist(); client.connect(members.stream().map(Member::clientAddress).collect(Collectors.toList())).thenRun(this::resume); @@ -1226,6 +1233,16 @@ private CopycatClient createClient(RecoveryStrategy strategy) throws Throwable { return client; } + /** + * Creates a test session. + */ + private CopycatSession createSession(CopycatClient client) { + return client.sessionBuilder() + .withName("test") + .withType("test") + .build(); + } + @BeforeMethod @AfterMethod public void clearTests() throws Exception { From c9727f9cd44c5a144a9b5cc88b78c04773388575 Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Mon, 29 May 2017 15:42:27 -0700 Subject: [PATCH 02/14] Fix compile errors. --- client/pom.xml | 2 +- .../client/session/CopycatSession.java | 8 ++++++ .../session/impl/DefaultCopycatSession.java | 9 +++++- examples/pom.xml | 2 +- examples/value-client/pom.xml | 2 +- .../copycat/examples/ValueClientExample.java | 20 +++++++------ examples/value-state-machine/pom.xml | 2 +- .../examples/ValueStateMachineExample.java | 2 +- pom.xml | 4 +-- protocol/pom.xml | 2 +- server/pom.xml | 2 +- test/pom.xml | 2 +- .../atomix/copycat/test/PerformanceTest.java | 28 +++++++++++-------- 13 files changed, 54 insertions(+), 31 deletions(-) diff --git a/client/pom.xml b/client/pom.xml index 1644a154..f332f403 100644 --- a/client/pom.xml +++ b/client/pom.xml @@ -19,7 +19,7 @@ io.atomix.copycat copycat-parent - 1.2.9-SNAPSHOT + 2.0.0-SNAPSHOT bundle diff --git a/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java b/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java index 83576ab6..7f161913 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java +++ b/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java @@ -16,6 +16,7 @@ package io.atomix.copycat.client.session; import io.atomix.catalyst.concurrent.Listener; +import io.atomix.catalyst.concurrent.ThreadContext; import io.atomix.catalyst.util.Assert; import io.atomix.catalyst.util.Builder; import io.atomix.copycat.Command; @@ -54,6 +55,13 @@ public interface CopycatSession { */ String type(); + /** + * Returns the session thread context. + * + * @return The session thread context. + */ + ThreadContext context(); + /** * Submits an operation to the Copycat cluster. *

diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java b/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java index 232459a3..31190d17 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java @@ -33,7 +33,7 @@ * The client session is responsible for maintaining a client's connection to a Copycat cluster and coordinating * the submission of {@link Command commands} and {@link Query queries} to various nodes in the cluster. Client * sessions are single-use objects that represent the context within which a cluster can guarantee linearizable - * semantics for state machine operations. When a session is {@link #register() opened}, the session will register + * semantics for state machine operations. When a session is opened, the session will register * itself with the cluster by attempting to contact each of the known servers. Once the session has been successfully * registered, kee-alive requests will be periodically sent to keep the session alive. *

@@ -51,6 +51,7 @@ public class DefaultCopycatSession implements CopycatSession { private final CopycatSessionManager sessionManager; private final CopycatSessionListener sessionListener; private final CopycatSessionSubmitter sessionSubmitter; + private final ThreadContext context; public DefaultCopycatSession(CopycatSessionState state, CopycatConnection leaderConnection, CopycatConnection sessionConnection, ThreadContext threadContext, CopycatSessionManager sessionManager) { this.state = Assert.notNull(state, "state"); @@ -58,6 +59,7 @@ public DefaultCopycatSession(CopycatSessionState state, CopycatConnection leader CopycatSessionSequencer sequencer = new CopycatSessionSequencer(state); this.sessionListener = new CopycatSessionListener(sessionConnection, state, sequencer, threadContext); this.sessionSubmitter = new CopycatSessionSubmitter(leaderConnection, sessionConnection, state, sequencer, sessionManager, threadContext); + this.context = threadContext; } @Override @@ -75,6 +77,11 @@ public String type() { return state.getSessionType(); } + @Override + public ThreadContext context() { + return context; + } + /** * Submits an operation to the session. * diff --git a/examples/pom.xml b/examples/pom.xml index b0389beb..80cd0cc1 100644 --- a/examples/pom.xml +++ b/examples/pom.xml @@ -19,7 +19,7 @@ io.atomix.copycat copycat-parent - 1.2.9-SNAPSHOT + 2.0.0-SNAPSHOT copycat-examples-parent diff --git a/examples/value-client/pom.xml b/examples/value-client/pom.xml index 5ad01b6a..16738363 100644 --- a/examples/value-client/pom.xml +++ b/examples/value-client/pom.xml @@ -19,7 +19,7 @@ io.atomix.copycat copycat-examples-parent - 1.2.9-SNAPSHOT + 2.0.0-SNAPSHOT copycat-value-client-example diff --git a/examples/value-client/src/main/java/io/atomix/copycat/examples/ValueClientExample.java b/examples/value-client/src/main/java/io/atomix/copycat/examples/ValueClientExample.java index 3554c841..14141f1c 100644 --- a/examples/value-client/src/main/java/io/atomix/copycat/examples/ValueClientExample.java +++ b/examples/value-client/src/main/java/io/atomix/copycat/examples/ValueClientExample.java @@ -17,10 +17,10 @@ import io.atomix.catalyst.transport.Address; import io.atomix.catalyst.transport.netty.NettyTransport; +import io.atomix.copycat.client.CommunicationStrategies; import io.atomix.copycat.client.ConnectionStrategies; import io.atomix.copycat.client.CopycatClient; -import io.atomix.copycat.client.RecoveryStrategies; -import io.atomix.copycat.client.ServerSelectionStrategies; +import io.atomix.copycat.client.session.CopycatSession; import java.time.Duration; import java.util.ArrayList; @@ -57,8 +57,7 @@ public static void main(String[] args) throws Exception { CopycatClient client = CopycatClient.builder() .withTransport(new NettyTransport()) .withConnectionStrategy(ConnectionStrategies.FIBONACCI_BACKOFF) - .withRecoveryStrategy(RecoveryStrategies.RECOVER) - .withServerSelectionStrategy(ServerSelectionStrategies.LEADER) + .withServerSelectionStrategy(CommunicationStrategies.LEADER) .withSessionTimeout(Duration.ofSeconds(15)) .build(); @@ -68,7 +67,12 @@ public static void main(String[] args) throws Exception { client.connect(members).join(); - recursiveSet(client); + CopycatSession session = client.sessionBuilder() + .withType("value") + .withName("test") + .build(); + + recursiveSet(session); while (client.state() != CopycatClient.State.CLOSED) { try { @@ -82,9 +86,9 @@ public static void main(String[] args) throws Exception { /** * Recursively sets state machine values. */ - private static void recursiveSet(CopycatClient client) { - client.submit(new SetCommand(UUID.randomUUID().toString())).whenComplete((result, error) -> { - client.context().schedule(Duration.ofSeconds(5), () -> recursiveSet(client)); + private static void recursiveSet(CopycatSession session) { + session.submit(new SetCommand(UUID.randomUUID().toString())).whenComplete((result, error) -> { + session.context().schedule(Duration.ofSeconds(5), () -> recursiveSet(session)); }); } diff --git a/examples/value-state-machine/pom.xml b/examples/value-state-machine/pom.xml index 2766881a..dda31ff3 100644 --- a/examples/value-state-machine/pom.xml +++ b/examples/value-state-machine/pom.xml @@ -19,7 +19,7 @@ io.atomix.copycat copycat-examples-parent - 1.2.9-SNAPSHOT + 2.0.0-SNAPSHOT copycat-value-state-machine-example diff --git a/examples/value-state-machine/src/main/java/io/atomix/copycat/examples/ValueStateMachineExample.java b/examples/value-state-machine/src/main/java/io/atomix/copycat/examples/ValueStateMachineExample.java index f6782ef6..3c102850 100644 --- a/examples/value-state-machine/src/main/java/io/atomix/copycat/examples/ValueStateMachineExample.java +++ b/examples/value-state-machine/src/main/java/io/atomix/copycat/examples/ValueStateMachineExample.java @@ -57,7 +57,7 @@ public static void main(String[] args) throws Exception { } CopycatServer server = CopycatServer.builder(address) - .withStateMachine(ValueStateMachine::new) + .addStateMachine("value", ValueStateMachine::new) .withTransport(new NettyTransport()) .withStorage(Storage.builder() .withDirectory(args[0]) diff --git a/pom.xml b/pom.xml index 642b1c42..5a87b917 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ io.atomix.copycat copycat-parent - 1.2.9-SNAPSHOT + 2.0.0-SNAPSHOT pom Copycat Parent Pom Feature complete implementation of the Raft consensus algorithm. @@ -22,7 +22,7 @@ 1.8 1.7.7 1.1.2 - 1.2.1 + 1.2.2-SNAPSHOT 2.2.1 3.0 diff --git a/protocol/pom.xml b/protocol/pom.xml index f7bf3f4b..ca2219f7 100644 --- a/protocol/pom.xml +++ b/protocol/pom.xml @@ -19,7 +19,7 @@ io.atomix.copycat copycat-parent - 1.2.9-SNAPSHOT + 2.0.0-SNAPSHOT bundle diff --git a/server/pom.xml b/server/pom.xml index 4011a8e1..ecb55912 100644 --- a/server/pom.xml +++ b/server/pom.xml @@ -19,7 +19,7 @@ io.atomix.copycat copycat-parent - 1.2.9-SNAPSHOT + 2.0.0-SNAPSHOT bundle diff --git a/test/pom.xml b/test/pom.xml index 1d511dc0..0979b1be 100644 --- a/test/pom.xml +++ b/test/pom.xml @@ -19,7 +19,7 @@ io.atomix.copycat copycat-parent - 1.2.9-SNAPSHOT + 2.0.0-SNAPSHOT copycat-test diff --git a/test/src/main/java/io/atomix/copycat/test/PerformanceTest.java b/test/src/main/java/io/atomix/copycat/test/PerformanceTest.java index ce31cb1c..c5cb0cab 100644 --- a/test/src/main/java/io/atomix/copycat/test/PerformanceTest.java +++ b/test/src/main/java/io/atomix/copycat/test/PerformanceTest.java @@ -21,6 +21,7 @@ import io.atomix.copycat.Command; import io.atomix.copycat.Query; import io.atomix.copycat.client.*; +import io.atomix.copycat.client.session.CopycatSession; import io.atomix.copycat.server.Commit; import io.atomix.copycat.server.CopycatServer; import io.atomix.copycat.server.Snapshottable; @@ -68,7 +69,7 @@ public static void main(String[] args) { private static final int NUM_CLIENTS = 5; private static final Query.ConsistencyLevel QUERY_CONSISTENCY = Query.ConsistencyLevel.LINEARIZABLE; - private static final ServerSelectionStrategy SERVER_SELECTION_STRATEGY = ServerSelectionStrategies.ANY; + private static final CommunicationStrategy SERVER_SELECTION_STRATEGY = CommunicationStrategies.ANY; private int port = 5000; private List members = new ArrayList<>(); @@ -121,13 +122,17 @@ private long runIteration() throws Exception { CopycatClient[] clients = new CopycatClient[NUM_CLIENTS]; for (int i = 0; i < NUM_CLIENTS; i++) { CompletableFuture future = new CompletableFuture<>(); - clients[i] = createClient(RecoveryStrategies.RECOVER); + clients[i] = createClient(); futures[i] = future; } long startTime = System.currentTimeMillis(); for (int i = 0; i < clients.length; i++) { - runClient(clients[i], futures[i]); + CopycatSession session = clients[i].sessionBuilder() + .withType("test") + .withName("test") + .build(); + runSession(session, futures[i]); } CompletableFuture.allOf(futures).join(); long endTime = System.currentTimeMillis(); @@ -142,25 +147,25 @@ private long runIteration() throws Exception { } /** - * Runs operations for a single client. + * Runs operations for a single session. */ - private void runClient(CopycatClient client, CompletableFuture future) { + private void runSession(CopycatSession session, CompletableFuture future) { int count = totalOperations.incrementAndGet(); if (count > TOTAL_OPERATIONS) { future.complete(null); } else if (count % 10 < WRITE_RATIO) { - client.submit(new Put(randomKey(), UUID.randomUUID().toString())).whenComplete((result, error) -> { + session.submit(new Put(randomKey(), UUID.randomUUID().toString())).whenComplete((result, error) -> { if (error == null) { writeCount.incrementAndGet(); } - runClient(client, future); + runSession(session, future); }); } else { - client.submit(new Get(randomKey(), QUERY_CONSISTENCY)).whenComplete((result, error) -> { + session.submit(new Get(randomKey(), QUERY_CONSISTENCY)).whenComplete((result, error) -> { if (error == null) { readCount.incrementAndGet(); } - runClient(client, future); + runSession(session, future); }); } } @@ -277,7 +282,7 @@ private CopycatServer createServer(Member member) { .withDirectory(new File(String.format("target/performance-logs/%d", member.address().hashCode()))) .withCompactionThreads(1) .build()) - .withStateMachine(PerformanceStateMachine::new); + .addStateMachine("test", PerformanceStateMachine::new); CopycatServer server = builder.build(); server.serializer().disableWhitelist(); @@ -288,11 +293,10 @@ private CopycatServer createServer(Member member) { /** * Creates a Copycat client. */ - private CopycatClient createClient(RecoveryStrategy strategy) throws Exception { + private CopycatClient createClient() throws Exception { CopycatClient client = CopycatClient.builder() .withTransport(new NettyTransport()) .withConnectionStrategy(ConnectionStrategies.FIBONACCI_BACKOFF) - .withRecoveryStrategy(strategy) .withServerSelectionStrategy(SERVER_SELECTION_STRATEGY) .build(); client.serializer().disableWhitelist(); From 9e2765072a89c32c6d8591032ef5ef94e9ed438b Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Mon, 29 May 2017 16:01:47 -0700 Subject: [PATCH 03/14] Use lastApplied index for queries. --- .../copycat/server/state/ServerCommit.java | 4 +-- .../server/state/ServerSessionContext.java | 32 +++++-------------- .../state/ServerStateMachineExecutor.java | 4 +-- .../state/ServerStateMachineManager.java | 2 +- .../copycat/server/storage/LogCleaner.java | 17 +++------- 5 files changed, 16 insertions(+), 43 deletions(-) diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerCommit.java b/server/src/main/java/io/atomix/copycat/server/state/ServerCommit.java index a907eafe..4d049442 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerCommit.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerCommit.java @@ -119,9 +119,7 @@ public void close() { * Cleans up the commit. */ private void cleanup() { - if (operation instanceof Command) { - cleaner.clean(index); - } + cleaner.clean(index); } @Override diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java b/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java index ed01caa5..15804d86 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java @@ -146,16 +146,14 @@ boolean setRequestSequence(long requestSequence) { * Resets the current request sequence number. * * @param requestSequence The request sequence number. - * @return The server session context. */ - ServerSessionContext resetRequestSequence(long requestSequence) { + void resetRequestSequence(long requestSequence) { // If the request sequence number is less than the applied sequence number, update the request // sequence number. This is necessary to ensure that if the local server is a follower that is // later elected leader, its sequences are consistent for commands. if (requestSequence > this.requestSequence) { this.requestSequence = requestSequence; } - return this; } /** @@ -180,9 +178,8 @@ long nextCommandSequence() { * Sets the session operation sequence number. * * @param sequence The session operation sequence number. - * @return The server session. */ - ServerSessionContext setCommandSequence(long sequence) { + void setCommandSequence(long sequence) { // For each increment of the sequence number, trigger query callbacks that are dependent on the specific sequence. for (long i = commandSequence + 1; i <= sequence; i++) { commandSequence = i; @@ -193,7 +190,6 @@ ServerSessionContext setCommandSequence(long sequence) { } } } - return this; } /** @@ -209,9 +205,8 @@ long getLastApplied() { * Sets the session index. * * @param index The session index. - * @return The server session. */ - ServerSessionContext setLastApplied(long index) { + void setLastApplied(long index) { // Query callbacks for this session are added to the indexQueries map to be executed once the required index // for the query is reached. For each increment of the index, trigger query callbacks that are dependent // on the specific index. @@ -224,8 +219,6 @@ ServerSessionContext setLastApplied(long index) { } } } - - return this; } /** @@ -233,13 +226,11 @@ ServerSessionContext setLastApplied(long index) { * * @param sequence The session sequence number at which to execute the query. * @param query The query to execute. - * @return The server session. */ - ServerSessionContext registerSequenceQuery(long sequence, Runnable query) { + void registerSequenceQuery(long sequence, Runnable query) { // Add a query to be run once the session's sequence number reaches the given sequence number. List queries = this.sequenceQueries.computeIfAbsent(sequence, v -> new LinkedList()); queries.add(query); - return this; } /** @@ -247,13 +238,11 @@ ServerSessionContext registerSequenceQuery(long sequence, Runnable query) { * * @param index The state machine index at which to execute the query. * @param query The query to execute. - * @return The server session. */ - ServerSessionContext registerIndexQuery(long index, Runnable query) { + void registerIndexQuery(long index, Runnable query) { // Add a query to be run once the session's index reaches the given index. List queries = this.indexQueries.computeIfAbsent(index, v -> new LinkedList<>()); queries.add(query); - return this; } /** @@ -265,11 +254,9 @@ ServerSessionContext registerIndexQuery(long index, Runnable query) { * * @param sequence The result sequence number. * @param result The result. - * @return The server session. */ - ServerSessionContext registerResult(long sequence, OperationResult result) { + void registerResult(long sequence, OperationResult result) { results.put(sequence, result); - return this; } /** @@ -280,16 +267,14 @@ ServerSessionContext registerResult(long sequence, OperationResult result) { * from memory as well. * * @param sequence The sequence to clear. - * @return The server session. */ - ServerSessionContext clearResults(long sequence) { + void clearResults(long sequence) { if (sequence > commandLowWaterMark) { for (long i = commandLowWaterMark + 1; i <= sequence; i++) { results.remove(i); commandLowWaterMark = i; } } - return this; } /** @@ -305,9 +290,8 @@ OperationResult getResult(long sequence) { /** * Sets the session connection. */ - ServerSessionContext setConnection(Connection connection) { + void setConnection(Connection connection) { this.connection = connection; - return this; } /** diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineExecutor.java b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineExecutor.java index a7ad576c..050a7874 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineExecutor.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineExecutor.java @@ -326,7 +326,7 @@ private void sequenceCommand(long sequence, ServerSessionContext session, Comple * Applies the given commit to the state machine. */ private void applyCommand(long index, long sequence, long timestamp, Command command, ServerSessionContext session, CompletableFuture future) { - ServerCommit commit = new ServerCommit(index, command, session, timestamp, new LogCleaner(server.getLog())); + ServerCommit commit = new ServerCommit(index, command, session, timestamp, server.getLog()::release); context.update(commit.index(), commit.time(), ServerStateMachineContext.Type.COMMAND); long eventIndex = session.getEventIndex(); @@ -406,7 +406,7 @@ private void indexQuery(long index, long timestamp, ServerSessionContext session * Applies a query to the state machine. */ private void applyQuery(long index, long timestamp, ServerSessionContext session, Query query, CompletableFuture future) { - ServerCommit commit = new ServerCommit(index, query, session, timestamp, new LogCleaner(server.getLog())); + ServerCommit commit = new ServerCommit(session.getLastApplied(), query, session, timestamp, i -> {}); context.update(commit.index(), commit.time(), ServerStateMachineContext.Type.QUERY); long eventIndex = session.getEventIndex(); diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java index 5cbc8ae6..4ecb2405 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java @@ -282,7 +282,7 @@ private CompletableFuture apply(RegisterEntry entry) { long index = entry.getIndex(); long timestamp = updateTimestamp(entry); - ClientContext client = new ClientContext(entry.getIndex(), entry.getTimeout(), new LogCleaner(log)); + ClientContext client = new ClientContext(entry.getIndex(), entry.getTimeout(), log::release); clientManager.registerClient(client); // Update the session timestamp *after* executing any scheduled operations. The executor's timestamp diff --git a/server/src/main/java/io/atomix/copycat/server/storage/LogCleaner.java b/server/src/main/java/io/atomix/copycat/server/storage/LogCleaner.java index 18e3bedb..2fa02138 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/LogCleaner.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/LogCleaner.java @@ -18,23 +18,14 @@ /** * Log cleaner. */ -public class LogCleaner { - private final Log log; - - public LogCleaner(Log log) { - this.log = log; - } +@FunctionalInterface +public interface LogCleaner { /** * Cleans the entry at the given index. * * @param index The index at which to clean the entry. */ - public void clean(long index) { - try { - log.release(index); - } catch (IllegalStateException e) { - // Do nothing. - } - } + void clean(long index); + } From ee354623c8b80321887a01a2bb2138e79e3efc3a Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Mon, 29 May 2017 16:42:28 -0700 Subject: [PATCH 04/14] Ensure session connection reconnects after failures. --- .../impl/CopycatSessionConnection.java | 35 ++++++++++++++++--- .../session/impl/CopycatSessionManager.java | 2 +- 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionConnection.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionConnection.java index 043dd577..ddfd8f16 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionConnection.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionConnection.java @@ -15,6 +15,7 @@ */ package io.atomix.copycat.client.session.impl; +import io.atomix.catalyst.concurrent.ThreadContext; import io.atomix.catalyst.transport.Address; import io.atomix.catalyst.transport.Connection; import io.atomix.copycat.client.util.AddressSelector; @@ -25,6 +26,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.time.Duration; import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.function.Function; @@ -34,13 +36,20 @@ */ public class CopycatSessionConnection extends CopycatConnection { private static final Logger LOGGER = LoggerFactory.getLogger(CopycatSessionConnection.class); + + private static final long BASE_RECONNECT_INTERVAL = 10; + private static final long MAX_RECONNECT_INTERVAL = 1000; + private final CopycatSessionState state; private final String sessionString; + private final ThreadContext context; + private long reconnectInterval; - public CopycatSessionConnection(CopycatSessionState state, ClientConnectionManager connections, AddressSelector selector) { + public CopycatSessionConnection(CopycatSessionState state, ClientConnectionManager connections, AddressSelector selector, ThreadContext context) { super(connections, selector); this.state = state; this.sessionString = String.valueOf(state.getSessionId()); + this.context = context; } @Override @@ -53,6 +62,22 @@ protected Logger logger() { return LOGGER; } + /** + * Reconnects to the cluster. + */ + private void reconnect() { + if (open) { + reset().connect().whenComplete((connection, error) -> { + if (connection == null || error != null) { + reconnectInterval = Math.max(reconnectInterval * 2, MAX_RECONNECT_INTERVAL); + context.schedule(Duration.ofMillis(reconnectInterval), this::reconnect); + } else { + reconnectInterval = BASE_RECONNECT_INTERVAL; + } + }); + } + } + @Override @SuppressWarnings("unchecked") protected void setupConnection(Address address, Connection connection, CompletableFuture future) { @@ -61,17 +86,17 @@ protected void setupConnection(Address address, Connection connection, Completab this.connection = connection; connection.onClose(c -> { - if (c.equals(this.connection)) { + if (c == this.connection) { logger().debug("{} - Connection closed", name()); this.connection = null; - connect(); + reconnect(); } }); connection.onException(c -> { - if (c.equals(this.connection)) { + if (c == this.connection) { logger().debug("{} - Connection lost", name()); this.connection = null; - connect(); + reconnect(); } }); diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java index f843119d..3daa4bc1 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java @@ -165,7 +165,7 @@ public CompletableFuture openSession(String name, String type, C CopycatSessionState state = new CopycatSessionState(response.session(), name, type); sessions.put(state.getSessionId(), state); CopycatConnection leaderConnection = new CopycatLeaderConnection(state, connectionManager, selectorManager.createSelector(CommunicationStrategies.LEADER)); - CopycatConnection sessionConnection = new CopycatSessionConnection(state, connectionManager, selectorManager.createSelector(communicationStrategy)); + CopycatConnection sessionConnection = new CopycatSessionConnection(state, connectionManager, selectorManager.createSelector(communicationStrategy), threadContext); leaderConnection.open().thenCompose(v -> sessionConnection.open()).whenComplete((connectResult, connectError) -> { if (connectError == null) { future.complete(new DefaultCopycatSession(state, leaderConnection, sessionConnection, threadContext, this)); From de097b77fa487d79eba619b241981500e085d74e Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Mon, 29 May 2017 17:07:20 -0700 Subject: [PATCH 05/14] Bump Catalyst dependency version. --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 5a87b917..7ca3910d 100644 --- a/pom.xml +++ b/pom.xml @@ -22,7 +22,7 @@ 1.8 1.7.7 1.1.2 - 1.2.2-SNAPSHOT + 1.3.0-SNAPSHOT 2.2.1 3.0 From 95f3615278524903acc25c6a0a2a1ff10af739cc Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Mon, 29 May 2017 19:15:12 -0700 Subject: [PATCH 06/14] Add state change events to CopycatSession. --- .../client/session/CopycatSession.java | 34 +++++++++++++--- .../session/impl/CopycatSessionManager.java | 4 +- .../session/impl/CopycatSessionState.java | 39 ++++++++++++++++++- .../session/impl/DefaultCopycatSession.java | 21 ++++++---- .../atomix/copycat/server/StateMachine.java | 3 +- 5 files changed, 84 insertions(+), 17 deletions(-) diff --git a/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java b/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java index 7f161913..aedc2b7c 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java +++ b/client/src/main/java/io/atomix/copycat/client/session/CopycatSession.java @@ -18,7 +18,6 @@ import io.atomix.catalyst.concurrent.Listener; import io.atomix.catalyst.concurrent.ThreadContext; import io.atomix.catalyst.util.Assert; -import io.atomix.catalyst.util.Builder; import io.atomix.copycat.Command; import io.atomix.copycat.Operation; import io.atomix.copycat.Query; @@ -35,11 +34,21 @@ public interface CopycatSession { /** - * Returns the unique session identifier. - * - * @return The session identifier. + * Indicates the session's state. */ - long id(); + enum State { + + /** + * Indicates that the session is open. + */ + OPEN, + + /** + * Indicates that the session is closed. + */ + CLOSED + + } /** * Returns the client proxy name. @@ -55,6 +64,21 @@ public interface CopycatSession { */ String type(); + /** + * Returns the session state. + * + * @return The session state. + */ + State state(); + + /** + * Registers a session state change listener. + * + * @param callback The callback to call when the session state changes. + * @return The session state change listener context. + */ + Listener onStateChange(Consumer callback); + /** * Returns the session thread context. * diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java index 3daa4bc1..aa2e0fed 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java @@ -115,10 +115,12 @@ private void setState(State state) { } break; case EXPIRED: - clientState.setState(CopycatClient.State.SUSPENDED); + clientState.setState(CopycatClient.State.CLOSED); + sessions.values().forEach(CopycatSessionState::close); break; case CLOSED: clientState.setState(CopycatClient.State.CLOSED); + sessions.values().forEach(CopycatSessionState::close); break; } } diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionState.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionState.java index d472253e..9905ddf6 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionState.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionState.java @@ -15,7 +15,13 @@ */ package io.atomix.copycat.client.session.impl; +import io.atomix.catalyst.concurrent.Listener; +import io.atomix.copycat.client.session.CopycatSession; + +import java.util.Set; +import java.util.concurrent.CopyOnWriteArraySet; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; /** * Client state. @@ -32,7 +38,7 @@ public final class CopycatSessionState { private long responseIndex; private long eventIndex; private long connection; - + private final Set> changeListeners = new CopyOnWriteArraySet<>(); CopycatSessionState(long sessionId, String name, String type) { this.sessionId = sessionId; @@ -69,6 +75,15 @@ public String getSessionType() { return type; } + /** + * Returns the session state. + * + * @return The session state. + */ + public CopycatSession.State getState() { + return isOpen() ? CopycatSession.State.OPEN : CopycatSession.State.CLOSED; + } + /** * Returns a boolean indicating whether the session is open. * @@ -83,6 +98,28 @@ public boolean isOpen() { */ void close() { open.set(false); + changeListeners.forEach(l -> l.accept(CopycatSession.State.CLOSED)); + } + + /** + * Registers a state change listener on the session manager. + * + * @param callback The state change listener callback. + * @return The state change listener. + */ + public Listener onStateChange(Consumer callback) { + Listener listener = new Listener() { + @Override + public void accept(CopycatSession.State state) { + callback.accept(state); + } + @Override + public void close() { + changeListeners.remove(this); + } + }; + changeListeners.add(listener); + return listener; } /** diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java b/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java index 31190d17..1e8759a2 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/DefaultCopycatSession.java @@ -62,11 +62,6 @@ public DefaultCopycatSession(CopycatSessionState state, CopycatConnection leader this.context = threadContext; } - @Override - public long id() { - return state.getSessionId(); - } - @Override public String name() { return state.getSessionName(); @@ -77,6 +72,16 @@ public String type() { return state.getSessionType(); } + @Override + public State state() { + return state.getState(); + } + + @Override + public Listener onStateChange(Consumer callback) { + return state.onStateChange(s -> context.execute(() -> callback.accept(s))); + } + @Override public ThreadContext context() { return context; @@ -171,19 +176,19 @@ public CompletableFuture close() { @Override public int hashCode() { int hashCode = 31; - long id = id(); + long id = state.getSessionId(); hashCode = 37 * hashCode + (int) (id ^ (id >>> 32)); return hashCode; } @Override public boolean equals(Object object) { - return object instanceof DefaultCopycatSession && ((DefaultCopycatSession) object).id() == id(); + return object instanceof DefaultCopycatSession && ((DefaultCopycatSession) object).state.getSessionId() == state.getSessionId(); } @Override public String toString() { - return String.format("%s[id=%d]", getClass().getSimpleName(), id()); + return String.format("%s[id=%d]", getClass().getSimpleName(), state.getSessionId()); } } diff --git a/server/src/main/java/io/atomix/copycat/server/StateMachine.java b/server/src/main/java/io/atomix/copycat/server/StateMachine.java index a1814f6b..3d56df7b 100644 --- a/server/src/main/java/io/atomix/copycat/server/StateMachine.java +++ b/server/src/main/java/io/atomix/copycat/server/StateMachine.java @@ -213,7 +213,7 @@ * * @author Jordan Halterman */ -public abstract class StateMachine implements AutoCloseable { +public abstract class StateMachine { protected StateMachineExecutor executor; protected StateMachineContext context; protected Clock clock; @@ -255,7 +255,6 @@ protected void configure(StateMachineExecutor executor) { /** * Closes the state machine. */ - @Override public void close() { } From 3c6a335e0d3e09b08af2024a3e57d246299c6864 Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Wed, 31 May 2017 01:55:17 -0700 Subject: [PATCH 07/14] Add metadata request methods to CopycatClient. --- .../atomix/copycat/client/CopycatClient.java | 7 + .../copycat/client/CopycatMetadata.java | 50 +++++ .../client/impl/DefaultCopycatClient.java | 16 +- .../client/impl/DefaultCopycatMetadata.java | 79 ++++++++ .../session/impl/CopycatSessionManager.java | 5 +- .../metadata/CopycatClientMetadata.java | 58 ++++++ .../metadata/CopycatSessionMetadata.java | 81 ++++++++ .../protocol/ClientRequestTypeResolver.java | 1 + .../protocol/ClientResponseTypeResolver.java | 19 +- .../copycat/protocol/MetadataRequest.java | 51 +++++ .../copycat/protocol/MetadataResponse.java | 175 ++++++++++++++++++ .../copycat/util/ProtocolSerialization.java | 2 +- .../copycat/server/state/InactiveState.java | 5 + .../copycat/server/state/LeaderState.java | 26 +++ .../copycat/server/state/MetadataResult.java | 34 ++++ .../copycat/server/state/ReserveState.java | 20 ++ .../server/state/ServerSessionContext.java | 30 ++- .../copycat/server/state/ServerState.java | 8 + .../state/ServerStateMachineManager.java | 28 ++- .../server/storage/entry/ClientEntry.java | 5 + .../server/storage/entry/MetadataEntry.java | 30 +++ .../storage/util/StorageSerialization.java | 14 +- .../server/util/ServerSerialization.java | 34 ++-- .../server/state/ServerSessionTest.java | 9 +- test/src/test/resources/logback.xml | 2 +- 25 files changed, 737 insertions(+), 52 deletions(-) create mode 100644 client/src/main/java/io/atomix/copycat/client/CopycatMetadata.java create mode 100644 client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatMetadata.java create mode 100644 protocol/src/main/java/io/atomix/copycat/metadata/CopycatClientMetadata.java create mode 100644 protocol/src/main/java/io/atomix/copycat/metadata/CopycatSessionMetadata.java create mode 100644 protocol/src/main/java/io/atomix/copycat/protocol/MetadataRequest.java create mode 100644 protocol/src/main/java/io/atomix/copycat/protocol/MetadataResponse.java create mode 100644 server/src/main/java/io/atomix/copycat/server/state/MetadataResult.java create mode 100644 server/src/main/java/io/atomix/copycat/server/storage/entry/MetadataEntry.java diff --git a/client/src/main/java/io/atomix/copycat/client/CopycatClient.java b/client/src/main/java/io/atomix/copycat/client/CopycatClient.java index 99e6ade6..926ecc2d 100644 --- a/client/src/main/java/io/atomix/copycat/client/CopycatClient.java +++ b/client/src/main/java/io/atomix/copycat/client/CopycatClient.java @@ -273,6 +273,13 @@ static Builder builder(Collection

cluster) { */ Listener onStateChange(Consumer callback); + /** + * Returns the Copycat metadata. + * + * @return The Copycat metadata. + */ + CopycatMetadata metadata(); + /** * Returns the client execution context. *

diff --git a/client/src/main/java/io/atomix/copycat/client/CopycatMetadata.java b/client/src/main/java/io/atomix/copycat/client/CopycatMetadata.java new file mode 100644 index 00000000..8fa826ab --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/CopycatMetadata.java @@ -0,0 +1,50 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client; + +import io.atomix.copycat.metadata.CopycatClientMetadata; +import io.atomix.copycat.metadata.CopycatSessionMetadata; + +import java.util.Set; +import java.util.concurrent.CompletableFuture; + +/** + * Copycat metadata. + */ +public interface CopycatMetadata { + + /** + * Returns a list of clients connected to the cluster. + * + * @return A completable future to be completed with a list of clients connected to the cluster. + */ + CompletableFuture> getClients(); + + /** + * Returns a list of open sessions. + * + * @return A completable future to be completed with a list of open sessions. + */ + CompletableFuture> getSessions(); + + /** + * Returns a list of open sessions of the given type. + * + * @return A completable future to be completed with a list of open sessions of the given type. + */ + CompletableFuture> getSessions(String type); + +} diff --git a/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatClient.java b/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatClient.java index ac91260c..4765020b 100644 --- a/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatClient.java +++ b/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatClient.java @@ -24,8 +24,10 @@ import io.atomix.catalyst.util.Assert; import io.atomix.copycat.client.ConnectionStrategy; import io.atomix.copycat.client.CopycatClient; +import io.atomix.copycat.client.CopycatMetadata; import io.atomix.copycat.client.session.CopycatSession; import io.atomix.copycat.client.session.impl.CopycatSessionManager; +import io.atomix.copycat.client.util.AddressSelectorManager; import io.atomix.copycat.client.util.ClientConnectionManager; import java.time.Duration; @@ -46,6 +48,9 @@ public class DefaultCopycatClient implements CopycatClient { private final Collection

cluster; private final CopycatClientState state; private final ThreadContext threadContext; + private final ClientConnectionManager connectionManager; + private final CopycatMetadata metadata; + private final AddressSelectorManager selectorManager = new AddressSelectorManager(); private final CopycatSessionManager sessionManager; private volatile CompletableFuture openFuture; private volatile CompletableFuture closeFuture; @@ -54,7 +59,9 @@ public DefaultCopycatClient(String clientId, Collection
cluster, Client this.cluster = Assert.notNull(cluster, "cluster"); this.threadContext = new ThreadPoolContext(threadPoolExecutor, serializer.clone()); this.state = new CopycatClientState(clientId); - this.sessionManager = new CopycatSessionManager(state, new ClientConnectionManager(client), threadContext, threadPoolExecutor, connectionStrategy, sessionTimeout, unstableTimeout); + this.connectionManager = new ClientConnectionManager(client); + this.metadata = new DefaultCopycatMetadata(connectionManager, selectorManager); + this.sessionManager = new CopycatSessionManager(state, connectionManager, selectorManager, threadContext, threadPoolExecutor, connectionStrategy, sessionTimeout, unstableTimeout); } @Override @@ -67,6 +74,11 @@ public Listener onStateChange(Consumer callback) { return state.onStateChange(callback); } + @Override + public CopycatMetadata metadata() { + return metadata; + } + @Override public Serializer serializer() { return threadContext.serializer(); @@ -121,7 +133,7 @@ public synchronized CompletableFuture close() { return CompletableFuture.completedFuture(null); if (closeFuture == null) { - closeFuture = sessionManager.close(); + closeFuture = sessionManager.close().whenComplete((r, e) -> connectionManager.close()); } return closeFuture; } diff --git a/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatMetadata.java b/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatMetadata.java new file mode 100644 index 00000000..7bca8c30 --- /dev/null +++ b/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatMetadata.java @@ -0,0 +1,79 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.client.impl; + +import io.atomix.copycat.client.CommunicationStrategies; +import io.atomix.copycat.metadata.CopycatClientMetadata; +import io.atomix.copycat.client.CopycatMetadata; +import io.atomix.copycat.metadata.CopycatSessionMetadata; +import io.atomix.copycat.client.session.impl.CopycatClientConnection; +import io.atomix.copycat.client.util.AddressSelectorManager; +import io.atomix.copycat.client.util.ClientConnectionManager; +import io.atomix.copycat.protocol.MetadataRequest; +import io.atomix.copycat.protocol.MetadataResponse; +import io.atomix.copycat.protocol.Response; + +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; + +/** + * Default Copycat metadata. + */ +public class DefaultCopycatMetadata implements CopycatMetadata { + private final CopycatClientConnection connection; + + public DefaultCopycatMetadata(ClientConnectionManager connectionManager, AddressSelectorManager selectorManager) { + this.connection = new CopycatClientConnection(connectionManager, selectorManager.createSelector(CommunicationStrategies.LEADER)); + } + + /** + * Requests metadata from the cluster. + * + * @return A completable future to be completed with cluster metadata. + */ + private CompletableFuture getMetadata() { + CompletableFuture future = new CompletableFuture<>(); + connection.sendAndReceive(MetadataRequest.NAME, MetadataRequest.builder().build()).whenComplete((response, error) -> { + if (error == null) { + if (response.status() == Response.Status.OK) { + future.complete(response); + } else { + future.completeExceptionally(response.error().createException()); + } + } else { + future.completeExceptionally(error); + } + }); + return future; + } + + @Override + public CompletableFuture> getClients() { + return getMetadata().thenApply(MetadataResponse::clients); + } + + @Override + public CompletableFuture> getSessions() { + return getMetadata().thenApply(MetadataResponse::sessions); + } + + @Override + public CompletableFuture> getSessions(String type) { + return getMetadata().thenApply(response -> response.sessions().stream().filter(s -> s.type().equals(type)).collect(Collectors.toSet())); + } + +} diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java index aa2e0fed..f2e527c5 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java @@ -57,7 +57,7 @@ public class CopycatSessionManager { private final CopycatConnection connection; private final ThreadContext threadContext; private final ScheduledExecutorService threadPoolExecutor; - private final AddressSelectorManager selectorManager = new AddressSelectorManager(); + private final AddressSelectorManager selectorManager; private final ConnectionStrategy strategy; private final Duration sessionTimeout; private final Duration unstableTimeout; @@ -67,9 +67,10 @@ public class CopycatSessionManager { private Duration interval; private Scheduled keepAlive; - public CopycatSessionManager(CopycatClientState clientState, ClientConnectionManager connectionManager, ThreadContext threadContext, ScheduledExecutorService threadPoolExecutor, ConnectionStrategy connectionStrategy, Duration sessionTimeout, Duration unstableTimeout) { + public CopycatSessionManager(CopycatClientState clientState, ClientConnectionManager connectionManager, AddressSelectorManager selectorManager, ThreadContext threadContext, ScheduledExecutorService threadPoolExecutor, ConnectionStrategy connectionStrategy, Duration sessionTimeout, Duration unstableTimeout) { this.clientState = Assert.notNull(clientState, "clientState"); this.connectionManager = Assert.notNull(connectionManager, "connectionManager"); + this.selectorManager = Assert.notNull(selectorManager, "selectorManager"); this.connection = new CopycatClientConnection(connectionManager, selectorManager.createSelector(CommunicationStrategies.ANY)); this.threadContext = Assert.notNull(threadContext, "threadContext"); this.threadPoolExecutor = Assert.notNull(threadPoolExecutor, "threadPoolExecutor"); diff --git a/protocol/src/main/java/io/atomix/copycat/metadata/CopycatClientMetadata.java b/protocol/src/main/java/io/atomix/copycat/metadata/CopycatClientMetadata.java new file mode 100644 index 00000000..ce2930cf --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/metadata/CopycatClientMetadata.java @@ -0,0 +1,58 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.metadata; + +import java.util.Objects; + +/** + * Copycat client metadata. + */ +public class CopycatClientMetadata { + private final long id; + + public CopycatClientMetadata(long id) { + this.id = id; + } + + /** + * Returns the client identifier. + * + * @return The client identifier. + */ + public long id() { + return id; + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public boolean equals(Object object) { + if (object instanceof CopycatClientMetadata) { + CopycatClientMetadata metadata = (CopycatClientMetadata) object; + return metadata.id == id; + } + return false; + } + + @Override + public String toString() { + return String.format("%s[id=%d]", getClass().getSimpleName(), id); + } + +} diff --git a/protocol/src/main/java/io/atomix/copycat/metadata/CopycatSessionMetadata.java b/protocol/src/main/java/io/atomix/copycat/metadata/CopycatSessionMetadata.java new file mode 100644 index 00000000..8178cc72 --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/metadata/CopycatSessionMetadata.java @@ -0,0 +1,81 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.metadata; + +import io.atomix.catalyst.util.Assert; + +import java.util.Objects; + +/** + * Copycat session metadata. + */ +public final class CopycatSessionMetadata { + private final long id; + private final String name; + private final String type; + + public CopycatSessionMetadata(long id, String name, String type) { + this.id = id; + this.name = Assert.notNull(name, "name"); + this.type = Assert.notNull(type, "type"); + } + + /** + * Returns the globally unique session identifier. + * + * @return The globally unique session identifier. + */ + public long id() { + return id; + } + + /** + * Returns the session name. + * + * @return The session name. + */ + public String name() { + return name; + } + + /** + * Returns the session type. + * + * @return The session type. + */ + public String type() { + return type; + } + + @Override + public int hashCode() { + return Objects.hash(id, type, name); + } + + @Override + public boolean equals(Object object) { + if (object instanceof CopycatSessionMetadata) { + CopycatSessionMetadata metadata = (CopycatSessionMetadata) object; + return metadata.id == id && Objects.equals(metadata.name, name) && Objects.equals(metadata.type, type); + } + return false; + } + + @Override + public String toString() { + return String.format("%s[id=%s, name=%s, type=%s]", getClass().getSimpleName(), id, name, type); + } +} diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequestTypeResolver.java b/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequestTypeResolver.java index f301097e..442c8a69 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequestTypeResolver.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/ClientRequestTypeResolver.java @@ -38,6 +38,7 @@ public final class ClientRequestTypeResolver implements SerializableTypeResolver put(UnregisterRequest.class, -9); put(OpenSessionRequest.class, -10); put(CloseSessionRequest.class, -11); + put(MetadataRequest.class, -12); }}; @Override diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponseTypeResolver.java b/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponseTypeResolver.java index 67fd2a1d..e4bf1514 100644 --- a/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponseTypeResolver.java +++ b/protocol/src/main/java/io/atomix/copycat/protocol/ClientResponseTypeResolver.java @@ -29,15 +29,16 @@ public final class ClientResponseTypeResolver implements SerializableTypeResolver { @SuppressWarnings("unchecked") private static final Map, Integer> TYPES = new HashMap() {{ - put(CommandResponse.class, -12); - put(ConnectResponse.class, -13); - put(KeepAliveResponse.class, -14); - put(ResetRequest.class, -15); - put(QueryResponse.class, -16); - put(RegisterResponse.class, -17); - put(UnregisterResponse.class, -18); - put(OpenSessionResponse.class, -19); - put(CloseSessionResponse.class, -20); + put(CommandResponse.class, -13); + put(ConnectResponse.class, -14); + put(KeepAliveResponse.class, -15); + put(ResetRequest.class, -16); + put(QueryResponse.class, -17); + put(RegisterResponse.class, -18); + put(UnregisterResponse.class, -19); + put(OpenSessionResponse.class, -20); + put(CloseSessionResponse.class, -21); + put(MetadataResponse.class, -22); }}; @Override diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/MetadataRequest.java b/protocol/src/main/java/io/atomix/copycat/protocol/MetadataRequest.java new file mode 100644 index 00000000..67b81bce --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/protocol/MetadataRequest.java @@ -0,0 +1,51 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.protocol; + +/** + * Cluster metadata request. + */ +public class MetadataRequest extends ClientRequest { + public static final String NAME = "metadata"; + + /** + * Returns a new metadata request builder. + * + * @return A new metadata request builder. + */ + public static Builder builder() { + return new Builder(new MetadataRequest()); + } + + /** + * Returns a new metadata request builder. + * + * @param request The metadata request to build. + * @return A new metadata request builder. + */ + public static Builder builder(MetadataRequest request) { + return new Builder(request); + } + + /** + * Metadata request builder. + */ + public static class Builder extends AbstractRequest.Builder { + public Builder(MetadataRequest request) { + super(request); + } + } +} diff --git a/protocol/src/main/java/io/atomix/copycat/protocol/MetadataResponse.java b/protocol/src/main/java/io/atomix/copycat/protocol/MetadataResponse.java new file mode 100644 index 00000000..c8eac9e8 --- /dev/null +++ b/protocol/src/main/java/io/atomix/copycat/protocol/MetadataResponse.java @@ -0,0 +1,175 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.protocol; + +import io.atomix.catalyst.buffer.BufferInput; +import io.atomix.catalyst.buffer.BufferOutput; +import io.atomix.catalyst.serializer.Serializer; +import io.atomix.catalyst.util.Assert; +import io.atomix.copycat.error.CopycatError; +import io.atomix.copycat.metadata.CopycatClientMetadata; +import io.atomix.copycat.metadata.CopycatSessionMetadata; + +import java.util.Arrays; +import java.util.Collection; +import java.util.HashSet; +import java.util.Set; + +/** + * Cluster metadata response. + */ +public class MetadataResponse extends AbstractResponse { + + /** + * Returns a new metadata response builder. + * + * @return A new metadata response builder. + */ + public static Builder builder() { + return new Builder(new MetadataResponse()); + } + + /** + * Returns a new metadata response builder. + * + * @param request The metadata response to build. + * @return A new metadata response builder. + */ + public static Builder builder(MetadataResponse request) { + return new Builder(request); + } + + private Set clients; + private Set sessions; + + /** + * Returns the client metadata. + * + * @return Client metadata. + */ + public Set clients() { + return clients; + } + + /** + * Returns the session metadata. + * + * @return Session metadata. + */ + public Set sessions() { + return sessions; + } + + @Override + public void readObject(BufferInput buffer, Serializer serializer) { + status = Status.forId(buffer.readByte()); + if (status == Status.OK) { + error = null; + int clientCount = buffer.readInt(); + clients = new HashSet<>(); + for (int i = 0; i < clientCount; i++) { + clients.add(new CopycatClientMetadata(buffer.readLong())); + } + + int sessionCount = buffer.readInt(); + sessions = new HashSet<>(); + for (int i = 0; i < sessionCount; i++) { + sessions.add(new CopycatSessionMetadata(buffer.readLong(), buffer.readString(), buffer.readString())); + } + } else { + error = CopycatError.forId(buffer.readByte()); + } + } + + @Override + public void writeObject(BufferOutput buffer, Serializer serializer) { + buffer.writeByte(status.id()); + if (status == Status.ERROR) { + buffer.writeByte(error.id()); + } else { + buffer.writeInt(clients.size()); + for (CopycatClientMetadata client : clients) { + buffer.writeLong(client.id()); + } + + buffer.writeInt(sessions.size()); + for (CopycatSessionMetadata session : sessions) { + buffer.writeLong(session.id()); + buffer.writeString(session.name()); + buffer.writeString(session.type()); + } + } + } + + /** + * Metadata response builder. + */ + public static class Builder extends AbstractResponse.Builder { + public Builder(MetadataResponse request) { + super(request); + } + + /** + * Sets the client metadata. + * + * @param clients The client metadata. + * @return The metadata response builder. + */ + public Builder withClients(CopycatClientMetadata... clients) { + return withClients(Arrays.asList(Assert.notNull(clients, "clients"))); + } + + /** + * Sets the client metadata. + * + * @param clients The client metadata. + * @return The metadata response builder. + */ + public Builder withClients(Collection clients) { + response.clients = new HashSet<>(Assert.notNull(clients, "clients")); + return this; + } + + /** + * Sets the session metadata. + * + * @param sessions The client metadata. + * @return The metadata response builder. + */ + public Builder withSessions(CopycatSessionMetadata... sessions) { + return withSessions(Arrays.asList(Assert.notNull(sessions, "sessions"))); + } + + /** + * Sets the session metadata. + * + * @param sessions The client metadata. + * @return The metadata response builder. + */ + public Builder withSessions(Collection sessions) { + response.sessions = new HashSet<>(Assert.notNull(sessions, "sessions")); + return this; + } + + @Override + public MetadataResponse build() { + super.build(); + Assert.notNull(response.clients, "clients"); + Assert.notNull(response.sessions, "sessions"); + return response; + } + } +} diff --git a/protocol/src/main/java/io/atomix/copycat/util/ProtocolSerialization.java b/protocol/src/main/java/io/atomix/copycat/util/ProtocolSerialization.java index 39361b0d..ca97e1df 100644 --- a/protocol/src/main/java/io/atomix/copycat/util/ProtocolSerialization.java +++ b/protocol/src/main/java/io/atomix/copycat/util/ProtocolSerialization.java @@ -35,7 +35,7 @@ public final class ProtocolSerialization implements SerializableTypeResolver { private static final Map, Integer> TYPES = new HashMap() {{ put(Address.class, -1); put(Event.class, -2); - put(NoOpCommand.class, -45); + put(NoOpCommand.class, -47); }}; @Override diff --git a/server/src/main/java/io/atomix/copycat/server/state/InactiveState.java b/server/src/main/java/io/atomix/copycat/server/state/InactiveState.java index 77adba7d..31475495 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/InactiveState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/InactiveState.java @@ -64,6 +64,11 @@ public CompletableFuture configure(ConfigureRequest request) .build())); } + @Override + public CompletableFuture metadata(MetadataRequest request) { + return Futures.exceptionalFuture(new IllegalStateException("inactive state")); + } + @Override public CompletableFuture register(RegisterRequest request) { return Futures.exceptionalFuture(new IllegalStateException("inactive state")); diff --git a/server/src/main/java/io/atomix/copycat/server/state/LeaderState.java b/server/src/main/java/io/atomix/copycat/server/state/LeaderState.java index 697a5494..3fe3c36a 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/LeaderState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/LeaderState.java @@ -473,6 +473,32 @@ public CompletableFuture append(final AppendRequest request) { } } + @Override + public CompletableFuture metadata(MetadataRequest request) { + context.checkThread(); + logRequest(request); + + CompletableFuture future = new CompletableFuture<>(); + context.getStateMachine().apply(new MetadataEntry()).whenComplete((result, error) -> { + context.checkThread(); + if (isOpen()) { + if (error == null) { + future.complete(logResponse(MetadataResponse.builder() + .withStatus(Response.Status.OK) + .withClients(result.clients) + .withSessions(result.sessions) + .build())); + } else { + future.complete(logResponse(MetadataResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.INTERNAL_ERROR) + .build())); + } + } + }); + return future; + } + @Override public CompletableFuture command(final CommandRequest request) { context.checkThread(); diff --git a/server/src/main/java/io/atomix/copycat/server/state/MetadataResult.java b/server/src/main/java/io/atomix/copycat/server/state/MetadataResult.java new file mode 100644 index 00000000..9a4cce1d --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/state/MetadataResult.java @@ -0,0 +1,34 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.state; + +import io.atomix.copycat.metadata.CopycatClientMetadata; +import io.atomix.copycat.metadata.CopycatSessionMetadata; + +import java.util.Set; + +/** + * Metadata result. + */ +final class MetadataResult { + final Set clients; + final Set sessions; + + MetadataResult(Set clients, Set sessions) { + this.clients = clients; + this.sessions = sessions; + } +} diff --git a/server/src/main/java/io/atomix/copycat/server/state/ReserveState.java b/server/src/main/java/io/atomix/copycat/server/state/ReserveState.java index 04b6932a..e1569243 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ReserveState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ReserveState.java @@ -49,6 +49,26 @@ public CompletableFuture open() { }).thenApply(v -> this); } + @Override + public CompletableFuture metadata(MetadataRequest request) { + context.checkThread(); + logRequest(request); + + if (context.getLeader() == null) { + return CompletableFuture.completedFuture(logResponse(MetadataResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.NO_LEADER_ERROR) + .build())); + } else { + return this.forward(MetadataRequest.NAME, request) + .exceptionally(error -> MetadataResponse.builder() + .withStatus(Response.Status.ERROR) + .withError(CopycatError.Type.NO_LEADER_ERROR) + .build()) + .thenApply(this::logResponse); + } + } + @Override public CompletableFuture append(AppendRequest request) { context.checkThread(); diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java b/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java index 15804d86..9b7888cc 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerSessionContext.java @@ -22,6 +22,7 @@ import io.atomix.copycat.protocol.PublishRequest; import io.atomix.copycat.server.session.ServerSession; import io.atomix.copycat.server.storage.Log; +import io.atomix.copycat.server.storage.LogCleaner; import io.atomix.copycat.session.Event; import io.atomix.copycat.session.Session; import org.slf4j.Logger; @@ -38,32 +39,31 @@ class ServerSessionContext implements ServerSession { private static final Logger LOGGER = LoggerFactory.getLogger(ServerSessionContext.class); private final long id; + private final String name; + private final String type; private final long client; - private final Log log; private final ServerStateMachineExecutor executor; private volatile State state = State.OPEN; private Connection connection; private final String messageType; - private volatile long references; private long requestSequence; private long commandSequence; private long lastApplied; private long commandLowWaterMark; private long eventIndex; private long completeIndex; - private long closeIndex; private final Map> sequenceQueries = new HashMap<>(); private final Map> indexQueries = new HashMap<>(); private final Map results = new HashMap<>(); private final Queue events = new LinkedList<>(); private EventHolder event; - private boolean unregistering; private final Listeners changeListeners = new Listeners<>(); - ServerSessionContext(long id, long client, Log log, ServerStateMachineExecutor executor) { + ServerSessionContext(long id, String name, String type, long client, ServerStateMachineExecutor executor) { this.id = id; + this.name = name; + this.type = type; this.client = client; - this.log = Assert.notNull(log, "log"); this.eventIndex = id; this.completeIndex = id; this.lastApplied = id; @@ -76,6 +76,24 @@ public long id() { return id; } + /** + * Returns the session name. + * + * @return The session name. + */ + public String name() { + return name; + } + + /** + * Returns the session type. + * + * @return The session type. + */ + public String type() { + return type; + } + /** * Returns the session client ID. * diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerState.java b/server/src/main/java/io/atomix/copycat/server/state/ServerState.java index 573c03cd..03f79547 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerState.java @@ -37,6 +37,14 @@ public interface ServerState extends Managed { */ CopycatServer.State type(); + /** + * Handles a metadata request. + * + * @param request The request to handle. + * @return A completable future to be completed with the request response. + */ + CompletableFuture metadata(MetadataRequest request); + /** * Handles a register request. * diff --git a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java index 4ecb2405..4602dddc 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java +++ b/server/src/main/java/io/atomix/copycat/server/state/ServerStateMachineManager.java @@ -26,9 +26,10 @@ import io.atomix.copycat.error.UnknownClientException; import io.atomix.copycat.error.UnknownSessionException; import io.atomix.copycat.error.UnknownStateMachineException; +import io.atomix.copycat.metadata.CopycatClientMetadata; +import io.atomix.copycat.metadata.CopycatSessionMetadata; import io.atomix.copycat.server.StateMachine; import io.atomix.copycat.server.storage.Log; -import io.atomix.copycat.server.storage.LogCleaner; import io.atomix.copycat.server.storage.entry.*; import io.atomix.copycat.server.storage.snapshot.Snapshot; import io.atomix.copycat.session.Session; @@ -36,7 +37,9 @@ import org.slf4j.LoggerFactory; import java.util.HashMap; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; import java.util.function.BiConsumer; @@ -238,6 +241,8 @@ public CompletableFuture apply(Entry entry) { ((CompletableFuture) apply((OpenSessionEntry) entry)).whenComplete(callback); } else if (entry instanceof CloseSessionEntry) { ((CompletableFuture) apply((CloseSessionEntry) entry)).whenComplete(callback); + } else if (entry instanceof MetadataEntry) { + ((CompletableFuture) apply((MetadataEntry) entry)).whenComplete(callback); } else if (entry instanceof RegisterEntry) { ((CompletableFuture) apply((RegisterEntry) entry)).whenComplete(callback); } else if (entry instanceof KeepAliveEntry) { @@ -487,7 +492,7 @@ else if (client.state() == ClientContext.State.CLOSED) { } // Create and register the session. - ServerSessionContext session = new ServerSessionContext(entry.getIndex(), entry.getClient(), log, stateMachineExecutor); + ServerSessionContext session = new ServerSessionContext(entry.getIndex(), entry.getName(), entry.getType(), entry.getClient(), stateMachineExecutor); sessionManager.registerSession(session); // Return the session ID for commands. @@ -519,6 +524,25 @@ else if (session.state() == Session.State.CLOSED) { return stateMachineExecutor.unregister(index, timestamp, session).thenApplyAsync(v -> v, threadContext); } + /** + * Applies a metadata entry to the state machine. + */ + private CompletableFuture apply(MetadataEntry entry) { + updateTimestamp(entry); + + Set clients = new HashSet<>(); + for (ClientContext client : clientManager.getClients()) { + clients.add(new CopycatClientMetadata(client.id())); + } + + Set sessions = new HashSet<>(); + for (ServerSessionContext session : sessionManager.getSessions()) { + sessions.add(new CopycatSessionMetadata(session.id(), session.name(), session.type())); + } + + return CompletableFuture.completedFuture(new MetadataResult(clients, sessions)); + } + /** * Applies a command entry to the state machine. *

diff --git a/server/src/main/java/io/atomix/copycat/server/storage/entry/ClientEntry.java b/server/src/main/java/io/atomix/copycat/server/storage/entry/ClientEntry.java index d5b06f7b..70612ffd 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/entry/ClientEntry.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/entry/ClientEntry.java @@ -68,4 +68,9 @@ public void readObject(BufferInput buffer, Serializer serializer) { client = buffer.readLong(); } + @Override + public String toString() { + return String.format("%s[index=%d, term=%d, client=%s, timestamp=%d]", getClass().getSimpleName(), getIndex(), getTerm(), getClient(), getTimestamp()); + } + } diff --git a/server/src/main/java/io/atomix/copycat/server/storage/entry/MetadataEntry.java b/server/src/main/java/io/atomix/copycat/server/storage/entry/MetadataEntry.java new file mode 100644 index 00000000..053678fd --- /dev/null +++ b/server/src/main/java/io/atomix/copycat/server/storage/entry/MetadataEntry.java @@ -0,0 +1,30 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.server.storage.entry; + +import io.atomix.catalyst.util.reference.ReferenceManager; + +/** + * Metadata entry. + */ +public class MetadataEntry extends ClientEntry { + public MetadataEntry() { + } + + public MetadataEntry(ReferenceManager> referenceManager) { + super(referenceManager); + } +} diff --git a/server/src/main/java/io/atomix/copycat/server/storage/util/StorageSerialization.java b/server/src/main/java/io/atomix/copycat/server/storage/util/StorageSerialization.java index 9ada4e6a..65dfd626 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/util/StorageSerialization.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/util/StorageSerialization.java @@ -31,13 +31,13 @@ public final class StorageSerialization implements SerializableTypeResolver { @SuppressWarnings("unchecked") private static final Map, Integer> TYPES = new HashMap() {{ - put(CommandEntry.class, -38); - put(ConfigurationEntry.class, -39); - put(KeepAliveEntry.class, -40); - put(InitializeEntry.class, -41); - put(QueryEntry.class, -42); - put(RegisterEntry.class, -43); - put(UnregisterEntry.class, -44); + put(CommandEntry.class, -40); + put(ConfigurationEntry.class, -41); + put(KeepAliveEntry.class, -42); + put(InitializeEntry.class, -43); + put(QueryEntry.class, -44); + put(RegisterEntry.class, -45); + put(UnregisterEntry.class, -46); }}; @Override diff --git a/server/src/main/java/io/atomix/copycat/server/util/ServerSerialization.java b/server/src/main/java/io/atomix/copycat/server/util/ServerSerialization.java index 292eba0d..a131efc6 100644 --- a/server/src/main/java/io/atomix/copycat/server/util/ServerSerialization.java +++ b/server/src/main/java/io/atomix/copycat/server/util/ServerSerialization.java @@ -32,23 +32,23 @@ public final class ServerSerialization implements SerializableTypeResolver { @SuppressWarnings("unchecked") private static final Map, Integer> TYPES = new HashMap() {{ - put(AppendRequest.class, -21); - put(ConfigureRequest.class, -22); - put(InstallRequest.class, -23); - put(JoinRequest.class, -24); - put(LeaveRequest.class, -25); - put(PollRequest.class, -26); - put(ReconfigureRequest.class, -27); - put(VoteRequest.class, -28); - put(AppendResponse.class, -29); - put(ConfigureResponse.class, -30); - put(InstallResponse.class, -31); - put(JoinResponse.class, -32); - put(LeaveResponse.class, -33); - put(PollResponse.class, -34); - put(ReconfigureResponse.class, -35); - put(VoteResponse.class, -36); - put(ServerMember.class, -37); + put(AppendRequest.class, -23); + put(ConfigureRequest.class, -24); + put(InstallRequest.class, -25); + put(JoinRequest.class, -26); + put(LeaveRequest.class, -27); + put(PollRequest.class, -28); + put(ReconfigureRequest.class, -29); + put(VoteRequest.class, -30); + put(AppendResponse.class, -31); + put(ConfigureResponse.class, -32); + put(InstallResponse.class, -33); + put(JoinResponse.class, -34); + put(LeaveResponse.class, -35); + put(PollResponse.class, -36); + put(ReconfigureResponse.class, -37); + put(VoteResponse.class, -38); + put(ServerMember.class, -39); }}; @Override diff --git a/server/src/test/java/io/atomix/copycat/server/state/ServerSessionTest.java b/server/src/test/java/io/atomix/copycat/server/state/ServerSessionTest.java index a769d2c7..97d5e26f 100644 --- a/server/src/test/java/io/atomix/copycat/server/state/ServerSessionTest.java +++ b/server/src/test/java/io/atomix/copycat/server/state/ServerSessionTest.java @@ -15,7 +15,6 @@ */ package io.atomix.copycat.server.state; -import io.atomix.copycat.server.storage.Log; import org.testng.annotations.Test; import java.util.concurrent.atomic.AtomicBoolean; @@ -36,7 +35,7 @@ public class ServerSessionTest { */ public void testInitializeSession() throws Throwable { ServerStateMachineContext context = mock(ServerStateMachineContext.class); - ServerSessionContext session = new ServerSessionContext(10, 1, mock(Log.class), mock(ServerStateMachineExecutor.class)); + ServerSessionContext session = new ServerSessionContext(10, "test", "test", 1, mock(ServerStateMachineExecutor.class)); assertEquals(session.id(), 10); assertEquals(session.getLastCompleted(), 9); assertEquals(session.getLastApplied(), 9); @@ -47,7 +46,7 @@ public void testInitializeSession() throws Throwable { */ public void testSequenceIndexQuery() throws Throwable { ServerStateMachineContext context = mock(ServerStateMachineContext.class); - ServerSessionContext session = new ServerSessionContext(10, 1, mock(Log.class), mock(ServerStateMachineExecutor.class)); + ServerSessionContext session = new ServerSessionContext(10, "test", "test", 1, mock(ServerStateMachineExecutor.class)); AtomicBoolean complete = new AtomicBoolean(); session.registerIndexQuery(10, () -> complete.set(true)); assertFalse(complete.get()); @@ -62,7 +61,7 @@ public void testSequenceIndexQuery() throws Throwable { */ public void testSequenceSequenceQuery() throws Throwable { ServerStateMachineContext context = mock(ServerStateMachineContext.class); - ServerSessionContext session = new ServerSessionContext(10, 1, mock(Log.class), mock(ServerStateMachineExecutor.class)); + ServerSessionContext session = new ServerSessionContext(10, "test", "test", 1, mock(ServerStateMachineExecutor.class)); AtomicBoolean complete = new AtomicBoolean(); session.registerSequenceQuery(10, () -> complete.set(true)); assertFalse(complete.get()); @@ -77,7 +76,7 @@ public void testSequenceSequenceQuery() throws Throwable { */ public void testCacheResponse() throws Throwable { ServerStateMachineContext context = mock(ServerStateMachineContext.class); - ServerSessionContext session = new ServerSessionContext(10, 1, mock(Log.class), mock(ServerStateMachineExecutor.class)); + ServerSessionContext session = new ServerSessionContext(10, "test", "test", 1, mock(ServerStateMachineExecutor.class)); session.registerResult(2, new OperationResult(2, 2, "Hello world!")); assertEquals(session.getResult(2).result, "Hello world!"); session.clearResults(3); diff --git a/test/src/test/resources/logback.xml b/test/src/test/resources/logback.xml index 9508e4f2..e9cfe967 100644 --- a/test/src/test/resources/logback.xml +++ b/test/src/test/resources/logback.xml @@ -21,7 +21,7 @@ - + \ No newline at end of file From 68bc1b22b3c4b1b1d6c1a27e0ee4c40a57b15a83 Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Wed, 31 May 2017 01:55:49 -0700 Subject: [PATCH 08/14] Revert test log level for Travis. --- test/src/test/resources/logback.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/src/test/resources/logback.xml b/test/src/test/resources/logback.xml index e9cfe967..9508e4f2 100644 --- a/test/src/test/resources/logback.xml +++ b/test/src/test/resources/logback.xml @@ -21,7 +21,7 @@ - + \ No newline at end of file From 98f8d3b3a3f79b0d5bb2c1ba3ae890c4001d2ea8 Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Wed, 31 May 2017 14:51:30 -0700 Subject: [PATCH 09/14] Add copycat-all bundle for OSGi --- all/pom.xml | 61 +++++++++++++++++++ .../atomix/copycat/bundle/CopycatBundle.java | 25 ++++++++ .../atomix/copycat/bundle/package-info.java | 20 ++++++ pom.xml | 1 + 4 files changed, 107 insertions(+) create mode 100644 all/pom.xml create mode 100644 all/src/main/java/io/atomix/copycat/bundle/CopycatBundle.java create mode 100644 all/src/main/java/io/atomix/copycat/bundle/package-info.java diff --git a/all/pom.xml b/all/pom.xml new file mode 100644 index 00000000..e519ea58 --- /dev/null +++ b/all/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + + + io.atomix.copycat + copycat-parent + 2.0.0-SNAPSHOT + + + bundle + copycat-all + Copycat All + + + + io.atomix.copycat + copycat-client + ${project.version} + + + io.atomix.copycat + copycat-server + ${project.version} + + + + + + + org.apache.felix + maven-bundle-plugin + true + + + + io.atomix.copycat.* + + + !sun.nio.ch,!sun.misc,* + + + + + + + diff --git a/all/src/main/java/io/atomix/copycat/bundle/CopycatBundle.java b/all/src/main/java/io/atomix/copycat/bundle/CopycatBundle.java new file mode 100644 index 00000000..354ba9f9 --- /dev/null +++ b/all/src/main/java/io/atomix/copycat/bundle/CopycatBundle.java @@ -0,0 +1,25 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.atomix.copycat.bundle; + +/** + * Empty class required to get the copycat-all module to build properly. + * + * NOTE Required for bundle plugin to operate. + */ +public class CopycatBundle { + +} \ No newline at end of file diff --git a/all/src/main/java/io/atomix/copycat/bundle/package-info.java b/all/src/main/java/io/atomix/copycat/bundle/package-info.java new file mode 100644 index 00000000..ffaede5d --- /dev/null +++ b/all/src/main/java/io/atomix/copycat/bundle/package-info.java @@ -0,0 +1,20 @@ +/* + * Copyright 2017-present Open Networking Laboratory + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Empty package for bundle plugin + */ +package io.atomix.copycat.bundle; diff --git a/pom.xml b/pom.xml index 7ca3910d..b40769cf 100644 --- a/pom.xml +++ b/pom.xml @@ -65,6 +65,7 @@ protocol server client + all test examples From f88142a220ed4ebcc2683299dd2cd7c0c6b34dc9 Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Wed, 31 May 2017 15:03:34 -0700 Subject: [PATCH 10/14] Fix bundle export in copycat-all pom.xml --- all/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/all/pom.xml b/all/pom.xml index e519ea58..9e19ca1c 100644 --- a/all/pom.xml +++ b/all/pom.xml @@ -48,7 +48,7 @@ - io.atomix.copycat.* + io.atomix.* !sun.nio.ch,!sun.misc,* From 1da69941317ca15ff174fd63883327467c49f236 Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Wed, 31 May 2017 16:25:19 -0700 Subject: [PATCH 11/14] Fix missing error constants. --- .../main/java/io/atomix/copycat/error/CopycatError.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/protocol/src/main/java/io/atomix/copycat/error/CopycatError.java b/protocol/src/main/java/io/atomix/copycat/error/CopycatError.java index 74f4a24a..2333e7fa 100644 --- a/protocol/src/main/java/io/atomix/copycat/error/CopycatError.java +++ b/protocol/src/main/java/io/atomix/copycat/error/CopycatError.java @@ -45,10 +45,14 @@ static CopycatError forId(int id) { case 5: return Type.ILLEGAL_MEMBER_STATE_ERROR; case 6: - return Type.UNKNOWN_SESSION_ERROR; + return Type.UNKNOWN_CLIENT_ERROR; case 7: - return Type.INTERNAL_ERROR; + return Type.UNKNOWN_SESSION_ERROR; case 8: + return Type.UNKNOWN_STATE_MACHINE_ERROR; + case 9: + return Type.INTERNAL_ERROR; + case 10: return Type.CONFIGURATION_ERROR; default: throw new IllegalArgumentException("invalid error identifier: " + id); From 60f411d47a67ff2d416ebcc51ba41b463286e967 Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Wed, 31 May 2017 17:28:58 -0700 Subject: [PATCH 12/14] Add cluster/leader methods to CopycatMetadata client. --- .../atomix/copycat/client/CopycatMetadata.java | 16 ++++++++++++++++ .../client/impl/DefaultCopycatMetadata.java | 15 +++++++++++++++ .../client/util/AddressSelectorManager.java | 18 ++++++++++++++++++ 3 files changed, 49 insertions(+) diff --git a/client/src/main/java/io/atomix/copycat/client/CopycatMetadata.java b/client/src/main/java/io/atomix/copycat/client/CopycatMetadata.java index 8fa826ab..70f96993 100644 --- a/client/src/main/java/io/atomix/copycat/client/CopycatMetadata.java +++ b/client/src/main/java/io/atomix/copycat/client/CopycatMetadata.java @@ -15,9 +15,11 @@ */ package io.atomix.copycat.client; +import io.atomix.catalyst.transport.Address; import io.atomix.copycat.metadata.CopycatClientMetadata; import io.atomix.copycat.metadata.CopycatSessionMetadata; +import java.util.Collection; import java.util.Set; import java.util.concurrent.CompletableFuture; @@ -26,6 +28,20 @@ */ public interface CopycatMetadata { + /** + * Returns the current cluster leader. + * + * @return The current cluster leader. + */ + Address leader(); + + /** + * Returns the set of known servers in the cluster. + * + * @return The set of known servers in the cluster. + */ + Collection

servers(); + /** * Returns a list of clients connected to the cluster. * diff --git a/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatMetadata.java b/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatMetadata.java index 7bca8c30..90495c1c 100644 --- a/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatMetadata.java +++ b/client/src/main/java/io/atomix/copycat/client/impl/DefaultCopycatMetadata.java @@ -15,6 +15,8 @@ */ package io.atomix.copycat.client.impl; +import io.atomix.catalyst.transport.Address; +import io.atomix.catalyst.util.Assert; import io.atomix.copycat.client.CommunicationStrategies; import io.atomix.copycat.metadata.CopycatClientMetadata; import io.atomix.copycat.client.CopycatMetadata; @@ -26,6 +28,7 @@ import io.atomix.copycat.protocol.MetadataResponse; import io.atomix.copycat.protocol.Response; +import java.util.Collection; import java.util.Set; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; @@ -34,12 +37,24 @@ * Default Copycat metadata. */ public class DefaultCopycatMetadata implements CopycatMetadata { + private final AddressSelectorManager selectorManager; private final CopycatClientConnection connection; public DefaultCopycatMetadata(ClientConnectionManager connectionManager, AddressSelectorManager selectorManager) { + this.selectorManager = Assert.notNull(selectorManager, "selectorManager"); this.connection = new CopycatClientConnection(connectionManager, selectorManager.createSelector(CommunicationStrategies.LEADER)); } + @Override + public Address leader() { + return selectorManager.leader(); + } + + @Override + public Collection
servers() { + return selectorManager.servers(); + } + /** * Requests metadata from the cluster. * diff --git a/client/src/main/java/io/atomix/copycat/client/util/AddressSelectorManager.java b/client/src/main/java/io/atomix/copycat/client/util/AddressSelectorManager.java index 148fea91..eeeac6ab 100644 --- a/client/src/main/java/io/atomix/copycat/client/util/AddressSelectorManager.java +++ b/client/src/main/java/io/atomix/copycat/client/util/AddressSelectorManager.java @@ -32,6 +32,24 @@ public final class AddressSelectorManager { private volatile Address leader; private volatile Collection
servers = Collections.emptyList(); + /** + * Returns the current cluster leader. + * + * @return The current cluster leader. + */ + public Address leader() { + return leader; + } + + /** + * Returns the set of servers in the cluster. + * + * @return The set of servers in the cluster. + */ + public Collection
servers() { + return servers; + } + /** * Creates a new address selector. * From 63e33fd2d5023c367a67364c898143b4f9ef7f13 Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Thu, 1 Jun 2017 10:41:19 -0700 Subject: [PATCH 13/14] Ensure client state change listeners are called only on state changes. --- .../client/impl/CopycatClientState.java | 6 ++- .../session/impl/CopycatSessionManager.java | 44 ++++++++++--------- .../session/impl/CopycatSessionState.java | 5 ++- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/client/src/main/java/io/atomix/copycat/client/impl/CopycatClientState.java b/client/src/main/java/io/atomix/copycat/client/impl/CopycatClientState.java index c5317c1e..7a39ce3a 100644 --- a/client/src/main/java/io/atomix/copycat/client/impl/CopycatClientState.java +++ b/client/src/main/java/io/atomix/copycat/client/impl/CopycatClientState.java @@ -83,8 +83,10 @@ public CopycatClient.State getState() { * @return The session state. */ public CopycatClientState setState(CopycatClient.State state) { - this.state = state; - changeListeners.forEach(l -> l.accept(state)); + if (this.state != state) { + this.state = state; + changeListeners.forEach(l -> l.accept(state)); + } return this; } diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java index f2e527c5..21859512 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionManager.java @@ -102,27 +102,29 @@ public void resetConnections(Address leader, Collection
servers) { * @param state The session manager state. */ private void setState(State state) { - this.state = state; - switch (state) { - case OPEN: - clientState.setState(CopycatClient.State.CONNECTED); - break; - case UNSTABLE: - clientState.setState(CopycatClient.State.SUSPENDED); - if (unstableTime == null) { - unstableTime = System.currentTimeMillis(); - } else if (System.currentTimeMillis() - unstableTime > unstableTimeout.toMillis()) { - setState(State.EXPIRED); - } - break; - case EXPIRED: - clientState.setState(CopycatClient.State.CLOSED); - sessions.values().forEach(CopycatSessionState::close); - break; - case CLOSED: - clientState.setState(CopycatClient.State.CLOSED); - sessions.values().forEach(CopycatSessionState::close); - break; + if (this.state != state) { + this.state = state; + switch (state) { + case OPEN: + clientState.setState(CopycatClient.State.CONNECTED); + break; + case UNSTABLE: + clientState.setState(CopycatClient.State.SUSPENDED); + if (unstableTime == null) { + unstableTime = System.currentTimeMillis(); + } else if (System.currentTimeMillis() - unstableTime > unstableTimeout.toMillis()) { + setState(State.EXPIRED); + } + break; + case EXPIRED: + clientState.setState(CopycatClient.State.CLOSED); + sessions.values().forEach(CopycatSessionState::close); + break; + case CLOSED: + clientState.setState(CopycatClient.State.CLOSED); + sessions.values().forEach(CopycatSessionState::close); + break; + } } } diff --git a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionState.java b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionState.java index 9905ddf6..294208a3 100644 --- a/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionState.java +++ b/client/src/main/java/io/atomix/copycat/client/session/impl/CopycatSessionState.java @@ -97,8 +97,9 @@ public boolean isOpen() { * Closes the session. */ void close() { - open.set(false); - changeListeners.forEach(l -> l.accept(CopycatSession.State.CLOSED)); + if (open.compareAndSet(true, false)) { + changeListeners.forEach(l -> l.accept(CopycatSession.State.CLOSED)); + } } /** From 9eb81e45d29eaa01a632dce42c324216a8ec7351 Mon Sep 17 00:00:00 2001 From: Jordan Halterman Date: Sat, 3 Jun 2017 20:36:48 -0700 Subject: [PATCH 14/14] Support per-state-machine snapshots in SnapshotStore. --- .../server/protocol/InstallRequest.java | 44 ++++++-- .../server/state/AbstractAppender.java | 14 ++- .../server/state/FollowerAppender.java | 6 +- .../copycat/server/state/LeaderAppender.java | 19 ++-- .../copycat/server/state/MemberState.java | 21 ---- .../copycat/server/state/PassiveState.java | 14 ++- .../server/storage/snapshot/FileSnapshot.java | 5 + .../storage/snapshot/MemorySnapshot.java | 5 + .../server/storage/snapshot/Snapshot.java | 7 ++ .../storage/snapshot/SnapshotDescriptor.java | 39 +++++-- .../server/storage/snapshot/SnapshotFile.java | 68 +++++++++--- .../storage/snapshot/SnapshotStore.java | 100 +++++++++--------- .../storage/AbstractSnapshotStoreTest.java | 22 ++-- .../server/storage/FileSnapshotStoreTest.java | 13 ++- 14 files changed, 248 insertions(+), 129 deletions(-) diff --git a/server/src/main/java/io/atomix/copycat/server/protocol/InstallRequest.java b/server/src/main/java/io/atomix/copycat/server/protocol/InstallRequest.java index ea75f8c3..2f344343 100644 --- a/server/src/main/java/io/atomix/copycat/server/protocol/InstallRequest.java +++ b/server/src/main/java/io/atomix/copycat/server/protocol/InstallRequest.java @@ -60,10 +60,11 @@ public static Builder builder(InstallRequest request) { private long term; private int leader; - protected long index; - protected int offset; - protected byte[] data; - protected boolean complete; + private long id; + private long index; + private int offset; + private byte[] data; + private boolean complete; /** * Returns the requesting node's current term. @@ -83,6 +84,15 @@ public int leader() { return leader; } + /** + * Returns the snapshot identifier. + * + * @return The snapshot identifier. + */ + public long id() { + return id; + } + /** * Returns the snapshot index. * @@ -123,25 +133,29 @@ public boolean complete() { public void writeObject(BufferOutput buffer, Serializer serializer) { buffer.writeLong(term) .writeInt(leader) + .writeLong(id) .writeLong(index) .writeInt(offset) - .writeBoolean(complete); - serializer.writeObject(data, buffer); + .writeBoolean(complete) + .writeInt(data.length) + .write(data); } @Override public void readObject(BufferInput buffer, Serializer serializer) { term = buffer.readLong(); leader = buffer.readInt(); + id = buffer.readLong(); index = buffer.readLong(); offset = buffer.readInt(); complete = buffer.readBoolean(); - data = serializer.readObject(buffer); + data = new byte[buffer.readInt()]; + buffer.read(data); } @Override public int hashCode() { - return Objects.hash(getClass(), term, leader, index, offset, complete, data); + return Objects.hash(getClass(), term, leader, id, index, offset, complete, data); } @Override @@ -150,6 +164,7 @@ public boolean equals(Object object) { InstallRequest request = (InstallRequest) object; return request.term == term && request.leader == leader + && request.id == id && request.index == index && request.offset == offset && request.complete == complete @@ -160,7 +175,7 @@ public boolean equals(Object object) { @Override public String toString() { - return String.format("%s[term=%d, leader=%d, index=%d, offset=%d, data=%s, complete=%b]", getClass().getSimpleName(), term, leader, index, offset, data, complete); + return String.format("%s[term=%d, leader=%d, id=%d, index=%d, offset=%d, data=%s, complete=%b]", getClass().getSimpleName(), term, leader, id, index, offset, data, complete); } /** @@ -195,6 +210,17 @@ public Builder withLeader(int leader) { return this; } + /** + * Sets the request snapshot identifier. + * + * @param id The request snapshot identifier. + * @return The request builder. + */ + public Builder withId(long id) { + request.id = Assert.argNot(id, id <= 0, "id must be positive"); + return this; + } + /** * Sets the request index. * diff --git a/server/src/main/java/io/atomix/copycat/server/state/AbstractAppender.java b/server/src/main/java/io/atomix/copycat/server/state/AbstractAppender.java index 0f8111fa..501cbf50 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/AbstractAppender.java +++ b/server/src/main/java/io/atomix/copycat/server/state/AbstractAppender.java @@ -124,6 +124,13 @@ protected AppendRequest buildAppendEntriesRequest(MemberState member, long lastI // Iterate through remaining entries in the log up to the last index. for (long i = index; i <= lastIndex; i++) { + // If a snapshot exists at this index, complete the request. This will ensure the snapshot + // is sent on the next index. + Snapshot snapshot = context.getSnapshotStore().getSnapshotByIndex(i); + if (snapshot != null) { + break; + } + // Get the entry from the log and append it if it's not null. Entries in the log can be null // if they've been cleaned or compacted from the log. Each entry sent in the append request // has a unique index to handle gaps in the log. @@ -478,7 +485,7 @@ protected void handleConfigureResponseError(MemberState member, ConfigureRequest * Builds an install request for the given member. */ protected InstallRequest buildInstallRequest(MemberState member) { - Snapshot snapshot = context.getSnapshotStore().currentSnapshot(); + Snapshot snapshot = context.getSnapshotStore().getSnapshotByIndex(member.getNextIndex()); if (member.getNextSnapshotIndex() != snapshot.index()) { member.setNextSnapshotIndex(snapshot.index()).setNextSnapshotOffset(0); } @@ -600,9 +607,10 @@ protected void handleInstallResponseOk(MemberState member, InstallRequest reques // If the install request was completed successfully, set the member's snapshotIndex and reset // the next snapshot index/offset. if (request.complete()) { - member.setSnapshotIndex(request.index()) + member .setNextSnapshotIndex(0) - .setNextSnapshotOffset(0); + .setNextSnapshotOffset(0) + .setNextIndex(request.index() + 1); } // If more install requests remain, increment the member's snapshot offset. else { diff --git a/server/src/main/java/io/atomix/copycat/server/state/FollowerAppender.java b/server/src/main/java/io/atomix/copycat/server/state/FollowerAppender.java index 15e2f033..3030fec0 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/FollowerAppender.java +++ b/server/src/main/java/io/atomix/copycat/server/state/FollowerAppender.java @@ -16,6 +16,7 @@ package io.atomix.copycat.server.state; import io.atomix.copycat.server.cluster.Member; +import io.atomix.copycat.server.storage.snapshot.Snapshot; /** * Follower appender. @@ -52,9 +53,8 @@ protected void appendEntries(MemberState member) { // If the member's current snapshot index is less than the latest snapshot index and the latest snapshot index // is less than the nextIndex, send a snapshot request. - if (context.getSnapshotStore().currentSnapshot() != null - && context.getSnapshotStore().currentSnapshot().index() >= member.getNextIndex() - && context.getSnapshotStore().currentSnapshot().index() > member.getSnapshotIndex()) { + Snapshot snapshot = context.getSnapshotStore().getSnapshotByIndex(member.getNextIndex()); + if (snapshot != null) { if (member.canInstall()) { sendInstallRequest(member, buildInstallRequest(member)); } diff --git a/server/src/main/java/io/atomix/copycat/server/state/LeaderAppender.java b/server/src/main/java/io/atomix/copycat/server/state/LeaderAppender.java index bc6a92e9..3f6625f5 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/LeaderAppender.java +++ b/server/src/main/java/io/atomix/copycat/server/state/LeaderAppender.java @@ -26,12 +26,15 @@ import io.atomix.copycat.server.protocol.ConfigureResponse; import io.atomix.copycat.server.protocol.InstallRequest; import io.atomix.copycat.server.protocol.InstallResponse; +import io.atomix.copycat.server.storage.snapshot.Snapshot; import java.time.Instant; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; /** * The leader appender is responsible for sending {@link AppendRequest}s on behalf of a leader to followers. @@ -199,13 +202,15 @@ else if (member.getMember().type() == Member.Type.RESERVE || member.getMember(). sendAppendRequest(member, buildAppendEmptyRequest(member)); } } - // If the member's current snapshot index is less than the latest snapshot index and the latest snapshot index - // is less than the nextIndex, send a snapshot request. - else if (member.getMember().type() == Member.Type.ACTIVE && context.getSnapshotStore().currentSnapshot() != null - && context.getSnapshotStore().currentSnapshot().index() >= member.getNextIndex() - && context.getSnapshotStore().currentSnapshot().index() > member.getSnapshotIndex()) { - if (member.canInstall()) { - sendInstallRequest(member, buildInstallRequest(member)); + // If there's a snapshot at the member's nextIndex, replicate the snapshot. + else if (member.getMember().type() == Member.Type.ACTIVE) { + Snapshot snapshot = context.getSnapshotStore().getSnapshotByIndex(member.getNextIndex()); + if (snapshot != null) { + if (member.canInstall()) { + sendInstallRequest(member, buildInstallRequest(member)); + } + } else if (member.canAppend()) { + sendAppendRequest(member, buildAppendRequest(member, context.getLog().lastIndex())); } } // If no AppendRequest is already being sent, send an AppendRequest. diff --git a/server/src/main/java/io/atomix/copycat/server/state/MemberState.java b/server/src/main/java/io/atomix/copycat/server/state/MemberState.java index 172c1000..032c31d3 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/MemberState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/MemberState.java @@ -28,7 +28,6 @@ final class MemberState { private final ServerMember member; private long term; private long configIndex; - private long snapshotIndex; private long nextSnapshotIndex; private int nextSnapshotOffset; private long matchIndex; @@ -114,26 +113,6 @@ MemberState setConfigIndex(long configIndex) { return this; } - /** - * Returns the member's snapshot index. - * - * @return The member's snapshot index. - */ - long getSnapshotIndex() { - return snapshotIndex; - } - - /** - * Sets the member's snapshot index. - * - * @param snapshotIndex The member's snapshot index. - * @return The member state. - */ - MemberState setSnapshotIndex(long snapshotIndex) { - this.snapshotIndex = snapshotIndex; - return this; - } - /** * Returns the member's next snapshot index. * diff --git a/server/src/main/java/io/atomix/copycat/server/state/PassiveState.java b/server/src/main/java/io/atomix/copycat/server/state/PassiveState.java index 004c0a76..0009475f 100644 --- a/server/src/main/java/io/atomix/copycat/server/state/PassiveState.java +++ b/server/src/main/java/io/atomix/copycat/server/state/PassiveState.java @@ -31,6 +31,8 @@ import io.atomix.copycat.server.storage.snapshot.Snapshot; import io.atomix.copycat.server.storage.snapshot.SnapshotWriter; +import java.util.HashMap; +import java.util.Map; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; import java.util.stream.Collectors; @@ -41,7 +43,7 @@ * @author Jordan Halterman */ class PassiveState extends ReserveState { - private Snapshot pendingSnapshot; + private final Map pendingSnapshots = new HashMap<>(); private int nextSnapshotOffset; public PassiveState(ServerContext context) { @@ -352,6 +354,9 @@ public CompletableFuture install(InstallRequest request) { .build())); } + // Get the pending snapshot for the associated snapshot ID. + Snapshot pendingSnapshot = pendingSnapshots.get(request.id()); + // If a snapshot is currently being received and the snapshot versions don't match, simply // close the existing snapshot. This is a naive implementation that assumes that the leader // will be responsible in sending the correct snapshot to this server. Leaders must dictate @@ -375,7 +380,7 @@ public CompletableFuture install(InstallRequest request) { .build())); } - pendingSnapshot = context.getSnapshotStore().createSnapshot(request.index()); + pendingSnapshot = context.getSnapshotStore().createSnapshot(request.id(), request.index()); nextSnapshotOffset = 0; } @@ -395,7 +400,7 @@ public CompletableFuture install(InstallRequest request) { // If the snapshot is complete, store the snapshot and reset state, otherwise update the next snapshot offset. if (request.complete()) { pendingSnapshot.complete(); - pendingSnapshot = null; + pendingSnapshots.remove(request.id()); nextSnapshotOffset = 0; } else { nextSnapshotOffset++; @@ -408,10 +413,9 @@ public CompletableFuture install(InstallRequest request) { @Override public CompletableFuture close() { - if (pendingSnapshot != null) { + for (Snapshot pendingSnapshot : pendingSnapshots.values()) { pendingSnapshot.close(); pendingSnapshot.delete(); - pendingSnapshot = null; } return super.close(); } diff --git a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/FileSnapshot.java b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/FileSnapshot.java index b0f2790e..a8eecca7 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/FileSnapshot.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/FileSnapshot.java @@ -38,6 +38,11 @@ final class FileSnapshot extends Snapshot { this.store = Assert.notNull(store, "store"); } + @Override + public long id() { + return file.id(); + } + @Override public long index() { return file.index(); diff --git a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/MemorySnapshot.java b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/MemorySnapshot.java index a842c0a7..37e1f2ab 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/MemorySnapshot.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/MemorySnapshot.java @@ -37,6 +37,11 @@ final class MemorySnapshot extends Snapshot { this.store = Assert.notNull(store, "store"); } + @Override + public long id() { + return descriptor.id(); + } + @Override public long index() { return descriptor.index(); diff --git a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/Snapshot.java b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/Snapshot.java index 327c7827..74e59e64 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/Snapshot.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/Snapshot.java @@ -59,6 +59,13 @@ protected Snapshot(SnapshotStore store) { this.store = Assert.notNull(store, "store"); } + /** + * Returns the identifier of the state machine to which the snapshot belongs. + * + * @return The snapshot identifier. + */ + public abstract long id(); + /** * Returns the snapshot index. *

diff --git a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotDescriptor.java b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotDescriptor.java index d5a1745a..15d605f0 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotDescriptor.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotDescriptor.java @@ -55,6 +55,7 @@ public static Builder builder(Buffer buffer) { } private Buffer buffer; + private final long id; private final long index; private final long timestamp; private boolean locked; @@ -64,12 +65,22 @@ public static Builder builder(Buffer buffer) { */ public SnapshotDescriptor(Buffer buffer) { this.buffer = Assert.notNull(buffer, "buffer"); + this.id = buffer.readLong(); this.index = buffer.readLong(); this.timestamp = buffer.readLong(); this.locked = buffer.readBoolean(); buffer.skip(BYTES - buffer.position()); } + /** + * Returns the snapshot identifier. + * + * @return The snapshot identifier. + */ + public long id() { + return id; + } + /** * Returns the snapshot index. * @@ -104,7 +115,7 @@ public boolean locked() { */ public void lock() { buffer.flush() - .writeBoolean(16, true) + .writeBoolean(24, true) .flush(); locked = true; } @@ -114,6 +125,7 @@ public void lock() { */ SnapshotDescriptor copyTo(Buffer buffer) { this.buffer = buffer + .writeLong(id) .writeLong(index) .writeLong(timestamp) .writeBoolean(locked) @@ -146,13 +158,24 @@ private Builder(Buffer buffer) { } /** - * Sets the segment index. + * Sets the snapshot identifier. * - * @param index The segment index. - * @return The segment descriptor builder. + * @param id The snapshot identifier. + * @return The snapshot builder. + */ + public Builder withId(long id) { + buffer.writeLong(0, id); + return this; + } + + /** + * Sets the snapshot index. + * + * @param index The snapshot index. + * @return The snapshot builder. */ public Builder withIndex(long index) { - buffer.writeLong(0, index); + buffer.writeLong(8, index); return this; } @@ -163,14 +186,14 @@ public Builder withIndex(long index) { * @return The snapshot builder. */ public Builder withTimestamp(long timestamp) { - buffer.writeLong(8, timestamp); + buffer.writeLong(16, timestamp); return this; } /** - * Builds the segment descriptor. + * Builds the snapshot descriptor. * - * @return The built segment descriptor. + * @return The built snapshot descriptor. */ public SnapshotDescriptor build() { return new SnapshotDescriptor(buffer); diff --git a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotFile.java b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotFile.java index 9d952b65..8cf126b9 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotFile.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotFile.java @@ -42,32 +42,67 @@ public static boolean isSnapshotFile(String name, File file) { Assert.notNull(name, "name"); Assert.notNull(file, "file"); String fileName = file.getName(); - if (fileName.lastIndexOf(EXTENSION_SEPARATOR) == -1 || fileName.lastIndexOf(PART_SEPARATOR) == -1 || fileName.lastIndexOf(EXTENSION_SEPARATOR) < fileName.lastIndexOf(PART_SEPARATOR) || !fileName.endsWith(EXTENSION)) + + // The file name should contain an extension separator. + if (fileName.lastIndexOf(EXTENSION_SEPARATOR) == -1) { return false; + } - for (int i = fileName.lastIndexOf(PART_SEPARATOR) + 1; i < fileName.lastIndexOf(EXTENSION_SEPARATOR); i++) { - if (!Character.isDigit(fileName.charAt(i))) { - return false; - } + // The file name should end with the snapshot extension. + if (!fileName.endsWith("." + EXTENSION)) { + return false; + } + + // Parse the file name parts. + String[] parts = fileName.split(String.valueOf(PART_SEPARATOR)); + + // The total number of file name parts should be at least 4. + if (parts.length >= 4) { + return false; + } + + // The first part of the file name should be the provided snapshot file name. + // Subtract from the number of parts to ensure PART_SEPARATOR can be used in snapshot names. + if (!parts[parts.length-4].equals(name)) { + return false; + } + + // The second part of the file name should be numeric. + // Subtract from the number of parts to ensure PART_SEPARATOR can be used in snapshot names. + if (!isNumeric(parts[parts.length-3])) { + return false; } - if (fileName.lastIndexOf(PART_SEPARATOR, fileName.lastIndexOf(PART_SEPARATOR) - 1) == -1) + // The third part of the file name should be numeric. + // Subtract from the number of parts to ensure PART_SEPARATOR can be used in snapshot names. + if (!isNumeric(parts[parts.length-2])) { return false; + } + + // Otherwise, assume this is a snapshot file. + return true; + } - for (int i = fileName.lastIndexOf(PART_SEPARATOR, fileName.lastIndexOf(PART_SEPARATOR) - 1) + 1; i < fileName.lastIndexOf(PART_SEPARATOR); i++) { - if (!Character.isDigit(fileName.charAt(i))) { + /** + * Returns a boolean indicating whether the given string value is numeric. + * + * @param value The value to check. + * @return Indicates whether the given string value is numeric. + */ + private static boolean isNumeric(String value) { + for (char c : value.toCharArray()) { + if (!Character.isDigit(c)) { return false; } } - - return fileName.substring(0, fileName.lastIndexOf(PART_SEPARATOR, fileName.lastIndexOf(PART_SEPARATOR) - 1)).equals(name); + return true; } /** * Creates a snapshot file for the given directory, log name, and snapshot index. */ - static File createSnapshotFile(String name, File directory, long index, long timestamp) { - return new File(directory, String.format("%s-%d-%s.snapshot", Assert.notNull(name, "name"), index, TIMESTAMP_FORMAT.format(new Date(timestamp)))); + static File createSnapshotFile(String name, File directory, long id, long index, long timestamp) { + return new File(directory, String.format("%s-%d-%d-%s.%s", Assert.notNull(name, "name"), id, index, TIMESTAMP_FORMAT.format(new Date(timestamp)), EXTENSION)); } /** @@ -86,6 +121,15 @@ public File file() { return file; } + /** + * Returns the snapshot identifier. + * + * @return The snapshot identifier. + */ + public long id() { + return Long.valueOf(file.getName().substring(file.getName().lastIndexOf(PART_SEPARATOR, file.getName().lastIndexOf(PART_SEPARATOR) - 1) + 1, file.getName().lastIndexOf(PART_SEPARATOR))); + } + /** * Returns the snapshot index. * diff --git a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotStore.java b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotStore.java index 4e039aeb..20fa58de 100644 --- a/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotStore.java +++ b/server/src/main/java/io/atomix/copycat/server/storage/snapshot/SnapshotStore.java @@ -26,7 +26,11 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.util.*; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; /** * Persists server snapshots via the {@link Storage} module. @@ -46,7 +50,7 @@ * Snapshot snapshot = snapshots.snapshot(1); * } *

- * To create a new {@link Snapshot}, use the {@link #createSnapshot(long)} method. Each snapshot must + * To create a new {@link Snapshot}, use the {@link #createSnapshot(long, long)} method. Each snapshot must * be created with a unique {@code index} which represents the index of the server state machine at * the point at which the snapshot was taken. Snapshot indices are used to sort snapshots loaded from * disk and apply them at the correct point in the state machine. @@ -75,8 +79,8 @@ public class SnapshotStore implements AutoCloseable { private final String name; final Storage storage; private final Serializer serializer; - private final TreeMap snapshots = new TreeMap<>(); - private Snapshot currentSnapshot; + private final Map indexSnapshots = new ConcurrentHashMap<>(); + private final Map stateMachineSnapshots = new ConcurrentHashMap<>(); public SnapshotStore(String name, Storage storage, Serializer serializer) { this.name = Assert.notNull(name, "name"); @@ -90,11 +94,20 @@ public SnapshotStore(String name, Storage storage, Serializer serializer) { */ private void open() { for (Snapshot snapshot : loadSnapshots()) { - snapshots.put(snapshot.index(), snapshot); + Snapshot existingSnapshot = stateMachineSnapshots.get(snapshot.id()); + if (existingSnapshot == null || existingSnapshot.index() < snapshot.index()) { + stateMachineSnapshots.put(snapshot.id(), snapshot); + + // If a newer snapshot was found, delete the old snapshot if necessary. + if (existingSnapshot != null && !storage.retainStaleSnapshots()) { + existingSnapshot.close(); + existingSnapshot.delete(); + } + } } - if (!snapshots.isEmpty()) { - currentSnapshot = snapshots.lastEntry().getValue(); + for (Snapshot snapshot : stateMachineSnapshots.values()) { + indexSnapshots.put(snapshot.index(), snapshot); } } @@ -108,39 +121,23 @@ public Serializer serializer() { } /** - * Returns the most recent completed snapshot. - *

- * The current snapshot is the last {@link Snapshot} successfully written and - * {@link Snapshot#complete() completed}. It represents the most recent snapshot successfully - * written to disk and from which the server state can be restored. - * - * @return The most recent completed snapshot. - */ - public Snapshot currentSnapshot() { - return currentSnapshot; - } - - /** - * Returns a collection of all snapshots stored on disk. - *

- * Snapshots will be loaded from the underlying {@link Storage} when the snapshot store is created. - * The returned collection of {@link Snapshot}s will include any stored {@link Snapshot#complete() complete} - * snapshots. Both stored and new incomplete snapshots will be excluded from this list. + * Returns the last snapshot for the given state machine identifier. * - * @return A collection of all snapshots. + * @param id The state machine identifier for which to return the snapshot. + * @return The latest snapshot for the given state machine. */ - public Collection snapshots() { - return snapshots.values(); + public Snapshot getSnapshotById(long id) { + return stateMachineSnapshots.get(id); } /** - * Returns a snapshot by index. + * Returns the snapshot at the given index. * - * @param index The snapshot index. - * @return The snapshot. + * @param index The index for which to return the snapshot. + * @return The snapshot at the given index. */ - public Snapshot snapshot(long index) { - return snapshots.get(index); + public Snapshot getSnapshotByIndex(long index) { + return indexSnapshots.get(index); } /** @@ -184,11 +181,13 @@ private Collection loadSnapshots() { /** * Creates a new snapshot. * + * @param id The snapshot identifier. * @param index The snapshot index. * @return The snapshot. */ - public Snapshot createSnapshot(long index) { + public Snapshot createSnapshot(long id, long index) { SnapshotDescriptor descriptor = SnapshotDescriptor.builder() + .withId(id) .withIndex(index) .withTimestamp(System.currentTimeMillis()) .build(); @@ -220,7 +219,7 @@ private Snapshot createMemorySnapshot(SnapshotDescriptor descriptor) { * Creates a disk snapshot. */ private Snapshot createDiskSnapshot(SnapshotDescriptor descriptor) { - SnapshotFile file = new SnapshotFile(SnapshotFile.createSnapshotFile(name, storage.directory(), descriptor.index(), descriptor.timestamp())); + SnapshotFile file = new SnapshotFile(SnapshotFile.createSnapshotFile(name, storage.directory(), descriptor.id(), descriptor.index(), descriptor.timestamp())); Snapshot snapshot = new FileSnapshot(file, this); LOGGER.debug("Created disk snapshot: {}", snapshot); return snapshot; @@ -229,26 +228,29 @@ private Snapshot createDiskSnapshot(SnapshotDescriptor descriptor) { /** * Completes writing a snapshot. */ - protected void completeSnapshot(Snapshot snapshot) { + protected synchronized void completeSnapshot(Snapshot snapshot) { Assert.notNull(snapshot, "snapshot"); - snapshots.put(snapshot.index(), snapshot); - if (currentSnapshot == null || snapshot.index() > currentSnapshot.index()) { - currentSnapshot = snapshot; - } + // Only store the snapshot if no existing snapshot exists. + Snapshot existingSnapshot = stateMachineSnapshots.get(snapshot.id()); + if (existingSnapshot == null || existingSnapshot.index() < snapshot.index()) { + stateMachineSnapshots.put(snapshot.id(), snapshot); + indexSnapshots.put(snapshot.index(), snapshot); - // Delete old snapshots if necessary. - if (!storage.retainStaleSnapshots()) { - Iterator> iterator = snapshots.entrySet().iterator(); - while (iterator.hasNext()) { - Snapshot oldSnapshot = iterator.next().getValue(); - if (oldSnapshot.index() < currentSnapshot.index()) { - iterator.remove(); - oldSnapshot.close(); - oldSnapshot.delete(); + // Delete the old snapshot if necessary. + if (existingSnapshot != null) { + indexSnapshots.remove(existingSnapshot.index()); + if (!storage.retainStaleSnapshots()) { + existingSnapshot.close(); + existingSnapshot.delete(); } } } + // If the snapshot was old, delete it if necessary. + else if (!storage.retainStaleSnapshots()) { + snapshot.close(); + snapshot.delete(); + } } @Override diff --git a/server/src/test/java/io/atomix/copycat/server/storage/AbstractSnapshotStoreTest.java b/server/src/test/java/io/atomix/copycat/server/storage/AbstractSnapshotStoreTest.java index 135f2948..f5168d4f 100644 --- a/server/src/test/java/io/atomix/copycat/server/storage/AbstractSnapshotStoreTest.java +++ b/server/src/test/java/io/atomix/copycat/server/storage/AbstractSnapshotStoreTest.java @@ -41,33 +41,39 @@ public abstract class AbstractSnapshotStoreTest { */ public void testWriteSnapshotChunks() { SnapshotStore store = createSnapshotStore(); - Snapshot snapshot = store.createSnapshot(1); + Snapshot snapshot = store.createSnapshot(1, 2); assertEquals(snapshot.index(), 1); assertNotEquals(snapshot.timestamp(), 0); - assertNull(store.currentSnapshot()); + assertNull(store.getSnapshotById(1)); + assertNull(store.getSnapshotByIndex(2)); try (SnapshotWriter writer = snapshot.writer()) { writer.writeLong(10); } - assertNull(store.currentSnapshot()); + assertNull(store.getSnapshotById(1)); + assertNull(store.getSnapshotByIndex(2)); try (SnapshotWriter writer = snapshot.writer()) { writer.writeLong(11); } - assertNull(store.currentSnapshot()); + assertNull(store.getSnapshotById(1)); + assertNull(store.getSnapshotByIndex(2)); try (SnapshotWriter writer = snapshot.writer()) { writer.writeLong(12); } - assertNull(store.currentSnapshot()); + assertNull(store.getSnapshotById(1)); + assertNull(store.getSnapshotByIndex(2)); snapshot.complete(); - assertNotNull(store.currentSnapshot()); - assertEquals(store.currentSnapshot().index(), 1); + assertEquals(store.getSnapshotById(1).id(), 1); + assertEquals(store.getSnapshotById(1).index(), 2); + assertEquals(store.getSnapshotByIndex(1).id(), 1); + assertEquals(store.getSnapshotByIndex(1).index(), 2); - try (SnapshotReader reader = store.currentSnapshot().reader()) { + try (SnapshotReader reader = store.getSnapshotById(1).reader()) { assertEquals(reader.readLong(), 10); assertEquals(reader.readLong(), 11); assertEquals(reader.readLong(), 12); diff --git a/server/src/test/java/io/atomix/copycat/server/storage/FileSnapshotStoreTest.java b/server/src/test/java/io/atomix/copycat/server/storage/FileSnapshotStoreTest.java index 6892f388..e2c18cd0 100644 --- a/server/src/test/java/io/atomix/copycat/server/storage/FileSnapshotStoreTest.java +++ b/server/src/test/java/io/atomix/copycat/server/storage/FileSnapshotStoreTest.java @@ -58,17 +58,22 @@ protected SnapshotStore createSnapshotStore() { public void testStoreLoadSnapshot() { SnapshotStore store = createSnapshotStore(); - Snapshot snapshot = store.createSnapshot(1); + Snapshot snapshot = store.createSnapshot(1, 2); try (SnapshotWriter writer = snapshot.writer()) { writer.writeLong(10); } snapshot.complete(); - assertNotNull(store.currentSnapshot()); + assertNotNull(store.getSnapshotById(1)); + assertNotNull(store.getSnapshotByIndex(2)); store.close(); store = createSnapshotStore(); - assertNotNull(store.currentSnapshot()); - assertEquals(store.currentSnapshot().index(), 1); + assertNotNull(store.getSnapshotById(1)); + assertNotNull(store.getSnapshotByIndex(2)); + assertEquals(store.getSnapshotById(1).id(), 1); + assertEquals(store.getSnapshotById(1).index(), 2); + assertEquals(store.getSnapshotByIndex(1).id(), 1); + assertEquals(store.getSnapshotByIndex(1).index(), 2); } @BeforeMethod