Skip to content

Commit

Permalink
Introduce ping connecion handler
Browse files Browse the repository at this point in the history
PingConnectionHandler will periodically send PING commands to Redis Server, and decide whether to reconnect based on whether it fails (the current strategy is to reconnect after three consecutive failures)

This is essentially because KeepAlive that only relies on TCP is unreliable, for details, please refer to: redis#2082
yangbodong22011 committed Dec 16, 2022
1 parent f48c227 commit 4476bab
Showing 7 changed files with 350 additions and 11 deletions.
73 changes: 73 additions & 0 deletions src/main/java/io/lettuce/core/BlockingCommands.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
/*
* Copyright 2020-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.lettuce.core;

import java.util.HashMap;
import java.util.Map;

import io.lettuce.core.protocol.CommandArgs;
import io.lettuce.core.protocol.CommandType;
import io.lettuce.core.protocol.ProtocolKeyword;

/**
* Includes all blocking commands, some commands must have a blocking parameter.
*
* @author Yang Bodong
*/
class BlockingCommands {

private static final Map<ProtocolKeyword, String> BLOCKING_COMMANDS = new HashMap<>();

static {

BLOCKING_COMMANDS.put(CommandType.BLMPOP, "");
BLOCKING_COMMANDS.put(CommandType.BLMOVE, "");
BLOCKING_COMMANDS.put(CommandType.BLPOP, "");
BLOCKING_COMMANDS.put(CommandType.BRPOP, "");
BLOCKING_COMMANDS.put(CommandType.BRPOPLPUSH, "");
BLOCKING_COMMANDS.put(CommandType.BZPOPMAX, "");
BLOCKING_COMMANDS.put(CommandType.BZPOPMIN, "");
BLOCKING_COMMANDS.put(CommandType.XREAD, "block");
BLOCKING_COMMANDS.put(CommandType.XREADGROUP, "block");
}

/**
* Users can customize and add Module Blocking commands
* @param protocolKeyword the command name
* @param blockingArg the blocking arg.
*/
public static void addBlockingCommand(ProtocolKeyword protocolKeyword, String blockingArg) {
BLOCKING_COMMANDS.put(protocolKeyword, blockingArg);
}

/**
* Determine whether a command is a block command, Some commands are block commands only when specific parameters
* are added, for example: xread block xxx.
* @param protocolKeyword the command name
* @param commandArgs the command args
* @return true, blocking; false, not blocking.
*/
public static boolean isBlockingCommand(ProtocolKeyword protocolKeyword, CommandArgs<?, ?> commandArgs) {
String blockingArg = BLOCKING_COMMANDS.get(protocolKeyword);
if (blockingArg == null) {
return false;
}
if (blockingArg.isEmpty()) {
return true;
}
return commandArgs.containArg(blockingArg);
}
}
32 changes: 31 additions & 1 deletion src/main/java/io/lettuce/core/ClientOptions.java
Original file line number Diff line number Diff line change
@@ -61,6 +61,8 @@ public class ClientOptions implements Serializable {

public static final TimeoutOptions DEFAULT_TIMEOUT_OPTIONS = TimeoutOptions.create();

public static final int DEFAULT_PING_CONNECTION_INTERVAL = 0;

private final boolean autoReconnect;

private final boolean cancelCommandsOnReconnectFailure;
@@ -87,6 +89,8 @@ public class ClientOptions implements Serializable {

private final TimeoutOptions timeoutOptions;

private final int pingConnectionInterval;

protected ClientOptions(Builder builder) {
this.autoReconnect = builder.autoReconnect;
this.cancelCommandsOnReconnectFailure = builder.cancelCommandsOnReconnectFailure;
@@ -101,6 +105,7 @@ protected ClientOptions(Builder builder) {
this.sslOptions = builder.sslOptions;
this.suspendReconnectOnProtocolFailure = builder.suspendReconnectOnProtocolFailure;
this.timeoutOptions = builder.timeoutOptions;
this.pingConnectionInterval = builder.pingConnectionInterval;
}

protected ClientOptions(ClientOptions original) {
@@ -117,6 +122,7 @@ protected ClientOptions(ClientOptions original) {
this.sslOptions = original.getSslOptions();
this.suspendReconnectOnProtocolFailure = original.isSuspendReconnectOnProtocolFailure();
this.timeoutOptions = original.getTimeoutOptions();
this.pingConnectionInterval = original.getPingConnectionInterval();
}

/**
@@ -178,6 +184,8 @@ public static class Builder {

private TimeoutOptions timeoutOptions = DEFAULT_TIMEOUT_OPTIONS;

private int pingConnectionInterval = DEFAULT_PING_CONNECTION_INTERVAL;

protected Builder() {
}

@@ -393,6 +401,19 @@ public Builder timeoutOptions(TimeoutOptions timeoutOptions) {
return this;
}

/**
* Set the period of sending PING command. see {@link PingConnectionHandler}
* @param pingConnectionInterval the period
* @return {@code this}
*/
public Builder pingConnectionInterval(int pingConnectionInterval) {
if (pingConnectionInterval <= 0) {
throw new IllegalArgumentException("PingConnectionInterval must be greater than 0, unit:ms");
}
this.pingConnectionInterval = pingConnectionInterval;
return this;
}

/**
* Create a new instance of {@link ClientOptions}.
*
@@ -421,7 +442,8 @@ public ClientOptions.Builder mutate() {
.publishOnScheduler(isPublishOnScheduler()).pingBeforeActivateConnection(isPingBeforeActivateConnection())
.protocolVersion(getConfiguredProtocolVersion()).requestQueueSize(getRequestQueueSize())
.scriptCharset(getScriptCharset()).socketOptions(getSocketOptions()).sslOptions(getSslOptions())
.suspendReconnectOnProtocolFailure(isSuspendReconnectOnProtocolFailure()).timeoutOptions(getTimeoutOptions());
.suspendReconnectOnProtocolFailure(isSuspendReconnectOnProtocolFailure()).timeoutOptions(getTimeoutOptions())
.pingConnectionInterval(getPingConnectionInterval());

return builder;
}
@@ -598,6 +620,14 @@ public TimeoutOptions getTimeoutOptions() {
return timeoutOptions;
}

/**
* Return the pingConnectionInterval
* @return the pingConnectionInterval
*/
public int getPingConnectionInterval() {
return pingConnectionInterval;
}

/**
* Behavior of connections in disconnected state.
*/
17 changes: 17 additions & 0 deletions src/main/java/io/lettuce/core/ConnectionBuilder.java
Original file line number Diff line number Diff line change
@@ -87,6 +87,8 @@ public class ConnectionBuilder {

private ConnectionWatchdog connectionWatchdog;

private PingConnectionHandler pingConnectionHandler;

private RedisURI redisURI;

public static ConnectionBuilder connectionBuilder() {
@@ -129,6 +131,10 @@ protected List<ChannelHandler> buildHandlers() {
handlers.add(createConnectionWatchdog());
}

if (clientOptions.getPingConnectionInterval() > 0) {
handlers.add(createPingConnectionHandler());
}

return handlers;
}

@@ -155,6 +161,17 @@ protected ConnectionWatchdog createConnectionWatchdog() {
return watchdog;
}

protected PingConnectionHandler createPingConnectionHandler() {

if (pingConnectionHandler != null) {
return pingConnectionHandler;
}

PingConnectionHandler handler = new PingConnectionHandler(clientResources, clientOptions);
pingConnectionHandler = handler;
return handler;
}

public ChannelInitializer<Channel> build(SocketAddress socketAddress) {
return new PlainChannelInitializer(this::buildHandlers, clientResources);
}
156 changes: 156 additions & 0 deletions src/main/java/io/lettuce/core/PingConnectionHandler.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,156 @@
package io.lettuce.core;

import java.util.ArrayDeque;
import java.util.concurrent.CancellationException;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CompletionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;

import io.lettuce.core.codec.StringCodec;
import io.lettuce.core.protocol.AsyncCommand;
import io.lettuce.core.protocol.Command;
import io.lettuce.core.protocol.CommandHandler;
import io.lettuce.core.protocol.RedisCommand;
import io.lettuce.core.resource.ClientResources;
import io.netty.channel.Channel;
import io.netty.channel.ChannelHandler;
import io.netty.channel.ChannelHandlerContext;
import io.netty.channel.ChannelInboundHandlerAdapter;
import io.netty.util.internal.logging.InternalLogger;
import io.netty.util.internal.logging.InternalLoggerFactory;

/**
* A netty {@link ChannelHandler} responsible for monitoring (By periodically sending PING commands) Redis alive.
*
* @author Yang Bodong
* @date 2022/11/11
*/
@ChannelHandler.Sharable
public class PingConnectionHandler extends ChannelInboundHandlerAdapter {
private static final InternalLogger logger = InternalLoggerFactory.getInstance(PingConnectionHandler.class);

private final RedisCommandBuilder<String, String> commandBuilder = new RedisCommandBuilder<>(StringCodec.UTF8);

private static final int MAX_PING_FAILED_TIMES = 3;

private final AtomicInteger pingFailed = new AtomicInteger(0);

private final ClientResources clientResources;

private final ClientOptions clientOptions;

public PingConnectionHandler(ClientResources clientResources, ClientOptions clientOptions) {
this.clientResources = clientResources;
this.clientOptions = clientOptions;
}

@Override
public void channelActive(ChannelHandlerContext ctx) throws Exception {
sendPing(ctx);
ctx.fireChannelActive();
}

/**
* Periodically send the PING command, if it fails three times in a row, it will initiate a reconnection.
* Ignore exceptions returned by the following PING commands: {@link RedisLoadingException}
* {@link RedisBusyException}
*
* @param ctx the ctx
*/
private void sendPing(ChannelHandlerContext ctx) {
AsyncCommand<String, String, String> dispatch;
RedisCommand<?, ?, ?> currentCommand = getCurrentCommand(ctx);
if (currentCommand == null || !isBlockingCommand(currentCommand)) {
dispatch = dispatch(ctx.channel(), commandBuilder.ping());
} else {
dispatch = null;
}

clientResources.timer().newTimeout(timeout -> {
if (ctx.isRemoved() || !ctx.channel().isActive()) {
return;
}

RedisCommand<?, ?, ?> cmd = getCurrentCommand(ctx);
if (cmd != null && isBlockingCommand(cmd)) {
sendPing(ctx);
return;
}

if (dispatch != null && (dispatch.cancel(false) || cause(dispatch) != null)) {

Throwable cause = cause(dispatch);

if (!(cause instanceof RedisLoadingException
|| cause instanceof RedisBusyException)) {
if (!dispatch.isCancelled()) {
logger.error("Unable to send PING command over channel: " + ctx.channel(), cause);
}

if (pingFailed.incrementAndGet() == MAX_PING_FAILED_TIMES) {
logger.error("channel: {} closed due to {} consecutive PING response timeout set in {} ms",
ctx.channel(), MAX_PING_FAILED_TIMES, clientOptions.getPingConnectionInterval());
pingFailed.set(0);
ctx.channel().close();
} else {
sendPing(ctx);
}
} else {
pingFailed.set(0);
sendPing(ctx);
}
} else {
pingFailed.set(0);
sendPing(ctx);
}
}, clientOptions.getPingConnectionInterval(), TimeUnit.MILLISECONDS);
}

private <T> AsyncCommand<String, String, T> dispatch(Channel channel, Command<String, String, T> command) {

AsyncCommand<String, String, T> future = new AsyncCommand<>(command);

channel.writeAndFlush(future).addListener(writeFuture -> {

if (!writeFuture.isSuccess()) {
future.completeExceptionally(writeFuture.cause());
}
});

return future;
}

protected Throwable cause(CompletableFuture<?> future) {
try {
future.getNow(null);
return null;
} catch (CompletionException ex2) {
return ex2.getCause();
} catch (CancellationException ex1) {
return ex1;
}
}

/**
* Determine whether a command is a Blocking command
* @param command the command
* @return true, blocking; false, not blocking.
*/
private boolean isBlockingCommand(RedisCommand<?, ?, ?> command) {
return BlockingCommands.isBlockingCommand(command.getType(), command.getArgs());
}

/**
* Get the command being executed by the current Channel
* @param ctx the ctx
* @return the command.
*/
private RedisCommand<?, ?, ?> getCurrentCommand(ChannelHandlerContext ctx) {
ArrayDeque<RedisCommand<?, ?, ?>> stack = ctx.channel().attr(CommandHandler.COMMANDS_STACK).get();
if (stack != null && !stack.isEmpty()) {
return stack.peek();
}
return null;
}
}
14 changes: 14 additions & 0 deletions src/main/java/io/lettuce/core/protocol/CommandArgs.java
Original file line number Diff line number Diff line change
@@ -350,6 +350,20 @@ public ByteBuffer getFirstEncodedKey() {
return CommandArgsAccessor.encodeFirstKey(this);
}

/**
* Determine whether the command parameter contains a specific parameter, is O(n) complexity, but is rarely called.
* @param arg the arg
* @return true: contain; false: not contain.
*/
public boolean containArg(final String arg) {
for (SingularArgument sa : singularArguments) {
if (sa.toString().equalsIgnoreCase(arg)) {
return true;
}
}
return false;
}

/**
* Encode the {@link CommandArgs} and write the arguments to the {@link ByteBuf}.
*
Loading

0 comments on commit 4476bab

Please sign in to comment.