From c298d9028e02424cbb6e4b30d17632397d659e34 Mon Sep 17 00:00:00 2001 From: Yohei Ueki Date: Sun, 17 Jan 2021 17:31:57 +0900 Subject: [PATCH] parse subnet ourselves and add ReadFrom.regex --- src/main/java/io/lettuce/core/ReadFrom.java | 19 +- .../java/io/lettuce/core/ReadFromImpl.java | 191 +++++++++++++++--- .../core/cluster/ReadFromUnitTests.java | 97 +++++++-- .../RedisClusterReadFromIntegrationTests.java | 13 ++ 4 files changed, 273 insertions(+), 47 deletions(-) diff --git a/src/main/java/io/lettuce/core/ReadFrom.java b/src/main/java/io/lettuce/core/ReadFrom.java index d7e7698f47..18515b04c8 100644 --- a/src/main/java/io/lettuce/core/ReadFrom.java +++ b/src/main/java/io/lettuce/core/ReadFrom.java @@ -16,6 +16,7 @@ package io.lettuce.core; import java.util.List; +import java.util.regex.Pattern; import io.lettuce.core.internal.LettuceStrings; import io.lettuce.core.models.role.RedisNodeDescription; @@ -114,7 +115,8 @@ public abstract class ReadFrom { /** * Setting to read from any node in the subnets. * - * @param cidrNotations CIDR-block notation strings, e.g., "192.168.0.0/16". + * @param cidrNotations CIDR-block notation strings, e.g., "192.168.0.0/16", "2001:db8:abcd:0000::/52". Must not be + * {@code null}. * @return an instance of {@link ReadFromImpl.ReadFromSubnet}. * @since x.x.x */ @@ -122,6 +124,17 @@ public static ReadFrom subnet(String... cidrNotations) { return new ReadFromImpl.ReadFromSubnet(cidrNotations); } + /** + * Read from any node that has {@link RedisURI} matching with the given pattern. + * + * @param pattern regex pattern, e.g., {@code Pattern.compile(".*region-1.*")}. Must not be {@code null}. + * @return an instance of {@link ReadFromImpl.ReadFromRegex}. + * @since x.x.x + */ + public static ReadFrom regex(Pattern pattern) { + return new ReadFromImpl.ReadFromRegex(pattern); + } + /** * Chooses the nodes from the matching Redis nodes that match this read selector. * @@ -194,6 +207,10 @@ public static ReadFrom valueOf(String name) { throw new IllegalArgumentException("subnet must be created via ReadFrom#subnet"); } + if (name.equalsIgnoreCase("regex")) { + throw new IllegalArgumentException("regex must be created via ReadFrom#regex"); + } + throw new IllegalArgumentException("ReadFrom " + name + " not supported"); } diff --git a/src/main/java/io/lettuce/core/ReadFromImpl.java b/src/main/java/io/lettuce/core/ReadFromImpl.java index 802bebac5f..320348f467 100644 --- a/src/main/java/io/lettuce/core/ReadFromImpl.java +++ b/src/main/java/io/lettuce/core/ReadFromImpl.java @@ -15,20 +15,17 @@ */ package io.lettuce.core; -import java.net.InetAddress; -import java.net.InetSocketAddress; -import java.net.UnknownHostException; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.function.Predicate; +import java.util.regex.Pattern; import io.lettuce.core.internal.LettuceAssert; import io.lettuce.core.internal.LettuceLists; import io.lettuce.core.internal.LettuceStrings; import io.lettuce.core.models.role.RedisNodeDescription; -import io.netty.handler.ipfilter.IpFilterRuleType; -import io.netty.handler.ipfilter.IpSubnetFilterRule; import io.netty.util.NetUtil; /** @@ -140,28 +137,24 @@ public ReadFromAnyReplica() { /** * Read from any node in the subnets. This class does not provide DNS resolution and supports only IP address style - * {@link RedisURI} i.e. unavailable when using {@link io.lettuce.core.masterreplica.MasterReplica} with - * static setup (provided hosts) and Redis Sentinel with {@literal announce-hostname yes}. + * {@link RedisURI} i.e. unavailable when using {@link io.lettuce.core.masterreplica.MasterReplica} with static setup + * (provided hosts) and Redis Sentinel with {@literal announce-hostname yes}. Both IPv4 and IPv6 style subnets are supported + * but they never match with IP addresses of different version. * * @since x.x.x */ static final class ReadFromSubnet extends ReadFrom { - private final List rules = new ArrayList<>(); + private final List rules = new ArrayList<>(); /** - * @param cidrNotations CIDR-block notation strings, e.g., "192.168.0.0/16". + * @param cidrNotations CIDR-block notation strings, e.g., "192.168.0.0/16" or "2001:db8:abcd:0000::/52". */ ReadFromSubnet(String... cidrNotations) { LettuceAssert.notEmpty(cidrNotations, "cidrNotations must not be empty"); for (String cidrNotation : cidrNotations) { - // parts[0]: ipAddress (e.g., "192.168.0.0") - // parts[1]: cidrPrefix (e.g., "16") - String[] parts = cidrNotation.split("/"); - LettuceAssert.isTrue(parts.length == 2, "cidrNotation must have exact one '/'"); - - rules.add(new IpSubnetFilterRule(parts[0], Integer.parseInt(parts[1]), IpFilterRuleType.ACCEPT)); + rules.add(createSubnetRule(cidrNotation)); } } @@ -169,37 +162,169 @@ static final class ReadFromSubnet extends ReadFrom { public List select(Nodes nodes) { List result = new ArrayList<>(nodes.getNodes().size()); for (RedisNodeDescription node : nodes) { - if (isInSubnet(node.getUri())) { - result.add(node); + for (SubnetRule rule : rules) { + if (rule.isInSubnet(node.getUri().getHost())) { + result.add(node); + break; + } } } return result; } - private boolean isInSubnet(RedisURI redisURI) { - String host = redisURI.getHost(); - if (LettuceStrings.isEmpty(host)) { - return false; + interface SubnetRule { + + boolean isInSubnet(String ipAddress); + + } + + static SubnetRule createSubnetRule(String cidrNotation) { + String[] parts = cidrNotation.split("/"); + LettuceAssert.isTrue(parts.length == 2, "cidrNotation must have exact one '/'"); + + String ipAddress = parts[0]; + int cidrPrefix = Integer.parseInt(parts[1]); + + if (NetUtil.isValidIpV4Address(ipAddress)) { + return new Ipv4SubnetRule(ipAddress, cidrPrefix); + } else if (NetUtil.isValidIpV6Address(ipAddress)) { + return new Ipv6SubnetRule(ipAddress, cidrPrefix); + } else { + throw new IllegalArgumentException("invalid cidrNotation. cidrNotation=" + cidrNotation); } + } + + static class Ipv4SubnetRule implements SubnetRule { + + private static final int IPV4_BYTE_COUNT = 4; + + private final int networkAddress; - LettuceAssert.isTrue(NetUtil.isValidIpV4Address(host) || NetUtil.isValidIpV6Address(host), - "ReadFromSubnet supports only IP address-style redisURI. host=" + host); + private final int subnetMask; - InetSocketAddress address; - try { - // This won't make DNS lookup because host is already validated as an IP address. - address = new InetSocketAddress(InetAddress.getByName(host), redisURI.getPort()); - } catch (UnknownHostException e) { - throw new IllegalStateException("Should not reach here. host=" + host, e); + Ipv4SubnetRule(String ipAddress, int cidrPrefix) { + LettuceAssert.isTrue(NetUtil.isValidIpV4Address(ipAddress), + () -> "invalid ipv4 IP address. ipAddress=" + ipAddress); + LettuceAssert.isTrue(0 <= cidrPrefix && cidrPrefix <= 32, () -> "invalid cidrPrefix. cidrPrefix=" + cidrPrefix); + + subnetMask = toSubnetMask(cidrPrefix); + networkAddress = toNetworkAddress(ipAddress, subnetMask); } - for (IpSubnetFilterRule rule : rules) { - if (rule.matches(address)) { - return true; + /** + * return {@code true} if the {@code ipAddress} is in this subnet. If {@code ipAddress} is not valid IPv4 style + * (e.g., IPv6 style) {@code false} is always returned. + */ + @Override + public boolean isInSubnet(String ipAddress) { + if (LettuceStrings.isEmpty(ipAddress) || !NetUtil.isValidIpV4Address(ipAddress)) { + return false; } + + return (toInt(ipAddress) & subnetMask) == networkAddress; } - return false; + + private static int toSubnetMask(int cidrPrefix) { + return (int) (-1L << (32 - cidrPrefix)); + } + + private static int toNetworkAddress(String ipAddress, int subnetMask) { + return toInt(ipAddress) & subnetMask; + } + + private static int toInt(String ipAddress) { + byte[] octets = NetUtil.createByteArrayFromIpAddressString(ipAddress); + + LettuceAssert.isTrue(octets != null && octets.length == IPV4_BYTE_COUNT, + () -> "invalid IP address. ipAddress=" + ipAddress); + + return ((octets[0] & 0xff) << 24) | ((octets[1] & 0xff) << 16) | ((octets[2] & 0xff) << 8) | (octets[3] & 0xff); + } + + } + + static class Ipv6SubnetRule implements SubnetRule { + + private static final int IPV6_BYTE_COUNT = 16; + + private final BigInteger networkAddress; + + private final BigInteger subnetMask; + + public Ipv6SubnetRule(String ipAddress, int cidrPrefix) { + LettuceAssert.isTrue(NetUtil.isValidIpV6Address(ipAddress), + () -> "invalid ipv6 IP address. ipAddress=" + ipAddress); + LettuceAssert.isTrue(0 <= cidrPrefix && cidrPrefix <= 128, + () -> "invalid cidrPrefix. cidrPrefix=" + cidrPrefix); + + subnetMask = toSubnetMask(cidrPrefix); + networkAddress = toNetworkAddress(ipAddress, subnetMask); + } + + /** + * return {@code true} if the {@code ipAddress} is in this subnet. If {@code ipAddress} is not valid IPv6 style + * (e.g., IPv4 style) {@code false} is always returned. + */ + @Override + public boolean isInSubnet(String ipAddress) { + if (LettuceStrings.isEmpty(ipAddress) || !NetUtil.isValidIpV6Address(ipAddress)) { + return false; + } + + return toBigInteger(ipAddress).and(subnetMask).equals(networkAddress); + } + + private static BigInteger toSubnetMask(int cidrPrefix) { + return BigInteger.valueOf(-1).shiftLeft(128 - cidrPrefix); + } + + private static BigInteger toNetworkAddress(String ipAddress, BigInteger subnetMask) { + return toBigInteger(ipAddress).and(subnetMask); + } + + private static BigInteger toBigInteger(String ipAddress) { + byte[] octets = NetUtil.createByteArrayFromIpAddressString(ipAddress); + + LettuceAssert.isTrue(octets != null && octets.length == IPV6_BYTE_COUNT, + () -> "invalid IP address. ipAddress=" + ipAddress); + + return new BigInteger(octets); + } + + } + + } + + /** + * Read from any node that has {@link RedisURI} matching with the given pattern. + * + * @since x.x.x + */ + static class ReadFromRegex extends ReadFrom { + + private final ReadFrom delegate; + + public ReadFromRegex(Pattern pattern) { + LettuceAssert.notNull(pattern, "Pattern must not be null"); + + delegate = new UnorderedPredicateReadFromAdapter(redisNodeDescription -> { + String host = redisNodeDescription.getUri().getHost(); + if (LettuceStrings.isEmpty(host)) { + return false; + } + return pattern.matcher(host).matches(); + }); + } + + @Override + public List select(Nodes nodes) { + return delegate.select(nodes); + } + + @Override + protected boolean isOrderSensitive() { + return delegate.isOrderSensitive(); } } diff --git a/src/test/java/io/lettuce/core/cluster/ReadFromUnitTests.java b/src/test/java/io/lettuce/core/cluster/ReadFromUnitTests.java index bd981e0ce6..0566f1f9ed 100644 --- a/src/test/java/io/lettuce/core/cluster/ReadFromUnitTests.java +++ b/src/test/java/io/lettuce/core/cluster/ReadFromUnitTests.java @@ -17,17 +17,18 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import java.util.Arrays; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.regex.Pattern; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import io.lettuce.core.ReadFrom; -import io.lettuce.core.ReadFrom.Nodes; import io.lettuce.core.RedisURI; import io.lettuce.core.cluster.models.partitions.Partitions; import io.lettuce.core.cluster.models.partitions.RedisClusterNode; @@ -95,7 +96,47 @@ void anyReplica() { } @Test - void subnet() { + void subnetIpv4RuleIpv6NodeGiven() { + ReadFrom sut = ReadFrom.subnet("0.0.0.0/0"); + RedisClusterNode ipv6node = createNodeWithHost("2001:db8:abcd:1000::"); + + List result = sut.select(getNodes(ipv6node)); + + assertThat(result).isEmpty(); + } + + @Test + void subnetIpv4RuleAnyNode() { + ReadFrom sut = ReadFrom.subnet("0.0.0.0/0"); + RedisClusterNode node = createNodeWithHost("192.0.2.1"); + + List result = sut.select(getNodes(node)); + + assertThat(result).hasSize(1).containsExactly(node); + } + + @Test + void subnetIpv6RuleIpv4NodeGiven() { + ReadFrom sut = ReadFrom.subnet("::/0"); + RedisClusterNode node = createNodeWithHost("192.0.2.1"); + + List result = sut.select(getNodes(node)); + + assertThat(result).isEmpty(); + } + + @Test + void subnetIpv6RuleAnyNode() { + ReadFrom sut = ReadFrom.subnet("::/0"); + RedisClusterNode node = createNodeWithHost("2001:db8:abcd:1000::"); + + List result = sut.select(getNodes(node)); + + assertThat(result).hasSize(1).containsExactly(node); + } + + @Test + void subnetIpv4Ipv6Mixed() { ReadFrom sut = ReadFrom.subnet("192.0.2.0/24", "2001:db8:abcd:0000::/52"); RedisClusterNode nodeInSubnetIpv4 = createNodeWithHost("192.0.2.1"); @@ -111,30 +152,55 @@ void subnet() { @Test void subnetNodeWithHostname() { - ReadFrom sut = ReadFrom.subnet("0.0.0.0/24"); + ReadFrom sut = ReadFrom.subnet("0.0.0.0/0"); - Nodes hostNode = getNodes(createNodeWithHost("example.com")); - assertThatThrownBy(() -> sut.select(hostNode)).isInstanceOf(IllegalArgumentException.class); + RedisClusterNode hostNode = createNodeWithHost("example.com"); + RedisClusterNode localhostNode = createNodeWithHost("localhost"); - Nodes localhostNode = getNodes(createNodeWithHost("localhost")); - assertThatThrownBy(() -> sut.select(localhostNode)).isInstanceOf(IllegalArgumentException.class); - } + List result = sut.select(getNodes(hostNode, localhostNode)); - private RedisClusterNode createNodeWithHost(String host) { - RedisClusterNode node = new RedisClusterNode(); - node.setUri(RedisURI.Builder.redis(host).build()); - return node; + assertThat(result).isEmpty(); } @Test - void subnetInvalidCidr() { + void subnetCidrValidation() { // malformed CIDR notation assertThatThrownBy(() -> ReadFrom.subnet("192.0.2.1//1")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ReadFrom.subnet("2001:db8:abcd:0000:://52")).isInstanceOf(IllegalArgumentException.class); // malformed ipAddress assertThatThrownBy(() -> ReadFrom.subnet("foo.bar/12")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ReadFrom.subnet("zzzz:db8:abcd:0000:://52")).isInstanceOf(IllegalArgumentException.class); // malformed cidrPrefix assertThatThrownBy(() -> ReadFrom.subnet("192.0.2.1/40")).isInstanceOf(IllegalArgumentException.class); assertThatThrownBy(() -> ReadFrom.subnet("192.0.2.1/foo")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ReadFrom.subnet("2001:db8:abcd:0000/129")).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> ReadFrom.subnet("2001:db8:abcd:0000/-1")).isInstanceOf(IllegalArgumentException.class); + + // acceptable cidrPrefix + assertDoesNotThrow(() -> ReadFrom.subnet("0.0.0.0/0")); + assertDoesNotThrow(() -> ReadFrom.subnet("0.0.0.0/32")); + assertDoesNotThrow(() -> ReadFrom.subnet("::/0")); + assertDoesNotThrow(() -> ReadFrom.subnet("::/128")); + } + + @Test + void regex() { + ReadFrom sut = ReadFrom.regex(Pattern.compile(".*region-1.*")); + + RedisClusterNode node1 = createNodeWithHost("redis-node-1.region-1.example.com"); + RedisClusterNode node2 = createNodeWithHost("redis-node-2.region-1.example.com"); + RedisClusterNode node3 = createNodeWithHost("redis-node-1.region-2.example.com"); + RedisClusterNode node4 = createNodeWithHost("redis-node-2.region-2.example.com"); + + List result = sut.select(getNodes(node1, node2, node3, node4)); + + assertThat(result).hasSize(2).containsExactly(node1, node2); + } + + private RedisClusterNode createNodeWithHost(String host) { + RedisClusterNode node = new RedisClusterNode(); + node.setUri(RedisURI.Builder.redis(host).build()); + return node; } @Test @@ -182,6 +248,11 @@ void valueOfSubnet() { assertThatThrownBy(() -> ReadFrom.valueOf("subnet")).isInstanceOf(IllegalArgumentException.class); } + @Test + void valueOfRegex() { + assertThatThrownBy(() -> ReadFrom.valueOf("regex")).isInstanceOf(IllegalArgumentException.class); + } + private ReadFrom.Nodes getNodes() { return new ReadFrom.Nodes() { @Override diff --git a/src/test/java/io/lettuce/core/cluster/RedisClusterReadFromIntegrationTests.java b/src/test/java/io/lettuce/core/cluster/RedisClusterReadFromIntegrationTests.java index 35393be764..4ced09985f 100644 --- a/src/test/java/io/lettuce/core/cluster/RedisClusterReadFromIntegrationTests.java +++ b/src/test/java/io/lettuce/core/cluster/RedisClusterReadFromIntegrationTests.java @@ -17,6 +17,8 @@ import static org.assertj.core.api.Assertions.assertThat; +import java.util.regex.Pattern; + import javax.inject.Inject; import org.junit.jupiter.api.AfterEach; @@ -125,4 +127,15 @@ void readWriteSubnet() { assertThat(sync.get(key)).isEqualTo("value1"); } + @Test + void readWriteRegex() { + + connection.setReadFrom(ReadFrom.regex(Pattern.compile(".*"))); + + sync.set(key, "value1"); + + connection.getConnection(ClusterTestSettings.host, ClusterTestSettings.port2).sync().waitForReplication(1, 1000); + assertThat(sync.get(key)).isEqualTo("value1"); + } + }