From 1e95256b84ff9684d53f4c8383008f49613cd5e0 Mon Sep 17 00:00:00 2001 From: Julien Viet Date: Tue, 11 May 2021 11:10:34 +0200 Subject: [PATCH] Non proxy host options for TCP clients, per HTTP request proxy options. Clients have been modified to filter proxy options based on a list of hosts support. Host declaration accept wildcard match like JVM nonProxyHosts list. HTTP requests declares now a ProxyOptions property that will set the proxy options per request and override the client configuration. fixes #2600 fixes #3795 --- src/main/asciidoc/http.adoc | 18 ++ src/main/asciidoc/net.adoc | 8 + .../core/http/RequestOptionsConverter.java | 8 + .../core/net/ClientOptionsBaseConverter.java | 15 + src/main/java/examples/HTTPExamples.java | 25 ++ src/main/java/examples/NetExamples.java | 13 + .../java/io/vertx/core/http/HttpClient.java | 2 + .../io/vertx/core/http/HttpClientOptions.java | 10 + .../io/vertx/core/http/RequestOptions.java | 30 ++ .../core/http/WebSocketConnectOptions.java | 30 ++ .../io/vertx/core/http/impl/EndpointKey.java | 5 +- .../core/http/impl/HttpChannelConnector.java | 9 +- .../vertx/core/http/impl/HttpClientImpl.java | 127 +++++---- .../io/vertx/core/net/ClientOptionsBase.java | 39 +++ .../io/vertx/core/net/NetClientOptions.java | 10 + .../io/vertx/core/net/impl/NetClientImpl.java | 11 +- .../io/vertx/core/net/impl/ProxyFilter.java | 58 ++++ .../io/vertx/core/http/Http1xProxyTest.java | 258 ++++++++++++++++-- src/test/java/io/vertx/core/net/NetTest.java | 20 ++ 19 files changed, 606 insertions(+), 90 deletions(-) create mode 100644 src/main/java/io/vertx/core/net/impl/ProxyFilter.java diff --git a/src/main/asciidoc/http.adoc b/src/main/asciidoc/http.adoc index f1d7dd510ed..55769d1348e 100644 --- a/src/main/asciidoc/http.adoc +++ b/src/main/asciidoc/http.adoc @@ -1828,6 +1828,24 @@ For a SOCKS5 proxy: The DNS resolution is always done on the proxy server, to achieve the functionality of a SOCKS4 client, it is necessary to resolve the DNS address locally. +Proxy options can also be set per request: + +[source,$lang] +---- +{@link examples.HTTPExamples#perRequestProxyOptions} +---- + +NOTE: A given host should always use the same proxy options: as HTTP requests are pooled, per request proxy +options are used when establishing the connection + +You can use {@link io.vertx.core.http.HttpClientOptions#setNonProxyHosts} to configure a list of host bypassing +the proxy. The lists accepts `*` wildcard for matching domains: + +[source,$lang] +---- +{@link examples.HTTPExamples#nonProxyHosts} +---- + ==== Handling of other protocols The HTTP proxy implementation supports getting ftp:// urls if the proxy supports diff --git a/src/main/asciidoc/net.adoc b/src/main/asciidoc/net.adoc index 437bd1ad283..deee3733405 100644 --- a/src/main/asciidoc/net.adoc +++ b/src/main/asciidoc/net.adoc @@ -779,6 +779,14 @@ Here's an example: The DNS resolution is always done on the proxy server, to achieve the functionality of a SOCKS4 client, it is necessary to resolve the DNS address locally. +You can use {@link io.vertx.core.net.NetClientOptions#setNonProxyHosts} to configure a list of host bypassing +the proxy. The lists accepts `*` wildcard for matching domains: + +[source,$lang] +---- +{@link examples.NetExamples#nonProxyHosts} +---- + === Using HA PROXY protocol diff --git a/src/main/generated/io/vertx/core/http/RequestOptionsConverter.java b/src/main/generated/io/vertx/core/http/RequestOptionsConverter.java index d4f8975c1bf..9cb60eac4c3 100644 --- a/src/main/generated/io/vertx/core/http/RequestOptionsConverter.java +++ b/src/main/generated/io/vertx/core/http/RequestOptionsConverter.java @@ -36,6 +36,11 @@ public static void fromJson(Iterable> json, obj.setPort(((Number)member.getValue()).intValue()); } break; + case "proxyOptions": + if (member.getValue() instanceof JsonObject) { + obj.setProxyOptions(new io.vertx.core.net.ProxyOptions((io.vertx.core.json.JsonObject)member.getValue())); + } + break; case "ssl": if (member.getValue() instanceof Boolean) { obj.setSsl((Boolean)member.getValue()); @@ -69,6 +74,9 @@ public static void toJson(RequestOptions obj, java.util.Map json if (obj.getPort() != null) { json.put("port", obj.getPort()); } + if (obj.getProxyOptions() != null) { + json.put("proxyOptions", obj.getProxyOptions().toJson()); + } if (obj.isSsl() != null) { json.put("ssl", obj.isSsl()); } diff --git a/src/main/generated/io/vertx/core/net/ClientOptionsBaseConverter.java b/src/main/generated/io/vertx/core/net/ClientOptionsBaseConverter.java index 0dbfe23d08d..04e170409fa 100644 --- a/src/main/generated/io/vertx/core/net/ClientOptionsBaseConverter.java +++ b/src/main/generated/io/vertx/core/net/ClientOptionsBaseConverter.java @@ -31,6 +31,16 @@ static void fromJson(Iterable> json, ClientO obj.setMetricsName((String)member.getValue()); } break; + case "nonProxyHosts": + if (member.getValue() instanceof JsonArray) { + java.util.ArrayList list = new java.util.ArrayList<>(); + ((Iterable)member.getValue()).forEach( item -> { + if (item instanceof String) + list.add((String)item); + }); + obj.setNonProxyHosts(list); + } + break; case "proxyOptions": if (member.getValue() instanceof JsonObject) { obj.setProxyOptions(new io.vertx.core.net.ProxyOptions((io.vertx.core.json.JsonObject)member.getValue())); @@ -57,6 +67,11 @@ static void toJson(ClientOptionsBase obj, java.util.Map json) { if (obj.getMetricsName() != null) { json.put("metricsName", obj.getMetricsName()); } + if (obj.getNonProxyHosts() != null) { + JsonArray array = new JsonArray(); + obj.getNonProxyHosts().forEach(item -> array.add(item)); + json.put("nonProxyHosts", array); + } if (obj.getProxyOptions() != null) { json.put("proxyOptions", obj.getProxyOptions().toJson()); } diff --git a/src/main/java/examples/HTTPExamples.java b/src/main/java/examples/HTTPExamples.java index 710fbef3b67..ad4fb970153 100644 --- a/src/main/java/examples/HTTPExamples.java +++ b/src/main/java/examples/HTTPExamples.java @@ -1022,6 +1022,31 @@ public void example59(Vertx vertx) { } + public void nonProxyHosts(Vertx vertx) { + + HttpClientOptions options = new HttpClientOptions() + .setProxyOptions(new ProxyOptions().setType(ProxyType.SOCKS5) + .setHost("localhost").setPort(1080) + .setUsername("username").setPassword("secret")) + .addNonProxyHost("*.foo.com") + .addNonProxyHost("localhost"); + HttpClient client = vertx.createHttpClient(options); + + } + + public void perRequestProxyOptions(HttpClient client, ProxyOptions proxyOptions) { + + client.request(new RequestOptions() + .setHost("example.com") + .setProxyOptions(proxyOptions)) + .compose(request -> request + .send() + .compose(HttpClientResponse::body)) + .onSuccess(body -> { + System.out.println("Received response"); + }); + } + public void example60(Vertx vertx) { HttpClientOptions options = new HttpClientOptions() diff --git a/src/main/java/examples/NetExamples.java b/src/main/java/examples/NetExamples.java index 9817d6ea0ee..b15cc8e9cfd 100755 --- a/src/main/java/examples/NetExamples.java +++ b/src/main/java/examples/NetExamples.java @@ -15,6 +15,8 @@ import io.vertx.core.Vertx; import io.vertx.core.buffer.Buffer; import io.vertx.core.http.ClientAuth; +import io.vertx.core.http.HttpClient; +import io.vertx.core.http.HttpClientOptions; import io.vertx.core.http.HttpServerOptions; import io.vertx.core.net.*; @@ -564,6 +566,17 @@ public void example47(Vertx vertx) { NetClient client = vertx.createNetClient(options); } + public void nonProxyHosts(Vertx vertx) { + + NetClientOptions options = new NetClientOptions() + .setProxyOptions(new ProxyOptions().setType(ProxyType.SOCKS5) + .setHost("localhost").setPort(1080) + .setUsername("username").setPassword("secret")) + .addNonProxyHost("*.foo.com") + .addNonProxyHost("localhost"); + NetClient client = vertx.createNetClient(options); + } + public void example48(Vertx vertx) throws CertificateException { SelfSignedCertificate certificate = SelfSignedCertificate.create(); diff --git a/src/main/java/io/vertx/core/http/HttpClient.java b/src/main/java/io/vertx/core/http/HttpClient.java index ea6aa29869e..4e48ba959f2 100644 --- a/src/main/java/io/vertx/core/http/HttpClient.java +++ b/src/main/java/io/vertx/core/http/HttpClient.java @@ -20,10 +20,12 @@ import io.vertx.core.MultiMap; import io.vertx.core.buffer.Buffer; import io.vertx.core.metrics.Measured; +import io.vertx.core.net.SocketAddress; import io.vertx.core.streams.ReadStream; import java.util.List; import java.util.function.Function; +import java.util.function.Predicate; /** * An asynchronous HTTP client. diff --git a/src/main/java/io/vertx/core/http/HttpClientOptions.java b/src/main/java/io/vertx/core/http/HttpClientOptions.java index 7c1db53f00c..4e6a9248a5a 100755 --- a/src/main/java/io/vertx/core/http/HttpClientOptions.java +++ b/src/main/java/io/vertx/core/http/HttpClientOptions.java @@ -1138,6 +1138,16 @@ public HttpClientOptions setProxyOptions(ProxyOptions proxyOptions) { return (HttpClientOptions) super.setProxyOptions(proxyOptions); } + @Override + public HttpClientOptions setNonProxyHosts(List nonProxyHosts) { + return (HttpClientOptions) super.setNonProxyHosts(nonProxyHosts); + } + + @Override + public HttpClientOptions addNonProxyHost(String nonProxyHost) { + return (HttpClientOptions) super.addNonProxyHost(nonProxyHost); + } + @Override public HttpClientOptions setLocalAddress(String localAddress) { return (HttpClientOptions) super.setLocalAddress(localAddress); diff --git a/src/main/java/io/vertx/core/http/RequestOptions.java b/src/main/java/io/vertx/core/http/RequestOptions.java index 053e1aa82ea..2bb69d27e09 100644 --- a/src/main/java/io/vertx/core/http/RequestOptions.java +++ b/src/main/java/io/vertx/core/http/RequestOptions.java @@ -16,6 +16,7 @@ import io.vertx.core.MultiMap; import io.vertx.core.VertxException; import io.vertx.core.json.JsonObject; +import io.vertx.core.net.ProxyOptions; import io.vertx.core.net.SocketAddress; import java.net.MalformedURLException; @@ -32,6 +33,11 @@ @DataObject(generateConverter = true) public class RequestOptions { + /** + * The default value for proxy options = {@code null} + */ + public static final ProxyOptions DEFAULT_PROXY_OPTIONS = null; + /** * The default value for server method = {@code null} */ @@ -72,6 +78,7 @@ public class RequestOptions { */ public static final long DEFAULT_TIMEOUT = 0; + private ProxyOptions proxyOptions; private SocketAddress server; private HttpMethod method; private String host; @@ -86,6 +93,7 @@ public class RequestOptions { * Default constructor */ public RequestOptions() { + proxyOptions = DEFAULT_PROXY_OPTIONS; server = DEFAULT_SERVER; method = DEFAULT_HTTP_METHOD; host = DEFAULT_HOST; @@ -102,6 +110,7 @@ public RequestOptions() { * @param other the options to copy */ public RequestOptions(RequestOptions other) { + setProxyOptions(other.proxyOptions); setServer(other.server); setMethod(other.method); setHost(other.host); @@ -155,6 +164,27 @@ public RequestOptions(JsonObject json) { } } + /** + * Get the proxy options override for connections + * + * @return proxy options override + */ + public ProxyOptions getProxyOptions() { + return proxyOptions; + } + + /** + * Override the {@link HttpClientOptions#setProxyOptions(ProxyOptions)} proxy options + * for connections. + * + * @param proxyOptions proxy options override object + * @return a reference to this, so the API can be used fluently + */ + public RequestOptions setProxyOptions(ProxyOptions proxyOptions) { + this.proxyOptions = proxyOptions; + return this; + } + /** * Get the server address to be used by the client request. * diff --git a/src/main/java/io/vertx/core/http/WebSocketConnectOptions.java b/src/main/java/io/vertx/core/http/WebSocketConnectOptions.java index 59923df4b4e..18c5e5e5f77 100644 --- a/src/main/java/io/vertx/core/http/WebSocketConnectOptions.java +++ b/src/main/java/io/vertx/core/http/WebSocketConnectOptions.java @@ -14,6 +14,7 @@ import io.vertx.codegen.annotations.GenIgnore; import io.vertx.core.MultiMap; import io.vertx.core.json.JsonObject; +import io.vertx.core.net.ProxyOptions; import java.util.ArrayList; import java.util.List; @@ -26,6 +27,11 @@ @DataObject(generateConverter = true) public class WebSocketConnectOptions extends RequestOptions { + /** + * The default value for proxy options = {@code null} + */ + public static final ProxyOptions DEFAULT_PROXY_OPTIONS = null; + /** * The default WebSocket version = {@link WebsocketVersion#V13} */ @@ -36,16 +42,19 @@ public class WebSocketConnectOptions extends RequestOptions { */ public static final List DEFAULT_SUB_PROTOCOLS = null; + private ProxyOptions proxyOptions; private WebsocketVersion version; private List subProtocols; public WebSocketConnectOptions() { + proxyOptions = DEFAULT_PROXY_OPTIONS; version = DEFAULT_VERSION; subProtocols = DEFAULT_SUB_PROTOCOLS; } public WebSocketConnectOptions(WebSocketConnectOptions other) { super(other); + this.proxyOptions = other.proxyOptions != null ? new ProxyOptions(other.proxyOptions) : null; this.version = other.version; this.subProtocols = other.subProtocols; } @@ -102,6 +111,27 @@ public WebSocketConnectOptions addSubProtocol(String subprotocol) { return this; } + /** + * Get the proxy options override for connections + * + * @return proxy options override + */ + public ProxyOptions getProxyOptions() { + return proxyOptions; + } + + /** + * Override the {@link HttpClientOptions#setProxyOptions(ProxyOptions)} proxy options + * for connections. + * + * @param proxyOptions proxy options override object + * @return a reference to this, so the API can be used fluently + */ + public RequestOptions setProxyOptions(ProxyOptions proxyOptions) { + this.proxyOptions = proxyOptions; + return this; + } + @Override public WebSocketConnectOptions setHost(String host) { return (WebSocketConnectOptions) super.setHost(host); diff --git a/src/main/java/io/vertx/core/http/impl/EndpointKey.java b/src/main/java/io/vertx/core/http/impl/EndpointKey.java index c0e19333e72..ba8c16adfde 100644 --- a/src/main/java/io/vertx/core/http/impl/EndpointKey.java +++ b/src/main/java/io/vertx/core/http/impl/EndpointKey.java @@ -10,6 +10,7 @@ */ package io.vertx.core.http.impl; +import io.vertx.core.net.ProxyOptions; import io.vertx.core.net.SocketAddress; final class EndpointKey { @@ -17,8 +18,9 @@ final class EndpointKey { final boolean ssl; final SocketAddress serverAddr; final SocketAddress peerAddr; + final ProxyOptions proxyOptions; - EndpointKey(boolean ssl, SocketAddress serverAddr, SocketAddress peerAddr) { + EndpointKey(boolean ssl, ProxyOptions proxyOptions, SocketAddress serverAddr, SocketAddress peerAddr) { if (serverAddr == null) { throw new NullPointerException("No null server address"); } @@ -26,6 +28,7 @@ final class EndpointKey { throw new NullPointerException("No null peer address"); } this.ssl = ssl; + this.proxyOptions = proxyOptions; this.peerAddr = peerAddr; this.serverAddr = serverAddr; } diff --git a/src/main/java/io/vertx/core/http/impl/HttpChannelConnector.java b/src/main/java/io/vertx/core/http/impl/HttpChannelConnector.java index 3af93552d9f..6d5a2e0d113 100644 --- a/src/main/java/io/vertx/core/http/impl/HttpChannelConnector.java +++ b/src/main/java/io/vertx/core/http/impl/HttpChannelConnector.java @@ -26,7 +26,6 @@ import io.vertx.core.impl.ContextInternal; import io.vertx.core.net.NetSocket; import io.vertx.core.net.ProxyOptions; -import io.vertx.core.net.ProxyType; import io.vertx.core.net.SocketAddress; import io.vertx.core.net.impl.NetClientImpl; import io.vertx.core.net.impl.NetSocketInternal; @@ -48,6 +47,7 @@ public class HttpChannelConnector { private final HttpClientImpl client; private final NetClientImpl netClient; private final HttpClientOptions options; + private final ProxyOptions proxyOptions; private final ClientMetrics metrics; private final boolean ssl; private final boolean useAlpn; @@ -57,6 +57,7 @@ public class HttpChannelConnector { public HttpChannelConnector(HttpClientImpl client, NetClientImpl netClient, + ProxyOptions proxyOptions, ClientMetrics metrics, HttpVersion version, boolean ssl, @@ -67,6 +68,7 @@ public HttpChannelConnector(HttpClientImpl client, this.netClient = netClient; this.metrics = metrics; this.options = client.getOptions(); + this.proxyOptions = proxyOptions; this.ssl = ssl; this.useAlpn = useAlpn; this.version = version; @@ -79,11 +81,6 @@ public SocketAddress server() { } private void connect(EventLoopContext context, Promise promise) { - ProxyOptions proxyOptions = this.options.getProxyOptions(); - if (proxyOptions != null && !ssl && proxyOptions.getType()== ProxyType.HTTP) { - // http proxy requests are handled in HttpClientImpl, everything else can use netty proxy handler - proxyOptions = null; - } netClient.connectInternal(proxyOptions, server, peerAddress, this.options.isForceSni() ? peerAddress.host() : null, ssl, useAlpn, promise, context, 0); } diff --git a/src/main/java/io/vertx/core/http/impl/HttpClientImpl.java b/src/main/java/io/vertx/core/http/impl/HttpClientImpl.java index 9a5e56ba0e8..e4f5bfcf36a 100644 --- a/src/main/java/io/vertx/core/http/impl/HttpClientImpl.java +++ b/src/main/java/io/vertx/core/http/impl/HttpClientImpl.java @@ -23,6 +23,7 @@ import io.vertx.core.impl.EventLoopContext; import io.vertx.core.net.NetClientOptions; import io.vertx.core.net.impl.NetClientImpl; +import io.vertx.core.net.impl.ProxyFilter; import io.vertx.core.net.impl.pool.ConnectionManager; import io.vertx.core.impl.ContextInternal; import io.vertx.core.impl.future.PromiseInternal; @@ -49,6 +50,7 @@ import java.util.Objects; import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.Predicate; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -123,13 +125,13 @@ public class HttpClientImpl implements HttpClient, MetricsProvider, Closeable { private final HttpClientOptions options; private final ConnectionManager webSocketCM; private final ConnectionManager> httpCM; - private final ProxyType proxyType; private final NetClientImpl netClient; private final HttpClientMetrics metrics; private final boolean keepAlive; private final boolean pipelining; private final CloseFuture closeFuture; private long timerID; + private Predicate proxyFilter; private volatile Handler connectionHandler; private volatile Function> redirectHandler = DEFAULT_HANDLER; @@ -155,6 +157,7 @@ public HttpClientImpl(VertxInternal vertx, HttpClientOptions options, CloseFutur if (!keepAlive && pipelining) { throw new IllegalStateException("Cannot have pipelining with no keep alive"); } + this.proxyFilter = options.getNonProxyHosts() != null ? ProxyFilter.nonProxyHosts(options.getNonProxyHosts()) : ProxyFilter.DEFAULT_PROXY_FILTER; this.netClient = new NetClientImpl( vertx, new NetClientOptions(options) @@ -167,7 +170,6 @@ public HttpClientImpl(VertxInternal vertx, HttpClientOptions options, CloseFutur closeFuture); webSocketCM = webSocketConnectionManager(); httpCM = httpConnectionManager(); - proxyType = options.getProxyOptions() != null ? options.getProxyOptions().getType() : null; if (options.getPoolCleanerPeriod() > 0 && (options.getKeepAliveTimeout() > 0L || options.getHttp2KeepAliveTimeout() > 0L)) { PoolChecker checker = new PoolChecker(this); timerID = vertx.setTimer(options.getPoolCleanerPeriod(), checker); @@ -210,7 +212,7 @@ private ConnectionManager> httpConnecti int maxPoolSize = Math.max(options.getMaxPoolSize(), options.getHttp2MaxPoolSize()); return new ConnectionManager<>((key, ctx, dispose) -> { ClientMetrics metrics = this.metrics != null ? this.metrics.createEndpointMetrics(key.serverAddr, maxPoolSize) : null; - HttpChannelConnector connector = new HttpChannelConnector(this, netClient, metrics, options.getProtocolVersion(), key.ssl, options.isUseAlpn(), key.peerAddr, key.serverAddr); + HttpChannelConnector connector = new HttpChannelConnector(this, netClient, key.proxyOptions, metrics, options.getProtocolVersion(), key.ssl, options.isUseAlpn(), key.peerAddr, key.serverAddr); return new SharedClientHttpStreamEndpoint( this, metrics, @@ -226,7 +228,7 @@ private ConnectionManager webSocketConnection int maxPoolSize = options.getMaxWebSockets(); return new ConnectionManager<>((key, ctx, dispose) -> { ClientMetrics metrics = this.metrics != null ? this.metrics.createEndpointMetrics(key.serverAddr, maxPoolSize) : null; - HttpChannelConnector connector = new HttpChannelConnector(this, netClient, metrics, HttpVersion.HTTP_1_1, key.ssl, false, key.peerAddr, key.serverAddr); + HttpChannelConnector connector = new HttpChannelConnector(this, netClient, key.proxyOptions, metrics, HttpVersion.HTTP_1_1, key.ssl, false, key.peerAddr, key.serverAddr); return new WebSocketEndpoint(null, maxPoolSize, connector, dispose); }); } @@ -243,6 +245,13 @@ private int getPort(RequestOptions request) { return options.getDefaultPort(); } + private ProxyOptions getProxyOptions(ProxyOptions proxyOptions) { + if (proxyOptions == null) { + proxyOptions = options.getProxyOptions(); + } + return proxyOptions; + } + private String getHost(RequestOptions request) { String host = request.getHost(); if (host != null) { @@ -271,7 +280,7 @@ public Future connect(SocketAddress server) { */ public Future connect(SocketAddress server, SocketAddress peer) { EventLoopContext context = (EventLoopContext) vertx.getOrCreateContext(); - HttpChannelConnector connector = new HttpChannelConnector(this, netClient, null, options.getProtocolVersion(), options.isSsl(), options.isUseAlpn(), peer, server); + HttpChannelConnector connector = new HttpChannelConnector(this, netClient, null, null, options.getProtocolVersion(), options.isSsl(), options.isUseAlpn(), peer, server); return connector.httpConnect(context); } @@ -281,10 +290,16 @@ public void webSocket(WebSocketConnectOptions connectOptions, Handler promise) { + ProxyOptions proxyOptions = getProxyOptions(connectOptions.getProxyOptions()); int port = getPort(connectOptions); String host = getHost(connectOptions); SocketAddress addr = SocketAddress.inetSocketAddress(port, host); - EndpointKey key = new EndpointKey(connectOptions.isSsl() != null ? connectOptions.isSsl() : options.isSsl(), addr, addr); + if (proxyFilter != null) { + if (!proxyFilter.test(addr)) { + proxyOptions = null; + } + } + EndpointKey key = new EndpointKey(connectOptions.isSsl() != null ? connectOptions.isSsl() : options.isSsl(), proxyOptions, addr, addr); ContextInternal ctx = promise.context(); EventLoopContext eventLoopContext; if (ctx instanceof EventLoopContext) { @@ -394,38 +409,25 @@ public void webSocketAbs(String url, MultiMap headers, WebsocketVersion version, public void request(RequestOptions options, Handler> handler) { ContextInternal ctx = vertx.getOrCreateContext(); PromiseInternal promise = ctx.promise(handler); - request(options, promise); + doRequest(options, promise); } @Override public Future request(RequestOptions options) { ContextInternal ctx = vertx.getOrCreateContext(); PromiseInternal promise = ctx.promise(); - request(options, promise); + doRequest(options, promise); return promise.future(); } - private void request(RequestOptions options, PromiseInternal promise) { - request(options.getMethod(), options.getServer(), getHost(options), getPort(options), options.isSsl(), options.getURI(), options.getHeaders(), options.getTimeout(), options.getFollowRedirects(), promise); - } - @Override public void request(HttpMethod method, int port, String host, String requestURI, Handler> handler) { - ContextInternal ctx = vertx.getOrCreateContext(); - PromiseInternal promise = ctx.promise(handler); - request(method, port, host, requestURI, promise); + request(new RequestOptions().setMethod(method).setPort(port).setHost(host).setURI(requestURI), handler); } @Override public Future request(HttpMethod method, int port, String host, String requestURI) { - ContextInternal ctx = vertx.getOrCreateContext(); - PromiseInternal promise = ctx.promise(); - request(method, port, host, requestURI, promise); - return promise.future(); - } - - private void request(HttpMethod method, int port, String host, String requestURI, PromiseInternal promise) { - request(method, null, host, port, null, requestURI, null, 0L, null, promise); + return request(new RequestOptions().setMethod(method).setPort(port).setHost(host).setURI(requestURI)); } @Override @@ -508,6 +510,11 @@ public Function> redirectHandler() { return redirectHandler; } + public HttpClient proxyFilter(Predicate filter) { + proxyFilter = filter; + return this; + } + public HttpClientOptions getOptions() { return options; } @@ -519,45 +526,52 @@ public VertxInternal getVertx() { return vertx; } - private void request(HttpMethod method, - SocketAddress server, - String host, - int port, - Boolean ssl, - String requestURI, - MultiMap headers, - long timeout, - Boolean followRedirects, - PromiseInternal requestPromise) { + private void doRequest(RequestOptions request, PromiseInternal promise) { + String host = getHost(request); + int port = getPort(request); + SocketAddress server = request.getServer(); + if (server == null) { + server = SocketAddress.inetSocketAddress(port, host); + } + ProxyOptions proxyOptions = getProxyOptions(request.getProxyOptions()); + HttpMethod method = request.getMethod(); + String requestURI = request.getURI(); + Boolean ssl = request.isSsl(); + MultiMap headers = request.getHeaders(); + long timeout = request.getTimeout(); + Boolean followRedirects = request.getFollowRedirects(); Objects.requireNonNull(method, "no null method accepted"); Objects.requireNonNull(host, "no null host accepted"); Objects.requireNonNull(requestURI, "no null requestURI accepted"); - boolean useAlpn = options.isUseAlpn(); - boolean useSSL = ssl != null ? ssl : options.isSsl(); - if (!useAlpn && useSSL && options.getProtocolVersion() == HttpVersion.HTTP_2) { + boolean useAlpn = this.options.isUseAlpn(); + boolean useSSL = ssl != null ? ssl : this.options.isSsl(); + if (!useAlpn && useSSL && this.options.getProtocolVersion() == HttpVersion.HTTP_2) { throw new IllegalArgumentException("Must enable ALPN when using H2"); } checkClosed(); - boolean useProxy = !useSSL && proxyType == ProxyType.HTTP; - - if (useProxy) { - // If the requestURI is as not absolute URI then we do not recompute one for the proxy - if (!ABS_URI_START_PATTERN.matcher(requestURI).find()) { - int defaultPort = 80; - String addPort = (port != -1 && port != defaultPort) ? (":" + port) : ""; - requestURI = (ssl == Boolean.TRUE ? "https://" : "http://") + host + addPort + requestURI; + if (proxyFilter != null) { + if (!proxyFilter.test(server)) { + proxyOptions = null; } - ProxyOptions proxyOptions = options.getProxyOptions(); - if (proxyOptions.getUsername() != null && proxyOptions.getPassword() != null) { - if (headers == null) { - headers = HttpHeaders.headers(); + } + if (proxyOptions != null) { + if (!useSSL && proxyOptions.getType() == ProxyType.HTTP) { + // If the requestURI is as not absolute URI then we do not recompute one for the proxy + if (!ABS_URI_START_PATTERN.matcher(requestURI).find()) { + int defaultPort = 80; + String addPort = (port != -1 && port != defaultPort) ? (":" + port) : ""; + requestURI = (ssl == Boolean.TRUE ? "https://" : "http://") + host + addPort + requestURI; } - headers.add("Proxy-Authorization", "Basic " + Base64.getEncoder() - .encodeToString((proxyOptions.getUsername() + ":" + proxyOptions.getPassword()).getBytes())); + if (proxyOptions.getUsername() != null && proxyOptions.getPassword() != null) { + if (headers == null) { + headers = HttpHeaders.headers(); + } + headers.add("Proxy-Authorization", "Basic " + Base64.getEncoder() + .encodeToString((proxyOptions.getUsername() + ":" + proxyOptions.getPassword()).getBytes())); + } + server = SocketAddress.inetSocketAddress(proxyOptions.getPort(), proxyOptions.getHost()); + proxyOptions = null; } - server = SocketAddress.inetSocketAddress(proxyOptions.getPort(), proxyOptions.getHost()); - } else if (server == null) { - server = SocketAddress.inetSocketAddress(port, host); } String peerHost = host; @@ -565,10 +579,10 @@ private void request(HttpMethod method, peerHost = peerHost.substring(0, peerHost.length() - 1); } SocketAddress peerAddress = SocketAddress.inetSocketAddress(port, peerHost); - request(method, peerAddress, server, host, port, useSSL, requestURI, headers, timeout, followRedirects, requestPromise); + doRequest(method, peerAddress, server, host, port, useSSL, requestURI, headers, timeout, followRedirects, proxyOptions, promise); } - private void request( + private void doRequest( HttpMethod method, SocketAddress peerAddress, SocketAddress server, @@ -579,9 +593,10 @@ private void request( MultiMap headers, long timeout, Boolean followRedirects, + ProxyOptions proxyOptions, PromiseInternal requestPromise) { ContextInternal ctx = requestPromise.context(); - EndpointKey key = new EndpointKey(useSSL, server, peerAddress); + EndpointKey key = new EndpointKey(useSSL, proxyOptions, server, peerAddress); EventLoopContext eventLoopContext; if (ctx instanceof EventLoopContext) { eventLoopContext = (EventLoopContext) ctx; diff --git a/src/main/java/io/vertx/core/net/ClientOptionsBase.java b/src/main/java/io/vertx/core/net/ClientOptionsBase.java index c58bb90b64c..15da129cc71 100755 --- a/src/main/java/io/vertx/core/net/ClientOptionsBase.java +++ b/src/main/java/io/vertx/core/net/ClientOptionsBase.java @@ -15,6 +15,8 @@ import io.vertx.core.buffer.Buffer; import io.vertx.core.json.JsonObject; +import java.util.ArrayList; +import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; @@ -46,6 +48,7 @@ public abstract class ClientOptionsBase extends TCPSSLOptions { private String metricsName; private ProxyOptions proxyOptions; private String localAddress; + private List nonProxyHosts; /** * Default constructor @@ -178,6 +181,42 @@ public ProxyOptions getProxyOptions() { return proxyOptions; } + /** + * @return the list of non proxies hosts + */ + public List getNonProxyHosts() { + return nonProxyHosts; + } + + /** + * Set a list of remote hosts that are not proxied when the client is configured to use a proxy. This + * list serves the same purpose than the JVM {@code nonProxyHosts} configuration. + * + *

Entries can use the * wildcard character for pattern matching, e.g *.example.com matches + * www.example.com. + * + * @param nonProxyHosts the list of non proxies hosts + * @return a reference to this, so the API can be used fluently + */ + public ClientOptionsBase setNonProxyHosts(List nonProxyHosts) { + this.nonProxyHosts = nonProxyHosts; + return this; + } + + /** + * Add a {@code host} to the {@link #getNonProxyHosts()} list. + * + * @param host the added host + * @return a reference to this, so the API can be used fluently + */ + public ClientOptionsBase addNonProxyHost(String host) { + if (nonProxyHosts == null) { + nonProxyHosts = new ArrayList<>(); + } + nonProxyHosts.add(host); + return this; + } + /** * @return the local interface to bind for network connections. */ diff --git a/src/main/java/io/vertx/core/net/NetClientOptions.java b/src/main/java/io/vertx/core/net/NetClientOptions.java index bf4460487d6..1f7072cf193 100755 --- a/src/main/java/io/vertx/core/net/NetClientOptions.java +++ b/src/main/java/io/vertx/core/net/NetClientOptions.java @@ -377,6 +377,16 @@ public NetClientOptions setProxyOptions(ProxyOptions proxyOptions) { return (NetClientOptions) super.setProxyOptions(proxyOptions); } + @Override + public NetClientOptions setNonProxyHosts(List nonProxyHosts) { + return (NetClientOptions) super.setNonProxyHosts(nonProxyHosts); + } + + @Override + public NetClientOptions addNonProxyHost(String nonProxyHost) { + return (NetClientOptions) super.addNonProxyHost(nonProxyHost); + } + @Override public NetClientOptions setLocalAddress(String localAddress) { return (NetClientOptions) super.setLocalAddress(localAddress); diff --git a/src/main/java/io/vertx/core/net/impl/NetClientImpl.java b/src/main/java/io/vertx/core/net/impl/NetClientImpl.java index 4bb6b64e02b..3b7d5954bbf 100644 --- a/src/main/java/io/vertx/core/net/impl/NetClientImpl.java +++ b/src/main/java/io/vertx/core/net/impl/NetClientImpl.java @@ -45,6 +45,7 @@ import java.net.ConnectException; import java.util.Objects; import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; /** * @@ -65,6 +66,7 @@ public class NetClientImpl implements MetricsProvider, NetClient, Closeable { private final ChannelGroup channelGroup; private final TCPMetrics metrics; private final CloseFuture closeFuture; + private final Predicate proxyFilter; public NetClientImpl(VertxInternal vertx, NetClientOptions options, CloseFuture closeFuture) { this.vertx = vertx; @@ -76,6 +78,7 @@ public NetClientImpl(VertxInternal vertx, NetClientOptions options, CloseFuture this.idleTimeout = options.getIdleTimeout(); this.idleTimeoutUnit = options.getIdleTimeoutUnit(); this.closeFuture = closeFuture; + this.proxyFilter = options.getNonProxyHosts() != null ? ProxyFilter.nonProxyHosts(options.getNonProxyHosts()) : ProxyFilter.DEFAULT_PROXY_FILTER; sslHelper.validate(vertx); } @@ -194,7 +197,13 @@ private void connect(SocketAddress remoteAddress, String serverName, PromiseJulien Viet + */ +public interface ProxyFilter extends Predicate { + + ProxyFilter DEFAULT_PROXY_FILTER = so -> !so.isDomainSocket(); + + static ProxyFilter nonProxyHosts(List nonProxyHosts) { + List filterElts = nonProxyHosts.stream().map(nonProxyHost -> { + if (nonProxyHost.contains("*")) { + String pattern = nonProxyHost + .replaceAll("\\.", "\\.") + .replaceAll("\\*", ".*"); + return Pattern.compile(pattern); + } else { + return nonProxyHost; + } + }).collect(Collectors.toList()); + return so -> { + if (so.isDomainSocket()) { + return false; + } else { + String host = so.host(); + for (Object filterElt : filterElts) { + if (filterElt instanceof Pattern) { + if (((Pattern) filterElt).matcher(host).matches()) { + return false; + } + } else if (filterElt.equals(host)) { + return false; + } + } + } + return true; + }; + } +} diff --git a/src/test/java/io/vertx/core/http/Http1xProxyTest.java b/src/test/java/io/vertx/core/http/Http1xProxyTest.java index 1951f0798ae..d9169223d5d 100644 --- a/src/test/java/io/vertx/core/http/Http1xProxyTest.java +++ b/src/test/java/io/vertx/core/http/Http1xProxyTest.java @@ -10,31 +10,140 @@ */ package io.vertx.core.http; -import io.vertx.core.AsyncResult; -import io.vertx.core.Handler; +import io.vertx.core.Future; +import io.vertx.core.VertxOptions; +import io.vertx.core.buffer.Buffer; +import io.vertx.core.http.impl.HttpClientImpl; import io.vertx.core.net.ProxyOptions; import io.vertx.core.net.ProxyType; -import org.junit.Ignore; +import io.vertx.core.net.SocketAddress; +import io.vertx.test.tls.Cert; import org.junit.Test; -import java.util.function.Consumer; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.function.Supplier; public class Http1xProxyTest extends HttpTestBase { + @Override + protected VertxOptions getOptions() { + VertxOptions options = super.getOptions(); + options.getAddressResolverOptions().setHostsValue(Buffer.buffer("" + + "127.0.0.1 localhost\n" + + "127.0.0.1 www1.example1.com\n" + + "127.0.0.1 www2.example1.com\n" + + "127.0.0.1 www1.example2.com\n" + + "127.0.0.1 www2.example2.com\n" + )); + return options; + } + @Test public void testHttpProxyRequest() throws Exception { startProxy(null, ProxyType.HTTP); client.close(); client = vertx.createHttpClient(new HttpClientOptions() .setProxyOptions(new ProxyOptions().setType(ProxyType.HTTP).setHost("localhost").setPort(proxy.getPort()))); - testHttpProxyRequest2(handler -> { - client.request(new RequestOptions() - .setHost(DEFAULT_HTTP_HOST) - .setPort(DEFAULT_HTTP_PORT) - .setURI("/") - ).compose(HttpClientRequest::send) - .onComplete(handler); + testHttpProxyRequest(() -> client.request(new RequestOptions() + .setHost(DEFAULT_HTTP_HOST) + .setPort(DEFAULT_HTTP_PORT) + .setURI("/") + ).compose(HttpClientRequest::send)).onComplete(onSuccess(v -> { + assertProxiedRequest(DEFAULT_HTTP_HOST); + testComplete(); + })); + await(); + } + + @Test + public void testHttpProxyRequest2() throws Exception { + startProxy(null, ProxyType.HTTP); + testHttpProxyRequest(() -> client.request(new RequestOptions() + .setProxyOptions(new ProxyOptions().setType(ProxyType.HTTP).setHost("localhost").setPort(proxy.getPort())) + .setHost(DEFAULT_HTTP_HOST) + .setPort(DEFAULT_HTTP_PORT) + .setURI("/") + ).compose(HttpClientRequest::send)).onComplete(onSuccess(v -> { + assertProxiedRequest(DEFAULT_HTTP_HOST); + testComplete(); + })); + await(); + } + + @Test + public void testAcceptFilter() throws Exception { + testFilter(true); + } + + @Test + public void testRejectFilter() throws Exception { + testFilter(false); + } + + private void testFilter(boolean accept) throws Exception { + startProxy(null, ProxyType.HTTP); + client.close(); + client = vertx.createHttpClient(new HttpClientOptions() + .setProxyOptions(new ProxyOptions().setType(ProxyType.HTTP).setHost("localhost").setPort(proxy.getPort()))); + Set filtered = Collections.synchronizedSet(new HashSet<>()); + ((HttpClientImpl)client).proxyFilter(so -> { + filtered.add(so); + return accept; }); + testHttpProxyRequest(() -> client.request(new RequestOptions() + .setHost(DEFAULT_HTTP_HOST) + .setPort(DEFAULT_HTTP_PORT) + .setURI("/") + ).compose(HttpClientRequest::send)).onComplete(onSuccess(v -> { + if (accept) { + assertProxiedRequest(DEFAULT_HTTP_HOST); + } + assertEquals(Collections.singleton(SocketAddress.inetSocketAddress(8080, "localhost")), filtered); + testComplete(); + })); + await(); + } + + @Test + public void testNonProxyHosts1() throws Exception { + testNonProxyHosts(Collections.singletonList("www1.example1.com"), "www1.example1.com", false); + } + + @Test + public void testNonProxyHosts2() throws Exception { + testNonProxyHosts(Collections.singletonList("www1.example1.com"), "www2.example1.com", true); + } + + @Test + public void testNonProxyHosts3() throws Exception { + testNonProxyHosts(Collections.singletonList("*.example2.com"), "www1.example2.com", false); + } + + @Test + public void testNonProxyHosts4() throws Exception { + testNonProxyHosts(Collections.singletonList("*.example2.com"), "www2.example2.com", false); + } + + private void testNonProxyHosts(List nonProxyHosts, String host, boolean proxied) throws Exception { + startProxy(null, ProxyType.HTTP); + client.close(); + client = vertx.createHttpClient(new HttpClientOptions() + .setNonProxyHosts(nonProxyHosts) + .setProxyOptions(new ProxyOptions().setType(ProxyType.HTTP).setHost("localhost").setPort(proxy.getPort()))); + testHttpProxyRequest(() -> client.request(new RequestOptions() + .setHost(host) + .setPort(DEFAULT_HTTP_PORT) + .setURI("/") + ).compose(HttpClientRequest::send)).onComplete(onSuccess(v -> { + if (proxied) { + assertProxiedRequest(host); + } + testComplete(); + })); + await(); } @Test @@ -43,27 +152,35 @@ public void testHttpProxyRequestOverrideClientSsl() throws Exception { client.close(); client = vertx.createHttpClient(new HttpClientOptions() .setSsl(true).setProxyOptions(new ProxyOptions().setType(ProxyType.HTTP).setHost("localhost").setPort(proxy.getPort()))); - testHttpProxyRequest2(handler -> { - client.request(new RequestOptions().setSsl(false).setHost("localhost").setPort(8080)) - .compose(HttpClientRequest::send) - .onComplete(handler); - }); + testHttpProxyRequest(() -> client + .request(new RequestOptions().setSsl(false).setHost("localhost").setPort(8080)) + .compose(HttpClientRequest::send)).onComplete(onSuccess(v -> { + assertProxiedRequest(DEFAULT_HTTP_HOST); + testComplete(); + })); + await(); + } + + private void assertProxiedRequest(String host) { + assertNotNull("request did not go through proxy", proxy.getLastUri()); + assertEquals("Host header doesn't contain target host", host + ":8080", proxy.getLastRequestHeaders().get("Host")); } - private void testHttpProxyRequest2(Consumer>> reqFact) throws Exception { + private Future testHttpProxyRequest(Supplier> reqFact) throws Exception { server.requestHandler(req -> { req.response().end(); }); - server.listen(onSuccess(s -> { - reqFact.accept(onSuccess(resp -> { - assertEquals(200, resp.statusCode()); - assertNotNull("request did not go through proxy", proxy.getLastUri()); - assertEquals("Host header doesn't contain target host", "localhost:8080", proxy.getLastRequestHeaders().get("Host")); - testComplete(); - })); - })); - await(); + return server.listen().compose(s -> { + return reqFact.get().compose(resp -> { + int sc = resp.statusCode(); + if (sc == 200) { + return Future.succeededFuture(); + } else { + return Future.failedFuture("Was expected 200 response instead of " + sc); + } + }); + }); } @Test @@ -173,4 +290,93 @@ public void testHttpSocksProxyRequestAuth() throws Exception { })); await(); } + + @Test + public void testWssHttpProxy() throws Exception { + startProxy(null, ProxyType.HTTP); + testWebSocket(createBaseServerOptions().setSsl(true) + .setKeyCertOptions(Cert.SERVER_JKS.get()), new HttpClientOptions() + .setSsl(true) + .setTrustOptions(Cert.SERVER_JKS.get()) + .setProxyOptions(new ProxyOptions() + .setType(ProxyType.HTTP) + .setHost(DEFAULT_HTTP_HOST) + .setPort(proxy.getPort())), true); + } + + @Test + public void testWsHttpProxy() throws Exception { + startProxy(null, ProxyType.HTTP); + testWebSocket(createBaseServerOptions(), new HttpClientOptions() + .setProxyOptions(new ProxyOptions() + .setType(ProxyType.HTTP) + .setHost(DEFAULT_HTTP_HOST) + .setPort(proxy.getPort())), true); + } + + @Test + public void testWssSocks5Proxy() throws Exception { + startProxy(null, ProxyType.SOCKS5); + testWebSocket(createBaseServerOptions().setSsl(true) + .setKeyCertOptions(Cert.SERVER_JKS.get()), new HttpClientOptions() + .setSsl(true) + .setTrustOptions(Cert.SERVER_JKS.get()) + .setProxyOptions(new ProxyOptions() + .setType(ProxyType.SOCKS5) + .setHost(DEFAULT_HTTP_HOST) + .setPort(proxy.getPort())), true); + } + + @Test + public void testWsSocks5Proxy() throws Exception { + startProxy(null, ProxyType.SOCKS5); + testWebSocket(createBaseServerOptions(), new HttpClientOptions() + .setProxyOptions(new ProxyOptions() + .setType(ProxyType.SOCKS5) + .setHost(DEFAULT_HTTP_HOST) + .setPort(proxy.getPort())), true); + } + + @Test + public void testWsNonProxyHosts() throws Exception { + startProxy(null, ProxyType.HTTP); + testWebSocket(createBaseServerOptions(), new HttpClientOptions() + .addNonProxyHost("localhost") + .setProxyOptions(new ProxyOptions() + .setType(ProxyType.HTTP) + .setHost(DEFAULT_HTTP_HOST) + .setPort(proxy.getPort())), false); + } + + private void testWebSocket(HttpServerOptions serverOptions, HttpClientOptions clientOptions, boolean proxied) throws Exception { + server.close(); + server = vertx.createHttpServer(serverOptions); + client.close(); + client = vertx.createHttpClient(clientOptions); + server.webSocketHandler(ws -> { + ws.handler(buff -> { + ws.write(buff); + ws.close(); + }); + }); + server.listen(DEFAULT_HTTPS_PORT, DEFAULT_HTTPS_HOST).onSuccess(s -> { + client.webSocket(DEFAULT_HTTPS_PORT, DEFAULT_HTTPS_HOST, "/").onSuccess(ws -> { + ws.handler(buff -> { + ws.close(onSuccess(v -> { + if (proxied) { + assertNotNull("request did not go through proxy", proxy.getLastUri()); + if (clientOptions.getProxyOptions().getType() == ProxyType.HTTP) { + assertEquals("Host header doesn't contain target host", "localhost:4043", proxy.getLastRequestHeaders().get("Host")); + } + } else { + assertNull("request did go through proxy", proxy.getLastUri()); + } + testComplete(); + })); + }); + ws.write(Buffer.buffer("Hello world")); + }); + }); + await(); + } } diff --git a/src/test/java/io/vertx/core/net/NetTest.java b/src/test/java/io/vertx/core/net/NetTest.java index 8fbf8d76b2a..2c9e9f034a5 100755 --- a/src/test/java/io/vertx/core/net/NetTest.java +++ b/src/test/java/io/vertx/core/net/NetTest.java @@ -3047,6 +3047,26 @@ public void testWithSocks4LocalResolver() throws Exception { await(); } + @Test + public void testNonProxyHosts() throws Exception { + NetClientOptions clientOptions = new NetClientOptions() + .addNonProxyHost("example.com") + .setProxyOptions(new ProxyOptions().setType(ProxyType.HTTP).setPort(13128)); + NetClient client = vertx.createNetClient(clientOptions); + server.connectHandler(sock -> { + + }); + proxy = new HttpProxy(null); + proxy.start(vertx); + server.listen(1234, "localhost", onSuccess(s -> { + client.connect(1234, "example.com", onSuccess(so -> { + assertNull(proxy.getLastUri()); + testComplete(); + })); + })); + await(); + } + @Test public void testTLSHostnameCertCheckCorrect() { server.close();