Non proxy host options for TCP clients, per HTTP request proxy options.

Clients have been modified to filter proxy options based on a list of hosts support. Host declaration accept wildcard match like JVM nonProxyHosts list.

HTTP requests declares now a ProxyOptions property that will set the proxy options per request and override the client configuration.

fixes eclipse-vertx#2600
fixes eclipse-vertx#3795

src/main/asciidoc/http.adoc

@@ -1828,6 +1828,24 @@ For a SOCKS5 proxy:
 The DNS resolution is always done on the proxy server, to achieve the functionality of a SOCKS4 client, it is necessary
 to resolve the DNS address locally.
 
+Proxy options can also be set per request:
+
+[source,$lang]
+----
+{@link examples.HTTPExamples#perRequestProxyOptions}
+----
+
+NOTE: A given host should always use the same proxy options: as HTTP requests are pooled, per request proxy
+options are used when establishing the connection
+
+You can use {@link io.vertx.core.http.HttpClientOptions#setNonProxyHosts} to configure a list of host bypassing
+the proxy. The lists accepts `*` wildcard for matching domains:
+
+[source,$lang]
+----
+{@link examples.HTTPExamples#nonProxyHosts}
+----
+
 ==== Handling of other protocols
 
 The HTTP proxy implementation supports getting ftp:// urls if the proxy supports

src/main/asciidoc/net.adoc

@@ -779,6 +779,14 @@ Here's an example:
 The DNS resolution is always done on the proxy server, to achieve the functionality of a SOCKS4 client, it is necessary
 to resolve the DNS address locally.
 
+You can use {@link io.vertx.core.net.NetClientOptions#setNonProxyHosts} to configure a list of host bypassing
+the proxy. The lists accepts `*` wildcard for matching domains:
+
+[source,$lang]
+----
+{@link examples.NetExamples#nonProxyHosts}
+----
+
 
 === Using HA PROXY protocol
 

src/main/generated/io/vertx/core/http/RequestOptionsConverter.java

@@ -36,6 +36,11 @@ public static void fromJson(Iterable<java.util.Map.Entry<String, Object>> json,
             obj.setPort(((Number)member.getValue()).intValue());
           }
           break;
+        case "proxyOptions":
+          if (member.getValue() instanceof JsonObject) {
+            obj.setProxyOptions(new io.vertx.core.net.ProxyOptions((io.vertx.core.json.JsonObject)member.getValue()));
+          }
+          break;
         case "ssl":
           if (member.getValue() instanceof Boolean) {
             obj.setSsl((Boolean)member.getValue());
@@ -69,6 +74,9 @@ public static void toJson(RequestOptions obj, java.util.Map<String, Object> json
     if (obj.getPort() != null) {
       json.put("port", obj.getPort());
     }
+    if (obj.getProxyOptions() != null) {
+      json.put("proxyOptions", obj.getProxyOptions().toJson());
+    }
     if (obj.isSsl() != null) {
       json.put("ssl", obj.isSsl());
     }

src/main/generated/io/vertx/core/net/ClientOptionsBaseConverter.java

@@ -31,6 +31,16 @@ static void fromJson(Iterable<java.util.Map.Entry<String, Object>> json, ClientO
             obj.setMetricsName((String)member.getValue());
           }
           break;
+        case "nonProxyHosts":
+          if (member.getValue() instanceof JsonArray) {
+            java.util.ArrayList<java.lang.String> list =  new java.util.ArrayList<>();
+            ((Iterable<Object>)member.getValue()).forEach( item -> {
+              if (item instanceof String)
+                list.add((String)item);
+            });
+            obj.setNonProxyHosts(list);
+          }
+          break;
         case "proxyOptions":
           if (member.getValue() instanceof JsonObject) {
             obj.setProxyOptions(new io.vertx.core.net.ProxyOptions((io.vertx.core.json.JsonObject)member.getValue()));
@@ -57,6 +67,11 @@ static void toJson(ClientOptionsBase obj, java.util.Map<String, Object> json) {
     if (obj.getMetricsName() != null) {
       json.put("metricsName", obj.getMetricsName());
     }
+    if (obj.getNonProxyHosts() != null) {
+      JsonArray array = new JsonArray();
+      obj.getNonProxyHosts().forEach(item -> array.add(item));
+      json.put("nonProxyHosts", array);
+    }
     if (obj.getProxyOptions() != null) {
       json.put("proxyOptions", obj.getProxyOptions().toJson());
     }

src/main/java/examples/HTTPExamples.java

@@ -1022,6 +1022,31 @@ public void example59(Vertx vertx) {
 
   }
 
+  public void nonProxyHosts(Vertx vertx) {
+
+    HttpClientOptions options = new HttpClientOptions()
+      .setProxyOptions(new ProxyOptions().setType(ProxyType.SOCKS5)
+        .setHost("localhost").setPort(1080)
+        .setUsername("username").setPassword("secret"))
+      .addNonProxyHost("*.foo.com")
+      .addNonProxyHost("localhost");
+    HttpClient client = vertx.createHttpClient(options);
+
+  }
+
+  public void perRequestProxyOptions(HttpClient client, ProxyOptions proxyOptions) {
+
+    client.request(new RequestOptions()
+      .setHost("example.com")
+      .setProxyOptions(proxyOptions))
+      .compose(request -> request
+        .send()
+        .compose(HttpClientResponse::body))
+      .onSuccess(body -> {
+        System.out.println("Received response");
+      });
+  }
+
   public void example60(Vertx vertx) {
 
     HttpClientOptions options = new HttpClientOptions()

src/main/java/examples/NetExamples.java

@@ -15,6 +15,8 @@
 import io.vertx.core.Vertx;
 import io.vertx.core.buffer.Buffer;
 import io.vertx.core.http.ClientAuth;
+import io.vertx.core.http.HttpClient;
+import io.vertx.core.http.HttpClientOptions;
 import io.vertx.core.http.HttpServerOptions;
 import io.vertx.core.net.*;
 
@@ -564,6 +566,17 @@ public void example47(Vertx vertx) {
     NetClient client = vertx.createNetClient(options);
   }
 
+  public void nonProxyHosts(Vertx vertx) {
+
+    NetClientOptions options = new NetClientOptions()
+      .setProxyOptions(new ProxyOptions().setType(ProxyType.SOCKS5)
+        .setHost("localhost").setPort(1080)
+        .setUsername("username").setPassword("secret"))
+      .addNonProxyHost("*.foo.com")
+      .addNonProxyHost("localhost");
+    NetClient client = vertx.createNetClient(options);
+  }
+
   public void example48(Vertx vertx) throws CertificateException {
     SelfSignedCertificate certificate = SelfSignedCertificate.create();
 

src/main/java/io/vertx/core/http/HttpClient.java

@@ -20,10 +20,12 @@
 import io.vertx.core.MultiMap;
 import io.vertx.core.buffer.Buffer;
 import io.vertx.core.metrics.Measured;
+import io.vertx.core.net.SocketAddress;
 import io.vertx.core.streams.ReadStream;
 
 import java.util.List;
 import java.util.function.Function;
+import java.util.function.Predicate;
 
 /**
  * An asynchronous HTTP client.

src/main/java/io/vertx/core/http/HttpClientOptions.java

@@ -1138,6 +1138,16 @@ public HttpClientOptions setProxyOptions(ProxyOptions proxyOptions) {
     return (HttpClientOptions) super.setProxyOptions(proxyOptions);
   }
 
+  @Override
+  public HttpClientOptions setNonProxyHosts(List<String> nonProxyHosts) {
+    return (HttpClientOptions) super.setNonProxyHosts(nonProxyHosts);
+  }
+
+  @Override
+  public HttpClientOptions addNonProxyHost(String nonProxyHost) {
+    return (HttpClientOptions) super.addNonProxyHost(nonProxyHost);
+  }
+
   @Override
   public HttpClientOptions setLocalAddress(String localAddress) {
     return (HttpClientOptions) super.setLocalAddress(localAddress);

src/main/java/io/vertx/core/http/RequestOptions.java

@@ -16,6 +16,7 @@
 import io.vertx.core.MultiMap;
 import io.vertx.core.VertxException;
 import io.vertx.core.json.JsonObject;
+import io.vertx.core.net.ProxyOptions;
 import io.vertx.core.net.SocketAddress;
 
 import java.net.MalformedURLException;
@@ -32,6 +33,11 @@
 @DataObject(generateConverter = true)
 public class RequestOptions {
 
+  /**
+   * The default value for proxy options = {@code null}
+   */
+  public static final ProxyOptions DEFAULT_PROXY_OPTIONS = null;
+
   /**
    * The default value for server method = {@code null}
    */
@@ -72,6 +78,7 @@ public class RequestOptions {
    */
   public static final long DEFAULT_TIMEOUT = 0;
 
+  private ProxyOptions proxyOptions;
   private SocketAddress server;
   private HttpMethod method;
   private String host;
@@ -86,6 +93,7 @@ public class RequestOptions {
    * Default constructor
    */
   public RequestOptions() {
+    proxyOptions = DEFAULT_PROXY_OPTIONS;
     server = DEFAULT_SERVER;
     method = DEFAULT_HTTP_METHOD;
     host = DEFAULT_HOST;
@@ -102,6 +110,7 @@ public RequestOptions() {
    * @param other  the options to copy
    */
   public RequestOptions(RequestOptions other) {
+    setProxyOptions(other.proxyOptions);
     setServer(other.server);
     setMethod(other.method);
     setHost(other.host);
@@ -155,6 +164,27 @@ public RequestOptions(JsonObject json) {
     }
   }
 
+  /**
+   * Get the proxy options override for connections
+   *
+   * @return proxy options override
+   */
+  public ProxyOptions getProxyOptions() {
+    return proxyOptions;
+  }
+
+  /**
+   * Override the {@link HttpClientOptions#setProxyOptions(ProxyOptions)} proxy options
+   * for connections.
+   *
+   * @param proxyOptions proxy options override object
+   * @return a reference to this, so the API can be used fluently
+   */
+  public RequestOptions setProxyOptions(ProxyOptions proxyOptions) {
+    this.proxyOptions = proxyOptions;
+    return this;
+  }
+
   /**
    * Get the server address to be used by the client request.
    *

src/main/java/io/vertx/core/http/WebSocketConnectOptions.java

@@ -14,6 +14,7 @@
 import io.vertx.codegen.annotations.GenIgnore;
 import io.vertx.core.MultiMap;
 import io.vertx.core.json.JsonObject;
+import io.vertx.core.net.ProxyOptions;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -26,6 +27,11 @@
 @DataObject(generateConverter = true)
 public class WebSocketConnectOptions extends RequestOptions {
 
+  /**
+   * The default value for proxy options = {@code null}
+   */
+  public static final ProxyOptions DEFAULT_PROXY_OPTIONS = null;
+
   /**
    * The default WebSocket version = {@link WebsocketVersion#V13}
    */
@@ -36,16 +42,19 @@ public class WebSocketConnectOptions extends RequestOptions {
    */
   public static final List<String> DEFAULT_SUB_PROTOCOLS = null;
 
+  private ProxyOptions proxyOptions;
   private WebsocketVersion version;
   private List<String> subProtocols;
 
   public WebSocketConnectOptions() {
+    proxyOptions = DEFAULT_PROXY_OPTIONS;
     version = DEFAULT_VERSION;
     subProtocols = DEFAULT_SUB_PROTOCOLS;
   }
 
   public WebSocketConnectOptions(WebSocketConnectOptions other) {
     super(other);
+    this.proxyOptions = other.proxyOptions != null ? new ProxyOptions(other.proxyOptions) : null;
     this.version = other.version;
     this.subProtocols = other.subProtocols;
   }
@@ -102,6 +111,27 @@ public WebSocketConnectOptions addSubProtocol(String subprotocol) {
     return this;
   }
 
+  /**
+   * Get the proxy options override for connections
+   *
+   * @return proxy options override
+   */
+  public ProxyOptions getProxyOptions() {
+    return proxyOptions;
+  }
+
+  /**
+   * Override the {@link HttpClientOptions#setProxyOptions(ProxyOptions)} proxy options
+   * for connections.
+   *
+   * @param proxyOptions proxy options override object
+   * @return a reference to this, so the API can be used fluently
+   */
+  public RequestOptions setProxyOptions(ProxyOptions proxyOptions) {
+    this.proxyOptions = proxyOptions;
+    return this;
+  }
+
   @Override
   public WebSocketConnectOptions setHost(String host) {
     return (WebSocketConnectOptions) super.setHost(host);

src/main/java/io/vertx/core/http/impl/EndpointKey.java

@@ -10,22 +10,25 @@
  */
 package io.vertx.core.http.impl;
 
+import io.vertx.core.net.ProxyOptions;
 import io.vertx.core.net.SocketAddress;
 
 final class EndpointKey {
 
   final boolean ssl;
   final SocketAddress serverAddr;
   final SocketAddress peerAddr;
+  final ProxyOptions proxyOptions;
 
-  EndpointKey(boolean ssl, SocketAddress serverAddr, SocketAddress peerAddr) {
+  EndpointKey(boolean ssl, ProxyOptions proxyOptions, SocketAddress serverAddr, SocketAddress peerAddr) {
     if (serverAddr == null) {
       throw new NullPointerException("No null server address");
     }
     if (peerAddr == null) {
       throw new NullPointerException("No null peer address");
     }
     this.ssl = ssl;
+    this.proxyOptions = proxyOptions;
     this.peerAddr = peerAddr;
     this.serverAddr = serverAddr;
   }

src/main/java/io/vertx/core/http/impl/HttpChannelConnector.java

@@ -26,7 +26,6 @@
 import io.vertx.core.impl.ContextInternal;
 import io.vertx.core.net.NetSocket;
 import io.vertx.core.net.ProxyOptions;
-import io.vertx.core.net.ProxyType;
 import io.vertx.core.net.SocketAddress;
 import io.vertx.core.net.impl.NetClientImpl;
 import io.vertx.core.net.impl.NetSocketInternal;
@@ -48,6 +47,7 @@ public class HttpChannelConnector {
   private final HttpClientImpl client;
   private final NetClientImpl netClient;
   private final HttpClientOptions options;
+  private final ProxyOptions proxyOptions;
   private final ClientMetrics metrics;
   private final boolean ssl;
   private final boolean useAlpn;
@@ -57,6 +57,7 @@ public class HttpChannelConnector {
 
   public HttpChannelConnector(HttpClientImpl client,
                               NetClientImpl netClient,
+                              ProxyOptions proxyOptions,
                               ClientMetrics metrics,
                               HttpVersion version,
                               boolean ssl,
@@ -67,6 +68,7 @@ public HttpChannelConnector(HttpClientImpl client,
     this.netClient = netClient;
     this.metrics = metrics;
     this.options = client.getOptions();
+    this.proxyOptions = proxyOptions;
     this.ssl = ssl;
     this.useAlpn = useAlpn;
     this.version = version;
@@ -79,11 +81,6 @@ public SocketAddress server() {
   }
 
   private void connect(EventLoopContext context, Promise<NetSocket> promise) {
-    ProxyOptions proxyOptions = this.options.getProxyOptions();
-    if (proxyOptions != null && !ssl && proxyOptions.getType()== ProxyType.HTTP) {
-      // http proxy requests are handled in HttpClientImpl, everything else can use netty proxy handler
-      proxyOptions = null;
-    }
     netClient.connectInternal(proxyOptions, server, peerAddress, this.options.isForceSni() ? peerAddress.host() : null, ssl, useAlpn, promise, context, 0);
   }