Initial take at replacing AsyncHttpClient with own HTTP client

Motivation:

AsyncHttpClient is a huge beast aiming at being a generic HTTP client
with tons of features that Gatling doesn’t use.
It would be more convenient that Gatling has its own tailor made HTTP
client.

Modifications:

* Introduce `gatling-http-client` and `gatling-netty-utils` modules

Results:

Gatling no longer depends on AsyncHttpClient.

Some features haven’t been implemented yet:

* NTLM support
* Expect-Continue
* Proxy support for WebSockets

build.sbt

@@ -11,7 +11,7 @@ import sbt._
 lazy val root = Project("gatling-parent", file("."))
   .enablePlugins(AutomateHeaderPlugin, SonatypeReleasePlugin)
   .dependsOn(Seq(commons, core, http, jms, jdbc, redis).map(_ % "compile->compile;test->test"): _*)
-  .aggregate(commons, core, jdbc, redis, http, jms, charts, metrics, app, recorder, testFramework, bundle, compiler)
+  .aggregate(nettyUtil, commons, core, jdbc, redis, httpClient, http, jms, charts, metrics, app, recorder, testFramework, bundle, compiler)
   .settings(basicSettings: _*)
   .settings(noArtifactToPublish)
   .settings(libraryDependencies ++= docDependencies)
@@ -24,7 +24,11 @@ def gatlingModule(id: String) = Project(id, file(id))
   .settings(gatlingModuleSettings: _*)
   .settings(updateOptions := updateOptions.value.withGigahorse(false))
 
+lazy val nettyUtil = gatlingModule("gatling-netty-util")
+  .settings(libraryDependencies ++= nettyUtilDependencies)
+
 lazy val commons = gatlingModule("gatling-commons")
+  .dependsOn(nettyUtil % "compile->compile;test->test")
   .settings(libraryDependencies ++= commonsDependencies(scalaVersion.value))
   .settings(generateVersionFileSettings: _*)
 
@@ -41,8 +45,14 @@ lazy val redis = gatlingModule("gatling-redis")
   .dependsOn(core % "compile->compile;test->test")
   .settings(libraryDependencies ++= redisDependencies)
 
+lazy val httpClient = gatlingModule("gatling-http-client")
+  .dependsOn(nettyUtil % "compile->compile;test->test")
+  .settings(libraryDependencies ++= httpClientDependencies)
+
 lazy val http = gatlingModule("gatling-http")
-  .dependsOn(core % "compile->compile;test->test")
+  .dependsOn(
+    core % "compile->compile;test->test",
+    httpClient % "compile->compile;test->test")
   .settings(libraryDependencies ++= httpDependencies)
 
 lazy val jms = gatlingModule("gatling-jms")

gatling-benchmarks/README.md

@@ -0,0 +1,11 @@
+# HttpRequestExpressionBuilderBenchmark
+
+```
+[info] Benchmark                                                                Mode  Cnt        Score         Error  Units
+[info] HttpRequestExpressionBuilderBenchmark.testReference                     thrpt   10  6748944,966 ± 1470265,342  ops/s
+[info] HttpRequestExpressionBuilderBenchmark.testRequestWithStaticAbsoluteUrl  thrpt   10  8232440,517 ±  941764,501  ops/s
+[info] HttpRequestExpressionBuilderBenchmark.testRequestWithStaticRelativeUrl  thrpt   10  8233697,239 ± 1448120,428  ops/s
+[info] HttpRequestExpressionBuilderBenchmark.testRequestWithStaticHeaders      thrpt   10  6052629,328 ±  770041,000  ops/s
+[info] HttpRequestExpressionBuilderBenchmark.testRequestWithStaticQueryParams  thrpt   10  3729282,348 ± 1308478,718  ops/s
+[info] HttpRequestExpressionBuilderBenchmark.testRequestWithDynamicQuery       thrpt   10  2063526,548 ±  960711,859  ops/s
+```

gatling-benchmarks/src/main/scala/io/gatling/http/request/builder/HttpRequestExpressionBuilderBenchmark.scala

@@ -17,15 +17,16 @@
 package io.gatling.http.request.builder
 
 import io.gatling.commons.validation._
-import io.gatling.core.{ ValidationImplicits, CoreComponents }
+import io.gatling.core.{ CoreComponents, ValidationImplicits }
 import io.gatling.core.config.GatlingConfiguration
 import io.gatling.core.session._
 import io.gatling.http.cache.HttpCaches
 import io.gatling.http.protocol._
+import io.gatling.http.client.{ Request => AhcRequest, RequestBuilder => AhcRequesBuilder }
+import io.gatling.http.client.ahc.uri.Uri
 
 import com.softwaremill.quicklens._
-import org.asynchttpclient.Request
-import org.asynchttpclient.Dsl._
+import io.netty.handler.codec.http.HttpMethod
 import org.openjdk.jmh.annotations.Benchmark
 
 object HttpRequestExpressionBuilderBenchmark extends ValidationImplicits {
@@ -54,25 +55,29 @@ object HttpRequestExpressionBuilderBenchmark extends ValidationImplicits {
     responseProcessor = null
   )
 
-  val Reference: Expression[Request] = _ =>
-    request("GET", "http://localhost:8000/ping")
-      .build.success
+  val Reference: Expression[AhcRequest] = _ =>
+    new AhcRequesBuilder(HttpMethod.GET, Uri.create("http://localhost:8000/ping"))
+      .build(true).success
 
-  val SimpleRequest: Expression[Request] =
+  val RequestWithStaticAbsoluteUrl: Expression[AhcRequest] =
+    new Http("requestName").get("http://localhost:8000/ping")
+      .build(coreComponents, httpComponents, throttled = false).ahcRequest
+
+  val RequestWithStaticRelativeUrl: Expression[AhcRequest] =
     new Http("requestName").get("/ping")
       .build(coreComponents, httpComponents, throttled = false).ahcRequest
 
-  val RequestWithStaticQueryParams: Expression[Request] =
+  val RequestWithStaticQueryParams: Expression[AhcRequest] =
     new Http("requestName").get("/ping")
       .queryParam("hello", "world")
       .queryParam("foo", "bar")
       .build(coreComponents, httpComponents, throttled = false).ahcRequest
 
-  val RequestWithDynamicUrl: Expression[Request] =
+  val RequestWithDynamicQuery: Expression[AhcRequest] =
     new Http("requestName").get("/ping?foo=${key}")
       .build(coreComponents, httpComponents, throttled = false).ahcRequest
 
-  val RequestWithStaticHeaders: Expression[Request] = {
+  val RequestWithStaticHeaders: Expression[AhcRequest] = {
 
     val httpProtocol = HttpProtocolBuilder(config)
       .baseURL("http://localhost:8000")
@@ -87,7 +92,6 @@ object HttpRequestExpressionBuilderBenchmark extends ValidationImplicits {
         httpComponents.modify(_.httpProtocol).setTo(httpProtocol),
         throttled = false
       ).ahcRequest
-
   }
 
   val EmptySession: Session = Session("scenario", 0)
@@ -100,22 +104,39 @@ class HttpRequestExpressionBuilderBenchmark {
   import HttpRequestExpressionBuilderBenchmark._
 
   @Benchmark
-  def testReference(): Validation[Request] =
+  def testReference(): Validation[AhcRequest] =
     Reference(EmptySession)
 
   @Benchmark
-  def testSimpleRequest(): Validation[Request] =
-    SimpleRequest(EmptySession)
+  def testRequestWithStaticRelativeUrl(): Validation[AhcRequest] =
+    RequestWithStaticRelativeUrl(EmptySession)
+
+  @Benchmark
+  def testRequestWithStaticAbsoluteUrl(): Validation[AhcRequest] =
+    RequestWithStaticAbsoluteUrl(EmptySession)
 
   @Benchmark
-  def testRequestWithStaticQueryParams(): Validation[Request] =
+  def testRequestWithStaticQueryParams(): Validation[AhcRequest] =
     RequestWithStaticQueryParams(EmptySession)
 
   @Benchmark
-  def testRequestWithStaticHeaders(): Validation[Request] =
+  def testRequestWithStaticHeaders(): Validation[AhcRequest] =
     RequestWithStaticHeaders(EmptySession)
 
   @Benchmark
-  def testRequestWithDynamicUrl(): Validation[Request] =
-    RequestWithDynamicUrl(NonEmptySession)
+  def testRequestWithDynamicQuery(): Validation[AhcRequest] =
+    RequestWithDynamicQuery(NonEmptySession)
+}
+
+object Test {
+  def main(args: Array[String]): Unit = {
+    val test = new HttpRequestExpressionBuilderBenchmark
+
+    System.out.println("testReference=" + test.testReference())
+    System.out.println("testRequestWithStaticRelativeUrl=" + test.testRequestWithStaticRelativeUrl())
+    System.out.println("testRequestWithStaticAbsoluteUrl" + test.testRequestWithStaticAbsoluteUrl())
+    System.out.println("testRequestWithStaticQueryParams=" + test.testRequestWithStaticQueryParams())
+    System.out.println("testRequestWithStaticHeaders=" + test.testRequestWithStaticHeaders())
+    System.out.println("testRequestWithDynamicQuery=" + test.testRequestWithDynamicQuery())
+  }
 }

gatling-commons/src/main/scala/io/gatling/commons/util/Bytes.scala

@@ -22,9 +22,9 @@ import java.nio.charset.Charset
 import scala.annotation.switch
 
 import io.gatling.commons.util.Collections._
+import io.gatling.netty.util.ahc.ByteBufUtils
 
 import io.netty.buffer.Unpooled
-import org.asynchttpclient.netty.util.ByteBufUtils
 
 object Bytes {
 

gatling-commons/src/main/scala/io/gatling/commons/util/ClassHelper.scala

@@ -16,11 +16,13 @@
 
 package io.gatling.commons.util
 
+import io.gatling.netty.util.ahc.StringBuilderPool
+
 object ClassHelper {
 
   def toClassShortName(className: String): String = {
     val parts = className.split("\\.")
-    val sb = StringBuilderPool.Global.get()
+    val sb = StringBuilderPool.DEFAULT.get()
     var i = 0
     while (i < parts.length - 1) {
       sb.append(parts(i).charAt(0)).append('.')

gatling-commons/src/main/scala/io/gatling/commons/util/StringReplace.scala

@@ -19,6 +19,7 @@ package io.gatling.commons.util
 import java.lang.{ StringBuilder => JStringBuilder }
 
 import io.gatling.commons.util.Spire._
+import io.gatling.netty.util.ahc.StringBuilderPool
 
 object StringReplace {
 
@@ -56,7 +57,7 @@ object StringReplace {
         if (replaced(c)) {
           if (!matchFound) {
             // first match
-            sb = StringBuilderPool.Global.get()
+            sb = SbPool.get()
             sb.append(text, 0, i)
             matchFound = true
           }

gatling-core/src/main/resources/config-removed.properties

@@ -65,3 +65,10 @@ gatling.http.ahc.usePooledMemory=dropped, use gatling.http.ahc.allocator instead
 gatling.jms.acknowledgedMessagesBufferSize=dropped
 gatling.core.directory.data=dropped, merged with bodies into resources
 gatling.core.directory.bodies=dropped, merged with data into resources
+gatling.http.ahc.keepAlive=dropped, control keep-alive with Connection request header
+gatling.http.ahc.readTimeout=dropped
+gatling.http.ahc.soSndBuf=dropped
+gatling.http.ahc.soRcvBuf=dropped
+gatling.http.ahc.httpClientCodecMaxChunkSize=dropped
+gatling.http.ahc.httpClientCodecInitialBufferSize=dropped
+gatling.http.ahc.soLinger=dropped

gatling-core/src/main/resources/gatling-defaults.conf

@@ -72,29 +72,23 @@ gatling {
       }
     }
     ahc {
-      keepAlive = true                                    # Allow pooling HTTP connections (keep-alive header automatically added)
       connectTimeout = 10000                              # Timeout when establishing a connection
       handshakeTimeout = 10000                            # Timeout when performing TLS hashshake
       pooledConnectionIdleTimeout = 60000                 # Timeout when a connection stays unused in the pool
-      readTimeout = 60000                                 # Timeout when a used connection stays idle
       maxRetry = 2                                        # Number of times that a request should be tried again
       requestTimeout = 60000                              # Timeout of the requests
       disableHttpsEndpointIdentificationAlgorithm = true  # When set to true, don't enable SSL algorithm on the SSLEngine
       useInsecureTrustManager = true                      # Use an insecure TrustManager that trusts all server certificates
       filterInsecureCipherSuites = true                   # Turn to false to not filter out insecure and weak cipher suites
-      httpClientCodecMaxChunkSize = 8192                  # Maximum length of the content or each chunk
-      httpClientCodecInitialBufferSize = 128              # Initial HttpClientCodec buffer size
       sslEnabledProtocols = [TLSv1.2, TLSv1.1, TLSv1]     # Array of enabled protocols for HTTPS, if empty use the JDK defaults
       sslEnabledCipherSuites = []                         # Array of enabled cipher suites for HTTPS, if empty use the AHC defaults
       sslSessionCacheSize = 0                             # SSLSession cache size, set to 0 to use JDK's default
       sslSessionTimeout = 0                               # SSLSession timeout in seconds, set to 0 to use JDK's default (24h)
+      disableSslSessionResumption = false                 # if true, SSLSessions won't be resumed
       useOpenSsl = false                                  # if OpenSSL should be used instead of JSSE (requires tcnative jar)
       useNativeTransport = false                          # if native transport should be used instead of Java NIO (requires netty-transport-native-epoll, currently Linux only)
       tcpNoDelay = true
       soReuseAddress = false
-      soLinger = -1
-      soSndBuf = -1
-      soRcvBuf = -1
       allocator = "pooled"                            # switch to unpooled for unpooled ByteBufAllocator
       maxThreadLocalCharBufferSize = 200000           # Netty's default is 16k
     }

gatling-core/src/main/scala/io/gatling/core/ConfigKeys.scala

@@ -90,29 +90,23 @@ object ConfigKeys {
     }
 
     object ahc {
-      val KeepAlive = "gatling.http.ahc.keepAlive"
       val ConnectTimeout = "gatling.http.ahc.connectTimeout"
       val HandshakeTimeout = "gatling.http.ahc.handshakeTimeout"
       val PooledConnectionIdleTimeout = "gatling.http.ahc.pooledConnectionIdleTimeout"
-      val ReadTimeout = "gatling.http.ahc.readTimeout"
       val MaxRetry = "gatling.http.ahc.maxRetry"
       val RequestTimeout = "gatling.http.ahc.requestTimeout"
       val DisableHttpsEndpointIdentificationAlgorithm = "gatling.http.ahc.disableHttpsEndpointIdentificationAlgorithm"
       val UseInsecureTrustManager = "gatling.http.ahc.useInsecureTrustManager"
       val FilterInsecureCipherSuites = "gatling.http.ahc.filterInsecureCipherSuites"
-      val HttpClientCodecMaxChunkSize = "gatling.http.ahc.httpClientCodecMaxChunkSize"
-      val HttpClientCodecInitialBufferSize = "gatling.http.ahc.httpClientCodecInitialBufferSize"
       val SslEnabledProtocols = "gatling.http.ahc.sslEnabledProtocols"
       val SslEnabledCipherSuites = "gatling.http.ahc.sslEnabledCipherSuites"
       val SslSessionCacheSize = "gatling.http.ahc.sslSessionCacheSize"
       val SslSessionTimeout = "gatling.http.ahc.sslSessionTimeout"
+      val DisableSslSessionResumption = "gatling.http.ahc.disableSslSessionResumption"
       val UseOpenSsl = "gatling.http.ahc.useOpenSsl"
       val UseNativeTransport = "gatling.http.ahc.useNativeTransport"
       val TcpNoDelay = "gatling.http.ahc.tcpNoDelay"
       val SoReuseAddress = "gatling.http.ahc.soReuseAddress"
-      val SoLinger = "gatling.http.ahc.soLinger"
-      val SoSndBuf = "gatling.http.ahc.soSndBuf"
-      val SoRcvBuf = "gatling.http.ahc.soRcvBuf"
       val Allocator = "gatling.http.ahc.allocator"
       val MaxThreadLocalCharBufferSize = "gatling.http.ahc.maxThreadLocalCharBufferSize"
     }

gatling-core/src/main/scala/io/gatling/core/body/Body.scala

@@ -19,11 +19,12 @@ package io.gatling.core.body
 import java.io.InputStream
 import java.nio.charset.Charset
 
-import io.gatling.commons.util.{ CompositeByteArrayInputStream, StringBuilderPool }
+import io.gatling.commons.util.CompositeByteArrayInputStream
 import io.gatling.commons.validation._
 import io.gatling.core.config.GatlingConfiguration
 import io.gatling.core.session._
 import io.gatling.core.session.el.ElCompiler
+import io.gatling.netty.util.ahc.StringBuilderPool
 
 import com.mitchellbosecke.pebble.template.PebbleTemplate
 import com.typesafe.scalalogging.StrictLogging
@@ -69,7 +70,7 @@ object CompositeByteArrayBody {
 case class CompositeByteArrayBody(bytes: Expression[Seq[Array[Byte]]], charset: Charset) extends Body with Expression[String] {
 
   override def apply(session: Session): Validation[String] = bytes(session).map { bs =>
-    val sb = StringBuilderPool.Global.get()
+    val sb = StringBuilderPool.DEFAULT.get()
     bs.foreach(b => sb.append(new String(b, charset)))
     sb.toString
   }