KTOR-7934 Fix content length check for Options and Head for wasm, js,…

… and native engines

buildSrc/src/main/kotlin/test/server/tests/Encoding.kt

@@ -12,6 +12,7 @@ import io.ktor.server.response.*
 import io.ktor.server.routing.*
 import io.ktor.util.*
 import io.ktor.utils.io.*
+import kotlinx.io.readByteArray
 
 internal fun Application.encodingTestServer() {
     routing {
@@ -73,7 +74,44 @@ internal fun Application.encodingTestServer() {
                     }
                 }
             }
+            route("/gzip-with-content-length") {
+                get {
+                    val content = "Hello, world"
+                    val compressed: ByteArray = GZipEncoder.encode(ByteReadChannel(content.toByteArray()))
+                        .readRemaining().readByteArray()
+
+                    call.respond(object : OutgoingContent.ReadChannelContent() {
+
+                        override val contentLength: Long = compressed.size.toLong()
+                        override val contentType: ContentType = ContentType.Text.Plain
+                        override val headers: Headers = Headers.build {
+                            append(HttpHeaders.ContentEncoding, "gzip")
+                        }
+
+                        override fun readFrom(): ByteReadChannel {
+                            return ByteReadChannel(compressed)
+                        }
+
+                    })
+                }
+            }
+            route("/head-gzip-with-content-length") {
+                head {
+                    val content = "Hello, world"
+                    val compressed: ByteArray = GZipEncoder.encode(ByteReadChannel(content.toByteArray()))
+                        .readRemaining().readByteArray()
+
+                    call.respond(object : OutgoingContent.NoContent() {
+                        override val contentLength: Long = compressed.size.toLong()
+                        override val contentType: ContentType = ContentType.Text.Plain
+                        override val headers: Headers = Headers.build {
+                            append(HttpHeaders.ContentEncoding, "gzip")
+                        }
+                    })
+                }
+            }
         }
+
     }
 }
 

gradle.properties

@@ -31,7 +31,7 @@ version=3.0.3-SNAPSHOT
 #   To save some memory, shutting down Kotlin Daemon used for buildSrc compilation after 30 seconds of idle.
 #   See: https://github.com/gradle/gradle/issues/29331
 org.gradle.jvmargs=-Xms2g -Xmx8g -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC -Dfile.encoding=UTF-8 -Dkotlin.daemon.jvm.options=-Xmx512m,Xms256m,-XX:MaxMetaspaceSize=256m,XX:+HeapDumpOnOutOfMemoryError
-kotlin.daemon.jvmargs=-Xms512m -Xmx2g -XX:MaxMetaspaceSize=256m -XX:+HeapDumpOnOutOfMemoryError
+kotlin.daemon.jvmargs=-Xms512m -Xmx4g -XX:MaxMetaspaceSize=256m -XX:+HeapDumpOnOutOfMemoryError
 # Gradle Doctor might increase memory consumption when task monitoring is enabled, so it is disabled by default.
 # Some features can't work without task monitoring:
 #  doctor-negative-savings, doctor-slow-build-cache-connection, doctor-slow-maven-connection

ktor-client/ktor-client-cio/jvmAndPosix/src/io/ktor/client/engine/cio/utils.kt

@@ -53,7 +53,7 @@ internal suspend fun writeHeaders(
     val expected = headers[HttpHeaders.Expect]
 
     try {
-        val normalizedUrl = if (url.pathSegments.isEmpty()) URLBuilder(url).apply { encodedPath = "/" }.build() else url
+        val normalizedUrl = if (url.rawSegments.isEmpty()) URLBuilder(url).apply { encodedPath = "/" }.build() else url
         val urlString = if (overProxy) normalizedUrl.toString() else normalizedUrl.fullPath
 
         builder.requestLine(method, urlString, HttpProtocolVersion.HTTP_1_1.toString())

ktor-client/ktor-client-core/api/ktor-client-core.api

@@ -1516,6 +1516,10 @@ public final class io/ktor/client/utils/HeadersKt {
 	public static synthetic fun buildHeaders$default (Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lio/ktor/http/Headers;
 }
 
+public final class io/ktor/client/utils/HeadersUtilsKt {
+	public static final fun dropCompressionHeaders (Lio/ktor/http/HeadersBuilder;Lio/ktor/http/HttpMethod;Lio/ktor/util/Attributes;)V
+}
+
 public final class io/ktor/client/utils/HttpResponseReceiveFail {
 	public fun <init> (Lio/ktor/client/statement/HttpResponse;Ljava/lang/Throwable;)V
 	public final fun getCause ()Ljava/lang/Throwable;

ktor-client/ktor-client-core/api/ktor-client-core.klib.api

@@ -1362,6 +1362,7 @@ final fun (io.ktor.client/HttpClientConfig<*>).io.ktor.client.plugins/defaultReq
 final fun (io.ktor.http.content/OutgoingContent).io.ktor.client.utils/wrapHeaders(kotlin/Function1<io.ktor.http/Headers, io.ktor.http/Headers>): io.ktor.http.content/OutgoingContent // io.ktor.client.utils/wrapHeaders|[email protected](kotlin.Function1<io.ktor.http.Headers,io.ktor.http.Headers>){}[0]
 final fun (io.ktor.http/Cookie).io.ktor.client.plugins.cookies/fillDefaults(io.ktor.http/Url): io.ktor.http/Cookie // io.ktor.client.plugins.cookies/fillDefaults|[email protected](io.ktor.http.Url){}[0]
 final fun (io.ktor.http/Cookie).io.ktor.client.plugins.cookies/matches(io.ktor.http/Url): kotlin/Boolean // io.ktor.client.plugins.cookies/matches|[email protected](io.ktor.http.Url){}[0]
+final fun (io.ktor.http/HeadersBuilder).io.ktor.client.utils/dropCompressionHeaders(io.ktor.http/HttpMethod, io.ktor.util/Attributes) // io.ktor.client.utils/dropCompressionHeaders|[email protected](io.ktor.http.HttpMethod;io.ktor.util.Attributes){}[0]
 final fun (io.ktor.http/HttpMessageBuilder).io.ktor.client.request/accept(io.ktor.http/ContentType) // io.ktor.client.request/accept|[email protected](io.ktor.http.ContentType){}[0]
 final fun (io.ktor.http/HttpMessageBuilder).io.ktor.client.request/basicAuth(kotlin/String, kotlin/String) // io.ktor.client.request/basicAuth|[email protected](kotlin.String;kotlin.String){}[0]
 final fun (io.ktor.http/HttpMessageBuilder).io.ktor.client.request/bearerAuth(kotlin/String) // io.ktor.client.request/bearerAuth|[email protected](kotlin.String){}[0]

ktor-client/ktor-client-core/common/src/io/ktor/client/utils/HeadersUtils.kt

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license.
+ */
+
+package io.ktor.client.utils
+
+import io.ktor.http.*
+import io.ktor.util.*
+import io.ktor.utils.io.*
+
+private val DecompressionListAttribute: AttributeKey<MutableList<String>> = AttributeKey("DecompressionListAttribute")
+
+/**
+ * This function should be used for engines which apply decompression but don't drop compression headers
+ * (like js and Curl) to make sure all the plugins and checks work with the correct content length and encoding.
+ */
+@InternalAPI
+public fun HeadersBuilder.dropCompressionHeaders(method: HttpMethod, attributes: Attributes) {
+    if (method == HttpMethod.Head || method == HttpMethod.Options) return
+    val header = get(HttpHeaders.ContentEncoding) ?: return
+    attributes.computeIfAbsent(DecompressionListAttribute) { mutableListOf<String>() }.add(header)
+    remove(HttpHeaders.ContentEncoding)
+    remove(HttpHeaders.ContentLength)
+}

ktor-client/ktor-client-core/js/src/io/ktor/client/engine/js/JsClientEngine.kt

@@ -52,7 +52,7 @@ internal class JsClientEngine(
         val rawResponse = commonFetch(data.url.toString(), rawRequest, config)
 
         val status = HttpStatusCode(rawResponse.status.toInt(), rawResponse.statusText)
-        val headers = rawResponse.headers.mapToKtor()
+        val headers = rawResponse.headers.mapToKtor(data.method, data.attributes)
         val version = HttpProtocolVersion.HTTP_1_1
 
         val body = CoroutineScope(callContext).readBody(rawResponse)
@@ -153,12 +153,13 @@ private fun Event.asString(): String = buildString {
     append(JSON.stringify(this@asString, arrayOf("message", "target", "type", "isTrusted")))
 }
 
-private fun org.w3c.fetch.Headers.mapToKtor(): Headers = buildHeaders {
+@OptIn(InternalAPI::class)
+private fun org.w3c.fetch.Headers.mapToKtor(method: HttpMethod, attributes: Attributes): Headers = buildHeaders {
     this@mapToKtor.asDynamic().forEach { value: String, key: String ->
         append(key, value)
     }
 
-    Unit
+    dropCompressionHeaders(method, attributes)
 }
 
 /**

ktor-client/ktor-client-core/wasmJs/src/io/ktor/client/engine/js/WasmJsClientEngine.kt

@@ -16,9 +16,11 @@ import io.ktor.util.*
 import io.ktor.util.date.*
 import io.ktor.utils.io.*
 import kotlinx.coroutines.*
-import org.w3c.dom.*
-import org.w3c.dom.events.*
-import kotlin.coroutines.*
+import org.w3c.dom.WebSocket
+import org.w3c.dom.events.Event
+import kotlin.coroutines.CoroutineContext
+import kotlin.coroutines.resume
+import kotlin.coroutines.resumeWithException
 
 @Suppress("UNUSED_PARAMETER")
 private fun createBrowserWebSocket(urlString_capturingHack: String, vararg protocols: String): WebSocket =
@@ -62,7 +64,7 @@ internal class JsClientEngine(
 
         val rawResponse = commonFetch(data.url.toString(), rawRequest, config)
         val status = HttpStatusCode(rawResponse.status.toInt(), rawResponse.statusText)
-        val headers = rawResponse.headers.mapToKtor()
+        val headers = rawResponse.headers.mapToKtor(data.method, data.attributes)
         val version = HttpProtocolVersion.HTTP_1_1
 
         val body = CoroutineScope(callContext).readBody(rawResponse)
@@ -162,13 +164,16 @@ private fun eventAsString(event: Event): String =
 private fun getKeys(headers: org.w3c.fetch.Headers): JsArray<JsString> =
     js("Array.from(headers.keys())")
 
-internal fun org.w3c.fetch.Headers.mapToKtor(): Headers = buildHeaders {
+@OptIn(InternalAPI::class)
+internal fun org.w3c.fetch.Headers.mapToKtor(method: HttpMethod, attributes: Attributes): Headers = buildHeaders {
     val keys = getKeys(this@mapToKtor)
     for (i in 0 until keys.length) {
         val key = keys[i].toString()
         val value = this@mapToKtor.get(key)!!
         append(key, value)
     }
+
+    dropCompressionHeaders(method, attributes)
 }
 
 /**

ktor-client/ktor-client-curl/desktop/src/io/ktor/client/engine/curl/CurlClientEngine.kt

@@ -9,6 +9,7 @@ import io.ktor.client.engine.curl.internal.*
 import io.ktor.client.plugins.*
 import io.ktor.client.plugins.sse.*
 import io.ktor.client.request.*
+import io.ktor.client.utils.dropCompressionHeaders
 import io.ktor.http.*
 import io.ktor.http.cio.*
 import io.ktor.util.date.*
@@ -38,12 +39,15 @@ internal class CurlClientEngine(
                 readUTF8Line()
             }
             val rawHeaders = parseHeaders(headerBytes)
+            val headers = rawHeaders
+                .toBuilder().apply {
+                    dropCompressionHeaders(data.method, data.attributes)
+                }.build()
 
-            val status = HttpStatusCode.fromValue(status)
-
-            val headers = filterCurlHeaders(rawHeaders)
             rawHeaders.release()
 
+            val status = HttpStatusCode.fromValue(status)
+
             val responseBody: Any = data.attributes.getOrNull(ResponseAdapterAttributeKey)
                 ?.adapt(data, status, headers, bodyChannel, data.body, callContext)
                 ?: bodyChannel
@@ -65,23 +69,6 @@ internal class CurlClientEngine(
     }
 }
 
-/**
- * Curl provides raw response headers while performing request decoding.
- * This can lead to an invalid content-length header or trigger the content encoding plugin wrongly.
- *
- * We need to filter out the headers that are no longer valid.
- */
-internal fun filterCurlHeaders(raw: HttpHeadersMap): Headers {
-    val builder = raw.toBuilder()
-
-    if (builder.contains(HttpHeaders.ContentEncoding)) {
-        builder.remove(HttpHeaders.ContentEncoding)
-        builder.remove(HttpHeaders.ContentLength)
-    }
-
-    return builder.build()
-}
-
 @Deprecated("This exception will be removed in a future release in favor of a better error handling.")
 public class CurlIllegalStateException(cause: String) : IllegalStateException(cause)
 

ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyResponseUtils.kt

@@ -6,8 +6,12 @@ package io.ktor.client.engine.darwin.internal.legacy
 
 import io.ktor.client.utils.*
 import io.ktor.http.*
+import io.ktor.util.Attributes
+import io.ktor.utils.io.InternalAPI
 import platform.Foundation.*
 
-internal fun NSHTTPURLResponse.readHeaders(): Headers = buildHeaders {
+@OptIn(InternalAPI::class)
+internal fun NSHTTPURLResponse.readHeaders(method: HttpMethod, attributes: Attributes): Headers = buildHeaders {
     allHeaderFields.mapKeys { (key, value) -> append(key as String, value as String) }
+    dropCompressionHeaders(method, attributes)
 }

ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyTaskHandler.kt

@@ -4,7 +4,6 @@
 
 package io.ktor.client.engine.darwin.internal.legacy
 
-import io.ktor.client.plugins.sse.*
 import io.ktor.client.request.*
 import io.ktor.http.*
 import io.ktor.util.date.*
@@ -71,7 +70,7 @@ internal class DarwinLegacyTaskHandler(
     @OptIn(UnsafeNumber::class, ExperimentalForeignApi::class, InternalAPI::class)
     fun NSHTTPURLResponse.toResponseData(requestData: HttpRequestData): HttpResponseData {
         val status = HttpStatusCode.fromValue(statusCode.convert())
-        val headers = readHeaders()
+        val headers = readHeaders(requestData.method, requestData.attributes)
         val responseBody: Any = requestData.attributes.getOrNull(ResponseAdapterAttributeKey)
             ?.adapt(requestData, status, headers, body, requestData.body, callContext)
             ?: body

ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyUrlUtils.kt

@@ -42,7 +42,7 @@ internal fun Url.toNSUrl(): NSURL {
 
     components.percentEncodedPath = when {
         pathEncoded -> encodedPath
-        else -> pathSegments.joinToString("/").sanitize(NSCharacterSet.URLPathAllowedCharacterSet)
+        else -> rawSegments.joinToString("/").sanitize(NSCharacterSet.URLPathAllowedCharacterSet)
     }
 
     when {

ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinResponseUtils.kt

@@ -6,8 +6,13 @@ package io.ktor.client.engine.darwin.internal
 
 import io.ktor.client.utils.*
 import io.ktor.http.*
+import io.ktor.util.Attributes
+import io.ktor.utils.io.InternalAPI
 import platform.Foundation.*
 
-internal fun NSHTTPURLResponse.readHeaders(): Headers = buildHeaders {
+@OptIn(InternalAPI::class)
+internal fun NSHTTPURLResponse.readHeaders(method: HttpMethod, attributes: Attributes): Headers = buildHeaders {
     allHeaderFields.mapKeys { (key, value) -> append(key as String, value as String) }
+
+    dropCompressionHeaders(method, attributes)
 }

ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinTaskHandler.kt

@@ -5,10 +5,8 @@
 package io.ktor.client.engine.darwin.internal
 
 import io.ktor.client.engine.darwin.*
-import io.ktor.client.plugins.sse.*
 import io.ktor.client.request.*
 import io.ktor.http.*
-import io.ktor.util.*
 import io.ktor.util.date.*
 import io.ktor.utils.io.*
 import io.ktor.utils.io.CancellationException
@@ -73,7 +71,7 @@ internal class DarwinTaskHandler(
     @OptIn(UnsafeNumber::class, ExperimentalForeignApi::class, InternalAPI::class)
     fun NSHTTPURLResponse.toResponseData(requestData: HttpRequestData): HttpResponseData {
         val status = HttpStatusCode.fromValue(statusCode.convert())
-        val headers = readHeaders()
+        val headers = readHeaders(requestData.method, requestData.attributes)
         val responseBody: Any = requestData.attributes.getOrNull(ResponseAdapterAttributeKey)
             ?.adapt(requestData, status, headers, body, requestData.body, callContext)
             ?: body

ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinUrlUtils.kt

@@ -42,7 +42,7 @@ internal fun Url.toNSUrl(): NSURL {
 
     components.percentEncodedPath = when {
         pathEncoded -> encodedPath
-        else -> pathSegments.joinToString("/").sanitize(NSCharacterSet.URLPathAllowedCharacterSet)
+        else -> rawSegments.joinToString("/").sanitize(NSCharacterSet.URLPathAllowedCharacterSet)
     }
 
     when {