diff --git a/buildSrc/src/main/kotlin/test/server/tests/Encoding.kt b/buildSrc/src/main/kotlin/test/server/tests/Encoding.kt index 9870e917799..b3bdc02695a 100644 --- a/buildSrc/src/main/kotlin/test/server/tests/Encoding.kt +++ b/buildSrc/src/main/kotlin/test/server/tests/Encoding.kt @@ -12,6 +12,7 @@ import io.ktor.server.response.* import io.ktor.server.routing.* import io.ktor.util.* import io.ktor.utils.io.* +import kotlinx.io.readByteArray internal fun Application.encodingTestServer() { routing { @@ -73,7 +74,44 @@ internal fun Application.encodingTestServer() { } } } + route("/gzip-with-content-length") { + get { + val content = "Hello, world" + val compressed: ByteArray = GZipEncoder.encode(ByteReadChannel(content.toByteArray())) + .readRemaining().readByteArray() + + call.respond(object : OutgoingContent.ReadChannelContent() { + + override val contentLength: Long = compressed.size.toLong() + override val contentType: ContentType = ContentType.Text.Plain + override val headers: Headers = Headers.build { + append(HttpHeaders.ContentEncoding, "gzip") + } + + override fun readFrom(): ByteReadChannel { + return ByteReadChannel(compressed) + } + + }) + } + } + route("/head-gzip-with-content-length") { + head { + val content = "Hello, world" + val compressed: ByteArray = GZipEncoder.encode(ByteReadChannel(content.toByteArray())) + .readRemaining().readByteArray() + + call.respond(object : OutgoingContent.NoContent() { + override val contentLength: Long = compressed.size.toLong() + override val contentType: ContentType = ContentType.Text.Plain + override val headers: Headers = Headers.build { + append(HttpHeaders.ContentEncoding, "gzip") + } + }) + } + } } + } } diff --git a/gradle.properties b/gradle.properties index dd7e997ae56..539af967128 100644 --- a/gradle.properties +++ b/gradle.properties @@ -31,7 +31,7 @@ version=3.0.3-SNAPSHOT # To save some memory, shutting down Kotlin Daemon used for buildSrc compilation after 30 seconds of idle. # See: https://github.com/gradle/gradle/issues/29331 org.gradle.jvmargs=-Xms2g -Xmx8g -XX:+HeapDumpOnOutOfMemoryError -XX:+UseParallelGC -Dfile.encoding=UTF-8 -Dkotlin.daemon.jvm.options=-Xmx512m,Xms256m,-XX:MaxMetaspaceSize=256m,XX:+HeapDumpOnOutOfMemoryError -kotlin.daemon.jvmargs=-Xms512m -Xmx2g -XX:MaxMetaspaceSize=256m -XX:+HeapDumpOnOutOfMemoryError +kotlin.daemon.jvmargs=-Xms512m -Xmx4g -XX:MaxMetaspaceSize=256m -XX:+HeapDumpOnOutOfMemoryError # Gradle Doctor might increase memory consumption when task monitoring is enabled, so it is disabled by default. # Some features can't work without task monitoring: # doctor-negative-savings, doctor-slow-build-cache-connection, doctor-slow-maven-connection diff --git a/ktor-client/ktor-client-cio/jvmAndPosix/src/io/ktor/client/engine/cio/utils.kt b/ktor-client/ktor-client-cio/jvmAndPosix/src/io/ktor/client/engine/cio/utils.kt index 8ec2b097ed1..325ca90bd92 100644 --- a/ktor-client/ktor-client-cio/jvmAndPosix/src/io/ktor/client/engine/cio/utils.kt +++ b/ktor-client/ktor-client-cio/jvmAndPosix/src/io/ktor/client/engine/cio/utils.kt @@ -53,7 +53,7 @@ internal suspend fun writeHeaders( val expected = headers[HttpHeaders.Expect] try { - val normalizedUrl = if (url.pathSegments.isEmpty()) URLBuilder(url).apply { encodedPath = "/" }.build() else url + val normalizedUrl = if (url.rawSegments.isEmpty()) URLBuilder(url).apply { encodedPath = "/" }.build() else url val urlString = if (overProxy) normalizedUrl.toString() else normalizedUrl.fullPath builder.requestLine(method, urlString, HttpProtocolVersion.HTTP_1_1.toString()) diff --git a/ktor-client/ktor-client-core/api/ktor-client-core.api b/ktor-client/ktor-client-core/api/ktor-client-core.api index 99aafdb303f..7568285b61b 100644 --- a/ktor-client/ktor-client-core/api/ktor-client-core.api +++ b/ktor-client/ktor-client-core/api/ktor-client-core.api @@ -1516,6 +1516,10 @@ public final class io/ktor/client/utils/HeadersKt { public static synthetic fun buildHeaders$default (Lkotlin/jvm/functions/Function1;ILjava/lang/Object;)Lio/ktor/http/Headers; } +public final class io/ktor/client/utils/HeadersUtilsKt { + public static final fun dropCompressionHeaders (Lio/ktor/http/HeadersBuilder;Lio/ktor/http/HttpMethod;Lio/ktor/util/Attributes;)V +} + public final class io/ktor/client/utils/HttpResponseReceiveFail { public fun (Lio/ktor/client/statement/HttpResponse;Ljava/lang/Throwable;)V public final fun getCause ()Ljava/lang/Throwable; diff --git a/ktor-client/ktor-client-core/api/ktor-client-core.klib.api b/ktor-client/ktor-client-core/api/ktor-client-core.klib.api index 854e339bc84..06cf826d0b6 100644 --- a/ktor-client/ktor-client-core/api/ktor-client-core.klib.api +++ b/ktor-client/ktor-client-core/api/ktor-client-core.klib.api @@ -1362,6 +1362,7 @@ final fun (io.ktor.client/HttpClientConfig<*>).io.ktor.client.plugins/defaultReq final fun (io.ktor.http.content/OutgoingContent).io.ktor.client.utils/wrapHeaders(kotlin/Function1): io.ktor.http.content/OutgoingContent // io.ktor.client.utils/wrapHeaders|wrapHeaders@io.ktor.http.content.OutgoingContent(kotlin.Function1){}[0] final fun (io.ktor.http/Cookie).io.ktor.client.plugins.cookies/fillDefaults(io.ktor.http/Url): io.ktor.http/Cookie // io.ktor.client.plugins.cookies/fillDefaults|fillDefaults@io.ktor.http.Cookie(io.ktor.http.Url){}[0] final fun (io.ktor.http/Cookie).io.ktor.client.plugins.cookies/matches(io.ktor.http/Url): kotlin/Boolean // io.ktor.client.plugins.cookies/matches|matches@io.ktor.http.Cookie(io.ktor.http.Url){}[0] +final fun (io.ktor.http/HeadersBuilder).io.ktor.client.utils/dropCompressionHeaders(io.ktor.http/HttpMethod, io.ktor.util/Attributes) // io.ktor.client.utils/dropCompressionHeaders|dropCompressionHeaders@io.ktor.http.HeadersBuilder(io.ktor.http.HttpMethod;io.ktor.util.Attributes){}[0] final fun (io.ktor.http/HttpMessageBuilder).io.ktor.client.request/accept(io.ktor.http/ContentType) // io.ktor.client.request/accept|accept@io.ktor.http.HttpMessageBuilder(io.ktor.http.ContentType){}[0] final fun (io.ktor.http/HttpMessageBuilder).io.ktor.client.request/basicAuth(kotlin/String, kotlin/String) // io.ktor.client.request/basicAuth|basicAuth@io.ktor.http.HttpMessageBuilder(kotlin.String;kotlin.String){}[0] final fun (io.ktor.http/HttpMessageBuilder).io.ktor.client.request/bearerAuth(kotlin/String) // io.ktor.client.request/bearerAuth|bearerAuth@io.ktor.http.HttpMessageBuilder(kotlin.String){}[0] diff --git a/ktor-client/ktor-client-core/common/src/io/ktor/client/utils/HeadersUtils.kt b/ktor-client/ktor-client-core/common/src/io/ktor/client/utils/HeadersUtils.kt new file mode 100644 index 00000000000..1b78fb2e00a --- /dev/null +++ b/ktor-client/ktor-client-core/common/src/io/ktor/client/utils/HeadersUtils.kt @@ -0,0 +1,24 @@ +/* + * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package io.ktor.client.utils + +import io.ktor.http.* +import io.ktor.util.* +import io.ktor.utils.io.* + +private val DecompressionListAttribute: AttributeKey> = AttributeKey("DecompressionListAttribute") + +/** + * This function should be used for engines which apply decompression but don't drop compression headers + * (like js and Curl) to make sure all the plugins and checks work with the correct content length and encoding. + */ +@InternalAPI +public fun HeadersBuilder.dropCompressionHeaders(method: HttpMethod, attributes: Attributes) { + if (method == HttpMethod.Head || method == HttpMethod.Options) return + val header = get(HttpHeaders.ContentEncoding) ?: return + attributes.computeIfAbsent(DecompressionListAttribute) { mutableListOf() }.add(header) + remove(HttpHeaders.ContentEncoding) + remove(HttpHeaders.ContentLength) +} diff --git a/ktor-client/ktor-client-core/js/src/io/ktor/client/engine/js/JsClientEngine.kt b/ktor-client/ktor-client-core/js/src/io/ktor/client/engine/js/JsClientEngine.kt index 78687960fa5..cb7c413e060 100644 --- a/ktor-client/ktor-client-core/js/src/io/ktor/client/engine/js/JsClientEngine.kt +++ b/ktor-client/ktor-client-core/js/src/io/ktor/client/engine/js/JsClientEngine.kt @@ -52,7 +52,7 @@ internal class JsClientEngine( val rawResponse = commonFetch(data.url.toString(), rawRequest, config) val status = HttpStatusCode(rawResponse.status.toInt(), rawResponse.statusText) - val headers = rawResponse.headers.mapToKtor() + val headers = rawResponse.headers.mapToKtor(data.method, data.attributes) val version = HttpProtocolVersion.HTTP_1_1 val body = CoroutineScope(callContext).readBody(rawResponse) @@ -153,12 +153,13 @@ private fun Event.asString(): String = buildString { append(JSON.stringify(this@asString, arrayOf("message", "target", "type", "isTrusted"))) } -private fun org.w3c.fetch.Headers.mapToKtor(): Headers = buildHeaders { +@OptIn(InternalAPI::class) +private fun org.w3c.fetch.Headers.mapToKtor(method: HttpMethod, attributes: Attributes): Headers = buildHeaders { this@mapToKtor.asDynamic().forEach { value: String, key: String -> append(key, value) } - Unit + dropCompressionHeaders(method, attributes) } /** diff --git a/ktor-client/ktor-client-core/wasmJs/src/io/ktor/client/engine/js/WasmJsClientEngine.kt b/ktor-client/ktor-client-core/wasmJs/src/io/ktor/client/engine/js/WasmJsClientEngine.kt index e04f1cdcdb6..1a1e6067770 100644 --- a/ktor-client/ktor-client-core/wasmJs/src/io/ktor/client/engine/js/WasmJsClientEngine.kt +++ b/ktor-client/ktor-client-core/wasmJs/src/io/ktor/client/engine/js/WasmJsClientEngine.kt @@ -16,9 +16,11 @@ import io.ktor.util.* import io.ktor.util.date.* import io.ktor.utils.io.* import kotlinx.coroutines.* -import org.w3c.dom.* -import org.w3c.dom.events.* -import kotlin.coroutines.* +import org.w3c.dom.WebSocket +import org.w3c.dom.events.Event +import kotlin.coroutines.CoroutineContext +import kotlin.coroutines.resume +import kotlin.coroutines.resumeWithException @Suppress("UNUSED_PARAMETER") private fun createBrowserWebSocket(urlString_capturingHack: String, vararg protocols: String): WebSocket = @@ -62,7 +64,7 @@ internal class JsClientEngine( val rawResponse = commonFetch(data.url.toString(), rawRequest, config) val status = HttpStatusCode(rawResponse.status.toInt(), rawResponse.statusText) - val headers = rawResponse.headers.mapToKtor() + val headers = rawResponse.headers.mapToKtor(data.method, data.attributes) val version = HttpProtocolVersion.HTTP_1_1 val body = CoroutineScope(callContext).readBody(rawResponse) @@ -162,13 +164,16 @@ private fun eventAsString(event: Event): String = private fun getKeys(headers: org.w3c.fetch.Headers): JsArray = js("Array.from(headers.keys())") -internal fun org.w3c.fetch.Headers.mapToKtor(): Headers = buildHeaders { +@OptIn(InternalAPI::class) +internal fun org.w3c.fetch.Headers.mapToKtor(method: HttpMethod, attributes: Attributes): Headers = buildHeaders { val keys = getKeys(this@mapToKtor) for (i in 0 until keys.length) { val key = keys[i].toString() val value = this@mapToKtor.get(key)!! append(key, value) } + + dropCompressionHeaders(method, attributes) } /** diff --git a/ktor-client/ktor-client-curl/desktop/src/io/ktor/client/engine/curl/CurlClientEngine.kt b/ktor-client/ktor-client-curl/desktop/src/io/ktor/client/engine/curl/CurlClientEngine.kt index a74934b2159..7f862268f87 100644 --- a/ktor-client/ktor-client-curl/desktop/src/io/ktor/client/engine/curl/CurlClientEngine.kt +++ b/ktor-client/ktor-client-curl/desktop/src/io/ktor/client/engine/curl/CurlClientEngine.kt @@ -9,6 +9,7 @@ import io.ktor.client.engine.curl.internal.* import io.ktor.client.plugins.* import io.ktor.client.plugins.sse.* import io.ktor.client.request.* +import io.ktor.client.utils.dropCompressionHeaders import io.ktor.http.* import io.ktor.http.cio.* import io.ktor.util.date.* @@ -38,12 +39,15 @@ internal class CurlClientEngine( readUTF8Line() } val rawHeaders = parseHeaders(headerBytes) + val headers = rawHeaders + .toBuilder().apply { + dropCompressionHeaders(data.method, data.attributes) + }.build() - val status = HttpStatusCode.fromValue(status) - - val headers = filterCurlHeaders(rawHeaders) rawHeaders.release() + val status = HttpStatusCode.fromValue(status) + val responseBody: Any = data.attributes.getOrNull(ResponseAdapterAttributeKey) ?.adapt(data, status, headers, bodyChannel, data.body, callContext) ?: bodyChannel @@ -65,23 +69,6 @@ internal class CurlClientEngine( } } -/** - * Curl provides raw response headers while performing request decoding. - * This can lead to an invalid content-length header or trigger the content encoding plugin wrongly. - * - * We need to filter out the headers that are no longer valid. - */ -internal fun filterCurlHeaders(raw: HttpHeadersMap): Headers { - val builder = raw.toBuilder() - - if (builder.contains(HttpHeaders.ContentEncoding)) { - builder.remove(HttpHeaders.ContentEncoding) - builder.remove(HttpHeaders.ContentLength) - } - - return builder.build() -} - @Deprecated("This exception will be removed in a future release in favor of a better error handling.") public class CurlIllegalStateException(cause: String) : IllegalStateException(cause) diff --git a/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyResponseUtils.kt b/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyResponseUtils.kt index 32c5add4918..20c326de10d 100644 --- a/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyResponseUtils.kt +++ b/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyResponseUtils.kt @@ -6,8 +6,12 @@ package io.ktor.client.engine.darwin.internal.legacy import io.ktor.client.utils.* import io.ktor.http.* +import io.ktor.util.Attributes +import io.ktor.utils.io.InternalAPI import platform.Foundation.* -internal fun NSHTTPURLResponse.readHeaders(): Headers = buildHeaders { +@OptIn(InternalAPI::class) +internal fun NSHTTPURLResponse.readHeaders(method: HttpMethod, attributes: Attributes): Headers = buildHeaders { allHeaderFields.mapKeys { (key, value) -> append(key as String, value as String) } + dropCompressionHeaders(method, attributes) } diff --git a/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyTaskHandler.kt b/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyTaskHandler.kt index c5074fe5560..18eaf85e87a 100644 --- a/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyTaskHandler.kt +++ b/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyTaskHandler.kt @@ -4,7 +4,6 @@ package io.ktor.client.engine.darwin.internal.legacy -import io.ktor.client.plugins.sse.* import io.ktor.client.request.* import io.ktor.http.* import io.ktor.util.date.* @@ -71,7 +70,7 @@ internal class DarwinLegacyTaskHandler( @OptIn(UnsafeNumber::class, ExperimentalForeignApi::class, InternalAPI::class) fun NSHTTPURLResponse.toResponseData(requestData: HttpRequestData): HttpResponseData { val status = HttpStatusCode.fromValue(statusCode.convert()) - val headers = readHeaders() + val headers = readHeaders(requestData.method, requestData.attributes) val responseBody: Any = requestData.attributes.getOrNull(ResponseAdapterAttributeKey) ?.adapt(requestData, status, headers, body, requestData.body, callContext) ?: body diff --git a/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyUrlUtils.kt b/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyUrlUtils.kt index 994ccf671a6..f718f156b2f 100644 --- a/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyUrlUtils.kt +++ b/ktor-client/ktor-client-darwin-legacy/darwin/src/io/ktor/client/engine/darwin/internal/legacy/DarwinLegacyUrlUtils.kt @@ -42,7 +42,7 @@ internal fun Url.toNSUrl(): NSURL { components.percentEncodedPath = when { pathEncoded -> encodedPath - else -> pathSegments.joinToString("/").sanitize(NSCharacterSet.URLPathAllowedCharacterSet) + else -> rawSegments.joinToString("/").sanitize(NSCharacterSet.URLPathAllowedCharacterSet) } when { diff --git a/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinResponseUtils.kt b/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinResponseUtils.kt index 2a5d9c430cd..90f246d6805 100644 --- a/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinResponseUtils.kt +++ b/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinResponseUtils.kt @@ -6,8 +6,13 @@ package io.ktor.client.engine.darwin.internal import io.ktor.client.utils.* import io.ktor.http.* +import io.ktor.util.Attributes +import io.ktor.utils.io.InternalAPI import platform.Foundation.* -internal fun NSHTTPURLResponse.readHeaders(): Headers = buildHeaders { +@OptIn(InternalAPI::class) +internal fun NSHTTPURLResponse.readHeaders(method: HttpMethod, attributes: Attributes): Headers = buildHeaders { allHeaderFields.mapKeys { (key, value) -> append(key as String, value as String) } + + dropCompressionHeaders(method, attributes) } diff --git a/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinTaskHandler.kt b/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinTaskHandler.kt index 668f4afd5dd..d04cce01127 100644 --- a/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinTaskHandler.kt +++ b/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinTaskHandler.kt @@ -5,10 +5,8 @@ package io.ktor.client.engine.darwin.internal import io.ktor.client.engine.darwin.* -import io.ktor.client.plugins.sse.* import io.ktor.client.request.* import io.ktor.http.* -import io.ktor.util.* import io.ktor.util.date.* import io.ktor.utils.io.* import io.ktor.utils.io.CancellationException @@ -73,7 +71,7 @@ internal class DarwinTaskHandler( @OptIn(UnsafeNumber::class, ExperimentalForeignApi::class, InternalAPI::class) fun NSHTTPURLResponse.toResponseData(requestData: HttpRequestData): HttpResponseData { val status = HttpStatusCode.fromValue(statusCode.convert()) - val headers = readHeaders() + val headers = readHeaders(requestData.method, requestData.attributes) val responseBody: Any = requestData.attributes.getOrNull(ResponseAdapterAttributeKey) ?.adapt(requestData, status, headers, body, requestData.body, callContext) ?: body diff --git a/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinUrlUtils.kt b/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinUrlUtils.kt index 8c85004d5f1..82bdacf045c 100644 --- a/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinUrlUtils.kt +++ b/ktor-client/ktor-client-darwin/darwin/src/io/ktor/client/engine/darwin/internal/DarwinUrlUtils.kt @@ -42,7 +42,7 @@ internal fun Url.toNSUrl(): NSURL { components.percentEncodedPath = when { pathEncoded -> encodedPath - else -> pathSegments.joinToString("/").sanitize(NSCharacterSet.URLPathAllowedCharacterSet) + else -> rawSegments.joinToString("/").sanitize(NSCharacterSet.URLPathAllowedCharacterSet) } when { diff --git a/ktor-client/ktor-client-plugins/ktor-client-encoding/common/src/ContentEncoding.kt b/ktor-client/ktor-client-plugins/ktor-client-encoding/common/src/ContentEncoding.kt index bd5ce105198..be270150f39 100644 --- a/ktor-client/ktor-client-plugins/ktor-client-encoding/common/src/ContentEncoding.kt +++ b/ktor-client/ktor-client-plugins/ktor-client-encoding/common/src/ContentEncoding.kt @@ -90,102 +90,105 @@ public class ContentEncodingConfig { * You can learn more from [Content encoding](https://ktor.io/docs/content-encoding.html). */ @OptIn(InternalAPI::class) -public val ContentEncoding: ClientPlugin = - createClientPlugin("HttpEncoding", ::ContentEncodingConfig) { - val encoders: Map = pluginConfig.encoders - val qualityValues: Map = pluginConfig.qualityValues - val mode = pluginConfig.mode +public val ContentEncoding: ClientPlugin = createClientPlugin( + "HttpEncoding", + ::ContentEncodingConfig +) { + val encoders: Map = pluginConfig.encoders + val qualityValues: Map = pluginConfig.qualityValues + val mode = pluginConfig.mode - val requestHeader = buildString { - for (encoder in encoders.values) { - if (isNotEmpty()) append(',') + val requestHeader = buildString { + for (encoder in encoders.values) { + if (isNotEmpty()) append(',') - append(encoder.name) + append(encoder.name) - val quality = qualityValues[encoder.name] ?: continue - check(quality in 0.0..1.0) { "Invalid quality value: $quality for encoder: $encoder" } + val quality = qualityValues[encoder.name] ?: continue + check(quality in 0.0..1.0) { "Invalid quality value: $quality for encoder: $encoder" } - val qualityValue = quality.toString().take(5) - append(";q=$qualityValue") - } + val qualityValue = quality.toString().take(5) + append(";q=$qualityValue") } + } - fun CoroutineScope.decode(response: HttpResponse): HttpResponse { - val encodings = response.headers[HttpHeaders.ContentEncoding]?.split(",")?.map { it.trim().lowercase() } - ?: run { - LOGGER.trace( - "Empty or no Content-Encoding header in response. " + - "Skipping ContentEncoding for ${response.call.request.url}" - ) - return response - } + fun CoroutineScope.decode(response: HttpResponse): HttpResponse { + val encodings = + response.headers[HttpHeaders.ContentEncoding]?.split(",")?.map { it.trim().lowercase() } ?: run { + LOGGER.trace( + "Empty or no Content-Encoding header in response. " + "Skipping ContentEncoding for ${response.call.request.url}" + ) + return response + } - var current = response.rawContent - for (encoding in encodings.reversed()) { - val encoder: Encoder = encoders[encoding] ?: throw UnsupportedContentEncodingException(encoding) + var current = response.rawContent + for (encoding in encodings.reversed()) { + val encoder: Encoder = encoders[encoding] ?: throw UnsupportedContentEncodingException(encoding) - LOGGER.trace("Decoding response with $encoder for ${response.call.request.url}") - with(encoder) { - current = decode(current, response.coroutineContext) - } + LOGGER.trace("Decoding response with $encoder for ${response.call.request.url}") + with(encoder) { + current = decode(current, response.coroutineContext) } + } - val headers = headers { - response.headers.forEach { name, values -> - if (name.equals(HttpHeaders.ContentEncoding, ignoreCase = true) || - name.equals(HttpHeaders.ContentLength, ignoreCase = true) - ) { - return@forEach - } - appendAll(name, values) - } - val remainingEncodings = encodings.filter { !encodings.contains(it) } - if (remainingEncodings.isNotEmpty()) { - append(HttpHeaders.ContentEncoding, remainingEncodings.joinToString(",")) + val headers = headers { + response.headers.forEach { name, values -> + if (name.equals( + HttpHeaders.ContentEncoding, + ignoreCase = true + ) || name.equals(HttpHeaders.ContentLength, ignoreCase = true) + ) { + return@forEach } + appendAll(name, values) + } + val remainingEncodings = encodings.filter { !encodings.contains(it) } + if (remainingEncodings.isNotEmpty()) { + append(HttpHeaders.ContentEncoding, remainingEncodings.joinToString(",")) } - response.call.attributes.put(DecompressionListAttribute, encodings) - return response.call.wrap(current, headers).response } + response.call.attributes.put(DecompressionListAttribute, encodings) + return response.call.wrap(current, headers).response + } - onRequest { request, _ -> - if (!mode.response) return@onRequest - if (request.headers.contains(HttpHeaders.AcceptEncoding)) return@onRequest - LOGGER.trace("Adding Accept-Encoding=$requestHeader for ${request.url}") - request.headers[HttpHeaders.AcceptEncoding] = requestHeader - } + onRequest { request, _ -> + if (!mode.response) return@onRequest + if (request.headers.contains(HttpHeaders.AcceptEncoding)) return@onRequest + LOGGER.trace("Adding Accept-Encoding=$requestHeader for ${request.url}") + request.headers[HttpHeaders.AcceptEncoding] = requestHeader + } - on(AfterRenderHook) { request, content -> - if (!mode.request) return@on null + on(AfterRenderHook) { request, content -> + if (!mode.request) return@on null - val encoderNames = request.attributes.getOrNull(CompressionListAttribute) ?: run { - LOGGER.trace("Skipping request compression for ${request.url} because no compressions set") - return@on null - } + val encoderNames = request.attributes.getOrNull(CompressionListAttribute) ?: run { + LOGGER.trace("Skipping request compression for ${request.url} because no compressions set") + return@on null + } - LOGGER.trace("Compressing request body for ${request.url} using $encoderNames") - val selectedEncoders = encoderNames.map { - encoders[it] ?: throw UnsupportedContentEncodingException(it) - } + LOGGER.trace("Compressing request body for ${request.url} using $encoderNames") + val selectedEncoders = encoderNames.map { + encoders[it] ?: throw UnsupportedContentEncodingException(it) + } - if (selectedEncoders.isEmpty()) return@on null - selectedEncoders.fold(content) { compressed, encoder -> - compressed.compressed(encoder, request.executionContext) ?: compressed - } + if (selectedEncoders.isEmpty()) return@on null + selectedEncoders.fold(content) { compressed, encoder -> + compressed.compressed(encoder, request.executionContext) ?: compressed } + } - on(ReceiveStateHook) { response -> - if (!mode.response) return@on null + on(ReceiveStateHook) { response -> + if (!mode.response) return@on null - val method = response.request.method - val contentLength = response.contentLength() + val method = response.request.method + val contentLength = response.contentLength() - if (contentLength == 0L) return@on null - if (contentLength == null && method == HttpMethod.Head) return@on null + if (contentLength == 0L) return@on null + if (contentLength == null && method == HttpMethod.Head) return@on null - return@on response.call.decode(response) - } + return@on response.call.decode(response) } +} internal object AfterRenderHook : ClientHook OutgoingContent?> { val afterRenderPhase = PipelinePhase("AfterRender") diff --git a/ktor-client/ktor-client-plugins/ktor-client-encoding/common/test/ContentEncodingTest.kt b/ktor-client/ktor-client-plugins/ktor-client-encoding/common/test/ContentEncodingTest.kt index 0e4a6fd8aa6..84400cce50f 100644 --- a/ktor-client/ktor-client-plugins/ktor-client-encoding/common/test/ContentEncodingTest.kt +++ b/ktor-client/ktor-client-plugins/ktor-client-encoding/common/test/ContentEncodingTest.kt @@ -8,7 +8,10 @@ import io.ktor.client.call.* import io.ktor.client.request.* import io.ktor.client.tests.utils.* import io.ktor.http.* -import kotlin.test.* +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertNull private const val TEST_URL = "$TEST_SERVER/compression" diff --git a/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/ConnectionTest.kt b/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/ConnectionTest.kt index 3a4beb799c1..00a4801420b 100644 --- a/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/ConnectionTest.kt +++ b/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/ConnectionTest.kt @@ -8,6 +8,7 @@ import io.ktor.client.* import io.ktor.client.call.* import io.ktor.client.request.* import io.ktor.client.statement.* +import io.ktor.client.statement.readRawBytes import io.ktor.client.tests.utils.* import io.ktor.http.* import kotlin.test.* @@ -28,7 +29,7 @@ class ConnectionTest : ClientLoader() { response.also { assertTrue(it.status.isSuccess()) - assertTrue(it.readBytes().isEmpty()) + assertTrue(it.readRawBytes().isEmpty()) } } } diff --git a/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/ContentEncodingIntegrationTest.kt b/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/ContentEncodingIntegrationTest.kt new file mode 100644 index 00000000000..74d4508735a --- /dev/null +++ b/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/ContentEncodingIntegrationTest.kt @@ -0,0 +1,47 @@ +/* + * Copyright 2014-2024 JetBrains s.r.o and contributors. Use of this source code is governed by the Apache 2.0 license. + */ + +package io.ktor.client.tests + +import io.ktor.client.request.* +import io.ktor.client.statement.* +import io.ktor.client.tests.utils.* +import io.ktor.http.* +import io.ktor.util.* +import io.ktor.utils.io.* +import kotlinx.io.readString +import kotlin.test.Test +import kotlin.test.assertEquals + +private const val TEST_URL = "$TEST_SERVER/compression" + +class ContentEncodingIntegrationTest : ClientLoader() { + + @Test + fun testGzipWithContentLengthWithoutPlugin() = clientTests { + test { client -> + val response = client.get("$TEST_URL/gzip-with-content-length") + val byteContent = response.bodyAsBytes() + + val content = if (response.headers[HttpHeaders.ContentEncoding] == "gzip") { + GZipEncoder.decode(ByteReadChannel(byteContent)).readRemaining().readString() + } else { + byteContent.decodeToString() + } + + assertEquals("Hello, world", content) + } + } + + @Test + fun testHeadGzipWithContentLengthWithoutPlugin() = clientTests { + test { client -> + val response = client.head("$TEST_URL/head-gzip-with-content-length") + + assertEquals(HttpStatusCode.OK, response.status) + assertEquals("gzip", response.headers[HttpHeaders.ContentEncoding], "gzip") + assertEquals("32", response.headers[HttpHeaders.ContentLength]) + } + } +} diff --git a/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/LoggingTest.kt b/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/LoggingTest.kt index 2b5eab275d7..91cd602be0a 100644 --- a/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/LoggingTest.kt +++ b/ktor-client/ktor-client-tests/common/test/io/ktor/client/tests/LoggingTest.kt @@ -11,6 +11,7 @@ import io.ktor.client.plugins.logging.* import io.ktor.client.request.* import io.ktor.client.request.forms.* import io.ktor.client.statement.* +import io.ktor.client.statement.readRawBytes import io.ktor.client.tests.utils.* import io.ktor.http.* import io.ktor.serialization.kotlinx.json.* @@ -37,7 +38,7 @@ class LoggingTest : ClientLoader() { test { client -> val size = 4 * 1024 * 1024 client.prepareGet("$TEST_SERVER/bytes?size=$size").execute { - assertEquals(size, it.readBytes().size) + assertEquals(size, it.readRawBytes().size) } } @@ -239,7 +240,7 @@ class LoggingTest : ClientLoader() { setBody(byteArrayOf(-77, 111)) }.execute { - it.readBytes() + it.readRawBytes() } } diff --git a/ktor-http/common/src/io/ktor/http/URLUtils.kt b/ktor-http/common/src/io/ktor/http/URLUtils.kt index 57a495384d2..c8e5aac4bcc 100644 --- a/ktor-http/common/src/io/ktor/http/URLUtils.kt +++ b/ktor-http/common/src/io/ktor/http/URLUtils.kt @@ -160,7 +160,7 @@ public fun Appendable.appendUrlFullPath( /** * Checks if [Url] has absolute path. */ -public val Url.isAbsolutePath: Boolean get() = pathSegments.firstOrNull() == "" +public val Url.isAbsolutePath: Boolean get() = rawSegments.firstOrNull() == "" /** * Checks if [Url] has absolute path. diff --git a/ktor-http/common/test/io/ktor/tests/http/URLBuilderTest.kt b/ktor-http/common/test/io/ktor/tests/http/URLBuilderTest.kt index 13a1700f162..93894a1c77e 100644 --- a/ktor-http/common/test/io/ktor/tests/http/URLBuilderTest.kt +++ b/ktor-http/common/test/io/ktor/tests/http/URLBuilderTest.kt @@ -348,7 +348,7 @@ internal class URLBuilderTest { builder.appendPathSegments("path2") assertEquals(listOf("value1"), url.parameters.getAll("key")) - assertEquals(listOf("path1"), url.pathSegments) + assertEquals(listOf("path1"), url.rawSegments) } @Test @@ -364,9 +364,9 @@ internal class URLBuilderTest { val url2 = builder.build() assertEquals(listOf("value1"), url1.parameters.getAll("key")) - assertEquals(listOf("path1"), url1.pathSegments) + assertEquals(listOf("path1"), url1.rawSegments) assertEquals(listOf("value1", "value2"), url2.parameters.getAll("key")) - assertEquals(listOf("path1", "path2"), url2.pathSegments) + assertEquals(listOf("path1", "path2"), url2.rawSegments) } @Test @@ -410,7 +410,7 @@ internal class URLBuilderTest { assertEquals(segments, builder.pathSegments) assertEquals(path, builder.encodedPath) - assertEquals(segments, url.pathSegments) + assertEquals(segments, url.rawSegments) } } @@ -436,7 +436,7 @@ internal class URLBuilderTest { assertEquals(segments, builder.pathSegments) assertEquals(urlString, builder.buildString()) - assertEquals(segments, url.pathSegments) + assertEquals(segments, url.rawSegments) } } diff --git a/ktor-http/common/test/io/ktor/tests/http/UrlTest.kt b/ktor-http/common/test/io/ktor/tests/http/UrlTest.kt index b973cd52d6f..d6f725cf1bc 100644 --- a/ktor-http/common/test/io/ktor/tests/http/UrlTest.kt +++ b/ktor-http/common/test/io/ktor/tests/http/UrlTest.kt @@ -17,7 +17,7 @@ class UrlTest { assertEquals(443, url.port) assertEquals(443, url.protocol.defaultPort) assertEquals("ktor.io", url.host) - assertEquals(listOf("", "quickstart", ""), url.pathSegments) + assertEquals(listOf("", "quickstart", ""), url.rawSegments) assertEquals(parametersOf("query" to listOf("string"), "param" to listOf("value", "value2")), url.parameters) assertEquals("fragment", url.fragment) assertEquals(null, url.user) @@ -86,7 +86,7 @@ class UrlTest { assertEquals("https", url.protocol.name) assertEquals(8080, url.port) assertEquals("[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", url.host) - assertEquals(listOf("", "hello"), url.pathSegments) + assertEquals(listOf("", "hello"), url.rawSegments) assertEquals(null, url.user) assertEquals(null, url.password) assertEquals(false, url.trailingQuery) @@ -100,7 +100,7 @@ class UrlTest { assertEquals("http", url.protocol.name) assertEquals("127.0.0.1", url.host) - assertEquals(listOf("", "hello"), url.pathSegments) + assertEquals(listOf("", "hello"), url.rawSegments) assertEquals(null, url.user) assertEquals(null, url.password) assertEquals(false, url.trailingQuery) @@ -119,7 +119,7 @@ class UrlTest { assertEquals("http", url.protocol.name) assertNull(url.user) assertNull(url.password) - assertEquals(listOf("", "foo${case}bar"), url.pathSegments) + assertEquals(listOf("", "foo${case}bar"), url.rawSegments) assertEquals("http://localhost/foo${case}bar", url.toString()) } @@ -137,7 +137,7 @@ class UrlTest { assertEquals("http://httpbin.org/response-headers?message=foo%25bar", urlBuilder().buildString()) assertEquals("http://httpbin.org/response-headers?message=foo%25bar", url.toString()) - assertEquals(listOf("", "response-headers"), url.pathSegments) + assertEquals(listOf("", "response-headers"), url.rawSegments) assertEquals("/response-headers?message=foo%25bar", url.fullPath) } @@ -187,7 +187,7 @@ class UrlTest { with(url) { assertEquals(URLProtocol.HTTPS, protocol) assertEquals("www.test.com", host) - assertEquals(emptyList(), pathSegments) + assertEquals(emptyList(), rawSegments) assertEquals("https://www.test.com?test=ok&authtoken=testToken", url.toString()) } } @@ -250,7 +250,7 @@ class UrlTest { val result = Url(expectedUrl) assertEquals("file", result.protocol.name) assertEquals("", result.host) - assertEquals(listOf("", "var", "www"), result.pathSegments) + assertEquals(listOf("", "var", "www"), result.rawSegments) assertEquals(expectedUrl, result.toString()) } @@ -260,7 +260,7 @@ class UrlTest { val result = Url(expectedUrl) assertEquals("file", result.protocol.name) assertEquals("localhost", result.host) - assertEquals(listOf("", "var", "www"), result.pathSegments) + assertEquals(listOf("", "var", "www"), result.rawSegments) assertEquals(expectedUrl, result.toString()) } @@ -301,7 +301,7 @@ class UrlTest { val urlString = "https://ktor.io/quickstar%25t?query=strin%25g" val url = Url(urlString) assertEquals("/quickstar%25t", url.encodedPath) - assertEquals("quickstar%t", url.pathSegments[1]) + assertEquals("quickstar%t", url.rawSegments[1]) assertEquals("query=strin%25g", url.encodedQuery) assertEquals("strin%g", url.parameters["query"]) assertEquals("/quickstar%25t?query=strin%25g", url.encodedPathAndQuery) diff --git a/ktor-utils/common/src/io/ktor/util/cio/Readers.kt b/ktor-utils/common/src/io/ktor/util/cio/Readers.kt index 37b47fee49f..cd1cbfe8fe2 100644 --- a/ktor-utils/common/src/io/ktor/util/cio/Readers.kt +++ b/ktor-utils/common/src/io/ktor/util/cio/Readers.kt @@ -32,6 +32,7 @@ public inline fun ByteWriteChannel.use(block: ByteWriteChannel.() -> Unit) { close(cause) throw cause } finally { + @Suppress("DEPRECATION") close() } }