From 0a39f3f3c1ccb4e4fc56128a4862610129280d30 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Mon, 11 Feb 2019 13:22:23 -0800 Subject: [PATCH 01/29] Initial prototyping --- .../Http/HttpProtocol.FeatureCollection.cs | 36 ++++- .../Core/src/Internal/Http/HttpProtocol.cs | 7 +- .../Internal/Http/HttpRequestPipeReader.cs | 126 ++++++++++++++++++ .../src/Internal/Http/HttpRequestStream.cs | 58 -------- .../Core/src/Internal/Http/MessageBody.cs | 14 ++ 5 files changed, 180 insertions(+), 61 deletions(-) create mode 100644 src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs index 4289a46b883e..5875e6b1d881 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs @@ -20,6 +20,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http public partial class HttpProtocol : IHttpRequestFeature, IHttpResponseFeature, IResponseBodyPipeFeature, + IRequestBodyPipeFeature, IHttpUpgradeFeature, IHttpConnectionFeature, IHttpRequestLifetimeFeature, @@ -94,8 +95,39 @@ IHeaderDictionary IHttpRequestFeature.Headers Stream IHttpRequestFeature.Body { - get => RequestBody; - set => RequestBody = value; + get + { + return RequestBody; + } + set + { + RequestBody = value; + var requestPipeReader = new StreamPipeReader(RequestBody, new StreamPipeReaderOptions( + minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize, + minimumReadThreshold: KestrelMemoryPool.MinimumSegmentSize / 4, + _context.MemoryPool)); + RequestPipeReader = requestPipeReader; + + // The StreamPipeWrapper needs to be disposed as it hold onto blocks of memory + if (_wrapperObjectsToDispose == null) + { + _wrapperObjectsToDispose = new List(); + } + _wrapperObjectsToDispose.Add(requestPipeReader); + } + } + + PipeReader IRequestBodyPipeFeature.RequestBodyPipe + { + get + { + return RequestPipeReader; + } + set + { + RequestPipeReader = value; + RequestBody = new ReadOnlyPipeStream(RequestPipeReader); + } } int IHttpResponseFeature.StatusCode diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index d560f9a1eebb..a6a3392bd7ce 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -31,6 +31,7 @@ public abstract partial class HttpProtocol : IDefaultHttpContextContainer, IHttp protected Streams _streams; private HttpResponsePipeWriter _originalPipeWriter; + private HttpRequestPipeReader _originalPipeReader; private Stack, object>> _onStarting; private Stack, object>> _onCompleted; @@ -75,7 +76,7 @@ public HttpProtocol(HttpConnectionContext context) public IHttpResponseControl HttpResponseControl { get; set; } - public Pipe RequestBodyPipe { get; protected set; } + public Pipe RequestBodyPipe { get; protected set; } // TODO make sure this is removed or BodyPipe returns correct value public ServiceContext ServiceContext => _context.ServiceContext; private IPEndPoint LocalEndPoint => _context.LocalEndPoint; @@ -193,6 +194,7 @@ private void HttpVersionSetSlow(string value) public IHeaderDictionary RequestHeaders { get; set; } public Stream RequestBody { get; set; } + public PipeReader RequestPipeReader { get; set; } private int _statusCode; public int StatusCode @@ -298,12 +300,15 @@ public void InitializeStreams(MessageBody messageBody) if (_streams == null) { var pipeWriter = new HttpResponsePipeWriter(this); + var pipeReader = new HttpRequestPipeReader(this); _streams = new Streams(bodyControl: this, pipeWriter); _originalPipeWriter = pipeWriter; + _originalPipeReader = pipeReader; } (RequestBody, ResponseBody) = _streams.Start(messageBody); ResponsePipeWriter = _originalPipeWriter; + RequestPipeReader = _originalPipeReader; } public void StopStreams() => _streams.Stop(); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs new file mode 100644 index 000000000000..34e906201058 --- /dev/null +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs @@ -0,0 +1,126 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO.Pipelines; +using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http.Features; + +namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http +{ + internal class HttpRequestPipeReader : PipeReader + { + private MessageBody _body; + private HttpStreamState _state; + private Exception _error; + + // All of these will just call into MessageBody + public HttpRequestPipeReader() + { + _state = HttpStreamState.Closed; + } + + private HttpProtocol httpProtocol; + + public HttpRequestPipeReader(HttpProtocol httpProtocol) + { + this.httpProtocol = httpProtocol; + } + + public override void AdvanceTo(SequencePosition consumed) + { + _body.AdvanceTo(consumed); + } + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + _body.AdvanceTo(consumed, examined); + } + + public override void CancelPendingRead() + { + throw new NotImplementedException(); + } + + public override void Complete(Exception exception = null) + { + throw new NotImplementedException(); + } + + public override void OnWriterCompleted(Action callback, object state) + { + throw new NotImplementedException(); + } + + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override bool TryRead(out ReadResult result) + { + throw new NotImplementedException(); + } + + public void StartAcceptingReads(MessageBody body) + { + // Only start if not aborted + if (_state == HttpStreamState.Closed) + { + _state = HttpStreamState.Open; + _body = body; + } + } + + public void StopAcceptingReads() + { + // Can't use dispose (or close) as can be disposed too early by user code + // As exampled in EngineTests.ZeroContentLengthNotSetAutomaticallyForCertainStatusCodes + _state = HttpStreamState.Closed; + _body = null; + } + + public void Abort(Exception error = null) + { + // We don't want to throw an ODE until the app func actually completes. + // If the request is aborted, we throw a TaskCanceledException instead, + // unless error is not null, in which case we throw it. + if (_state != HttpStreamState.Closed) + { + _state = HttpStreamState.Aborted; + _error = error; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ValidateState(CancellationToken cancellationToken) + { + var state = _state; + if (state == HttpStreamState.Open) + { + cancellationToken.ThrowIfCancellationRequested(); + } + else if (state == HttpStreamState.Closed) + { + ThrowObjectDisposedException(); + } + else + { + if (_error != null) + { + ExceptionDispatchInfo.Capture(_error).Throw(); + } + else + { + ThrowTaskCanceledException(); + } + } + + void ThrowObjectDisposedException() => throw new ObjectDisposedException(nameof(HttpRequestStream)); + void ThrowTaskCanceledException() => throw new TaskCanceledException(); + } + } +} diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs index 31d73b248187..61dab59719c5 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs @@ -157,63 +157,5 @@ private async Task CopyToAsyncInternal(Stream destination, CancellationToken can throw new TaskCanceledException("The request was aborted", ex); } } - - public void StartAcceptingReads(MessageBody body) - { - // Only start if not aborted - if (_state == HttpStreamState.Closed) - { - _state = HttpStreamState.Open; - _body = body; - } - } - - public void StopAcceptingReads() - { - // Can't use dispose (or close) as can be disposed too early by user code - // As exampled in EngineTests.ZeroContentLengthNotSetAutomaticallyForCertainStatusCodes - _state = HttpStreamState.Closed; - _body = null; - } - - public void Abort(Exception error = null) - { - // We don't want to throw an ODE until the app func actually completes. - // If the request is aborted, we throw a TaskCanceledException instead, - // unless error is not null, in which case we throw it. - if (_state != HttpStreamState.Closed) - { - _state = HttpStreamState.Aborted; - _error = error; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ValidateState(CancellationToken cancellationToken) - { - var state = _state; - if (state == HttpStreamState.Open) - { - cancellationToken.ThrowIfCancellationRequested(); - } - else if (state == HttpStreamState.Closed) - { - ThrowObjectDisposedException(); - } - else - { - if (_error != null) - { - ExceptionDispatchInfo.Capture(_error).Throw(); - } - else - { - ThrowTaskCanceledException(); - } - } - - void ThrowObjectDisposedException() => throw new ObjectDisposedException(nameof(HttpRequestStream)); - void ThrowTaskCanceledException() => throw new TaskCanceledException(); - } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index 5dfe29549795..9a83fa352d84 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -27,6 +27,8 @@ public abstract class MessageBody private bool _backpressure; private long _alreadyTimedBytes; + + protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) { _context = context; @@ -45,6 +47,18 @@ protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) protected IKestrelTrace Log => _context.ServiceContext.Log; + // Make these methods pipe-like + + internal void AdvanceTo(SequencePosition consumed) + { + throw new NotImplementedException(); + } + + internal void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + throw new NotImplementedException(); + } + public virtual async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default(CancellationToken)) { TryStart(); From 281efb4740c5151ed73c3a525ce8a7bfa7163e3b Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 12 Feb 2019 14:55:26 -0800 Subject: [PATCH 02/29] Majority of refactor for HttpRequest --- .../Core/src/Internal/Http/HttpProtocol.cs | 5 +- .../Internal/Http/HttpRequestPipeReader.cs | 61 +++++++-- .../src/Internal/Http/HttpRequestStream.cs | 125 ++++-------------- .../Core/src/Internal/Http/MessageBody.cs | 57 +++++++- .../src/Internal/Infrastructure/Streams.cs | 6 +- .../Core/test/HttpRequestStreamTests.cs | 42 +++--- .../Kestrel/Core/test/MessageBodyTests.cs | 34 ++--- src/Servers/Kestrel/Core/test/StreamsTests.cs | 6 +- 8 files changed, 173 insertions(+), 163 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index a6a3392bd7ce..9c1f24cafea1 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -299,9 +299,10 @@ public void InitializeStreams(MessageBody messageBody) { if (_streams == null) { + // TODO really drop the streams stuff. It's broken var pipeWriter = new HttpResponsePipeWriter(this); - var pipeReader = new HttpRequestPipeReader(this); - _streams = new Streams(bodyControl: this, pipeWriter); + var pipeReader = new HttpRequestPipeReader(); + _streams = new Streams(bodyControl: this, pipeWriter, pipeReader); _originalPipeWriter = pipeWriter; _originalPipeReader = pipeReader; } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs index 34e906201058..a36d48db81a3 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs @@ -2,34 +2,28 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.IO; using System.IO.Pipelines; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Http.Features; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { - internal class HttpRequestPipeReader : PipeReader + public class HttpRequestPipeReader : PipeReader { private MessageBody _body; private HttpStreamState _state; private Exception _error; - // All of these will just call into MessageBody public HttpRequestPipeReader() { _state = HttpStreamState.Closed; } - private HttpProtocol httpProtocol; - - public HttpRequestPipeReader(HttpProtocol httpProtocol) - { - this.httpProtocol = httpProtocol; - } - public override void AdvanceTo(SequencePosition consumed) { _body.AdvanceTo(consumed); @@ -42,27 +36,31 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami public override void CancelPendingRead() { - throw new NotImplementedException(); + _body.CancelPendingRead(); } public override void Complete(Exception exception = null) { - throw new NotImplementedException(); + // TODO going to let this noop for now, I think we can support it but avoiding for now. + //throw new NotImplementedException(); } public override void OnWriterCompleted(Action callback, object state) { - throw new NotImplementedException(); + _body.OnWriterCompleted(callback, state); } public override ValueTask ReadAsync(CancellationToken cancellationToken = default) { - throw new NotImplementedException(); + ValidateState(cancellationToken); + + return _body.ReadAsync(cancellationToken); } public override bool TryRead(out ReadResult result) { - throw new NotImplementedException(); + // TODO validate state + return _body.TryRead(out result); } public void StartAcceptingReads(MessageBody body) @@ -75,6 +73,13 @@ public void StartAcceptingReads(MessageBody body) } } + public ValueTask ReadAsyncForStream(Memory buffer, CancellationToken cancellationToken) + { + ValidateState(cancellationToken); + + return _body.ReadAsync(buffer, cancellationToken); + } + public void StopAcceptingReads() { // Can't use dispose (or close) as can be disposed too early by user code @@ -122,5 +127,33 @@ private void ValidateState(CancellationToken cancellationToken) void ThrowObjectDisposedException() => throw new ObjectDisposedException(nameof(HttpRequestStream)); void ThrowTaskCanceledException() => throw new TaskCanceledException(); } + + public Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { + if (destination == null) + { + throw new ArgumentNullException(nameof(destination)); + } + if (bufferSize <= 0) + { + throw new ArgumentException(CoreStrings.PositiveNumberRequired, nameof(bufferSize)); + } + + ValidateState(cancellationToken); + + return CopyToAsyncInternal(destination, cancellationToken); + } + + private async Task CopyToAsyncInternal(Stream destination, CancellationToken cancellationToken) + { + try + { + await _body.CopyToAsync(destination, cancellationToken); + } + catch (ConnectionAbortedException ex) + { + throw new TaskCanceledException("The request was aborted", ex); + } + } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs index 61dab59719c5..d7e55e5f8940 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs @@ -3,57 +3,34 @@ using System; using System.IO; -using System.Runtime.CompilerServices; -using System.Runtime.ExceptionServices; +using System.IO.Pipelines; using System.Threading; using System.Threading.Tasks; -using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.Connections; -using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; +using Microsoft.AspNetCore.Http.Features; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { - internal class HttpRequestStream : ReadOnlyStream + internal class HttpRequestStream : ReadOnlyPipeStream { + private HttpRequestPipeReader _pipeReader; private readonly IHttpBodyControlFeature _bodyControl; - private MessageBody _body; - private HttpStreamState _state; - private Exception _error; - public HttpRequestStream(IHttpBodyControlFeature bodyControl) + public HttpRequestStream(IHttpBodyControlFeature bodyControl, HttpRequestPipeReader pipeReader) + : base (pipeReader) { _bodyControl = bodyControl; - _state = HttpStreamState.Closed; - } - - public override bool CanSeek => false; - - public override long Length - => throw new NotSupportedException(); - - public override long Position - { - get => throw new NotSupportedException(); - set => throw new NotSupportedException(); + _pipeReader = pipeReader; } - public override void Flush() - { - } - - public override Task FlushAsync(CancellationToken cancellationToken) - { - return Task.CompletedTask; - } - - public override long Seek(long offset, SeekOrigin origin) + public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) { - throw new NotSupportedException(); + return ReadAsyncInternal(new Memory(buffer, offset, count), cancellationToken).AsTask(); } - public override void SetLength(long value) + public override ValueTask ReadAsync(Memory destination, CancellationToken cancellationToken = default) { - throw new NotSupportedException(); + return ReadAsyncInternal(destination, cancellationToken); } public override int Read(byte[] buffer, int offset, int count) @@ -66,63 +43,27 @@ public override int Read(byte[] buffer, int offset, int count) return ReadAsync(buffer, offset, count).GetAwaiter().GetResult(); } - public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback callback, object state) - { - var task = ReadAsync(buffer, offset, count, default, state); - if (callback != null) - { - task.ContinueWith(t => callback.Invoke(t)); - } - return task; - } - - public override int EndRead(IAsyncResult asyncResult) - { - return ((Task)asyncResult).GetAwaiter().GetResult(); - } - - private Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken, object state) + public void StartAcceptingReads(MessageBody body) { - var tcs = new TaskCompletionSource(state); - var task = ReadAsync(buffer, offset, count, cancellationToken); - task.ContinueWith((task2, state2) => - { - var tcs2 = (TaskCompletionSource)state2; - if (task2.IsCanceled) - { - tcs2.SetCanceled(); - } - else if (task2.IsFaulted) - { - tcs2.SetException(task2.Exception); - } - else - { - tcs2.SetResult(task2.Result); - } - }, tcs, cancellationToken); - return tcs.Task; + _pipeReader.StartAcceptingReads(body); } - public override Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + public void StopAcceptingReads() { - ValidateState(cancellationToken); - - return ReadAsyncInternal(new Memory(buffer, offset, count), cancellationToken).AsTask(); + _pipeReader.StopAcceptingReads(); } - public override ValueTask ReadAsync(Memory destination, CancellationToken cancellationToken = default) + public void Abort(Exception exception = null) { - ValidateState(cancellationToken); - - return ReadAsyncInternal(destination, cancellationToken); + _pipeReader.Abort(exception); } private async ValueTask ReadAsyncInternal(Memory buffer, CancellationToken cancellationToken) { try { - return await _body.ReadAsync(buffer, cancellationToken); + // TODO why await here. + return await _pipeReader.ReadAsyncForStream(buffer, cancellationToken); } catch (ConnectionAbortedException ex) { @@ -130,32 +71,18 @@ private async ValueTask ReadAsyncInternal(Memory buffer, Cancellation } } - public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + public override void Flush() { - if (destination == null) - { - throw new ArgumentNullException(nameof(destination)); - } - if (bufferSize <= 0) - { - throw new ArgumentException(CoreStrings.PositiveNumberRequired, nameof(bufferSize)); - } - - ValidateState(cancellationToken); + } - return CopyToAsyncInternal(destination, cancellationToken); + public override Task FlushAsync(CancellationToken cancellationToken) + { + return Task.CompletedTask; } - private async Task CopyToAsyncInternal(Stream destination, CancellationToken cancellationToken) + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) { - try - { - await _body.CopyToAsync(destination, cancellationToken); - } - catch (ConnectionAbortedException ex) - { - throw new TaskCanceledException("The request was aborted", ex); - } + return _pipeReader.CopyToAsync(destination, bufferSize, cancellationToken); } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index 9a83fa352d84..7b08676cba4f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -27,8 +27,6 @@ public abstract class MessageBody private bool _backpressure; private long _alreadyTimedBytes; - - protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) { _context = context; @@ -51,12 +49,45 @@ protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) internal void AdvanceTo(SequencePosition consumed) { - throw new NotImplementedException(); + _context.RequestBodyPipe.Reader.AdvanceTo(consumed); } internal void AdvanceTo(SequencePosition consumed, SequencePosition examined) { - throw new NotImplementedException(); + _context.RequestBodyPipe.Reader.AdvanceTo(consumed, examined); + } + + public virtual async ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + TryStart(); + + while (true) + { + var result = await StartTimingReadAsync(cancellationToken); + var readableBuffer = result.Buffer; + var readableBufferLength = readableBuffer.Length; + StopTimingRead(readableBufferLength); + + try + { + if (readableBufferLength != 0) + { + return result; + } + + if (result.IsCompleted) + { + TryStop(); + return result; + } + } + finally + { + // Update the flow-control window after advancing the pipe reader, so we don't risk overfilling + // the pipe despite the client being well-behaved. + OnDataRead(readableBuffer.Length); + } + } } public virtual async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default(CancellationToken)) @@ -107,6 +138,24 @@ internal void AdvanceTo(SequencePosition consumed, SequencePosition examined) } } + public virtual bool TryRead(out ReadResult result) + { + return _context.RequestBodyPipe.Reader.TryRead(out result); + } + + public virtual void OnWriterCompleted(Action callback, object state) + { + _context.RequestBodyPipe.Reader.OnWriterCompleted(callback, state); + } + + public virtual void CancelPendingRead() + { + _context.RequestBodyPipe.Reader.CancelPendingRead(); + } + + // I think this can be removed or modified. + // Also how do we care about timing. + public virtual async Task CopyToAsync(Stream destination, CancellationToken cancellationToken = default(CancellationToken)) { TryStart(); diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/Streams.cs b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/Streams.cs index ee75017f6342..c93caf7dd737 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/Streams.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/Streams.cs @@ -18,10 +18,10 @@ private static readonly ThrowingWasUpgradedWriteOnlyStream _throwingResponseStre private readonly HttpRequestStream _emptyRequest; private readonly Stream _upgradeStream; - public Streams(IHttpBodyControlFeature bodyControl, HttpResponsePipeWriter writer) + public Streams(IHttpBodyControlFeature bodyControl, HttpResponsePipeWriter writer, HttpRequestPipeReader reader) { - _request = new HttpRequestStream(bodyControl); - _emptyRequest = new HttpRequestStream(bodyControl); + _request = new HttpRequestStream(bodyControl, reader); + _emptyRequest = new HttpRequestStream(bodyControl, new HttpRequestPipeReader()); _response = new HttpResponseStream(bodyControl, writer); _upgradeableResponse = new WrappingStream(_response); _upgradeStream = new HttpUpgradeStream(_request, _response); diff --git a/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs b/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs index ee3c21a042cf..5ff51dbb79bc 100644 --- a/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs +++ b/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs @@ -17,49 +17,49 @@ public class HttpRequestStreamTests [Fact] public void CanReadReturnsTrue() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); Assert.True(stream.CanRead); } [Fact] public void CanSeekReturnsFalse() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); Assert.False(stream.CanSeek); } [Fact] public void CanWriteReturnsFalse() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); Assert.False(stream.CanWrite); } [Fact] public void SeekThrows() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); Assert.Throws(() => stream.Seek(0, SeekOrigin.Begin)); } [Fact] public void LengthThrows() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); Assert.Throws(() => stream.Length); } [Fact] public void SetLengthThrows() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); Assert.Throws(() => stream.SetLength(0)); } [Fact] public void PositionThrows() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); Assert.Throws(() => stream.Position); Assert.Throws(() => stream.Position = 0); } @@ -67,21 +67,21 @@ public void PositionThrows() [Fact] public void WriteThrows() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); Assert.Throws(() => stream.Write(new byte[1], 0, 1)); } [Fact] public void WriteByteThrows() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); Assert.Throws(() => stream.WriteByte(0)); } [Fact] public async Task WriteAsyncThrows() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); await Assert.ThrowsAsync(() => stream.WriteAsync(new byte[1], 0, 1)); } @@ -89,14 +89,14 @@ public async Task WriteAsyncThrows() // Read-only streams should support Flush according to https://github.com/dotnet/corefx/pull/27327#pullrequestreview-98384813 public void FlushDoesNotThrow() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.Flush(); } [Fact] public async Task FlushAsyncDoesNotThrow() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); await stream.FlushAsync(); } @@ -109,7 +109,7 @@ public async Task SynchronousReadsThrowIfDisallowedByIHttpBodyControlFeature() var mockMessageBody = new Mock(null, null); mockMessageBody.Setup(m => m.ReadAsync(It.IsAny>(), CancellationToken.None)).Returns(new ValueTask(0)); - var stream = new HttpRequestStream(mockBodyControl.Object); + var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); stream.StartAcceptingReads(mockMessageBody.Object); Assert.Equal(0, await stream.ReadAsync(new byte[1], 0, 1)); @@ -127,7 +127,7 @@ public async Task SynchronousReadsThrowIfDisallowedByIHttpBodyControlFeature() [Fact] public async Task AbortCausesReadToCancel() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); stream.Abort(); await Assert.ThrowsAsync(() => stream.ReadAsync(new byte[1], 0, 1)); @@ -136,7 +136,7 @@ public async Task AbortCausesReadToCancel() [Fact] public async Task AbortWithErrorCausesReadToCancel() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); var error = new Exception(); stream.Abort(error); @@ -147,7 +147,7 @@ public async Task AbortWithErrorCausesReadToCancel() [Fact] public void StopAcceptingReadsCausesReadToThrowObjectDisposedException() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); stream.StopAcceptingReads(); Assert.Throws(() => { stream.ReadAsync(new byte[1], 0, 1); }); @@ -156,7 +156,7 @@ public void StopAcceptingReadsCausesReadToThrowObjectDisposedException() [Fact] public async Task AbortCausesCopyToAsyncToCancel() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); stream.Abort(); await Assert.ThrowsAsync(() => stream.CopyToAsync(Mock.Of())); @@ -165,7 +165,7 @@ public async Task AbortCausesCopyToAsyncToCancel() [Fact] public async Task AbortWithErrorCausesCopyToAsyncToCancel() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); var error = new Exception(); stream.Abort(error); @@ -176,7 +176,7 @@ public async Task AbortWithErrorCausesCopyToAsyncToCancel() [Fact] public void StopAcceptingReadsCausesCopyToAsyncToThrowObjectDisposedException() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); stream.StopAcceptingReads(); Assert.Throws(() => { stream.CopyToAsync(Mock.Of()); }); @@ -185,7 +185,7 @@ public void StopAcceptingReadsCausesCopyToAsyncToThrowObjectDisposedException() [Fact] public void NullDestinationCausesCopyToAsyncToThrowArgumentNullException() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); Assert.Throws(() => { stream.CopyToAsync(null); }); } @@ -193,7 +193,7 @@ public void NullDestinationCausesCopyToAsyncToThrowArgumentNullException() [Fact] public void ZeroBufferSizeCausesCopyToAsyncToThrowArgumentException() { - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); Assert.Throws(() => { stream.CopyToAsync(Mock.Of(), 0); }); } diff --git a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs index 159f1ef14fb2..2f762f85672f 100644 --- a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs +++ b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs @@ -33,7 +33,7 @@ public async Task CanReadFromContentLength(HttpVersion httpVersion) var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); - var stream = new HttpRequestStream(mockBodyControl.Object); + var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("Hello"); @@ -60,7 +60,7 @@ public async Task CanReadAsyncFromContentLength(HttpVersion httpVersion) using (var input = new TestInput()) { var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("Hello"); @@ -87,7 +87,7 @@ public async Task CanReadFromChunkedEncoding() var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); - var stream = new HttpRequestStream(mockBodyControl.Object); + var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("5\r\nHello\r\n"); @@ -114,7 +114,7 @@ public async Task CanReadAsyncFromChunkedEncoding() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("5\r\nHello\r\n"); @@ -141,7 +141,7 @@ public async Task ReadExitsGivenIncompleteChunkedExtension() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("5;\r\0"); @@ -167,7 +167,7 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThanMaxInt() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("80000000\r\n"); @@ -189,7 +189,7 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThan8Bytes() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("012345678\r"); @@ -215,7 +215,7 @@ public async Task CanReadFromRemainingData(HttpVersion httpVersion) var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderConnection = "upgrade" }, input.Http1Connection); var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); - var stream = new HttpRequestStream(mockBodyControl.Object); + var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("Hello"); @@ -241,7 +241,7 @@ public async Task CanReadAsyncFromRemainingData(HttpVersion httpVersion) using (var input = new TestInput()) { var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderConnection = "upgrade" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("Hello"); @@ -269,7 +269,7 @@ public async Task ReadFromNoContentLengthReturnsZero(HttpVersion httpVersion) var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders(), input.Http1Connection); var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); - var stream = new HttpRequestStream(mockBodyControl.Object); + var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("Hello"); @@ -289,7 +289,7 @@ public async Task ReadAsyncFromNoContentLengthReturnsZero(HttpVersion httpVersio using (var input = new TestInput()) { var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders(), input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("Hello"); @@ -307,7 +307,7 @@ public async Task CanHandleLargeBlocks() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http10, new HttpRequestHeaders { HeaderContentLength = "8197" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); // Input needs to be greater than 4032 bytes to allocate a block not backed by a slab. @@ -495,7 +495,7 @@ public async Task ConnectionUpgradeKeepAlive(string headerConnection) using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderConnection = headerConnection }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("Hello"); @@ -522,7 +522,7 @@ public async Task UpgradeConnectionAcceptsContentLengthZero() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderConnection = headerConnection, ContentLength = 0 }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); input.Add("Hello"); @@ -544,7 +544,7 @@ public async Task PumpAsyncDoesNotReturnAfterCancelingInput() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "2" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); // Add some input and consume it to ensure PumpAsync is running @@ -659,7 +659,7 @@ public async Task LogsWhenStartsReadingRequestBody() input.Http1Connection.TraceIdentifier = "RequestId"; var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "2" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); // Add some input and consume it to ensure PumpAsync is running @@ -690,7 +690,7 @@ public async Task LogsWhenStopsReadingRequestBody() input.Http1Connection.TraceIdentifier = "RequestId"; var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "2" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of()); + var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(body); // Add some input and consume it to ensure PumpAsync is running diff --git a/src/Servers/Kestrel/Core/test/StreamsTests.cs b/src/Servers/Kestrel/Core/test/StreamsTests.cs index a3c09848e338..5cab28250b77 100644 --- a/src/Servers/Kestrel/Core/test/StreamsTests.cs +++ b/src/Servers/Kestrel/Core/test/StreamsTests.cs @@ -16,7 +16,7 @@ public class StreamsTests [Fact] public async Task StreamsThrowAfterAbort() { - var streams = new Streams(Mock.Of(), new HttpResponsePipeWriter(Mock.Of())); + var streams = new Streams(Mock.Of(), new HttpResponsePipeWriter(Mock.Of()), new HttpRequestPipeReader()); var (request, response) = streams.Start(new MockMessageBody()); var ex = new Exception("My error"); @@ -30,7 +30,7 @@ public async Task StreamsThrowAfterAbort() [Fact] public async Task StreamsThrowOnAbortAfterUpgrade() { - var streams = new Streams(Mock.Of(), new HttpResponsePipeWriter(Mock.Of())); + var streams = new Streams(Mock.Of(), new HttpResponsePipeWriter(Mock.Of()), new HttpRequestPipeReader()); var (request, response) = streams.Start(new MockMessageBody(upgradeable: true)); var upgrade = streams.Upgrade(); @@ -52,7 +52,7 @@ public async Task StreamsThrowOnAbortAfterUpgrade() [Fact] public async Task StreamsThrowOnUpgradeAfterAbort() { - var streams = new Streams(Mock.Of(), new HttpResponsePipeWriter(Mock.Of())); + var streams = new Streams(Mock.Of(), new HttpResponsePipeWriter(Mock.Of()), new HttpRequestPipeReader()); var (request, response) = streams.Start(new MockMessageBody(upgradeable: true)); var ex = new Exception("My error"); From df26a51fee59f19b661e6811363ce94b7d3f6a12 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 12 Feb 2019 15:28:08 -0800 Subject: [PATCH 03/29] remove streams => bodyControl --- .../Http/HttpProtocol.FeatureCollection.cs | 2 +- .../Core/src/Internal/Http/HttpProtocol.cs | 27 ++++-------- .../{Streams.cs => BodyControl.cs} | 42 +++++++++++-------- .../{StreamsTests.cs => BodyControlTests.cs} | 26 ++++++------ .../Kestrel/Core/test/Http1ConnectionTests.cs | 4 +- .../Http1WritingBenchmark.cs | 2 +- 6 files changed, 51 insertions(+), 52 deletions(-) rename src/Servers/Kestrel/Core/src/Internal/Infrastructure/{Streams.cs => BodyControl.cs} (52%) rename src/Servers/Kestrel/Core/test/{StreamsTests.cs => BodyControlTests.cs} (69%) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs index 5875e6b1d881..1333c42a6d3d 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs @@ -307,7 +307,7 @@ async Task IHttpUpgradeFeature.UpgradeAsync() await FlushAsync(); - return _streams.Upgrade(); + return bodyControl.Upgrade(); } void IHttpRequestLifetimeFeature.Abort() diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index 9c1f24cafea1..dfbf192337b9 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -29,9 +29,7 @@ public abstract partial class HttpProtocol : IDefaultHttpContextContainer, IHttp private static readonly byte[] _bytesTransferEncodingChunked = Encoding.ASCII.GetBytes("\r\nTransfer-Encoding: chunked"); private static readonly byte[] _bytesServer = Encoding.ASCII.GetBytes("\r\nServer: " + Constants.ServerName); - protected Streams _streams; - private HttpResponsePipeWriter _originalPipeWriter; - private HttpRequestPipeReader _originalPipeReader; + protected BodyControl bodyControl; private Stack, object>> _onStarting; private Stack, object>> _onCompleted; @@ -295,24 +293,17 @@ DefaultHttpContext IDefaultHttpContextContainer.HttpContext } } - public void InitializeStreams(MessageBody messageBody) + public void InitializeBodyControl(MessageBody messageBody) { - if (_streams == null) + if (bodyControl == null) { - // TODO really drop the streams stuff. It's broken - var pipeWriter = new HttpResponsePipeWriter(this); - var pipeReader = new HttpRequestPipeReader(); - _streams = new Streams(bodyControl: this, pipeWriter, pipeReader); - _originalPipeWriter = pipeWriter; - _originalPipeReader = pipeReader; + bodyControl = new BodyControl(bodyControl: this, this); } - (RequestBody, ResponseBody) = _streams.Start(messageBody); - ResponsePipeWriter = _originalPipeWriter; - RequestPipeReader = _originalPipeReader; + (RequestBody, ResponseBody, RequestPipeReader, ResponsePipeWriter) = bodyControl.Start(messageBody); } - public void StopStreams() => _streams.Stop(); + public void StopBodies() => bodyControl.Stop(); // For testing internal void ResetState() @@ -466,7 +457,7 @@ protected void AbortRequest() protected void PoisonRequestBodyStream(Exception abortReason) { - _streams?.Abort(abortReason); + bodyControl?.Abort(abortReason); } // Prevents the RequestAborted token from firing for the duration of the request. @@ -572,7 +563,7 @@ private async Task ProcessRequests(IHttpApplication applicat IsUpgradableRequest = messageBody.RequestUpgrade; - InitializeStreams(messageBody); + InitializeBodyControl(messageBody); var context = application.CreateContext(this); @@ -614,7 +605,7 @@ private async Task ProcessRequests(IHttpApplication applicat // At this point all user code that needs use to the request or response streams has completed. // Using these streams in the OnCompleted callback is not allowed. - StopStreams(); + StopBodies(); // 4XX responses are written by TryProduceInvalidRequestResponse during connection tear down. if (_requestRejectedException == null) diff --git a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/Streams.cs b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/BodyControl.cs similarity index 52% rename from src/Servers/Kestrel/Core/src/Internal/Infrastructure/Streams.cs rename to src/Servers/Kestrel/Core/src/Internal/Infrastructure/BodyControl.cs index c93caf7dd737..5fbeead03e39 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Infrastructure/Streams.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Infrastructure/BodyControl.cs @@ -3,26 +3,34 @@ using System; using System.IO; +using System.IO.Pipelines; using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure { - public class Streams + public class BodyControl { private static readonly ThrowingWasUpgradedWriteOnlyStream _throwingResponseStream = new ThrowingWasUpgradedWriteOnlyStream(); private readonly HttpResponseStream _response; + private readonly HttpResponsePipeWriter _responseWriter; + private readonly HttpRequestPipeReader _requestReader; private readonly HttpRequestStream _request; + private readonly HttpRequestPipeReader _emptyRequestReader; private readonly WrappingStream _upgradeableResponse; private readonly HttpRequestStream _emptyRequest; private readonly Stream _upgradeStream; - public Streams(IHttpBodyControlFeature bodyControl, HttpResponsePipeWriter writer, HttpRequestPipeReader reader) + public BodyControl(IHttpBodyControlFeature bodyControl, IHttpResponseControl responseControl) { - _request = new HttpRequestStream(bodyControl, reader); - _emptyRequest = new HttpRequestStream(bodyControl, new HttpRequestPipeReader()); - _response = new HttpResponseStream(bodyControl, writer); + _requestReader = new HttpRequestPipeReader(); + _request = new HttpRequestStream(bodyControl, _requestReader); + _emptyRequestReader = new HttpRequestPipeReader(); + _emptyRequest = new HttpRequestStream(bodyControl, _emptyRequestReader); + + _responseWriter = new HttpResponsePipeWriter(responseControl); + _response = new HttpResponseStream(bodyControl, _responseWriter); _upgradeableResponse = new WrappingStream(_response); _upgradeStream = new HttpUpgradeStream(_request, _response); } @@ -35,37 +43,37 @@ public Stream Upgrade() return _upgradeStream; } - public (Stream request, Stream response) Start(MessageBody body) + public (Stream request, Stream response, PipeReader reader, PipeWriter writer) Start(MessageBody body) { - _request.StartAcceptingReads(body); - _emptyRequest.StartAcceptingReads(MessageBody.ZeroContentLengthClose); - _response.StartAcceptingWrites(); + _requestReader.StartAcceptingReads(body); + _emptyRequestReader.StartAcceptingReads(MessageBody.ZeroContentLengthClose); + _responseWriter.StartAcceptingWrites(); if (body.RequestUpgrade) { // until Upgrade() is called, context.Response.Body should use the normal output stream _upgradeableResponse.SetInnerStream(_response); // upgradeable requests should never have a request body - return (_emptyRequest, _upgradeableResponse); + return (_emptyRequest, _upgradeableResponse, _emptyRequestReader, _responseWriter); } else { - return (_request, _response); + return (_request, _response, _requestReader, _responseWriter); } } public void Stop() { - _request.StopAcceptingReads(); - _emptyRequest.StopAcceptingReads(); - _response.StopAcceptingWrites(); + _requestReader.StopAcceptingReads(); + _emptyRequestReader.StopAcceptingReads(); + _responseWriter.StopAcceptingWrites(); } public void Abort(Exception error) { - _request.Abort(error); - _emptyRequest.Abort(error); - _response.Abort(); + _requestReader.Abort(error); + _emptyRequestReader.Abort(error); + _responseWriter.Abort(); } } } diff --git a/src/Servers/Kestrel/Core/test/StreamsTests.cs b/src/Servers/Kestrel/Core/test/BodyControlTests.cs similarity index 69% rename from src/Servers/Kestrel/Core/test/StreamsTests.cs rename to src/Servers/Kestrel/Core/test/BodyControlTests.cs index 5cab28250b77..b172c4b17502 100644 --- a/src/Servers/Kestrel/Core/test/StreamsTests.cs +++ b/src/Servers/Kestrel/Core/test/BodyControlTests.cs @@ -11,16 +11,16 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests { - public class StreamsTests + public class BodyControlTests { [Fact] - public async Task StreamsThrowAfterAbort() + public async Task BodyControlThrowAfterAbort() { - var streams = new Streams(Mock.Of(), new HttpResponsePipeWriter(Mock.Of()), new HttpRequestPipeReader()); - var (request, response) = streams.Start(new MockMessageBody()); + var bodyControl = new BodyControl(Mock.Of(), Mock.Of()); + var (request, response, _, _) = bodyControl.Start(new MockMessageBody()); var ex = new Exception("My error"); - streams.Abort(ex); + bodyControl.Abort(ex); await response.WriteAsync(new byte[1], 0, 1); Assert.Same(ex, @@ -30,12 +30,12 @@ public async Task StreamsThrowAfterAbort() [Fact] public async Task StreamsThrowOnAbortAfterUpgrade() { - var streams = new Streams(Mock.Of(), new HttpResponsePipeWriter(Mock.Of()), new HttpRequestPipeReader()); - var (request, response) = streams.Start(new MockMessageBody(upgradeable: true)); + var bodyControl = new BodyControl(Mock.Of(), Mock.Of()); + var (request, response, _, _) = bodyControl.Start(new MockMessageBody(upgradeable: true)); - var upgrade = streams.Upgrade(); + var upgrade = bodyControl.Upgrade(); var ex = new Exception("My error"); - streams.Abort(ex); + bodyControl.Abort(ex); var writeEx = await Assert.ThrowsAsync(() => response.WriteAsync(new byte[1], 0, 1)); Assert.Equal(CoreStrings.ResponseStreamWasUpgraded, writeEx.Message); @@ -52,13 +52,13 @@ public async Task StreamsThrowOnAbortAfterUpgrade() [Fact] public async Task StreamsThrowOnUpgradeAfterAbort() { - var streams = new Streams(Mock.Of(), new HttpResponsePipeWriter(Mock.Of()), new HttpRequestPipeReader()); + var bodyControl = new BodyControl(Mock.Of(), Mock.Of()); - var (request, response) = streams.Start(new MockMessageBody(upgradeable: true)); + var (request, response, _, _) = bodyControl.Start(new MockMessageBody(upgradeable: true)); var ex = new Exception("My error"); - streams.Abort(ex); + bodyControl.Abort(ex); - var upgrade = streams.Upgrade(); + var upgrade = bodyControl.Upgrade(); var writeEx = await Assert.ThrowsAsync(() => response.WriteAsync(new byte[1], 0, 1)); Assert.Equal(CoreStrings.ResponseStreamWasUpgraded, writeEx.Message); diff --git a/src/Servers/Kestrel/Core/test/Http1ConnectionTests.cs b/src/Servers/Kestrel/Core/test/Http1ConnectionTests.cs index 17263e2f6daa..533c24c9ca4a 100644 --- a/src/Servers/Kestrel/Core/test/Http1ConnectionTests.cs +++ b/src/Servers/Kestrel/Core/test/Http1ConnectionTests.cs @@ -353,7 +353,7 @@ public void InitializeStreamsResetsStreams() { // Arrange var messageBody = Http1MessageBody.For(Kestrel.Core.Internal.Http.HttpVersion.Http11, (HttpRequestHeaders)_http1Connection.RequestHeaders, _http1Connection); - _http1Connection.InitializeStreams(messageBody); + _http1Connection.InitializeBodyControl(messageBody); var originalRequestBody = _http1Connection.RequestBody; var originalResponseBody = _http1Connection.ResponseBody; @@ -361,7 +361,7 @@ public void InitializeStreamsResetsStreams() _http1Connection.ResponseBody = new MemoryStream(); // Act - _http1Connection.InitializeStreams(messageBody); + _http1Connection.InitializeBodyControl(messageBody); // Assert Assert.Same(originalRequestBody, _http1Connection.RequestBody); diff --git a/src/Servers/Kestrel/perf/Kestrel.Performance/Http1WritingBenchmark.cs b/src/Servers/Kestrel/perf/Kestrel.Performance/Http1WritingBenchmark.cs index f272a24b6104..78f8359f06a0 100644 --- a/src/Servers/Kestrel/perf/Kestrel.Performance/Http1WritingBenchmark.cs +++ b/src/Servers/Kestrel/perf/Kestrel.Performance/Http1WritingBenchmark.cs @@ -119,7 +119,7 @@ private TestHttp1Connection MakeHttp1Connection() }); http1Connection.Reset(); - http1Connection.InitializeStreams(MessageBody.ZeroContentLengthKeepAlive); + http1Connection.InitializeBodyControl(MessageBody.ZeroContentLengthKeepAlive); serviceContext.DateHeaderValueManager.OnHeartbeat(DateTimeOffset.UtcNow); return http1Connection; From f0bca46fea46f4d8f0fec48b77280d874b6c992c Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 12 Feb 2019 17:24:28 -0800 Subject: [PATCH 04/29] Things are generally working. TODO adding tests for everything --- .../Internal/Http/HttpProtocol.Generated.cs | 23 +++ .../Internal/Http/HttpRequestPipeReader.cs | 1 - .../Core/src/Internal/Http/MessageBody.cs | 55 ++++--- .../ChunkedRequestTests.cs | 134 ++++++++++++++++++ .../InMemory.FunctionalTests/RequestTests.cs | 51 +++++++ .../HttpProtocolFeatureCollection.cs | 2 + 6 files changed, 245 insertions(+), 21 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.Generated.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.Generated.cs index 8e5fbc44aab1..aec1e2318f4b 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.Generated.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.Generated.cs @@ -16,6 +16,7 @@ public partial class HttpProtocol : IFeatureCollection private static readonly Type IHttpRequestFeatureType = typeof(IHttpRequestFeature); private static readonly Type IHttpResponseFeatureType = typeof(IHttpResponseFeature); private static readonly Type IResponseBodyPipeFeatureType = typeof(IResponseBodyPipeFeature); + private static readonly Type IRequestBodyPipeFeatureType = typeof(IRequestBodyPipeFeature); private static readonly Type IHttpRequestIdentifierFeatureType = typeof(IHttpRequestIdentifierFeature); private static readonly Type IServiceProvidersFeatureType = typeof(IServiceProvidersFeature); private static readonly Type IHttpRequestLifetimeFeatureType = typeof(IHttpRequestLifetimeFeature); @@ -41,6 +42,7 @@ public partial class HttpProtocol : IFeatureCollection private object _currentIHttpRequestFeature; private object _currentIHttpResponseFeature; private object _currentIResponseBodyPipeFeature; + private object _currentIRequestBodyPipeFeature; private object _currentIHttpRequestIdentifierFeature; private object _currentIServiceProvidersFeature; private object _currentIHttpRequestLifetimeFeature; @@ -72,6 +74,7 @@ private void FastReset() _currentIHttpRequestFeature = this; _currentIHttpResponseFeature = this; _currentIResponseBodyPipeFeature = this; + _currentIRequestBodyPipeFeature = this; _currentIHttpUpgradeFeature = this; _currentIHttpRequestIdentifierFeature = this; _currentIHttpRequestLifetimeFeature = this; @@ -160,6 +163,10 @@ object IFeatureCollection.this[Type key] { feature = _currentIResponseBodyPipeFeature; } + else if (key == IRequestBodyPipeFeatureType) + { + feature = _currentIRequestBodyPipeFeature; + } else if (key == IHttpRequestIdentifierFeatureType) { feature = _currentIHttpRequestIdentifierFeature; @@ -268,6 +275,10 @@ object IFeatureCollection.this[Type key] { _currentIResponseBodyPipeFeature = value; } + else if (key == IRequestBodyPipeFeatureType) + { + _currentIRequestBodyPipeFeature = value; + } else if (key == IHttpRequestIdentifierFeatureType) { _currentIHttpRequestIdentifierFeature = value; @@ -374,6 +385,10 @@ TFeature IFeatureCollection.Get() { feature = (TFeature)_currentIResponseBodyPipeFeature; } + else if (typeof(TFeature) == typeof(IRequestBodyPipeFeature)) + { + feature = (TFeature)_currentIRequestBodyPipeFeature; + } else if (typeof(TFeature) == typeof(IHttpRequestIdentifierFeature)) { feature = (TFeature)_currentIHttpRequestIdentifierFeature; @@ -486,6 +501,10 @@ void IFeatureCollection.Set(TFeature feature) { _currentIResponseBodyPipeFeature = feature; } + else if (typeof(TFeature) == typeof(IRequestBodyPipeFeature)) + { + _currentIRequestBodyPipeFeature = feature; + } else if (typeof(TFeature) == typeof(IHttpRequestIdentifierFeature)) { _currentIHttpRequestIdentifierFeature = feature; @@ -590,6 +609,10 @@ private IEnumerable> FastEnumerable() { yield return new KeyValuePair(IResponseBodyPipeFeatureType, _currentIResponseBodyPipeFeature); } + if (_currentIRequestBodyPipeFeature != null) + { + yield return new KeyValuePair(IRequestBodyPipeFeatureType, _currentIRequestBodyPipeFeature); + } if (_currentIHttpRequestIdentifierFeature != null) { yield return new KeyValuePair(IHttpRequestIdentifierFeatureType, _currentIHttpRequestIdentifierFeature); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs index a36d48db81a3..bbf9a07cbb5c 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs @@ -9,7 +9,6 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Connections; -using Microsoft.AspNetCore.Http.Features; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index 7b08676cba4f..aeb1ff607116 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -47,16 +47,33 @@ protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) // Make these methods pipe-like - internal void AdvanceTo(SequencePosition consumed) + public virtual void AdvanceTo(SequencePosition consumed) { _context.RequestBodyPipe.Reader.AdvanceTo(consumed); } - internal void AdvanceTo(SequencePosition consumed, SequencePosition examined) + public virtual void AdvanceTo(SequencePosition consumed, SequencePosition examined) { _context.RequestBodyPipe.Reader.AdvanceTo(consumed, examined); } + public virtual bool TryRead(out ReadResult result) + { + TryStart(); + + return _context.RequestBodyPipe.Reader.TryRead(out result); + } + + public virtual void OnWriterCompleted(Action callback, object state) + { + _context.RequestBodyPipe.Reader.OnWriterCompleted(callback, state); + } + + public virtual void CancelPendingRead() + { + _context.RequestBodyPipe.Reader.CancelPendingRead(); + } + public virtual async ValueTask ReadAsync(CancellationToken cancellationToken = default) { TryStart(); @@ -138,24 +155,6 @@ public virtual async ValueTask ReadAsync(CancellationToken cancellat } } - public virtual bool TryRead(out ReadResult result) - { - return _context.RequestBodyPipe.Reader.TryRead(out result); - } - - public virtual void OnWriterCompleted(Action callback, object state) - { - _context.RequestBodyPipe.Reader.OnWriterCompleted(callback, state); - } - - public virtual void CancelPendingRead() - { - _context.RequestBodyPipe.Reader.CancelPendingRead(); - } - - // I think this can be removed or modified. - // Also how do we care about timing. - public virtual async Task CopyToAsync(Stream destination, CancellationToken cancellationToken = default(CancellationToken)) { TryStart(); @@ -332,11 +331,27 @@ public ForZeroContentLength(bool keepAlive) public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default(CancellationToken)) => new ValueTask(0); + public override ValueTask ReadAsync(CancellationToken cancellationToken = default(CancellationToken)) => new ValueTask(new ReadResult(default, isCanceled: false, isCompleted: true)); + public override Task CopyToAsync(Stream destination, CancellationToken cancellationToken = default(CancellationToken)) => Task.CompletedTask; public override Task ConsumeAsync() => Task.CompletedTask; public override Task StopAsync() => Task.CompletedTask; + + public override void AdvanceTo(SequencePosition consumed) { } + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { } + + public override bool TryRead(out ReadResult result) + { + result = new ReadResult(default, isCanceled: false, isCompleted: true); + return true; + } + + public override void OnWriterCompleted(Action callback, object state) { } + + public override void CancelPendingRead() { } } } } diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs index 03ba3d3c23a3..93c879376e71 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Buffers; using System.Collections.Generic; using System.IO; using System.Linq; @@ -35,6 +36,24 @@ private async Task App(HttpContext httpContext) } } + private async Task PipeApp(HttpContext httpContext) + { + var request = httpContext.Request; + var response = httpContext.Response; + while (true) + { + var readResult = await request.BodyPipe.ReadAsync(); + if (readResult.IsCompleted) + { + break; + } + // Need to copy here. + await response.BodyPipe.WriteAsync(readResult.Buffer.ToArray()); + + request.BodyPipe.AdvanceTo(readResult.Buffer.End); + } + } + private async Task AppChunked(HttpContext httpContext) { var request = httpContext.Request; @@ -76,6 +95,35 @@ await connection.ReceiveEnd( } } + [Fact] + public async Task Http10TransferEncodingPipes() + { + var testContext = new TestServiceContext(LoggerFactory); + + using (var server = new TestServer(PipeApp, testContext)) + { + using (var connection = server.CreateConnection()) + { + await connection.Send( + "POST / HTTP/1.0", + "Host:", + "Transfer-Encoding: chunked", + "", + "5", "Hello", + "6", " World", + "0", + "", + ""); + await connection.ReceiveEnd( + "HTTP/1.1 200 OK", + "Connection: close", + $"Date: {testContext.DateHeaderValue}", + "", + "Hello World"); + } + } + } + [Fact] public async Task Http10KeepAliveTransferEncoding() { @@ -261,6 +309,92 @@ public async Task TrailingHeadersAreParsed() } } + [Fact] + public async Task TrailingHeadersAreParsedWithPipe() + { + var requestCount = 10; + var requestsReceived = 0; + + using (var server = new TestServer(async httpContext => + { + var response = httpContext.Response; + var request = httpContext.Request; + + var buffer = new byte[200]; + var result = await request.BodyPipe.ReadAsync(); + while (!result.IsCompleted) + { + request.BodyPipe.AdvanceTo(result.Buffer.End); + } + + if (requestsReceived < requestCount) + { + Assert.Equal(new string('a', requestsReceived), request.Headers["X-Trailer-Header"].ToString()); + } + else + { + Assert.True(string.IsNullOrEmpty(request.Headers["X-Trailer-Header"])); + } + + requestsReceived++; + + response.Headers["Content-Length"] = new[] { "11" }; + + await response.Body.WriteAsync(Encoding.ASCII.GetBytes("Hello World"), 0, 11); + }, new TestServiceContext(LoggerFactory))) + { + var response = string.Join("\r\n", new string[] { + "HTTP/1.1 200 OK", + $"Date: {server.Context.DateHeaderValue}", + "Content-Length: 11", + "", + "Hello World"}); + + var expectedFullResponse = string.Join("", Enumerable.Repeat(response, requestCount + 1)); + + IEnumerable sendSequence = new string[] { + "POST / HTTP/1.1", + "Host:", + "Transfer-Encoding: chunked", + "", + "C", + "HelloChunked", + "0", + ""}; + + for (var i = 1; i < requestCount; i++) + { + sendSequence = sendSequence.Concat(new string[] { + "POST / HTTP/1.1", + "Host:", + "Transfer-Encoding: chunked", + "", + "C", + $"HelloChunk{i:00}", + "0", + string.Concat("X-Trailer-Header: ", new string('a', i)), + "" }); + } + + sendSequence = sendSequence.Concat(new string[] { + "POST / HTTP/1.1", + "Host:", + "Content-Length: 7", + "", + "Goodbye" + }); + + var fullRequest = sendSequence.ToArray(); + + using (var connection = server.CreateConnection()) + { + await connection.Send(fullRequest); + await connection.Receive(expectedFullResponse); + } + + await server.StopAsync(); + } + } [Fact] public async Task TrailingHeadersCountTowardsHeadersTotalSizeLimit() { diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs index c482c63b4f0e..1c6cc964e861 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs @@ -649,6 +649,57 @@ await connection.ReceiveEnd( } } + [Fact] + public async Task ZeroContentLengthAssumedOnNonKeepAliveRequestsWithoutContentLengthOrTransferEncodingHeaderPipeReader() + { + var testContext = new TestServiceContext(LoggerFactory); + + using (var server = new TestServer(async httpContext => + { + var readResult = await httpContext.Request.BodyPipe.ReadAsync().AsTask().DefaultTimeout(); + // This will hang if 0 content length is not assumed by the server + Assert.True(readResult.IsCompleted); + }, testContext)) + { + using (var connection = server.CreateConnection()) + { + // Use Send instead of SendEnd to ensure the connection will remain open while + // the app runs and reads 0 bytes from the body nonetheless. This checks that + // https://github.com/aspnet/KestrelHttpServer/issues/1104 is not regressing. + await connection.Send( + "GET / HTTP/1.1", + "Host:", + "Connection: close", + "", + ""); + await connection.ReceiveEnd( + "HTTP/1.1 200 OK", + "Connection: close", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 0", + "", + ""); + } + + using (var connection = server.CreateConnection()) + { + await connection.Send( + "GET / HTTP/1.0", + "Host:", + "", + ""); + await connection.ReceiveEnd( + "HTTP/1.1 200 OK", + "Connection: close", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 0", + "", + ""); + } + await server.StopAsync(); + } + } + [Fact] public async Task ConnectionClosesWhenFinReceivedBeforeRequestCompletes() { diff --git a/src/Servers/Kestrel/tools/CodeGenerator/HttpProtocolFeatureCollection.cs b/src/Servers/Kestrel/tools/CodeGenerator/HttpProtocolFeatureCollection.cs index 4e7b1a689fb5..f61df14ff2c9 100644 --- a/src/Servers/Kestrel/tools/CodeGenerator/HttpProtocolFeatureCollection.cs +++ b/src/Servers/Kestrel/tools/CodeGenerator/HttpProtocolFeatureCollection.cs @@ -14,6 +14,7 @@ public static string GenerateFile() "IHttpRequestFeature", "IHttpResponseFeature", "IResponseBodyPipeFeature", + "IRequestBodyPipeFeature", "IHttpRequestIdentifierFeature", "IServiceProvidersFeature", "IHttpRequestLifetimeFeature", @@ -62,6 +63,7 @@ public static string GenerateFile() "IHttpRequestFeature", "IHttpResponseFeature", "IResponseBodyPipeFeature", + "IRequestBodyPipeFeature", "IHttpUpgradeFeature", "IHttpRequestIdentifierFeature", "IHttpRequestLifetimeFeature", From 9c5ca05fd501ccdfe1d9417912aa0dfa86c68b9c Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Wed, 13 Feb 2019 09:30:06 -0800 Subject: [PATCH 05/29] Rename and organization; Add Complete() --- .../Internal/Http/HttpRequestPipeReader.cs | 79 ++++++++++--------- .../src/Internal/Http/HttpRequestStream.cs | 4 +- .../Core/src/Internal/Http/MessageBody.cs | 15 ++-- 3 files changed, 51 insertions(+), 47 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs index bbf9a07cbb5c..1366919a8a33 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs @@ -40,8 +40,7 @@ public override void CancelPendingRead() public override void Complete(Exception exception = null) { - // TODO going to let this noop for now, I think we can support it but avoiding for now. - //throw new NotImplementedException(); + _body.Complete(exception); } public override void OnWriterCompleted(Action callback, object state) @@ -58,10 +57,49 @@ public override ValueTask ReadAsync(CancellationToken cancellationTo public override bool TryRead(out ReadResult result) { - // TODO validate state return _body.TryRead(out result); } + // Leaving stream APIs in HttpRequestPipeReader + // Upgrade overrides stream.ReadAsync and CopyToAsync to directly access the connection + // Also want to keep validation in the same place (only have HttpRequestPipeReader access the MessageBody). + + // TODO we can probably remove ForUpgrade.ReadAsync/CopyToAsync and use the ReadOnlyPipeStream for everything. + public ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) + { + ValidateState(cancellationToken); + + return _body.ReadAsync(buffer, cancellationToken); + } + + public Task CopyToStreamAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { + if (destination == null) + { + throw new ArgumentNullException(nameof(destination)); + } + if (bufferSize <= 0) + { + throw new ArgumentException(CoreStrings.PositiveNumberRequired, nameof(bufferSize)); + } + + ValidateState(cancellationToken); + + return CopyToAsyncInternal(destination, cancellationToken); + } + + private async Task CopyToAsyncInternal(Stream destination, CancellationToken cancellationToken) + { + try + { + await _body.CopyToAsync(destination, cancellationToken); + } + catch (ConnectionAbortedException ex) + { + throw new TaskCanceledException("The request was aborted", ex); + } + } + public void StartAcceptingReads(MessageBody body) { // Only start if not aborted @@ -72,13 +110,6 @@ public void StartAcceptingReads(MessageBody body) } } - public ValueTask ReadAsyncForStream(Memory buffer, CancellationToken cancellationToken) - { - ValidateState(cancellationToken); - - return _body.ReadAsync(buffer, cancellationToken); - } - public void StopAcceptingReads() { // Can't use dispose (or close) as can be disposed too early by user code @@ -126,33 +157,5 @@ private void ValidateState(CancellationToken cancellationToken) void ThrowObjectDisposedException() => throw new ObjectDisposedException(nameof(HttpRequestStream)); void ThrowTaskCanceledException() => throw new TaskCanceledException(); } - - public Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) - { - if (destination == null) - { - throw new ArgumentNullException(nameof(destination)); - } - if (bufferSize <= 0) - { - throw new ArgumentException(CoreStrings.PositiveNumberRequired, nameof(bufferSize)); - } - - ValidateState(cancellationToken); - - return CopyToAsyncInternal(destination, cancellationToken); - } - - private async Task CopyToAsyncInternal(Stream destination, CancellationToken cancellationToken) - { - try - { - await _body.CopyToAsync(destination, cancellationToken); - } - catch (ConnectionAbortedException ex) - { - throw new TaskCanceledException("The request was aborted", ex); - } - } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs index d7e55e5f8940..baa68f23f9a8 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs @@ -63,7 +63,7 @@ private async ValueTask ReadAsyncInternal(Memory buffer, Cancellation try { // TODO why await here. - return await _pipeReader.ReadAsyncForStream(buffer, cancellationToken); + return await _pipeReader.ReadAsync(buffer, cancellationToken); } catch (ConnectionAbortedException ex) { @@ -82,7 +82,7 @@ public override Task FlushAsync(CancellationToken cancellationToken) public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) { - return _pipeReader.CopyToAsync(destination, bufferSize, cancellationToken); + return _pipeReader.CopyToStreamAsync(destination, bufferSize, cancellationToken); } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index aeb1ff607116..d549d228a519 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -45,11 +45,9 @@ protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) protected IKestrelTrace Log => _context.ServiceContext.Log; - // Make these methods pipe-like - public virtual void AdvanceTo(SequencePosition consumed) { - _context.RequestBodyPipe.Reader.AdvanceTo(consumed); + AdvanceTo(consumed, consumed); } public virtual void AdvanceTo(SequencePosition consumed, SequencePosition examined) @@ -69,6 +67,11 @@ public virtual void OnWriterCompleted(Action callback, object _context.RequestBodyPipe.Reader.OnWriterCompleted(callback, state); } + public virtual void Complete(Exception exception) + { + _context.RequestBodyPipe.Reader.Complete(exception); + } + public virtual void CancelPendingRead() { _context.RequestBodyPipe.Reader.CancelPendingRead(); @@ -329,12 +332,8 @@ public ForZeroContentLength(bool keepAlive) public override bool IsEmpty => true; - public override ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default(CancellationToken)) => new ValueTask(0); - public override ValueTask ReadAsync(CancellationToken cancellationToken = default(CancellationToken)) => new ValueTask(new ReadResult(default, isCanceled: false, isCompleted: true)); - public override Task CopyToAsync(Stream destination, CancellationToken cancellationToken = default(CancellationToken)) => Task.CompletedTask; - public override Task ConsumeAsync() => Task.CompletedTask; public override Task StopAsync() => Task.CompletedTask; @@ -351,6 +350,8 @@ public override bool TryRead(out ReadResult result) public override void OnWriterCompleted(Action callback, object state) { } + public override void Complete(Exception ex) { } + public override void CancelPendingRead() { } } } From 879021a801576e7a1636133c549f850a2d4c836d Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Wed, 13 Feb 2019 13:02:14 -0800 Subject: [PATCH 06/29] Make things pipe like :D --- .../src/Internal/Http/Http1MessageBody.cs | 88 ++---- .../Internal/Http/HttpRequestPipeReader.cs | 40 --- .../src/Internal/Http/HttpRequestStream.cs | 10 +- .../Core/src/Internal/Http/MessageBody.cs | 123 ++------- .../HttpProtocolFeatureCollectionTests.cs | 3 + .../Core/test/HttpRequestStreamTests.cs | 18 +- .../Kestrel/Core/test/MessageBodyTests.cs | 252 +++++++++--------- .../ChunkedRequestTests.cs | 1 + .../Http2/Http2StreamTests.cs | 8 +- 9 files changed, 200 insertions(+), 343 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs index fb34300a4bd4..ad0d284cfd1b 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs @@ -160,6 +160,11 @@ protected override Task OnConsumeAsync() _context.SetBadRequestState(ex); return Task.CompletedTask; } + catch (Exception) + { + //_context.SetBadRequestState(ex); + return Task.CompletedTask; + } return OnConsumeAsyncAwaited(); } @@ -302,74 +307,39 @@ public ForUpgrade(Http1Connection context) // This returns IsEmpty so we can avoid draining the body (since it's basically an endless stream) public override bool IsEmpty => true; - public override async Task CopyToAsync(Stream destination, CancellationToken cancellationToken = default) + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) { - while (true) - { - var result = await _context.Input.ReadAsync(cancellationToken); - var readableBuffer = result.Buffer; - var readableBufferLength = readableBuffer.Length; - - try - { - if (!readableBuffer.IsEmpty) - { - foreach (var memory in readableBuffer) - { - // REVIEW: This *could* be slower if 2 things are true - // - The WriteAsync(ReadOnlyMemory) isn't overridden on the destination - // - We change the Kestrel Memory Pool to not use pinned arrays but instead use native memory - await destination.WriteAsync(memory, cancellationToken); - } - } - - if (result.IsCompleted) - { - return; - } - } - finally - { - _context.Input.AdvanceTo(readableBuffer.End); - } - } + return _context.Input.ReadAsync(cancellationToken); } - public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + public override bool TryRead(out ReadResult result) { - while (true) - { - var result = await _context.Input.ReadAsync(cancellationToken); - var readableBuffer = result.Buffer; - var readableBufferLength = readableBuffer.Length; + return _context.Input.TryRead(out result); + } - var consumed = readableBuffer.End; - var actual = 0; + public override void AdvanceTo(SequencePosition consumed) + { + _context.Input.AdvanceTo(consumed); + } - try - { - if (readableBufferLength != 0) - { - // buffer.Length is int - actual = (int)Math.Min(readableBufferLength, buffer.Length); + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + _context.Input.AdvanceTo(consumed, examined); + } - var slice = actual == readableBufferLength ? readableBuffer : readableBuffer.Slice(0, actual); - consumed = slice.End; - slice.CopyTo(buffer.Span); + public override void Complete(Exception exception) + { + // Noop as we don't want to complete the connection pipe. + } - return actual; - } + public override void CancelPendingRead() + { + _context.Input.CancelPendingRead(); + } - if (result.IsCompleted) - { - return 0; - } - } - finally - { - _context.Input.AdvanceTo(consumed); - } - } + public override void OnWriterCompleted(Action callback, object state) + { + _context.Input.OnWriterCompleted(callback, state); } public override Task ConsumeAsync() diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs index 1366919a8a33..7605304bc98e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs @@ -60,46 +60,6 @@ public override bool TryRead(out ReadResult result) return _body.TryRead(out result); } - // Leaving stream APIs in HttpRequestPipeReader - // Upgrade overrides stream.ReadAsync and CopyToAsync to directly access the connection - // Also want to keep validation in the same place (only have HttpRequestPipeReader access the MessageBody). - - // TODO we can probably remove ForUpgrade.ReadAsync/CopyToAsync and use the ReadOnlyPipeStream for everything. - public ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken) - { - ValidateState(cancellationToken); - - return _body.ReadAsync(buffer, cancellationToken); - } - - public Task CopyToStreamAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) - { - if (destination == null) - { - throw new ArgumentNullException(nameof(destination)); - } - if (bufferSize <= 0) - { - throw new ArgumentException(CoreStrings.PositiveNumberRequired, nameof(bufferSize)); - } - - ValidateState(cancellationToken); - - return CopyToAsyncInternal(destination, cancellationToken); - } - - private async Task CopyToAsyncInternal(Stream destination, CancellationToken cancellationToken) - { - try - { - await _body.CopyToAsync(destination, cancellationToken); - } - catch (ConnectionAbortedException ex) - { - throw new TaskCanceledException("The request was aborted", ex); - } - } - public void StartAcceptingReads(MessageBody body) { // Only start if not aborted diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs index baa68f23f9a8..a32041648be3 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs @@ -58,12 +58,11 @@ public void Abort(Exception exception = null) _pipeReader.Abort(exception); } - private async ValueTask ReadAsyncInternal(Memory buffer, CancellationToken cancellationToken) + private ValueTask ReadAsyncInternal(Memory buffer, CancellationToken cancellationToken) { try { - // TODO why await here. - return await _pipeReader.ReadAsync(buffer, cancellationToken); + return base.ReadAsync(buffer, cancellationToken); } catch (ConnectionAbortedException ex) { @@ -79,10 +78,5 @@ public override Task FlushAsync(CancellationToken cancellationToken) { return Task.CompletedTask; } - - public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) - { - return _pipeReader.CopyToStreamAsync(destination, bufferSize, cancellationToken); - } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index d549d228a519..2eb565561773 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -45,6 +45,8 @@ protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) protected IKestrelTrace Log => _context.ServiceContext.Log; + private ReadResult _previousReadResult; + public virtual void AdvanceTo(SequencePosition consumed) { AdvanceTo(consumed, consumed); @@ -52,14 +54,19 @@ public virtual void AdvanceTo(SequencePosition consumed) public virtual void AdvanceTo(SequencePosition consumed, SequencePosition examined) { + var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; _context.RequestBodyPipe.Reader.AdvanceTo(consumed, examined); + OnDataRead(dataLength); } - public virtual bool TryRead(out ReadResult result) + public virtual bool TryRead(out ReadResult readResult) { TryStart(); - return _context.RequestBodyPipe.Reader.TryRead(out result); + var hasReadResult = _context.RequestBodyPipe.Reader.TryRead(out _previousReadResult); + readResult = _previousReadResult; + + return hasReadResult; } public virtual void OnWriterCompleted(Action callback, object state) @@ -83,118 +90,20 @@ public virtual async ValueTask ReadAsync(CancellationToken cancellat while (true) { - var result = await StartTimingReadAsync(cancellationToken); - var readableBuffer = result.Buffer; - var readableBufferLength = readableBuffer.Length; - StopTimingRead(readableBufferLength); - - try - { - if (readableBufferLength != 0) - { - return result; - } - - if (result.IsCompleted) - { - TryStop(); - return result; - } - } - finally - { - // Update the flow-control window after advancing the pipe reader, so we don't risk overfilling - // the pipe despite the client being well-behaved. - OnDataRead(readableBuffer.Length); - } - } - } - - public virtual async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default(CancellationToken)) - { - TryStart(); - - while (true) - { - var result = await StartTimingReadAsync(cancellationToken); - var readableBuffer = result.Buffer; + _previousReadResult = await StartTimingReadAsync(cancellationToken); + var readableBuffer = _previousReadResult.Buffer; var readableBufferLength = readableBuffer.Length; StopTimingRead(readableBufferLength); - var consumed = readableBuffer.End; - var actual = 0; - - try - { - if (readableBufferLength != 0) - { - // buffer.Length is int - actual = (int)Math.Min(readableBufferLength, buffer.Length); - - // Make sure we don't double-count bytes on the next read. - _alreadyTimedBytes = readableBufferLength - actual; - - var slice = actual == readableBufferLength ? readableBuffer : readableBuffer.Slice(0, actual); - consumed = slice.End; - slice.CopyTo(buffer.Span); - - return actual; - } - - if (result.IsCompleted) - { - TryStop(); - return 0; - } - } - finally + if (readableBufferLength != 0) { - _context.RequestBodyPipe.Reader.AdvanceTo(consumed); - - // Update the flow-control window after advancing the pipe reader, so we don't risk overfilling - // the pipe despite the client being well-behaved. - OnDataRead(actual); + return _previousReadResult; } - } - } - - public virtual async Task CopyToAsync(Stream destination, CancellationToken cancellationToken = default(CancellationToken)) - { - TryStart(); - while (true) - { - var result = await StartTimingReadAsync(cancellationToken); - var readableBuffer = result.Buffer; - var readableBufferLength = readableBuffer.Length; - StopTimingRead(readableBufferLength); - - try + if (_previousReadResult.IsCompleted) { - if (readableBufferLength != 0) - { - foreach (var memory in readableBuffer) - { - // REVIEW: This *could* be slower if 2 things are true - // - The WriteAsync(ReadOnlyMemory) isn't overridden on the destination - // - We change the Kestrel Memory Pool to not use pinned arrays but instead use native memory - await destination.WriteAsync(memory, cancellationToken); - } - } - - if (result.IsCompleted) - { - TryStop(); - return; - } - } - finally - { - _context.RequestBodyPipe.Reader.AdvanceTo(readableBuffer.End); - - // Update the flow-control window after advancing the pipe reader, so we don't risk overfilling - // the pipe despite the client being well-behaved. - OnDataRead(readableBufferLength); + TryStop(); + return _previousReadResult; } } } diff --git a/src/Servers/Kestrel/Core/test/HttpProtocolFeatureCollectionTests.cs b/src/Servers/Kestrel/Core/test/HttpProtocolFeatureCollectionTests.cs index 5021fe334c9a..28b1df7c7ab5 100644 --- a/src/Servers/Kestrel/Core/test/HttpProtocolFeatureCollectionTests.cs +++ b/src/Servers/Kestrel/Core/test/HttpProtocolFeatureCollectionTests.cs @@ -118,6 +118,7 @@ public void FeaturesSetByTypeSameAsGeneric() _collection[typeof(IHttpRequestFeature)] = CreateHttp1Connection(); _collection[typeof(IHttpResponseFeature)] = CreateHttp1Connection(); _collection[typeof(IResponseBodyPipeFeature)] = CreateHttp1Connection(); + _collection[typeof(IRequestBodyPipeFeature)] = CreateHttp1Connection(); _collection[typeof(IHttpRequestIdentifierFeature)] = CreateHttp1Connection(); _collection[typeof(IHttpRequestLifetimeFeature)] = CreateHttp1Connection(); _collection[typeof(IHttpConnectionFeature)] = CreateHttp1Connection(); @@ -138,6 +139,7 @@ public void FeaturesSetByGenericSameAsByType() _collection.Set(CreateHttp1Connection()); _collection.Set(CreateHttp1Connection()); _collection.Set(CreateHttp1Connection()); + _collection.Set(CreateHttp1Connection()); _collection.Set(CreateHttp1Connection()); _collection.Set(CreateHttp1Connection()); _collection.Set(CreateHttp1Connection()); @@ -176,6 +178,7 @@ private void CompareGenericGetterToIndexer() Assert.Same(_collection.Get(), _collection[typeof(IHttpRequestFeature)]); Assert.Same(_collection.Get(), _collection[typeof(IHttpResponseFeature)]); Assert.Same(_collection.Get(), _collection[typeof(IResponseBodyPipeFeature)]); + Assert.Same(_collection.Get(), _collection[typeof(IRequestBodyPipeFeature)]); Assert.Same(_collection.Get(), _collection[typeof(IHttpRequestIdentifierFeature)]); Assert.Same(_collection.Get(), _collection[typeof(IHttpRequestLifetimeFeature)]); Assert.Same(_collection.Get(), _collection[typeof(IHttpConnectionFeature)]); diff --git a/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs b/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs index 5ff51dbb79bc..08c8788c8b93 100644 --- a/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs +++ b/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs @@ -3,6 +3,7 @@ using System; using System.IO; +using System.IO.Pipelines; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http.Features; @@ -104,10 +105,11 @@ public async Task FlushAsyncDoesNotThrow() public async Task SynchronousReadsThrowIfDisallowedByIHttpBodyControlFeature() { var allowSynchronousIO = false; + var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(() => allowSynchronousIO); var mockMessageBody = new Mock(null, null); - mockMessageBody.Setup(m => m.ReadAsync(It.IsAny>(), CancellationToken.None)).Returns(new ValueTask(0)); + mockMessageBody.Setup(m => m.ReadAsync(CancellationToken.None)).Returns(new ValueTask(new ReadResult(default, isCanceled: false, isCompleted: true))); var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); stream.StartAcceptingReads(mockMessageBody.Object); @@ -145,12 +147,14 @@ public async Task AbortWithErrorCausesReadToCancel() } [Fact] - public void StopAcceptingReadsCausesReadToThrowObjectDisposedException() + public async Task StopAcceptingReadsCausesReadToThrowObjectDisposedException() { var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); stream.StopAcceptingReads(); - Assert.Throws(() => { stream.ReadAsync(new byte[1], 0, 1); }); + + // Validation for ReadAsync occurs in an async method in ReadOnlyPipeStream. + await Assert.ThrowsAsync(async () => { await stream.ReadAsync(new byte[1], 0, 1); }); } [Fact] @@ -174,12 +178,13 @@ public async Task AbortWithErrorCausesCopyToAsyncToCancel() } [Fact] - public void StopAcceptingReadsCausesCopyToAsyncToThrowObjectDisposedException() + public async Task StopAcceptingReadsCausesCopyToAsyncToThrowObjectDisposedException() { var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); stream.StopAcceptingReads(); - Assert.Throws(() => { stream.CopyToAsync(Mock.Of()); }); + // Validation for CopyToAsync occurs in an async method in ReadOnlyPipeStream. + await Assert.ThrowsAsync(async () => { await stream.CopyToAsync(Mock.Of()); }); } [Fact] @@ -195,7 +200,8 @@ public void ZeroBufferSizeCausesCopyToAsyncToThrowArgumentException() { var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); stream.StartAcceptingReads(null); - Assert.Throws(() => { stream.CopyToAsync(Mock.Of(), 0); }); + // This is technically a breaking change, to throw an ArgumentoutOfRangeException rather than an ArgumentException + Assert.Throws(() => { stream.CopyToAsync(Mock.Of(), 0); }); } } } diff --git a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs index 2f762f85672f..e14f595d859b 100644 --- a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs +++ b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs @@ -375,26 +375,26 @@ public void ForThrowsWhenMethodRequiresLengthButNoContentLengthSetHttp10(HttpMet } } - [Fact] - public async Task CopyToAsyncDoesNotCompletePipeReader() - { - using (var input = new TestInput()) - { - var body = Http1MessageBody.For(HttpVersion.Http10, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + //[Fact] + //public async Task CopyToAsyncDoesNotCompletePipeReader() + //{ + // using (var input = new TestInput()) + // { + // var body = Http1MessageBody.For(HttpVersion.Http10, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); - input.Add("Hello"); + // input.Add("Hello"); - using (var ms = new MemoryStream()) - { - await body.CopyToAsync(ms); - } + // using (var ms = new MemoryStream()) + // { + // await body.CopyToAsync(ms); + // } - Assert.Equal(0, await body.ReadAsync(new ArraySegment(new byte[1]))); + // Assert.Equal(0, await body.ReadAsync(new ArraySegment(new byte[1]))); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); - await body.StopAsync(); - } - } + // input.Http1Connection.RequestBodyPipe.Reader.Complete(); + // await body.StopAsync(); + // } + //} [Fact] public async Task ConsumeAsyncConsumesAllRemainingInput() @@ -407,83 +407,83 @@ public async Task ConsumeAsyncConsumesAllRemainingInput() await body.ConsumeAsync(); - Assert.Equal(0, await body.ReadAsync(new ArraySegment(new byte[1]))); + Assert.True((await body.ReadAsync()).IsCompleted); input.Http1Connection.RequestBodyPipe.Reader.Complete(); await body.StopAsync(); } } - [Fact] - public async Task CopyToAsyncDoesNotCopyBlocks() - { - var writeCount = 0; - var writeTcs = new TaskCompletionSource<(byte[], int, int)>(TaskCreationOptions.RunContinuationsAsynchronously); - var mockDestination = new Mock { CallBase = true }; - - mockDestination - .Setup(m => m.WriteAsync(It.IsAny(), It.IsAny(), It.IsAny(), CancellationToken.None)) - .Callback((byte[] buffer, int offset, int count, CancellationToken cancellationToken) => - { - writeTcs.SetResult((buffer, offset, count)); - writeCount++; - }) - .Returns(Task.CompletedTask); - - using (var memoryPool = KestrelMemoryPool.Create()) - { - var options = new PipeOptions(pool: memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false); - var pair = DuplexPipe.CreateConnectionPair(options, options); - var transport = pair.Transport; - var http1ConnectionContext = new HttpConnectionContext - { - ServiceContext = new TestServiceContext(), - ConnectionFeatures = new FeatureCollection(), - Transport = transport, - MemoryPool = memoryPool, - TimeoutControl = Mock.Of() - }; - var http1Connection = new Http1Connection(http1ConnectionContext) - { - HasStartedConsumingRequestBody = true - }; - - var headers = new HttpRequestHeaders { HeaderContentLength = "12" }; - var body = Http1MessageBody.For(HttpVersion.Http11, headers, http1Connection); - - var copyToAsyncTask = body.CopyToAsync(mockDestination.Object); - - var bytes = Encoding.ASCII.GetBytes("Hello "); - var buffer = http1Connection.RequestBodyPipe.Writer.GetMemory(2048); - Assert.True(MemoryMarshal.TryGetArray(buffer, out ArraySegment segment)); - Buffer.BlockCopy(bytes, 0, segment.Array, segment.Offset, bytes.Length); - http1Connection.RequestBodyPipe.Writer.Advance(bytes.Length); - await http1Connection.RequestBodyPipe.Writer.FlushAsync(); - - // Verify the block passed to Stream.WriteAsync() is the same one incoming data was written into. - Assert.Equal((segment.Array, segment.Offset, bytes.Length), await writeTcs.Task); - - // Verify the again when GetMemory returns the tail space of the same block. - writeTcs = new TaskCompletionSource<(byte[], int, int)>(TaskCreationOptions.RunContinuationsAsynchronously); - bytes = Encoding.ASCII.GetBytes("World!"); - buffer = http1Connection.RequestBodyPipe.Writer.GetMemory(2048); - Assert.True(MemoryMarshal.TryGetArray(buffer, out segment)); - Buffer.BlockCopy(bytes, 0, segment.Array, segment.Offset, bytes.Length); - http1Connection.RequestBodyPipe.Writer.Advance(bytes.Length); - await http1Connection.RequestBodyPipe.Writer.FlushAsync(); - - Assert.Equal((segment.Array, segment.Offset, bytes.Length), await writeTcs.Task); - - http1Connection.RequestBodyPipe.Writer.Complete(); - - await copyToAsyncTask; - - Assert.Equal(2, writeCount); - - // Don't call body.StopAsync() because PumpAsync() was never called. - http1Connection.RequestBodyPipe.Reader.Complete(); - } - } + //[Fact] + //public async Task CopyToAsyncDoesNotCopyBlocks() + //{ + // var writeCount = 0; + // var writeTcs = new TaskCompletionSource<(byte[], int, int)>(TaskCreationOptions.RunContinuationsAsynchronously); + // var mockDestination = new Mock { CallBase = true }; + + // mockDestination + // .Setup(m => m.WriteAsync(It.IsAny(), It.IsAny(), It.IsAny(), CancellationToken.None)) + // .Callback((byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + // { + // writeTcs.SetResult((buffer, offset, count)); + // writeCount++; + // }) + // .Returns(Task.CompletedTask); + + // using (var memoryPool = KestrelMemoryPool.Create()) + // { + // var options = new PipeOptions(pool: memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false); + // var pair = DuplexPipe.CreateConnectionPair(options, options); + // var transport = pair.Transport; + // var http1ConnectionContext = new HttpConnectionContext + // { + // ServiceContext = new TestServiceContext(), + // ConnectionFeatures = new FeatureCollection(), + // Transport = transport, + // MemoryPool = memoryPool, + // TimeoutControl = Mock.Of() + // }; + // var http1Connection = new Http1Connection(http1ConnectionContext) + // { + // HasStartedConsumingRequestBody = true + // }; + + // var headers = new HttpRequestHeaders { HeaderContentLength = "12" }; + // var body = Http1MessageBody.For(HttpVersion.Http11, headers, http1Connection); + + // var copyToAsyncTask = body.CopyToAsync(mockDestination.Object); + + // var bytes = Encoding.ASCII.GetBytes("Hello "); + // var buffer = http1Connection.RequestBodyPipe.Writer.GetMemory(2048); + // Assert.True(MemoryMarshal.TryGetArray(buffer, out ArraySegment segment)); + // Buffer.BlockCopy(bytes, 0, segment.Array, segment.Offset, bytes.Length); + // http1Connection.RequestBodyPipe.Writer.Advance(bytes.Length); + // await http1Connection.RequestBodyPipe.Writer.FlushAsync(); + + // // Verify the block passed to Stream.WriteAsync() is the same one incoming data was written into. + // Assert.Equal((segment.Array, segment.Offset, bytes.Length), await writeTcs.Task); + + // // Verify the again when GetMemory returns the tail space of the same block. + // writeTcs = new TaskCompletionSource<(byte[], int, int)>(TaskCreationOptions.RunContinuationsAsynchronously); + // bytes = Encoding.ASCII.GetBytes("World!"); + // buffer = http1Connection.RequestBodyPipe.Writer.GetMemory(2048); + // Assert.True(MemoryMarshal.TryGetArray(buffer, out segment)); + // Buffer.BlockCopy(bytes, 0, segment.Array, segment.Offset, bytes.Length); + // http1Connection.RequestBodyPipe.Writer.Advance(bytes.Length); + // await http1Connection.RequestBodyPipe.Writer.FlushAsync(); + + // Assert.Equal((segment.Array, segment.Offset, bytes.Length), await writeTcs.Task); + + // http1Connection.RequestBodyPipe.Writer.Complete(); + + // await copyToAsyncTask; + + // Assert.Equal(2, writeCount); + + // // Don't call body.StopAsync() because PumpAsync() was never called. + // http1Connection.RequestBodyPipe.Reader.Complete(); + // } + //} [Theory] [InlineData("keep-alive, upgrade")] @@ -575,12 +575,14 @@ public async Task ReadAsyncThrowsOnTimeout() // Add some input and read it to start PumpAsync input.Add("a"); - Assert.Equal(1, await body.ReadAsync(new ArraySegment(new byte[1]))); + var readResult = await body.ReadAsync(); + Assert.Equal(1, readResult.Buffer.Length); + body.AdvanceTo(readResult.Buffer.End); // Time out on the next read input.Http1Connection.SendTimeoutResponse(); - var exception = await Assert.ThrowsAsync(async () => await body.ReadAsync(new Memory(new byte[1]))); + var exception = await Assert.ThrowsAsync(async () => await body.ReadAsync()); Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); input.Http1Connection.RequestBodyPipe.Reader.Complete(); @@ -603,7 +605,11 @@ public async Task ConsumeAsyncCompletesAndDoesNotThrowOnTimeout() // Add some input and read it to start PumpAsync input.Add("a"); - Assert.Equal(1, await body.ReadAsync(new ArraySegment(new byte[1]))); + var readResult = await body.ReadAsync(); + Assert.Equal(1, readResult.Buffer.Length); + + // need to advance to make PipeReader in ReadCompleted state + body.AdvanceTo(readResult.Buffer.End); // Time out on the next read input.Http1Connection.SendTimeoutResponse(); @@ -619,34 +625,34 @@ public async Task ConsumeAsyncCompletesAndDoesNotThrowOnTimeout() } } - [Fact] - public async Task CopyToAsyncThrowsOnTimeout() - { - using (var input = new TestInput()) - { - var mockTimeoutControl = new Mock(); + //[Fact] + //public async Task CopyToAsyncThrowsOnTimeout() + //{ + // using (var input = new TestInput()) + // { + // var mockTimeoutControl = new Mock(); - input.Http1ConnectionContext.TimeoutControl = mockTimeoutControl.Object; + // input.Http1ConnectionContext.TimeoutControl = mockTimeoutControl.Object; - var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + // var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); - // Add some input and read it to start PumpAsync - input.Add("a"); - Assert.Equal(1, await body.ReadAsync(new ArraySegment(new byte[1]))); + // // Add some input and read it to start PumpAsync + // input.Add("a"); + // Assert.Equal(1, (await body.ReadAsync()).Buffer.Length); - // Time out on the next read - input.Http1Connection.SendTimeoutResponse(); + // // Time out on the next read + // input.Http1Connection.SendTimeoutResponse(); - using (var ms = new MemoryStream()) - { - var exception = await Assert.ThrowsAsync(() => body.CopyToAsync(ms)); - Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); - } + // using (var ms = new MemoryStream()) + // { + // var exception = await Assert.ThrowsAsync(() => body.CopyToAsync(ms)); + // Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); + // } - input.Http1Connection.RequestBodyPipe.Reader.Complete(); - await body.StopAsync(); - } - } + // input.Http1Connection.RequestBodyPipe.Reader.Complete(); + // await body.StopAsync(); + // } + //} [Fact] public async Task LogsWhenStartsReadingRequestBody() @@ -717,13 +723,17 @@ public async Task PausesAndResumesRequestBodyTimeoutOnBackpressure() var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "12" }, input.Http1Connection); // Add some input and read it to start PumpAsync - var readTask1 = body.ReadAsync(new ArraySegment(new byte[6])); + var readTask1 = body.ReadAsync(); input.Add("hello,"); - Assert.Equal(6, await readTask1); + var readResult = await readTask1; + Assert.Equal(6, readResult.Buffer.Length); + body.AdvanceTo(readResult.Buffer.End); - var readTask2 = body.ReadAsync(new ArraySegment(new byte[6])); + var readTask2 = body.ReadAsync(); input.Add(" world"); - Assert.Equal(6, await readTask2); + readResult = await readTask2; + Assert.Equal(6, readResult.Buffer.Length); + body.AdvanceTo(readResult.Buffer.End); // Due to the limits set on HttpProtocol.RequestBodyPipe, backpressure should be triggered on every write to that pipe. mockTimeoutControl.Verify(timeoutControl => timeoutControl.StopTimingRead(), Times.Exactly(2)); @@ -751,7 +761,7 @@ public async Task OnlyEnforcesRequestBodyTimeoutAfterFirstRead() Assert.False(startRequestBodyCalled); // Add some input and read it to start PumpAsync - var readTask = body.ReadAsync(new ArraySegment(new byte[1])); + var readTask = body.ReadAsync(); Assert.True(startRequestBodyCalled); @@ -776,11 +786,15 @@ public async Task DoesNotEnforceRequestBodyTimeoutOnUpgradeRequests() // Add some input and read it to start PumpAsync input.Add("a"); - Assert.Equal(1, await body.ReadAsync(new ArraySegment(new byte[1]))); + var readResult = await body.ReadAsync(); + Assert.Equal(1, readResult.Buffer.Length); + + // need to advance to make PipeReader in ReadCompleted state + body.AdvanceTo(readResult.Buffer.End); input.Fin(); - Assert.Equal(0, await body.ReadAsync(new ArraySegment(new byte[1]))); + Assert.True((await body.ReadAsync()).IsCompleted); mockTimeoutControl.Verify(timeoutControl => timeoutControl.StartRequestBody(minReadRate), Times.Never); mockTimeoutControl.Verify(timeoutControl => timeoutControl.StopRequestBody(), Times.Never); diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs index 93c879376e71..697492093c1d 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs @@ -325,6 +325,7 @@ public async Task TrailingHeadersAreParsedWithPipe() while (!result.IsCompleted) { request.BodyPipe.AdvanceTo(result.Buffer.End); + result = await request.BodyPipe.ReadAsync(); } if (requestsReceived < requestCount) diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs index a35fdb388b5f..2512e27c9c4a 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs @@ -2301,10 +2301,10 @@ await InitializeConnectionAsync(async context => await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); Assert.NotNull(thrownEx); - Assert.IsType(thrownEx); - Assert.Equal("The request was aborted", thrownEx.Message); - Assert.IsType(thrownEx.InnerException); - Assert.Equal(CoreStrings.ConnectionAbortedByApplication, thrownEx.InnerException.Message); + // TODO due to the method going async, we don't rethrow the connection aborted exception. + // We can look into fixing this. + Assert.IsType(thrownEx); + Assert.Equal(CoreStrings.ConnectionAbortedByApplication, thrownEx.Message); } // Sync writes after async writes could block the write loop if the callback is not dispatched. From eb74db844189911354edeb553fa1465c7c4e454c Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Wed, 13 Feb 2019 13:07:33 -0800 Subject: [PATCH 07/29] minor nits --- .../Core/src/Internal/Http/Http1MessageBody.cs | 5 ----- .../Core/src/Internal/Http/HttpProtocol.cs | 2 +- .../Core/src/Internal/Http/HttpResponseStream.cs | 15 --------------- .../Kestrel/Core/test/HttpResponseStreamTests.cs | 6 +++--- .../Kestrel/test/FunctionalTests/RequestTests.cs | 2 +- 5 files changed, 5 insertions(+), 25 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs index ad0d284cfd1b..e70faea85ec9 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs @@ -160,11 +160,6 @@ protected override Task OnConsumeAsync() _context.SetBadRequestState(ex); return Task.CompletedTask; } - catch (Exception) - { - //_context.SetBadRequestState(ex); - return Task.CompletedTask; - } return OnConsumeAsyncAwaited(); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index dfbf192337b9..b855591ee800 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -74,7 +74,7 @@ public HttpProtocol(HttpConnectionContext context) public IHttpResponseControl HttpResponseControl { get; set; } - public Pipe RequestBodyPipe { get; protected set; } // TODO make sure this is removed or BodyPipe returns correct value + public Pipe RequestBodyPipe { get; protected set; } public ServiceContext ServiceContext => _context.ServiceContext; private IPEndPoint LocalEndPoint => _context.LocalEndPoint; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpResponseStream.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpResponseStream.cs index 054ffaffa09f..0c9f19075e61 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpResponseStream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpResponseStream.cs @@ -38,20 +38,5 @@ public override void Flush() base.Flush(); } - - public void StartAcceptingWrites() - { - _pipeWriter.StartAcceptingWrites(); - } - - public void StopAcceptingWrites() - { - _pipeWriter.StopAcceptingWrites(); - } - - public void Abort() - { - _pipeWriter.Abort(); - } } } diff --git a/src/Servers/Kestrel/Core/test/HttpResponseStreamTests.cs b/src/Servers/Kestrel/Core/test/HttpResponseStreamTests.cs index 3f00c58e0acc..4b7e167e9b94 100644 --- a/src/Servers/Kestrel/Core/test/HttpResponseStreamTests.cs +++ b/src/Servers/Kestrel/Core/test/HttpResponseStreamTests.cs @@ -98,8 +98,8 @@ public void StopAcceptingWritesCausesWriteToThrowObjectDisposedException() { var pipeWriter = new HttpResponsePipeWriter(Mock.Of()); var stream = new HttpResponseStream(Mock.Of(), pipeWriter); - stream.StartAcceptingWrites(); - stream.StopAcceptingWrites(); + pipeWriter.StartAcceptingWrites(); + pipeWriter.StopAcceptingWrites(); var ex = Assert.Throws(() => { stream.WriteAsync(new byte[1], 0, 1); }); Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message); } @@ -115,7 +115,7 @@ public async Task SynchronousWritesThrowIfDisallowedByIHttpBodyControlFeature() var pipeWriter = new HttpResponsePipeWriter(mockHttpResponseControl.Object); var stream = new HttpResponseStream(mockBodyControl.Object, pipeWriter); - stream.StartAcceptingWrites(); + pipeWriter.StartAcceptingWrites(); // WriteAsync doesn't throw. await stream.WriteAsync(new byte[1], 0, 1); diff --git a/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs index 46d722aec939..4132de863f77 100644 --- a/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs @@ -697,7 +697,7 @@ await connection.Send("POST / HTTP/1.1", await server.StopAsync(); } - await Assert.ThrowsAsync(async () => await readTcs.Task); + await Assert.ThrowsAsync(async () => await readTcs.Task); // The cancellation token for only the last request should be triggered. var abortedRequestId = await registrationTcs.Task.DefaultTimeout(); From 54e0f5dc5f5eb5452a3211a91fab343c118b73a5 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Wed, 13 Feb 2019 14:22:43 -0800 Subject: [PATCH 08/29] updating tests --- .../Internal/Http/HttpRequestPipeReader.cs | 14 +++- .../src/Internal/Http/HttpRequestStream.cs | 15 ---- .../Kestrel/Core/test/BodyControlTests.cs | 79 +++++++++++++++++-- .../Core/test/HttpRequestPipeReaderTests.cs | 10 +++ .../Core/test/HttpRequestStreamTests.cs | 57 +++++++------ 5 files changed, 131 insertions(+), 44 deletions(-) create mode 100644 src/Servers/Kestrel/Core/test/HttpRequestPipeReaderTests.cs diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs index 7605304bc98e..b6bd80935412 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs @@ -25,26 +25,36 @@ public HttpRequestPipeReader() public override void AdvanceTo(SequencePosition consumed) { + ValidateState(); + _body.AdvanceTo(consumed); } public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { + ValidateState(); + _body.AdvanceTo(consumed, examined); } public override void CancelPendingRead() { + ValidateState(); + _body.CancelPendingRead(); } public override void Complete(Exception exception = null) { + ValidateState(); + _body.Complete(exception); } public override void OnWriterCompleted(Action callback, object state) { + ValidateState(); + _body.OnWriterCompleted(callback, state); } @@ -57,6 +67,8 @@ public override ValueTask ReadAsync(CancellationToken cancellationTo public override bool TryRead(out ReadResult result) { + ValidateState(); + return _body.TryRead(out result); } @@ -91,7 +103,7 @@ public void Abort(Exception error = null) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private void ValidateState(CancellationToken cancellationToken) + private void ValidateState(CancellationToken cancellationToken = default) { var state = _state; if (state == HttpStreamState.Open) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs index a32041648be3..587fdc55c060 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestStream.cs @@ -43,21 +43,6 @@ public override int Read(byte[] buffer, int offset, int count) return ReadAsync(buffer, offset, count).GetAwaiter().GetResult(); } - public void StartAcceptingReads(MessageBody body) - { - _pipeReader.StartAcceptingReads(body); - } - - public void StopAcceptingReads() - { - _pipeReader.StopAcceptingReads(); - } - - public void Abort(Exception exception = null) - { - _pipeReader.Abort(exception); - } - private ValueTask ReadAsyncInternal(Memory buffer, CancellationToken cancellationToken) { try diff --git a/src/Servers/Kestrel/Core/test/BodyControlTests.cs b/src/Servers/Kestrel/Core/test/BodyControlTests.cs index b172c4b17502..2ac070d56b04 100644 --- a/src/Servers/Kestrel/Core/test/BodyControlTests.cs +++ b/src/Servers/Kestrel/Core/test/BodyControlTests.cs @@ -17,7 +17,7 @@ public class BodyControlTests public async Task BodyControlThrowAfterAbort() { var bodyControl = new BodyControl(Mock.Of(), Mock.Of()); - var (request, response, _, _) = bodyControl.Start(new MockMessageBody()); + var (request, response, requestPipe, responsePipe) = bodyControl.Start(new MockMessageBody()); var ex = new Exception("My error"); bodyControl.Abort(ex); @@ -25,13 +25,15 @@ public async Task BodyControlThrowAfterAbort() await response.WriteAsync(new byte[1], 0, 1); Assert.Same(ex, await Assert.ThrowsAsync(() => request.ReadAsync(new byte[1], 0, 1))); + Assert.Same(ex, + await Assert.ThrowsAsync(async () => await requestPipe.ReadAsync())); } [Fact] - public async Task StreamsThrowOnAbortAfterUpgrade() + public async Task BodyControlThrowOnAbortAfterUpgrade() { var bodyControl = new BodyControl(Mock.Of(), Mock.Of()); - var (request, response, _, _) = bodyControl.Start(new MockMessageBody(upgradeable: true)); + var (request, response, requestPipe, responsePipe) = bodyControl.Start(new MockMessageBody(upgradeable: true)); var upgrade = bodyControl.Upgrade(); var ex = new Exception("My error"); @@ -46,15 +48,18 @@ public async Task StreamsThrowOnAbortAfterUpgrade() Assert.Same(ex, await Assert.ThrowsAsync(() => upgrade.ReadAsync(new byte[1], 0, 1))); + Assert.Same(ex, + await Assert.ThrowsAsync(async () => await requestPipe.ReadAsync())); + await upgrade.WriteAsync(new byte[1], 0, 1); } [Fact] - public async Task StreamsThrowOnUpgradeAfterAbort() + public async Task BodyControlThrowOnUpgradeAfterAbort() { var bodyControl = new BodyControl(Mock.Of(), Mock.Of()); - var (request, response, _, _) = bodyControl.Start(new MockMessageBody(upgradeable: true)); + var (request, response, requestPipe, responsePipe) = bodyControl.Start(new MockMessageBody(upgradeable: true)); var ex = new Exception("My error"); bodyControl.Abort(ex); @@ -68,10 +73,74 @@ public async Task StreamsThrowOnUpgradeAfterAbort() Assert.Same(ex, await Assert.ThrowsAsync(() => upgrade.ReadAsync(new byte[1], 0, 1))); + Assert.Same(ex, + await Assert.ThrowsAsync(async () => await requestPipe.ReadAsync())); await upgrade.WriteAsync(new byte[1], 0, 1); } + [Fact] + public async Task RequestPipeMethodsThrowAfterAbort() + { + var bodyControl = new BodyControl(Mock.Of(), Mock.Of()); + + var (_, response, requestPipe, responsePipe) = bodyControl.Start(new MockMessageBody(upgradeable: true)); + var ex = new Exception("My error"); + bodyControl.Abort(ex); + + await response.WriteAsync(new byte[1], 0, 1); + Assert.Same(ex, + Assert.Throws(() => requestPipe.AdvanceTo(new SequencePosition()))); + Assert.Same(ex, + Assert.Throws(() => requestPipe.AdvanceTo(new SequencePosition(), new SequencePosition()))); + Assert.Same(ex, + Assert.Throws(() => requestPipe.CancelPendingRead())); + Assert.Same(ex, + Assert.Throws(() => requestPipe.TryRead(out var res))); + Assert.Same(ex, + Assert.Throws(() => requestPipe.Complete())); + Assert.Same(ex, + Assert.Throws(() => requestPipe.OnWriterCompleted(null, null))); + } + + [Fact] + public async Task RequestPipeThrowsObjectDisposedExceptionAfterStop() + { + var bodyControl = new BodyControl(Mock.Of(), Mock.Of()); + + var (_, response, requestPipe, responsePipe) = bodyControl.Start(new MockMessageBody()); + + bodyControl.Stop(); + + Assert.Throws(() => requestPipe.AdvanceTo(new SequencePosition())); + Assert.Throws(() => requestPipe.AdvanceTo(new SequencePosition(), new SequencePosition())); + Assert.Throws(() => requestPipe.CancelPendingRead()); + Assert.Throws(() => requestPipe.TryRead(out var res)); + Assert.Throws(() => requestPipe.Complete()); + Assert.Throws(() => requestPipe.OnWriterCompleted(null, null)); + await Assert.ThrowsAsync(async () => await requestPipe.ReadAsync()); + } + + [Fact] + public async Task ResponsePipeThrowsObjectDisposedExceptionAfterStop() + { + var bodyControl = new BodyControl(Mock.Of(), Mock.Of()); + + var (_, response, requestPipe, responsePipe) = bodyControl.Start(new MockMessageBody()); + + bodyControl.Stop(); + + Assert.Throws(() => responsePipe.Advance(1)); + Assert.Throws(() => responsePipe.CancelPendingFlush()); + Assert.Throws(() => responsePipe.GetMemory()); + Assert.Throws(() => responsePipe.GetSpan()); + Assert.Throws(() => responsePipe.Complete()); + Assert.Throws(() => responsePipe.OnReaderCompleted(null, null)); + await Assert.ThrowsAsync(async () => await responsePipe.WriteAsync(new Memory())); + await Assert.ThrowsAsync(async () => await responsePipe.FlushAsync()); + } + + private class MockMessageBody : MessageBody { public MockMessageBody(bool upgradeable = false) diff --git a/src/Servers/Kestrel/Core/test/HttpRequestPipeReaderTests.cs b/src/Servers/Kestrel/Core/test/HttpRequestPipeReaderTests.cs new file mode 100644 index 000000000000..abda6aaff120 --- /dev/null +++ b/src/Servers/Kestrel/Core/test/HttpRequestPipeReaderTests.cs @@ -0,0 +1,10 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + + +namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests +{ + public class HttpRequestPipeReaderTests + { + } +} diff --git a/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs b/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs index 08c8788c8b93..5eaaf242b2c3 100644 --- a/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs +++ b/src/Servers/Kestrel/Core/test/HttpRequestStreamTests.cs @@ -111,8 +111,9 @@ public async Task SynchronousReadsThrowIfDisallowedByIHttpBodyControlFeature() var mockMessageBody = new Mock(null, null); mockMessageBody.Setup(m => m.ReadAsync(CancellationToken.None)).Returns(new ValueTask(new ReadResult(default, isCanceled: false, isCompleted: true))); - var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); - stream.StartAcceptingReads(mockMessageBody.Object); + var pipeReader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(mockBodyControl.Object, pipeReader); + pipeReader.StartAcceptingReads(mockMessageBody.Object); Assert.Equal(0, await stream.ReadAsync(new byte[1], 0, 1)); @@ -129,19 +130,23 @@ public async Task SynchronousReadsThrowIfDisallowedByIHttpBodyControlFeature() [Fact] public async Task AbortCausesReadToCancel() { - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(null); - stream.Abort(); + var pipeReader = new HttpRequestPipeReader(); + + var stream = new HttpRequestStream(Mock.Of(), pipeReader); + pipeReader.StartAcceptingReads(null); + pipeReader.Abort(); await Assert.ThrowsAsync(() => stream.ReadAsync(new byte[1], 0, 1)); } [Fact] public async Task AbortWithErrorCausesReadToCancel() { - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(null); + var pipeReader = new HttpRequestPipeReader(); + + var stream = new HttpRequestStream(Mock.Of(), pipeReader); + pipeReader.StartAcceptingReads(null); var error = new Exception(); - stream.Abort(error); + pipeReader.Abort(error); var exception = await Assert.ThrowsAsync(() => stream.ReadAsync(new byte[1], 0, 1)); Assert.Same(error, exception); } @@ -149,9 +154,10 @@ public async Task AbortWithErrorCausesReadToCancel() [Fact] public async Task StopAcceptingReadsCausesReadToThrowObjectDisposedException() { - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(null); - stream.StopAcceptingReads(); + var pipeReader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), pipeReader); + pipeReader.StartAcceptingReads(null); + pipeReader.StopAcceptingReads(); // Validation for ReadAsync occurs in an async method in ReadOnlyPipeStream. await Assert.ThrowsAsync(async () => { await stream.ReadAsync(new byte[1], 0, 1); }); @@ -160,19 +166,21 @@ public async Task StopAcceptingReadsCausesReadToThrowObjectDisposedException() [Fact] public async Task AbortCausesCopyToAsyncToCancel() { - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(null); - stream.Abort(); + var pipeReader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), pipeReader); + pipeReader.StartAcceptingReads(null); + pipeReader.Abort(); await Assert.ThrowsAsync(() => stream.CopyToAsync(Mock.Of())); } [Fact] public async Task AbortWithErrorCausesCopyToAsyncToCancel() { - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(null); + var pipeReader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), pipeReader); + pipeReader.StartAcceptingReads(null); var error = new Exception(); - stream.Abort(error); + pipeReader.Abort(error); var exception = await Assert.ThrowsAsync(() => stream.CopyToAsync(Mock.Of())); Assert.Same(error, exception); } @@ -180,9 +188,10 @@ public async Task AbortWithErrorCausesCopyToAsyncToCancel() [Fact] public async Task StopAcceptingReadsCausesCopyToAsyncToThrowObjectDisposedException() { - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(null); - stream.StopAcceptingReads(); + var pipeReader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), pipeReader); + pipeReader.StartAcceptingReads(null); + pipeReader.StopAcceptingReads(); // Validation for CopyToAsync occurs in an async method in ReadOnlyPipeStream. await Assert.ThrowsAsync(async () => { await stream.CopyToAsync(Mock.Of()); }); } @@ -190,16 +199,18 @@ public async Task StopAcceptingReadsCausesCopyToAsyncToThrowObjectDisposedExcept [Fact] public void NullDestinationCausesCopyToAsyncToThrowArgumentNullException() { - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(null); + var pipeReader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), pipeReader); + pipeReader.StartAcceptingReads(null); Assert.Throws(() => { stream.CopyToAsync(null); }); } [Fact] public void ZeroBufferSizeCausesCopyToAsyncToThrowArgumentException() { + var pipeReader = new HttpRequestPipeReader(); var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(null); + pipeReader.StartAcceptingReads(null); // This is technically a breaking change, to throw an ArgumentoutOfRangeException rather than an ArgumentException Assert.Throws(() => { stream.CopyToAsync(Mock.Of(), 0); }); } From e0a67d4456aa356a3c8d6d56e050c87d3dd64d04 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Wed, 13 Feb 2019 14:43:37 -0800 Subject: [PATCH 09/29] Start removing response pipe --- .../src/Internal/Http/Http1MessageBody.cs | 131 ++---------------- .../Core/src/Internal/Http/HttpProtocol.cs | 6 +- 2 files changed, 16 insertions(+), 121 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs index e70faea85ec9..5be0134e3769 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs @@ -18,129 +18,19 @@ public abstract class Http1MessageBody : MessageBody { private readonly Http1Connection _context; - private volatile bool _canceled; - private Task _pumpTask; - protected Http1MessageBody(Http1Connection context) : base(context, context.MinRequestBodyDataRate) { _context = context; } - private async Task PumpAsync() - { - Debug.Assert(!RequestUpgrade, "Upgraded connections should never use this code path!"); - - Exception error = null; - - try - { - var awaitable = _context.Input.ReadAsync(); - - if (!awaitable.IsCompleted) - { - TryProduceContinue(); - } - - while (true) - { - var result = await awaitable; - - if (_context.RequestTimedOut) - { - BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); - } - - var readableBuffer = result.Buffer; - var consumed = readableBuffer.Start; - var examined = readableBuffer.Start; - - try - { - if (_canceled) - { - break; - } - - if (!readableBuffer.IsEmpty) - { - bool done; - done = Read(readableBuffer, _context.RequestBodyPipe.Writer, out consumed, out examined); - - await _context.RequestBodyPipe.Writer.FlushAsync(); - - if (done) - { - break; - } - } - - // Read() will have already have greedily consumed the entire request body if able. - if (result.IsCompleted) - { - // OnInputOrOutputCompleted() is an idempotent method that closes the connection. Sometimes - // input completion is observed here before the Input.OnWriterCompleted() callback is fired, - // so we call OnInputOrOutputCompleted() now to prevent a race in our tests where a 400 - // response is written after observing the unexpected end of request content instead of just - // closing the connection without a response as expected. - _context.OnInputOrOutputCompleted(); - - BadHttpRequestException.Throw(RequestRejectionReason.UnexpectedEndOfRequestContent); - } - } - finally - { - _context.Input.AdvanceTo(consumed, examined); - } - - awaitable = _context.Input.ReadAsync(); - } - } - catch (Exception ex) - { - error = ex; - } - finally - { - _context.RequestBodyPipe.Writer.Complete(error); - } - } - - protected override Task OnStopAsync() - { - if (!_context.HasStartedConsumingRequestBody) - { - return Task.CompletedTask; - } - - // PumpTask catches all Exceptions internally. - if (_pumpTask.IsCompleted) - { - // At this point both the request body pipe reader and writer should be completed. - _context.RequestBodyPipe.Reset(); - return Task.CompletedTask; - } - - return StopAsyncAwaited(); - } - - private async Task StopAsyncAwaited() - { - _canceled = true; - _context.Input.CancelPendingRead(); - await _pumpTask; - - // At this point both the request body pipe reader and writer should be completed. - _context.RequestBodyPipe.Reset(); - } - protected override Task OnConsumeAsync() { try { - if (_context.RequestBodyPipe.Reader.TryRead(out var readResult)) + if (_context.RequestBodyPipeReader.TryRead(out var readResult)) { - _context.RequestBodyPipe.Reader.AdvanceTo(readResult.Buffer.End); + _context.RequestBodyPipeReader.AdvanceTo(readResult.Buffer.End); if (readResult.IsCompleted) { @@ -175,8 +65,8 @@ private async Task OnConsumeAsyncAwaited() ReadResult result; do { - result = await _context.RequestBodyPipe.Reader.ReadAsync(); - _context.RequestBodyPipe.Reader.AdvanceTo(result.Buffer.End); + result = await _context.RequestBodyPipeReader.ReadAsync(); + _context.RequestBodyPipeReader.AdvanceTo(result.Buffer.End); } while (!result.IsCompleted); } catch (BadHttpRequestException ex) @@ -208,10 +98,8 @@ protected void Copy(ReadOnlySequence readableBuffer, PipeWriter writableBu } } - protected override void OnReadStarted() - { - _pumpTask = PumpAsync(); - } + public abstract void Advance(long consumedBytes); + protected virtual bool Read(ReadOnlySequence readableBuffer, PipeWriter writableBuffer, out SequencePosition consumed, out SequencePosition examined) { @@ -242,6 +130,7 @@ public static MessageBody For( BadHttpRequestException.Throw(RequestRejectionReason.UpgradeRequestCannotHavePayload); } + context.RequestBodyPipeReader = new HttpRequestPipeReader(); return new ForUpgrade(context); } @@ -261,6 +150,7 @@ public static MessageBody For( BadHttpRequestException.Throw(RequestRejectionReason.FinalTransferCodingNotChunked, in transferEncoding); } + context.RequestBodyPipeReader = new HttpRequestPipeReader(); return new ForChunkedEncoding(keepAlive, context); } @@ -273,6 +163,7 @@ public static MessageBody For( return keepAlive ? MessageBody.ZeroContentLengthKeepAlive : MessageBody.ZeroContentLengthClose; } + context.RequestBodyPipeReader = new HttpRequestPipeReader(); return new ForContentLength(keepAlive, contentLength, context); } @@ -322,6 +213,10 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami _context.Input.AdvanceTo(consumed, examined); } + public override void Advance(long consumedBytes) + { + } + public override void Complete(Exception exception) { // Noop as we don't want to complete the connection pipe. diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index b855591ee800..c2b6aa286c23 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -74,7 +74,7 @@ public HttpProtocol(HttpConnectionContext context) public IHttpResponseControl HttpResponseControl { get; set; } - public Pipe RequestBodyPipe { get; protected set; } + public PipeReader RequestBodyPipeReader { get; set; } public ServiceContext ServiceContext => _context.ServiceContext; private IPEndPoint LocalEndPoint => _context.LocalEndPoint; @@ -649,10 +649,10 @@ private async Task ProcessRequests(IHttpApplication applicat if (HasStartedConsumingRequestBody) { - RequestBodyPipe.Reader.Complete(); + RequestBodyPipeReader.Complete(); // Wait for Http1MessageBody.PumpAsync() to call RequestBodyPipe.Writer.Complete(). - await messageBody.StopAsync(); + messageBody.Stop(); } } } From d3eb44a61d48bb80bf2ac5d3dcc46f6ce5cf0b1d Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 14 Feb 2019 08:34:15 -0800 Subject: [PATCH 10/29] Major refactor --- .../Http/ForChunkedEncodingMessageBody.cs | 558 ++++++++++++++++++ .../Http/ForContentLengthMessageBody.cs | 133 +++++ .../Internal/Http/ForUpgradeMessageBody.cs | 71 +++ .../Http/ForZeroContentLengthMessageBody.cs | 43 ++ .../Core/src/Internal/Http/Http1Connection.cs | 17 +- .../src/Internal/Http/Http1MessageBody.cs | 455 +------------- .../Core/src/Internal/Http/HttpProtocol.cs | 2 +- .../Core/src/Internal/Http/MessageBody.cs | 123 +--- .../src/Internal/Http2/Http2MessageBody.cs | 60 +- .../Core/src/Internal/Http2/Http2Stream.cs | 2 + .../Kestrel/Core/test/BodyControlTests.cs | 22 + .../Kestrel/Core/test/MessageBodyTests.cs | 132 +++-- 12 files changed, 981 insertions(+), 637 deletions(-) create mode 100644 src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs create mode 100644 src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs create mode 100644 src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs create mode 100644 src/Servers/Kestrel/Core/src/Internal/Http/ForZeroContentLengthMessageBody.cs diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs new file mode 100644 index 000000000000..66243db406a4 --- /dev/null +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs @@ -0,0 +1,558 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Buffers; +using System.Diagnostics; +using System.IO; +using System.IO.Pipelines; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; +using Microsoft.AspNetCore.Server.Kestrel.Transport.Abstractions.Internal; + +namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http +{ + /// + /// http://tools.ietf.org/html/rfc2616#section-3.6.1 + /// + public class ForChunkedEncoding : Http1MessageBody + { + // byte consts don't have a data type annotation so we pre-cast it + private const byte ByteCR = (byte)'\r'; + // "7FFFFFFF\r\n" is the largest chunk size that could be returned as an int. + private const int MaxChunkPrefixBytes = 10; + + private long _inputLength; + + private Mode _mode = Mode.Prefix; + private volatile bool _canceled; + private Task _pumpTask; + private Pipe _requestBodyPipe; + + public ForChunkedEncoding(bool keepAlive, Http1Connection context) + : base(context) + { + RequestKeepAlive = keepAlive; + + // For now, chunking will use the request body pipe + _requestBodyPipe = CreateRequestBodyPipe(context); + } + + private Pipe CreateRequestBodyPipe(Http1Connection context) + => new Pipe(new PipeOptions + ( + pool: context.MemoryPool, + readerScheduler: context.ServiceContext.Scheduler, + writerScheduler: PipeScheduler.Inline, + pauseWriterThreshold: 1, + resumeWriterThreshold: 1, + useSynchronizationContext: false, + minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize + )); + + private async Task PumpAsync() + { + Debug.Assert(!RequestUpgrade, "Upgraded connections should never use this code path!"); + + Exception error = null; + + try + { + var awaitable = _context.Input.ReadAsync(); + + if (!awaitable.IsCompleted) + { + TryProduceContinue(); + } + + while (true) + { + var result = await awaitable; + + if (_context.RequestTimedOut) + { + BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); + } + + var readableBuffer = result.Buffer; + var consumed = readableBuffer.Start; + var examined = readableBuffer.Start; + + try + { + if (_canceled) + { + break; + } + + if (!readableBuffer.IsEmpty) + { + bool done; + done = Read(readableBuffer, _requestBodyPipe.Writer, out consumed, out examined); + + await _requestBodyPipe.Writer.FlushAsync(); + + if (done) + { + break; + } + } + + // Read() will have already have greedily consumed the entire request body if able. + if (result.IsCompleted) + { + // OnInputOrOutputCompleted() is an idempotent method that closes the connection. Sometimes + // input completion is observed here before the Input.OnWriterCompleted() callback is fired, + // so we call OnInputOrOutputCompleted() now to prevent a race in our tests where a 400 + // response is written after observing the unexpected end of request content instead of just + // closing the connection without a response as expected. + _context.OnInputOrOutputCompleted(); + + BadHttpRequestException.Throw(RequestRejectionReason.UnexpectedEndOfRequestContent); + } + } + finally + { + _context.Input.AdvanceTo(consumed, examined); + } + + awaitable = _context.Input.ReadAsync(); + } + } + catch (Exception ex) + { + error = ex; + } + finally + { + _requestBodyPipe.Writer.Complete(error); + } + } + + protected override Task OnStopAsync() + { + if (!_context.HasStartedConsumingRequestBody) + { + return Task.CompletedTask; + } + + // PumpTask catches all Exceptions internally. + if (_pumpTask.IsCompleted) + { + // At this point both the request body pipe reader and writer should be completed. + _requestBodyPipe.Reset(); + return Task.CompletedTask; + } + + return StopAsyncAwaited(); + } + + private async Task StopAsyncAwaited() + { + _canceled = true; + _context.Input.CancelPendingRead(); + await _pumpTask; + + // At this point both the request body pipe reader and writer should be completed. + _requestBodyPipe.Reset(); + } + + protected override Task OnConsumeAsync() + { + try + { + if (_requestBodyPipe.Reader.TryRead(out var readResult)) + { + _requestBodyPipe.Reader.AdvanceTo(readResult.Buffer.End); + + if (readResult.IsCompleted) + { + return Task.CompletedTask; + } + } + } + catch (OperationCanceledException) + { + // TryRead can throw OperationCanceledException https://github.com/dotnet/corefx/issues/32029 + // because of buggy logic, this works around that for now + } + catch (BadHttpRequestException ex) + { + // At this point, the response has already been written, so this won't result in a 4XX response; + // however, we still need to stop the request processing loop and log. + _context.SetBadRequestState(ex); + return Task.CompletedTask; + } + + return OnConsumeAsyncAwaited(); + } + + private async Task OnConsumeAsyncAwaited() + { + Log.RequestBodyNotEntirelyRead(_context.ConnectionIdFeature, _context.TraceIdentifier); + + _context.TimeoutControl.SetTimeout(Constants.RequestBodyDrainTimeout.Ticks, TimeoutReason.RequestBodyDrain); + + try + { + ReadResult result; + do + { + result = await _requestBodyPipe.Reader.ReadAsync(); + _requestBodyPipe.Reader.AdvanceTo(result.Buffer.End); + } while (!result.IsCompleted); + } + catch (BadHttpRequestException ex) + { + _context.SetBadRequestState(ex); + } + catch (ConnectionAbortedException) + { + Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); + } + finally + { + _context.TimeoutControl.CancelTimeout(); + } + } + + protected void Copy(ReadOnlySequence readableBuffer, PipeWriter writableBuffer) + { + if (readableBuffer.IsSingleSegment) + { + writableBuffer.Write(readableBuffer.First.Span); + } + else + { + foreach (var memory in readableBuffer) + { + writableBuffer.Write(memory.Span); + } + } + } + + protected override void OnReadStarted() + { + _pumpTask = PumpAsync(); + } + + protected bool Read(ReadOnlySequence readableBuffer, PipeWriter writableBuffer, out SequencePosition consumed, out SequencePosition examined) + { + consumed = default; + examined = default; + + while (_mode < Mode.Trailer) + { + if (_mode == Mode.Prefix) + { + ParseChunkedPrefix(readableBuffer, out consumed, out examined); + + if (_mode == Mode.Prefix) + { + return false; + } + + readableBuffer = readableBuffer.Slice(consumed); + } + + if (_mode == Mode.Extension) + { + ParseExtension(readableBuffer, out consumed, out examined); + + if (_mode == Mode.Extension) + { + return false; + } + + readableBuffer = readableBuffer.Slice(consumed); + } + + if (_mode == Mode.Data) + { + ReadChunkedData(readableBuffer, writableBuffer, out consumed, out examined); + + if (_mode == Mode.Data) + { + return false; + } + + readableBuffer = readableBuffer.Slice(consumed); + } + + if (_mode == Mode.Suffix) + { + ParseChunkedSuffix(readableBuffer, out consumed, out examined); + + if (_mode == Mode.Suffix) + { + return false; + } + + readableBuffer = readableBuffer.Slice(consumed); + } + } + + // Chunks finished, parse trailers + if (_mode == Mode.Trailer) + { + ParseChunkedTrailer(readableBuffer, out consumed, out examined); + + if (_mode == Mode.Trailer) + { + return false; + } + + readableBuffer = readableBuffer.Slice(consumed); + } + + // _consumedBytes aren't tracked for trailer headers, since headers have separate limits. + if (_mode == Mode.TrailerHeaders) + { + if (_context.TakeMessageHeaders(readableBuffer, out consumed, out examined)) + { + _mode = Mode.Complete; + } + } + + return _mode == Mode.Complete; + } + + private void ParseChunkedPrefix(ReadOnlySequence buffer, out SequencePosition consumed, out SequencePosition examined) + { + consumed = buffer.Start; + examined = buffer.Start; + var reader = new SequenceReader(buffer); + + if (!reader.TryRead(out var ch1) || !reader.TryRead(out var ch2)) + { + examined = reader.Position; + return; + } + + var chunkSize = CalculateChunkSize(ch1, 0); + ch1 = ch2; + + while (reader.Consumed < MaxChunkPrefixBytes) + { + if (ch1 == ';') + { + consumed = reader.Position; + examined = reader.Position; + + AddAndCheckConsumedBytes(reader.Consumed); + _inputLength = chunkSize; + _mode = Mode.Extension; + return; + } + + if (!reader.TryRead(out ch2)) + { + examined = reader.Position; + return; + } + + if (ch1 == '\r' && ch2 == '\n') + { + consumed = reader.Position; + examined = reader.Position; + + AddAndCheckConsumedBytes(reader.Consumed); + _inputLength = chunkSize; + _mode = chunkSize > 0 ? Mode.Data : Mode.Trailer; + return; + } + + chunkSize = CalculateChunkSize(ch1, chunkSize); + ch1 = ch2; + } + + // At this point, 10 bytes have been consumed which is enough to parse the max value "7FFFFFFF\r\n". + BadHttpRequestException.Throw(RequestRejectionReason.BadChunkSizeData); + } + + private void ParseExtension(ReadOnlySequence buffer, out SequencePosition consumed, out SequencePosition examined) + { + // Chunk-extensions not currently parsed + // Just drain the data + consumed = buffer.Start; + examined = buffer.Start; + + do + { + SequencePosition? extensionCursorPosition = buffer.PositionOf(ByteCR); + if (extensionCursorPosition == null) + { + // End marker not found yet + consumed = buffer.End; + examined = buffer.End; + AddAndCheckConsumedBytes(buffer.Length); + return; + }; + + var extensionCursor = extensionCursorPosition.Value; + var charsToByteCRExclusive = buffer.Slice(0, extensionCursor).Length; + + var suffixBuffer = buffer.Slice(extensionCursor); + if (suffixBuffer.Length < 2) + { + consumed = extensionCursor; + examined = buffer.End; + AddAndCheckConsumedBytes(charsToByteCRExclusive); + return; + } + + suffixBuffer = suffixBuffer.Slice(0, 2); + var suffixSpan = suffixBuffer.ToSpan(); + + if (suffixSpan[1] == '\n') + { + // We consumed the \r\n at the end of the extension, so switch modes. + _mode = _inputLength > 0 ? Mode.Data : Mode.Trailer; + + consumed = suffixBuffer.End; + examined = suffixBuffer.End; + AddAndCheckConsumedBytes(charsToByteCRExclusive + 2); + } + else + { + // Don't consume suffixSpan[1] in case it is also a \r. + buffer = buffer.Slice(charsToByteCRExclusive + 1); + consumed = extensionCursor; + AddAndCheckConsumedBytes(charsToByteCRExclusive + 1); + } + } while (_mode == Mode.Extension); + } + + private void ReadChunkedData(ReadOnlySequence buffer, PipeWriter writableBuffer, out SequencePosition consumed, out SequencePosition examined) + { + var actual = Math.Min(buffer.Length, _inputLength); + consumed = buffer.GetPosition(actual); + examined = consumed; + + Copy(buffer.Slice(0, actual), writableBuffer); + + _inputLength -= actual; + AddAndCheckConsumedBytes(actual); + + if (_inputLength == 0) + { + _mode = Mode.Suffix; + } + } + + private void ParseChunkedSuffix(ReadOnlySequence buffer, out SequencePosition consumed, out SequencePosition examined) + { + consumed = buffer.Start; + examined = buffer.Start; + + if (buffer.Length < 2) + { + examined = buffer.End; + return; + } + + var suffixBuffer = buffer.Slice(0, 2); + var suffixSpan = suffixBuffer.ToSpan(); + if (suffixSpan[0] == '\r' && suffixSpan[1] == '\n') + { + consumed = suffixBuffer.End; + examined = suffixBuffer.End; + AddAndCheckConsumedBytes(2); + _mode = Mode.Prefix; + } + else + { + BadHttpRequestException.Throw(RequestRejectionReason.BadChunkSuffix); + } + } + + private void ParseChunkedTrailer(ReadOnlySequence buffer, out SequencePosition consumed, out SequencePosition examined) + { + consumed = buffer.Start; + examined = buffer.Start; + + if (buffer.Length < 2) + { + examined = buffer.End; + return; + } + + var trailerBuffer = buffer.Slice(0, 2); + var trailerSpan = trailerBuffer.ToSpan(); + + if (trailerSpan[0] == '\r' && trailerSpan[1] == '\n') + { + consumed = trailerBuffer.End; + examined = trailerBuffer.End; + AddAndCheckConsumedBytes(2); + _mode = Mode.Complete; + } + else + { + _mode = Mode.TrailerHeaders; + } + } + + private int CalculateChunkSize(int extraHexDigit, int currentParsedSize) + { + try + { + checked + { + if (extraHexDigit >= '0' && extraHexDigit <= '9') + { + return currentParsedSize * 0x10 + (extraHexDigit - '0'); + } + else if (extraHexDigit >= 'A' && extraHexDigit <= 'F') + { + return currentParsedSize * 0x10 + (extraHexDigit - ('A' - 10)); + } + else if (extraHexDigit >= 'a' && extraHexDigit <= 'f') + { + return currentParsedSize * 0x10 + (extraHexDigit - ('a' - 10)); + } + } + } + catch (OverflowException ex) + { + throw new IOException(CoreStrings.BadRequest_BadChunkSizeData, ex); + } + + BadHttpRequestException.Throw(RequestRejectionReason.BadChunkSizeData); + return -1; // can't happen, but compiler complains + } + + public override void AdvanceTo(SequencePosition consumed) + { + throw new NotImplementedException(); + } + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + throw new NotImplementedException(); + } + + public override bool TryRead(out ReadResult readResult) + { + throw new NotImplementedException(); + } + + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + private enum Mode + { + Prefix, + Extension, + Data, + Suffix, + Trailer, + TrailerHeaders, + Complete + }; + } +} diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs new file mode 100644 index 000000000000..b0557b717621 --- /dev/null +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs @@ -0,0 +1,133 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO.Pipelines; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http +{ + // Think this is close to good + public class ForContentLength : Http1MessageBody + { + private readonly long _contentLength; + private long _inputLength; + private ReadResult _previousReadResult; // TODO we can probably make this in Http1MessageBody or even MessageBody + + public ForContentLength(bool keepAlive, long contentLength, Http1Connection context) + : base(context) + { + RequestKeepAlive = keepAlive; + _contentLength = contentLength; + _inputLength = _contentLength; + } + + public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + if (_inputLength == 0) + { + throw new InvalidOperationException("Attempted to read from completed Content-Length request body."); + } + + TryStart(); + + while (true) + { + _previousReadResult = await StartTimingReadAsync(cancellationToken); + var readableBuffer = _previousReadResult.Buffer; + var readableBufferLength = readableBuffer.Length; + StopTimingRead(readableBufferLength); + + if (readableBufferLength != 0) + { + break; + } + + if (_previousReadResult.IsCompleted) + { + TryStop(); + break; + } + } + + // handle cases where we send more data than the content length + if (_previousReadResult.Buffer.Length > _inputLength) + { + _previousReadResult = new ReadResult(_previousReadResult.Buffer.Slice(0, _inputLength), _previousReadResult.IsCanceled, isCompleted: true); + + } + else if (_previousReadResult.Buffer.Length == _inputLength) + { + _previousReadResult = new ReadResult(_previousReadResult.Buffer, _previousReadResult.IsCanceled, isCompleted: true); + } + + return _previousReadResult; + } + + public override bool TryRead(out ReadResult readResult) + { + var res = _context.Input.TryRead(out _previousReadResult); + + if (_previousReadResult.Buffer.Length > _inputLength) + { + _previousReadResult = new ReadResult(_previousReadResult.Buffer.Slice(0, _inputLength), _previousReadResult.IsCanceled, isCompleted: true); + + } + else if (_previousReadResult.Buffer.Length == _inputLength) + { + _previousReadResult = new ReadResult(_previousReadResult.Buffer, _previousReadResult.IsCanceled, isCompleted: true); + } + + readResult = _previousReadResult; + return res; + } + + public override void AdvanceTo(SequencePosition consumed) + { + var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; + _inputLength -= dataLength; + _context.Input.AdvanceTo(consumed); + } + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; + _inputLength -= dataLength; + _context.Input.AdvanceTo(consumed, examined); + } + + protected override void OnReadStarting() + { + if (_contentLength > _context.MaxRequestBodySize) + { + BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTooLarge); + } + } + + private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) + { + var readAwaitable = _context.Input.ReadAsync(cancellationToken); + + if (!readAwaitable.IsCompleted && _timingEnabled) + { + _backpressure = true; + _context.TimeoutControl.StartTimingRead(); + } + + return readAwaitable; + } + + private void StopTimingRead(long bytesRead) + { + _context.TimeoutControl.BytesRead(bytesRead - _alreadyTimedBytes); + _alreadyTimedBytes = 0; + + if (_backpressure) + { + _backpressure = false; + _context.TimeoutControl.StopTimingRead(); + } + } + } +} diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs new file mode 100644 index 000000000000..91f464d9766e --- /dev/null +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO.Pipelines; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http +{ + /// + /// The upgrade stream uses the raw connection stream instead of going through the RequestBodyPipe. This + /// removes the redundant copy from the transport pipe to the body pipe. + /// + public class ForUpgrade : Http1MessageBody + { + public ForUpgrade(Http1Connection context) + : base(context) + { + RequestUpgrade = true; + } + + // This returns IsEmpty so we can avoid draining the body (since it's basically an endless stream) + public override bool IsEmpty => true; + + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + return _context.Input.ReadAsync(cancellationToken); + } + + public override bool TryRead(out ReadResult result) + { + return _context.Input.TryRead(out result); + } + + public override void AdvanceTo(SequencePosition consumed) + { + _context.Input.AdvanceTo(consumed); + } + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + _context.Input.AdvanceTo(consumed, examined); + } + + public override void Complete(Exception exception) + { + // Noop as we don't want to complete the connection pipe. + } + + public override void CancelPendingRead() + { + _context.Input.CancelPendingRead(); + } + + public override void OnWriterCompleted(Action callback, object state) + { + _context.Input.OnWriterCompleted(callback, state); + } + + public override Task ConsumeAsync() + { + return Task.CompletedTask; + } + + public override Task StopAsync() + { + return Task.CompletedTask; + } + } +} diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForZeroContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForZeroContentLengthMessageBody.cs new file mode 100644 index 000000000000..919421d1e48d --- /dev/null +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForZeroContentLengthMessageBody.cs @@ -0,0 +1,43 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.IO.Pipelines; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http +{ + public class ForZeroContentLength : MessageBody + { + public ForZeroContentLength(bool keepAlive) + : base(null, null) + { + RequestKeepAlive = keepAlive; + } + + public override bool IsEmpty => true; + + public override ValueTask ReadAsync(CancellationToken cancellationToken = default(CancellationToken)) => new ValueTask(new ReadResult(default, isCanceled: false, isCompleted: true)); + + public override Task ConsumeAsync() => Task.CompletedTask; + + public override Task StopAsync() => Task.CompletedTask; + + public override void AdvanceTo(SequencePosition consumed) { } + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { } + + public override bool TryRead(out ReadResult result) + { + result = new ReadResult(default, isCanceled: false, isCompleted: true); + return true; + } + + public override void OnWriterCompleted(Action callback, object state) { } + + public override void Complete(Exception ex) { } + + public override void CancelPendingRead() { } + } +} diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1Connection.cs index 2baffd56fd17..f418b475b7aa 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1Connection.cs @@ -44,8 +44,6 @@ public Http1Connection(HttpConnectionContext context) _keepAliveTicks = ServerOptions.Limits.KeepAliveTimeout.Ticks; _requestHeadersTimeoutTicks = ServerOptions.Limits.RequestHeadersTimeout.Ticks; - RequestBodyPipe = CreateRequestBodyPipe(); - _http1Output = new Http1OutputProducer( _context.Transport.Output, _context.ConnectionId, @@ -57,6 +55,7 @@ public Http1Connection(HttpConnectionContext context) Input = _context.Transport.Input; Output = _http1Output; + MemoryPool = _context.MemoryPool; } public PipeReader Input { get; } @@ -67,6 +66,8 @@ public Http1Connection(HttpConnectionContext context) public MinDataRate MinResponseDataRate { get; set; } + public MemoryPool MemoryPool { get; } + protected override void OnRequestProcessingEnded() { Input.Complete(); @@ -531,17 +532,5 @@ protected override bool TryParseRequest(ReadResult result, out bool endConnectio } void IRequestProcessor.Tick(DateTimeOffset now) { } - - private Pipe CreateRequestBodyPipe() - => new Pipe(new PipeOptions - ( - pool: _context.MemoryPool, - readerScheduler: ServiceContext.Scheduler, - writerScheduler: PipeScheduler.Inline, - pauseWriterThreshold: 1, - resumeWriterThreshold: 1, - useSynchronizationContext: false, - minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize - )); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs index 5be0134e3769..801a687187ac 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs @@ -2,21 +2,16 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Buffers; -using System.Diagnostics; -using System.IO; using System.IO.Pipelines; -using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Connections; -using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { public abstract class Http1MessageBody : MessageBody { - private readonly Http1Connection _context; + protected readonly Http1Connection _context; protected Http1MessageBody(Http1Connection context) : base(context, context.MinRequestBodyDataRate) @@ -83,29 +78,6 @@ private async Task OnConsumeAsyncAwaited() } } - protected void Copy(ReadOnlySequence readableBuffer, PipeWriter writableBuffer) - { - if (readableBuffer.IsSingleSegment) - { - writableBuffer.Write(readableBuffer.First.Span); - } - else - { - foreach (var memory in readableBuffer) - { - writableBuffer.Write(memory.Span); - } - } - } - - public abstract void Advance(long consumedBytes); - - - protected virtual bool Read(ReadOnlySequence readableBuffer, PipeWriter writableBuffer, out SequencePosition consumed, out SequencePosition examined) - { - throw new NotImplementedException(); - } - public static MessageBody For( HttpVersion httpVersion, HttpRequestHeaders headers, @@ -151,6 +123,8 @@ public static MessageBody For( } context.RequestBodyPipeReader = new HttpRequestPipeReader(); + // TODO may push more into the wrapper rather than just calling into the message body + // NBD for now. return new ForChunkedEncoding(keepAlive, context); } @@ -177,428 +151,5 @@ public static MessageBody For( return keepAlive ? MessageBody.ZeroContentLengthKeepAlive : MessageBody.ZeroContentLengthClose; } - - /// - /// The upgrade stream uses the raw connection stream instead of going through the RequestBodyPipe. This - /// removes the redundant copy from the transport pipe to the body pipe. - /// - private class ForUpgrade : Http1MessageBody - { - public ForUpgrade(Http1Connection context) - : base(context) - { - RequestUpgrade = true; - } - - // This returns IsEmpty so we can avoid draining the body (since it's basically an endless stream) - public override bool IsEmpty => true; - - public override ValueTask ReadAsync(CancellationToken cancellationToken = default) - { - return _context.Input.ReadAsync(cancellationToken); - } - - public override bool TryRead(out ReadResult result) - { - return _context.Input.TryRead(out result); - } - - public override void AdvanceTo(SequencePosition consumed) - { - _context.Input.AdvanceTo(consumed); - } - - public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) - { - _context.Input.AdvanceTo(consumed, examined); - } - - public override void Advance(long consumedBytes) - { - } - - public override void Complete(Exception exception) - { - // Noop as we don't want to complete the connection pipe. - } - - public override void CancelPendingRead() - { - _context.Input.CancelPendingRead(); - } - - public override void OnWriterCompleted(Action callback, object state) - { - _context.Input.OnWriterCompleted(callback, state); - } - - public override Task ConsumeAsync() - { - return Task.CompletedTask; - } - - public override Task StopAsync() - { - return Task.CompletedTask; - } - } - - private class ForContentLength : Http1MessageBody - { - private readonly long _contentLength; - private long _inputLength; - - public ForContentLength(bool keepAlive, long contentLength, Http1Connection context) - : base(context) - { - RequestKeepAlive = keepAlive; - _contentLength = contentLength; - _inputLength = _contentLength; - } - - protected override bool Read(ReadOnlySequence readableBuffer, PipeWriter writableBuffer, out SequencePosition consumed, out SequencePosition examined) - { - if (_inputLength == 0) - { - throw new InvalidOperationException("Attempted to read from completed Content-Length request body."); - } - - var actual = (int)Math.Min(readableBuffer.Length, _inputLength); - _inputLength -= actual; - - consumed = readableBuffer.GetPosition(actual); - examined = consumed; - - Copy(readableBuffer.Slice(0, actual), writableBuffer); - - return _inputLength == 0; - } - - protected override void OnReadStarting() - { - if (_contentLength > _context.MaxRequestBodySize) - { - BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTooLarge); - } - } - } - - /// - /// http://tools.ietf.org/html/rfc2616#section-3.6.1 - /// - private class ForChunkedEncoding : Http1MessageBody - { - // byte consts don't have a data type annotation so we pre-cast it - private const byte ByteCR = (byte)'\r'; - // "7FFFFFFF\r\n" is the largest chunk size that could be returned as an int. - private const int MaxChunkPrefixBytes = 10; - - private long _inputLength; - - private Mode _mode = Mode.Prefix; - - public ForChunkedEncoding(bool keepAlive, Http1Connection context) - : base(context) - { - RequestKeepAlive = keepAlive; - } - - protected override bool Read(ReadOnlySequence readableBuffer, PipeWriter writableBuffer, out SequencePosition consumed, out SequencePosition examined) - { - consumed = default(SequencePosition); - examined = default(SequencePosition); - - while (_mode < Mode.Trailer) - { - if (_mode == Mode.Prefix) - { - ParseChunkedPrefix(readableBuffer, out consumed, out examined); - - if (_mode == Mode.Prefix) - { - return false; - } - - readableBuffer = readableBuffer.Slice(consumed); - } - - if (_mode == Mode.Extension) - { - ParseExtension(readableBuffer, out consumed, out examined); - - if (_mode == Mode.Extension) - { - return false; - } - - readableBuffer = readableBuffer.Slice(consumed); - } - - if (_mode == Mode.Data) - { - ReadChunkedData(readableBuffer, writableBuffer, out consumed, out examined); - - if (_mode == Mode.Data) - { - return false; - } - - readableBuffer = readableBuffer.Slice(consumed); - } - - if (_mode == Mode.Suffix) - { - ParseChunkedSuffix(readableBuffer, out consumed, out examined); - - if (_mode == Mode.Suffix) - { - return false; - } - - readableBuffer = readableBuffer.Slice(consumed); - } - } - - // Chunks finished, parse trailers - if (_mode == Mode.Trailer) - { - ParseChunkedTrailer(readableBuffer, out consumed, out examined); - - if (_mode == Mode.Trailer) - { - return false; - } - - readableBuffer = readableBuffer.Slice(consumed); - } - - // _consumedBytes aren't tracked for trailer headers, since headers have separate limits. - if (_mode == Mode.TrailerHeaders) - { - if (_context.TakeMessageHeaders(readableBuffer, out consumed, out examined)) - { - _mode = Mode.Complete; - } - } - - return _mode == Mode.Complete; - } - - private void ParseChunkedPrefix(ReadOnlySequence buffer, out SequencePosition consumed, out SequencePosition examined) - { - consumed = buffer.Start; - examined = buffer.Start; - var reader = new SequenceReader(buffer); - - if (!reader.TryRead(out var ch1) || !reader.TryRead(out var ch2)) - { - examined = reader.Position; - return; - } - - var chunkSize = CalculateChunkSize(ch1, 0); - ch1 = ch2; - - while (reader.Consumed < MaxChunkPrefixBytes) - { - if (ch1 == ';') - { - consumed = reader.Position; - examined = reader.Position; - - AddAndCheckConsumedBytes(reader.Consumed); - _inputLength = chunkSize; - _mode = Mode.Extension; - return; - } - - if (!reader.TryRead(out ch2)) - { - examined = reader.Position; - return; - } - - if (ch1 == '\r' && ch2 == '\n') - { - consumed = reader.Position; - examined = reader.Position; - - AddAndCheckConsumedBytes(reader.Consumed); - _inputLength = chunkSize; - _mode = chunkSize > 0 ? Mode.Data : Mode.Trailer; - return; - } - - chunkSize = CalculateChunkSize(ch1, chunkSize); - ch1 = ch2; - } - - // At this point, 10 bytes have been consumed which is enough to parse the max value "7FFFFFFF\r\n". - BadHttpRequestException.Throw(RequestRejectionReason.BadChunkSizeData); - } - - private void ParseExtension(ReadOnlySequence buffer, out SequencePosition consumed, out SequencePosition examined) - { - // Chunk-extensions not currently parsed - // Just drain the data - consumed = buffer.Start; - examined = buffer.Start; - - do - { - SequencePosition? extensionCursorPosition = buffer.PositionOf(ByteCR); - if (extensionCursorPosition == null) - { - // End marker not found yet - consumed = buffer.End; - examined = buffer.End; - AddAndCheckConsumedBytes(buffer.Length); - return; - }; - - var extensionCursor = extensionCursorPosition.Value; - var charsToByteCRExclusive = buffer.Slice(0, extensionCursor).Length; - - var suffixBuffer = buffer.Slice(extensionCursor); - if (suffixBuffer.Length < 2) - { - consumed = extensionCursor; - examined = buffer.End; - AddAndCheckConsumedBytes(charsToByteCRExclusive); - return; - } - - suffixBuffer = suffixBuffer.Slice(0, 2); - var suffixSpan = suffixBuffer.ToSpan(); - - if (suffixSpan[1] == '\n') - { - // We consumed the \r\n at the end of the extension, so switch modes. - _mode = _inputLength > 0 ? Mode.Data : Mode.Trailer; - - consumed = suffixBuffer.End; - examined = suffixBuffer.End; - AddAndCheckConsumedBytes(charsToByteCRExclusive + 2); - } - else - { - // Don't consume suffixSpan[1] in case it is also a \r. - buffer = buffer.Slice(charsToByteCRExclusive + 1); - consumed = extensionCursor; - AddAndCheckConsumedBytes(charsToByteCRExclusive + 1); - } - } while (_mode == Mode.Extension); - } - - private void ReadChunkedData(ReadOnlySequence buffer, PipeWriter writableBuffer, out SequencePosition consumed, out SequencePosition examined) - { - var actual = Math.Min(buffer.Length, _inputLength); - consumed = buffer.GetPosition(actual); - examined = consumed; - - Copy(buffer.Slice(0, actual), writableBuffer); - - _inputLength -= actual; - AddAndCheckConsumedBytes(actual); - - if (_inputLength == 0) - { - _mode = Mode.Suffix; - } - } - - private void ParseChunkedSuffix(ReadOnlySequence buffer, out SequencePosition consumed, out SequencePosition examined) - { - consumed = buffer.Start; - examined = buffer.Start; - - if (buffer.Length < 2) - { - examined = buffer.End; - return; - } - - var suffixBuffer = buffer.Slice(0, 2); - var suffixSpan = suffixBuffer.ToSpan(); - if (suffixSpan[0] == '\r' && suffixSpan[1] == '\n') - { - consumed = suffixBuffer.End; - examined = suffixBuffer.End; - AddAndCheckConsumedBytes(2); - _mode = Mode.Prefix; - } - else - { - BadHttpRequestException.Throw(RequestRejectionReason.BadChunkSuffix); - } - } - - private void ParseChunkedTrailer(ReadOnlySequence buffer, out SequencePosition consumed, out SequencePosition examined) - { - consumed = buffer.Start; - examined = buffer.Start; - - if (buffer.Length < 2) - { - examined = buffer.End; - return; - } - - var trailerBuffer = buffer.Slice(0, 2); - var trailerSpan = trailerBuffer.ToSpan(); - - if (trailerSpan[0] == '\r' && trailerSpan[1] == '\n') - { - consumed = trailerBuffer.End; - examined = trailerBuffer.End; - AddAndCheckConsumedBytes(2); - _mode = Mode.Complete; - } - else - { - _mode = Mode.TrailerHeaders; - } - } - - private int CalculateChunkSize(int extraHexDigit, int currentParsedSize) - { - try - { - checked - { - if (extraHexDigit >= '0' && extraHexDigit <= '9') - { - return currentParsedSize * 0x10 + (extraHexDigit - '0'); - } - else if (extraHexDigit >= 'A' && extraHexDigit <= 'F') - { - return currentParsedSize * 0x10 + (extraHexDigit - ('A' - 10)); - } - else if (extraHexDigit >= 'a' && extraHexDigit <= 'f') - { - return currentParsedSize * 0x10 + (extraHexDigit - ('a' - 10)); - } - } - } - catch (OverflowException ex) - { - throw new IOException(CoreStrings.BadRequest_BadChunkSizeData, ex); - } - - BadHttpRequestException.Throw(RequestRejectionReason.BadChunkSizeData); - return -1; // can't happen, but compiler complains - } - - private enum Mode - { - Prefix, - Extension, - Data, - Suffix, - Trailer, - TrailerHeaders, - Complete - }; - } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index c2b6aa286c23..14fd1bb8c57e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -652,7 +652,7 @@ private async Task ProcessRequests(IHttpApplication applicat RequestBodyPipeReader.Complete(); // Wait for Http1MessageBody.PumpAsync() to call RequestBodyPipe.Writer.Complete(). - messageBody.Stop(); + await messageBody.StopAsync(); } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index 2eb565561773..0c7f3a4a6e76 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -2,8 +2,6 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Buffers; -using System.IO; using System.IO.Pipelines; using System.Threading; using System.Threading.Tasks; @@ -23,9 +21,9 @@ public abstract class MessageBody private long _consumedBytes; private bool _stopped; - private bool _timingEnabled; - private bool _backpressure; - private long _alreadyTimedBytes; + protected bool _timingEnabled; + protected bool _backpressure; + protected long _alreadyTimedBytes; protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) { @@ -45,68 +43,27 @@ protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) protected IKestrelTrace Log => _context.ServiceContext.Log; - private ReadResult _previousReadResult; + public abstract void AdvanceTo(SequencePosition consumed); - public virtual void AdvanceTo(SequencePosition consumed) - { - AdvanceTo(consumed, consumed); - } + public abstract void AdvanceTo(SequencePosition consumed, SequencePosition examined); - public virtual void AdvanceTo(SequencePosition consumed, SequencePosition examined) - { - var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; - _context.RequestBodyPipe.Reader.AdvanceTo(consumed, examined); - OnDataRead(dataLength); - } - - public virtual bool TryRead(out ReadResult readResult) - { - TryStart(); - - var hasReadResult = _context.RequestBodyPipe.Reader.TryRead(out _previousReadResult); - readResult = _previousReadResult; - - return hasReadResult; - } + public abstract bool TryRead(out ReadResult readResult); public virtual void OnWriterCompleted(Action callback, object state) { - _context.RequestBodyPipe.Reader.OnWriterCompleted(callback, state); + _context.RequestBodyPipeReader.OnWriterCompleted(callback, state); } public virtual void Complete(Exception exception) { - _context.RequestBodyPipe.Reader.Complete(exception); + _context.RequestBodyPipeReader.Complete(exception); } public virtual void CancelPendingRead() { - _context.RequestBodyPipe.Reader.CancelPendingRead(); } - public virtual async ValueTask ReadAsync(CancellationToken cancellationToken = default) - { - TryStart(); - - while (true) - { - _previousReadResult = await StartTimingReadAsync(cancellationToken); - var readableBuffer = _previousReadResult.Buffer; - var readableBufferLength = readableBuffer.Length; - StopTimingRead(readableBufferLength); - - if (readableBufferLength != 0) - { - return _previousReadResult; - } - - if (_previousReadResult.IsCompleted) - { - TryStop(); - return _previousReadResult; - } - } - } + public abstract ValueTask ReadAsync(CancellationToken cancellationToken = default); public virtual Task ConsumeAsync() { @@ -135,7 +92,7 @@ protected void TryProduceContinue() } } - private void TryStart() + protected void TryStart() { if (_context.HasStartedConsumingRequestBody) { @@ -159,7 +116,7 @@ private void TryStart() OnReadStarted(); } - private void TryStop() + protected void TryStop() { if (_stopped) { @@ -205,63 +162,5 @@ protected void AddAndCheckConsumedBytes(long consumedBytes) BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTooLarge); } } - - private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) - { - var readAwaitable = _context.RequestBodyPipe.Reader.ReadAsync(cancellationToken); - - if (!readAwaitable.IsCompleted && _timingEnabled) - { - _backpressure = true; - _context.TimeoutControl.StartTimingRead(); - } - - return readAwaitable; - } - - private void StopTimingRead(long bytesRead) - { - _context.TimeoutControl.BytesRead(bytesRead - _alreadyTimedBytes); - _alreadyTimedBytes = 0; - - if (_backpressure) - { - _backpressure = false; - _context.TimeoutControl.StopTimingRead(); - } - } - - private class ForZeroContentLength : MessageBody - { - public ForZeroContentLength(bool keepAlive) - : base(null, null) - { - RequestKeepAlive = keepAlive; - } - - public override bool IsEmpty => true; - - public override ValueTask ReadAsync(CancellationToken cancellationToken = default(CancellationToken)) => new ValueTask(new ReadResult(default, isCanceled: false, isCompleted: true)); - - public override Task ConsumeAsync() => Task.CompletedTask; - - public override Task StopAsync() => Task.CompletedTask; - - public override void AdvanceTo(SequencePosition consumed) { } - - public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { } - - public override bool TryRead(out ReadResult result) - { - result = new ReadResult(default, isCanceled: false, isCompleted: true); - return true; - } - - public override void OnWriterCompleted(Action callback, object state) { } - - public override void Complete(Exception ex) { } - - public override void CancelPendingRead() { } - } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs index 7427e98e14a5..dca2c157ae70 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs @@ -1,15 +1,18 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using System.IO.Pipelines; +using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; -using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 { public class Http2MessageBody : MessageBody { private readonly Http2Stream _context; + private ReadResult _previousReadResult; private Http2MessageBody(Http2Stream context, MinDataRate minRequestBodyDataRate) : base(context, minRequestBodyDataRate) @@ -51,5 +54,60 @@ public static MessageBody For(Http2Stream context, MinDataRate minRequestBodyDat return new Http2MessageBody(context, minRequestBodyDataRate); } + + public override void AdvanceTo(SequencePosition consumed) + { + AdvanceTo(consumed, consumed); + } + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; + _context.RequestBodyPipe.Reader.AdvanceTo(consumed, examined); + OnDataRead(dataLength); + } + + public override bool TryRead(out ReadResult readResult) + { + return _context.RequestBodyPipe.Reader.TryRead(out readResult); + } + + public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + _previousReadResult = await StartTimingReadAsync(cancellationToken); + StopTimingRead(_previousReadResult.Buffer.Length); + + if (_previousReadResult.IsCompleted) + { + TryStop(); + } + + return _previousReadResult; + } + + private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) + { + var readAwaitable = _context.RequestBodyPipe.Reader.ReadAsync(cancellationToken); + + if (!readAwaitable.IsCompleted && _timingEnabled) + { + _backpressure = true; + _context.TimeoutControl.StartTimingRead(); + } + + return readAwaitable; + } + + private void StopTimingRead(long bytesRead) + { + _context.TimeoutControl.BytesRead(bytesRead - _alreadyTimedBytes); + _alreadyTimedBytes = 0; + + if (_backpressure) + { + _backpressure = false; + _context.TimeoutControl.StopTimingRead(); + } + } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs index 661c2ae06895..e2a5b3b83d2e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs @@ -25,6 +25,8 @@ public abstract partial class Http2Stream : HttpProtocol, IThreadPoolWorkItem private readonly StreamInputFlowControl _inputFlowControl; private readonly StreamOutputFlowControl _outputFlowControl; + public Pipe RequestBodyPipe { get; } + internal long DrainExpirationTicks { get; set; } private StreamCompletionFlags _completionState; diff --git a/src/Servers/Kestrel/Core/test/BodyControlTests.cs b/src/Servers/Kestrel/Core/test/BodyControlTests.cs index 2ac070d56b04..1c33fb67cd6f 100644 --- a/src/Servers/Kestrel/Core/test/BodyControlTests.cs +++ b/src/Servers/Kestrel/Core/test/BodyControlTests.cs @@ -2,6 +2,8 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.IO.Pipelines; +using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; @@ -148,6 +150,26 @@ public MockMessageBody(bool upgradeable = false) { RequestUpgrade = upgradeable; } + + public override void AdvanceTo(SequencePosition consumed) + { + throw new NotImplementedException(); + } + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + throw new NotImplementedException(); + } + + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public override bool TryRead(out ReadResult readResult) + { + throw new NotImplementedException(); + } } } } diff --git a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs index e14f595d859b..e0a5d62f9c28 100644 --- a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs +++ b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs @@ -33,8 +33,9 @@ public async Task CanReadFromContentLength(HttpVersion httpVersion) var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); - var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(mockBodyControl.Object, reader); + reader.StartAcceptingReads(body); input.Add("Hello"); @@ -47,7 +48,7 @@ public async Task CanReadFromContentLength(HttpVersion httpVersion) count = stream.Read(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -60,8 +61,10 @@ public async Task CanReadAsyncFromContentLength(HttpVersion httpVersion) using (var input = new TestInput()) { var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("Hello"); @@ -74,7 +77,7 @@ public async Task CanReadAsyncFromContentLength(HttpVersion httpVersion) count = await stream.ReadAsync(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -87,8 +90,9 @@ public async Task CanReadFromChunkedEncoding() var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); - var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("5\r\nHello\r\n"); @@ -103,7 +107,7 @@ public async Task CanReadFromChunkedEncoding() count = stream.Read(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -114,8 +118,9 @@ public async Task CanReadAsyncFromChunkedEncoding() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("5\r\nHello\r\n"); @@ -130,7 +135,7 @@ public async Task CanReadAsyncFromChunkedEncoding() count = await stream.ReadAsync(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -141,8 +146,9 @@ public async Task ReadExitsGivenIncompleteChunkedExtension() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("5;\r\0"); @@ -156,7 +162,7 @@ public async Task ReadExitsGivenIncompleteChunkedExtension() Assert.Equal(5, await readTask.DefaultTimeout()); Assert.Equal(0, await stream.ReadAsync(buffer, 0, buffer.Length)); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -167,8 +173,9 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThanMaxInt() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("80000000\r\n"); @@ -178,7 +185,7 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThanMaxInt() Assert.IsType(ex.InnerException); Assert.Equal(CoreStrings.BadRequest_BadChunkSizeData, ex.Message); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -189,8 +196,9 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThan8Bytes() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("012345678\r"); @@ -200,7 +208,7 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThan8Bytes() Assert.Equal(CoreStrings.BadRequest_BadChunkSizeData, ex.Message); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -215,8 +223,9 @@ public async Task CanReadFromRemainingData(HttpVersion httpVersion) var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderConnection = "upgrade" }, input.Http1Connection); var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); - var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("Hello"); @@ -228,7 +237,7 @@ public async Task CanReadFromRemainingData(HttpVersion httpVersion) input.Fin(); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -241,8 +250,9 @@ public async Task CanReadAsyncFromRemainingData(HttpVersion httpVersion) using (var input = new TestInput()) { var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderConnection = "upgrade" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("Hello"); @@ -254,7 +264,7 @@ public async Task CanReadAsyncFromRemainingData(HttpVersion httpVersion) input.Fin(); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -269,8 +279,9 @@ public async Task ReadFromNoContentLengthReturnsZero(HttpVersion httpVersion) var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders(), input.Http1Connection); var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); - var stream = new HttpRequestStream(mockBodyControl.Object, new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("Hello"); @@ -289,8 +300,9 @@ public async Task ReadAsyncFromNoContentLengthReturnsZero(HttpVersion httpVersio using (var input = new TestInput()) { var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders(), input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("Hello"); @@ -307,8 +319,9 @@ public async Task CanHandleLargeBlocks() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http10, new HttpRequestHeaders { HeaderContentLength = "8197" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); // Input needs to be greater than 4032 bytes to allocate a block not backed by a slab. var largeInput = new string('a', 8192); @@ -325,7 +338,7 @@ public async Task CanHandleLargeBlocks() Assert.Equal(8197, requestArray.Length); AssertASCII(largeInput + "Hello", new ArraySegment(requestArray, 0, requestArray.Length)); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -391,7 +404,7 @@ public void ForThrowsWhenMethodRequiresLengthButNoContentLengthSetHttp10(HttpMet // Assert.Equal(0, await body.ReadAsync(new ArraySegment(new byte[1]))); - // input.Http1Connection.RequestBodyPipe.Reader.Complete(); + // input.Http1Connection.RequestBodyPipeReader.Complete(); // await body.StopAsync(); // } //} @@ -409,7 +422,7 @@ public async Task ConsumeAsyncConsumesAllRemainingInput() Assert.True((await body.ReadAsync()).IsCompleted); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -481,7 +494,7 @@ public async Task ConsumeAsyncConsumesAllRemainingInput() // Assert.Equal(2, writeCount); // // Don't call body.StopAsync() because PumpAsync() was never called. - // http1Connection.RequestBodyPipe.Reader.Complete(); + // http1Connection.RequestBodyPipeReader.Complete(); // } //} @@ -495,8 +508,9 @@ public async Task ConnectionUpgradeKeepAlive(string headerConnection) using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderConnection = headerConnection }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("Hello"); @@ -506,7 +520,7 @@ public async Task ConnectionUpgradeKeepAlive(string headerConnection) input.Fin(); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -522,8 +536,9 @@ public async Task UpgradeConnectionAcceptsContentLengthZero() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderConnection = headerConnection, ContentLength = 0 }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("Hello"); @@ -533,7 +548,7 @@ public async Task UpgradeConnectionAcceptsContentLengthZero() input.Fin(); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -544,8 +559,9 @@ public async Task PumpAsyncDoesNotReturnAfterCancelingInput() using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "2" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); // Add some input and consume it to ensure PumpAsync is running input.Add("a"); @@ -557,7 +573,7 @@ public async Task PumpAsyncDoesNotReturnAfterCancelingInput() input.Add("b"); Assert.Equal(1, await stream.ReadAsync(new byte[1], 0, 1)); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -585,7 +601,7 @@ public async Task ReadAsyncThrowsOnTimeout() var exception = await Assert.ThrowsAsync(async () => await body.ReadAsync()); Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -620,7 +636,7 @@ public async Task ConsumeAsyncCompletesAndDoesNotThrowOnTimeout() It.IsAny(), It.Is(ex => ex.Reason == RequestRejectionReason.RequestBodyTimeout))); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -649,7 +665,7 @@ public async Task ConsumeAsyncCompletesAndDoesNotThrowOnTimeout() // Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); // } - // input.Http1Connection.RequestBodyPipe.Reader.Complete(); + // input.Http1Connection.RequestBodyPipeReader.Complete(); // await body.StopAsync(); // } //} @@ -665,8 +681,9 @@ public async Task LogsWhenStartsReadingRequestBody() input.Http1Connection.TraceIdentifier = "RequestId"; var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "2" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); // Add some input and consume it to ensure PumpAsync is running input.Add("a"); @@ -676,7 +693,7 @@ public async Task LogsWhenStartsReadingRequestBody() input.Fin(); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -696,8 +713,9 @@ public async Task LogsWhenStopsReadingRequestBody() input.Http1Connection.TraceIdentifier = "RequestId"; var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "2" }, input.Http1Connection); - var stream = new HttpRequestStream(Mock.Of(), new HttpRequestPipeReader()); - stream.StartAcceptingReads(body); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); // Add some input and consume it to ensure PumpAsync is running input.Add("a"); @@ -705,7 +723,7 @@ public async Task LogsWhenStopsReadingRequestBody() input.Fin(); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); await logEvent.Task.DefaultTimeout(); @@ -768,7 +786,7 @@ public async Task OnlyEnforcesRequestBodyTimeoutAfterFirstRead() input.Add("a"); await readTask; - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -805,7 +823,7 @@ public async Task DoesNotEnforceRequestBodyTimeoutOnUpgradeRequests() mockTimeoutControl.Verify(timeoutControl => timeoutControl.StopTimingRead(), Times.Never); mockTimeoutControl.Verify(timeoutControl => timeoutControl.StartTimingRead(), Times.Never); - input.Http1Connection.RequestBodyPipe.Reader.Complete(); + input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } From bfba404e62869e87cc0027498a345a7cdae77e84 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 14 Feb 2019 12:11:21 -0800 Subject: [PATCH 11/29] Majority of refactor done. TODO tests --- src/Http/Http/src/ReadOnlyPipeStream.cs | 1 + .../Http/ForChunkedEncodingMessageBody.cs | 77 ++++++++++++- .../Http/ForContentLengthMessageBody.cs | 106 ++++++++++++------ .../src/Internal/Http/Http1MessageBody.cs | 11 +- .../Http/HttpProtocol.FeatureCollection.cs | 8 +- .../Core/src/Internal/Http/HttpProtocol.cs | 11 +- .../Internal/Http/HttpResponsePipeWriter.cs | 2 + .../Core/src/Internal/Http/MessageBody.cs | 7 +- .../src/Internal/Http2/Http2MessageBody.cs | 7 ++ .../Kestrel/Core/test/BodyControlTests.cs | 6 +- .../Core/test/HttpResponsePipeWriterTests.cs | 14 ++- .../Kestrel/Core/test/MessageBodyTests.cs | 70 +++++++----- .../test/FunctionalTests/RequestTests.cs | 1 + .../ChunkedRequestTests.cs | 9 +- 14 files changed, 230 insertions(+), 100 deletions(-) diff --git a/src/Http/Http/src/ReadOnlyPipeStream.cs b/src/Http/Http/src/ReadOnlyPipeStream.cs index 919745bc5dcc..c93ef8ccf011 100644 --- a/src/Http/Http/src/ReadOnlyPipeStream.cs +++ b/src/Http/Http/src/ReadOnlyPipeStream.cs @@ -171,6 +171,7 @@ private async ValueTask ReadAsyncInternal(Memory buffer, Cancellation var readableBuffer = result.Buffer; var readableBufferLength = readableBuffer.Length; + // TODO make this throw if the result is canceled var consumed = readableBuffer.End; var actual = 0; try diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs index 66243db406a4..b5949bc45486 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs @@ -38,6 +38,7 @@ public ForChunkedEncoding(bool keepAlive, Http1Connection context) // For now, chunking will use the request body pipe _requestBodyPipe = CreateRequestBodyPipe(context); + context.InternalRequestBodyPipeReader = _requestBodyPipe.Reader; } private Pipe CreateRequestBodyPipe(Http1Connection context) @@ -523,25 +524,91 @@ private int CalculateChunkSize(int extraHexDigit, int currentParsedSize) BadHttpRequestException.Throw(RequestRejectionReason.BadChunkSizeData); return -1; // can't happen, but compiler complains } + private ReadResult _previousReadResult; public override void AdvanceTo(SequencePosition consumed) { - throw new NotImplementedException(); + AdvanceTo(consumed, consumed); } public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { - throw new NotImplementedException(); + var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; + _context.InternalRequestBodyPipeReader.AdvanceTo(consumed, examined); + OnDataRead(dataLength); } public override bool TryRead(out ReadResult readResult) { - throw new NotImplementedException(); + TryStart(); + + var res =_requestBodyPipe.Reader.TryRead(out _previousReadResult); + readResult = _previousReadResult; + + if (_previousReadResult.IsCompleted) + { + TryStop(); + } + return res; + } + + public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + TryStart(); + + while (true) + { + _previousReadResult = await StartTimingReadAsync(cancellationToken); + var readableBuffer = _previousReadResult.Buffer; + var readableBufferLength = readableBuffer.Length; + StopTimingRead(readableBufferLength); + + if (readableBufferLength != 0) + { + break; + } + + if (_previousReadResult.IsCompleted) + { + TryStop(); + break; + } + } + + return _previousReadResult; + } + + private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) + { + // The only difference is which reader to use. Let's do the following. + // Make an internal reader that will always be used for whatever operation is needed here + // Keep external one the same always. + var readAwaitable = _context.InternalRequestBodyPipeReader.ReadAsync(cancellationToken); + + if (!readAwaitable.IsCompleted && _timingEnabled) + { + _backpressure = true; + _context.TimeoutControl.StartTimingRead(); + } + + return readAwaitable; + } + + private void StopTimingRead(long bytesRead) + { + _context.TimeoutControl.BytesRead(bytesRead - _alreadyTimedBytes); + _alreadyTimedBytes = 0; + + if (_backpressure) + { + _backpressure = false; + _context.TimeoutControl.StopTimingRead(); + } } - public override ValueTask ReadAsync(CancellationToken cancellationToken = default) + public override void Complete(Exception exception) { - throw new NotImplementedException(); + _context.InternalRequestBodyPipeReader.Complete(exception); } private enum Mode diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs index b0557b717621..3663dedbb75d 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Diagnostics; using System.IO.Pipelines; using System.Threading; using System.Threading.Tasks; @@ -13,7 +14,8 @@ public class ForContentLength : Http1MessageBody { private readonly long _contentLength; private long _inputLength; - private ReadResult _previousReadResult; // TODO we can probably make this in Http1MessageBody or even MessageBody + private ReadResult _readResult; // TODO we can probably make this in Http1MessageBody or even MessageBody + private bool _completed; public ForContentLength(bool keepAlive, long contentLength, Http1Connection context) : base(context) @@ -25,76 +27,104 @@ public ForContentLength(bool keepAlive, long contentLength, Http1Connection cont public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) { - if (_inputLength == 0) + if (_inputLength == 0 || _completed) { - throw new InvalidOperationException("Attempted to read from completed Content-Length request body."); + _readResult = new ReadResult(default, isCanceled: false, isCompleted: true); + return _readResult; } TryStart(); - while (true) + // This isn't great. The issue is that TryRead can get a canceled read result + // which is unknown to StartTimingReadAsync. + if (_context.RequestTimedOut) + { + BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); + } + + _readResult = await StartTimingReadAsync(cancellationToken); + + if (_context.RequestTimedOut) + { + Debug.Assert(_readResult.IsCanceled); + BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); + } + + var readableBuffer = _readResult.Buffer; + var readableBufferLength = readableBuffer.Length; + StopTimingRead(readableBufferLength); + + if (_readResult.IsCompleted) { - _previousReadResult = await StartTimingReadAsync(cancellationToken); - var readableBuffer = _previousReadResult.Buffer; - var readableBufferLength = readableBuffer.Length; - StopTimingRead(readableBufferLength); - - if (readableBufferLength != 0) - { - break; - } - - if (_previousReadResult.IsCompleted) - { - TryStop(); - break; - } + // OnInputOrOutputCompleted() is an idempotent method that closes the connection. Sometimes + // input completion is observed here before the Input.OnWriterCompleted() callback is fired, + // so we call OnInputOrOutputCompleted() now to prevent a race in our tests where a 400 + // response is written after observing the unexpected end of request content instead of just + // closing the connection without a response as expected. + _context.OnInputOrOutputCompleted(); + + BadHttpRequestException.Throw(RequestRejectionReason.UnexpectedEndOfRequestContent); } // handle cases where we send more data than the content length - if (_previousReadResult.Buffer.Length > _inputLength) + if (_readResult.Buffer.Length > _inputLength) { - _previousReadResult = new ReadResult(_previousReadResult.Buffer.Slice(0, _inputLength), _previousReadResult.IsCanceled, isCompleted: true); + _readResult = new ReadResult(_readResult.Buffer.Slice(0, _inputLength), _readResult.IsCanceled, isCompleted: true); } - else if (_previousReadResult.Buffer.Length == _inputLength) + else if (_readResult.Buffer.Length == _inputLength) + { + _readResult = new ReadResult(_readResult.Buffer, _readResult.IsCanceled, isCompleted: true); + } + + if (_readResult.IsCompleted) { - _previousReadResult = new ReadResult(_previousReadResult.Buffer, _previousReadResult.IsCanceled, isCompleted: true); + TryStop(); } - return _previousReadResult; + return _readResult; } public override bool TryRead(out ReadResult readResult) { - var res = _context.Input.TryRead(out _previousReadResult); + if (_inputLength == 0 || _completed) + { + // TODO should this muck with _readResult + readResult = default; + return false; + } + + var res = _context.Input.TryRead(out _readResult); - if (_previousReadResult.Buffer.Length > _inputLength) + if (_readResult.Buffer.Length > _inputLength) { - _previousReadResult = new ReadResult(_previousReadResult.Buffer.Slice(0, _inputLength), _previousReadResult.IsCanceled, isCompleted: true); + _readResult = new ReadResult(_readResult.Buffer.Slice(0, _inputLength), _readResult.IsCanceled, isCompleted: true); } - else if (_previousReadResult.Buffer.Length == _inputLength) + else if (_readResult.Buffer.Length == _inputLength) { - _previousReadResult = new ReadResult(_previousReadResult.Buffer, _previousReadResult.IsCanceled, isCompleted: true); + _readResult = new ReadResult(_readResult.Buffer, _readResult.IsCanceled, isCompleted: true); } - readResult = _previousReadResult; + readResult = _readResult; return res; } public override void AdvanceTo(SequencePosition consumed) { - var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; - _inputLength -= dataLength; - _context.Input.AdvanceTo(consumed); + AdvanceTo(consumed, consumed); } public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { - var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; + if (_inputLength == 0) + { + return; + } + var dataLength = _readResult.Buffer.Slice(_readResult.Buffer.Start, consumed).Length; _inputLength -= dataLength; _context.Input.AdvanceTo(consumed, examined); + OnDataRead(dataLength); } protected override void OnReadStarting() @@ -111,6 +141,8 @@ private ValueTask StartTimingReadAsync(CancellationToken cancellatio if (!readAwaitable.IsCompleted && _timingEnabled) { + TryProduceContinue(); + _backpressure = true; _context.TimeoutControl.StartTimingRead(); } @@ -129,5 +161,11 @@ private void StopTimingRead(long bytesRead) _context.TimeoutControl.StopTimingRead(); } } + + public override void Complete(Exception exception) + { + // Make this noop for now TODO + _completed = true; + } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs index 801a687187ac..8e71db4575c8 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs @@ -23,9 +23,9 @@ protected override Task OnConsumeAsync() { try { - if (_context.RequestBodyPipeReader.TryRead(out var readResult)) + if (TryRead(out var readResult)) { - _context.RequestBodyPipeReader.AdvanceTo(readResult.Buffer.End); + AdvanceTo(readResult.Buffer.End); if (readResult.IsCompleted) { @@ -60,8 +60,8 @@ private async Task OnConsumeAsyncAwaited() ReadResult result; do { - result = await _context.RequestBodyPipeReader.ReadAsync(); - _context.RequestBodyPipeReader.AdvanceTo(result.Buffer.End); + result = await ReadAsync(); + AdvanceTo(result.Buffer.End); } while (!result.IsCompleted); } catch (BadHttpRequestException ex) @@ -102,7 +102,6 @@ public static MessageBody For( BadHttpRequestException.Throw(RequestRejectionReason.UpgradeRequestCannotHavePayload); } - context.RequestBodyPipeReader = new HttpRequestPipeReader(); return new ForUpgrade(context); } @@ -122,7 +121,6 @@ public static MessageBody For( BadHttpRequestException.Throw(RequestRejectionReason.FinalTransferCodingNotChunked, in transferEncoding); } - context.RequestBodyPipeReader = new HttpRequestPipeReader(); // TODO may push more into the wrapper rather than just calling into the message body // NBD for now. return new ForChunkedEncoding(keepAlive, context); @@ -137,7 +135,6 @@ public static MessageBody For( return keepAlive ? MessageBody.ZeroContentLengthKeepAlive : MessageBody.ZeroContentLengthClose; } - context.RequestBodyPipeReader = new HttpRequestPipeReader(); return new ForContentLength(keepAlive, contentLength, context); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs index 1333c42a6d3d..9be0b8a6dc2d 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs @@ -106,7 +106,7 @@ Stream IHttpRequestFeature.Body minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize, minimumReadThreshold: KestrelMemoryPool.MinimumSegmentSize / 4, _context.MemoryPool)); - RequestPipeReader = requestPipeReader; + RequestBodyPipeReader = requestPipeReader; // The StreamPipeWrapper needs to be disposed as it hold onto blocks of memory if (_wrapperObjectsToDispose == null) @@ -121,12 +121,12 @@ PipeReader IRequestBodyPipeFeature.RequestBodyPipe { get { - return RequestPipeReader; + return RequestBodyPipeReader; } set { - RequestPipeReader = value; - RequestBody = new ReadOnlyPipeStream(RequestPipeReader); + RequestBodyPipeReader = value; + RequestBody = new ReadOnlyPipeStream(RequestBodyPipeReader); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index 14fd1bb8c57e..cbe928db09fc 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -62,6 +62,7 @@ public abstract partial class HttpProtocol : IDefaultHttpContextContainer, IHttp protected string _methodText = null; private string _scheme = null; + private List _wrapperObjectsToDispose; public HttpProtocol(HttpConnectionContext context) @@ -74,7 +75,7 @@ public HttpProtocol(HttpConnectionContext context) public IHttpResponseControl HttpResponseControl { get; set; } - public PipeReader RequestBodyPipeReader { get; set; } + public PipeReader InternalRequestBodyPipeReader { get; set; } public ServiceContext ServiceContext => _context.ServiceContext; private IPEndPoint LocalEndPoint => _context.LocalEndPoint; @@ -192,7 +193,7 @@ private void HttpVersionSetSlow(string value) public IHeaderDictionary RequestHeaders { get; set; } public Stream RequestBody { get; set; } - public PipeReader RequestPipeReader { get; set; } + public PipeReader RequestBodyPipeReader { get; set; } private int _statusCode; public int StatusCode @@ -300,7 +301,7 @@ public void InitializeBodyControl(MessageBody messageBody) bodyControl = new BodyControl(bodyControl: this, this); } - (RequestBody, ResponseBody, RequestPipeReader, ResponsePipeWriter) = bodyControl.Start(messageBody); + (RequestBody, ResponseBody, RequestBodyPipeReader, ResponsePipeWriter) = bodyControl.Start(messageBody); } public void StopBodies() => bodyControl.Stop(); @@ -649,7 +650,9 @@ private async Task ProcessRequests(IHttpApplication applicat if (HasStartedConsumingRequestBody) { - RequestBodyPipeReader.Complete(); + // This shouldn't be happening for http1 content length. + // Maybe we can go through the body? + InternalRequestBodyPipeReader?.Complete(); // Wait for Http1MessageBody.PumpAsync() to call RequestBodyPipe.Writer.Complete(). await messageBody.StopAsync(); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpResponsePipeWriter.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpResponsePipeWriter.cs index 0cc5a65fa0b2..2a406b067f86 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpResponsePipeWriter.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpResponsePipeWriter.cs @@ -34,6 +34,7 @@ public override void CancelPendingFlush() public override void Complete(Exception exception = null) { + ValidateState(); _pipeControl.Complete(exception); } @@ -57,6 +58,7 @@ public override Span GetSpan(int sizeHint = 0) public override void OnReaderCompleted(Action callback, object state) { + ValidateState(); throw new NotSupportedException(); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index 0c7f3a4a6e76..52539f8006b6 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -51,13 +51,10 @@ protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) public virtual void OnWriterCompleted(Action callback, object state) { - _context.RequestBodyPipeReader.OnWriterCompleted(callback, state); + _context.InternalRequestBodyPipeReader.OnWriterCompleted(callback, state); } - public virtual void Complete(Exception exception) - { - _context.RequestBodyPipeReader.Complete(exception); - } + public abstract void Complete(Exception exception); public virtual void CancelPendingRead() { diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs index dca2c157ae70..3351da27ceee 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs @@ -74,6 +74,8 @@ public override bool TryRead(out ReadResult readResult) public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) { + TryStart(); + _previousReadResult = await StartTimingReadAsync(cancellationToken); StopTimingRead(_previousReadResult.Buffer.Length); @@ -109,5 +111,10 @@ private void StopTimingRead(long bytesRead) _context.TimeoutControl.StopTimingRead(); } } + + public override void Complete(Exception exception) + { + _context.RequestBodyPipe.Reader.Complete(exception); + } } } diff --git a/src/Servers/Kestrel/Core/test/BodyControlTests.cs b/src/Servers/Kestrel/Core/test/BodyControlTests.cs index 1c33fb67cd6f..2e18586c06f9 100644 --- a/src/Servers/Kestrel/Core/test/BodyControlTests.cs +++ b/src/Servers/Kestrel/Core/test/BodyControlTests.cs @@ -142,7 +142,6 @@ public async Task ResponsePipeThrowsObjectDisposedExceptionAfterStop() await Assert.ThrowsAsync(async () => await responsePipe.FlushAsync()); } - private class MockMessageBody : MessageBody { public MockMessageBody(bool upgradeable = false) @@ -161,6 +160,11 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami throw new NotImplementedException(); } + public override void Complete(Exception exception) + { + throw new NotImplementedException(); + } + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) { throw new NotImplementedException(); diff --git a/src/Servers/Kestrel/Core/test/HttpResponsePipeWriterTests.cs b/src/Servers/Kestrel/Core/test/HttpResponsePipeWriterTests.cs index 9f57f1dffb32..41a2f743b75e 100644 --- a/src/Servers/Kestrel/Core/test/HttpResponsePipeWriterTests.cs +++ b/src/Servers/Kestrel/Core/test/HttpResponsePipeWriterTests.cs @@ -14,6 +14,7 @@ public class HttpResponsePipeWriterTests public void OnReaderCompletedThrowsNotSupported() { var pipeWriter = CreateHttpResponsePipeWriter(); + pipeWriter.StartAcceptingWrites(); Assert.Throws(() => pipeWriter.OnReaderCompleted((a, b) => { }, null)); } @@ -48,30 +49,31 @@ public void GetSpanAfterStopAcceptingWritesThrowsObjectDisposedException() } [Fact] - public void FlushAsyncAfterStopAcceptingWritesThrowsObjectDisposedException() + public void CompleteAfterStopAcceptingWritesThrowsObjectDisposedException() { var pipeWriter = CreateHttpResponsePipeWriter(); pipeWriter.StartAcceptingWrites(); pipeWriter.StopAcceptingWrites(); - var ex = Assert.Throws(() => { pipeWriter.FlushAsync(); }); + var ex = Assert.Throws(() => { pipeWriter.Complete(); }); Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message); } [Fact] - public void WriteAsyncAfterStopAcceptingWritesThrowsObjectDisposedException() + public void FlushAsyncAfterStopAcceptingWritesThrowsObjectDisposedException() { var pipeWriter = CreateHttpResponsePipeWriter(); pipeWriter.StartAcceptingWrites(); pipeWriter.StopAcceptingWrites(); - var ex = Assert.Throws(() => { pipeWriter.WriteAsync(new Memory()); }); + var ex = Assert.Throws(() => { pipeWriter.FlushAsync(); }); Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message); } [Fact] - public void CompleteCallsStopAcceptingWrites() + public void WriteAsyncAfterStopAcceptingWritesThrowsObjectDisposedException() { var pipeWriter = CreateHttpResponsePipeWriter(); - pipeWriter.Complete(); + pipeWriter.StartAcceptingWrites(); + pipeWriter.StopAcceptingWrites(); var ex = Assert.Throws(() => { pipeWriter.WriteAsync(new Memory()); }); Assert.Contains(CoreStrings.WritingToResponseBodyAfterResponseCompleted, ex.Message); } diff --git a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs index e0a5d62f9c28..a0a261b6ffea 100644 --- a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs +++ b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs @@ -3,18 +3,13 @@ using System; using System.IO; -using System.IO.Pipelines; -using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http.Features; -using Microsoft.AspNetCore.Server.Kestrel.Core.Internal; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; -using Microsoft.AspNetCore.Server.Kestrel.Transport.Abstractions.Internal; -using Microsoft.AspNetCore.Testing; using Moq; using Xunit; using Xunit.Sdk; @@ -48,7 +43,7 @@ public async Task CanReadFromContentLength(HttpVersion httpVersion) count = stream.Read(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -77,7 +72,7 @@ public async Task CanReadAsyncFromContentLength(HttpVersion httpVersion) count = await stream.ReadAsync(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -91,7 +86,7 @@ public async Task CanReadFromChunkedEncoding() var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); var reader = new HttpRequestPipeReader(); - var stream = new HttpRequestStream(Mock.Of(), reader); + var stream = new HttpRequestStream(mockBodyControl.Object, reader); reader.StartAcceptingReads(body); input.Add("5\r\nHello\r\n"); @@ -107,7 +102,7 @@ public async Task CanReadFromChunkedEncoding() count = stream.Read(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -135,7 +130,7 @@ public async Task CanReadAsyncFromChunkedEncoding() count = await stream.ReadAsync(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -160,9 +155,18 @@ public async Task ReadExitsGivenIncompleteChunkedExtension() input.Add("\r\r\r\nHello\r\n0\r\n\r\n"); Assert.Equal(5, await readTask.DefaultTimeout()); - Assert.Equal(0, await stream.ReadAsync(buffer, 0, buffer.Length)); + try + { + var res = await stream.ReadAsync(buffer, 0, buffer.Length); + Assert.Equal(0, res); + } + catch (Exception ex) + { + throw ex; + } + + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); - input.Http1Connection.RequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -185,7 +189,7 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThanMaxInt() Assert.IsType(ex.InnerException); Assert.Equal(CoreStrings.BadRequest_BadChunkSizeData, ex.Message); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -208,7 +212,7 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThan8Bytes() Assert.Equal(CoreStrings.BadRequest_BadChunkSizeData, ex.Message); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -224,7 +228,7 @@ public async Task CanReadFromRemainingData(HttpVersion httpVersion) var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); var reader = new HttpRequestPipeReader(); - var stream = new HttpRequestStream(Mock.Of(), reader); + var stream = new HttpRequestStream(mockBodyControl.Object, reader); reader.StartAcceptingReads(body); input.Add("Hello"); @@ -237,7 +241,7 @@ public async Task CanReadFromRemainingData(HttpVersion httpVersion) input.Fin(); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -264,7 +268,7 @@ public async Task CanReadAsyncFromRemainingData(HttpVersion httpVersion) input.Fin(); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -280,7 +284,7 @@ public async Task ReadFromNoContentLengthReturnsZero(HttpVersion httpVersion) var mockBodyControl = new Mock(); mockBodyControl.Setup(m => m.AllowSynchronousIO).Returns(true); var reader = new HttpRequestPipeReader(); - var stream = new HttpRequestStream(Mock.Of(), reader); + var stream = new HttpRequestStream(mockBodyControl.Object, reader); reader.StartAcceptingReads(body); input.Add("Hello"); @@ -288,6 +292,7 @@ public async Task ReadFromNoContentLengthReturnsZero(HttpVersion httpVersion) var buffer = new byte[1024]; Assert.Equal(0, stream.Read(buffer, 0, buffer.Length)); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -309,6 +314,7 @@ public async Task ReadAsyncFromNoContentLengthReturnsZero(HttpVersion httpVersio var buffer = new byte[1024]; Assert.Equal(0, await stream.ReadAsync(buffer, 0, buffer.Length)); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -338,7 +344,7 @@ public async Task CanHandleLargeBlocks() Assert.Equal(8197, requestArray.Length); AssertASCII(largeInput + "Hello", new ArraySegment(requestArray, 0, requestArray.Length)); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -404,7 +410,7 @@ public void ForThrowsWhenMethodRequiresLengthButNoContentLengthSetHttp10(HttpMet // Assert.Equal(0, await body.ReadAsync(new ArraySegment(new byte[1]))); - // input.Http1Connection.RequestBodyPipeReader.Complete(); + // input.Http1Connection.InternalRequestBodyPipeReader.Complete(); // await body.StopAsync(); // } //} @@ -420,9 +426,10 @@ public async Task ConsumeAsyncConsumesAllRemainingInput() await body.ConsumeAsync(); + // TODO should this throw an exception or not? Assert.True((await body.ReadAsync()).IsCompleted); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -507,6 +514,7 @@ public async Task ConnectionUpgradeKeepAlive(string headerConnection) { using (var input = new TestInput()) { + // note the http1connection request body pipe reader should be the same. var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderConnection = headerConnection }, input.Http1Connection); var reader = new HttpRequestPipeReader(); var stream = new HttpRequestStream(Mock.Of(), reader); @@ -520,7 +528,7 @@ public async Task ConnectionUpgradeKeepAlive(string headerConnection) input.Fin(); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -548,7 +556,7 @@ public async Task UpgradeConnectionAcceptsContentLengthZero() input.Fin(); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -573,7 +581,7 @@ public async Task PumpAsyncDoesNotReturnAfterCancelingInput() input.Add("b"); Assert.Equal(1, await stream.ReadAsync(new byte[1], 0, 1)); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -601,7 +609,7 @@ public async Task ReadAsyncThrowsOnTimeout() var exception = await Assert.ThrowsAsync(async () => await body.ReadAsync()); Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -636,7 +644,7 @@ public async Task ConsumeAsyncCompletesAndDoesNotThrowOnTimeout() It.IsAny(), It.Is(ex => ex.Reason == RequestRejectionReason.RequestBodyTimeout))); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -665,7 +673,7 @@ public async Task ConsumeAsyncCompletesAndDoesNotThrowOnTimeout() // Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); // } - // input.Http1Connection.RequestBodyPipeReader.Complete(); + // input.Http1Connection.InternalRequestBodyPipeReader.Complete(); // await body.StopAsync(); // } //} @@ -693,7 +701,7 @@ public async Task LogsWhenStartsReadingRequestBody() input.Fin(); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -723,7 +731,7 @@ public async Task LogsWhenStopsReadingRequestBody() input.Fin(); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); await logEvent.Task.DefaultTimeout(); @@ -786,7 +794,7 @@ public async Task OnlyEnforcesRequestBodyTimeoutAfterFirstRead() input.Add("a"); await readTask; - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -823,7 +831,7 @@ public async Task DoesNotEnforceRequestBodyTimeoutOnUpgradeRequests() mockTimeoutControl.Verify(timeoutControl => timeoutControl.StopTimingRead(), Times.Never); mockTimeoutControl.Verify(timeoutControl => timeoutControl.StartTimingRead(), Times.Never); - input.Http1Connection.RequestBodyPipeReader.Complete(); + input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } diff --git a/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs index 4132de863f77..06ffc72a8316 100644 --- a/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs @@ -621,6 +621,7 @@ await connection.Send( [MemberData(nameof(ConnectionAdapterData))] public async Task RequestsCanBeAbortedMidRead(ListenOptions listenOptions) { + // This needs a timeout. const int applicationAbortedConnectionId = 34; var testContext = new TestServiceContext(LoggerFactory); diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs index 697492093c1d..6a41eb24a983 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs @@ -321,11 +321,14 @@ public async Task TrailingHeadersAreParsedWithPipe() var request = httpContext.Request; var buffer = new byte[200]; - var result = await request.BodyPipe.ReadAsync(); - while (!result.IsCompleted) + while (true) { + var result = await request.BodyPipe.ReadAsync(); request.BodyPipe.AdvanceTo(result.Buffer.End); - result = await request.BodyPipe.ReadAsync(); + if (result.IsCompleted) + { + break; + } } if (requestsReceived < requestCount) From 3da4058191cf73603c4f8d8c49c43f5cd4a6fac3 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 14 Feb 2019 12:48:55 -0800 Subject: [PATCH 12/29] Remove InternalRequestBodyPipeReader --- .../Http/ForChunkedEncodingMessageBody.cs | 22 ++++++++-- .../Http/ForContentLengthMessageBody.cs | 12 ++++++ .../Http/HttpProtocol.FeatureCollection.cs | 8 ++-- .../Core/src/Internal/Http/HttpProtocol.cs | 9 ++--- .../Internal/Http/HttpRequestPipeReader.cs | 5 ++- .../Core/src/Internal/Http/MessageBody.cs | 9 +---- .../src/Internal/Http2/Http2MessageBody.cs | 31 ++++++++++++-- .../Core/src/Internal/Http2/Http2Stream.cs | 16 ++++---- .../Kestrel/Core/test/BodyControlTests.cs | 10 +++++ .../Kestrel/Core/test/MessageBodyTests.cs | 40 +++++++------------ 10 files changed, 103 insertions(+), 59 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs index b5949bc45486..0099ee70f9d8 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs @@ -38,7 +38,7 @@ public ForChunkedEncoding(bool keepAlive, Http1Connection context) // For now, chunking will use the request body pipe _requestBodyPipe = CreateRequestBodyPipe(context); - context.InternalRequestBodyPipeReader = _requestBodyPipe.Reader; + //context.InternalRequestBodyPipeReader = _requestBodyPipe.Reader; } private Pipe CreateRequestBodyPipe(Http1Connection context) @@ -139,6 +139,9 @@ protected override Task OnStopAsync() return Task.CompletedTask; } + // call complete here on the reader + _requestBodyPipe.Reader.Complete(); + // PumpTask catches all Exceptions internally. if (_pumpTask.IsCompleted) { @@ -147,6 +150,7 @@ protected override Task OnStopAsync() return Task.CompletedTask; } + // Should I call complete here? return StopAsyncAwaited(); } @@ -534,7 +538,7 @@ public override void AdvanceTo(SequencePosition consumed) public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; - _context.InternalRequestBodyPipeReader.AdvanceTo(consumed, examined); + _requestBodyPipe.Reader.AdvanceTo(consumed, examined); OnDataRead(dataLength); } @@ -583,7 +587,7 @@ private ValueTask StartTimingReadAsync(CancellationToken cancellatio // The only difference is which reader to use. Let's do the following. // Make an internal reader that will always be used for whatever operation is needed here // Keep external one the same always. - var readAwaitable = _context.InternalRequestBodyPipeReader.ReadAsync(cancellationToken); + var readAwaitable = _requestBodyPipe.Reader.ReadAsync(cancellationToken); if (!readAwaitable.IsCompleted && _timingEnabled) { @@ -608,7 +612,17 @@ private void StopTimingRead(long bytesRead) public override void Complete(Exception exception) { - _context.InternalRequestBodyPipeReader.Complete(exception); + _requestBodyPipe.Reader.Complete(exception); + } + + public override void OnWriterCompleted(Action callback, object state) + { + throw new NotImplementedException(); + } + + public override void CancelPendingRead() + { + throw new NotImplementedException(); } private enum Mode diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs index 3663dedbb75d..5d5658a3a565 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs @@ -167,5 +167,17 @@ public override void Complete(Exception exception) // Make this noop for now TODO _completed = true; } + + public override void OnWriterCompleted(Action callback, object state) + { + throw new NotImplementedException(); + } + + public override void CancelPendingRead() + { + throw new NotImplementedException(); + } + + // TODO maybe override OnStopAsync here. } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs index 9be0b8a6dc2d..751db24ac44c 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs @@ -106,7 +106,7 @@ Stream IHttpRequestFeature.Body minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize, minimumReadThreshold: KestrelMemoryPool.MinimumSegmentSize / 4, _context.MemoryPool)); - RequestBodyPipeReader = requestPipeReader; + RequestBodyPipe = requestPipeReader; // The StreamPipeWrapper needs to be disposed as it hold onto blocks of memory if (_wrapperObjectsToDispose == null) @@ -121,12 +121,12 @@ PipeReader IRequestBodyPipeFeature.RequestBodyPipe { get { - return RequestBodyPipeReader; + return RequestBodyPipe; } set { - RequestBodyPipeReader = value; - RequestBody = new ReadOnlyPipeStream(RequestBodyPipeReader); + RequestBodyPipe = value; + RequestBody = new ReadOnlyPipeStream(RequestBodyPipe); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index cbe928db09fc..0791bb3d3b36 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -75,7 +75,7 @@ public HttpProtocol(HttpConnectionContext context) public IHttpResponseControl HttpResponseControl { get; set; } - public PipeReader InternalRequestBodyPipeReader { get; set; } + //public PipeReader InternalRequestBodyPipeReader { get; set; } public ServiceContext ServiceContext => _context.ServiceContext; private IPEndPoint LocalEndPoint => _context.LocalEndPoint; @@ -193,7 +193,7 @@ private void HttpVersionSetSlow(string value) public IHeaderDictionary RequestHeaders { get; set; } public Stream RequestBody { get; set; } - public PipeReader RequestBodyPipeReader { get; set; } + public PipeReader RequestBodyPipe { get; set; } private int _statusCode; public int StatusCode @@ -301,7 +301,7 @@ public void InitializeBodyControl(MessageBody messageBody) bodyControl = new BodyControl(bodyControl: this, this); } - (RequestBody, ResponseBody, RequestBodyPipeReader, ResponsePipeWriter) = bodyControl.Start(messageBody); + (RequestBody, ResponseBody, RequestBodyPipe, ResponsePipeWriter) = bodyControl.Start(messageBody); } public void StopBodies() => bodyControl.Stop(); @@ -652,7 +652,7 @@ private async Task ProcessRequests(IHttpApplication applicat { // This shouldn't be happening for http1 content length. // Maybe we can go through the body? - InternalRequestBodyPipeReader?.Complete(); + //InternalRequestBodyPipeReader?.Complete(); // Wait for Http1MessageBody.PumpAsync() to call RequestBodyPipe.Writer.Complete(). await messageBody.StopAsync(); @@ -695,7 +695,6 @@ protected Task FireOnStarting() { return FireOnStartingMayAwait(onStarting); } - } private Task FireOnStartingMayAwait(Stack, object>> onStarting) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs index b6bd80935412..b7bce1a005c6 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpRequestPipeReader.cs @@ -2,16 +2,17 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.IO; using System.IO.Pipelines; using System.Runtime.CompilerServices; using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; -using Microsoft.AspNetCore.Connections; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { + /// + /// Default HttpRequest PipeReader implementation to be used by Kestrel. + /// public class HttpRequestPipeReader : PipeReader { private MessageBody _body; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index 52539f8006b6..7361af4bffd5 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -49,16 +49,11 @@ protected MessageBody(HttpProtocol context, MinDataRate minRequestBodyDataRate) public abstract bool TryRead(out ReadResult readResult); - public virtual void OnWriterCompleted(Action callback, object state) - { - _context.InternalRequestBodyPipeReader.OnWriterCompleted(callback, state); - } + public abstract void OnWriterCompleted(Action callback, object state); public abstract void Complete(Exception exception); - public virtual void CancelPendingRead() - { - } + public abstract void CancelPendingRead(); public abstract ValueTask ReadAsync(CancellationToken cancellationToken = default); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs index 3351da27ceee..ea871774ecb1 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs @@ -63,13 +63,13 @@ public override void AdvanceTo(SequencePosition consumed) public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; - _context.RequestBodyPipe.Reader.AdvanceTo(consumed, examined); + _context.RequestPipe.Reader.AdvanceTo(consumed, examined); OnDataRead(dataLength); } public override bool TryRead(out ReadResult readResult) { - return _context.RequestBodyPipe.Reader.TryRead(out readResult); + return _context.RequestPipe.Reader.TryRead(out readResult); } public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) @@ -89,7 +89,7 @@ public override async ValueTask ReadAsync(CancellationToken cancella private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) { - var readAwaitable = _context.RequestBodyPipe.Reader.ReadAsync(cancellationToken); + var readAwaitable = _context.RequestPipe.Reader.ReadAsync(cancellationToken); if (!readAwaitable.IsCompleted && _timingEnabled) { @@ -114,7 +114,30 @@ private void StopTimingRead(long bytesRead) public override void Complete(Exception exception) { - _context.RequestBodyPipe.Reader.Complete(exception); + _context.RequestPipe.Reader.Complete(exception); + } + + public override void OnWriterCompleted(Action callback, object state) + { + throw new NotImplementedException(); + } + + public override void CancelPendingRead() + { + throw new NotImplementedException(); + } + + protected override Task OnStopAsync() + { + if (!_context.HasStartedConsumingRequestBody) + { + return Task.CompletedTask; + } + + // call complete here on the reader + _context.RequestPipe.Reader.Complete(); + + return Task.CompletedTask; } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs index e2a5b3b83d2e..e0b9934db98e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs @@ -25,7 +25,7 @@ public abstract partial class Http2Stream : HttpProtocol, IThreadPoolWorkItem private readonly StreamInputFlowControl _inputFlowControl; private readonly StreamOutputFlowControl _outputFlowControl; - public Pipe RequestBodyPipe { get; } + public Pipe RequestPipe { get; } internal long DrainExpirationTicks { get; set; } @@ -57,7 +57,7 @@ public Http2Stream(Http2StreamContext context) this, context.ServiceContext.Log); - RequestBodyPipe = CreateRequestBodyPipe(context.ServerPeerSettings.InitialWindowSize); + RequestPipe = CreateRequestBodyPipe(context.ServerPeerSettings.InitialWindowSize); Output = _http2Output; } @@ -101,13 +101,13 @@ protected override void OnRequestProcessingEnded() { // Don't block on IO. This never faults. _ = _http2Output.WriteRstStreamAsync(Http2ErrorCode.NO_ERROR); - RequestBodyPipe.Writer.Complete(); + RequestPipe.Writer.Complete(); } } _http2Output.Dispose(); - RequestBodyPipe.Reader.Complete(); + RequestPipe.Reader.Complete(); // The app can no longer read any more of the request body, so return any bytes that weren't read to the // connection's flow-control window. @@ -366,9 +366,9 @@ public Task OnDataAsync(Http2Frame dataFrame, ReadOnlySequence payload) { foreach (var segment in dataPayload) { - RequestBodyPipe.Writer.Write(segment.Span); + RequestPipe.Writer.Write(segment.Span); } - var flushTask = RequestBodyPipe.Writer.FlushAsync(); + var flushTask = RequestPipe.Writer.FlushAsync(); // It shouldn't be possible for the RequestBodyPipe to fill up an return an incomplete task if // _inputFlowControl.Advance() didn't throw. @@ -398,7 +398,7 @@ public void OnEndStreamReceived() } } - RequestBodyPipe.Writer.Complete(); + RequestPipe.Writer.Complete(); _inputFlowControl.StopWindowUpdates(); } @@ -472,7 +472,7 @@ private void AbortCore(Exception abortReason) // Unblock the request body. PoisonRequestBodyStream(abortReason); - RequestBodyPipe.Writer.Complete(abortReason); + RequestPipe.Writer.Complete(abortReason); _inputFlowControl.Abort(); } diff --git a/src/Servers/Kestrel/Core/test/BodyControlTests.cs b/src/Servers/Kestrel/Core/test/BodyControlTests.cs index 2e18586c06f9..397b92da856f 100644 --- a/src/Servers/Kestrel/Core/test/BodyControlTests.cs +++ b/src/Servers/Kestrel/Core/test/BodyControlTests.cs @@ -160,11 +160,21 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami throw new NotImplementedException(); } + public override void CancelPendingRead() + { + throw new NotImplementedException(); + } + public override void Complete(Exception exception) { throw new NotImplementedException(); } + public override void OnWriterCompleted(Action callback, object state) + { + throw new NotImplementedException(); + } + public override ValueTask ReadAsync(CancellationToken cancellationToken = default) { throw new NotImplementedException(); diff --git a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs index a0a261b6ffea..92a4ec8649e4 100644 --- a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs +++ b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs @@ -43,7 +43,6 @@ public async Task CanReadFromContentLength(HttpVersion httpVersion) count = stream.Read(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -72,7 +71,6 @@ public async Task CanReadAsyncFromContentLength(HttpVersion httpVersion) count = await stream.ReadAsync(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -102,7 +100,6 @@ public async Task CanReadFromChunkedEncoding() count = stream.Read(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -130,7 +127,6 @@ public async Task CanReadAsyncFromChunkedEncoding() count = await stream.ReadAsync(buffer, 0, buffer.Length); Assert.Equal(0, count); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -165,8 +161,6 @@ public async Task ReadExitsGivenIncompleteChunkedExtension() throw ex; } - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); - await body.StopAsync(); } } @@ -189,7 +183,6 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThanMaxInt() Assert.IsType(ex.InnerException); Assert.Equal(CoreStrings.BadRequest_BadChunkSizeData, ex.Message); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -212,7 +205,6 @@ public async Task ReadThrowsGivenChunkPrefixGreaterThan8Bytes() Assert.Equal(CoreStrings.BadRequest_BadChunkSizeData, ex.Message); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -241,7 +233,6 @@ public async Task CanReadFromRemainingData(HttpVersion httpVersion) input.Fin(); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -268,7 +259,6 @@ public async Task CanReadAsyncFromRemainingData(HttpVersion httpVersion) input.Fin(); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); await body.StopAsync(); } } @@ -292,7 +282,7 @@ public async Task ReadFromNoContentLengthReturnsZero(HttpVersion httpVersion) var buffer = new byte[1024]; Assert.Equal(0, stream.Read(buffer, 0, buffer.Length)); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -314,7 +304,7 @@ public async Task ReadAsyncFromNoContentLengthReturnsZero(HttpVersion httpVersio var buffer = new byte[1024]; Assert.Equal(0, await stream.ReadAsync(buffer, 0, buffer.Length)); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -344,7 +334,7 @@ public async Task CanHandleLargeBlocks() Assert.Equal(8197, requestArray.Length); AssertASCII(largeInput + "Hello", new ArraySegment(requestArray, 0, requestArray.Length)); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -410,7 +400,7 @@ public void ForThrowsWhenMethodRequiresLengthButNoContentLengthSetHttp10(HttpMet // Assert.Equal(0, await body.ReadAsync(new ArraySegment(new byte[1]))); - // input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + // // await body.StopAsync(); // } //} @@ -429,7 +419,7 @@ public async Task ConsumeAsyncConsumesAllRemainingInput() // TODO should this throw an exception or not? Assert.True((await body.ReadAsync()).IsCompleted); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -528,7 +518,7 @@ public async Task ConnectionUpgradeKeepAlive(string headerConnection) input.Fin(); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -556,7 +546,7 @@ public async Task UpgradeConnectionAcceptsContentLengthZero() input.Fin(); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -581,7 +571,7 @@ public async Task PumpAsyncDoesNotReturnAfterCancelingInput() input.Add("b"); Assert.Equal(1, await stream.ReadAsync(new byte[1], 0, 1)); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -609,7 +599,7 @@ public async Task ReadAsyncThrowsOnTimeout() var exception = await Assert.ThrowsAsync(async () => await body.ReadAsync()); Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -644,7 +634,7 @@ public async Task ConsumeAsyncCompletesAndDoesNotThrowOnTimeout() It.IsAny(), It.Is(ex => ex.Reason == RequestRejectionReason.RequestBodyTimeout))); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -673,7 +663,7 @@ public async Task ConsumeAsyncCompletesAndDoesNotThrowOnTimeout() // Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); // } - // input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + // // await body.StopAsync(); // } //} @@ -701,7 +691,7 @@ public async Task LogsWhenStartsReadingRequestBody() input.Fin(); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -731,7 +721,7 @@ public async Task LogsWhenStopsReadingRequestBody() input.Fin(); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); await logEvent.Task.DefaultTimeout(); @@ -794,7 +784,7 @@ public async Task OnlyEnforcesRequestBodyTimeoutAfterFirstRead() input.Add("a"); await readTask; - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } @@ -831,7 +821,7 @@ public async Task DoesNotEnforceRequestBodyTimeoutOnUpgradeRequests() mockTimeoutControl.Verify(timeoutControl => timeoutControl.StopTimingRead(), Times.Never); mockTimeoutControl.Verify(timeoutControl => timeoutControl.StartTimingRead(), Times.Never); - input.Http1Connection.InternalRequestBodyPipeReader.Complete(); + await body.StopAsync(); } } From 9886fc0cbccd72c075f43ca0d10c629f9b5ccf6f Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 14 Feb 2019 15:57:51 -0800 Subject: [PATCH 13/29] Fix cancel --- .../Http/ForChunkedEncodingMessageBody.cs | 17 ++ .../Http/ForContentLengthMessageBody.cs | 181 ++++++++--- .../Internal/Http/ForUpgradeMessageBody.cs | 1 + .../src/Internal/Http/Http1MessageBody.cs | 65 ---- .../Core/src/Internal/Http/HttpProtocol.cs | 7 +- .../src/Internal/Http2/Http2MessageBody.cs | 5 +- .../Kestrel/Core/test/MessageBodyTests.cs | 282 ++++++++++-------- .../ChunkedRequestTests.cs | 1 - .../InMemory.FunctionalTests/RequestTests.cs | 54 ++++ 9 files changed, 383 insertions(+), 230 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs index 0099ee70f9d8..0b29dbfd1645 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs @@ -190,6 +190,15 @@ protected override Task OnConsumeAsync() _context.SetBadRequestState(ex); return Task.CompletedTask; } + catch (InvalidOperationException ex) + { + var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); + _context.ReportApplicationError(connectionAbortedException); + + // Have to abort the connection because we can't finish draining the request + _context.StopProcessingNextRequest(); + return Task.CompletedTask; + } return OnConsumeAsyncAwaited(); } @@ -217,6 +226,14 @@ private async Task OnConsumeAsyncAwaited() { Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); } + catch (InvalidOperationException ex) + { + var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); + _context.ReportApplicationError(connectionAbortedException); + + // Have to abort the connection because we can't finish draining the request + _context.StopProcessingNextRequest(); + } finally { _context.TimeoutControl.CancelTimeout(); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs index 5d5658a3a565..cfb7ea61447f 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs @@ -6,6 +6,8 @@ using System.IO.Pipelines; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { @@ -16,6 +18,7 @@ public class ForContentLength : Http1MessageBody private long _inputLength; private ReadResult _readResult; // TODO we can probably make this in Http1MessageBody or even MessageBody private bool _completed; + private int _userCanceled; public ForContentLength(bool keepAlive, long contentLength, Http1Connection context) : base(context) @@ -35,35 +38,52 @@ public override async ValueTask ReadAsync(CancellationToken cancella TryStart(); - // This isn't great. The issue is that TryRead can get a canceled read result - // which is unknown to StartTimingReadAsync. - if (_context.RequestTimedOut) + while (true) { - BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); - } - - _readResult = await StartTimingReadAsync(cancellationToken); - - if (_context.RequestTimedOut) - { - Debug.Assert(_readResult.IsCanceled); - BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); - } - - var readableBuffer = _readResult.Buffer; - var readableBufferLength = readableBuffer.Length; - StopTimingRead(readableBufferLength); - - if (_readResult.IsCompleted) - { - // OnInputOrOutputCompleted() is an idempotent method that closes the connection. Sometimes - // input completion is observed here before the Input.OnWriterCompleted() callback is fired, - // so we call OnInputOrOutputCompleted() now to prevent a race in our tests where a 400 - // response is written after observing the unexpected end of request content instead of just - // closing the connection without a response as expected. - _context.OnInputOrOutputCompleted(); - - BadHttpRequestException.Throw(RequestRejectionReason.UnexpectedEndOfRequestContent); + // This isn't great. The issue is that TryRead can get a canceled read result + // which is unknown to StartTimingReadAsync. + if (_context.RequestTimedOut) + { + BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); + } + + _readResult = await StartTimingReadAsync(cancellationToken); + + if (_context.RequestTimedOut) + { + Debug.Assert(_readResult.IsCanceled); + BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); + } + + if (_readResult.IsCanceled) + { + if (Interlocked.CompareExchange(ref _userCanceled, 0, 1) == 1) + { + // Ignore the readResult if it wasn't by the user. + break; + } + } + + var readableBuffer = _readResult.Buffer; + var readableBufferLength = readableBuffer.Length; + StopTimingRead(readableBufferLength); + + if (_readResult.IsCompleted) + { + // OnInputOrOutputCompleted() is an idempotent method that closes the connection. Sometimes + // input completion is observed here before the Input.OnWriterCompleted() callback is fired, + // so we call OnInputOrOutputCompleted() now to prevent a race in our tests where a 400 + // response is written after observing the unexpected end of request content instead of just + // closing the connection without a response as expected. + _context.OnInputOrOutputCompleted(); + + BadHttpRequestException.Throw(RequestRejectionReason.UnexpectedEndOfRequestContent); + } + + if (readableBufferLength > 0) + { + break; + } } // handle cases where we send more data than the content length @@ -89,12 +109,13 @@ public override bool TryRead(out ReadResult readResult) { if (_inputLength == 0 || _completed) { - // TODO should this muck with _readResult - readResult = default; + readResult = new ReadResult(default, isCanceled: false, isCompleted: true); return false; } - var res = _context.Input.TryRead(out _readResult); + TryStart(); + + var boolResult = _context.Input.TryRead(out _readResult); if (_readResult.Buffer.Length > _inputLength) { @@ -107,7 +128,13 @@ public override bool TryRead(out ReadResult readResult) } readResult = _readResult; - return res; + + if (_readResult.IsCompleted) + { + TryStop(); + } + + return boolResult; } public override void AdvanceTo(SequencePosition consumed) @@ -121,6 +148,7 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami { return; } + var dataLength = _readResult.Buffer.Slice(_readResult.Buffer.Start, consumed).Length; _inputLength -= dataLength; _context.Input.AdvanceTo(consumed, examined); @@ -164,20 +192,101 @@ private void StopTimingRead(long bytesRead) public override void Complete(Exception exception) { - // Make this noop for now TODO + _context.ReportApplicationError(exception); _completed = true; } public override void OnWriterCompleted(Action callback, object state) { - throw new NotImplementedException(); + // TODO make this work with ContentLength. } public override void CancelPendingRead() { - throw new NotImplementedException(); + Interlocked.Exchange(ref _userCanceled, 1); + _context.Input.CancelPendingRead(); + } + + protected override Task OnStopAsync() + { + Complete(null); + return Task.CompletedTask; + } + + protected override Task OnConsumeAsync() + { + try + { + if (TryRead(out var readResult)) + { + AdvanceTo(readResult.Buffer.End); + + if (readResult.IsCompleted) + { + return Task.CompletedTask; + } + } + } + catch (OperationCanceledException) + { + // TryRead can throw OperationCanceledException https://github.com/dotnet/corefx/issues/32029 + // because of buggy logic, this works around that for now + } + catch (BadHttpRequestException ex) + { + // At this point, the response has already been written, so this won't result in a 4XX response; + // however, we still need to stop the request processing loop and log. + _context.SetBadRequestState(ex); + return Task.CompletedTask; + } + catch (InvalidOperationException ex) + { + var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); + _context.ReportApplicationError(connectionAbortedException); + + // Have to abort the connection because we can't finish draining the request + _context.StopProcessingNextRequest(); + return Task.CompletedTask; + } + + return OnConsumeAsyncAwaited(); } - // TODO maybe override OnStopAsync here. + private async Task OnConsumeAsyncAwaited() + { + Log.RequestBodyNotEntirelyRead(_context.ConnectionIdFeature, _context.TraceIdentifier); + + _context.TimeoutControl.SetTimeout(Constants.RequestBodyDrainTimeout.Ticks, TimeoutReason.RequestBodyDrain); + + try + { + ReadResult result; + do + { + result = await ReadAsync(); + AdvanceTo(result.Buffer.End); + } while (!result.IsCompleted); + } + catch (BadHttpRequestException ex) + { + _context.SetBadRequestState(ex); + } + catch (ConnectionAbortedException) + { + Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); + } + catch (InvalidOperationException ex) + { + var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); + _context.ReportApplicationError(connectionAbortedException); + + // Have to abort the connection because we can't finish draining the request + _context.StopProcessingNextRequest(); + } + finally + { + _context.TimeoutControl.CancelTimeout(); + } + } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs index 91f464d9766e..da89a5c93379 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs @@ -46,6 +46,7 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami public override void Complete(Exception exception) { // Noop as we don't want to complete the connection pipe. + // actually we should complete this, just keep it internal } public override void CancelPendingRead() diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs index 8e71db4575c8..c0c492d0f9f3 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs @@ -1,12 +1,6 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. -using System; -using System.IO.Pipelines; -using System.Threading.Tasks; -using Microsoft.AspNetCore.Connections; -using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; - namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { public abstract class Http1MessageBody : MessageBody @@ -19,65 +13,6 @@ protected Http1MessageBody(Http1Connection context) _context = context; } - protected override Task OnConsumeAsync() - { - try - { - if (TryRead(out var readResult)) - { - AdvanceTo(readResult.Buffer.End); - - if (readResult.IsCompleted) - { - return Task.CompletedTask; - } - } - } - catch (OperationCanceledException) - { - // TryRead can throw OperationCanceledException https://github.com/dotnet/corefx/issues/32029 - // because of buggy logic, this works around that for now - } - catch (BadHttpRequestException ex) - { - // At this point, the response has already been written, so this won't result in a 4XX response; - // however, we still need to stop the request processing loop and log. - _context.SetBadRequestState(ex); - return Task.CompletedTask; - } - - return OnConsumeAsyncAwaited(); - } - - private async Task OnConsumeAsyncAwaited() - { - Log.RequestBodyNotEntirelyRead(_context.ConnectionIdFeature, _context.TraceIdentifier); - - _context.TimeoutControl.SetTimeout(Constants.RequestBodyDrainTimeout.Ticks, TimeoutReason.RequestBodyDrain); - - try - { - ReadResult result; - do - { - result = await ReadAsync(); - AdvanceTo(result.Buffer.End); - } while (!result.IsCompleted); - } - catch (BadHttpRequestException ex) - { - _context.SetBadRequestState(ex); - } - catch (ConnectionAbortedException) - { - Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); - } - finally - { - _context.TimeoutControl.CancelTimeout(); - } - } - public static MessageBody For( HttpVersion httpVersion, HttpRequestHeaders headers, diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index 0791bb3d3b36..3f93631e2750 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -650,11 +650,6 @@ private async Task ProcessRequests(IHttpApplication applicat if (HasStartedConsumingRequestBody) { - // This shouldn't be happening for http1 content length. - // Maybe we can go through the body? - //InternalRequestBodyPipeReader?.Complete(); - - // Wait for Http1MessageBody.PumpAsync() to call RequestBodyPipe.Writer.Complete(). await messageBody.StopAsync(); } } @@ -1249,7 +1244,7 @@ public void SetBadRequestState(BadHttpRequestException ex) _requestRejectedException = ex; } - protected void ReportApplicationError(Exception ex) + public void ReportApplicationError(Exception ex) { if (_applicationException == null) { diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs index ea871774ecb1..b0ca40fad1a6 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs @@ -119,12 +119,12 @@ public override void Complete(Exception exception) public override void OnWriterCompleted(Action callback, object state) { - throw new NotImplementedException(); + _context.RequestPipe.Reader.OnWriterCompleted(callback, state); } public override void CancelPendingRead() { - throw new NotImplementedException(); + _context.RequestPipe.Reader.CancelPendingRead(); } protected override Task OnStopAsync() @@ -134,7 +134,6 @@ protected override Task OnStopAsync() return Task.CompletedTask; } - // call complete here on the reader _context.RequestPipe.Reader.Complete(); return Task.CompletedTask; diff --git a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs index 92a4ec8649e4..722d950bd448 100644 --- a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs +++ b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs @@ -2,6 +2,7 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; +using System.Buffers; using System.IO; using System.Text; using System.Threading; @@ -47,6 +48,99 @@ public async Task CanReadFromContentLength(HttpVersion httpVersion) } } + [Theory] + [InlineData(HttpVersion.Http10)] + [InlineData(HttpVersion.Http11)] + public async Task CanReadFromContentLengthPipeApis(HttpVersion httpVersion) + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("Hello"); + + var readResult = await reader.ReadAsync(); + + Assert.Equal(5, readResult.Buffer.Length); + AssertASCII("Hello", readResult.Buffer); + reader.AdvanceTo(readResult.Buffer.End); + + readResult = await reader.ReadAsync(); + Assert.True(readResult.IsCompleted); + reader.AdvanceTo(readResult.Buffer.End); + + await body.StopAsync(); + } + } + + [Theory] + [InlineData(HttpVersion.Http10)] + [InlineData(HttpVersion.Http11)] + public async Task CanTryReadFromContentLengthPipeApis(HttpVersion httpVersion) + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("Hello"); + Assert.True(reader.TryRead(out var readResult)); + + Assert.Equal(5, readResult.Buffer.Length); + AssertASCII("Hello", readResult.Buffer); + reader.AdvanceTo(readResult.Buffer.End); + + reader.TryRead(out readResult); + Assert.True(readResult.IsCompleted); + reader.AdvanceTo(readResult.Buffer.End); + + await body.StopAsync(); + } + } + + [Theory] + [InlineData(HttpVersion.Http10)] + [InlineData(HttpVersion.Http11)] + public async Task ReadAsyncWithoutAdvanceFromContentLengthThrows(HttpVersion httpVersion) + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("Hello"); + var readResult = await reader.ReadAsync(); + + await Assert.ThrowsAsync(async () => await reader.ReadAsync()); + + await body.StopAsync(); + } + } + + [Theory] + [InlineData(HttpVersion.Http10)] + [InlineData(HttpVersion.Http11)] + public async Task TryReadWithoutAdvanceFromContentLengthThrows(HttpVersion httpVersion) + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(httpVersion, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("Hello"); + Assert.True(reader.TryRead(out var readResult)); + + Assert.Throws(() => reader.TryRead(out readResult)); + + await body.StopAsync(); + } + } + [Theory] [InlineData(HttpVersion.Http10)] [InlineData(HttpVersion.Http11)] @@ -384,26 +478,27 @@ public void ForThrowsWhenMethodRequiresLengthButNoContentLengthSetHttp10(HttpMet } } - //[Fact] - //public async Task CopyToAsyncDoesNotCompletePipeReader() - //{ - // using (var input = new TestInput()) - // { - // var body = Http1MessageBody.For(HttpVersion.Http10, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + [Fact] + public async Task CopyToAsyncDoesNotCompletePipeReader() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http10, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); - // input.Add("Hello"); + input.Add("Hello"); - // using (var ms = new MemoryStream()) - // { - // await body.CopyToAsync(ms); - // } + using (var ms = new MemoryStream()) + { + await stream.CopyToAsync(ms); + } - // Assert.Equal(0, await body.ReadAsync(new ArraySegment(new byte[1]))); + Assert.Equal(0, await stream.ReadAsync(new ArraySegment(new byte[1]))); - // - // await body.StopAsync(); - // } - //} + await body.StopAsync(); + } + } [Fact] public async Task ConsumeAsyncConsumesAllRemainingInput() @@ -416,84 +511,29 @@ public async Task ConsumeAsyncConsumesAllRemainingInput() await body.ConsumeAsync(); - // TODO should this throw an exception or not? Assert.True((await body.ReadAsync()).IsCompleted); - await body.StopAsync(); } } - //[Fact] - //public async Task CopyToAsyncDoesNotCopyBlocks() - //{ - // var writeCount = 0; - // var writeTcs = new TaskCompletionSource<(byte[], int, int)>(TaskCreationOptions.RunContinuationsAsynchronously); - // var mockDestination = new Mock { CallBase = true }; - - // mockDestination - // .Setup(m => m.WriteAsync(It.IsAny(), It.IsAny(), It.IsAny(), CancellationToken.None)) - // .Callback((byte[] buffer, int offset, int count, CancellationToken cancellationToken) => - // { - // writeTcs.SetResult((buffer, offset, count)); - // writeCount++; - // }) - // .Returns(Task.CompletedTask); - - // using (var memoryPool = KestrelMemoryPool.Create()) - // { - // var options = new PipeOptions(pool: memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false); - // var pair = DuplexPipe.CreateConnectionPair(options, options); - // var transport = pair.Transport; - // var http1ConnectionContext = new HttpConnectionContext - // { - // ServiceContext = new TestServiceContext(), - // ConnectionFeatures = new FeatureCollection(), - // Transport = transport, - // MemoryPool = memoryPool, - // TimeoutControl = Mock.Of() - // }; - // var http1Connection = new Http1Connection(http1ConnectionContext) - // { - // HasStartedConsumingRequestBody = true - // }; - - // var headers = new HttpRequestHeaders { HeaderContentLength = "12" }; - // var body = Http1MessageBody.For(HttpVersion.Http11, headers, http1Connection); - - // var copyToAsyncTask = body.CopyToAsync(mockDestination.Object); - - // var bytes = Encoding.ASCII.GetBytes("Hello "); - // var buffer = http1Connection.RequestBodyPipe.Writer.GetMemory(2048); - // Assert.True(MemoryMarshal.TryGetArray(buffer, out ArraySegment segment)); - // Buffer.BlockCopy(bytes, 0, segment.Array, segment.Offset, bytes.Length); - // http1Connection.RequestBodyPipe.Writer.Advance(bytes.Length); - // await http1Connection.RequestBodyPipe.Writer.FlushAsync(); - - // // Verify the block passed to Stream.WriteAsync() is the same one incoming data was written into. - // Assert.Equal((segment.Array, segment.Offset, bytes.Length), await writeTcs.Task); - - // // Verify the again when GetMemory returns the tail space of the same block. - // writeTcs = new TaskCompletionSource<(byte[], int, int)>(TaskCreationOptions.RunContinuationsAsynchronously); - // bytes = Encoding.ASCII.GetBytes("World!"); - // buffer = http1Connection.RequestBodyPipe.Writer.GetMemory(2048); - // Assert.True(MemoryMarshal.TryGetArray(buffer, out segment)); - // Buffer.BlockCopy(bytes, 0, segment.Array, segment.Offset, bytes.Length); - // http1Connection.RequestBodyPipe.Writer.Advance(bytes.Length); - // await http1Connection.RequestBodyPipe.Writer.FlushAsync(); - - // Assert.Equal((segment.Array, segment.Offset, bytes.Length), await writeTcs.Task); - - // http1Connection.RequestBodyPipe.Writer.Complete(); - - // await copyToAsyncTask; - - // Assert.Equal(2, writeCount); - - // // Don't call body.StopAsync() because PumpAsync() was never called. - // http1Connection.RequestBodyPipeReader.Complete(); - // } - //} + [Fact] + public async Task ConsumeAsyncConsumesAllRemainingInputAfterStartingTryReadWithoutAdvance() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http10, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + + input.Add("Hello"); + body.TryRead(out var readResult); + + await body.ConsumeAsync(); // This will throw because someone called try read without advance. + + Assert.True((await body.ReadAsync()).IsCompleted); + + await body.StopAsync(); + } + } [Theory] [InlineData("keep-alive, upgrade")] @@ -518,7 +558,6 @@ public async Task ConnectionUpgradeKeepAlive(string headerConnection) input.Fin(); - await body.StopAsync(); } } @@ -546,7 +585,6 @@ public async Task UpgradeConnectionAcceptsContentLengthZero() input.Fin(); - await body.StopAsync(); } } @@ -571,7 +609,6 @@ public async Task PumpAsyncDoesNotReturnAfterCancelingInput() input.Add("b"); Assert.Equal(1, await stream.ReadAsync(new byte[1], 0, 1)); - await body.StopAsync(); } } @@ -599,7 +636,6 @@ public async Task ReadAsyncThrowsOnTimeout() var exception = await Assert.ThrowsAsync(async () => await body.ReadAsync()); Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); - await body.StopAsync(); } } @@ -634,39 +670,39 @@ public async Task ConsumeAsyncCompletesAndDoesNotThrowOnTimeout() It.IsAny(), It.Is(ex => ex.Reason == RequestRejectionReason.RequestBodyTimeout))); - await body.StopAsync(); } } - //[Fact] - //public async Task CopyToAsyncThrowsOnTimeout() - //{ - // using (var input = new TestInput()) - // { - // var mockTimeoutControl = new Mock(); + [Fact] + public async Task CopyToAsyncThrowsOnTimeout() + { + using (var input = new TestInput()) + { + var mockTimeoutControl = new Mock(); - // input.Http1ConnectionContext.TimeoutControl = mockTimeoutControl.Object; + input.Http1ConnectionContext.TimeoutControl = mockTimeoutControl.Object; - // var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + var stream = new HttpRequestStream(Mock.Of(), reader); - // // Add some input and read it to start PumpAsync - // input.Add("a"); - // Assert.Equal(1, (await body.ReadAsync()).Buffer.Length); + // Add some input and read it to start PumpAsync + input.Add("a"); + Assert.Equal(1, (await body.ReadAsync()).Buffer.Length); - // // Time out on the next read - // input.Http1Connection.SendTimeoutResponse(); + // Time out on the next read + input.Http1Connection.SendTimeoutResponse(); - // using (var ms = new MemoryStream()) - // { - // var exception = await Assert.ThrowsAsync(() => body.CopyToAsync(ms)); - // Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); - // } + using (var ms = new MemoryStream()) + { + var exception = await Assert.ThrowsAsync(() => stream.CopyToAsync(ms)); + Assert.Equal(StatusCodes.Status408RequestTimeout, exception.StatusCode); + } - // - // await body.StopAsync(); - // } - //} + await body.StopAsync(); + } + } [Fact] public async Task LogsWhenStartsReadingRequestBody() @@ -691,7 +727,6 @@ public async Task LogsWhenStartsReadingRequestBody() input.Fin(); - await body.StopAsync(); } } @@ -721,7 +756,6 @@ public async Task LogsWhenStopsReadingRequestBody() input.Fin(); - await body.StopAsync(); await logEvent.Task.DefaultTimeout(); @@ -784,7 +818,6 @@ public async Task OnlyEnforcesRequestBodyTimeoutAfterFirstRead() input.Add("a"); await readTask; - await body.StopAsync(); } } @@ -821,7 +854,6 @@ public async Task DoesNotEnforceRequestBodyTimeoutOnUpgradeRequests() mockTimeoutControl.Verify(timeoutControl => timeoutControl.StopTimingRead(), Times.Never); mockTimeoutControl.Verify(timeoutControl => timeoutControl.StartTimingRead(), Times.Never); - await body.StopAsync(); } } @@ -837,6 +869,18 @@ private void AssertASCII(string expected, ArraySegment actual) } } + private void AssertASCII(string expected, ReadOnlySequence actual) + { + var arr = actual.ToArray(); + var encoding = Encoding.ASCII; + var bytes = encoding.GetBytes(expected); + Assert.Equal(bytes.Length, actual.Length); + for (var index = 0; index < bytes.Length; index++) + { + Assert.Equal(bytes[index], arr[index]); + } + } + private class ThrowOnWriteSynchronousStream : Stream { public override void Flush() diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs index 6a41eb24a983..d67a191a5769 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs @@ -817,4 +817,3 @@ await connection.SendAll( } } } - diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs index 1c6cc964e861..5430c32cf5d6 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs @@ -700,6 +700,60 @@ await connection.ReceiveEnd( } } + [Fact] + public async Task ContentLengthTryReadPipeReader() + { + var testContext = new TestServiceContext(LoggerFactory); + + using (var server = new TestServer(httpContext => + { + var res = httpContext.Request.BodyPipe.TryRead(out var readResult); + // This will hang if 0 content length is not assumed by the server + Assert.Equal(5, readResult.Buffer.Length); + httpContext.Request.BodyPipe.AdvanceTo(readResult.Buffer.End); + + return Task.CompletedTask; + }, testContext)) + { + using (var connection = server.CreateConnection()) + { + // Use Send instead of SendEnd to ensure the connection will remain open while + // the app runs and reads 0 bytes from the body nonetheless. This checks that + // https://github.com/aspnet/KestrelHttpServer/issues/1104 is not regressing. + await connection.SendAll( + "POST / HTTP/1.0", + "Host:", + "Content-Length: 5", + "", + "hello"); + await connection.ReceiveEnd( + "HTTP/1.1 200 OK", + "Connection: close", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 0", + "", + ""); + } + + using (var connection = server.CreateConnection()) + { + await connection.Send( + "GET / HTTP/1.0", + "Host:", + "", + ""); + await connection.ReceiveEnd( + "HTTP/1.1 200 OK", + "Connection: close", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 0", + "", + ""); + } + await server.StopAsync(); + } + } + [Fact] public async Task ConnectionClosesWhenFinReceivedBeforeRequestCompletes() { From aeaf6b10b9708154d93e0ade3f3958cbbf73b193 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 14 Feb 2019 18:15:58 -0800 Subject: [PATCH 14/29] bunch of tests --- .../Http/ForChunkedEncodingMessageBody.cs | 33 +- .../Http/ForContentLengthMessageBody.cs | 16 +- .../Internal/Http/ForUpgradeMessageBody.cs | 14 +- .../Core/src/Internal/Http/HttpProtocol.cs | 5 + .../Kestrel/Core/test/MessageBodyTests.cs | 383 ++++++++++++++++++ 5 files changed, 435 insertions(+), 16 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs index 0b29dbfd1645..4b8ed862b618 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs @@ -30,6 +30,7 @@ public class ForChunkedEncoding : Http1MessageBody private volatile bool _canceled; private Task _pumpTask; private Pipe _requestBodyPipe; + private int _userCanceled; public ForChunkedEncoding(bool keepAlive, Http1Connection context) : base(context) @@ -545,7 +546,7 @@ private int CalculateChunkSize(int extraHexDigit, int currentParsedSize) BadHttpRequestException.Throw(RequestRejectionReason.BadChunkSizeData); return -1; // can't happen, but compiler complains } - private ReadResult _previousReadResult; + private ReadResult _readResult; public override void AdvanceTo(SequencePosition consumed) { @@ -554,7 +555,7 @@ public override void AdvanceTo(SequencePosition consumed) public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { - var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; + var dataLength = _readResult.Buffer.Slice(_readResult.Buffer.Start, consumed).Length; _requestBodyPipe.Reader.AdvanceTo(consumed, examined); OnDataRead(dataLength); } @@ -563,10 +564,10 @@ public override bool TryRead(out ReadResult readResult) { TryStart(); - var res =_requestBodyPipe.Reader.TryRead(out _previousReadResult); - readResult = _previousReadResult; + var res =_requestBodyPipe.Reader.TryRead(out _readResult); + readResult = _readResult; - if (_previousReadResult.IsCompleted) + if (_readResult.IsCompleted) { TryStop(); } @@ -579,24 +580,33 @@ public override async ValueTask ReadAsync(CancellationToken cancella while (true) { - _previousReadResult = await StartTimingReadAsync(cancellationToken); - var readableBuffer = _previousReadResult.Buffer; + _readResult = await StartTimingReadAsync(cancellationToken); + var readableBuffer = _readResult.Buffer; var readableBufferLength = readableBuffer.Length; StopTimingRead(readableBufferLength); + if (_readResult.IsCanceled) + { + if (Interlocked.CompareExchange(ref _userCanceled, 0, 1) == 1) + { + // Ignore the readResult if it wasn't by the user. + break; + } + } + if (readableBufferLength != 0) { break; } - if (_previousReadResult.IsCompleted) + if (_readResult.IsCompleted) { TryStop(); break; } } - return _previousReadResult; + return _readResult; } private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) @@ -634,12 +644,13 @@ public override void Complete(Exception exception) public override void OnWriterCompleted(Action callback, object state) { - throw new NotImplementedException(); + _requestBodyPipe.Reader.OnWriterCompleted(callback, state); } public override void CancelPendingRead() { - throw new NotImplementedException(); + Interlocked.Exchange(ref _userCanceled, 1); + _requestBodyPipe.Reader.CancelPendingRead(); } private enum Mode diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs index cfb7ea61447f..196d78e6fb51 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs @@ -30,7 +30,12 @@ public ForContentLength(bool keepAlive, long contentLength, Http1Connection cont public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) { - if (_inputLength == 0 || _completed) + if (_completed) + { + throw new InvalidOperationException("Reading is not allowed after the reader was completed."); + } + + if (_inputLength == 0) { _readResult = new ReadResult(default, isCanceled: false, isCompleted: true); return _readResult; @@ -107,10 +112,15 @@ public override async ValueTask ReadAsync(CancellationToken cancella public override bool TryRead(out ReadResult readResult) { - if (_inputLength == 0 || _completed) + if (_completed) + { + throw new InvalidOperationException("Reading is not allowed after the reader was completed."); + } + + if (_inputLength == 0) { readResult = new ReadResult(default, isCanceled: false, isCompleted: true); - return false; + return true; } TryStart(); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs index da89a5c93379..0e0f587caec3 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs @@ -14,6 +14,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http /// public class ForUpgrade : Http1MessageBody { + public bool _completed; public ForUpgrade(Http1Connection context) : base(context) { @@ -25,11 +26,19 @@ public ForUpgrade(Http1Connection context) public override ValueTask ReadAsync(CancellationToken cancellationToken = default) { + if (_completed) + { + throw new InvalidOperationException("Reading is not allowed after the reader was completed."); + } return _context.Input.ReadAsync(cancellationToken); } public override bool TryRead(out ReadResult result) { + if (_completed) + { + throw new InvalidOperationException("Reading is not allowed after the reader was completed."); + } return _context.Input.TryRead(out result); } @@ -45,8 +54,9 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami public override void Complete(Exception exception) { - // Noop as we don't want to complete the connection pipe. - // actually we should complete this, just keep it internal + // Don't call Connection.Complete. + _context.ReportApplicationError(exception); + _completed = true; } public override void CancelPendingRead() diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index 3f93631e2750..0ea24fb19d0b 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -1246,6 +1246,11 @@ public void SetBadRequestState(BadHttpRequestException ex) public void ReportApplicationError(Exception ex) { + if (ex == null) + { + return; + } + if (_applicationException == null) { _applicationException = ex; diff --git a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs index 722d950bd448..cefb6366364d 100644 --- a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs +++ b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs @@ -858,6 +858,389 @@ public async Task DoesNotEnforceRequestBodyTimeoutOnUpgradeRequests() } } + [Fact] + public async Task CancelPendingReadContentLengthWorks() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + var readResultTask = reader.ReadAsync(); + + reader.CancelPendingRead(); + + var readResult = await readResultTask; + + Assert.True(readResult.IsCanceled); + + await body.StopAsync(); + } + } + + [Fact] + public async Task CancelPendingReadChunkedWorks() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + var readResultTask = reader.ReadAsync(); + + reader.CancelPendingRead(); + + var readResult = await readResultTask; + + Assert.True(readResult.IsCanceled); + + await body.StopAsync(); + } + } + + [Fact] + public async Task CancelPendingReadUpgradeWorks() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderConnection = "upgrade" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + var readResultTask = reader.ReadAsync(); + + reader.CancelPendingRead(); + + var readResult = await readResultTask; + + Assert.True(readResult.IsCanceled); + + await body.StopAsync(); + } + } + + [Fact] + public async Task CancelPendingReadForZeroContentLengthCannotBeCanceled() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders(), input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + var readResultTask = reader.ReadAsync(); + + Assert.True(readResultTask.IsCompleted); + + reader.CancelPendingRead(); + + var readResult = await readResultTask; + + Assert.False(readResult.IsCanceled); + + await body.StopAsync(); + } + } + + [Fact] + public async Task TryReadReturnsCompletedResultAfterReadingEntireContentLength() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("Hello"); + + Assert.True(reader.TryRead(out var readResult)); + + Assert.True(readResult.IsCompleted); + + await body.StopAsync(); + } + } + + [Fact] + public async Task TryReadReturnsCompletedResultAfterReadingEntireChunk() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("5\r\nHello\r\n"); + + Assert.True(reader.TryRead(out var readResult)); + Assert.False(readResult.IsCompleted); + AssertASCII("Hello", readResult.Buffer); + + reader.AdvanceTo(readResult.Buffer.End); + + input.Add("0\r\n\r\n"); + Assert.True(reader.TryRead(out readResult)); + + Assert.True(readResult.IsCompleted); + reader.AdvanceTo(readResult.Buffer.End); + + await body.StopAsync(); + } + } + + [Fact] + public async Task TryReadDoesNotReturnCompletedReadResultFromUpgradeStreamUntilCompleted() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderConnection = "upgrade" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("Hello"); + + Assert.True(reader.TryRead(out var readResult)); + Assert.False(readResult.IsCompleted); + AssertASCII("Hello", readResult.Buffer); + + reader.AdvanceTo(readResult.Buffer.End); + + input.Fin(); + + reader.TryRead(out readResult); + Assert.True(readResult.IsCompleted); + + await body.StopAsync(); + } + } + + [Fact] + public async Task TryReadDoesReturnsCompletedReadResultForZeroContentLength() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders(), input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("Hello"); + + Assert.True(reader.TryRead(out var readResult)); + Assert.True(readResult.IsCompleted); + + reader.AdvanceTo(readResult.Buffer.End); + + reader.TryRead(out readResult); + Assert.True(readResult.IsCompleted); + + await body.StopAsync(); + } + } + + [Fact] // TODO + public async Task OnWriterCompletedForContentLengthDoesNotWork() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("Hello"); + var retVal = false; + + // Callback isn't fired at the moment. + reader.OnWriterCompleted((a, b) => retVal = true, null); + Assert.True(reader.TryRead(out var readResult)); + + Assert.True(readResult.IsCompleted); + Assert.False(retVal); + + await body.StopAsync(); + } + } + + [Fact] + public async Task OnWriterCompletedForChunkedWorks() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + var retVal = false; + reader.OnWriterCompleted((a, b) => retVal = true, null); + + input.Add("0\r\n\r\n"); + + Assert.True(reader.TryRead(out var readResult)); + + Assert.True(readResult.IsCompleted); + Assert.True(retVal); + + await body.StopAsync(); + } + } + + [Fact] + public async Task OnWriterCompletedForUpgradeWorks() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderConnection = "upgrade" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + var retVal = false; + reader.OnWriterCompleted((a, b) => retVal = true, null); + + input.Add("hi"); + + Assert.True(reader.TryRead(out var readResult)); + reader.AdvanceTo(readResult.Buffer.End); + + input.Fin(); + + Assert.True(retVal); + + await body.StopAsync(); + } + } + + [Fact] + public async Task OnWriterCompletedForNoContentLengthNoop() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders(), input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + var retVal = false; + reader.OnWriterCompleted((a, b) => retVal = true, null); + + input.Add("hi"); + + Assert.True(reader.TryRead(out var readResult)); + reader.AdvanceTo(readResult.Buffer.End); + + input.Fin(); + + Assert.False(retVal); + + await body.StopAsync(); + } + } + + [Fact] + public async Task CompleteForContentLengthDoesNotCompleteConnectionPipeMakesReadReturnThrow() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("a"); + + Assert.True(reader.TryRead(out var readResult)); + + Assert.False(readResult.IsCompleted); + + input.Add("asdf"); + + reader.Complete(); + reader.AdvanceTo(readResult.Buffer.End); + + Assert.Throws(() => reader.TryRead(out readResult)); + + await body.StopAsync(); + } + } + + [Fact] + public async Task CompleteForChunkedDoesNotCompleteConnectionPipeMakesReadThrow() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderTransferEncoding = "chunked" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("5\r\nHello\r\n"); + + Assert.True(reader.TryRead(out var readResult)); + + Assert.False(readResult.IsCompleted); + reader.AdvanceTo(readResult.Buffer.End); + + input.Add("1\r\nH\r\n"); + + reader.Complete(); + + Assert.Throws(() => reader.TryRead(out readResult)); + + await body.StopAsync(); + } + } + + [Fact] + public async Task CompleteForUpgradeDoesNotCompleteConnectionPipeMakesReadThrow() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderConnection = "upgrade" }, input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("asdf"); + + Assert.True(reader.TryRead(out var readResult)); + + Assert.False(readResult.IsCompleted); + reader.AdvanceTo(readResult.Buffer.End); + + input.Add("asdf"); + + reader.Complete(); + + Assert.Throws(() => reader.TryRead(out readResult)); + + await body.StopAsync(); + } + } + + + [Fact] + public async Task CompleteForZeroByteBodyDoesNotCompleteConnectionPipeNoopsReads() + { + using (var input = new TestInput()) + { + var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders(), input.Http1Connection); + var reader = new HttpRequestPipeReader(); + reader.StartAcceptingReads(body); + + input.Add("asdf"); + + Assert.True(reader.TryRead(out var readResult)); + + Assert.False(readResult.IsCompleted); + reader.AdvanceTo(readResult.Buffer.End); + + input.Add("asdf"); + + reader.Complete(); + + // TODO should this noop or throw? I think we should keep parity with normal pipe behavior. + reader.TryRead(out readResult); + + await body.StopAsync(); + } + } + private void AssertASCII(string expected, ArraySegment actual) { var encoding = Encoding.ASCII; From f1ea1ec4d601232f8431cb014563dd8da3a9649e Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 14 Feb 2019 21:57:36 -0800 Subject: [PATCH 15/29] minor nits --- src/Servers/Kestrel/Core/test/MessageBodyTests.cs | 3 ++- .../Kestrel/test/InMemory.FunctionalTests/RequestTests.cs | 8 +++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs index cefb6366364d..53ecb8eec156 100644 --- a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs +++ b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs @@ -1227,7 +1227,7 @@ public async Task CompleteForZeroByteBodyDoesNotCompleteConnectionPipeNoopsReads Assert.True(reader.TryRead(out var readResult)); - Assert.False(readResult.IsCompleted); + Assert.True(readResult.IsCompleted); reader.AdvanceTo(readResult.Buffer.End); input.Add("asdf"); @@ -1235,6 +1235,7 @@ public async Task CompleteForZeroByteBodyDoesNotCompleteConnectionPipeNoopsReads reader.Complete(); // TODO should this noop or throw? I think we should keep parity with normal pipe behavior. + // So maybe this should throw reader.TryRead(out readResult); await body.StopAsync(); diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs index 5430c32cf5d6..5f033526b55c 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs @@ -701,18 +701,16 @@ await connection.ReceiveEnd( } [Fact] - public async Task ContentLengthTryReadPipeReader() + public async Task ContentLengthReadAsyncPipeReader() { var testContext = new TestServiceContext(LoggerFactory); - using (var server = new TestServer(httpContext => + using (var server = new TestServer(async httpContext => { - var res = httpContext.Request.BodyPipe.TryRead(out var readResult); + var readResult = await httpContext.Request.BodyPipe.ReadAsync(); // This will hang if 0 content length is not assumed by the server Assert.Equal(5, readResult.Buffer.Length); httpContext.Request.BodyPipe.AdvanceTo(readResult.Buffer.End); - - return Task.CompletedTask; }, testContext)) { using (var connection = server.CreateConnection()) From 196c72f6b247ecacd3998aceb0c3950f2e615761 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 15 Feb 2019 09:03:49 -0800 Subject: [PATCH 16/29] Handle Chunked.ReadAsync better --- src/Http/Http/src/ReadOnlyPipeStream.cs | 1 - .../Http/ForChunkedEncodingMessageBody.cs | 168 ++++++++---------- .../Http/ForContentLengthMessageBody.cs | 4 + 3 files changed, 75 insertions(+), 98 deletions(-) diff --git a/src/Http/Http/src/ReadOnlyPipeStream.cs b/src/Http/Http/src/ReadOnlyPipeStream.cs index c93ef8ccf011..919745bc5dcc 100644 --- a/src/Http/Http/src/ReadOnlyPipeStream.cs +++ b/src/Http/Http/src/ReadOnlyPipeStream.cs @@ -171,7 +171,6 @@ private async ValueTask ReadAsyncInternal(Memory buffer, Cancellation var readableBuffer = result.Buffer; var readableBufferLength = readableBuffer.Length; - // TODO make this throw if the result is canceled var consumed = readableBuffer.End; var actual = 0; try diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs index 4b8ed862b618..813acc5311bd 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs @@ -30,29 +30,74 @@ public class ForChunkedEncoding : Http1MessageBody private volatile bool _canceled; private Task _pumpTask; private Pipe _requestBodyPipe; - private int _userCanceled; + private ReadResult _readResult; public ForChunkedEncoding(bool keepAlive, Http1Connection context) : base(context) { RequestKeepAlive = keepAlive; - // For now, chunking will use the request body pipe _requestBodyPipe = CreateRequestBodyPipe(context); - //context.InternalRequestBodyPipeReader = _requestBodyPipe.Reader; } - private Pipe CreateRequestBodyPipe(Http1Connection context) - => new Pipe(new PipeOptions - ( - pool: context.MemoryPool, - readerScheduler: context.ServiceContext.Scheduler, - writerScheduler: PipeScheduler.Inline, - pauseWriterThreshold: 1, - resumeWriterThreshold: 1, - useSynchronizationContext: false, - minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize - )); + public override void AdvanceTo(SequencePosition consumed) + { + AdvanceTo(consumed, consumed); + } + + public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) + { + var dataLength = _readResult.Buffer.Slice(_readResult.Buffer.Start, consumed).Length; + _requestBodyPipe.Reader.AdvanceTo(consumed, examined); + OnDataRead(dataLength); + } + + public override bool TryRead(out ReadResult readResult) + { + TryStart(); + + var res = _requestBodyPipe.Reader.TryRead(out _readResult); + readResult = _readResult; + + if (_readResult.IsCompleted) + { + TryStop(); + } + return res; + } + + public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) + { + TryStart(); + + _readResult = await StartTimingReadAsync(cancellationToken); + + var readableBuffer = _readResult.Buffer; + var readableBufferLength = readableBuffer.Length; + StopTimingRead(readableBufferLength); + + if (_readResult.IsCompleted) + { + TryStop(); + } + + return _readResult; + } + + public override void Complete(Exception exception) + { + _requestBodyPipe.Reader.Complete(exception); + } + + public override void OnWriterCompleted(Action callback, object state) + { + _requestBodyPipe.Reader.OnWriterCompleted(callback, state); + } + + public override void CancelPendingRead() + { + _requestBodyPipe.Reader.CancelPendingRead(); + } private async Task PumpAsync() { @@ -179,11 +224,6 @@ protected override Task OnConsumeAsync() } } } - catch (OperationCanceledException) - { - // TryRead can throw OperationCanceledException https://github.com/dotnet/corefx/issues/32029 - // because of buggy logic, this works around that for now - } catch (BadHttpRequestException ex) { // At this point, the response has already been written, so this won't result in a 4XX response; @@ -546,68 +586,6 @@ private int CalculateChunkSize(int extraHexDigit, int currentParsedSize) BadHttpRequestException.Throw(RequestRejectionReason.BadChunkSizeData); return -1; // can't happen, but compiler complains } - private ReadResult _readResult; - - public override void AdvanceTo(SequencePosition consumed) - { - AdvanceTo(consumed, consumed); - } - - public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) - { - var dataLength = _readResult.Buffer.Slice(_readResult.Buffer.Start, consumed).Length; - _requestBodyPipe.Reader.AdvanceTo(consumed, examined); - OnDataRead(dataLength); - } - - public override bool TryRead(out ReadResult readResult) - { - TryStart(); - - var res =_requestBodyPipe.Reader.TryRead(out _readResult); - readResult = _readResult; - - if (_readResult.IsCompleted) - { - TryStop(); - } - return res; - } - - public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) - { - TryStart(); - - while (true) - { - _readResult = await StartTimingReadAsync(cancellationToken); - var readableBuffer = _readResult.Buffer; - var readableBufferLength = readableBuffer.Length; - StopTimingRead(readableBufferLength); - - if (_readResult.IsCanceled) - { - if (Interlocked.CompareExchange(ref _userCanceled, 0, 1) == 1) - { - // Ignore the readResult if it wasn't by the user. - break; - } - } - - if (readableBufferLength != 0) - { - break; - } - - if (_readResult.IsCompleted) - { - TryStop(); - break; - } - } - - return _readResult; - } private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) { @@ -637,22 +615,6 @@ private void StopTimingRead(long bytesRead) } } - public override void Complete(Exception exception) - { - _requestBodyPipe.Reader.Complete(exception); - } - - public override void OnWriterCompleted(Action callback, object state) - { - _requestBodyPipe.Reader.OnWriterCompleted(callback, state); - } - - public override void CancelPendingRead() - { - Interlocked.Exchange(ref _userCanceled, 1); - _requestBodyPipe.Reader.CancelPendingRead(); - } - private enum Mode { Prefix, @@ -663,5 +625,17 @@ private enum Mode TrailerHeaders, Complete }; + + private Pipe CreateRequestBodyPipe(Http1Connection context) + => new Pipe(new PipeOptions + ( + pool: context.MemoryPool, + readerScheduler: context.ServiceContext.Scheduler, + writerScheduler: PipeScheduler.Inline, + pauseWriterThreshold: 1, + resumeWriterThreshold: 1, + useSynchronizationContext: false, + minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize + )); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs index 196d78e6fb51..389b15599bb4 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs @@ -43,6 +43,10 @@ public override async ValueTask ReadAsync(CancellationToken cancella TryStart(); + // The while(true) loop is required because the Http1 connection calls CancelPendingRead to unblock + // the call to StartTimingReadAsync to check if the request timed out. + // However, if the user called CancelPendingRead, we want that to return a canceled ReadResult + // We internally track an int for that. while (true) { // This isn't great. The issue is that TryRead can get a canceled read result From a89e13272eb37395460d09b37ffc731434da4b12 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 15 Feb 2019 09:40:46 -0800 Subject: [PATCH 17/29] Renames for files and cleanup read/tryread logic --- ....cs => Http1ChunkedEncodingMessageBody.cs} | 14 ++--- ...dy.cs => Http1ContentLengthMessageBody.cs} | 54 +++++++++---------- .../src/Internal/Http/Http1MessageBody.cs | 6 +-- ...sageBody.cs => Http1UpgradeMessageBody.cs} | 4 +- .../Core/src/Internal/Http/MessageBody.cs | 4 +- ...ody.cs => ZeroContentLengthMessageBody.cs} | 4 +- 6 files changed, 42 insertions(+), 44 deletions(-) rename src/Servers/Kestrel/Core/src/Internal/Http/{ForChunkedEncodingMessageBody.cs => Http1ChunkedEncodingMessageBody.cs} (98%) rename src/Servers/Kestrel/Core/src/Internal/Http/{ForContentLengthMessageBody.cs => Http1ContentLengthMessageBody.cs} (90%) rename src/Servers/Kestrel/Core/src/Internal/Http/{ForUpgradeMessageBody.cs => Http1UpgradeMessageBody.cs} (95%) rename src/Servers/Kestrel/Core/src/Internal/Http/{ForZeroContentLengthMessageBody.cs => ZeroContentLengthMessageBody.cs} (92%) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs similarity index 98% rename from src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs rename to src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs index 813acc5311bd..9958fd89b0a5 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs @@ -17,7 +17,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http /// /// http://tools.ietf.org/html/rfc2616#section-3.6.1 /// - public class ForChunkedEncoding : Http1MessageBody + public class Http1ChunkedEncodingMessageBody : Http1MessageBody { // byte consts don't have a data type annotation so we pre-cast it private const byte ByteCR = (byte)'\r'; @@ -32,7 +32,7 @@ public class ForChunkedEncoding : Http1MessageBody private Pipe _requestBodyPipe; private ReadResult _readResult; - public ForChunkedEncoding(bool keepAlive, Http1Connection context) + public Http1ChunkedEncodingMessageBody(bool keepAlive, Http1Connection context) : base(context) { RequestKeepAlive = keepAlive; @@ -56,14 +56,16 @@ public override bool TryRead(out ReadResult readResult) { TryStart(); - var res = _requestBodyPipe.Reader.TryRead(out _readResult); + var boolResult = _requestBodyPipe.Reader.TryRead(out _readResult); + readResult = _readResult; if (_readResult.IsCompleted) { TryStop(); } - return res; + + return boolResult; } public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) @@ -72,9 +74,7 @@ public override async ValueTask ReadAsync(CancellationToken cancella _readResult = await StartTimingReadAsync(cancellationToken); - var readableBuffer = _readResult.Buffer; - var readableBufferLength = readableBuffer.Length; - StopTimingRead(readableBufferLength); + StopTimingRead(_readResult.Buffer.Length); if (_readResult.IsCompleted) { diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs similarity index 90% rename from src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs rename to src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs index 389b15599bb4..692745ade5b0 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs @@ -11,16 +11,15 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { - // Think this is close to good - public class ForContentLength : Http1MessageBody + public class Http1ContentLengthMessageBody : Http1MessageBody { private readonly long _contentLength; private long _inputLength; - private ReadResult _readResult; // TODO we can probably make this in Http1MessageBody or even MessageBody + private ReadResult _readResult; private bool _completed; private int _userCanceled; - public ForContentLength(bool keepAlive, long contentLength, Http1Connection context) + public Http1ContentLengthMessageBody(bool keepAlive, long contentLength, Http1Connection context) : base(context) { RequestKeepAlive = keepAlive; @@ -30,10 +29,7 @@ public ForContentLength(bool keepAlive, long contentLength, Http1Connection cont public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) { - if (_completed) - { - throw new InvalidOperationException("Reading is not allowed after the reader was completed."); - } + ThrowIfCompleted(); if (_inputLength == 0) { @@ -95,31 +91,14 @@ public override async ValueTask ReadAsync(CancellationToken cancella } } - // handle cases where we send more data than the content length - if (_readResult.Buffer.Length > _inputLength) - { - _readResult = new ReadResult(_readResult.Buffer.Slice(0, _inputLength), _readResult.IsCanceled, isCompleted: true); - - } - else if (_readResult.Buffer.Length == _inputLength) - { - _readResult = new ReadResult(_readResult.Buffer, _readResult.IsCanceled, isCompleted: true); - } - - if (_readResult.IsCompleted) - { - TryStop(); - } + _readResult = CreateReadResultFromConnectionReadResult(); return _readResult; } public override bool TryRead(out ReadResult readResult) { - if (_completed) - { - throw new InvalidOperationException("Reading is not allowed after the reader was completed."); - } + ThrowIfCompleted(); if (_inputLength == 0) { @@ -131,6 +110,22 @@ public override bool TryRead(out ReadResult readResult) var boolResult = _context.Input.TryRead(out _readResult); + readResult = CreateReadResultFromConnectionReadResult(); + + return boolResult; + } + + private void ThrowIfCompleted() + { + if (_completed) + { + throw new InvalidOperationException("Reading is not allowed after the reader was completed."); + } + } + + private ReadResult CreateReadResultFromConnectionReadResult() + { + ReadResult readResult; if (_readResult.Buffer.Length > _inputLength) { _readResult = new ReadResult(_readResult.Buffer.Slice(0, _inputLength), _readResult.IsCanceled, isCompleted: true); @@ -148,7 +143,7 @@ public override bool TryRead(out ReadResult readResult) TryStop(); } - return boolResult; + return readResult; } public override void AdvanceTo(SequencePosition consumed) @@ -164,8 +159,11 @@ public override void AdvanceTo(SequencePosition consumed, SequencePosition exami } var dataLength = _readResult.Buffer.Slice(_readResult.Buffer.Start, consumed).Length; + _inputLength -= dataLength; + _context.Input.AdvanceTo(consumed, examined); + OnDataRead(dataLength); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs index c0c492d0f9f3..7f60c6805a55 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs @@ -37,7 +37,7 @@ public static MessageBody For( BadHttpRequestException.Throw(RequestRejectionReason.UpgradeRequestCannotHavePayload); } - return new ForUpgrade(context); + return new Http1UpgradeMessageBody(context); } if (headers.HasTransferEncoding) @@ -58,7 +58,7 @@ public static MessageBody For( // TODO may push more into the wrapper rather than just calling into the message body // NBD for now. - return new ForChunkedEncoding(keepAlive, context); + return new Http1ChunkedEncodingMessageBody(keepAlive, context); } if (headers.ContentLength.HasValue) @@ -70,7 +70,7 @@ public static MessageBody For( return keepAlive ? MessageBody.ZeroContentLengthKeepAlive : MessageBody.ZeroContentLengthClose; } - return new ForContentLength(keepAlive, contentLength, context); + return new Http1ContentLengthMessageBody(keepAlive, contentLength, context); } // If we got here, request contains no Content-Length or Transfer-Encoding header. diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1UpgradeMessageBody.cs similarity index 95% rename from src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs rename to src/Servers/Kestrel/Core/src/Internal/Http/Http1UpgradeMessageBody.cs index 0e0f587caec3..1fcf18c37c43 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForUpgradeMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1UpgradeMessageBody.cs @@ -12,10 +12,10 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http /// The upgrade stream uses the raw connection stream instead of going through the RequestBodyPipe. This /// removes the redundant copy from the transport pipe to the body pipe. /// - public class ForUpgrade : Http1MessageBody + public class Http1UpgradeMessageBody : Http1MessageBody { public bool _completed; - public ForUpgrade(Http1Connection context) + public Http1UpgradeMessageBody(Http1Connection context) : base(context) { RequestUpgrade = true; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index 7361af4bffd5..c45258dee56b 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -11,8 +11,8 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { public abstract class MessageBody { - private static readonly MessageBody _zeroContentLengthClose = new ForZeroContentLength(keepAlive: false); - private static readonly MessageBody _zeroContentLengthKeepAlive = new ForZeroContentLength(keepAlive: true); + private static readonly MessageBody _zeroContentLengthClose = new ZeroContentLengthMessageBody(keepAlive: false); + private static readonly MessageBody _zeroContentLengthKeepAlive = new ZeroContentLengthMessageBody(keepAlive: true); private readonly HttpProtocol _context; private readonly MinDataRate _minRequestBodyDataRate; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/ForZeroContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/ZeroContentLengthMessageBody.cs similarity index 92% rename from src/Servers/Kestrel/Core/src/Internal/Http/ForZeroContentLengthMessageBody.cs rename to src/Servers/Kestrel/Core/src/Internal/Http/ZeroContentLengthMessageBody.cs index 919421d1e48d..355f534be02b 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/ForZeroContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/ZeroContentLengthMessageBody.cs @@ -8,9 +8,9 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { - public class ForZeroContentLength : MessageBody + public class ZeroContentLengthMessageBody : MessageBody { - public ForZeroContentLength(bool keepAlive) + public ZeroContentLengthMessageBody(bool keepAlive) : base(null, null) { RequestKeepAlive = keepAlive; From 118f8d13faff3a498f85792ad4b2f666acd7c9ab Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 15 Feb 2019 10:27:26 -0800 Subject: [PATCH 18/29] make tests pass and some general cleanup --- .../Http/Http1ContentLengthMessageBody.cs | 19 +++++-------------- .../Core/src/Internal/Http/HttpProtocol.cs | 4 +--- .../src/Internal/Http2/Http2MessageBody.cs | 12 ++++++------ .../Kestrel/Core/test/MessageBodyTests.cs | 15 ++++++++------- .../InMemory.FunctionalTests/RequestTests.cs | 15 --------------- 5 files changed, 20 insertions(+), 45 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs index 692745ade5b0..32b741fa25e2 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs @@ -91,7 +91,7 @@ public override async ValueTask ReadAsync(CancellationToken cancella } } - _readResult = CreateReadResultFromConnectionReadResult(); + CreateReadResultFromConnectionReadResult(); return _readResult; } @@ -110,7 +110,9 @@ public override bool TryRead(out ReadResult readResult) var boolResult = _context.Input.TryRead(out _readResult); - readResult = CreateReadResultFromConnectionReadResult(); + CreateReadResultFromConnectionReadResult(); + + readResult = _readResult; return boolResult; } @@ -123,27 +125,21 @@ private void ThrowIfCompleted() } } - private ReadResult CreateReadResultFromConnectionReadResult() + private void CreateReadResultFromConnectionReadResult() { - ReadResult readResult; if (_readResult.Buffer.Length > _inputLength) { _readResult = new ReadResult(_readResult.Buffer.Slice(0, _inputLength), _readResult.IsCanceled, isCompleted: true); - } else if (_readResult.Buffer.Length == _inputLength) { _readResult = new ReadResult(_readResult.Buffer, _readResult.IsCanceled, isCompleted: true); } - readResult = _readResult; - if (_readResult.IsCompleted) { TryStop(); } - - return readResult; } public override void AdvanceTo(SequencePosition consumed) @@ -239,11 +235,6 @@ protected override Task OnConsumeAsync() } } } - catch (OperationCanceledException) - { - // TryRead can throw OperationCanceledException https://github.com/dotnet/corefx/issues/32029 - // because of buggy logic, this works around that for now - } catch (BadHttpRequestException ex) { // At this point, the response has already been written, so this won't result in a 4XX response; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index 0ea24fb19d0b..5ce3afd57c31 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -62,7 +62,6 @@ public abstract partial class HttpProtocol : IDefaultHttpContextContainer, IHttp protected string _methodText = null; private string _scheme = null; - private List _wrapperObjectsToDispose; public HttpProtocol(HttpConnectionContext context) @@ -75,8 +74,6 @@ public HttpProtocol(HttpConnectionContext context) public IHttpResponseControl HttpResponseControl { get; set; } - //public PipeReader InternalRequestBodyPipeReader { get; set; } - public ServiceContext ServiceContext => _context.ServiceContext; private IPEndPoint LocalEndPoint => _context.LocalEndPoint; private IPEndPoint RemoteEndPoint => _context.RemoteEndPoint; @@ -1246,6 +1243,7 @@ public void SetBadRequestState(BadHttpRequestException ex) public void ReportApplicationError(Exception ex) { + // ReportApplicationError can be called with a null exception from MessageBody if (ex == null) { return; diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs index b0ca40fad1a6..19ff0d442715 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs @@ -12,7 +12,7 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 public class Http2MessageBody : MessageBody { private readonly Http2Stream _context; - private ReadResult _previousReadResult; + private ReadResult _readResult; private Http2MessageBody(Http2Stream context, MinDataRate minRequestBodyDataRate) : base(context, minRequestBodyDataRate) @@ -62,7 +62,7 @@ public override void AdvanceTo(SequencePosition consumed) public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { - var dataLength = _previousReadResult.Buffer.Slice(_previousReadResult.Buffer.Start, consumed).Length; + var dataLength = _readResult.Buffer.Slice(_readResult.Buffer.Start, consumed).Length; _context.RequestPipe.Reader.AdvanceTo(consumed, examined); OnDataRead(dataLength); } @@ -76,15 +76,15 @@ public override async ValueTask ReadAsync(CancellationToken cancella { TryStart(); - _previousReadResult = await StartTimingReadAsync(cancellationToken); - StopTimingRead(_previousReadResult.Buffer.Length); + _readResult = await StartTimingReadAsync(cancellationToken); + StopTimingRead(_readResult.Buffer.Length); - if (_previousReadResult.IsCompleted) + if (_readResult.IsCompleted) { TryStop(); } - return _previousReadResult; + return _readResult; } private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) diff --git a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs index 53ecb8eec156..785097ab1768 100644 --- a/src/Servers/Kestrel/Core/test/MessageBodyTests.cs +++ b/src/Servers/Kestrel/Core/test/MessageBodyTests.cs @@ -486,6 +486,7 @@ public async Task CopyToAsyncDoesNotCompletePipeReader() var body = Http1MessageBody.For(HttpVersion.Http10, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); var reader = new HttpRequestPipeReader(); var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); input.Add("Hello"); @@ -518,18 +519,17 @@ public async Task ConsumeAsyncConsumesAllRemainingInput() } [Fact] - public async Task ConsumeAsyncConsumesAllRemainingInputAfterStartingTryReadWithoutAdvance() + public async Task ConsumeAsyncAbortsConnectionInputAfterStartingTryReadWithoutAdvance() { using (var input = new TestInput()) { var body = Http1MessageBody.For(HttpVersion.Http10, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); input.Add("Hello"); - body.TryRead(out var readResult); - await body.ConsumeAsync(); // This will throw because someone called try read without advance. + body.TryRead(out var readResult); - Assert.True((await body.ReadAsync()).IsCompleted); + await body.ConsumeAsync(); await body.StopAsync(); } @@ -686,6 +686,7 @@ public async Task CopyToAsyncThrowsOnTimeout() var body = Http1MessageBody.For(HttpVersion.Http11, new HttpRequestHeaders { HeaderContentLength = "5" }, input.Http1Connection); var reader = new HttpRequestPipeReader(); var stream = new HttpRequestStream(Mock.Of(), reader); + reader.StartAcceptingReads(body); // Add some input and read it to start PumpAsync input.Add("a"); @@ -1071,15 +1072,15 @@ public async Task OnWriterCompletedForChunkedWorks() var reader = new HttpRequestPipeReader(); reader.StartAcceptingReads(body); - var retVal = false; - reader.OnWriterCompleted((a, b) => retVal = true, null); + var tcs = new TaskCompletionSource(); + reader.OnWriterCompleted((a, b) => tcs.SetResult(null), null); input.Add("0\r\n\r\n"); Assert.True(reader.TryRead(out var readResult)); Assert.True(readResult.IsCompleted); - Assert.True(retVal); + Assert.Null(await tcs.Task.DefaultTimeout()); await body.StopAsync(); } diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs index 5f033526b55c..d23af2583f0d 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs @@ -733,21 +733,6 @@ await connection.ReceiveEnd( ""); } - using (var connection = server.CreateConnection()) - { - await connection.Send( - "GET / HTTP/1.0", - "Host:", - "", - ""); - await connection.ReceiveEnd( - "HTTP/1.1 200 OK", - "Connection: close", - $"Date: {testContext.DateHeaderValue}", - "Content-Length: 0", - "", - ""); - } await server.StopAsync(); } } From af4234ce04e2517c9ca6a78972dd4a1989f6518b Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 15 Feb 2019 11:31:48 -0800 Subject: [PATCH 19/29] Add some more http1 tests --- .../Http/Http1ChunkedEncodingMessageBody.cs | 1 + .../ChunkedRequestTests.cs | 159 +++++++++++++++++- .../Http2/Http2StreamTests.cs | 151 +++++++++++++++++ .../InMemory.FunctionalTests/RequestTests.cs | 129 ++++++++++++++ 4 files changed, 439 insertions(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs index 9958fd89b0a5..14cfa03c1afa 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs @@ -87,6 +87,7 @@ public override async ValueTask ReadAsync(CancellationToken cancella public override void Complete(Exception exception) { _requestBodyPipe.Reader.Complete(exception); + _context.ReportApplicationError(exception); } public override void OnWriterCompleted(Action callback, object state) diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs index d67a191a5769..fc5c3362f388 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs @@ -320,7 +320,6 @@ public async Task TrailingHeadersAreParsedWithPipe() var response = httpContext.Response; var request = httpContext.Request; - var buffer = new byte[200]; while (true) { var result = await request.BodyPipe.ReadAsync(); @@ -815,5 +814,163 @@ await connection.SendAll( await server.StopAsync(); } } + + [Fact] + public async Task ChunkedRequestCallCancelPendingReadWorks() + { + var tcs = new TaskCompletionSource(); + var testContext = new TestServiceContext(LoggerFactory); + + using (var server = new TestServer(async httpContext => + { + var response = httpContext.Response; + var request = httpContext.Request; + + Assert.Equal("POST", request.Method); + + var readResult = await request.BodyPipe.ReadAsync(); + request.BodyPipe.AdvanceTo(readResult.Buffer.End); + + var requestTask = httpContext.Request.BodyPipe.ReadAsync(); + + httpContext.Request.BodyPipe.CancelPendingRead(); + + Assert.True((await requestTask).IsCanceled); + + tcs.SetResult(null); + + response.Headers["Content-Length"] = new[] { "11" }; + + await response.BodyPipe.WriteAsync(new Memory(Encoding.ASCII.GetBytes("Hello World"), 0, 11)); + + }, testContext)) + { + using (var connection = server.CreateConnection()) + { + await connection.SendAll( + "POST / HTTP/1.1", + "Host:", + "Transfer-Encoding: chunked", + "", + "1", + "H"); + await tcs.Task; + await connection.Send( + "4", + "ello", + "0", + "", + ""); + + await connection.Receive( + "HTTP/1.1 200 OK", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 11", + "", + "Hello World"); + } + await server.StopAsync(); + } + } + + [Fact] + public async Task ChunkedRequestCallCompleteThrowsExceptionOnRead() + { + var testContext = new TestServiceContext(LoggerFactory); + + using (var server = new TestServer(async httpContext => + { + var response = httpContext.Response; + var request = httpContext.Request; + + Assert.Equal("POST", request.Method); + + var readResult = await request.BodyPipe.ReadAsync(); + request.BodyPipe.AdvanceTo(readResult.Buffer.End); + + httpContext.Request.BodyPipe.Complete(); + + await Assert.ThrowsAsync(async () => await request.BodyPipe.ReadAsync()); + + response.Headers["Content-Length"] = new[] { "11" }; + + await response.BodyPipe.WriteAsync(new Memory(Encoding.ASCII.GetBytes("Hello World"), 0, 11)); + + }, testContext)) + { + using (var connection = server.CreateConnection()) + { + await connection.SendAll( + "POST / HTTP/1.1", + "Host:", + "Transfer-Encoding: chunked", + "", + "1", + "H"); + await connection.Send( + "4", + "ello", + "0", + "", + ""); + + await connection.Receive( + "HTTP/1.1 200 OK", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 11", + "", + "Hello World"); + } + await server.StopAsync(); + } + } + + [Fact] + public async Task ChunkedRequestCallCompleteWithExceptionCauses500() + { + var tcs = new TaskCompletionSource(); + var testContext = new TestServiceContext(LoggerFactory); + + using (var server = new TestServer(async httpContext => + { + var response = httpContext.Response; + var request = httpContext.Request; + + Assert.Equal("POST", request.Method); + + var readResult = await request.BodyPipe.ReadAsync(); + request.BodyPipe.AdvanceTo(readResult.Buffer.End); + + httpContext.Request.BodyPipe.Complete(new Exception()); + + response.Headers["Content-Length"] = new[] { "11" }; + + await response.BodyPipe.WriteAsync(new Memory(Encoding.ASCII.GetBytes("Hello World"), 0, 11)); + + }, testContext)) + { + using (var connection = server.CreateConnection()) + { + await connection.SendAll( + "POST / HTTP/1.1", + "Host:", + "Transfer-Encoding: chunked", + "", + "1", + "H", + "0", + "", + ""); + + await connection.Receive( + "HTTP/1.1 500 Internal Server Error", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 0", + "", + ""); + } + await server.StopAsync(); + } + } } } diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs index 2512e27c9c4a..26687dbb20db 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs @@ -727,6 +727,110 @@ await ExpectAsync(Http2FrameType.DATA, Assert.Equal("0", _decodedHeaders[HeaderNames.ContentLength]); } + [Fact] + public async Task ContentLength_Received_MultipleDataFrame_ReadViaPipe_Verified() + { + var headers = new[] + { + new KeyValuePair(HeaderNames.Method, "POST"), + new KeyValuePair(HeaderNames.Path, "/"), + new KeyValuePair(HeaderNames.Scheme, "http"), + new KeyValuePair(HeaderNames.ContentLength, "12"), + }; + await InitializeConnectionAsync(async context => + { + var readResult = await context.Request.BodyPipe.ReadAsync(); + while (!readResult.IsCompleted) + { + context.Request.BodyPipe.AdvanceTo(readResult.Buffer.Start, readResult.Buffer.End); + readResult = await context.Request.BodyPipe.ReadAsync(); + } + + Assert.Equal(12, readResult.Buffer.Length); + context.Request.BodyPipe.AdvanceTo(readResult.Buffer.End); + }); + + await StartStreamAsync(1, headers, endStream: false); + await SendDataAsync(1, new byte[1], endStream: false); + await SendDataAsync(1, new byte[3], endStream: false); + await SendDataAsync(1, new byte[8], endStream: true); + + var headersFrame = await ExpectAsync(Http2FrameType.HEADERS, + withLength: 55, + withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS, + withStreamId: 1); + await ExpectAsync(Http2FrameType.DATA, + withLength: 0, + withFlags: (byte)Http2DataFrameFlags.END_STREAM, + withStreamId: 1); + + await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); + + _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this); + + Assert.Equal(3, _decodedHeaders.Count); + Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase); + Assert.Equal("200", _decodedHeaders[HeaderNames.Status]); + Assert.Equal("0", _decodedHeaders[HeaderNames.ContentLength]); + } + + [Fact] + public async Task ContentLength_Received_MultipleDataFrame_ReadViaPipeAndStream_Verified() + { + var tcs = new TaskCompletionSource(); + var headers = new[] + { + new KeyValuePair(HeaderNames.Method, "POST"), + new KeyValuePair(HeaderNames.Path, "/"), + new KeyValuePair(HeaderNames.Scheme, "http"), + new KeyValuePair(HeaderNames.ContentLength, "12"), + }; + await InitializeConnectionAsync(async context => + { + var readResult = await context.Request.BodyPipe.ReadAsync(); + Assert.Equal(1, readResult.Buffer.Length); + context.Request.BodyPipe.AdvanceTo(readResult.Buffer.End); + + tcs.SetResult(null); + + var buffer = new byte[100]; + + var read = await context.Request.Body.ReadAsync(buffer, 0, buffer.Length); + var total = read; + while (read > 0) + { + read = await context.Request.Body.ReadAsync(buffer, total, buffer.Length - total); + total += read; + } + + Assert.Equal(11, total); + }); + + await StartStreamAsync(1, headers, endStream: false); + await SendDataAsync(1, new byte[1], endStream: false); + await tcs.Task; + await SendDataAsync(1, new byte[3], endStream: false); + await SendDataAsync(1, new byte[8], endStream: true); + + var headersFrame = await ExpectAsync(Http2FrameType.HEADERS, + withLength: 55, + withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS, + withStreamId: 1); + await ExpectAsync(Http2FrameType.DATA, + withLength: 0, + withFlags: (byte)Http2DataFrameFlags.END_STREAM, + withStreamId: 1); + + await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); + + _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this); + + Assert.Equal(3, _decodedHeaders.Count); + Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase); + Assert.Equal("200", _decodedHeaders[HeaderNames.Status]); + Assert.Equal("0", _decodedHeaders[HeaderNames.ContentLength]); + } + [Fact] public async Task ContentLength_Received_NoDataFrames_Reset() { @@ -911,6 +1015,53 @@ await InitializeConnectionAsync(async context => Assert.IsType(thrownEx.InnerException); } + [Fact] + public async Task ContentLength_Received_ReadViaPipes() + { + await InitializeConnectionAsync(async context => + { + var readResult = await context.Request.BodyPipe.ReadAsync(); + Assert.Equal(12, readResult.Buffer.Length); + Assert.True(readResult.IsCompleted); + context.Request.BodyPipe.AdvanceTo(readResult.Buffer.End); + + readResult = await context.Request.BodyPipe.ReadAsync(); + Assert.True(readResult.IsCompleted); + }); + + var headers = new[] + { + new KeyValuePair(HeaderNames.Method, "POST"), + new KeyValuePair(HeaderNames.Path, "/"), + new KeyValuePair(HeaderNames.Scheme, "http"), + new KeyValuePair("a", _4kHeaderValue), + new KeyValuePair("b", _4kHeaderValue), + new KeyValuePair("c", _4kHeaderValue), + new KeyValuePair("d", _4kHeaderValue), + new KeyValuePair(HeaderNames.ContentLength, "12"), + }; + await StartStreamAsync(1, headers, endStream: false); + await SendDataAsync(1, new byte[12], endStream: true); + + var headersFrame = await ExpectAsync(Http2FrameType.HEADERS, + withLength: 55, + withFlags: (byte)Http2HeadersFrameFlags.END_HEADERS, + withStreamId: 1); + await ExpectAsync(Http2FrameType.DATA, + withLength: 0, + withFlags: (byte)Http2DataFrameFlags.END_STREAM, + withStreamId: 1); + + await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); + + _hpackDecoder.Decode(headersFrame.PayloadSequence, endHeaders: false, handler: this); + + Assert.Equal(3, _decodedHeaders.Count); + Assert.Contains("date", _decodedHeaders.Keys, StringComparer.OrdinalIgnoreCase); + Assert.Equal("200", _decodedHeaders[HeaderNames.Status]); + Assert.Equal("0", _decodedHeaders[HeaderNames.ContentLength]); + } + [Fact] // TODO https://github.com/aspnet/AspNetCore/issues/7034 public async Task ContentLength_Response_FirstWriteMoreBytesWritten_Throws_Sends500() { diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs index d23af2583f0d..15f8b7dbca07 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs @@ -1326,6 +1326,83 @@ public async Task SynchronousReadsCanBeAllowedGlobally() Assert.Equal(0, await context.Request.Body.ReadAsync(new byte[1], 0, 1)); Assert.Equal("Hello", Encoding.ASCII.GetString(buffer, 0, 5)); + })); + } + + [Fact] + public async Task ContentLengthRequestCallCancelPendingReadWorks() + { + var tcs = new TaskCompletionSource(); + var testContext = new TestServiceContext(LoggerFactory); + + using (var server = new TestServer(async httpContext => + { + var response = httpContext.Response; + var request = httpContext.Request; + + Assert.Equal("POST", request.Method); + + var readResult = await request.BodyPipe.ReadAsync(); + request.BodyPipe.AdvanceTo(readResult.Buffer.End); + + var requestTask = httpContext.Request.BodyPipe.ReadAsync(); + + httpContext.Request.BodyPipe.CancelPendingRead(); + + Assert.True((await requestTask).IsCanceled); + + tcs.SetResult(null); + + response.Headers["Content-Length"] = new[] { "11" }; + + await response.BodyPipe.WriteAsync(new Memory(Encoding.ASCII.GetBytes("Hello World"), 0, 11)); + + }, testContext)) + { + using (var connection = server.CreateConnection()) + { + await connection.Send( + "POST / HTTP/1.1", + "Host:", + "Content-Length: 5", + "", + "H"); + await tcs.Task; + await connection.Send( + "ello"); + await connection.Receive( + "HTTP/1.1 200 OK", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 11", + "", + "Hello World"); + } + await server.StopAsync(); + } + } + + [Fact] + public async Task ContentLengthRequestCallCompleteThrowsExceptionOnRead() + { + var testContext = new TestServiceContext(LoggerFactory); + + using (var server = new TestServer(async httpContext => + { + var response = httpContext.Response; + var request = httpContext.Request; + + Assert.Equal("POST", request.Method); + + var readResult = await request.BodyPipe.ReadAsync(); + request.BodyPipe.AdvanceTo(readResult.Buffer.End); + + httpContext.Request.BodyPipe.Complete(); + + await Assert.ThrowsAsync(async () => await request.BodyPipe.ReadAsync()); + + response.Headers["Content-Length"] = new[] { "11" }; + + await response.BodyPipe.WriteAsync(new Memory(Encoding.ASCII.GetBytes("Hello World"), 0, 11)); }, testContext)) { using (var connection = server.CreateConnection()) @@ -1342,7 +1419,59 @@ await connection.Receive( "Content-Length: 0", "", ""); + + await connection.Receive( + "HTTP/1.1 200 OK", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 11", + "", + "Hello World"); } + await server.StopAsync(); + } + } + + [Fact] + public async Task ContentLengthCallCompleteWithExceptionCauses500() + { + var tcs = new TaskCompletionSource(); + var testContext = new TestServiceContext(LoggerFactory); + + using (var server = new TestServer(async httpContext => + { + var response = httpContext.Response; + var request = httpContext.Request; + + Assert.Equal("POST", request.Method); + + var readResult = await request.BodyPipe.ReadAsync(); + request.BodyPipe.AdvanceTo(readResult.Buffer.End); + + httpContext.Request.BodyPipe.Complete(new Exception()); + + response.Headers["Content-Length"] = new[] { "11" }; + + await response.BodyPipe.WriteAsync(new Memory(Encoding.ASCII.GetBytes("Hello World"), 0, 11)); + + }, testContext)) + { + using (var connection = server.CreateConnection()) + { + await connection.Send( + "POST / HTTP/1.1", + "Host:", + "Content-Length: 5", + "", + "Hello"); + + await connection.Receive( + "HTTP/1.1 500 Internal Server Error", + $"Date: {testContext.DateHeaderValue}", + "Content-Length: 0", + "", + ""); + } + await server.StopAsync(); } } From d737360f1a2ef2eb386286d1db5ec098dc0cfaed Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 15 Feb 2019 11:58:17 -0800 Subject: [PATCH 20/29] RequestPipeReader tests --- .../Core/test/HttpRequestPipeReaderTests.cs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/Servers/Kestrel/Core/test/HttpRequestPipeReaderTests.cs b/src/Servers/Kestrel/Core/test/HttpRequestPipeReaderTests.cs index abda6aaff120..97f0c91cba7c 100644 --- a/src/Servers/Kestrel/Core/test/HttpRequestPipeReaderTests.cs +++ b/src/Servers/Kestrel/Core/test/HttpRequestPipeReaderTests.cs @@ -1,10 +1,45 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; +using Xunit; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Tests { public class HttpRequestPipeReaderTests { + [Fact] + public async Task StopAcceptingReadsCausesReadToThrowObjectDisposedException() + { + var pipeReader = new HttpRequestPipeReader(); + pipeReader.StartAcceptingReads(null); + pipeReader.StopAcceptingReads(); + + // Validation for ReadAsync occurs in an async method in ReadOnlyPipeStream. + await Assert.ThrowsAsync(async () => { await pipeReader.ReadAsync(); }); + } + [Fact] + public async Task AbortCausesReadToCancel() + { + var pipeReader = new HttpRequestPipeReader(); + + pipeReader.StartAcceptingReads(null); + pipeReader.Abort(); + await Assert.ThrowsAsync(() => pipeReader.ReadAsync().AsTask()); + } + + [Fact] + public async Task AbortWithErrorCausesReadToCancel() + { + var pipeReader = new HttpRequestPipeReader(); + + pipeReader.StartAcceptingReads(null); + var error = new Exception(); + pipeReader.Abort(error); + var exception = await Assert.ThrowsAsync(() => pipeReader.ReadAsync().AsTask()); + Assert.Same(error, exception); + } } } From d32295d503bacf6a436ef14f293b1cb7cf2577a0 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 15 Feb 2019 13:58:01 -0800 Subject: [PATCH 21/29] adding benchmarks even though they don't work --- .../Http1ReadingBenchmark.cs | 145 ++++++++++++++++++ .../Http1WritingBenchmark.cs | 2 +- 2 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 src/Servers/Kestrel/perf/Kestrel.Performance/Http1ReadingBenchmark.cs diff --git a/src/Servers/Kestrel/perf/Kestrel.Performance/Http1ReadingBenchmark.cs b/src/Servers/Kestrel/perf/Kestrel.Performance/Http1ReadingBenchmark.cs new file mode 100644 index 000000000000..122c5b694c19 --- /dev/null +++ b/src/Servers/Kestrel/perf/Kestrel.Performance/Http1ReadingBenchmark.cs @@ -0,0 +1,145 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +using System; +using System.Buffers; +using System.IO.Pipelines; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using BenchmarkDotNet.Attributes; +using Microsoft.AspNetCore.Http.Features; +using Microsoft.AspNetCore.Server.Kestrel.Core; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; +using Microsoft.AspNetCore.Server.Kestrel.Transport.Abstractions.Internal; +using Microsoft.AspNetCore.Testing; + +namespace Microsoft.AspNetCore.Server.Kestrel.Performance +{ + public class Http1ReadingBenchmark + { + // Standard completed task + private static readonly Func _syncTaskFunc = (obj) => Task.CompletedTask; + // Non-standard completed task + private static readonly Task _pseudoAsyncTask = Task.FromResult(27); + private static readonly Func _pseudoAsyncTaskFunc = (obj) => _pseudoAsyncTask; + + private TestHttp1Connection _http1Connection; + private DuplexPipe.DuplexPipePair _pair; + private MemoryPool _memoryPool; + + private readonly byte[] _readData = Encoding.ASCII.GetBytes(new string('a', 100)); + + [GlobalSetup] + public void GlobalSetup() + { + _memoryPool = KestrelMemoryPool.Create(); + _http1Connection = MakeHttp1Connection(); + } + + [Params(true, false)] + public bool WithHeaders { get; set; } + + //[Params(true, false)] + //public bool Chunked { get; set; } + + [Params(Startup.None, Startup.Sync, Startup.Async)] + public Startup OnStarting { get; set; } + + [IterationSetup] + public void Setup() + { + _http1Connection.Reset(); + + _http1Connection.RequestHeaders.ContentLength = _readData.Length; + + if (!WithHeaders) + { + _http1Connection.FlushAsync().GetAwaiter().GetResult(); + } + + ResetState(); + } + + private void ResetState() + { + if (WithHeaders) + { + _http1Connection.ResetState(); + + switch (OnStarting) + { + case Startup.Sync: + _http1Connection.OnStarting(_syncTaskFunc, null); + break; + case Startup.Async: + _http1Connection.OnStarting(_pseudoAsyncTaskFunc, null); + break; + } + } + } + + [Benchmark] + public Task ReadAsync() + { + ResetState(); + + return _http1Connection.ResponseBody.ReadAsync(new byte[100], default(CancellationToken)).AsTask(); + } + + private TestHttp1Connection MakeHttp1Connection() + { + var options = new PipeOptions(_memoryPool, readerScheduler: PipeScheduler.Inline, writerScheduler: PipeScheduler.Inline, useSynchronizationContext: false); + var pair = DuplexPipe.CreateConnectionPair(options, options); + _pair = pair; + + var serviceContext = new ServiceContext + { + DateHeaderValueManager = new DateHeaderValueManager(), + ServerOptions = new KestrelServerOptions(), + Log = new MockTrace(), + HttpParser = new HttpParser() + }; + + var http1Connection = new TestHttp1Connection(new HttpConnectionContext + { + ServiceContext = serviceContext, + ConnectionFeatures = new FeatureCollection(), + MemoryPool = _memoryPool, + TimeoutControl = new TimeoutControl(timeoutHandler: null), + Transport = pair.Transport + }); + + http1Connection.Reset(); + http1Connection.InitializeBodyControl(new Http1ContentLengthMessageBody(keepAlive: true, 100, http1Connection)); + serviceContext.DateHeaderValueManager.OnHeartbeat(DateTimeOffset.UtcNow); + + return http1Connection; + } + + [IterationCleanup] + public void Cleanup() + { + var reader = _pair.Application.Input; + if (reader.TryRead(out var readResult)) + { + reader.AdvanceTo(readResult.Buffer.End); + } + } + + public enum Startup + { + None, + Sync, + Async + } + + [GlobalCleanup] + public void Dispose() + { + _memoryPool?.Dispose(); + } + } +} diff --git a/src/Servers/Kestrel/perf/Kestrel.Performance/Http1WritingBenchmark.cs b/src/Servers/Kestrel/perf/Kestrel.Performance/Http1WritingBenchmark.cs index 78f8359f06a0..d0c1cf3370f7 100644 --- a/src/Servers/Kestrel/perf/Kestrel.Performance/Http1WritingBenchmark.cs +++ b/src/Servers/Kestrel/perf/Kestrel.Performance/Http1WritingBenchmark.cs @@ -1,4 +1,4 @@ -// Copyright (c) .NET Foundation. All rights reserved. +// Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; From 20a6ad3ca63ebeb84df2368ac2010e988fd3a0c5 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 15 Feb 2019 15:01:46 -0800 Subject: [PATCH 22/29] removing bad comments --- .../test/InMemory.FunctionalTests/RequestTests.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs index 15f8b7dbca07..99257710f265 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs @@ -612,9 +612,6 @@ public async Task ZeroContentLengthAssumedOnNonKeepAliveRequestsWithoutContentLe { using (var connection = server.CreateConnection()) { - // Use Send instead of SendEnd to ensure the connection will remain open while - // the app runs and reads 0 bytes from the body nonetheless. This checks that - // https://github.com/aspnet/KestrelHttpServer/issues/1104 is not regressing. await connection.Send( "GET / HTTP/1.1", "Host:", @@ -663,9 +660,6 @@ public async Task ZeroContentLengthAssumedOnNonKeepAliveRequestsWithoutContentLe { using (var connection = server.CreateConnection()) { - // Use Send instead of SendEnd to ensure the connection will remain open while - // the app runs and reads 0 bytes from the body nonetheless. This checks that - // https://github.com/aspnet/KestrelHttpServer/issues/1104 is not regressing. await connection.Send( "GET / HTTP/1.1", "Host:", @@ -715,9 +709,6 @@ public async Task ContentLengthReadAsyncPipeReader() { using (var connection = server.CreateConnection()) { - // Use Send instead of SendEnd to ensure the connection will remain open while - // the app runs and reads 0 bytes from the body nonetheless. This checks that - // https://github.com/aspnet/KestrelHttpServer/issues/1104 is not regressing. await connection.SendAll( "POST / HTTP/1.0", "Host:", From 86ac73b7eb4739c7837e304da29290f1e8194bb7 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Fri, 15 Feb 2019 17:49:34 -0800 Subject: [PATCH 23/29] Wrap cancellation exceptions with TaskCanceledException --- .../Internal/Http/Http1ChunkedEncodingMessageBody.cs | 9 ++++++++- .../Internal/Http/Http1ContentLengthMessageBody.cs | 9 ++++++++- .../Core/src/Internal/Http2/Http2MessageBody.cs | 11 ++++++++++- .../Kestrel/test/FunctionalTests/RequestTests.cs | 2 +- .../InMemory.FunctionalTests/ChunkedRequestTests.cs | 5 ++--- .../Http2/Http2StreamTests.cs | 8 ++++---- 6 files changed, 33 insertions(+), 11 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs index 14cfa03c1afa..c16d15d5d123 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs @@ -72,7 +72,14 @@ public override async ValueTask ReadAsync(CancellationToken cancella { TryStart(); - _readResult = await StartTimingReadAsync(cancellationToken); + try + { + _readResult = await StartTimingReadAsync(cancellationToken); + } + catch (ConnectionAbortedException ex) + { + throw new TaskCanceledException("The request was aborted", ex); + } StopTimingRead(_readResult.Buffer.Length); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs index 32b741fa25e2..0658162fc6e1 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs @@ -52,7 +52,14 @@ public override async ValueTask ReadAsync(CancellationToken cancella BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); } - _readResult = await StartTimingReadAsync(cancellationToken); + try + { + _readResult = await StartTimingReadAsync(cancellationToken); + } + catch (ConnectionAbortedException ex) + { + throw new TaskCanceledException("The request was aborted", ex); + } if (_context.RequestTimedOut) { diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs index 19ff0d442715..1fa4a49f17d3 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs @@ -5,6 +5,7 @@ using System.IO.Pipelines; using System.Threading; using System.Threading.Tasks; +using Microsoft.AspNetCore.Connections; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2 @@ -76,7 +77,15 @@ public override async ValueTask ReadAsync(CancellationToken cancella { TryStart(); - _readResult = await StartTimingReadAsync(cancellationToken); + try + { + _readResult = await StartTimingReadAsync(cancellationToken); + } + catch (ConnectionAbortedException ex) + { + throw new TaskCanceledException("The request was aborted", ex); + } + StopTimingRead(_readResult.Buffer.Length); if (_readResult.IsCompleted) diff --git a/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs index 06ffc72a8316..e2320b2d28c4 100644 --- a/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/FunctionalTests/RequestTests.cs @@ -698,7 +698,7 @@ await connection.Send("POST / HTTP/1.1", await server.StopAsync(); } - await Assert.ThrowsAsync(async () => await readTcs.Task); + await Assert.ThrowsAsync(async () => await readTcs.Task); // The cancellation token for only the last request should be triggered. var abortedRequestId = await registrationTcs.Task.DefaultTimeout(); diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs index fc5c3362f388..5d0be6b3e0cc 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/ChunkedRequestTests.cs @@ -900,14 +900,13 @@ public async Task ChunkedRequestCallCompleteThrowsExceptionOnRead() { using (var connection = server.CreateConnection()) { - await connection.SendAll( + await connection.Send( "POST / HTTP/1.1", "Host:", "Transfer-Encoding: chunked", "", "1", - "H"); - await connection.Send( + "H", "4", "ello", "0", diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs index 26687dbb20db..a427f7c4804c 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http2/Http2StreamTests.cs @@ -2452,10 +2452,10 @@ await InitializeConnectionAsync(async context => await StopConnectionAsync(expectedLastStreamId: 1, ignoreNonGoAwayFrames: false); Assert.NotNull(thrownEx); - // TODO due to the method going async, we don't rethrow the connection aborted exception. - // We can look into fixing this. - Assert.IsType(thrownEx); - Assert.Equal(CoreStrings.ConnectionAbortedByApplication, thrownEx.Message); + Assert.IsType(thrownEx); + Assert.Equal("The request was aborted", thrownEx.Message); + Assert.IsType(thrownEx.InnerException); + Assert.Equal(CoreStrings.ConnectionAbortedByApplication, thrownEx.InnerException.Message); } // Sync writes after async writes could block the write loop if the callback is not dispatched. From 1f35f3ac0f0609d10c1e35567ed9eff295dab433 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 19 Feb 2019 06:41:34 -0800 Subject: [PATCH 24/29] Majority of feedback. --- .../Http/Http1ChunkedEncodingMessageBody.cs | 115 +------------- .../Http/Http1ContentLengthMessageBody.cs | 147 ++++-------------- .../src/Internal/Http/Http1MessageBody.cs | 96 ++++++++++++ .../Http/HttpProtocol.FeatureCollection.cs | 8 +- .../Core/src/Internal/Http/HttpProtocol.cs | 4 +- .../Core/src/Internal/Http/MessageBody.cs | 26 ++++ .../src/Internal/Http2/Http2MessageBody.cs | 41 ++--- .../Core/src/Internal/Http2/Http2Stream.cs | 16 +- 8 files changed, 179 insertions(+), 274 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs index c16d15d5d123..81b123161ee8 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs @@ -74,7 +74,9 @@ public override async ValueTask ReadAsync(CancellationToken cancella try { - _readResult = await StartTimingReadAsync(cancellationToken); + var readAwaitable = _requestBodyPipe.Reader.ReadAsync(cancellationToken); + + _readResult = await StartTimingReadAsync(readAwaitable, cancellationToken); } catch (ConnectionAbortedException ex) { @@ -156,17 +158,7 @@ private async Task PumpAsync() } // Read() will have already have greedily consumed the entire request body if able. - if (result.IsCompleted) - { - // OnInputOrOutputCompleted() is an idempotent method that closes the connection. Sometimes - // input completion is observed here before the Input.OnWriterCompleted() callback is fired, - // so we call OnInputOrOutputCompleted() now to prevent a race in our tests where a 400 - // response is written after observing the unexpected end of request content instead of just - // closing the connection without a response as expected. - _context.OnInputOrOutputCompleted(); - - BadHttpRequestException.Throw(RequestRejectionReason.UnexpectedEndOfRequestContent); - } + CheckCompletedReadResult(result); } finally { @@ -218,77 +210,6 @@ private async Task StopAsyncAwaited() _requestBodyPipe.Reset(); } - protected override Task OnConsumeAsync() - { - try - { - if (_requestBodyPipe.Reader.TryRead(out var readResult)) - { - _requestBodyPipe.Reader.AdvanceTo(readResult.Buffer.End); - - if (readResult.IsCompleted) - { - return Task.CompletedTask; - } - } - } - catch (BadHttpRequestException ex) - { - // At this point, the response has already been written, so this won't result in a 4XX response; - // however, we still need to stop the request processing loop and log. - _context.SetBadRequestState(ex); - return Task.CompletedTask; - } - catch (InvalidOperationException ex) - { - var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); - _context.ReportApplicationError(connectionAbortedException); - - // Have to abort the connection because we can't finish draining the request - _context.StopProcessingNextRequest(); - return Task.CompletedTask; - } - - return OnConsumeAsyncAwaited(); - } - - private async Task OnConsumeAsyncAwaited() - { - Log.RequestBodyNotEntirelyRead(_context.ConnectionIdFeature, _context.TraceIdentifier); - - _context.TimeoutControl.SetTimeout(Constants.RequestBodyDrainTimeout.Ticks, TimeoutReason.RequestBodyDrain); - - try - { - ReadResult result; - do - { - result = await _requestBodyPipe.Reader.ReadAsync(); - _requestBodyPipe.Reader.AdvanceTo(result.Buffer.End); - } while (!result.IsCompleted); - } - catch (BadHttpRequestException ex) - { - _context.SetBadRequestState(ex); - } - catch (ConnectionAbortedException) - { - Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); - } - catch (InvalidOperationException ex) - { - var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); - _context.ReportApplicationError(connectionAbortedException); - - // Have to abort the connection because we can't finish draining the request - _context.StopProcessingNextRequest(); - } - finally - { - _context.TimeoutControl.CancelTimeout(); - } - } - protected void Copy(ReadOnlySequence readableBuffer, PipeWriter writableBuffer) { if (readableBuffer.IsSingleSegment) @@ -595,34 +516,6 @@ private int CalculateChunkSize(int extraHexDigit, int currentParsedSize) return -1; // can't happen, but compiler complains } - private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) - { - // The only difference is which reader to use. Let's do the following. - // Make an internal reader that will always be used for whatever operation is needed here - // Keep external one the same always. - var readAwaitable = _requestBodyPipe.Reader.ReadAsync(cancellationToken); - - if (!readAwaitable.IsCompleted && _timingEnabled) - { - _backpressure = true; - _context.TimeoutControl.StartTimingRead(); - } - - return readAwaitable; - } - - private void StopTimingRead(long bytesRead) - { - _context.TimeoutControl.BytesRead(bytesRead - _alreadyTimedBytes); - _alreadyTimedBytes = 0; - - if (_backpressure) - { - _backpressure = false; - _context.TimeoutControl.StopTimingRead(); - } - } - private enum Mode { Prefix, diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs index 0658162fc6e1..a44bb1f01fc2 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs @@ -45,7 +45,7 @@ public override async ValueTask ReadAsync(CancellationToken cancella // We internally track an int for that. while (true) { - // This isn't great. The issue is that TryRead can get a canceled read result + // _context.RequestTimedOut The issue is that TryRead can get a canceled read result // which is unknown to StartTimingReadAsync. if (_context.RequestTimedOut) { @@ -54,7 +54,8 @@ public override async ValueTask ReadAsync(CancellationToken cancella try { - _readResult = await StartTimingReadAsync(cancellationToken); + var readAwaitable = _context.Input.ReadAsync(cancellationToken); + _readResult = await StartTimingReadAsync(readAwaitable, cancellationToken); } catch (ConnectionAbortedException ex) { @@ -63,43 +64,39 @@ public override async ValueTask ReadAsync(CancellationToken cancella if (_context.RequestTimedOut) { - Debug.Assert(_readResult.IsCanceled); BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTimeout); } + // Make sure to handle when this is canceled here. if (_readResult.IsCanceled) { - if (Interlocked.CompareExchange(ref _userCanceled, 0, 1) == 1) + if (Interlocked.Exchange(ref _userCanceled, 0) == 1) { // Ignore the readResult if it wasn't by the user. break; } + else + { + // TODO should this reset the timing read? + StopTimingRead(0); + continue; + } } var readableBuffer = _readResult.Buffer; var readableBufferLength = readableBuffer.Length; StopTimingRead(readableBufferLength); - if (_readResult.IsCompleted) - { - // OnInputOrOutputCompleted() is an idempotent method that closes the connection. Sometimes - // input completion is observed here before the Input.OnWriterCompleted() callback is fired, - // so we call OnInputOrOutputCompleted() now to prevent a race in our tests where a 400 - // response is written after observing the unexpected end of request content instead of just - // closing the connection without a response as expected. - _context.OnInputOrOutputCompleted(); - - BadHttpRequestException.Throw(RequestRejectionReason.UnexpectedEndOfRequestContent); - } + CheckCompletedReadResult(_readResult); if (readableBufferLength > 0) { + CreateReadResultFromConnectionReadResult(); + break; } } - CreateReadResultFromConnectionReadResult(); - return _readResult; } @@ -115,13 +112,27 @@ public override bool TryRead(out ReadResult readResult) TryStart(); - var boolResult = _context.Input.TryRead(out _readResult); + if (!_context.Input.TryRead(out _readResult)) + { + readResult = default; + return false; + } + + if (_readResult.IsCanceled) + { + if (Interlocked.Exchange(ref _userCanceled, 0) == 0) + { + // Cancellation wasn't by the user, return default ReadResult + readResult = default; + return false; + } + } CreateReadResultFromConnectionReadResult(); readResult = _readResult; - return boolResult; + return true; } private void ThrowIfCompleted() @@ -178,33 +189,6 @@ protected override void OnReadStarting() } } - private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) - { - var readAwaitable = _context.Input.ReadAsync(cancellationToken); - - if (!readAwaitable.IsCompleted && _timingEnabled) - { - TryProduceContinue(); - - _backpressure = true; - _context.TimeoutControl.StartTimingRead(); - } - - return readAwaitable; - } - - private void StopTimingRead(long bytesRead) - { - _context.TimeoutControl.BytesRead(bytesRead - _alreadyTimedBytes); - _alreadyTimedBytes = 0; - - if (_backpressure) - { - _backpressure = false; - _context.TimeoutControl.StopTimingRead(); - } - } - public override void Complete(Exception exception) { _context.ReportApplicationError(exception); @@ -227,76 +211,5 @@ protected override Task OnStopAsync() Complete(null); return Task.CompletedTask; } - - protected override Task OnConsumeAsync() - { - try - { - if (TryRead(out var readResult)) - { - AdvanceTo(readResult.Buffer.End); - - if (readResult.IsCompleted) - { - return Task.CompletedTask; - } - } - } - catch (BadHttpRequestException ex) - { - // At this point, the response has already been written, so this won't result in a 4XX response; - // however, we still need to stop the request processing loop and log. - _context.SetBadRequestState(ex); - return Task.CompletedTask; - } - catch (InvalidOperationException ex) - { - var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); - _context.ReportApplicationError(connectionAbortedException); - - // Have to abort the connection because we can't finish draining the request - _context.StopProcessingNextRequest(); - return Task.CompletedTask; - } - - return OnConsumeAsyncAwaited(); - } - - private async Task OnConsumeAsyncAwaited() - { - Log.RequestBodyNotEntirelyRead(_context.ConnectionIdFeature, _context.TraceIdentifier); - - _context.TimeoutControl.SetTimeout(Constants.RequestBodyDrainTimeout.Ticks, TimeoutReason.RequestBodyDrain); - - try - { - ReadResult result; - do - { - result = await ReadAsync(); - AdvanceTo(result.Buffer.End); - } while (!result.IsCompleted); - } - catch (BadHttpRequestException ex) - { - _context.SetBadRequestState(ex); - } - catch (ConnectionAbortedException) - { - Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); - } - catch (InvalidOperationException ex) - { - var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); - _context.ReportApplicationError(connectionAbortedException); - - // Have to abort the connection because we can't finish draining the request - _context.StopProcessingNextRequest(); - } - finally - { - _context.TimeoutControl.CancelTimeout(); - } - } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs index 7f60c6805a55..3f6296d71535 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs @@ -1,6 +1,12 @@ // Copyright (c) .NET Foundation. All rights reserved. // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. +using System; +using System.IO.Pipelines; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; + namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { public abstract class Http1MessageBody : MessageBody @@ -13,6 +19,96 @@ protected Http1MessageBody(Http1Connection context) _context = context; } + protected void CheckCompletedReadResult(ReadResult result) + { + if (result.IsCompleted) + { + // OnInputOrOutputCompleted() is an idempotent method that closes the connection. Sometimes + // input completion is observed here before the Input.OnWriterCompleted() callback is fired, + // so we call OnInputOrOutputCompleted() now to prevent a race in our tests where a 400 + // response is written after observing the unexpected end of request content instead of just + // closing the connection without a response as expected. + _context.OnInputOrOutputCompleted(); + + BadHttpRequestException.Throw(RequestRejectionReason.UnexpectedEndOfRequestContent); + } + } + + protected override Task OnConsumeAsync() + { + try + { + if (TryRead(out var readResult)) + { + AdvanceTo(readResult.Buffer.End); + + if (readResult.IsCompleted) + { + return Task.CompletedTask; + } + } + } + catch (BadHttpRequestException ex) + { + // At this point, the response has already been written, so this won't result in a 4XX response; + // however, we still need to stop the request processing loop and log. + _context.SetBadRequestState(ex); + return Task.CompletedTask; + } + catch (ConnectionAbortedException) + { + Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); + } + catch (InvalidOperationException ex) + { + var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); + _context.ReportApplicationError(connectionAbortedException); + + // Have to abort the connection because we can't finish draining the request + _context.StopProcessingNextRequest(); + return Task.CompletedTask; + } + + return OnConsumeAsyncAwaited(); + } + + protected async Task OnConsumeAsyncAwaited() + { + Log.RequestBodyNotEntirelyRead(_context.ConnectionIdFeature, _context.TraceIdentifier); + + _context.TimeoutControl.SetTimeout(Constants.RequestBodyDrainTimeout.Ticks, TimeoutReason.RequestBodyDrain); + + try + { + ReadResult result; + do + { + result = await ReadAsync(); + AdvanceTo(result.Buffer.End); + } while (!result.IsCompleted); + } + catch (BadHttpRequestException ex) + { + _context.SetBadRequestState(ex); + } + catch (ConnectionAbortedException) + { + Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); + } + catch (InvalidOperationException ex) + { + var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); + _context.ReportApplicationError(connectionAbortedException); + + // Have to abort the connection because we can't finish draining the request + _context.StopProcessingNextRequest(); + } + finally + { + _context.TimeoutControl.CancelTimeout(); + } + } + public static MessageBody For( HttpVersion httpVersion, HttpRequestHeaders headers, diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs index 751db24ac44c..9be0b8a6dc2d 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.FeatureCollection.cs @@ -106,7 +106,7 @@ Stream IHttpRequestFeature.Body minimumSegmentSize: KestrelMemoryPool.MinimumSegmentSize, minimumReadThreshold: KestrelMemoryPool.MinimumSegmentSize / 4, _context.MemoryPool)); - RequestBodyPipe = requestPipeReader; + RequestBodyPipeReader = requestPipeReader; // The StreamPipeWrapper needs to be disposed as it hold onto blocks of memory if (_wrapperObjectsToDispose == null) @@ -121,12 +121,12 @@ PipeReader IRequestBodyPipeFeature.RequestBodyPipe { get { - return RequestBodyPipe; + return RequestBodyPipeReader; } set { - RequestBodyPipe = value; - RequestBody = new ReadOnlyPipeStream(RequestBodyPipe); + RequestBodyPipeReader = value; + RequestBody = new ReadOnlyPipeStream(RequestBodyPipeReader); } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs index 5ce3afd57c31..724aa8514ecc 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/HttpProtocol.cs @@ -190,7 +190,7 @@ private void HttpVersionSetSlow(string value) public IHeaderDictionary RequestHeaders { get; set; } public Stream RequestBody { get; set; } - public PipeReader RequestBodyPipe { get; set; } + public PipeReader RequestBodyPipeReader { get; set; } private int _statusCode; public int StatusCode @@ -298,7 +298,7 @@ public void InitializeBodyControl(MessageBody messageBody) bodyControl = new BodyControl(bodyControl: this, this); } - (RequestBody, ResponseBody, RequestBodyPipe, ResponsePipeWriter) = bodyControl.Start(messageBody); + (RequestBody, ResponseBody, RequestBodyPipeReader, ResponsePipeWriter) = bodyControl.Start(messageBody); } public void StopBodies() => bodyControl.Stop(); diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs index c45258dee56b..159daffcd085 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/MessageBody.cs @@ -154,5 +154,31 @@ protected void AddAndCheckConsumedBytes(long consumedBytes) BadHttpRequestException.Throw(RequestRejectionReason.RequestBodyTooLarge); } } + + protected ValueTask StartTimingReadAsync(ValueTask readAwaitable, CancellationToken cancellationToken) + { + + if (!readAwaitable.IsCompleted && _timingEnabled) + { + TryProduceContinue(); + + _backpressure = true; + _context.TimeoutControl.StartTimingRead(); + } + + return readAwaitable; + } + + protected void StopTimingRead(long bytesRead) + { + _context.TimeoutControl.BytesRead(bytesRead - _alreadyTimedBytes); + _alreadyTimedBytes = 0; + + if (_backpressure) + { + _backpressure = false; + _context.TimeoutControl.StopTimingRead(); + } + } } } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs index 1fa4a49f17d3..4dcf7faa9ff9 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs @@ -64,13 +64,13 @@ public override void AdvanceTo(SequencePosition consumed) public override void AdvanceTo(SequencePosition consumed, SequencePosition examined) { var dataLength = _readResult.Buffer.Slice(_readResult.Buffer.Start, consumed).Length; - _context.RequestPipe.Reader.AdvanceTo(consumed, examined); + _context.RequestBodyPipe.Reader.AdvanceTo(consumed, examined); OnDataRead(dataLength); } public override bool TryRead(out ReadResult readResult) { - return _context.RequestPipe.Reader.TryRead(out readResult); + return _context.RequestBodyPipe.Reader.TryRead(out readResult); } public override async ValueTask ReadAsync(CancellationToken cancellationToken = default) @@ -79,7 +79,9 @@ public override async ValueTask ReadAsync(CancellationToken cancella try { - _readResult = await StartTimingReadAsync(cancellationToken); + var readAwaitable = _context.RequestBodyPipe.Reader.ReadAsync(cancellationToken); + + _readResult = await StartTimingReadAsync(readAwaitable, cancellationToken); } catch (ConnectionAbortedException ex) { @@ -96,44 +98,19 @@ public override async ValueTask ReadAsync(CancellationToken cancella return _readResult; } - private ValueTask StartTimingReadAsync(CancellationToken cancellationToken) - { - var readAwaitable = _context.RequestPipe.Reader.ReadAsync(cancellationToken); - - if (!readAwaitable.IsCompleted && _timingEnabled) - { - _backpressure = true; - _context.TimeoutControl.StartTimingRead(); - } - - return readAwaitable; - } - - private void StopTimingRead(long bytesRead) - { - _context.TimeoutControl.BytesRead(bytesRead - _alreadyTimedBytes); - _alreadyTimedBytes = 0; - - if (_backpressure) - { - _backpressure = false; - _context.TimeoutControl.StopTimingRead(); - } - } - public override void Complete(Exception exception) { - _context.RequestPipe.Reader.Complete(exception); + _context.RequestBodyPipe.Reader.Complete(exception); } public override void OnWriterCompleted(Action callback, object state) { - _context.RequestPipe.Reader.OnWriterCompleted(callback, state); + _context.RequestBodyPipe.Reader.OnWriterCompleted(callback, state); } public override void CancelPendingRead() { - _context.RequestPipe.Reader.CancelPendingRead(); + _context.RequestBodyPipe.Reader.CancelPendingRead(); } protected override Task OnStopAsync() @@ -143,7 +120,7 @@ protected override Task OnStopAsync() return Task.CompletedTask; } - _context.RequestPipe.Reader.Complete(); + _context.RequestBodyPipe.Reader.Complete(); return Task.CompletedTask; } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs index e0b9934db98e..e2a5b3b83d2e 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs @@ -25,7 +25,7 @@ public abstract partial class Http2Stream : HttpProtocol, IThreadPoolWorkItem private readonly StreamInputFlowControl _inputFlowControl; private readonly StreamOutputFlowControl _outputFlowControl; - public Pipe RequestPipe { get; } + public Pipe RequestBodyPipe { get; } internal long DrainExpirationTicks { get; set; } @@ -57,7 +57,7 @@ public Http2Stream(Http2StreamContext context) this, context.ServiceContext.Log); - RequestPipe = CreateRequestBodyPipe(context.ServerPeerSettings.InitialWindowSize); + RequestBodyPipe = CreateRequestBodyPipe(context.ServerPeerSettings.InitialWindowSize); Output = _http2Output; } @@ -101,13 +101,13 @@ protected override void OnRequestProcessingEnded() { // Don't block on IO. This never faults. _ = _http2Output.WriteRstStreamAsync(Http2ErrorCode.NO_ERROR); - RequestPipe.Writer.Complete(); + RequestBodyPipe.Writer.Complete(); } } _http2Output.Dispose(); - RequestPipe.Reader.Complete(); + RequestBodyPipe.Reader.Complete(); // The app can no longer read any more of the request body, so return any bytes that weren't read to the // connection's flow-control window. @@ -366,9 +366,9 @@ public Task OnDataAsync(Http2Frame dataFrame, ReadOnlySequence payload) { foreach (var segment in dataPayload) { - RequestPipe.Writer.Write(segment.Span); + RequestBodyPipe.Writer.Write(segment.Span); } - var flushTask = RequestPipe.Writer.FlushAsync(); + var flushTask = RequestBodyPipe.Writer.FlushAsync(); // It shouldn't be possible for the RequestBodyPipe to fill up an return an incomplete task if // _inputFlowControl.Advance() didn't throw. @@ -398,7 +398,7 @@ public void OnEndStreamReceived() } } - RequestPipe.Writer.Complete(); + RequestBodyPipe.Writer.Complete(); _inputFlowControl.StopWindowUpdates(); } @@ -472,7 +472,7 @@ private void AbortCore(Exception abortReason) // Unblock the request body. PoisonRequestBodyStream(abortReason); - RequestPipe.Writer.Complete(abortReason); + RequestBodyPipe.Writer.Complete(abortReason); _inputFlowControl.Abort(); } From c9ac10d111578bb6c0da08770fb7bac09dc21107 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Tue, 19 Feb 2019 14:11:57 -0800 Subject: [PATCH 25/29] nits and bad merge conflict resolution --- .../Http/Http1ContentLengthMessageBody.cs | 2 +- .../InMemory.FunctionalTests/RequestTests.cs | 35 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs index a44bb1f01fc2..cc46903fc088 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs @@ -45,7 +45,7 @@ public override async ValueTask ReadAsync(CancellationToken cancella // We internally track an int for that. while (true) { - // _context.RequestTimedOut The issue is that TryRead can get a canceled read result + // The issue is that TryRead can get a canceled read result // which is unknown to StartTimingReadAsync. if (_context.RequestTimedOut) { diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs index 99257710f265..a28fb1cabdae 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/RequestTests.cs @@ -18,7 +18,6 @@ using Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests.TestTransport; using Microsoft.AspNetCore.Testing; using Microsoft.Extensions.Logging.Testing; -using Moq; using Xunit; namespace Microsoft.AspNetCore.Server.Kestrel.InMemory.FunctionalTests @@ -731,9 +730,11 @@ await connection.ReceiveEnd( [Fact] public async Task ConnectionClosesWhenFinReceivedBeforeRequestCompletes() { - var testContext = new TestServiceContext(LoggerFactory); - // FIN callbacks are scheduled so run inline to make this test more reliable - testContext.Scheduler = PipeScheduler.Inline; + var testContext = new TestServiceContext(LoggerFactory) + { + // FIN callbacks are scheduled so run inline to make this test more reliable + Scheduler = PipeScheduler.Inline + }; using (var server = new TestServer(TestApp.EchoAppChunked, testContext)) { @@ -1317,7 +1318,24 @@ public async Task SynchronousReadsCanBeAllowedGlobally() Assert.Equal(0, await context.Request.Body.ReadAsync(new byte[1], 0, 1)); Assert.Equal("Hello", Encoding.ASCII.GetString(buffer, 0, 5)); - })); + }, testContext)) + { + using (var connection = server.CreateConnection()) + { + await connection.Send( + "POST / HTTP/1.1", + "Host:", + "Content-Length: 5", + "", + "Hello"); + await connection.Receive( + "HTTP/1.1 200 OK", + $"Date: {server.Context.DateHeaderValue}", + "Content-Length: 0", + "", + ""); + } + } } [Fact] @@ -1394,6 +1412,7 @@ public async Task ContentLengthRequestCallCompleteThrowsExceptionOnRead() response.Headers["Content-Length"] = new[] { "11" }; await response.BodyPipe.WriteAsync(new Memory(Encoding.ASCII.GetBytes("Hello World"), 0, 11)); + }, testContext)) { using (var connection = server.CreateConnection()) @@ -1404,12 +1423,6 @@ await connection.Send( "Content-Length: 5", "", "Hello"); - await connection.Receive( - "HTTP/1.1 200 OK", - $"Date: {server.Context.DateHeaderValue}", - "Content-Length: 0", - "", - ""); await connection.Receive( "HTTP/1.1 200 OK", From 283396a7fa6a8afeb9fd6a6bcd150f52fff43086 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 21 Feb 2019 16:17:27 -0800 Subject: [PATCH 26/29] Log message --- .../Core/src/Internal/Http/Http1ContentLengthMessageBody.cs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs index cc46903fc088..2fa2d221ed6c 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs @@ -2,12 +2,10 @@ // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. using System; -using System.Diagnostics; using System.IO.Pipelines; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Connections; -using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http { @@ -59,6 +57,7 @@ public override async ValueTask ReadAsync(CancellationToken cancella } catch (ConnectionAbortedException ex) { + Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); throw new TaskCanceledException("The request was aborted", ex); } @@ -77,7 +76,7 @@ public override async ValueTask ReadAsync(CancellationToken cancella } else { - // TODO should this reset the timing read? + // Reset the timing read here for the next call to read. StopTimingRead(0); continue; } From c1799e173f02590cedeb5532d4990f89228c6cb8 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 21 Feb 2019 16:24:39 -0800 Subject: [PATCH 27/29] Correct fix --- .../Core/src/Internal/Http/Http1ContentLengthMessageBody.cs | 1 - .../Kestrel/Core/src/Internal/Http/Http1MessageBody.cs | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs index 2fa2d221ed6c..289ecac406f2 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ContentLengthMessageBody.cs @@ -57,7 +57,6 @@ public override async ValueTask ReadAsync(CancellationToken cancella } catch (ConnectionAbortedException ex) { - Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); throw new TaskCanceledException("The request was aborted", ex); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs index 3f6296d71535..71c346897081 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1MessageBody.cs @@ -55,10 +55,6 @@ protected override Task OnConsumeAsync() _context.SetBadRequestState(ex); return Task.CompletedTask; } - catch (ConnectionAbortedException) - { - Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); - } catch (InvalidOperationException ex) { var connectionAbortedException = new ConnectionAbortedException(CoreStrings.ConnectionAbortedByApplication, ex); @@ -91,7 +87,7 @@ protected async Task OnConsumeAsyncAwaited() { _context.SetBadRequestState(ex); } - catch (ConnectionAbortedException) + catch (OperationCanceledException ex) when (ex is ConnectionAbortedException || ex is TaskCanceledException) { Log.RequestBodyDrainTimedOut(_context.ConnectionIdFeature, _context.TraceIdentifier); } From 749537e24fc4eb15e65bb03ba0a2c499fcd9b9c6 Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 21 Feb 2019 17:10:39 -0800 Subject: [PATCH 28/29] Don't complete the reader in Complete --- .../Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs index 81b123161ee8..aa9d0dda66bd 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs @@ -95,7 +95,6 @@ public override async ValueTask ReadAsync(CancellationToken cancella public override void Complete(Exception exception) { - _requestBodyPipe.Reader.Complete(exception); _context.ReportApplicationError(exception); } From 9f97ab5f218e1e328c0ec285ab14a217979ef6ad Mon Sep 17 00:00:00 2001 From: Justin Kotalik Date: Thu, 21 Feb 2019 18:24:17 -0800 Subject: [PATCH 29/29] Misread feedback --- .../Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs | 1 + .../Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs index aa9d0dda66bd..963fd5d85c84 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http/Http1ChunkedEncodingMessageBody.cs @@ -95,6 +95,7 @@ public override async ValueTask ReadAsync(CancellationToken cancella public override void Complete(Exception exception) { + _requestBodyPipe.Reader.Complete(); _context.ReportApplicationError(exception); } diff --git a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs index 4dcf7faa9ff9..cba9b491b221 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http2/Http2MessageBody.cs @@ -100,7 +100,8 @@ public override async ValueTask ReadAsync(CancellationToken cancella public override void Complete(Exception exception) { - _context.RequestBodyPipe.Reader.Complete(exception); + _context.RequestBodyPipe.Reader.Complete(); + _context.ReportApplicationError(exception); } public override void OnWriterCompleted(Action callback, object state)