WinHttpHandler: Add support for bidirectional streaming

src/libraries/Common/src/Interop/Windows/WinHttp/Interop.winhttp_types.cs

@@ -61,6 +61,7 @@ internal static partial class WinHttp
 
         public const uint WINHTTP_FLAG_SECURE = 0x00800000;
         public const uint WINHTTP_FLAG_ESCAPE_DISABLE = 0x00000040;
+        public const uint WINHTTP_FLAG_AUTOMATIC_CHUNKING = 0x00000200;
 
         public const uint WINHTTP_QUERY_FLAG_NUMBER = 0x20000000;
         public const uint WINHTTP_QUERY_VERSION = 18;

src/libraries/Common/tests/System/Net/Http/Http2LoopbackConnection.cs

@@ -100,6 +100,9 @@ private async Task ReadPrefixAsync()
                 // The contents of what we send don't really matter, as long as it is interpreted by SocketsHttpHandler as an invalid response.
                 await _connectionStream.WriteAsync(Encoding.ASCII.GetBytes("HTTP/2.0 400 Bad Request\r\n\r\n"));
                 _connectionSocket.Shutdown(SocketShutdown.Send);
+                // If WinHTTP doesn't support streaming a request without a length then it will fallback
+                // to HTTP/1.1. Throwing an exception to detect this case in WinHttpHandler tests. 
+                throw new Exception("HTTP/1.1 request sent to HTTP/2 connection.");
             }
         }
 
@@ -347,13 +350,13 @@ public async Task ShutdownIgnoringErrorsAsync(int lastStreamId, ProtocolErrors e
             }
         }
 
-        public async Task<int> ReadRequestHeaderAsync()
+        public async Task<int> ReadRequestHeaderAsync(bool expectEndOfStream = true)
         {
-            HeadersFrame frame = await ReadRequestHeaderFrameAsync();
+            HeadersFrame frame = await ReadRequestHeaderFrameAsync(expectEndOfStream);
             return frame.StreamId;
         }
 
-        public async Task<HeadersFrame> ReadRequestHeaderFrameAsync()
+        public async Task<HeadersFrame> ReadRequestHeaderFrameAsync(bool expectEndOfStream = true)
         {
             // Receive HEADERS frame for request.
             Frame frame = await ReadFrameAsync(_timeout).ConfigureAwait(false);
@@ -363,10 +366,27 @@ public async Task<HeadersFrame> ReadRequestHeaderFrameAsync()
             }
 
             Assert.Equal(FrameType.Headers, frame.Type);
-            Assert.Equal(FrameFlags.EndHeaders | FrameFlags.EndStream, frame.Flags);
+            Assert.Equal(FrameFlags.EndHeaders, frame.Flags & FrameFlags.EndHeaders);
+            if (expectEndOfStream)
+            {
+                Assert.Equal(FrameFlags.EndStream, frame.Flags & FrameFlags.EndStream);
+            }
             return (HeadersFrame)frame;
         }
 
+        public async Task<Frame> ReadDataFrameAsync()
+        {
+            // Receive DATA frame for request.
+            Frame frame = await ReadFrameAsync(_timeout).ConfigureAwait(false);
+            if (frame == null)
+            {
+                throw new IOException("Failed to read Data frame.");
+            }
+
+            Assert.Equal(FrameType.Data, frame.Type);
+            return frame;
+        }
+
         private static (int bytesConsumed, int value) DecodeInteger(ReadOnlySpan<byte> headerBlock, byte prefixMask)
         {
             return QPackTestDecoder.DecodeInteger(headerBlock, prefixMask);
@@ -729,11 +749,16 @@ public async Task SendPingAckAsync(long payload)
             await WriteFrameAsync(pingAck).ConfigureAwait(false);
         }
 
-        public async Task SendDefaultResponseHeadersAsync(int streamId)
+        public async Task SendDefaultResponseHeadersAsync(int streamId, bool endStream = false)
         {
             byte[] headers = new byte[] { 0x88 };   // Encoding for ":status: 200"
+            FrameFlags flags = FrameFlags.EndHeaders;
+            if (endStream)
+            {
+                flags = flags | FrameFlags.EndStream;
+            }
 
-            HeadersFrame headersFrame = new HeadersFrame(headers, FrameFlags.EndHeaders, 0, 0, 0, streamId);
+            HeadersFrame headersFrame = new HeadersFrame(headers, flags, 0, 0, 0, streamId);
             await WriteFrameAsync(headersFrame).ConfigureAwait(false);
         }
 

src/libraries/System.Net.Http.WinHttpHandler/src/System.Net.Http.WinHttpHandler.csproj

@@ -82,6 +82,7 @@
     <Compile Include="System\Net\Http\WinHttpAuthHelper.cs" />
     <Compile Include="System\Net\Http\WinHttpCertificateHelper.cs" />
     <Compile Include="System\Net\Http\WinHttpChannelBinding.cs" />
+    <Compile Include="System\Net\Http\WinHttpChunkMode.cs" />
     <Compile Include="System\Net\Http\WinHttpCookieContainerAdapter.cs" />
     <Compile Include="System\Net\Http\WinHttpException.cs" />
     <Compile Include="System\Net\Http\WinHttpHandler.cs" />

src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpChunkMode.cs

@@ -0,0 +1,12 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+namespace System.Net.Http
+{
+    internal enum WinHttpChunkMode
+    {
+        None,
+        Manual,
+        Automatic
+    }
+}

src/libraries/System.Net.Http.WinHttpHandler/src/System/Net/Http/WinHttpHandler.cs

@@ -628,17 +628,22 @@ protected override Task<HttpResponseMessage> SendAsync(
             return tcs.Task;
         }
 
-        private static bool IsChunkedModeForSend(HttpRequestMessage requestMessage)
+        private static WinHttpChunkMode GetChunkedModeForSend(HttpRequestMessage requestMessage)
         {
-            bool chunkedMode = requestMessage.Headers.TransferEncodingChunked.HasValue &&
-                requestMessage.Headers.TransferEncodingChunked.Value;
+            WinHttpChunkMode chunkedMode = WinHttpChunkMode.None;
+
+            if (requestMessage.Headers.TransferEncodingChunked.HasValue &&
+                requestMessage.Headers.TransferEncodingChunked.Value)
+            {
+                chunkedMode = WinHttpChunkMode.Manual;
+            }
 
             HttpContent requestContent = requestMessage.Content;
             if (requestContent != null)
             {
                 if (requestContent.Headers.ContentLength.HasValue)
                 {
-                    if (chunkedMode)
+                    if (chunkedMode == WinHttpChunkMode.Manual)
                     {
                         // Deal with conflict between 'Content-Length' vs. 'Transfer-Encoding: chunked' semantics.
                         // Current .NET Desktop HttpClientHandler allows both headers to be specified but ends up
@@ -649,19 +654,28 @@ private static bool IsChunkedModeForSend(HttpRequestMessage requestMessage)
                 }
                 else
                 {
-                    if (!chunkedMode)
+                    if (chunkedMode == WinHttpChunkMode.None)
                     {
                         // Neither 'Content-Length' nor 'Transfer-Encoding: chunked' semantics was given.
-                        // Current .NET Desktop HttpClientHandler uses 'Content-Length' semantics and
-                        // buffers the content as well in some cases.  But the WinHttpHandler can't access
-                        // the protected internal TryComputeLength() method of the content.  So, it
-                        // will use'Transfer-Encoding: chunked' semantics.
-                        chunkedMode = true;
-                        requestMessage.Headers.TransferEncodingChunked = true;
+
+                        if (requestMessage.Version >= HttpVersion20)
+                        {
+                            // HTTP/2 supports automatic chunking (streaming the request body without a length).
+                            chunkedMode = WinHttpChunkMode.Automatic;
+                        }
+                        else
+                        {
+                            // Current .NET Desktop HttpClientHandler uses 'Content-Length' semantics and
+                            // buffers the content as well in some cases.  But the WinHttpHandler can't access
+                            // the protected internal TryComputeLength() method of the content.  So, it
+                            // will use 'Transfer-Encoding: chunked' semantics.
+                            chunkedMode = WinHttpChunkMode.Manual;
+                            requestMessage.Headers.TransferEncodingChunked = true;
+                        }
                     }
                 }
             }
-            else if (chunkedMode)
+            else if (chunkedMode == WinHttpChunkMode.Manual)
             {
                 throw new InvalidOperationException(SR.net_http_chunked_not_allowed_with_empty_content);
             }
@@ -860,6 +874,7 @@ private async Task StartRequestAsync(WinHttpRequestState state)
                 return;
             }
 
+            Task sendRequestBodyTask = null;
             SafeWinHttpHandle connectHandle = null;
             try
             {
@@ -888,25 +903,8 @@ private async Task StartRequestAsync(WinHttpRequestState state)
                     httpVersion = "HTTP/1.1";
                 }
 
-                // Turn off additional URI reserved character escaping (percent-encoding). This matches
-                // .NET Framework behavior. System.Uri establishes the baseline rules for percent-encoding
-                // of reserved characters.
-                uint flags = Interop.WinHttp.WINHTTP_FLAG_ESCAPE_DISABLE;
-                if (state.RequestMessage.RequestUri.Scheme == UriScheme.Https)
-                {
-                    flags |= Interop.WinHttp.WINHTTP_FLAG_SECURE;
-                }
-
-                // Create an HTTP request handle.
-                state.RequestHandle = Interop.WinHttp.WinHttpOpenRequest(
-                    connectHandle,
-                    state.RequestMessage.Method.Method,
-                    state.RequestMessage.RequestUri.PathAndQuery,
-                    httpVersion,
-                    Interop.WinHttp.WINHTTP_NO_REFERER,
-                    Interop.WinHttp.WINHTTP_DEFAULT_ACCEPT_TYPES,
-                    flags);
-                ThrowOnInvalidHandle(state.RequestHandle, nameof(Interop.WinHttp.WinHttpOpenRequest));
+                OpenRequestHandle(state, connectHandle, httpVersion, out WinHttpChunkMode chunkedModeForSend, out SafeWinHttpHandle requestHandle);
+                state.RequestHandle = requestHandle;
                 state.RequestHandle.SetParentHandle(connectHandle);
 
                 // Set callback function.
@@ -915,8 +913,6 @@ private async Task StartRequestAsync(WinHttpRequestState state)
                 // Set needed options on the request handle.
                 SetRequestHandleOptions(state);
 
-                bool chunkedModeForSend = IsChunkedModeForSend(state.RequestMessage);
-
                 AddRequestHeaders(
                     state.RequestHandle,
                     state.RequestMessage,
@@ -941,12 +937,36 @@ private async Task StartRequestAsync(WinHttpRequestState state)
 
                         await InternalSendRequestAsync(state);
 
-                        if (state.RequestMessage.Content != null)
+                        RendezvousAwaitable<int> receivedResponseTask;
+
+                        if (chunkedModeForSend == WinHttpChunkMode.Automatic)
                         {
-                            await InternalSendRequestBodyAsync(state, chunkedModeForSend).ConfigureAwait(false);
+                            // Start waiting to receive response headers before sending request body.
+                            // This order is important because the response could be returned immediately
+                            // with END_STREAM flag on headers. Trying to send request body after that
+                            // can cause the request to go into a bad state.
+                            //
+                            // We only use this order if chunk mode is automatic because Windows versions
+                            // prior to AUTOMATIC_CHUNKING didn't support it.
+                            receivedResponseTask = InternalReceiveResponseHeadersAsync(state);
+
+                            if (state.RequestMessage.Content != null)
+                            {
+                                sendRequestBodyTask = InternalSendRequestBodyAsync(state, chunkedModeForSend);
+                            }
+                        }
+                        else
+                        {
+                            if (state.RequestMessage.Content != null)
+                            {
+                                sendRequestBodyTask = InternalSendRequestBodyAsync(state, chunkedModeForSend);
+                                await sendRequestBodyTask.ConfigureAwait(false);
+                            }
+
+                            receivedResponseTask = InternalReceiveResponseHeadersAsync(state);
                         }
 
-                        bool receivedResponse = await InternalReceiveResponseHeadersAsync(state) != 0;
+                        bool receivedResponse = await receivedResponseTask != 0;
                         if (receivedResponse)
                         {
                             // If we're manually handling cookies, we need to add them to the container after
@@ -995,7 +1015,80 @@ private async Task StartRequestAsync(WinHttpRequestState state)
             finally
             {
                 SafeWinHttpHandle.DisposeAndClearHandle(ref connectHandle);
-                state.ClearSendRequestState();
+
+                try
+                {
+                    // Wait for request body to finish sending.
+                    if (sendRequestBodyTask != null)
+                    {
+                        await sendRequestBodyTask.ConfigureAwait(false);
+                    }
+                }
+                finally
+                {
+                    state.ClearSendRequestState();
+                }
+            }
+        }
+
+        private void OpenRequestHandle(WinHttpRequestState state, SafeWinHttpHandle connectHandle, string httpVersion, out WinHttpChunkMode chunkedModeForSend, out SafeWinHttpHandle requestHandle)
+        {
+            chunkedModeForSend = GetChunkedModeForSend(state.RequestMessage);
+
+            // Create an HTTP request handle.
+            requestHandle = Interop.WinHttp.WinHttpOpenRequest(
+                connectHandle,
+                state.RequestMessage.Method.Method,
+                state.RequestMessage.RequestUri.PathAndQuery,
+                httpVersion,
+                Interop.WinHttp.WINHTTP_NO_REFERER,
+                Interop.WinHttp.WINHTTP_DEFAULT_ACCEPT_TYPES,
+                GetRequestFlags(state, chunkedModeForSend));
+
+            // It is possible the request was made with the WINHTTP_FLAG_AUTOMATIC_CHUNKING flag
+            // and the platform doesn't support that flag.
+            if (requestHandle.IsInvalid)
+            {
+                int lastError = Marshal.GetLastWin32Error();
+                if (NetEventSource.Log.IsEnabled()) NetEventSource.Error(this, $"error={lastError}");
+                if (lastError != Interop.WinHttp.ERROR_INVALID_PARAMETER || chunkedModeForSend != WinHttpChunkMode.Automatic)
+                {
+                    ThrowOnInvalidHandle(requestHandle, nameof(Interop.WinHttp.WinHttpOpenRequest));
+                }
+
+                // Platform doesn't support WINHTTP_FLAG_AUTOMATIC_CHUNKING. Revert to manual chunking.
+                // Note that manual chunking with WinHttp downgrades HTTP/2 requests to HTTP/1.1.
+                chunkedModeForSend = WinHttpChunkMode.Manual;
+                state.RequestMessage.Headers.TransferEncodingChunked = true;
+
+                requestHandle = Interop.WinHttp.WinHttpOpenRequest(
+                    connectHandle,
+                    state.RequestMessage.Method.Method,
+                    state.RequestMessage.RequestUri.PathAndQuery,
+                    httpVersion,
+                    Interop.WinHttp.WINHTTP_NO_REFERER,
+                    Interop.WinHttp.WINHTTP_DEFAULT_ACCEPT_TYPES,
+                    GetRequestFlags(state, chunkedModeForSend));
+
+                ThrowOnInvalidHandle(requestHandle, nameof(Interop.WinHttp.WinHttpOpenRequest));
+            }
+
+            static uint GetRequestFlags(WinHttpRequestState state, WinHttpChunkMode chunkedModeForSend)
+            {
+                // Turn off additional URI reserved character escaping (percent-encoding). This matches
+                // .NET Framework behavior. System.Uri establishes the baseline rules for percent-encoding
+                // of reserved characters.
+                uint flags = Interop.WinHttp.WINHTTP_FLAG_ESCAPE_DISABLE;
+                if (state.RequestMessage.RequestUri.Scheme == UriScheme.Https)
+                {
+                    flags |= Interop.WinHttp.WINHTTP_FLAG_SECURE;
+                }
+                if (chunkedModeForSend == WinHttpChunkMode.Automatic)
+                {
+                    flags |= Interop.WinHttp.WINHTTP_FLAG_AUTOMATIC_CHUNKING;
+                }
+
+                return flags;
             }
         }
 
@@ -1527,7 +1620,7 @@ private RendezvousAwaitable<int> InternalSendRequestAsync(WinHttpRequestState st
             return state.LifecycleAwaitable;
         }
 
-        private async Task InternalSendRequestBodyAsync(WinHttpRequestState state, bool chunkedModeForSend)
+        private async Task InternalSendRequestBodyAsync(WinHttpRequestState state, WinHttpChunkMode chunkedModeForSend)
         {
             using (var requestStream = new WinHttpRequestStream(state, chunkedModeForSend))
             {