From 25360e7798c4738c0fbc77bdde010be8e1dfee12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marie=20P=C3=ADchov=C3=A1?= <11718369+ManickaP@users.noreply.github.com> Date: Sat, 13 May 2023 02:00:44 +0200 Subject: [PATCH] [release/7.0] Fix HTTP/3 and HTTP/2 header decoder buffer allocation (#47949) * Fix HTTP/3 and HTTTP/2 header decoder buffer allocation * Fix warnings in H/QPack tests. * Fix Asserts * Fix test --------- Co-authored-by: Aditya Mandaleeka --- .editorconfig | 12 +- .../runtime/Http2/Hpack/HPackDecoder.cs | 11 +- .../runtime/Http3/QPack/QPackDecoder.cs | 4 +- .../runtime/Http2/HPackDecoderTest.cs | 110 +++++++++++++++++- .../runtime/Http3/QPackDecoderTest.cs | 105 ++++++++++++++++- 5 files changed, 224 insertions(+), 18 deletions(-) diff --git a/.editorconfig b/.editorconfig index c532bc0d1f63..4ecd69ca10c0 100644 --- a/.editorconfig +++ b/.editorconfig @@ -133,10 +133,12 @@ dotnet_diagnostic.CA1829.severity = warning dotnet_diagnostic.CA1830.severity = warning # CA1831: Use AsSpan or AsMemory instead of Range-based indexers when appropriate -# CA1832: Use AsSpan or AsMemory instead of Range-based indexers when appropriate -# CA1833: Use AsSpan or AsMemory instead of Range-based indexers when appropriate dotnet_diagnostic.CA1831.severity = warning + +# CA1832: Use AsSpan or AsMemory instead of Range-based indexers when appropriate dotnet_diagnostic.CA1832.severity = warning + +# CA1833: Use AsSpan or AsMemory instead of Range-based indexers when appropriate dotnet_diagnostic.CA1833.severity = warning # CA1834: Consider using 'StringBuilder.Append(char)' when applicable @@ -290,6 +292,12 @@ dotnet_diagnostic.CA1826.severity = suggestion dotnet_diagnostic.CA1827.severity = suggestion # CA1829: Use Length/Count property instead of Count() when available dotnet_diagnostic.CA1829.severity = suggestion +# CA1831: Use AsSpan or AsMemory instead of Range-based indexers when appropriate +dotnet_diagnostic.CA1831.severity = suggestion +# CA1832: Use AsSpan or AsMemory instead of Range-based indexers when appropriate +dotnet_diagnostic.CA1832.severity = suggestion +# CA1833: Use AsSpan or AsMemory instead of Range-based indexers when appropriate +dotnet_diagnostic.CA1833.severity = suggestion # CA1834: Consider using 'StringBuilder.Append(char)' when applicable dotnet_diagnostic.CA1834.severity = suggestion # CA1835: Prefer the 'Memory'-based overloads for 'ReadAsync' and 'WriteAsync' diff --git a/src/Shared/runtime/Http2/Hpack/HPackDecoder.cs b/src/Shared/runtime/Http2/Hpack/HPackDecoder.cs index fb8739999a57..1f9083e29645 100644 --- a/src/Shared/runtime/Http2/Hpack/HPackDecoder.cs +++ b/src/Shared/runtime/Http2/Hpack/HPackDecoder.cs @@ -187,12 +187,11 @@ private void DecodeInternal(ReadOnlySpan data, IHttpStreamHeadersHandler h // will no longer be valid. if (_headerNameRange != null) { - EnsureStringCapacity(ref _headerNameOctets); + EnsureStringCapacity(ref _headerNameOctets, _headerNameLength); _headerName = _headerNameOctets; ReadOnlySpan headerBytes = data.Slice(_headerNameRange.GetValueOrDefault().start, _headerNameRange.GetValueOrDefault().length); headerBytes.CopyTo(_headerName); - _headerNameLength = headerBytes.Length; _headerNameRange = null; } } @@ -427,6 +426,7 @@ private void ParseHeaderName(ReadOnlySpan data, ref int currentIndex, IHtt { // Fast path. Store the range rather than copying. _headerNameRange = (start: currentIndex, count); + _headerNameLength = _stringLength; currentIndex += count; _state = State.HeaderValueLength; @@ -621,11 +621,12 @@ int Decode(ref byte[] dst) _state = nextState; } - private void EnsureStringCapacity(ref byte[] dst) + private void EnsureStringCapacity(ref byte[] dst, int stringLength = -1) { - if (dst.Length < _stringLength) + stringLength = stringLength >= 0 ? stringLength : _stringLength; + if (dst.Length < stringLength) { - dst = new byte[Math.Max(_stringLength, Math.Min(dst.Length * 2, _maxHeadersLength))]; + dst = new byte[Math.Max(stringLength, Math.Min(dst.Length * 2, _maxHeadersLength))]; } } diff --git a/src/Shared/runtime/Http3/QPack/QPackDecoder.cs b/src/Shared/runtime/Http3/QPack/QPackDecoder.cs index 394cc1aee72a..608a2ae73acf 100644 --- a/src/Shared/runtime/Http3/QPack/QPackDecoder.cs +++ b/src/Shared/runtime/Http3/QPack/QPackDecoder.cs @@ -243,12 +243,11 @@ private void DecodeInternal(ReadOnlySpan data, IHttpStreamHeadersHandler h // will no longer be valid. if (_headerNameRange != null) { - EnsureStringCapacity(ref _headerNameOctets, _stringLength, existingLength: 0); + EnsureStringCapacity(ref _headerNameOctets, _headerNameLength, existingLength: 0); _headerName = _headerNameOctets; ReadOnlySpan headerBytes = data.Slice(_headerNameRange.GetValueOrDefault().start, _headerNameRange.GetValueOrDefault().length); headerBytes.CopyTo(_headerName); - _headerNameLength = headerBytes.Length; _headerNameRange = null; } } @@ -294,6 +293,7 @@ private void ParseHeaderName(ReadOnlySpan data, ref int currentIndex, IHtt { // Fast path. Store the range rather than copying. _headerNameRange = (start: currentIndex, count); + _headerNameLength = _stringLength; currentIndex += count; _state = State.HeaderValueLength; diff --git a/src/Shared/test/Shared.Tests/runtime/Http2/HPackDecoderTest.cs b/src/Shared/test/Shared.Tests/runtime/Http2/HPackDecoderTest.cs index b7f4c19072a6..d070c179aa4c 100644 --- a/src/Shared/test/Shared.Tests/runtime/Http2/HPackDecoderTest.cs +++ b/src/Shared/test/Shared.Tests/runtime/Http2/HPackDecoderTest.cs @@ -46,8 +46,13 @@ public class HPackDecoderTests private const string _headerNameString = "new-header"; + // On purpose longer than 4096 (DefaultStringOctetsSize from HPackDecoder) to trigger https://github.com/dotnet/runtime/issues/78516 + private static readonly string _literalHeaderNameString = string.Concat(Enumerable.Range(0, 4100).Select(c => (char)('a' + (c % 26)))); + private static readonly byte[] _headerNameBytes = Encoding.ASCII.GetBytes(_headerNameString); + private static readonly byte[] _literalHeaderNameBytes = Encoding.ASCII.GetBytes(_literalHeaderNameString); + // n e w - h e a d e r * // 10101000 10111110 00010110 10011100 10100011 10010000 10110110 01111111 private static readonly byte[] _headerNameHuffmanBytes = new byte[] { 0xa8, 0xbe, 0x16, 0x9c, 0xa3, 0x90, 0xb6, 0x7f }; @@ -64,6 +69,12 @@ public class HPackDecoderTests .Concat(_headerNameBytes) .ToArray(); + // size = 4096 ==> 0x7f, 0x81, 0x1f (7+) prefixed integer + // size = 4100 ==> 0x7f, 0x85, 0x1f (7+) prefixed integer + private static readonly byte[] _literalHeaderName = new byte[] { 0x7f, 0x85, 0x1f } // 4100 + .Concat(_literalHeaderNameBytes) + .ToArray(); + private static readonly byte[] _headerNameHuffman = new byte[] { (byte)(0x80 | _headerNameHuffmanBytes.Length) } .Concat(_headerNameHuffmanBytes) .ToArray(); @@ -392,6 +403,101 @@ public void DecodesLiteralHeaderFieldNeverIndexed_IndexedName_OutOfRange_Error() Assert.Empty(_handler.DecodedHeaders); } + [Fact] + public void DecodesLiteralHeaderFieldNeverIndexed_NewName_SingleBuffer() + { + byte[] encoded = _literalHeaderFieldWithoutIndexingNewName + .Concat(_literalHeaderName) + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(encoded, endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]); + } + + [Fact] + public void DecodesLiteralHeaderFieldNeverIndexed_NewName_NameLengthBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalHeaderFieldWithoutIndexingNewName + .Concat(_literalHeaderName) + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(encoded[..1], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[1..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]); + } + + [Fact] + public void DecodesLiteralHeaderFieldNeverIndexed_NewName_NameBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalHeaderFieldWithoutIndexingNewName + .Concat(_literalHeaderName) + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(encoded[..(_literalHeaderNameString.Length / 2)], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[(_literalHeaderNameString.Length / 2)..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]); + } + + [Fact] + public void DecodesLiteralHeaderFieldNeverIndexed_NewName_NameAndValueBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalHeaderFieldWithoutIndexingNewName + .Concat(_literalHeaderName) + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(encoded[..^_headerValue.Length], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[^_headerValue.Length..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]); + } + + [Fact] + public void DecodesLiteralHeaderFieldNeverIndexed_NewName_ValueLengthBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalHeaderFieldWithoutIndexingNewName + .Concat(_literalHeaderName) + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(encoded[..^(_headerValue.Length - 1)], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[^(_headerValue.Length - 1)..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]); + } + + [Fact] + public void DecodesLiteralHeaderFieldNeverIndexed_NewName_ValueBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalHeaderFieldWithoutIndexingNewName + .Concat(_literalHeaderName) + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(encoded[..^(_headerValueString.Length / 2)], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[^(_headerValueString.Length / 2)..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderNameString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderNameString]); + } + [Fact] public void DecodesDynamicTableSizeUpdate() { @@ -500,10 +606,8 @@ public void DecodesStringLength_ExceedsLimit_Throws() string string8191 = new string('a', MaxHeaderFieldSize - 1); string string8193 = new string('a', MaxHeaderFieldSize + 1); string string8194 = new string('a', MaxHeaderFieldSize + 2); - var bytes = new byte[3]; var success = IntegerEncoder.Encode(8194, 7, bytes, out var written); - byte[] encoded = _literalHeaderFieldWithoutIndexingNewName .Concat(new byte[] { 0x7f, 0x80, 0x3f }) // 8191 encoded with 7-bit prefix, no Huffman encoding .Concat(Encoding.ASCII.GetBytes(string8191)) @@ -520,14 +624,12 @@ public void DecodesStringLength_ExceedsLimit_Throws() .Concat(new byte[] { 0x7f, 0x83, 0x3f }) // 8194 encoded with 7-bit prefix, no Huffman encoding .Concat(Encoding.ASCII.GetBytes(string8194)) .ToArray(); - var ex = Assert.Throws(() => decoder.Decode(encoded, endHeaders: true, handler: _handler)); Assert.Equal(SR.Format(SR.net_http_headers_exceeded_length, MaxHeaderFieldSize + 1), ex.Message); Assert.Equal(string8191, _handler.DecodedHeaders[string8191]); Assert.Equal(string8193, _handler.DecodedHeaders[string8193]); Assert.False(_handler.DecodedHeaders.ContainsKey(string8194)); } - [Fact] public void DecodesStringLength_IndividualBytes() { diff --git a/src/Shared/test/Shared.Tests/runtime/Http3/QPackDecoderTest.cs b/src/Shared/test/Shared.Tests/runtime/Http3/QPackDecoderTest.cs index 475fddeab09b..931ecbcd9bd2 100644 --- a/src/Shared/test/Shared.Tests/runtime/Http3/QPackDecoderTest.cs +++ b/src/Shared/test/Shared.Tests/runtime/Http3/QPackDecoderTest.cs @@ -25,11 +25,11 @@ public class QPackDecoderTests // 4.5.4 - Literal Header Field With Name Reference - Static Table - Index 44 (content-type) private static readonly byte[] _literalHeaderFieldWithNameReferenceStatic = new byte[] { 0x5f, 0x1d }; - // 4.5.6 - Literal Field Line With Literal Name - (translate) - private static readonly byte[] _literalFieldLineWithLiteralName = new byte[] { 0x37, 0x02, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x6c, 0x61, 0x74, 0x65 }; + // 4.5.6 - Literal Field Line With Literal Name - (literal-header-field) + private static readonly byte[] _literalFieldLineWithLiteralName = new byte[] { 0x37, 0x0d, 0x6c, 0x69, 0x74, 0x65, 0x72, 0x61, 0x6c, 0x2d, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x2d, 0x66, 0x69, 0x65, 0x6c, 0x64 }; private const string _contentTypeString = "content-type"; - private const string _translateString = "translate"; + private const string _literalHeaderFieldString = "literal-header-field"; // n e w - h e a d e r * // 10101000 10111110 00010110 10011100 10100011 10010000 10110110 01111111 @@ -97,7 +97,7 @@ public void DecodesLiteralFieldLineWithLiteralName_Value() .Concat(_headerValue) .ToArray(); - TestDecodeWithoutIndexing(encoded, _translateString, _headerValueString); + TestDecodeWithoutIndexing(encoded, _literalHeaderFieldString, _headerValueString); } [Fact] @@ -140,7 +140,7 @@ public void DecodesLiteralFieldLineWithLiteralName_HuffmanEncodedValue() .Concat(_headerValueHuffman) .ToArray(); - TestDecodeWithoutIndexing(encoded, _translateString, _headerValueString); + TestDecodeWithoutIndexing(encoded, _literalHeaderFieldString, _headerValueString); } [Fact] @@ -173,6 +173,101 @@ public void DecodesLiteralFieldLineWithLiteralName_LargeValues() }); } + [Fact] + public void LiteralFieldWithoutNameReference_SingleBuffer() + { + byte[] encoded = _literalFieldLineWithLiteralName + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler); + _decoder.Decode(encoded, endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]); + } + + [Fact] + public void LiteralFieldWithoutNameReference_NameLengthBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalFieldLineWithLiteralName + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler); + _decoder.Decode(encoded[..1], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[1..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]); + } + + [Fact] + public void LiteralFieldWithoutNameReference_NameBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalFieldLineWithLiteralName + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler); + _decoder.Decode(encoded[..(_literalHeaderFieldString.Length / 2)], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[(_literalHeaderFieldString.Length / 2)..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]); + } + + [Fact] + public void LiteralFieldWithoutNameReference_NameAndValueBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalFieldLineWithLiteralName + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler); + _decoder.Decode(encoded[..^_headerValue.Length], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[^_headerValue.Length..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]); + } + + [Fact] + public void LiteralFieldWithoutNameReference_ValueLengthBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalFieldLineWithLiteralName + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler); + _decoder.Decode(encoded[..^(_headerValue.Length - 1)], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[^(_headerValue.Length - 1)..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]); + } + + [Fact] + public void LiteralFieldWithoutNameReference_ValueBrokenIntoSeparateBuffers() + { + byte[] encoded = _literalFieldLineWithLiteralName + .Concat(_headerValue) + .ToArray(); + + _decoder.Decode(new byte[] { 0x00, 0x00 }, endHeaders: false, handler: _handler); + _decoder.Decode(encoded[..^(_headerValueString.Length / 2)], endHeaders: false, handler: _handler); + _decoder.Decode(encoded[^(_headerValueString.Length / 2)..], endHeaders: true, handler: _handler); + + Assert.Single(_handler.DecodedHeaders); + Assert.True(_handler.DecodedHeaders.ContainsKey(_literalHeaderFieldString)); + Assert.Equal(_headerValueString, _handler.DecodedHeaders[_literalHeaderFieldString]); + } + public static readonly TheoryData _incompleteHeaderBlockData = new TheoryData { // Incomplete header