diff --git a/src/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs b/src/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs index ab72151679ae..1be5a0f92803 100644 --- a/src/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs +++ b/src/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs @@ -66,7 +66,9 @@ public Utf8JsonReader(ReadOnlySequence jsonData, bool isFinalBlock, JsonRe { _currentPosition = jsonData.Start; _nextPosition = _currentPosition; - if (_buffer.Length == 0) + + bool firstSegmentIsEmpty = _buffer.Length == 0; + if (firstSegmentIsEmpty) { // Once we find a non-empty segment, we need to set current position to it. // Therefore, track the next position in a copy before it gets advanced to the next segment. @@ -84,7 +86,15 @@ public Utf8JsonReader(ReadOnlySequence jsonData, bool isFinalBlock, JsonRe } } - _isLastSegment = !jsonData.TryGet(ref _nextPosition, out _, advance: true) && isFinalBlock; // Don't re-order to avoid short-circuiting + // If firstSegmentIsEmpty is true, + // only check if we have reached the last segment but do not advance _nextPosition. The while loop above already advanced it. + // Otherwise, we would end up skipping a segment (i.e. advance = false). + // If firstSegmentIsEmpty is false, + // make sure to advance _nextPosition so that it is no longer the same as _currentPosition (i.e. advance = true). + _isLastSegment = !jsonData.TryGet(ref _nextPosition, out _, advance: !firstSegmentIsEmpty) && isFinalBlock; // Don't re-order to avoid short-circuiting + + Debug.Assert(!_nextPosition.Equals(_currentPosition)); + _isMultiSegment = true; } } @@ -1310,6 +1320,7 @@ private bool TryGetNumberMultiSegment(ReadOnlySpan data, out int consumed) private ConsumeNumberResult ConsumeNegativeSignMultiSegment(ref ReadOnlySpan data, ref int i) { + Debug.Assert(i == 0); byte nextByte = data[i]; if (nextByte == '-') @@ -1330,7 +1341,8 @@ private ConsumeNumberResult ConsumeNegativeSignMultiSegment(ref ReadOnlySpan data, ref int i) { Debug.Assert(data[i] == (byte)'0'); + Debug.Assert(i == 0 || i == 1); i++; _bytePositionInLine++; - byte nextByte = default; + byte nextByte; if (i < data.Length) { nextByte = data[i]; @@ -1378,7 +1391,7 @@ private ConsumeNumberResult ConsumeZeroMultiSegment(ref ReadOnlySpan data, return ConsumeNumberResult.NeedMoreData; } - _totalConsumed++; + _totalConsumed += i; HasValueSequence = true; i = 0; data = _buffer; @@ -1523,6 +1536,7 @@ private ConsumeNumberResult ConsumeSignMultiSegment(ref ReadOnlySpan data, } return ConsumeNumberResult.NeedMoreData; } + _totalConsumed += i; HasValueSequence = true; i = 0; data = _buffer; @@ -1548,7 +1562,7 @@ private ConsumeNumberResult ConsumeSignMultiSegment(ref ReadOnlySpan data, } return ConsumeNumberResult.NeedMoreData; } - _totalConsumed++; + _totalConsumed += i; HasValueSequence = true; i = 0; data = _buffer; diff --git a/src/System.Text.Json/tests/JsonTestHelper.cs b/src/System.Text.Json/tests/JsonTestHelper.cs index fbd74356981f..0ffdb327bab9 100644 --- a/src/System.Text.Json/tests/JsonTestHelper.cs +++ b/src/System.Text.Json/tests/JsonTestHelper.cs @@ -4,6 +4,7 @@ using System.Buffers; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.IO; using System.Text.Json.Tests; @@ -151,20 +152,49 @@ public static ReadOnlySequence CreateSegments(byte[] data) return new ReadOnlySequence(firstSegment, 0, secondSegment, secondMem.Length); } - public static ReadOnlySequence GetSequence(byte[] _dataUtf8, int segmentSize) + public static ReadOnlySequence CreateSegments(byte[] data, int splitLocation) { - int numberOfSegments = _dataUtf8.Length / segmentSize + 1; + Debug.Assert(splitLocation <= data.Length); + + ReadOnlyMemory dataMemory = data; + + var firstSegment = new BufferSegment(dataMemory.Slice(0, splitLocation)); + ReadOnlyMemory secondMem = dataMemory.Slice(splitLocation); + BufferSegment secondSegment = firstSegment.Append(secondMem); + + return new ReadOnlySequence(firstSegment, 0, secondSegment, secondMem.Length); + } + + public static ReadOnlySequence CreateSegments(byte[] data, int firstSplit, int secondSplit) + { + Debug.Assert(firstSplit <= data.Length && secondSplit <= data.Length && firstSplit <= secondSplit); + + ReadOnlyMemory dataMemory = data; + + var firstSegment = new BufferSegment(dataMemory.Slice(0, firstSplit)); + ReadOnlyMemory secondMem = dataMemory.Slice(firstSplit, secondSplit - firstSplit); + BufferSegment secondSegment = firstSegment.Append(secondMem); + + ReadOnlyMemory thirdMem = dataMemory.Slice(secondSplit); + BufferSegment thirdSegment = secondSegment.Append(thirdMem); + + return new ReadOnlySequence(firstSegment, 0, thirdSegment, thirdMem.Length); + } + + public static ReadOnlySequence GetSequence(byte[] dataUtf8, int segmentSize) + { + int numberOfSegments = dataUtf8.Length / segmentSize + 1; byte[][] buffers = new byte[numberOfSegments][]; for (int j = 0; j < numberOfSegments - 1; j++) { buffers[j] = new byte[segmentSize]; - Array.Copy(_dataUtf8, j * segmentSize, buffers[j], 0, segmentSize); + Array.Copy(dataUtf8, j * segmentSize, buffers[j], 0, segmentSize); } - int remaining = _dataUtf8.Length % segmentSize; + int remaining = dataUtf8.Length % segmentSize; buffers[numberOfSegments - 1] = new byte[remaining]; - Array.Copy(_dataUtf8, _dataUtf8.Length - remaining, buffers[numberOfSegments - 1], 0, remaining); + Array.Copy(dataUtf8, dataUtf8.Length - remaining, buffers[numberOfSegments - 1], 0, remaining); return BufferFactory.Create(buffers); } diff --git a/src/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs b/src/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs index 567eb8f7b8b9..6495750ddcc6 100644 --- a/src/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs +++ b/src/System.Text.Json/tests/Utf8JsonReaderTests.MultiSegment.cs @@ -37,6 +37,111 @@ public static void InitialStateMultiSegment() Assert.False(json.Read()); } + [Fact] + public static void EmptyJsonMultiSegmentIsInvalid() + { + ReadOnlyMemory dataMemory = Array.Empty(); + + var firstSegment = new BufferSegment(dataMemory.Slice(0, 0)); + ReadOnlyMemory secondMem = dataMemory.Slice(0, 0); + BufferSegment secondSegment = firstSegment.Append(secondMem); + + var sequence = new ReadOnlySequence(firstSegment, 0, secondSegment, secondMem.Length); + + Assert.ThrowsAny(() => + { + var json = new Utf8JsonReader(sequence, isFinalBlock: true, state: default); + + Assert.Equal(0, json.BytesConsumed); + Assert.Equal(0, json.TokenStartIndex); + Assert.Equal(0, json.CurrentDepth); + Assert.Equal(JsonTokenType.None, json.TokenType); + Assert.NotEqual(default, json.Position); + Assert.False(json.HasValueSequence); + Assert.True(json.ValueSpan.SequenceEqual(default)); + Assert.True(json.ValueSequence.IsEmpty); + + Assert.Equal(64, json.CurrentState.Options.MaxDepth); + Assert.False(json.CurrentState.Options.AllowTrailingCommas); + Assert.Equal(JsonCommentHandling.Disallow, json.CurrentState.Options.CommentHandling); + + json.Read(); // this should throw + }); + } + + [Theory] + [InlineData("2e2", 200, 3)] + [InlineData("2e+2", 200, 4)] + [InlineData("123e-01", 12.3, 7)] + [InlineData("0", 0, 1)] + [InlineData("0.1", 0.1, 3)] + [InlineData("-0", 0, 2)] + [InlineData("-0.1", -0.1, 4)] + [InlineData(" 2e2 ", 200, 3)] + [InlineData(" 2e+2 ", 200, 4)] + [InlineData(" 123e-01 ", 12.3, 7)] + [InlineData(" 0 ", 0, 1)] + [InlineData(" 0.1 ", 0.1, 3)] + [InlineData(" -0 ", 0, 2)] + [InlineData(" -0.1 ", -0.1, 4)] + [InlineData("[2e2]", 200, 3)] + [InlineData("[2e+2]", 200, 4)] + [InlineData("[123e-01]", 12.3, 7)] + [InlineData("[0]", 0, 1)] + [InlineData("[0.1]", 0.1, 3)] + [InlineData("[-0]", 0, 2)] + [InlineData("[-0.1]", -0.1, 4)] + [InlineData("{\"foo\": 2e2}", 200, 3)] + [InlineData("{\"foo\": 2e+2}", 200, 4)] + [InlineData("{\"foo\": 123e-01}", 12.3, 7)] + [InlineData("{\"foo\": 0}", 0, 1)] + [InlineData("{\"foo\": 0.1}", 0.1, 3)] + [InlineData("{\"foo\": -0}", 0, 2)] + [InlineData("{\"foo\": -0.1}", -0.1, 4)] + public static void ReadJsonWithNumberVariousSegmentSizes(string input, double expectedValue, long expectedTokenLength) + { + byte[] utf8 = Encoding.UTF8.GetBytes(input); + + var jsonReader = new Utf8JsonReader(utf8); + ReadDoubleHelper(ref jsonReader, utf8.Length, expectedValue, expectedTokenLength); + + ReadOnlySequence sequence = JsonTestHelper.GetSequence(utf8, 1); + jsonReader = new Utf8JsonReader(sequence); + ReadDoubleHelper(ref jsonReader, utf8.Length, expectedValue, expectedTokenLength); + + for (int splitLocation = 0; splitLocation < utf8.Length; splitLocation++) + { + sequence = JsonTestHelper.CreateSegments(utf8, splitLocation); + jsonReader = new Utf8JsonReader(sequence); + ReadDoubleHelper(ref jsonReader, utf8.Length, expectedValue, expectedTokenLength); + } + + for (int firstSplit = 0; firstSplit < utf8.Length; firstSplit++) + { + for (int secondSplit = firstSplit; secondSplit < utf8.Length; secondSplit++) + { + sequence = JsonTestHelper.CreateSegments(utf8, firstSplit, secondSplit); + jsonReader = new Utf8JsonReader(sequence); + ReadDoubleHelper(ref jsonReader, utf8.Length, expectedValue, expectedTokenLength); + } + } + } + + private static void ReadDoubleHelper(ref Utf8JsonReader jsonReader, int expectedBytesConsumed, double expectedValue, long expectedTokenLength) + { + while (jsonReader.Read()) + { + if (jsonReader.TokenType == JsonTokenType.Number) + { + long tokenLength = jsonReader.HasValueSequence ? jsonReader.ValueSequence.Length : jsonReader.ValueSpan.Length; + Assert.Equal(expectedTokenLength, tokenLength); + Assert.Equal(tokenLength, jsonReader.BytesConsumed - jsonReader.TokenStartIndex); + Assert.Equal(expectedValue, jsonReader.GetDouble()); + } + } + Assert.Equal(expectedBytesConsumed, jsonReader.BytesConsumed); + } + [Fact] public static void InitialStateSimpleCtorMultiSegment() {