Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add Utf8JsonWriter.WriteRawValue(System.Buffers.ReadOnlySequence). #76444

1 change: 1 addition & 0 deletions src/libraries/System.Text.Json/ref/System.Text.Json.cs
Original file line number Diff line number Diff line change
Expand Up @@ -583,6 +583,7 @@ public void WritePropertyName(System.ReadOnlySpan<char> propertyName) { }
public void WritePropertyName(string propertyName) { }
public void WritePropertyName(System.Text.Json.JsonEncodedText propertyName) { }
public void WriteRawValue(System.ReadOnlySpan<byte> utf8Json, bool skipInputValidation = false) { }
public void WriteRawValue(System.Buffers.ReadOnlySequence<byte> utf8Json, bool skipInputValidation = false) { }
public void WriteRawValue([System.Diagnostics.CodeAnalysis.StringSyntaxAttribute("Json")] System.ReadOnlySpan<char> json, bool skipInputValidation = false) { }
public void WriteRawValue([System.Diagnostics.CodeAnalysis.StringSyntaxAttribute("Json")] string json, bool skipInputValidation = false) { }
public void WriteStartArray() { }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ public static void ThrowArgumentException_PropertyNameTooLarge(int tokenLength)
}

[DoesNotReturn]
public static void ThrowArgumentException_ValueTooLarge(int tokenLength)
public static void ThrowArgumentException_ValueTooLarge(long tokenLength)
{
throw GetArgumentException(SR.Format(SR.ValueTooLarge, tokenLength));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ public void WriteRawValue([StringSyntax(StringSyntaxAttribute.Json)] ReadOnlySpa
/// </summary>
/// <param name="utf8Json">The raw JSON content to write.</param>
/// <param name="skipInputValidation">Whether to validate if the input is an RFC 8259-compliant JSON payload.</param>
/// <exception cref="ArgumentException">Thrown if the length of the input is zero or equal to <see cref="int.MaxValue"/>.</exception>
/// <exception cref="ArgumentException">Thrown if the length of the input is zero or greater than or equal to <see cref="int.MaxValue"/>.</exception>
/// <exception cref="JsonException">
/// Thrown if <paramref name="skipInputValidation"/> is <see langword="false"/>, and the input
/// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259)
Expand Down Expand Up @@ -113,6 +113,86 @@ public void WriteRawValue(ReadOnlySpan<byte> utf8Json, bool skipInputValidation
WriteRawValueCore(utf8Json, skipInputValidation);
}

/// <summary>
/// Writes the input as JSON content. It is expected that the input content is a single complete JSON value.
/// </summary>
/// <param name="utf8Json">The raw JSON content to write.</param>
/// <param name="skipInputValidation">Whether to validate if the input is an RFC 8259-compliant JSON payload.</param>
/// <exception cref="ArgumentException">Thrown if the length of the input is zero or equal to <see cref="int.MaxValue"/>.</exception>
/// <exception cref="JsonException">
/// Thrown if <paramref name="skipInputValidation"/> is <see langword="false"/>, and the input
/// is not a valid, complete, single JSON value according to the JSON RFC (https://tools.ietf.org/html/rfc8259)
/// or the input JSON exceeds a recursive depth of 64.
/// </exception>
/// <remarks>
/// When writing untrused JSON values, do not set <paramref name="skipInputValidation"/> to <see langword="true"/> as this can result in invalid JSON
/// being written, and/or the overall payload being written to the writer instance being invalid.
///
/// When using this method, the input content will be written to the writer destination as-is, unless validation fails (when it is enabled).
///
/// The <see cref="JsonWriterOptions.SkipValidation"/> value for the writer instance is honored when using this method.
///
/// The <see cref="JsonWriterOptions.Indented"/> and <see cref="JsonWriterOptions.Encoder"/> values for the writer instance are not applied when using this method.
/// </remarks>
public void WriteRawValue(ReadOnlySequence<byte> utf8Json, bool skipInputValidation = false)
{
if (!_options.SkipValidation)
{
ValidateWritingValue();
}

long utf8JsonLen = utf8Json.Length;

if (utf8JsonLen == 0)
{
ThrowHelper.ThrowArgumentException(SR.ExpectedJsonTokens);
}
if (utf8JsonLen >= int.MaxValue)
{
ThrowHelper.ThrowArgumentException_ValueTooLarge(utf8JsonLen);
}

if (skipInputValidation)
{
// Treat all unvalidated raw JSON value writes as string. If the payload is valid, this approach does
// not affect structural validation since a string token is equivalent to a complete object, array,
// or other complete JSON tokens when considering structural validation on subsequent writer calls.
// If the payload is not valid, then we make no guarantees about the structural validation of the final payload.
_tokenType = JsonTokenType.String;
}
else
{
// Utilize reader validation.
Utf8JsonReader reader = new(utf8Json);
while (reader.Read());
_tokenType = reader.TokenType;
}

Debug.Assert(utf8JsonLen < int.MaxValue);
int len = (int)utf8JsonLen;

// TODO (https://github.com/dotnet/runtime/issues/29293):
// investigate writing this in chunks, rather than requesting one potentially long, contiguous buffer.
int maxRequired = len + 1; // Optionally, 1 list separator. We've guarded against integer overflow earlier in the call stack.

if (_memory.Length - BytesPending < maxRequired)
{
Grow(maxRequired);
}

Span<byte> output = _memory.Span;

if (_currentDepth < 0)
{
output[BytesPending++] = JsonConstants.ListSeparator;
}

utf8Json.CopyTo(output.Slice(BytesPending));
BytesPending += len;

SetFlagToAddListSeparatorBeforeNextItem();
}

private void TranscodeAndWriteRawValue(ReadOnlySpan<char> json, bool skipInputValidation)
{
if (json.Length > JsonConstants.MaxUtf16RawValueLength)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Buffers;
using System.Collections.Generic;
using System.IO;
using System.Linq;
Expand All @@ -23,6 +24,8 @@ public static void WriteRawValidJson(byte[] rawJson, Action<byte[]> verifyWithDe
using MemoryStream ms = new();
using Utf8JsonWriter writer = new(ms);

string rawJsonAsStr = Encoding.UTF8.GetString(rawJson);

RunTests(skipInputValidation: true);
RunTests(skipInputValidation: false);

Expand All @@ -31,22 +34,29 @@ void RunTests(bool skipInputValidation)
// ROS<byte>
writer.Reset();
ms.SetLength(0);
writer.WriteRawValue(rawJson, skipInputValidation);
WriteRawValueWithSetting(writer, rawJsonAsStr, OverloadParamType.ByteArray, skipInputValidation);
writer.Flush();
verifyWithDeserialize(ms.ToArray());

// string
string rawJsonAsStr = Encoding.UTF8.GetString(rawJson);

writer.Reset();
ms.SetLength(0);
WriteRawValueWithSetting(writer, rawJsonAsStr, OverloadParamType.String, skipInputValidation);
writer.Flush();
verifyWithDeserialize(ms.ToArray());

// ROS<char>
writer.Reset();
ms.SetLength(0);
writer.WriteRawValue(rawJsonAsStr, skipInputValidation);
WriteRawValueWithSetting(writer, rawJsonAsStr, OverloadParamType.ROSChar, skipInputValidation);
writer.Flush();
verifyWithDeserialize(ms.ToArray());

// ROS<char>
writer.Reset();
ms.SetLength(0);
writer.WriteRawValue(rawJsonAsStr.AsSpan(), skipInputValidation);
WriteRawValueWithSetting(writer, rawJsonAsStr, OverloadParamType.ROSeqByte, skipInputValidation);
writer.Flush();
verifyWithDeserialize(ms.ToArray());
}
Expand Down Expand Up @@ -244,7 +254,7 @@ public static void WriteRawNullOrEmptyTokenInvalid()
Assert.Throws<ArgumentNullException>(() => writer.WriteRawValue(json: default(string)));
Assert.Throws<ArgumentException>(() => writer.WriteRawValue(json: ""));
Assert.Throws<ArgumentException>(() => writer.WriteRawValue(json: default(ReadOnlySpan<char>)));
Assert.Throws<ArgumentException>(() => writer.WriteRawValue(utf8Json: default));
Assert.Throws<ArgumentException>(() => writer.WriteRawValue(utf8Json: default(ReadOnlySpan<byte>)));
eiriktsarpalis marked this conversation as resolved.
Show resolved Hide resolved
}

[Theory]
Expand Down Expand Up @@ -406,6 +416,7 @@ public static void WriteRawMaxUtf16InputLength(JsonTokenType tokenType)
RunTest(OverloadParamType.ROSChar);
RunTest(OverloadParamType.String);
RunTest(OverloadParamType.ByteArray);
RunTest(OverloadParamType.ROSeqByte);

void RunTest(OverloadParamType paramType)
{
Expand Down Expand Up @@ -450,22 +461,27 @@ private enum OverloadParamType
{
ROSChar,
String,
ByteArray
ByteArray,
ROSeqByte
}

private static void WriteRawValueWithSetting(Utf8JsonWriter writer, string payload, OverloadParamType param)
private static void WriteRawValueWithSetting(Utf8JsonWriter writer, string payload, OverloadParamType param, bool skipInputValidation = false)
{
switch (param)
{
case OverloadParamType.ROSChar:
writer.WriteRawValue(payload.AsSpan());
writer.WriteRawValue(payload.AsSpan(), skipInputValidation);
break;
case OverloadParamType.String:
writer.WriteRawValue(payload);
writer.WriteRawValue(payload, skipInputValidation);
break;
case OverloadParamType.ByteArray:
byte[] payloadAsBytes = Encoding.UTF8.GetBytes(payload);
writer.WriteRawValue(payloadAsBytes);
writer.WriteRawValue(payloadAsBytes, skipInputValidation);
break;
case OverloadParamType.ROSeqByte:
ReadOnlySequence<byte> payloadAsSequence = new(Encoding.UTF8.GetBytes(payload));
writer.WriteRawValue(payloadAsSequence, skipInputValidation);
krwq marked this conversation as resolved.
Show resolved Hide resolved
break;
}
}
Expand Down Expand Up @@ -499,10 +515,32 @@ public static void WriteRawUtf16LengthGreaterThanMax(int len)
// UTF-8 overload is okay.
WriteRawValueWithSetting(writer, payload, OverloadParamType.ByteArray);
writer.Flush();
Assert.Equal(payload.Length, Encoding.UTF8.GetString(ms.ToArray()).Length);

writer.Reset();
ms.SetLength(0);
WriteRawValueWithSetting(writer, payload, OverloadParamType.ROSeqByte);
writer.Flush();
Assert.Equal(payload.Length, Encoding.UTF8.GetString(ms.ToArray()).Length);
}

[PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)]
[ConditionalTheory(typeof(Environment), nameof(Environment.Is64BitProcess))]
[InlineData(int.MaxValue)]
[InlineData((long)int.MaxValue + 1)]
[OuterLoop]
public void WriteRawUtf8LengthGreaterThanOrEqualToIntMax(long len)
{
try
{
using MemoryStream ms = new();
using Utf8JsonWriter writer = new(ms);
ReadOnlySequence<byte> readonlySeq = CreateLargeReadOnlySequence(len);
Assert.Throws<ArgumentException>(() => writer.WriteRawValue(readonlySeq));
}
catch (OutOfMemoryException) { } // Perhaps failed to allocate large arrays
}

[PlatformSpecific(TestPlatforms.Windows | TestPlatforms.OSX)]
[ConditionalFact(typeof(Environment), nameof(Environment.Is64BitProcess))]
[OuterLoop]
Expand All @@ -526,6 +564,7 @@ public static void WriteRawTranscodeFromUtf16ToUtf8TooLong()
RunTest(OverloadParamType.ROSChar);
RunTest(OverloadParamType.String);
RunTest(OverloadParamType.ByteArray);
RunTest(OverloadParamType.ROSeqByte);

void RunTest(OverloadParamType paramType)
{
Expand All @@ -544,5 +583,81 @@ void RunTest(OverloadParamType paramType)
catch (OutOfMemoryException) { } // OutOfMemoryException is okay since the transcoding output is probably too large.
}
}

private static ReadOnlySequence<byte> CreateLargeReadOnlySequence(long len)
{
Assert.InRange(len, int.MaxValue, long.MaxValue);

const int ArrayMaxLength = 0X7FFFFFC7; // Array.MaxLength

long totalLen = len;
TestSequenceSegment? startSegment = null;
TestSequenceSegment? endSegment = null;

// Construct ReadOnlySequence<byte> with length as 'len' which is greater than or equal to int.MaxValue
do
{
if (startSegment is null) // First segment
{
var bytes = new byte[ArrayMaxLength];
bytes[0] = (byte)'"';
FillArray(bytes, (byte)'a', 1, bytes.Length);
endSegment = startSegment = new(bytes);
}
else
{
byte[] bytes;
if (totalLen <= ArrayMaxLength) // The last segment
{
bytes = new byte[totalLen];
bytes[totalLen - 1] = (byte)'"';
}
else
{
bytes = new byte[ArrayMaxLength];
bytes[ArrayMaxLength - 1] = (byte)'a';
}

FillArray(bytes, (byte)'a', 0, bytes.Length - 1);
endSegment = endSegment!.Append(bytes);
}

totalLen -= endSegment.Memory.Length;
} while (totalLen > 0);

var readonlySeq = new ReadOnlySequence<byte>(startSegment, 0, endSegment, endSegment.Memory.Length);

Assert.Equal(len, readonlySeq.Length); // Make sure constructed ReadOnlySequence's length is as expected

return readonlySeq;

static void FillArray(byte[] bytes, byte value, int start, int end)
{
for (int i = start; i < end; i++)
{
bytes[i] = value;
}
}
}

private class TestSequenceSegment : ReadOnlySequenceSegment<byte>
{
public TestSequenceSegment(ReadOnlyMemory<byte> memory)
{
Memory = memory;
}

public TestSequenceSegment Append(ReadOnlyMemory<byte> memory)
{
var newSegment = new TestSequenceSegment(memory)
{
RunningIndex = RunningIndex + Memory.Length
};

Next = newSegment;

return newSegment;
}
}
}
}