Add Base64 APIs to Utf8JsonReader, Utf8JsonWriter, and JsonElement (#…

…38048)

* Add Utf8JsonWriter Base64 APIs.

* Add Utf8JsonReader Base64 APIs.

* Add JsonElement Base64 APIs.

* Update API shape based on review.

* Auto-generate the ref assembly.

* Address PR feedback so far.

* Add escaping step and update length counters accordingly.

* Add JsonWriter API tests.

* Add JsonReader and JsonElement tests.

src/System.Text.Json/ref/System.Text.Json.cs

@@ -38,6 +38,7 @@ public readonly partial struct JsonElement
         public System.Text.Json.JsonElement.ObjectEnumerator EnumerateObject() { throw null; }
         public int GetArrayLength() { throw null; }
         public bool GetBoolean() { throw null; }
+        public byte[] GetBytesFromBase64() { throw null; }
         public System.DateTime GetDateTime() { throw null; }
         public System.DateTimeOffset GetDateTimeOffset() { throw null; }
         public decimal GetDecimal() { throw null; }
@@ -56,6 +57,7 @@ public readonly partial struct JsonElement
         [System.CLSCompliantAttribute(false)]
         public ulong GetUInt64() { throw null; }
         public override string ToString() { throw null; }
+        public bool TryGetBytesFromBase64(out byte[] value) { throw null; }
         public bool TryGetDateTime(out System.DateTime value) { throw null; }
         public bool TryGetDateTimeOffset(out System.DateTimeOffset value) { throw null; }
         public bool TryGetDecimal(out decimal value) { throw null; }
@@ -196,9 +198,10 @@ public ref partial struct Utf8JsonReader
         public System.Buffers.ReadOnlySequence<byte> ValueSequence { get { throw null; } }
         public System.ReadOnlySpan<byte> ValueSpan { get { throw null; } }
         public bool GetBoolean() { throw null; }
+        public byte[] GetBytesFromBase64() { throw null; }
+        public string GetComment() { throw null; }
         public System.DateTime GetDateTime() { throw null; }
         public System.DateTimeOffset GetDateTimeOffset() { throw null; }
-        public string GetComment() { throw null; }
         public decimal GetDecimal() { throw null; }
         public double GetDouble() { throw null; }
         public System.Guid GetGuid() { throw null; }
@@ -214,6 +217,7 @@ public ref partial struct Utf8JsonReader
         public void Skip() { }
         public bool TextEquals(System.ReadOnlySpan<byte> otherUtf8Text) { throw null; }
         public bool TextEquals(System.ReadOnlySpan<char> otherText) { throw null; }
+        public bool TryGetBytesFromBase64(out byte[] value) { throw null; }
         public bool TryGetDateTime(out System.DateTime value) { throw null; }
         public bool TryGetDateTimeOffset(out System.DateTimeOffset value) { throw null; }
         public bool TryGetDecimal(out decimal value) { throw null; }
@@ -242,6 +246,11 @@ public void Flush() { }
         public void Reset() { }
         public void Reset(System.Buffers.IBufferWriter<byte> bufferWriter) { }
         public void Reset(System.IO.Stream utf8Json) { }
+        public void WriteBase64String(System.ReadOnlySpan<byte> utf8PropertyName, System.ReadOnlySpan<byte> bytes) { }
+        public void WriteBase64String(System.ReadOnlySpan<char> propertyName, System.ReadOnlySpan<byte> bytes) { }
+        public void WriteBase64String(string propertyName, System.ReadOnlySpan<byte> bytes) { }
+        public void WriteBase64String(System.Text.Json.JsonEncodedText propertyName, System.ReadOnlySpan<byte> bytes) { }
+        public void WriteBase64StringValue(System.ReadOnlySpan<byte> bytes) { }
         public void WriteBoolean(System.ReadOnlySpan<byte> utf8PropertyName, bool value) { }
         public void WriteBoolean(System.ReadOnlySpan<char> propertyName, bool value) { }
         public void WriteBoolean(string propertyName, bool value) { }
@@ -385,10 +394,10 @@ public static partial class JsonSerializer
         public static TValue Parse<TValue>(string json, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; }
         public static System.Threading.Tasks.ValueTask<object> ReadAsync(System.IO.Stream utf8Json, System.Type returnType, System.Text.Json.Serialization.JsonSerializerOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         public static System.Threading.Tasks.ValueTask<TValue> ReadAsync<TValue>(System.IO.Stream utf8Json, System.Text.Json.Serialization.JsonSerializerOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
-        public static byte[] ToUtf8Bytes(object value, System.Type type, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; }
-        public static byte[] ToUtf8Bytes<TValue>(TValue value, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; }
         public static string ToString(object value, System.Type type, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; }
         public static string ToString<TValue>(TValue value, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; }
+        public static byte[] ToUtf8Bytes(object value, System.Type type, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; }
+        public static byte[] ToUtf8Bytes<TValue>(TValue value, System.Text.Json.Serialization.JsonSerializerOptions options = null) { throw null; }
         public static System.Threading.Tasks.Task WriteAsync(object value, System.Type type, System.IO.Stream utf8Json, System.Text.Json.Serialization.JsonSerializerOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         public static System.Threading.Tasks.Task WriteAsync<TValue>(TValue value, System.IO.Stream utf8Json, System.Text.Json.Serialization.JsonSerializerOptions options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
     }

src/System.Text.Json/src/Resources/Strings.resx

@@ -137,6 +137,9 @@
   <data name="CannotTranscodeInvalidUtf8" xml:space="preserve">
     <value>Cannot transcode invalid UTF-8 JSON text to UTF-16 string.</value>
   </data>
+  <data name="CannotDecodeInvalidBase64" xml:space="preserve">
+    <value>Cannot decode JSON text that is not encoded as valid Base64 to bytes.</value>
+  </data>
   <data name="CannotTranscodeInvalidUtf16" xml:space="preserve">
     <value>Cannot transcode invalid UTF-16 string to UTF-8 JSON text.</value>
   </data>

src/System.Text.Json/src/System.Text.Json.csproj

@@ -11,7 +11,7 @@
     <!-- Workaround for overriding the XML comments related warnings that are being supressed repo wide (within arcade): -->
     <!-- https://github.com/dotnet/arcade/blob/ea6addfdc65e5df1b2c036f11614a5f922e36267/src/Microsoft.DotNet.Arcade.Sdk/tools/ProjectDefaults.props#L90 -->
     <!-- For this project, we want warnings if there are public APIs/types without properly formatted XML comments (particularly CS1591). -->
-    <NoWarn/>
+    <NoWarn />
   </PropertyGroup>
   <ItemGroup>
     <Compile Include="System\Text\Json\BitStack.cs" />
@@ -124,6 +124,7 @@
     <Compile Include="System\Text\Json\Writer\JsonWriterOptions.cs" />
     <Compile Include="System\Text\Json\Writer\SequenceValidity.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.cs" />
+    <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteProperties.Bytes.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteProperties.DateTime.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteProperties.DateTimeOffset.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteProperties.Decimal.cs" />
@@ -136,6 +137,7 @@
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteProperties.SignedNumber.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteProperties.String.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteProperties.UnsignedNumber.cs" />
+    <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.Bytes.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.Comment.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.DateTime.cs" />
     <Compile Include="System\Text\Json\Writer\Utf8JsonWriter.WriteValues.DateTimeOffset.cs" />

src/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs

@@ -263,6 +263,29 @@ internal string GetNameOfPropertyValue(int index)
             return GetString(index - DbRow.Size, JsonTokenType.PropertyName);
         }
 
+        internal bool TryGetValue(int index, out byte[] value)
+        {
+            CheckNotDisposed();
+
+            DbRow row = _parsedData.Get(index);
+
+            CheckExpectedType(JsonTokenType.String, row.TokenType);
+
+            ReadOnlySpan<byte> data = _utf8Json.Span;
+            ReadOnlySpan<byte> segment = data.Slice(row.Location, row.SizeOrLength);
+
+            // Segment needs to be unescaped
+            if (row.HasComplexChildren)
+            {
+                int idx = segment.IndexOf(JsonConstants.BackSlash);
+                Debug.Assert(idx != -1);
+                return JsonReaderHelper.TryGetUnescapedBase64Bytes(segment, idx, out value);
+            }
+
+            Debug.Assert(segment.IndexOf(JsonConstants.BackSlash) == -1);
+            return JsonReaderHelper.TryDecodeBase64(segment, out value);
+        }
+
         internal bool TryGetValue(int index, out int value)
         {
             CheckNotDisposed();

src/System.Text.Json/src/System/Text/Json/Document/JsonElement.cs

@@ -2,6 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
+using System.Buffers.Text;
 using System.Collections.Generic;
 using System.Diagnostics;
 
@@ -350,6 +351,57 @@ public string GetString()
             return _parent.GetString(_idx, JsonTokenType.String);
         }
 
+        /// <summary>
+        ///   Attempts to represent the current JSON string as bytes assuming it is base 64 encoded.
+        /// </summary>
+        /// <param name="value">Receives the value.</param>
+        /// <remarks>
+        ///  This method does not create a byte[] representation of values other than bsae 64 encoded JSON strings.
+        /// </remarks>
+        /// <returns>
+        ///   <see langword="true"/> if the entire token value is encoded as valid base 64 text and can be successfully decoded to bytes.
+        ///   <see langword="false"/> otherwise.
+        /// </returns>
+        /// <exception cref="InvalidOperationException">
+        ///   This value's <see cref="Type"/> is not <see cref="JsonValueType.String"/>.
+        /// </exception>
+        /// <exception cref="ObjectDisposedException">
+        ///   The parent <see cref="JsonDocument"/> has been disposed.
+        /// </exception>
+        public bool TryGetBytesFromBase64(out byte[] value)
+        {
+            CheckValidInstance();
+
+            return _parent.TryGetValue(_idx, out value);
+        }
+
+        /// <summary>
+        ///   Gets the value of the element as bytes.
+        /// </summary>
+        /// <remarks>
+        ///   This method does not create a byte[] representation of values other than base 64 encoded JSON strings.
+        /// </remarks>
+        /// <returns>The value decode to bytes.</returns>
+        /// <exception cref="InvalidOperationException">
+        ///   This value's <see cref="Type"/> is not <see cref="JsonValueType.String"/>.
+        /// </exception>
+        /// <exception cref="FormatException">
+        ///   The value is not encoded as base 64 text and hence cannot be decoded to bytes.
+        /// </exception>
+        /// <exception cref="ObjectDisposedException">
+        ///   The parent <see cref="JsonDocument"/> has been disposed.
+        /// </exception>
+        /// <seealso cref="ToString"/>
+        public byte[] GetBytesFromBase64()
+        {
+            if (TryGetBytesFromBase64(out byte[] value))
+            {
+                return value;
+            }
+
+            throw new FormatException();
+        }
+
         /// <summary>
         ///   Attempts to represent the current JSON number as an <see cref="int"/>.
         /// </summary>

src/System.Text.Json/src/System/Text/Json/JsonConstants.cs

@@ -57,8 +57,9 @@ internal static class JsonConstants
         // All other UTF-16 characters can be represented by either 1 or 2 UTF-8 bytes.
         public const int MaxExpansionFactorWhileTranscoding = 3;
 
-        public const int MaxTokenSize = 2_000_000_000 / MaxExpansionFactorWhileEscaping;  // 357_913_941 bytes
-        public const int MaxCharacterTokenSize = 2_000_000_000 / MaxExpansionFactorWhileEscaping; // 357_913_941 characters
+        public const int MaxTokenSize = 1_000_000_000 / MaxExpansionFactorWhileEscaping;  // 166_666_666 bytes
+        public const int MaxBase46ValueTokenSize = (1_000_000_000 >> 2 * 3) / MaxExpansionFactorWhileEscaping;  // 125_000_000 bytes
+        public const int MaxCharacterTokenSize = 1_000_000_000 / MaxExpansionFactorWhileEscaping; // 166_666_666 characters
 
         public const int MaximumFormatInt64Length = 20;   // 19 + sign (i.e. -9223372036854775808)
         public const int MaximumFormatUInt64Length = 20;  // i.e. 18446744073709551615

src/System.Text.Json/src/System/Text/Json/Reader/JsonReaderHelper.Unescaping.cs

@@ -10,6 +10,31 @@ namespace System.Text.Json
 {
     internal static partial class JsonReaderHelper
     {
+        public static bool TryGetUnescapedBase64Bytes(ReadOnlySpan<byte> utf8Source, int idx, out byte[] bytes)
+        {
+            byte[] unescapedArray = null;
+
+            Span<byte> utf8Unescaped = utf8Source.Length <= JsonConstants.StackallocThreshold ?
+                stackalloc byte[utf8Source.Length] :
+                (unescapedArray = ArrayPool<byte>.Shared.Rent(utf8Source.Length));
+
+            Unescape(utf8Source, utf8Unescaped, idx, out int written);
+            Debug.Assert(written > 0);
+
+            utf8Unescaped = utf8Unescaped.Slice(0, written);
+            Debug.Assert(!utf8Unescaped.IsEmpty);
+
+            bool result = TryDecodeBase64InPlace(utf8Unescaped, out bytes);
+
+            if (unescapedArray != null)
+            {
+                utf8Unescaped.Clear();
+                ArrayPool<byte>.Shared.Return(unescapedArray);
+            }
+
+            return result;
+        }
+
         // Reject any invalid UTF-8 data rather than silently replacing.
         public static readonly UTF8Encoding s_utf8Encoding = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true);
 
@@ -107,6 +132,53 @@ public static bool UnescapeAndCompare(ReadOnlySequence<byte> utf8Source, ReadOnl
             return result;
         }
 
+        public static bool TryDecodeBase64InPlace(Span<byte> utf8Unescaped, out byte[] bytes)
+        {
+            OperationStatus status = Base64.DecodeFromUtf8InPlace(utf8Unescaped, out int bytesWritten);
+            if (status != OperationStatus.Done)
+            {
+                bytes = null;
+                return false;
+            }
+            bytes = utf8Unescaped.Slice(0, bytesWritten).ToArray();
+            return true;
+        }
+
+        public static bool TryDecodeBase64(ReadOnlySpan<byte> utf8Unescaped, out byte[] bytes)
+        {
+            byte[] pooledArray = null;
+
+            Span<byte> byteSpan = utf8Unescaped.Length <= JsonConstants.StackallocThreshold ?
+                stackalloc byte[utf8Unescaped.Length] :
+                (pooledArray = ArrayPool<byte>.Shared.Rent(utf8Unescaped.Length));
+
+            OperationStatus status = Base64.DecodeFromUtf8(utf8Unescaped, byteSpan, out int bytesConsumed, out int bytesWritten);
+
+            if (status != OperationStatus.Done)
+            {
+                bytes = null;
+
+                if (pooledArray != null)
+                {
+                    byteSpan.Clear();
+                    ArrayPool<byte>.Shared.Return(pooledArray);
+                }
+
+                return false;
+            }
+            Debug.Assert(bytesConsumed == utf8Unescaped.Length);
+
+            bytes = byteSpan.Slice(0, bytesWritten).ToArray();
+
+            if (pooledArray != null)
+            {
+                byteSpan.Clear();
+                ArrayPool<byte>.Shared.Return(pooledArray);
+            }
+
+            return true;
+        }
+
         public static string TranscodeHelper(ReadOnlySpan<byte> utf8Unescaped)
         {
             try