Skip to content

Commit

Permalink
Moving the Utf8Formatter and Utf8Parser into S.P.Corelib (dotnet#20934)
Browse files Browse the repository at this point in the history
* Moving the Utf8Formatter and Utf8Parser into S.P.Corelib

* Doing some minimal cleanup to lineup types and get the Utf8Parser/Utf8Formatter building

* Updating the Utf8 Float Parser to have different buffers for Single vs Double

* Fixing the Utf8Parser to track trailing zero digits and to properly mark the end of the buffer

* Fixing a couple of issues in Utf8Parser.Number

Signed-off-by: dotnet-bot <[email protected]>
  • Loading branch information
tannergooding authored and jlennox committed Dec 16, 2018
1 parent 0a6249a commit de6a1b9
Show file tree
Hide file tree
Showing 26 changed files with 216 additions and 75 deletions.
50 changes: 50 additions & 0 deletions src/Common/src/CoreLib/System.Private.CoreLib.Shared.projitems
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\IPinnable.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\MemoryHandle.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\MemoryManager.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\StandardFormat.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\TlsOverPerCoreLockedStacksArrayPool.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Utilities.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Binary\Reader.cs" />
Expand All @@ -60,6 +61,55 @@
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Binary\WriterBigEndian.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Binary\WriterLittleEndian.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\FormattingHelpers.CountDigits.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Constants.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\FormattingHelpers.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Boolean.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Date.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Date.G.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Date.L.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Date.O.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Date.R.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Decimal.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Decimal.E.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Decimal.F.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Decimal.G.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Float.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Guid.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.Signed.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.Signed.D.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.Signed.Default.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.Signed.N.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.Unsigned.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.Unsigned.D.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.Unsigned.Default.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.Unsigned.N.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.Integer.Unsigned.X.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Formatter\Utf8Formatter.TimeSpan.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\ParserHelpers.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Boolean.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Date.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Date.Default.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Date.G.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Date.Helpers.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Date.O.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Date.R.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Decimal.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Float.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Guid.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Integer.Signed.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Integer.Signed.D.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Integer.Signed.N.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Integer.Unsigned.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Integer.Unsigned.D.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Integer.Unsigned.N.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Integer.Unsigned.X.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.Number.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.TimeSpan.BigG.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.TimeSpan.C.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.TimeSpan.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.TimeSpan.LittleG.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Buffers\Text\Utf8Parser\Utf8Parser.TimeSpanSplitter.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Byte.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\Char.cs" />
<Compile Include="$(MSBuildThisFileDirectory)System\CharEnumerator.cs" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -242,5 +242,15 @@ public static uint DivMod(uint numerator, uint denominator, out uint modulo)
}

#endregion Math Helper methods

//
// Enable use of ThrowHelper from TryFormat() routines without introducing dozens of non-code-coveraged "bytesWritten = 0; return false" boilerplate.
//
public static bool TryFormatThrowFormatException(out int bytesWritten)
{
bytesWritten = 0;
ThrowHelper.ThrowFormatException_BadFormatSpecifier();
return false;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ public static bool TryFormat(bool value, Span<byte> destination, out int bytesWr
return false;

BadFormat:
return ThrowHelper.TryFormatThrowFormatException(out bytesWritten);
return FormattingHelpers.TryFormatThrowFormatException(out bytesWritten);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,7 @@ public static bool TryFormat(DateTimeOffset value, Span<byte> destination, out i
return TryFormatDateTimeG(value.DateTime, offset, destination, out bytesWritten);

default:
return ThrowHelper.TryFormatThrowFormatException(out bytesWritten);
return FormattingHelpers.TryFormatThrowFormatException(out bytesWritten);
}
}

Expand Down Expand Up @@ -164,7 +164,7 @@ public static bool TryFormat(DateTime value, Span<byte> destination, out int byt
return TryFormatDateTimeG(value, Utf8Constants.NullUtcOffset, destination, out bytesWritten);

default:
return ThrowHelper.TryFormatThrowFormatException(out bytesWritten);
return FormattingHelpers.TryFormatThrowFormatException(out bytesWritten);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ namespace System.Buffers.Text
{
public static partial class Utf8Formatter
{
private static bool TryFormatDecimalE(ref NumberBuffer number, Span<byte> destination, out int bytesWritten, byte precision, byte exponentSymbol)
private static bool TryFormatDecimalE(ref Number.NumberBuffer number, Span<byte> destination, out int bytesWritten, byte precision, byte exponentSymbol)
{
const int NumExponentDigits = 3;

Expand Down Expand Up @@ -89,7 +89,7 @@ private static bool TryFormatDecimalE(ref NumberBuffer number, Span<byte> destin
exponent = -exponent;
}

Debug.Assert(exponent < Number.DECIMAL_PRECISION, "If you're trying to reuse this routine for double/float, you'll need to review the code carefully for Decimal-specific assumptions.");
Debug.Assert(exponent < Number.DecimalPrecision, "If you're trying to reuse this routine for double/float, you'll need to review the code carefully for Decimal-specific assumptions.");

// Emit exactly three digits for the exponent.
destination[dstIndex++] = (byte)'0'; // The exponent for Decimal can never exceed 28 (let alone 99)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ namespace System.Buffers.Text
{
public static partial class Utf8Formatter
{
private static bool TryFormatDecimalF(ref NumberBuffer number, Span<byte> destination, out int bytesWritten, byte precision)
private static bool TryFormatDecimalF(ref Number.NumberBuffer number, Span<byte> destination, out int bytesWritten, byte precision)
{
int scale = number.Scale;
ReadOnlySpan<byte> digits = number.Digits;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@ namespace System.Buffers.Text
{
public static partial class Utf8Formatter
{
private static bool TryFormatDecimalG(ref NumberBuffer number, Span<byte> destination, out int bytesWritten)
private static bool TryFormatDecimalG(ref Number.NumberBuffer number, Span<byte> destination, out int bytesWritten)
{
int scale = number.Scale;
ReadOnlySpan<byte> digits = number.Digits;
int numDigits = number.NumDigits;
int numDigits = number.DigitsCount;

bool isFraction = scale < numDigits;
int numBytesNeeded;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public static partial class Utf8Formatter
/// <exceptions>
/// <cref>System.FormatException</cref> if the format is not valid for this data type.
/// </exceptions>
public static bool TryFormat(decimal value, Span<byte> destination, out int bytesWritten, StandardFormat format = default)
public static unsafe bool TryFormat(decimal value, Span<byte> destination, out int bytesWritten, StandardFormat format = default)
{
if (format.IsDefault)
{
Expand All @@ -42,8 +42,11 @@ public static bool TryFormat(decimal value, Span<byte> destination, out int byte
{
if (format.Precision != StandardFormat.NoPrecision)
throw new NotSupportedException(SR.Argument_GWithPrecisionNotSupported);
NumberBuffer number = default;
Number.DecimalToNumber(value, ref number);

byte* pDigits = stackalloc byte[Number.DecimalNumberBufferLength];
Number.NumberBuffer number = new Number.NumberBuffer(Number.NumberBufferKind.Decimal, pDigits, Number.DecimalNumberBufferLength);

Number.DecimalToNumber(ref value, ref number);
bool success = TryFormatDecimalG(ref number, destination, out bytesWritten);
#if DEBUG
// This DEBUG segment exists to close a code coverage hole inside TryFormatDecimalG(). Because we don't call RoundNumber() on this path, we have no way to feed
Expand All @@ -52,7 +55,7 @@ public static bool TryFormat(decimal value, Span<byte> destination, out int byte
if (success)
{
Span<byte> digits = number.Digits;
int numDigits = number.NumDigits;
int numDigits = number.DigitsCount;
if (numDigits != 0 && number.Scale == numDigits && digits[numDigits - 1] == '0')
{
while (numDigits != 0 && digits[numDigits - 1] == '0')
Expand All @@ -61,6 +64,7 @@ public static bool TryFormat(decimal value, Span<byte> destination, out int byte
numDigits--;
}

number.DigitsCount = numDigits;
number.CheckConsistency();

byte[] buffer2 = new byte[destination.Length];
Expand All @@ -81,8 +85,10 @@ public static bool TryFormat(decimal value, Span<byte> destination, out int byte
case 'f':
case 'F':
{
NumberBuffer number = default;
Number.DecimalToNumber(value, ref number);
byte* pDigits = stackalloc byte[Number.DecimalNumberBufferLength];
Number.NumberBuffer number = new Number.NumberBuffer(Number.NumberBufferKind.Decimal, pDigits, Number.DecimalNumberBufferLength);

Number.DecimalToNumber(ref value, ref number);
byte precision = (format.Precision == StandardFormat.NoPrecision) ? (byte)2 : format.Precision;
Number.RoundNumber(ref number, number.Scale + precision);
return TryFormatDecimalF(ref number, destination, out bytesWritten, precision);
Expand All @@ -91,15 +97,17 @@ public static bool TryFormat(decimal value, Span<byte> destination, out int byte
case 'e':
case 'E':
{
NumberBuffer number = default;
Number.DecimalToNumber(value, ref number);
byte* pDigits = stackalloc byte[Number.DecimalNumberBufferLength];
Number.NumberBuffer number = new Number.NumberBuffer(Number.NumberBufferKind.Decimal, pDigits, Number.DecimalNumberBufferLength);

Number.DecimalToNumber(ref value, ref number);
byte precision = (format.Precision == StandardFormat.NoPrecision) ? (byte)6 : format.Precision;
Number.RoundNumber(ref number, precision + 1);
return TryFormatDecimalE(ref number, destination, out bytesWritten, precision, exponentSymbol: (byte)format.Symbol);
}

default:
return ThrowHelper.TryFormatThrowFormatException(out bytesWritten);
return FormattingHelpers.TryFormatThrowFormatException(out bytesWritten);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ private static bool TryFormatFloatingPoint<T>(T value, Span<byte> destination, o
break;

default:
return ThrowHelper.TryFormatThrowFormatException(out bytesWritten);
return FormattingHelpers.TryFormatThrowFormatException(out bytesWritten);
}

string formatString = format.ToString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ public static bool TryFormat(Guid value, Span<byte> destination, out int bytesWr
break;

default:
return ThrowHelper.TryFormatThrowFormatException(out bytesWritten);
return FormattingHelpers.TryFormatThrowFormatException(out bytesWritten);
}

// At this point, the low byte of flags contains the minimum required length
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ private static bool TryFormatInt64(long value, ulong mask, Span<byte> destinatio
return TryFormatUInt64X((ulong)value & mask, format.Precision, false, destination, out bytesWritten);

default:
return ThrowHelper.TryFormatThrowFormatException(out bytesWritten);
return FormattingHelpers.TryFormatThrowFormatException(out bytesWritten);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ private static bool TryFormatUInt64(ulong value, Span<byte> destination, out int
return TryFormatUInt64X(value, format.Precision, false /* useLower */, destination, out bytesWritten);

default:
return ThrowHelper.TryFormatThrowFormatException(out bytesWritten);
return FormattingHelpers.TryFormatThrowFormatException(out bytesWritten);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public static bool TryFormat(TimeSpan value, Span<byte> destination, out int byt
break;

default:
return ThrowHelper.TryFormatThrowFormatException(out bytesWritten);
return FormattingHelpers.TryFormatThrowFormatException(out bytesWritten);
}

// First, calculate how large an output buffer is needed to hold the entire output.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,5 +51,24 @@ public static bool IsDigit(int i)
{
return (uint)(i - '0') <= ('9' - '0');
}

//
// Enable use of ThrowHelper from TryParse() routines without introducing dozens of non-code-coveraged "value= default; bytesConsumed = 0; return false" boilerplate.
//
public static bool TryParseThrowFormatException(out int bytesConsumed)
{
bytesConsumed = 0;
ThrowHelper.ThrowFormatException_BadFormatSpecifier();
return false;
}

//
// Enable use of ThrowHelper from TryParse() routines without introducing dozens of non-code-coveraged "value= default; bytesConsumed = 0; return false" boilerplate.
//
public static bool TryParseThrowFormatException<T>(out T value, out int bytesConsumed)
{
value = default;
return TryParseThrowFormatException(out bytesConsumed);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public static partial class Utf8Parser
public static bool TryParse(ReadOnlySpan<byte> source, out bool value, out int bytesConsumed, char standardFormat = default)
{
if (!(standardFormat == default(char) || standardFormat == 'G' || standardFormat == 'l'))
return ThrowHelper.TryParseThrowFormatException(out value, out bytesConsumed);
return ParserHelpers.TryParseThrowFormatException(out value, out bytesConsumed);

if (source.Length >= 4)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ public static bool TryParse(ReadOnlySpan<byte> source, out DateTime value, out i
return TryParseDateTimeG(source, out value, out _, out bytesConsumed);

default:
return ThrowHelper.TryParseThrowFormatException(out value, out bytesConsumed);
return ParserHelpers.TryParseThrowFormatException(out value, out bytesConsumed);
}
}

Expand Down Expand Up @@ -139,7 +139,7 @@ public static bool TryParse(ReadOnlySpan<byte> source, out DateTimeOffset value,
return TryParseDateTimeG(source, out DateTime _, out value, out bytesConsumed);

default:
return ThrowHelper.TryParseThrowFormatException(out value, out bytesConsumed);
return ParserHelpers.TryParseThrowFormatException(out value, out bytesConsumed);
}
}

Expand Down
Loading

0 comments on commit de6a1b9

Please sign in to comment.