From d963fd9ac161b3ca9ad9dcf4964bf4b41db18c37 Mon Sep 17 00:00:00 2001 From: James Newton-King Date: Sat, 9 Mar 2019 10:54:54 +1300 Subject: [PATCH] Add CodedInputReader and CodedOutputWriter --- .../BenchmarkMessage1Proto3.cs | 683 ++++++++++++++- .../JamesBenchmarks.cs | 145 ++++ .../src/Google.Protobuf.Benchmarks/Program.cs | 13 +- .../Google.Protobuf.Test/ArrayBufferWriter.cs | 158 ++++ .../CodedInputReaderTest.cs | 579 +++++++++++++ .../CodedOutputWriterTest.cs | 444 ++++++++++ csharp/src/Google.Protobuf/BufferWriter.cs | 200 +++++ .../src/Google.Protobuf/CodedInputReader.cs | 787 ++++++++++++++++++ .../src/Google.Protobuf/CodedOutputWriter.cs | 585 +++++++++++++ .../Collections/RepeatedField.cs | 85 +- csharp/src/Google.Protobuf/FieldCodec.cs | 292 ++++++- .../Google.Protobuf/Google.Protobuf.csproj | 5 +- csharp/src/Google.Protobuf/ISpanMessage.cs | 69 ++ csharp/src/Google.Protobuf/SequenceReader.cs | 410 +++++++++ csharp/src/Google.Protobuf/UnknownField.cs | 53 ++ csharp/src/Google.Protobuf/UnknownFieldSet.cs | 771 +++++++++-------- 16 files changed, 4906 insertions(+), 373 deletions(-) create mode 100644 csharp/src/Google.Protobuf.Benchmarks/JamesBenchmarks.cs create mode 100644 csharp/src/Google.Protobuf.Test/ArrayBufferWriter.cs create mode 100644 csharp/src/Google.Protobuf.Test/CodedInputReaderTest.cs create mode 100644 csharp/src/Google.Protobuf.Test/CodedOutputWriterTest.cs create mode 100644 csharp/src/Google.Protobuf/BufferWriter.cs create mode 100644 csharp/src/Google.Protobuf/CodedInputReader.cs create mode 100644 csharp/src/Google.Protobuf/CodedOutputWriter.cs create mode 100644 csharp/src/Google.Protobuf/ISpanMessage.cs create mode 100644 csharp/src/Google.Protobuf/SequenceReader.cs diff --git a/csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs b/csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs index 9e8c330a2b63..86d54a5f39ab 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/BenchmarkMessage1Proto3.cs @@ -64,7 +64,7 @@ static BenchmarkMessage1Proto3Reflection() { } #region Messages - public sealed partial class GoogleMessage1 : pb::IMessage { + public sealed partial class GoogleMessage1 : pb::IMessage, pb::ISpanMessage { private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new GoogleMessage1()); private pb::UnknownFieldSet _unknownFields; [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -80,6 +80,11 @@ public sealed partial class GoogleMessage1 : pb::IMessage { get { return Descriptor; } } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + pbr::MessageDescriptor pb::ISpanMessage.Descriptor { + get { return Descriptor; } + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public GoogleMessage1() { OnConstruction(); @@ -868,7 +873,217 @@ public void WriteTo(pb::CodedOutputStream output) { } } - [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void WriteTo(ref pb::CodedOutputWriter output) + { + if (Field1.Length != 0) + { + output.WriteRawTag(10); + output.WriteString(Field1); + } + if (Field2 != 0) + { + output.WriteRawTag(16); + output.WriteInt32(Field2); + } + if (Field3 != 0) + { + output.WriteRawTag(24); + output.WriteInt32(Field3); + } + if (Field4.Length != 0) + { + output.WriteRawTag(34); + output.WriteString(Field4); + } + field5_.WriteTo(ref output, _repeated_field5_codec); + if (Field6 != 0) + { + output.WriteRawTag(48); + output.WriteInt32(Field6); + } + if (Field7.Length != 0) + { + output.WriteRawTag(58); + output.WriteString(Field7); + } + if (Field9.Length != 0) + { + output.WriteRawTag(74); + output.WriteString(Field9); + } + if (Field12 != false) + { + output.WriteRawTag(96); + output.WriteBool(Field12); + } + if (Field13 != false) + { + output.WriteRawTag(104); + output.WriteBool(Field13); + } + if (Field14 != false) + { + output.WriteRawTag(112); + output.WriteBool(Field14); + } + if (field15_ != null) + { + output.WriteRawTag(122); + output.WriteMessage(Field15); + } + if (Field16 != 0) + { + output.WriteRawTag(128, 1); + output.WriteInt32(Field16); + } + if (Field17 != false) + { + output.WriteRawTag(136, 1); + output.WriteBool(Field17); + } + if (Field18.Length != 0) + { + output.WriteRawTag(146, 1); + output.WriteString(Field18); + } + if (Field22 != 0L) + { + output.WriteRawTag(176, 1); + output.WriteInt64(Field22); + } + if (Field23 != 0) + { + output.WriteRawTag(184, 1); + output.WriteInt32(Field23); + } + if (Field24 != false) + { + output.WriteRawTag(192, 1); + output.WriteBool(Field24); + } + if (Field25 != 0) + { + output.WriteRawTag(200, 1); + output.WriteInt32(Field25); + } + if (Field29 != 0) + { + output.WriteRawTag(232, 1); + output.WriteInt32(Field29); + } + if (Field30 != false) + { + output.WriteRawTag(240, 1); + output.WriteBool(Field30); + } + if (Field59 != false) + { + output.WriteRawTag(216, 3); + output.WriteBool(Field59); + } + if (Field60 != 0) + { + output.WriteRawTag(224, 3); + output.WriteInt32(Field60); + } + if (Field67 != 0) + { + output.WriteRawTag(152, 4); + output.WriteInt32(Field67); + } + if (Field68 != 0) + { + output.WriteRawTag(160, 4); + output.WriteInt32(Field68); + } + if (Field78 != false) + { + output.WriteRawTag(240, 4); + output.WriteBool(Field78); + } + if (Field80 != false) + { + output.WriteRawTag(128, 5); + output.WriteBool(Field80); + } + if (Field81 != false) + { + output.WriteRawTag(136, 5); + output.WriteBool(Field81); + } + if (Field100 != 0) + { + output.WriteRawTag(160, 6); + output.WriteInt32(Field100); + } + if (Field101 != 0) + { + output.WriteRawTag(168, 6); + output.WriteInt32(Field101); + } + if (Field102.Length != 0) + { + output.WriteRawTag(178, 6); + output.WriteString(Field102); + } + if (Field103.Length != 0) + { + output.WriteRawTag(186, 6); + output.WriteString(Field103); + } + if (Field104 != 0) + { + output.WriteRawTag(192, 6); + output.WriteInt32(Field104); + } + if (Field128 != 0) + { + output.WriteRawTag(128, 8); + output.WriteInt32(Field128); + } + if (Field129.Length != 0) + { + output.WriteRawTag(138, 8); + output.WriteString(Field129); + } + if (Field130 != 0) + { + output.WriteRawTag(144, 8); + output.WriteInt32(Field130); + } + if (Field131 != 0) + { + output.WriteRawTag(152, 8); + output.WriteInt32(Field131); + } + if (Field150 != 0) + { + output.WriteRawTag(176, 9); + output.WriteInt32(Field150); + } + if (Field271 != 0) + { + output.WriteRawTag(248, 16); + output.WriteInt32(Field271); + } + if (Field272 != 0) + { + output.WriteRawTag(128, 17); + output.WriteInt32(Field272); + } + if (Field280 != 0) + { + output.WriteRawTag(192, 17); + output.WriteInt32(Field280); + } + if (_unknownFields != null) + { + _unknownFields.WriteTo(ref output); + } + } + + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public int CalculateSize() { int size = 0; if (Field1.Length != 0) { @@ -1310,9 +1525,234 @@ public void MergeFrom(pb::CodedInputStream input) { } } - } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public void MergeFrom(ref pb::CodedInputReader input) + { + uint tag; + while ((tag = input.ReadTag()) != 0) + { + switch (tag) + { + default: + _unknownFields = pb::UnknownFieldSet.MergeFieldFrom(_unknownFields, ref input); + break; + case 10: + { + Field1 = input.ReadString(); + break; + } + case 16: + { + Field2 = input.ReadInt32(); + break; + } + case 24: + { + Field3 = input.ReadInt32(); + break; + } + case 34: + { + Field4 = input.ReadString(); + break; + } + case 42: + case 41: + { + field5_.AddEntriesFrom(ref input, _repeated_field5_codec); + break; + } + case 48: + { + Field6 = input.ReadInt32(); + break; + } + case 58: + { + Field7 = input.ReadString(); + break; + } + case 74: + { + Field9 = input.ReadString(); + break; + } + case 96: + { + Field12 = input.ReadBool(); + break; + } + case 104: + { + Field13 = input.ReadBool(); + break; + } + case 112: + { + Field14 = input.ReadBool(); + break; + } + case 122: + { + if (field15_ == null) + { + Field15 = new global::Benchmarks.Proto3.GoogleMessage1SubMessage(); + } + input.ReadMessage(Field15); + break; + } + case 128: + { + Field16 = input.ReadInt32(); + break; + } + case 136: + { + Field17 = input.ReadBool(); + break; + } + case 146: + { + Field18 = input.ReadString(); + break; + } + case 176: + { + Field22 = input.ReadInt64(); + break; + } + case 184: + { + Field23 = input.ReadInt32(); + break; + } + case 192: + { + Field24 = input.ReadBool(); + break; + } + case 200: + { + Field25 = input.ReadInt32(); + break; + } + case 232: + { + Field29 = input.ReadInt32(); + break; + } + case 240: + { + Field30 = input.ReadBool(); + break; + } + case 472: + { + Field59 = input.ReadBool(); + break; + } + case 480: + { + Field60 = input.ReadInt32(); + break; + } + case 536: + { + Field67 = input.ReadInt32(); + break; + } + case 544: + { + Field68 = input.ReadInt32(); + break; + } + case 624: + { + Field78 = input.ReadBool(); + break; + } + case 640: + { + Field80 = input.ReadBool(); + break; + } + case 648: + { + Field81 = input.ReadBool(); + break; + } + case 800: + { + Field100 = input.ReadInt32(); + break; + } + case 808: + { + Field101 = input.ReadInt32(); + break; + } + case 818: + { + Field102 = input.ReadString(); + break; + } + case 826: + { + Field103 = input.ReadString(); + break; + } + case 832: + { + Field104 = input.ReadInt32(); + break; + } + case 1024: + { + Field128 = input.ReadInt32(); + break; + } + case 1034: + { + Field129 = input.ReadString(); + break; + } + case 1040: + { + Field130 = input.ReadInt32(); + break; + } + case 1048: + { + Field131 = input.ReadInt32(); + break; + } + case 1200: + { + Field150 = input.ReadInt32(); + break; + } + case 2168: + { + Field271 = input.ReadInt32(); + break; + } + case 2176: + { + Field272 = input.ReadInt32(); + break; + } + case 2240: + { + Field280 = input.ReadInt32(); + break; + } + } + } + } + + } - public sealed partial class GoogleMessage1SubMessage : pb::IMessage { + public sealed partial class GoogleMessage1SubMessage : pb::IMessage, pb::ISpanMessage { private static readonly pb::MessageParser _parser = new pb::MessageParser(() => new GoogleMessage1SubMessage()); private pb::UnknownFieldSet _unknownFields; [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -1328,6 +1768,11 @@ public sealed partial class GoogleMessage1SubMessage : pb::IMessage _readOnlySequence; + + [GlobalSetup] + public void GlobalSetup() + { + MemoryStream ms = new MemoryStream(); + CodedOutputStream output = new CodedOutputStream(ms); + + GoogleMessage1 googleMessage1 = new GoogleMessage1(); + googleMessage1.Field1 = "Text" + new string('!', 200); + googleMessage1.Field2 = 2; + googleMessage1.Field15 = new GoogleMessage1SubMessage(); + googleMessage1.Field15.Field1 = 1; + + googleMessage1.WriteTo(output); + output.Flush(); + + _message = googleMessage1; + _messageData = ms.ToArray(); + _messageSize = googleMessage1.CalculateSize(); + + _bufferWriter = new BufferWriter(new byte[_messageSize]); + _readOnlySequence = new ReadOnlySequence(_messageData); + } + + [Benchmark] + public void WriteToByteArray() + { + CodedOutputStream output = new CodedOutputStream(new byte[_messageSize]); + + _message.WriteTo(output); + } + + [Benchmark] + public void ParseFromByteArray() + { + var messageData = new byte[_messageData.Length]; + Array.Copy(_messageData, messageData, _messageData.Length); + + CodedInputStream input = new CodedInputStream(messageData); + + GoogleMessage1 message = new GoogleMessage1(); + message.MergeFrom(input); + } + + [Benchmark] + public void WriteToBufferWriter() + { + CodedOutputWriter output = new CodedOutputWriter(_bufferWriter); + + _message.WriteTo(ref output); + + _bufferWriter.Reset(); + } + + [Benchmark] + public void ParseFromReadOnlySequence() + { + CodedInputReader input = new CodedInputReader(_readOnlySequence); + + GoogleMessage1 message = new GoogleMessage1(); + message.MergeFrom(ref input); + } + } + + internal class BufferWriter : IBufferWriter + { + private readonly byte[] _buffer; + private int _position; + + public BufferWriter(byte[] buffer) + { + _buffer = buffer; + } + + public void Advance(int count) + { + _position += count; + } + + public void Reset() + { + _position = 0; + } + + public Memory GetMemory(int sizeHint = 0) + { + return _buffer.AsMemory(_position); + } + + public Span GetSpan(int sizeHint = 0) + { + return _buffer.AsSpan(_position); + } + } +} diff --git a/csharp/src/Google.Protobuf.Benchmarks/Program.cs b/csharp/src/Google.Protobuf.Benchmarks/Program.cs index 66f71d139154..4ca86cb253ef 100644 --- a/csharp/src/Google.Protobuf.Benchmarks/Program.cs +++ b/csharp/src/Google.Protobuf.Benchmarks/Program.cs @@ -41,6 +41,17 @@ namespace Google.Protobuf.Benchmarks /// class Program { - static void Main() => BenchmarkRunner.Run(); +#if true + static void Main() => BenchmarkRunner.Run(); +#else + static void Main() + { + var b = new JamesBenchmarks(); + b.GlobalSetup(); + + b.ParseFromReadOnlySequence(); + //b.ParseFromByteArray(); + } +#endif } } diff --git a/csharp/src/Google.Protobuf.Test/ArrayBufferWriter.cs b/csharp/src/Google.Protobuf.Test/ArrayBufferWriter.cs new file mode 100644 index 000000000000..91427dfd46dd --- /dev/null +++ b/csharp/src/Google.Protobuf.Test/ArrayBufferWriter.cs @@ -0,0 +1,158 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +#if NETCOREAPP2_1 +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Text; + +namespace Google.Protobuf +{ + internal class ArrayBufferWriter : IBufferWriter, IDisposable + { + private ResizableArray _buffer; + private readonly int _maximumSizeHint; + + public ArrayBufferWriter(int capacity, int? maximumSizeHint = null) + { + _buffer = new ResizableArray(ArrayPool.Shared.Rent(capacity)); + _maximumSizeHint = maximumSizeHint ?? int.MaxValue; + } + + public int CommitedByteCount => _buffer.Count; + + public void Clear() + { + _buffer.Count = 0; + } + + public ArraySegment Free => _buffer.Free; + + public ArraySegment Formatted => _buffer.Full; + + public Memory GetMemory(int minimumLength = 0) + { + minimumLength = Math.Min(minimumLength, _maximumSizeHint); + + if (minimumLength < 1) + { + minimumLength = 1; + } + + if (minimumLength > _buffer.FreeCount) + { + int doubleCount = _buffer.FreeCount * 2; + int newSize = minimumLength > doubleCount ? minimumLength : doubleCount; + byte[] newArray = ArrayPool.Shared.Rent(newSize + _buffer.Count); + byte[] oldArray = _buffer.Resize(newArray); + ArrayPool.Shared.Return(oldArray); + } + + return _buffer.FreeMemory; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Span GetSpan(int minimumLength = 0) + { + minimumLength = Math.Min(minimumLength, _maximumSizeHint); + + if (minimumLength < 1) + { + minimumLength = 1; + } + + if (minimumLength > _buffer.FreeCount) + { + int doubleCount = _buffer.FreeCount * 2; + int newSize = minimumLength > doubleCount ? minimumLength : doubleCount; + byte[] newArray = ArrayPool.Shared.Rent(newSize + _buffer.Count); + byte[] oldArray = _buffer.Resize(newArray); + ArrayPool.Shared.Return(oldArray); + } + + return _buffer.FreeSpan; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Advance(int bytes) + { + _buffer.Count += bytes; + if (_buffer.Count > _buffer.Capacity) + { + throw new InvalidOperationException("More bytes commited than returned from FreeBuffer"); + } + } + + public void Dispose() + { + byte[] array = _buffer.Array; + _buffer.Array = null; + ArrayPool.Shared.Return(array); + } + + private struct ResizableArray + { + public ResizableArray(T[] array, int count = 0) + { + Array = array; + Count = count; + } + + public T[] Array { get; set; } + + public int Count { get; set; } + + public int Capacity => Array.Length; + + public T[] Resize(T[] newArray) + { + T[] oldArray = Array; + Array.AsSpan(0, Count).CopyTo(newArray); // CopyTo will throw if newArray.Length < _count + Array = newArray; + return oldArray; + } + + public ArraySegment Full => new ArraySegment(Array, 0, Count); + + public ArraySegment Free => new ArraySegment(Array, Count, Array.Length - Count); + + public Span FreeSpan => new Span(Array, Count, Array.Length - Count); + + public Memory FreeMemory => new Memory(Array, Count, Array.Length - Count); + + public int FreeCount => Array.Length - Count; + } + } +} +#endif \ No newline at end of file diff --git a/csharp/src/Google.Protobuf.Test/CodedInputReaderTest.cs b/csharp/src/Google.Protobuf.Test/CodedInputReaderTest.cs new file mode 100644 index 000000000000..00420bfb260b --- /dev/null +++ b/csharp/src/Google.Protobuf.Test/CodedInputReaderTest.cs @@ -0,0 +1,579 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +#if NETCOREAPP2_1 +using System; +using System.Buffers; +using System.IO; +using Google.Protobuf.TestProtos; +using NUnit.Framework; + +namespace Google.Protobuf +{ + public class CodedInputReaderTest + { + /// + /// Helper to construct a byte array from a bunch of bytes. The inputs are + /// actually ints so that I can use hex notation and not get stupid errors + /// about precision. + /// + private static byte[] Bytes(params int[] bytesAsInts) + { + byte[] bytes = new byte[bytesAsInts.Length]; + for (int i = 0; i < bytesAsInts.Length; i++) + { + bytes[i] = (byte) bytesAsInts[i]; + } + return bytes; + } + + /// + /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() + /// + private static void AssertReadVarint(byte[] data, ulong value) + { + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(data)); + Assert.AreEqual((uint) value, input.ReadRawVarint32()); + + Assert.AreEqual(true, input.IsAtEnd); + + input = new CodedInputReader(new ReadOnlySequence(data)); + Assert.AreEqual(value, input.ReadRawVarint64()); + + Assert.AreEqual(true, input.IsAtEnd); + } + + /// + /// Parses the given bytes using ReadRawVarint32() and ReadRawVarint64() and + /// expects them to fail with an InvalidProtocolBufferException whose + /// description matches the given one. + /// + private static void AssertReadVarintFailure(InvalidProtocolBufferException expected, byte[] data) + { + var exception = Assert.Throws(() => + { + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(data)); + input.ReadRawVarint32(); + }); + Assert.AreEqual(expected.Message, exception.Message); + + exception = Assert.Throws(() => + { + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(data)); + input.ReadRawVarint64(); + }); + Assert.AreEqual(expected.Message, exception.Message); + } + + [Test] + public void ReadTag() + { + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(Bytes(0x7f))); + + Assert.AreEqual(127, input.ReadTag()); + Assert.IsTrue(input.IsAtEnd); + Assert.AreEqual(0, input.ReadTag()); + } + + [Test] + public void ReadVarint() + { + AssertReadVarint(Bytes(0x00), 0); + AssertReadVarint(Bytes(0x01), 1); + AssertReadVarint(Bytes(0x7f), 127); + // 14882 + AssertReadVarint(Bytes(0xa2, 0x74), (0x22 << 0) | (0x74 << 7)); + // 2961488830 + AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x0b), + (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | + (0x0bL << 28)); + + // 64-bit + // 7256456126 + AssertReadVarint(Bytes(0xbe, 0xf7, 0x92, 0x84, 0x1b), + (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | + (0x1bL << 28)); + // 41256202580718336 + AssertReadVarint(Bytes(0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49), + (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) | + (0x43L << 28) | (0x49L << 35) | (0x24L << 42) | (0x49L << 49)); + // 11964378330978735131 + AssertReadVarint(Bytes(0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01), + (0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) | + (0x3bUL << 28) | (0x56UL << 35) | (0x00UL << 42) | + (0x05UL << 49) | (0x26UL << 56) | (0x01UL << 63)); + + // Failures + AssertReadVarintFailure( + InvalidProtocolBufferException.MalformedVarint(), + Bytes(0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, + 0x00)); + AssertReadVarintFailure( + InvalidProtocolBufferException.TruncatedMessage(), + Bytes(0x80)); + } + + /// + /// Parses the given bytes using ReadRawLittleEndian32() and checks + /// that the result matches the given value. + /// + private static void AssertReadLittleEndian32(byte[] data, uint value) + { + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(data)); + Assert.AreEqual(value, input.ReadRawLittleEndian32()); + Assert.IsTrue(input.IsAtEnd); + } + + /// + /// Parses the given bytes using ReadRawLittleEndian64() and checks + /// that the result matches the given value. + /// + private static void AssertReadLittleEndian64(byte[] data, ulong value) + { + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(data)); + Assert.AreEqual(value, input.ReadRawLittleEndian64()); + Assert.IsTrue(input.IsAtEnd); + } + + [Test] + public void ReadLittleEndian() + { + AssertReadLittleEndian32(Bytes(0x78, 0x56, 0x34, 0x12), 0x12345678); + AssertReadLittleEndian32(Bytes(0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef0); + + AssertReadLittleEndian64(Bytes(0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12), + 0x123456789abcdef0L); + AssertReadLittleEndian64( + Bytes(0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a), 0x9abcdef012345678UL); + } + + [Test] + public void DecodeZigZag32() + { + Assert.AreEqual(0, CodedInputReader.DecodeZigZag32(0)); + Assert.AreEqual(-1, CodedInputReader.DecodeZigZag32(1)); + Assert.AreEqual(1, CodedInputReader.DecodeZigZag32(2)); + Assert.AreEqual(-2, CodedInputReader.DecodeZigZag32(3)); + Assert.AreEqual(0x3FFFFFFF, CodedInputReader.DecodeZigZag32(0x7FFFFFFE)); + Assert.AreEqual(unchecked((int) 0xC0000000), CodedInputReader.DecodeZigZag32(0x7FFFFFFF)); + Assert.AreEqual(0x7FFFFFFF, CodedInputReader.DecodeZigZag32(0xFFFFFFFE)); + Assert.AreEqual(unchecked((int) 0x80000000), CodedInputReader.DecodeZigZag32(0xFFFFFFFF)); + } + + [Test] + public void DecodeZigZag64() + { + Assert.AreEqual(0, CodedInputReader.DecodeZigZag64(0)); + Assert.AreEqual(-1, CodedInputReader.DecodeZigZag64(1)); + Assert.AreEqual(1, CodedInputReader.DecodeZigZag64(2)); + Assert.AreEqual(-2, CodedInputReader.DecodeZigZag64(3)); + Assert.AreEqual(0x000000003FFFFFFFL, CodedInputReader.DecodeZigZag64(0x000000007FFFFFFEL)); + Assert.AreEqual(unchecked((long) 0xFFFFFFFFC0000000L), CodedInputReader.DecodeZigZag64(0x000000007FFFFFFFL)); + Assert.AreEqual(0x000000007FFFFFFFL, CodedInputReader.DecodeZigZag64(0x00000000FFFFFFFEL)); + Assert.AreEqual(unchecked((long) 0xFFFFFFFF80000000L), CodedInputReader.DecodeZigZag64(0x00000000FFFFFFFFL)); + Assert.AreEqual(0x7FFFFFFFFFFFFFFFL, CodedInputReader.DecodeZigZag64(0xFFFFFFFFFFFFFFFEL)); + Assert.AreEqual(unchecked((long) 0x8000000000000000L), CodedInputReader.DecodeZigZag64(0xFFFFFFFFFFFFFFFFL)); + } + + [Test] + public void ReadHugeBlob() + { + // Allocate and initialize a 1MB blob. + byte[] blob = new byte[1 << 20]; + for (int i = 0; i < blob.Length; i++) + { + blob[i] = (byte) i; + } + + // Make a message containing it. + var message = new TestAllTypes { SingleBytes = ByteString.CopyFrom(blob) }; + + // Serialize and parse it. Make sure to parse from an InputStream, not + // directly from a ByteString, so that CodedInputStream uses buffered + // reading. + TestAllTypes message2 = TestAllTypes.Parser.ParseFrom(message.ToByteString()); + + Assert.AreEqual(message, message2); + } + + [Test] + public void ReadMaliciouslyLargeBlob() + { + MemoryStream ms = new MemoryStream(); + CodedOutputStream output = new CodedOutputStream(ms); + + uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); + output.WriteRawVarint32(tag); + output.WriteRawVarint32(0x7FFFFFFF); + output.WriteRawBytes(new byte[32]); // Pad with a few random bytes. + output.Flush(); + ms.Position = 0; + + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(ms.ToArray())); + Assert.AreEqual(tag, input.ReadTag()); + + try + { + input.ReadBytes(); + Assert.Fail(); + } + catch (InvalidProtocolBufferException) + { + } + } + + // Representations of a tag for field 0 with various wire types + [Test] + [TestCase(0)] + [TestCase(1)] + [TestCase(2)] + [TestCase(3)] + [TestCase(4)] + [TestCase(5)] + public void ReadTag_ZeroFieldRejected(byte tag) + { + CodedInputReader cis = new CodedInputReader(new ReadOnlySequence(new byte[] { tag })); + try + { + cis.ReadTag(); + Assert.Fail(); + } + catch (InvalidProtocolBufferException) + { + } + } + + internal static TestRecursiveMessage MakeRecursiveMessage(int depth) + { + if (depth == 0) + { + return new TestRecursiveMessage { I = 5 }; + } + else + { + return new TestRecursiveMessage { A = MakeRecursiveMessage(depth - 1) }; + } + } + + internal static void AssertMessageDepth(TestRecursiveMessage message, int depth) + { + if (depth == 0) + { + Assert.IsNull(message.A); + Assert.AreEqual(5, message.I); + } + else + { + Assert.IsNotNull(message.A); + AssertMessageDepth(message.A, depth - 1); + } + } + + [Test] + public void MaliciousRecursion() + { + ByteString atRecursiveLimit = MakeRecursiveMessage(CodedInputStream.DefaultRecursionLimit).ToByteString(); + ByteString beyondRecursiveLimit = MakeRecursiveMessage(CodedInputStream.DefaultRecursionLimit + 1).ToByteString(); + + AssertMessageDepth(TestRecursiveMessage.Parser.ParseFrom(atRecursiveLimit), CodedInputStream.DefaultRecursionLimit); + + Assert.Throws(() => TestRecursiveMessage.Parser.ParseFrom(beyondRecursiveLimit)); + + CodedInputStream input = CodedInputStream.CreateWithLimits(new MemoryStream(atRecursiveLimit.ToByteArray()), 1000000, CodedInputStream.DefaultRecursionLimit - 1); + Assert.Throws(() => TestRecursiveMessage.Parser.ParseFrom(input)); + } + + /// + /// Tests that if we read an string that contains invalid UTF-8, no exception + /// is thrown. Instead, the invalid bytes are replaced with the Unicode + /// "replacement character" U+FFFD. + /// + [Test] + public void ReadInvalidUtf8() + { + MemoryStream ms = new MemoryStream(); + CodedOutputStream output = new CodedOutputStream(ms); + + uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); + output.WriteRawVarint32(tag); + output.WriteRawVarint32(1); + output.WriteRawBytes(new byte[] {0x80}); + output.Flush(); + ms.Position = 0; + + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(ms.ToArray())); + + Assert.AreEqual(tag, input.ReadTag()); + string text = input.ReadString(); + Assert.AreEqual('\ufffd', text[0]); + } + + [Test] + public void ReadNegativeSizedStringThrowsInvalidProtocolBufferException() + { + MemoryStream ms = new MemoryStream(); + CodedOutputStream output = new CodedOutputStream(ms); + + uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); + output.WriteRawVarint32(tag); + output.WriteLength(-1); + output.Flush(); + ms.Position = 0; + + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(ms.ToArray())); + + Assert.AreEqual(tag, input.ReadTag()); + try + { + input.ReadString(); + Assert.Fail(); + } + catch (InvalidProtocolBufferException) + { + } + } + + [Test] + public void ReadNegativeSizedBytesThrowsInvalidProtocolBufferException() + { + MemoryStream ms = new MemoryStream(); + CodedOutputStream output = new CodedOutputStream(ms); + + uint tag = WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited); + output.WriteRawVarint32(tag); + output.WriteLength(-1); + output.Flush(); + ms.Position = 0; + + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(ms.ToArray())); + + Assert.AreEqual(tag, input.ReadTag()); + try + { + input.ReadBytes(); + Assert.Fail(); + } + catch (InvalidProtocolBufferException) + { + } + } + + [Test] + public void TestNegativeEnum() + { + byte[] bytes = { 0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x01 }; + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(bytes)); + Assert.AreEqual((int)SampleEnum.NegativeValue, input.ReadEnum()); + Assert.IsTrue(input.IsAtEnd); + } + + [Test] + public void Tag0Throws() + { + var input = new CodedInputReader(new ReadOnlySequence(new byte[] { 0 })); + try + { + input.ReadTag(); + Assert.Fail(); + } + catch (InvalidProtocolBufferException) + { + } + } + + [Test] + public void SkipGroup() + { + // Create an output stream with a group in: + // Field 1: string "field 1" + // Field 2: group containing: + // Field 1: fixed int32 value 100 + // Field 2: string "ignore me" + // Field 3: nested group containing + // Field 1: fixed int64 value 1000 + // Field 3: string "field 3" + var stream = new MemoryStream(); + var output = new CodedOutputStream(stream); + output.WriteTag(1, WireFormat.WireType.LengthDelimited); + output.WriteString("field 1"); + + // The outer group... + output.WriteTag(2, WireFormat.WireType.StartGroup); + output.WriteTag(1, WireFormat.WireType.Fixed32); + output.WriteFixed32(100); + output.WriteTag(2, WireFormat.WireType.LengthDelimited); + output.WriteString("ignore me"); + // The nested group... + output.WriteTag(3, WireFormat.WireType.StartGroup); + output.WriteTag(1, WireFormat.WireType.Fixed64); + output.WriteFixed64(1000); + // Note: Not sure the field number is relevant for end group... + output.WriteTag(3, WireFormat.WireType.EndGroup); + + // End the outer group + output.WriteTag(2, WireFormat.WireType.EndGroup); + + output.WriteTag(3, WireFormat.WireType.LengthDelimited); + output.WriteString("field 3"); + output.Flush(); + stream.Position = 0; + + // Now act like a generated client + var input = new CodedInputReader(new ReadOnlySequence(stream.ToArray())); + Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited), input.ReadTag()); + Assert.AreEqual("field 1", input.ReadString()); + Assert.AreEqual(WireFormat.MakeTag(2, WireFormat.WireType.StartGroup), input.ReadTag()); + input.SkipLastField(); // Should consume the whole group, including the nested one. + Assert.AreEqual(WireFormat.MakeTag(3, WireFormat.WireType.LengthDelimited), input.ReadTag()); + Assert.AreEqual("field 3", input.ReadString()); + } + + [Test] + public void SkipGroup_WrongEndGroupTag() + { + // Create an output stream with: + // Field 1: string "field 1" + // Start group 2 + // Field 3: fixed int32 + // End group 4 (should give an error) + var stream = new MemoryStream(); + var output = new CodedOutputStream(stream); + output.WriteTag(1, WireFormat.WireType.LengthDelimited); + output.WriteString("field 1"); + + // The outer group... + output.WriteTag(2, WireFormat.WireType.StartGroup); + output.WriteTag(3, WireFormat.WireType.Fixed32); + output.WriteFixed32(100); + output.WriteTag(4, WireFormat.WireType.EndGroup); + output.Flush(); + stream.Position = 0; + + // Now act like a generated client + var input = new CodedInputReader(new ReadOnlySequence(stream.ToArray())); + Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.LengthDelimited), input.ReadTag()); + Assert.AreEqual("field 1", input.ReadString()); + Assert.AreEqual(WireFormat.MakeTag(2, WireFormat.WireType.StartGroup), input.ReadTag()); + + try + { + input.SkipLastField(); + Assert.Fail(); + } + catch (InvalidProtocolBufferException) + { + } + } + + [Test] + public void RogueEndGroupTag() + { + // If we have an end-group tag without a leading start-group tag, generated + // code will just call SkipLastField... so that should fail. + + var stream = new MemoryStream(); + var output = new CodedOutputStream(stream); + output.WriteTag(1, WireFormat.WireType.EndGroup); + output.Flush(); + stream.Position = 0; + + var input = new CodedInputReader(new ReadOnlySequence(stream.ToArray())); + Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.EndGroup), input.ReadTag()); + + try + { + input.SkipLastField(); + Assert.Fail(); + } + catch (InvalidProtocolBufferException) + { + } + } + + [Test] + public void EndOfStreamReachedWhileSkippingGroup() + { + var stream = new MemoryStream(); + var output = new CodedOutputStream(stream); + output.WriteTag(1, WireFormat.WireType.StartGroup); + output.WriteTag(2, WireFormat.WireType.StartGroup); + output.WriteTag(2, WireFormat.WireType.EndGroup); + + output.Flush(); + stream.Position = 0; + + // Now act like a generated client + var input = new CodedInputReader(new ReadOnlySequence(stream.ToArray())); + input.ReadTag(); + + try + { + input.SkipLastField(); + Assert.Fail(); + } + catch (InvalidProtocolBufferException) + { + } + } + + [Test] + public void RecursionLimitAppliedWhileSkippingGroup() + { + var stream = new MemoryStream(); + var output = new CodedOutputStream(stream); + for (int i = 0; i < CodedInputStream.DefaultRecursionLimit + 1; i++) + { + output.WriteTag(1, WireFormat.WireType.StartGroup); + } + for (int i = 0; i < CodedInputStream.DefaultRecursionLimit + 1; i++) + { + output.WriteTag(1, WireFormat.WireType.EndGroup); + } + output.Flush(); + stream.Position = 0; + + // Now act like a generated client + var input = new CodedInputReader(new ReadOnlySequence(stream.ToArray())); + Assert.AreEqual(WireFormat.MakeTag(1, WireFormat.WireType.StartGroup), input.ReadTag()); + + try + { + input.SkipLastField(); + Assert.Fail(); + } + catch (InvalidProtocolBufferException) + { + } + } + } +} +#endif \ No newline at end of file diff --git a/csharp/src/Google.Protobuf.Test/CodedOutputWriterTest.cs b/csharp/src/Google.Protobuf.Test/CodedOutputWriterTest.cs new file mode 100644 index 000000000000..fad39e48e004 --- /dev/null +++ b/csharp/src/Google.Protobuf.Test/CodedOutputWriterTest.cs @@ -0,0 +1,444 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +#if NETCOREAPP2_1 +using System; +using System.Buffers; +using System.IO; +using System.Text; +using Google.Protobuf.TestProtos; +using NUnit.Framework; + +namespace Google.Protobuf +{ + public class CodedOutputWriterTest + { + /// + /// Writes the given value using WriteRawVarint32() and WriteRawVarint64() and + /// checks that the result matches the given bytes + /// + private static void AssertWriteVarint(byte[] data, ulong value) + { + // Only do 32-bit write if the value fits in 32 bits. + if ((value >> 32) == 0) + { + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024); + CodedOutputWriter output = new CodedOutputWriter(rawOutput); + output.WriteRawVarint32((uint) value); + output.Flush(); + Assert.AreEqual(data, rawOutput.Formatted.ToArray()); + // Also try computing size. + Assert.AreEqual(data.Length, CodedOutputStream.ComputeRawVarint32Size((uint) value)); + } + + { + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024); + CodedOutputWriter output = new CodedOutputWriter(rawOutput); + output.WriteRawVarint64(value); + output.Flush(); + Assert.AreEqual(data, rawOutput.Formatted.ToArray()); + + // Also try computing size. + Assert.AreEqual(data.Length, CodedOutputStream.ComputeRawVarint64Size(value)); + } + } + + /// + /// Tests WriteRawVarint32() and WriteRawVarint64() + /// + [Test] + public void WriteVarint() + { + AssertWriteVarint(new byte[] {0x00}, 0); + AssertWriteVarint(new byte[] {0x01}, 1); + AssertWriteVarint(new byte[] {0x7f}, 127); + // 14882 + AssertWriteVarint(new byte[] {0xa2, 0x74}, (0x22 << 0) | (0x74 << 7)); + // 2961488830 + AssertWriteVarint(new byte[] {0xbe, 0xf7, 0x92, 0x84, 0x0b}, + (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | + (0x0bL << 28)); + + // 64-bit + // 7256456126 + AssertWriteVarint(new byte[] {0xbe, 0xf7, 0x92, 0x84, 0x1b}, + (0x3e << 0) | (0x77 << 7) | (0x12 << 14) | (0x04 << 21) | + (0x1bL << 28)); + // 41256202580718336 + AssertWriteVarint( + new byte[] {0x80, 0xe6, 0xeb, 0x9c, 0xc3, 0xc9, 0xa4, 0x49}, + (0x00 << 0) | (0x66 << 7) | (0x6b << 14) | (0x1c << 21) | + (0x43UL << 28) | (0x49L << 35) | (0x24UL << 42) | (0x49UL << 49)); + // 11964378330978735131 + AssertWriteVarint( + new byte[] {0x9b, 0xa8, 0xf9, 0xc2, 0xbb, 0xd6, 0x80, 0x85, 0xa6, 0x01}, + unchecked((ulong) + ((0x1b << 0) | (0x28 << 7) | (0x79 << 14) | (0x42 << 21) | + (0x3bL << 28) | (0x56L << 35) | (0x00L << 42) | + (0x05L << 49) | (0x26L << 56) | (0x01L << 63)))); + } + + /// + /// Parses the given bytes using WriteRawLittleEndian32() and checks + /// that the result matches the given value. + /// + private static void AssertWriteLittleEndian32(byte[] data, uint value) + { + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024); + CodedOutputWriter output = new CodedOutputWriter(rawOutput); + output.WriteRawLittleEndian32(value); + output.Flush(); + Assert.AreEqual(data, rawOutput.Formatted.ToArray()); + } + + /// + /// Parses the given bytes using WriteRawLittleEndian64() and checks + /// that the result matches the given value. + /// + private static void AssertWriteLittleEndian64(byte[] data, ulong value) + { + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024); + CodedOutputWriter output = new CodedOutputWriter(rawOutput); + output.WriteRawLittleEndian64(value); + output.Flush(); + Assert.AreEqual(data, rawOutput.Formatted.ToArray()); + } + + /// + /// Tests writeRawLittleEndian32() and writeRawLittleEndian64(). + /// + [Test] + public void WriteLittleEndian() + { + AssertWriteLittleEndian32(new byte[] {0x78, 0x56, 0x34, 0x12}, 0x12345678); + AssertWriteLittleEndian32(new byte[] {0xf0, 0xde, 0xbc, 0x9a}, 0x9abcdef0); + + AssertWriteLittleEndian64( + new byte[] {0xf0, 0xde, 0xbc, 0x9a, 0x78, 0x56, 0x34, 0x12}, + 0x123456789abcdef0L); + AssertWriteLittleEndian64( + new byte[] {0x78, 0x56, 0x34, 0x12, 0xf0, 0xde, 0xbc, 0x9a}, + 0x9abcdef012345678UL); + } + + [Test] + public void EncodeZigZag32() + { + Assert.AreEqual(0u, CodedOutputStream.EncodeZigZag32(0)); + Assert.AreEqual(1u, CodedOutputStream.EncodeZigZag32(-1)); + Assert.AreEqual(2u, CodedOutputStream.EncodeZigZag32(1)); + Assert.AreEqual(3u, CodedOutputStream.EncodeZigZag32(-2)); + Assert.AreEqual(0x7FFFFFFEu, CodedOutputStream.EncodeZigZag32(0x3FFFFFFF)); + Assert.AreEqual(0x7FFFFFFFu, CodedOutputStream.EncodeZigZag32(unchecked((int) 0xC0000000))); + Assert.AreEqual(0xFFFFFFFEu, CodedOutputStream.EncodeZigZag32(0x7FFFFFFF)); + Assert.AreEqual(0xFFFFFFFFu, CodedOutputStream.EncodeZigZag32(unchecked((int) 0x80000000))); + } + + [Test] + public void EncodeZigZag64() + { + Assert.AreEqual(0u, CodedOutputStream.EncodeZigZag64(0)); + Assert.AreEqual(1u, CodedOutputStream.EncodeZigZag64(-1)); + Assert.AreEqual(2u, CodedOutputStream.EncodeZigZag64(1)); + Assert.AreEqual(3u, CodedOutputStream.EncodeZigZag64(-2)); + Assert.AreEqual(0x000000007FFFFFFEuL, + CodedOutputStream.EncodeZigZag64(unchecked((long) 0x000000003FFFFFFFUL))); + Assert.AreEqual(0x000000007FFFFFFFuL, + CodedOutputStream.EncodeZigZag64(unchecked((long) 0xFFFFFFFFC0000000UL))); + Assert.AreEqual(0x00000000FFFFFFFEuL, + CodedOutputStream.EncodeZigZag64(unchecked((long) 0x000000007FFFFFFFUL))); + Assert.AreEqual(0x00000000FFFFFFFFuL, + CodedOutputStream.EncodeZigZag64(unchecked((long) 0xFFFFFFFF80000000UL))); + Assert.AreEqual(0xFFFFFFFFFFFFFFFEL, + CodedOutputStream.EncodeZigZag64(unchecked((long) 0x7FFFFFFFFFFFFFFFUL))); + Assert.AreEqual(0xFFFFFFFFFFFFFFFFL, + CodedOutputStream.EncodeZigZag64(unchecked((long) 0x8000000000000000UL))); + } + + [Test] + public void RoundTripZigZag32() + { + // Some easier-to-verify round-trip tests. The inputs (other than 0, 1, -1) + // were chosen semi-randomly via keyboard bashing. + Assert.AreEqual(0, CodedInputStream.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(0))); + Assert.AreEqual(1, CodedInputStream.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(1))); + Assert.AreEqual(-1, CodedInputStream.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(-1))); + Assert.AreEqual(14927, CodedInputStream.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(14927))); + Assert.AreEqual(-3612, CodedInputStream.DecodeZigZag32(CodedOutputStream.EncodeZigZag32(-3612))); + } + + [Test] + public void RoundTripZigZag64() + { + Assert.AreEqual(0, CodedInputStream.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(0))); + Assert.AreEqual(1, CodedInputStream.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(1))); + Assert.AreEqual(-1, CodedInputStream.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(-1))); + Assert.AreEqual(14927, CodedInputStream.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(14927))); + Assert.AreEqual(-3612, CodedInputStream.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(-3612))); + + Assert.AreEqual(856912304801416L, + CodedInputStream.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(856912304801416L))); + Assert.AreEqual(-75123905439571256L, + CodedInputStream.DecodeZigZag64(CodedOutputStream.EncodeZigZag64(-75123905439571256L))); + } + + [Test] + public void TestNegativeEnumNoTag() + { + Assert.AreEqual(10, CodedOutputStream.ComputeInt32Size(-2)); + Assert.AreEqual(10, CodedOutputStream.ComputeEnumSize((int) SampleEnum.NegativeValue)); + + ArrayBufferWriter rawOutput = new ArrayBufferWriter(10); + CodedOutputWriter output = new CodedOutputWriter(rawOutput); + output.WriteEnum((int) SampleEnum.NegativeValue); + output.Flush(); + + Assert.AreEqual(10, rawOutput.Formatted.Count); + Assert.AreEqual("FE-FF-FF-FF-FF-FF-FF-FF-FF-01", BitConverter.ToString(rawOutput.Formatted.ToArray())); + } + + [Test] + public void TestCodedInputOutputPosition() + { + byte[] content = new byte[110]; + for (int i = 0; i < content.Length; i++) + content[i] = (byte)i; + + byte[] child; + { + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024); + CodedOutputWriter cout = new CodedOutputWriter(rawOutput); + // Field 11: numeric value: 500 + cout.WriteTag(11, WireFormat.WireType.Varint); + cout.Flush(); + Assert.AreEqual(1, rawOutput.CommitedByteCount); + cout.WriteInt32(500); + cout.Flush(); + Assert.AreEqual(3, rawOutput.CommitedByteCount); + //Field 12: length delimited 120 bytes + cout.WriteTag(12, WireFormat.WireType.LengthDelimited); + cout.Flush(); + Assert.AreEqual(4, rawOutput.CommitedByteCount); + cout.WriteBytes(ByteString.CopyFrom(content)); + cout.Flush(); + Assert.AreEqual(115, rawOutput.CommitedByteCount); + // Field 13: fixed numeric value: 501 + cout.WriteTag(13, WireFormat.WireType.Fixed32); + cout.Flush(); + Assert.AreEqual(116, rawOutput.CommitedByteCount); + cout.WriteSFixed32(501); + cout.Flush(); + Assert.AreEqual(120, rawOutput.CommitedByteCount); + + child = rawOutput.Formatted.ToArray(); + } + + byte[] bytes; + { + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024); + CodedOutputWriter cout = new CodedOutputWriter(rawOutput); + // Field 1: numeric value: 500 + cout.WriteTag(1, WireFormat.WireType.Varint); + cout.Flush(); + Assert.AreEqual(1, rawOutput.CommitedByteCount); + cout.WriteInt32(500); + cout.Flush(); + Assert.AreEqual(3, rawOutput.CommitedByteCount); + //Field 2: length delimited 120 bytes + cout.WriteTag(2, WireFormat.WireType.LengthDelimited); + cout.Flush(); + Assert.AreEqual(4, rawOutput.CommitedByteCount); + cout.WriteBytes(ByteString.CopyFrom(child)); + cout.Flush(); + Assert.AreEqual(125, rawOutput.CommitedByteCount); + // Field 3: fixed numeric value: 500 + cout.WriteTag(3, WireFormat.WireType.Fixed32); + cout.Flush(); + Assert.AreEqual(126, rawOutput.CommitedByteCount); + cout.WriteSFixed32(501); + cout.Flush(); + Assert.AreEqual(130, rawOutput.CommitedByteCount); + + bytes = rawOutput.Formatted.ToArray(); + } + + // Now test Input stream: + { + CodedInputReader cin = new CodedInputReader(new ReadOnlySequence(bytes)); + Assert.AreEqual(0, cin.Position); + // Field 1: + uint tag = cin.ReadTag(); + Assert.AreEqual(1, tag >> 3); + Assert.AreEqual(1, cin.Position); + Assert.AreEqual(500, cin.ReadInt32()); + Assert.AreEqual(3, cin.Position); + //Field 2: + tag = cin.ReadTag(); + Assert.AreEqual(2, tag >> 3); + Assert.AreEqual(4, cin.Position); + int childlen = cin.ReadLength(); + Assert.AreEqual(120, childlen); + Assert.AreEqual(5, cin.Position); + Assert.AreEqual(5, cin.Position); + // Now we are reading child message + { + // Field 11: numeric value: 500 + tag = cin.ReadTag(); + Assert.AreEqual(11, tag >> 3); + Assert.AreEqual(6, cin.Position); + Assert.AreEqual(500, cin.ReadInt32()); + Assert.AreEqual(8, cin.Position); + //Field 12: length delimited 120 bytes + tag = cin.ReadTag(); + Assert.AreEqual(12, tag >> 3); + Assert.AreEqual(9, cin.Position); + ByteString bstr = cin.ReadBytes(); + Assert.AreEqual(110, bstr.Length); + Assert.AreEqual((byte) 109, bstr[109]); + Assert.AreEqual(120, cin.Position); + // Field 13: fixed numeric value: 501 + tag = cin.ReadTag(); + Assert.AreEqual(13, tag >> 3); + // ROK - Previously broken here, this returned 126 failing to account for bufferSizeAfterLimit + Assert.AreEqual(121, cin.Position); + Assert.AreEqual(501, cin.ReadSFixed32()); + Assert.AreEqual(125, cin.Position); + } + Assert.AreEqual(125, cin.Position); + // Field 3: fixed numeric value: 501 + tag = cin.ReadTag(); + Assert.AreEqual(3, tag >> 3); + Assert.AreEqual(126, cin.Position); + Assert.AreEqual(501, cin.ReadSFixed32()); + Assert.AreEqual(130, cin.Position); + Assert.IsTrue(cin.IsAtEnd); + } + } + + [Test] + public void WriteAsciiSmallString() + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 32; i++) + { + sb.AppendLine(i.ToString()); + } + string s1 = sb.ToString(); + + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024, 1024); + CodedOutputWriter output = new CodedOutputWriter(rawOutput); + output.WriteString(s1); + output.Flush(); + + var textData = rawOutput.Formatted.Slice(1).ToArray(); + string s2 = Encoding.UTF8.GetString(textData); + + Assert.AreEqual(s1, s2); + + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(rawOutput.Formatted.ToArray())); + Assert.AreEqual(s1, input.ReadString()); + } + + [Test] + public void WriteUnicodeSmallString() + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 32; i++) + { + sb.AppendLine(i.ToString().Replace("0", "\x1369")); + } + string s1 = sb.ToString(); + + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024, 1024); + CodedOutputWriter output = new CodedOutputWriter(rawOutput); + output.WriteString(s1); + output.Flush(); + + var textData = rawOutput.Formatted.Slice(1).ToArray(); + string s2 = Encoding.UTF8.GetString(textData); + + Assert.AreEqual(s1, s2); + + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(rawOutput.Formatted.ToArray())); + Assert.AreEqual(s1, input.ReadString()); + } + + [Test] + public void WriteAsciiLargeString() + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 2048; i++) + { + sb.AppendLine(i.ToString()); + } + string s1 = sb.ToString(); + + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024, 1024); + CodedOutputWriter output = new CodedOutputWriter(rawOutput); + output.WriteString(s1); + output.Flush(); + + var textData = rawOutput.Formatted.Slice(2).ToArray(); + string s2 = Encoding.UTF8.GetString(textData); + + Assert.AreEqual(s1, s2); + + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(rawOutput.Formatted.ToArray())); + Assert.AreEqual(s1, input.ReadString()); + } + + [Test] + public void WriteUnicodeLargeString() + { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < 2048; i++) + { + sb.AppendLine(i.ToString().Replace("0", "\x1369")); + } + string s1 = sb.ToString(); + + ArrayBufferWriter rawOutput = new ArrayBufferWriter(1024, 1024); + CodedOutputWriter output = new CodedOutputWriter(rawOutput); + output.WriteString(s1); + output.Flush(); + + var textData = rawOutput.Formatted.Slice(2).ToArray(); + string s2 = Encoding.UTF8.GetString(textData); + + Assert.AreEqual(s1, s2); + + CodedInputReader input = new CodedInputReader(new ReadOnlySequence(rawOutput.Formatted.ToArray())); + Assert.AreEqual(s1, input.ReadString()); + } + } +} +#endif \ No newline at end of file diff --git a/csharp/src/Google.Protobuf/BufferWriter.cs b/csharp/src/Google.Protobuf/BufferWriter.cs new file mode 100644 index 000000000000..f77c023f97b9 --- /dev/null +++ b/csharp/src/Google.Protobuf/BufferWriter.cs @@ -0,0 +1,200 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +#if NETSTANDARD2_0 +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +namespace Google.Protobuf +{ + /// + /// A fast access struct that wraps . + /// + internal ref struct BufferWriter + { + /// + /// The underlying . + /// + private IBufferWriter _output; + + /// + /// The result of the last call to , less any bytes already "consumed" with . + /// Backing field for the property. + /// + private Span _span; + + /// + /// The number of uncommitted bytes (all the calls to since the last call to ). + /// + private int _buffered; + + /// + /// The total number of bytes written with this writer. + /// Backing field for the property. + /// + private long _bytesCommitted; + + /// + /// Initializes a new instance of the struct. + /// + /// The to be wrapped. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public BufferWriter(IBufferWriter output) + { + _buffered = 0; + _bytesCommitted = 0; + _output = output; + + _span = _output.GetSpan(); + } + + /// + /// Gets the result of the last call to . + /// + public Span Span => _span; + + /// + /// Gets the total number of bytes written with this writer. + /// + public long BytesCommitted => _bytesCommitted; + + /// + /// Gets the underlying this instance. + /// + internal IBufferWriter UnderlyingWriter => _output; + + public Span GetSpan(int sizeHint) + { + Ensure(sizeHint); + return this.Span; + } + + /// + /// Calls on the underlying writer + /// with the number of uncommitted bytes. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Commit() + { + var buffered = _buffered; + if (buffered > 0) + { + _bytesCommitted += buffered; + _buffered = 0; + _output.Advance(buffered); + _span = default; + } + } + + /// + /// Used to indicate that part of the buffer has been written to. + /// + /// The number of bytes written to. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Advance(int count) + { + _buffered += count; + _span = _span.Slice(count); + } + + /// + /// Copies the caller's buffer into this writer and calls with the length of the source buffer. + /// + /// The buffer to copy in. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Write(ReadOnlySpan source) + { + if (_span.Length >= source.Length) + { + source.CopyTo(_span); + Advance(source.Length); + } + else + { + WriteMultiBuffer(source); + } + } + + /// + /// Acquires a new buffer if necessary to ensure that some given number of bytes can be written to a single buffer. + /// + /// The number of bytes that must be allocated in a single buffer. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Ensure(int count = 1) + { + if (_span.Length < count) + { + EnsureMore(count); + } + } + + /// + /// Gets a fresh span to write to, with an optional minimum size. + /// + /// The minimum size for the next requested buffer. + [MethodImpl(MethodImplOptions.NoInlining)] + private void EnsureMore(int count = 0) + { + if (_buffered > 0) + { + Commit(); + } + + _span = _output.GetSpan(count); + } + + /// + /// Copies the caller's buffer into this writer, potentially across multiple buffers from the underlying writer. + /// + /// The buffer to copy into this writer. + private void WriteMultiBuffer(ReadOnlySpan source) + { + while (source.Length > 0) + { + if (_span.Length == 0) + { + EnsureMore(); + } + + var writable = Math.Min(source.Length, _span.Length); + source.Slice(0, writable).CopyTo(_span); + source = source.Slice(writable); + Advance(writable); + } + } + } +} +#endif \ No newline at end of file diff --git a/csharp/src/Google.Protobuf/CodedInputReader.cs b/csharp/src/Google.Protobuf/CodedInputReader.cs new file mode 100644 index 000000000000..c070297407cc --- /dev/null +++ b/csharp/src/Google.Protobuf/CodedInputReader.cs @@ -0,0 +1,787 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +#if NETSTANDARD2_0 +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.Collections.Generic; +using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +namespace Google.Protobuf +{ + // TODO: class name TBD (CodedInputReader, CodedInputContext, CodedInputByRefReader, WireFormatReader) + public ref struct CodedInputReader + { + private const int recursionLimit = 100; + + private SequenceReader reader; + private uint lastTag; + private int recursionDepth; + private long currentLimit; + + public CodedInputReader(ReadOnlySequence input) + { + reader = new SequenceReader(input); + lastTag = 0; + recursionDepth = 0; + currentLimit = long.MaxValue; + DiscardUnknownFields = false; + } + + public ReadOnlySequence Sequence => reader.Sequence; + + public long Position => reader.Consumed; + + public bool IsAtEnd => reader.End; + + /// + /// Returns the last tag read, or 0 if no tags have been read or we've read beyond + /// the end of the stream. + /// + internal uint LastTag { get { return lastTag; } } + + /// + /// Internal-only property; when set to true, unknown fields will be discarded while parsing. + /// + internal bool DiscardUnknownFields { get; set; } + + #region Reading of tags etc + + /// + /// Peeks at the next field tag. This is like calling , but the + /// tag is not consumed. (So a subsequent call to will return the + /// same value.) + /// + public uint PeekTag() + { + uint previousTag = lastTag; + long consumed = reader.Consumed; + + uint tag = ReadTag(); + + long rewindCount = reader.Consumed - consumed; + if (rewindCount > 0) + { + reader.Rewind(rewindCount); + lastTag = previousTag; + } + + return tag; + } + + private void ThrowEndOfStreamUnless(bool condition) + { + if (!condition) + { + throw InvalidProtocolBufferException.TruncatedMessage(); + } + } + + /// + /// Reads a field tag, returning the tag of 0 for "end of stream". + /// + /// + /// If this method returns 0, it doesn't necessarily mean the end of all + /// the data in this CodedInputStream; it may be the end of the logical stream + /// for an embedded message, for example. + /// + /// The next field tag, or 0 for end of stream. (0 is never a valid tag.) + public uint ReadTag() + { + uint tag; + byte value; + + bool hasValue = reader.TryRead(out value); + if (!hasValue) + { + ThrowEndOfStreamUnless(IsAtEnd); + + // End of stream + lastTag = 0; + return 0; + } + + if (value < 128) + { + tag = value; + } + else + { + int result = value & 0x7f; + ThrowEndOfStreamUnless(reader.TryRead(out value)); + if (value < 128) + { + result |= value << 7; + tag = (uint)result; + } + else + { + result = (value & 0x7f) << 7; + ThrowEndOfStreamUnless(reader.TryRead(out value)); + if (value < 128) + { + result |= value << 14; + tag = (uint)result; + } + else + { + result = (value & 0x7f) << 14; + ThrowEndOfStreamUnless(reader.TryRead(out value)); + result |= value << 21; + tag = (uint)result; + } + } + } + + if (WireFormat.GetTagFieldNumber(tag) == 0) + { + // If we actually read a tag with a field of 0, that's not a valid tag. + throw InvalidProtocolBufferException.InvalidTag(); + } + + lastTag = tag; + + return tag; + } + + /// + /// Skips the data for the field with the tag we've just read. + /// This should be called directly after , when + /// the caller wishes to skip an unknown field. + /// + /// + /// This method throws if the last-read tag was an end-group tag. + /// If a caller wishes to skip a group, they should skip the whole group, by calling this method after reading the + /// start-group tag. This behavior allows callers to call this method on any field they don't understand, correctly + /// resulting in an error if an end-group tag has not been paired with an earlier start-group tag. + /// + /// The last tag was an end-group tag + /// The last read operation read to the end of the logical stream + public void SkipLastField() + { + if (lastTag == 0) + { + throw new InvalidOperationException("SkipLastField cannot be called at the end of a stream"); + } + switch (WireFormat.GetTagWireType(lastTag)) + { + case WireFormat.WireType.StartGroup: + SkipGroup(lastTag); + break; + case WireFormat.WireType.EndGroup: + throw new InvalidProtocolBufferException( + "SkipLastField called on an end-group tag, indicating that the corresponding start-group was missing"); + case WireFormat.WireType.Fixed32: + ReadFixed32(); + break; + case WireFormat.WireType.Fixed64: + ReadFixed64(); + break; + case WireFormat.WireType.LengthDelimited: + var length = ReadLength(); + ThrowEndOfStreamUnless(reader.Remaining >= length); + reader.Advance(length); + break; + case WireFormat.WireType.Varint: + ReadRawVarint32(); + break; + } + } + + /// + /// Skip a group. + /// + internal void SkipGroup(uint startGroupTag) + { + // Note: Currently we expect this to be the way that groups are read. We could put the recursion + // depth changes into the ReadTag method instead, potentially... + recursionDepth++; + if (recursionDepth >= recursionLimit) + { + throw InvalidProtocolBufferException.RecursionLimitExceeded(); + } + uint tag; + while (true) + { + tag = ReadTag(); + if (tag == 0) + { + throw InvalidProtocolBufferException.TruncatedMessage(); + } + // Can't call SkipLastField for this case- that would throw. + if (WireFormat.GetTagWireType(tag) == WireFormat.WireType.EndGroup) + { + break; + } + // This recursion will allow us to handle nested groups. + SkipLastField(); + } + int startField = WireFormat.GetTagFieldNumber(startGroupTag); + int endField = WireFormat.GetTagFieldNumber(tag); + if (startField != endField) + { + throw new InvalidProtocolBufferException( + $"Mismatched end-group tag. Started with field {startField}; ended with field {endField}"); + } + recursionDepth--; + } + + /// + /// Reads a double field from the stream. + /// + public double ReadDouble() + { + return BitConverter.Int64BitsToDouble((long)ReadRawLittleEndian64()); + } + + /// + /// Reads a float field from the stream. + /// + public unsafe float ReadFloat() + { + byte* buffer = stackalloc byte[4]; + Span tempSpan = new Span(buffer, 4); + + ThrowEndOfStreamUnless(reader.TryCopyTo(tempSpan)); + reader.Advance(4); + + return Unsafe.ReadUnaligned(ref MemoryMarshal.GetReference(tempSpan)); + } + + /// + /// Reads a uint64 field from the stream. + /// + public ulong ReadUInt64() + { + return ReadRawVarint64(); + } + + /// + /// Reads an int64 field from the stream. + /// + public long ReadInt64() + { + return (long)ReadRawVarint64(); + } + + /// + /// Reads an int32 field from the stream. + /// + public int ReadInt32() + { + return (int)ReadRawVarint32(); + } + + /// + /// Reads a fixed64 field from the stream. + /// + public ulong ReadFixed64() + { + return ReadRawLittleEndian64(); + } + + /// + /// Reads a fixed32 field from the stream. + /// + public uint ReadFixed32() + { + return ReadRawLittleEndian32(); + } + + /// + /// Reads a bool field from the stream. + /// + public bool ReadBool() + { + return ReadRawVarint32() != 0; + } + + /// + /// Reads a string field from the stream. + /// + public unsafe string ReadString() + { + int length = ReadLength(); + + if (length == 0) + { + return string.Empty; + } + + if (length < 0) + { + throw InvalidProtocolBufferException.NegativeSize(); + } + + ReadOnlySpan unreadSpan = reader.UnreadSpan; + if (unreadSpan.Length >= length) + { + // Fast path: all bytes to decode appear in the same span. + ReadOnlySpan data = unreadSpan.Slice(0, length); + + string value; + fixed (byte* sourceBytes = &MemoryMarshal.GetReference(data)) + { + value = CodedOutputStream.Utf8Encoding.GetString(sourceBytes, length); + } + + reader.Advance(length); + return value; + } + else + { + return ReadStringSlow(length); + } + } + + /// + /// Reads a string assuming that it is spread across multiple spans in the . + /// + /// The length of the string to be decoded, in bytes. + /// The decoded string. + private string ReadStringSlow(int byteLength) + { + ThrowEndOfStreamUnless(reader.Remaining >= byteLength); + + // We need to decode bytes incrementally across multiple spans. + int maxCharLength = CodedOutputStream.Utf8Encoding.GetMaxCharCount(byteLength); + char[] charArray = ArrayPool.Shared.Rent(maxCharLength); + var decoder = CodedOutputStream.Utf8Encoding.GetDecoder(); + + int remainingByteLength = byteLength; + int initializedChars = 0; + while (remainingByteLength > 0) + { + int bytesRead = Math.Min(remainingByteLength, this.reader.UnreadSpan.Length); + remainingByteLength -= bytesRead; + bool flush = remainingByteLength == 0; + + unsafe + { + fixed (byte* pUnreadSpan = reader.UnreadSpan) + fixed (char* pCharArray = &charArray[initializedChars]) + { + initializedChars += decoder.GetChars(pUnreadSpan, bytesRead, pCharArray, charArray.Length - initializedChars, flush); + } + } + } + + string value = new string(charArray, 0, initializedChars); + ArrayPool.Shared.Return(charArray); + return value; + } + + /// + /// Reads an embedded message field value from the stream. + /// + public void ReadMessage(ISpanMessage builder) + { + int length = ReadLength(); + if (recursionDepth >= recursionLimit) + { + throw InvalidProtocolBufferException.RecursionLimitExceeded(); + } + + ++recursionDepth; + builder.MergeFrom(ref this); + CheckReadEndOfStreamTag(); + + --recursionDepth; + } + + /// + /// Reads an embedded group field from the stream. + /// + public void ReadGroup(ISpanMessage builder) + { + if (recursionDepth >= recursionLimit) + { + throw InvalidProtocolBufferException.RecursionLimitExceeded(); + } + ++recursionDepth; + builder.MergeFrom(ref this); + --recursionDepth; + } + + /// + /// Reads a bytes field value from the stream. + /// + public ByteString ReadBytes() + { + int length = ReadLength(); + + ThrowEndOfStreamUnless(reader.Remaining >= length); + + if (length < 0) + { + throw InvalidProtocolBufferException.NegativeSize(); + } + + var data = reader.Sequence.Slice(reader.Position, length); + + reader.Advance(length); + + return ByteString.AttachBytes(data.ToArray()); + } + + /// + /// Reads a uint32 field value from the stream. + /// + public uint ReadUInt32() + { + return ReadRawVarint32(); + } + + /// + /// Reads an enum field value from the stream. + /// + public int ReadEnum() + { + // Currently just a pass-through, but it's nice to separate it logically from WriteInt32. + return (int)ReadRawVarint32(); + } + + /// + /// Reads an sfixed32 field value from the stream. + /// + public int ReadSFixed32() + { + return (int)ReadRawLittleEndian32(); + } + + /// + /// Reads an sfixed64 field value from the stream. + /// + public long ReadSFixed64() + { + return (long)ReadRawLittleEndian64(); + } + + /// + /// Reads an sint32 field value from the stream. + /// + public int ReadSInt32() + { + return DecodeZigZag32(ReadRawVarint32()); + } + + /// + /// Reads an sint64 field value from the stream. + /// + public long ReadSInt64() + { + return DecodeZigZag64(ReadRawVarint64()); + } + + /// + /// Reads a length for length-delimited data. + /// + /// + /// This is internally just reading a varint, but this method exists + /// to make the calling code clearer. + /// + public int ReadLength() + { + return (int)ReadRawVarint32(); + } + + /// + /// Peeks at the next tag in the stream. If it matches , + /// the tag is consumed and the method returns true; otherwise, the + /// stream is left in the original position and the method returns false. + /// + public bool MaybeConsumeTag(uint tag) + { + return PeekTag() == tag; + } + + #endregion + + #region Underlying reading primitives + + /// + /// Reads a raw Varint from the stream. If larger than 32 bits, discard the upper bits. + /// This method is optimised for the case where we've got lots of data in the buffer. + /// That means we can check the size just once, then just read directly from the buffer + /// without constant rechecking of the buffer length. + /// + internal uint ReadRawVarint32() + { + byte value; + + ThrowEndOfStreamUnless(reader.TryRead(out value)); + int tmp = value; + if (tmp < 128) + { + return (uint)tmp; + } + int result = tmp & 0x7f; + ThrowEndOfStreamUnless(reader.TryRead(out value)); + tmp = value; + if (tmp < 128) + { + result |= tmp << 7; + } + else + { + result |= (tmp & 0x7f) << 7; + ThrowEndOfStreamUnless(reader.TryRead(out value)); + tmp = value; + if (tmp < 128) + { + result |= tmp << 14; + } + else + { + result |= (tmp & 0x7f) << 14; + ThrowEndOfStreamUnless(reader.TryRead(out value)); + tmp = value; + if (tmp < 128) + { + result |= tmp << 21; + } + else + { + result |= (tmp & 0x7f) << 21; + ThrowEndOfStreamUnless(reader.TryRead(out value)); + tmp = value; + result |= tmp << 28; + if (tmp >= 128) + { + // Discard upper 32 bits. + // Note that this has to use ReadRawByte() as we only ensure we've + // got at least 5 bytes at the start of the method. This lets us + // use the fast path in more cases, and we rarely hit this section of code. + for (int i = 0; i < 5; i++) + { + ThrowEndOfStreamUnless(reader.TryRead(out value)); + tmp = value; + if (tmp < 128) + { + return (uint)result; + } + } + throw InvalidProtocolBufferException.MalformedVarint(); + } + } + } + } + return (uint)result; + } + + /// + /// Reads a varint from the input one byte at a time, so that it does not + /// read any bytes after the end of the varint. If you simply wrapped the + /// stream in a CodedInputStream and used ReadRawVarint32(Stream) + /// then you would probably end up reading past the end of the varint since + /// CodedInputStream buffers its input. + /// + /// + /// + internal static uint ReadRawVarint32(Stream input) + { + int result = 0; + int offset = 0; + for (; offset < 32; offset += 7) + { + int b = input.ReadByte(); + if (b == -1) + { + throw InvalidProtocolBufferException.TruncatedMessage(); + } + result |= (b & 0x7f) << offset; + if ((b & 0x80) == 0) + { + return (uint)result; + } + } + // Keep reading up to 64 bits. + for (; offset < 64; offset += 7) + { + int b = input.ReadByte(); + if (b == -1) + { + throw InvalidProtocolBufferException.TruncatedMessage(); + } + if ((b & 0x80) == 0) + { + return (uint)result; + } + } + throw InvalidProtocolBufferException.MalformedVarint(); + } + + /// + /// Reads a raw varint from the stream. + /// + internal ulong ReadRawVarint64() + { + int shift = 0; + ulong result = 0; + while (shift < 64) + { + ThrowEndOfStreamUnless(reader.TryRead(out byte b)); + result |= (ulong)(b & 0x7F) << shift; + if ((b & 0x80) == 0) + { + return result; + } + shift += 7; + } + throw InvalidProtocolBufferException.MalformedVarint(); + } + + /// + /// Reads a 32-bit little-endian integer from the stream. + /// + internal unsafe uint ReadRawLittleEndian32() + { + byte* buffer = stackalloc byte[4]; + Span tempSpan = new Span(buffer, 4); + + ThrowEndOfStreamUnless(reader.TryCopyTo(tempSpan)); + reader.Advance(4); + + return BinaryPrimitives.ReadUInt32LittleEndian(tempSpan); + } + + /// + /// Reads a 64-bit little-endian integer from the stream. + /// + internal unsafe ulong ReadRawLittleEndian64() + { + byte* buffer = stackalloc byte[8]; + Span tempSpan = new Span(buffer, 8); + + ThrowEndOfStreamUnless(reader.TryCopyTo(tempSpan)); + reader.Advance(8); + + return BinaryPrimitives.ReadUInt64LittleEndian(tempSpan); + } + + /// + /// Decode a 32-bit value with ZigZag encoding. + /// + /// + /// ZigZag encodes signed integers into values that can be efficiently + /// encoded with varint. (Otherwise, negative values must be + /// sign-extended to 64 bits to be varint encoded, thus always taking + /// 10 bytes on the wire.) + /// + internal static int DecodeZigZag32(uint n) + { + return (int)(n >> 1) ^ -(int)(n & 1); + } + + /// + /// Decode a 32-bit value with ZigZag encoding. + /// + /// + /// ZigZag encodes signed integers into values that can be efficiently + /// encoded with varint. (Otherwise, negative values must be + /// sign-extended to 64 bits to be varint encoded, thus always taking + /// 10 bytes on the wire.) + /// + internal static long DecodeZigZag64(ulong n) + { + return (long)(n >> 1) ^ -(long)(n & 1); + } + #endregion + + /// + /// Sets currentLimit to (current position) + byteLimit. This is called + /// when descending into a length-delimited embedded message. The previous + /// limit is returned. + /// + /// The old limit. + internal long PushLimit(long byteLimit) + { + if (byteLimit < 0) + { + throw InvalidProtocolBufferException.NegativeSize(); + } + + byteLimit += reader.Consumed; + long oldLimit = currentLimit; + if (byteLimit > oldLimit) + { + throw InvalidProtocolBufferException.TruncatedMessage(); + } + currentLimit = byteLimit; + + return oldLimit; + } + + /// + /// Discards the current limit, returning the previous limit. + /// + internal void PopLimit(long oldLimit) + { + currentLimit = oldLimit; + } + + /// + /// Returns whether or not all the data before the limit has been read. + /// + /// + internal bool ReachedLimit + { + get + { + if (currentLimit == long.MaxValue) + { + return false; + } + return reader.Consumed >= currentLimit; + } + } + + /// + /// Verifies that the last call to ReadTag() returned tag 0 - in other words, + /// we've reached the end of the stream when we expected to. + /// + /// The + /// tag read was not the one specified + internal void CheckReadEndOfStreamTag() + { + if (lastTag != 0) + { + throw InvalidProtocolBufferException.MoreDataAvailable(); + } + } + } +} +#endif \ No newline at end of file diff --git a/csharp/src/Google.Protobuf/CodedOutputWriter.cs b/csharp/src/Google.Protobuf/CodedOutputWriter.cs new file mode 100644 index 000000000000..89c0360700bf --- /dev/null +++ b/csharp/src/Google.Protobuf/CodedOutputWriter.cs @@ -0,0 +1,585 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +#if NETSTANDARD2_0 +using System; +using System.Buffers; +using System.Buffers.Binary; +using System.IO; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +namespace Google.Protobuf +{ + /// + /// Encodes and writes protocol message fields. + /// + /// + /// + /// This class is generally used by generated code to write appropriate + /// primitives to the stream. It effectively encapsulates the lowest + /// levels of protocol buffer format. Unlike some other implementations, + /// this does not include combined "write tag and value" methods. Generated + /// code knows the exact byte representations of the tags they're going to write, + /// so there's no need to re-encode them each time. Manually-written code calling + /// this class should just call one of the WriteTag overloads before each value. + /// + /// + /// Repeated fields and map fields are not handled by this class; use RepeatedField<T> + /// and MapField<TKey, TValue> to serialize such fields. + /// + /// + public ref struct CodedOutputWriter + { + // "Local" copy of Encoding.UTF8, for efficiency. (Yes, it makes a difference.) + internal static readonly Encoding Utf8Encoding = Encoding.UTF8; + + private BufferWriter writer; + private Encoder encoder; + + public CodedOutputWriter(IBufferWriter writer) + { + this.writer = new BufferWriter(writer); + this.encoder = null; + } + + #region Writing of values (not including tags) + + /// + /// Writes a double field value, without a tag, to the stream. + /// + /// The value to write + public void WriteDouble(double value) + { + WriteRawLittleEndian64((ulong)BitConverter.DoubleToInt64Bits(value)); + } + + /// + /// Writes a float field value, without a tag, to the stream. + /// + /// The value to write + public unsafe void WriteFloat(float value) + { + // Cannot create a span directly since it gets passed to instance methods on a ref struct. + byte* ptr = stackalloc byte[sizeof(float)]; + Span floatSpan = new Span(ptr, sizeof(float)); + + Unsafe.WriteUnaligned(ref MemoryMarshal.GetReference(floatSpan), value); + + if (!BitConverter.IsLittleEndian) + { + floatSpan.Reverse(); + } + + writer.Write(floatSpan); + } + + /// + /// Writes a uint64 field value, without a tag, to the stream. + /// + /// The value to write + public void WriteUInt64(ulong value) + { + WriteRawVarint64(value); + } + + /// + /// Writes an int64 field value, without a tag, to the stream. + /// + /// The value to write + public void WriteInt64(long value) + { + WriteRawVarint64((ulong) value); + } + + /// + /// Writes an int32 field value, without a tag, to the stream. + /// + /// The value to write + public void WriteInt32(int value) + { + if (value >= 0) + { + WriteRawVarint32((uint) value); + } + else + { + // Must sign-extend. + WriteRawVarint64((ulong) value); + } + } + + /// + /// Writes a fixed64 field value, without a tag, to the stream. + /// + /// The value to write + public void WriteFixed64(ulong value) + { + WriteRawLittleEndian64(value); + } + + /// + /// Writes a fixed32 field value, without a tag, to the stream. + /// + /// The value to write + public void WriteFixed32(uint value) + { + WriteRawLittleEndian32(value); + } + + /// + /// Writes a bool field value, without a tag, to the stream. + /// + /// The value to write + public void WriteBool(bool value) + { + WriteRawByte(value ? (byte) 1 : (byte) 0); + } + + /// + /// Writes a string field value, without a tag, to the stream. + /// The data is length-prefixed. + /// + /// The value to write + public void WriteString(string value) + { + int length = Utf8Encoding.GetByteCount(value); + WriteLength(length); + + Span buffer = writer.GetSpan(length); + + if (buffer.Length >= length) + { + // Can write string to a single buffer + + if (length == value.Length) // Must be all ASCII... + { + for (int i = 0; i < length; i++) + { + buffer[i] = (byte)value[i]; + } + + writer.Advance(length); + } + else + { + ReadOnlySpan source = value.AsSpan(); + + int bytesUsed; + + unsafe + { + fixed (char* sourceChars = &MemoryMarshal.GetReference(source)) + fixed (byte* destinationBytes = &MemoryMarshal.GetReference(buffer)) + { + bytesUsed = Utf8Encoding.GetBytes(sourceChars, source.Length, destinationBytes, buffer.Length); + } + } + + writer.Advance(bytesUsed); + } + } + else + { + // The destination byte array might not be large enough so multiple writes are sometimes required + if (encoder == null) + { + encoder = Utf8Encoding.GetEncoder(); + } + + ReadOnlySpan source = value.AsSpan(); + int written = 0; + + while (true) + { + int bytesUsed; + int charsUsed; + + unsafe + { + fixed (char* sourceChars = &MemoryMarshal.GetReference(source)) + fixed (byte* destinationBytes = &MemoryMarshal.GetReference(buffer)) + { + encoder.Convert(sourceChars, source.Length, destinationBytes, buffer.Length, false, out charsUsed, out bytesUsed, out _); + } + } + + source = source.Slice(charsUsed); + written += bytesUsed; + + writer.Advance(bytesUsed); + + if (source.Length == 0) + { + break; + } + + buffer = writer.GetSpan(length - written); + } + + } + } + + /// + /// Writes a message, without a tag, to the stream. + /// The data is length-prefixed. + /// + /// The value to write + public void WriteMessage(ISpanMessage value) + { + WriteLength(value.CalculateSize()); + value.WriteTo(ref this); + } + + /// + /// Writes a group, without a tag, to the stream. + /// + /// The value to write + public void WriteGroup(ISpanMessage value) + { + value.WriteTo(ref this); + } + + /// + /// Write a byte string, without a tag, to the stream. + /// The data is length-prefixed. + /// + /// The value to write + public void WriteBytes(ByteString value) + { + WriteLength(value.Length); + + writer.Write(value.Span); + } + + /// + /// Writes a uint32 value, without a tag, to the stream. + /// + /// The value to write + public void WriteUInt32(uint value) + { + WriteRawVarint32(value); + } + + /// + /// Writes an enum value, without a tag, to the stream. + /// + /// The value to write + public void WriteEnum(int value) + { + WriteInt32(value); + } + + /// + /// Writes an sfixed32 value, without a tag, to the stream. + /// + /// The value to write. + public void WriteSFixed32(int value) + { + WriteRawLittleEndian32((uint) value); + } + + /// + /// Writes an sfixed64 value, without a tag, to the stream. + /// + /// The value to write + public void WriteSFixed64(long value) + { + WriteRawLittleEndian64((ulong) value); + } + + /// + /// Writes an sint32 value, without a tag, to the stream. + /// + /// The value to write + public void WriteSInt32(int value) + { + WriteRawVarint32(EncodeZigZag32(value)); + } + + /// + /// Writes an sint64 value, without a tag, to the stream. + /// + /// The value to write + public void WriteSInt64(long value) + { + WriteRawVarint64(EncodeZigZag64(value)); + } + + /// + /// Writes a length (in bytes) for length-delimited data. + /// + /// + /// This method simply writes a rawint, but exists for clarity in calling code. + /// + /// Length value, in bytes. + public void WriteLength(int length) + { + WriteRawVarint32((uint) length); + } + + #endregion + + #region Raw tag writing + /// + /// Encodes and writes a tag. + /// + /// The number of the field to write the tag for + /// The wire format type of the tag to write + public void WriteTag(int fieldNumber, WireFormat.WireType type) + { + WriteRawVarint32(WireFormat.MakeTag(fieldNumber, type)); + } + + /// + /// Writes an already-encoded tag. + /// + /// The encoded tag + public void WriteTag(uint tag) + { + WriteRawVarint32(tag); + } + + /// + /// Writes the given single-byte tag directly to the stream. + /// + /// The encoded tag + public void WriteRawTag(byte b1) + { + var span = writer.GetSpan(1); + span[0] = b1; + writer.Advance(1); + } + + /// + /// Writes the given two-byte tag directly to the stream. + /// + /// The first byte of the encoded tag + /// The second byte of the encoded tag + public void WriteRawTag(byte b1, byte b2) + { + var span = writer.GetSpan(2); + span[1] = b2; + span[0] = b1; + writer.Advance(2); + } + + /// + /// Writes the given three-byte tag directly to the stream. + /// + /// The first byte of the encoded tag + /// The second byte of the encoded tag + /// The third byte of the encoded tag + public void WriteRawTag(byte b1, byte b2, byte b3) + { + var span = writer.GetSpan(3); + span[2] = b3; + span[1] = b2; + span[0] = b1; + writer.Advance(3); + } + + /// + /// Writes the given four-byte tag directly to the stream. + /// + /// The first byte of the encoded tag + /// The second byte of the encoded tag + /// The third byte of the encoded tag + /// The fourth byte of the encoded tag + public void WriteRawTag(byte b1, byte b2, byte b3, byte b4) + { + var span = writer.GetSpan(4); + span[3] = b4; + span[2] = b3; + span[1] = b2; + span[0] = b1; + writer.Advance(4); + } + + /// + /// Writes the given five-byte tag directly to the stream. + /// + /// The first byte of the encoded tag + /// The second byte of the encoded tag + /// The third byte of the encoded tag + /// The fourth byte of the encoded tag + /// The fifth byte of the encoded tag + public void WriteRawTag(byte b1, byte b2, byte b3, byte b4, byte b5) + { + var span = writer.GetSpan(5); + span[4] = b5; + span[3] = b4; + span[2] = b3; + span[1] = b2; + span[0] = b1; + writer.Advance(5); + } + #endregion + + #region Underlying writing primitives + /// + /// Writes a 32 bit value as a varint. The fast route is taken when + /// there's enough buffer space left to whizz through without checking + /// for each byte; otherwise, we resort to calling WriteRawByte each time. + /// + internal void WriteRawVarint32(uint value) + { + // Optimize for the common case of a single byte value + if (value < 128) + { + var buffer = writer.GetSpan(1); + buffer[0] = (byte)value; + writer.Advance(1); + } + else + { + var buffer = writer.GetSpan(4); + var position = 0; + + while (value > 127) + { + buffer[position++] = (byte)((value & 0x7F) | 0x80); + value >>= 7; + } + + buffer[position++] = (byte)value; + + writer.Advance(position); + } + } + + internal void WriteRawVarint64(ulong value) + { + // Optimize for the common case of a single byte value + if (value < 128) + { + var buffer = writer.GetSpan(1); + buffer[0] = (byte)value; + writer.Advance(1); + } + else + { + var buffer = writer.GetSpan(8); + var position = 0; + + while (value > 127) + { + buffer[position++] = (byte)((value & 0x7F) | 0x80); + value >>= 7; + } + + buffer[position++] = (byte)value; + + writer.Advance(position); + } + } + + internal void WriteRawLittleEndian32(uint value) + { + var buffer = writer.GetSpan(4); + + BinaryPrimitives.WriteUInt32LittleEndian(buffer, value); + + writer.Advance(4); + } + + internal void WriteRawLittleEndian64(ulong value) + { + var buffer = writer.GetSpan(8); + + BinaryPrimitives.WriteUInt64LittleEndian(buffer, value); + + writer.Advance(8); + } + + internal void WriteRawByte(byte value) + { + var buffer = writer.GetSpan(1); + buffer[0] = value; + writer.Advance(1); + } + + /// + /// Writes out an array of bytes. + /// + internal void WriteRawBytes(byte[] value) + { + WriteRawBytes(value, 0, value.Length); + } + + /// + /// Writes out part of an array of bytes. + /// + internal void WriteRawBytes(byte[] value, int offset, int length) + { + writer.Write(value.AsSpan(offset, length)); + } + + #endregion + + /// + /// Encode a 32-bit value with ZigZag encoding. + /// + /// + /// ZigZag encodes signed integers into values that can be efficiently + /// encoded with varint. (Otherwise, negative values must be + /// sign-extended to 64 bits to be varint encoded, thus always taking + /// 10 bytes on the wire.) + /// + internal static uint EncodeZigZag32(int n) + { + // Note: the right-shift must be arithmetic + return (uint) ((n << 1) ^ (n >> 31)); + } + + /// + /// Encode a 64-bit value with ZigZag encoding. + /// + /// + /// ZigZag encodes signed integers into values that can be efficiently + /// encoded with varint. (Otherwise, negative values must be + /// sign-extended to 64 bits to be varint encoded, thus always taking + /// 10 bytes on the wire.) + /// + internal static ulong EncodeZigZag64(long n) + { + return (ulong) ((n << 1) ^ (n >> 63)); + } + + public void Flush() + { + writer.Commit(); + } + } +} +#endif \ No newline at end of file diff --git a/csharp/src/Google.Protobuf/Collections/RepeatedField.cs b/csharp/src/Google.Protobuf/Collections/RepeatedField.cs index 0e8bb617647b..cc806ec6626b 100644 --- a/csharp/src/Google.Protobuf/Collections/RepeatedField.cs +++ b/csharp/src/Google.Protobuf/Collections/RepeatedField.cs @@ -124,6 +124,44 @@ public void AddEntriesFrom(CodedInputStream input, FieldCodec codec) } } +#if NETSTANDARD2_0 + /// + /// Adds the entries from the given input stream, decoding them with the specified codec. + /// + /// The input stream to read from. + /// The codec to use in order to read each entry. + public void AddEntriesFrom(ref CodedInputReader input, FieldCodec codec) + { + // TODO: Inline some of the Add code, so we can avoid checking the size on every + // iteration. + uint tag = input.LastTag; + var reader = codec.ValueSpanReader; + // Non-nullable value types can be packed or not. + if (FieldCodec.IsPackedRepeatedField(tag)) + { + int length = input.ReadLength(); + if (length > 0) + { + long oldLimit = input.PushLimit(length); + while (!input.ReachedLimit) + { + Add(reader(ref input)); + } + input.PopLimit(oldLimit); + } + // Empty packed field. Odd, but valid - just ignore. + } + else + { + // Not packed... (possibly not packable) + do + { + Add(reader(ref input)); + } while (input.MaybeConsumeTag(tag)); + } + } +#endif + /// /// Calculates the size of this collection based on the given codec. /// @@ -220,6 +258,49 @@ public void WriteTo(CodedOutputStream output, FieldCodec codec) } } +#if NETSTANDARD2_0 + /// + /// Writes the contents of this collection to the given , + /// encoding each value using the specified codec. + /// + /// The output stream to write to. + /// The codec to use when encoding each value. + public void WriteTo(ref CodedOutputWriter output, FieldCodec codec) + { + if (count == 0) + { + return; + } + var writer = codec.ValueSpanWriter; + var tag = codec.Tag; + if (codec.PackedRepeatedField) + { + // Packed primitive type + uint size = (uint)CalculatePackedDataSize(codec); + output.WriteTag(tag); + output.WriteRawVarint32(size); + for (int i = 0; i < count; i++) + { + writer(ref output, array[i]); + } + } + else + { + // Not packed: a simple tag/value pair for each value. + // Can't use codec.WriteTagAndValue, as that omits default values. + for (int i = 0; i < count; i++) + { + output.WriteTag(tag); + writer(ref output, array[i]); + if (codec.EndTag != 0) + { + output.WriteTag(codec.EndTag); + } + } + } + } +#endif + /// /// Gets and sets the capacity of the RepeatedField's internal array. WHen set, the internal array is reallocated to the given capacity. /// The new value is less than Count -or- when Count is less than 0. @@ -579,7 +660,7 @@ public T this[int index] } } - #region Explicit interface implementation for IList and ICollection. +#region Explicit interface implementation for IList and ICollection. bool IList.IsFixedSize => false; void ICollection.CopyTo(Array array, int index) @@ -630,6 +711,6 @@ void IList.Remove(object value) } Remove((T)value); } - #endregion +#endregion } } diff --git a/csharp/src/Google.Protobuf/FieldCodec.cs b/csharp/src/Google.Protobuf/FieldCodec.cs index ebd00b702206..2f8da317b276 100644 --- a/csharp/src/Google.Protobuf/FieldCodec.cs +++ b/csharp/src/Google.Protobuf/FieldCodec.cs @@ -218,7 +218,16 @@ public static FieldCodec ForEnum(uint tag, Func toInt32, FuncA codec for the given tag. public static FieldCodec ForString(uint tag, string defaultValue) { - return new FieldCodec(input => input.ReadString(), (output, value) => output.WriteString(value), CodedOutputStream.ComputeStringSize, tag, defaultValue); + return new FieldCodec( + input => input.ReadString(), + (output, value) => output.WriteString(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadString(), + (ref CodedOutputWriter output, string value) => output.WriteString(value), +#endif + CodedOutputStream.ComputeStringSize, + tag, + defaultValue); } /// @@ -229,7 +238,16 @@ public static FieldCodec ForString(uint tag, string defaultValue) /// A codec for the given tag. public static FieldCodec ForBytes(uint tag, ByteString defaultValue) { - return new FieldCodec(input => input.ReadBytes(), (output, value) => output.WriteBytes(value), CodedOutputStream.ComputeBytesSize, tag, defaultValue); + return new FieldCodec( + input => input.ReadBytes(), + (output, value) => output.WriteBytes(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadBytes(), + (ref CodedOutputWriter output, ByteString value) => output.WriteBytes(value), +#endif + CodedOutputStream.ComputeBytesSize, + tag, + defaultValue); } /// @@ -240,7 +258,16 @@ public static FieldCodec ForBytes(uint tag, ByteString defaultValue) /// A codec for the given tag. public static FieldCodec ForBool(uint tag, bool defaultValue) { - return new FieldCodec(input => input.ReadBool(), (output, value) => output.WriteBool(value), CodedOutputStream.BoolSize, tag, defaultValue); + return new FieldCodec( + input => input.ReadBool(), + (output, value) => output.WriteBool(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadBool(), + (ref CodedOutputWriter output, bool value) => output.WriteBool(value), +#endif + CodedOutputStream.BoolSize, + tag, + defaultValue); } /// @@ -251,7 +278,16 @@ public static FieldCodec ForBool(uint tag, bool defaultValue) /// A codec for the given tag. public static FieldCodec ForInt32(uint tag, int defaultValue) { - return new FieldCodec(input => input.ReadInt32(), (output, value) => output.WriteInt32(value), CodedOutputStream.ComputeInt32Size, tag, defaultValue); + return new FieldCodec( + input => input.ReadInt32(), + (output, value) => output.WriteInt32(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadInt32(), + (ref CodedOutputWriter output, int value) => output.WriteInt32(value), +#endif + CodedOutputStream.ComputeInt32Size, + tag, + defaultValue); } /// @@ -262,7 +298,16 @@ public static FieldCodec ForInt32(uint tag, int defaultValue) /// A codec for the given tag. public static FieldCodec ForSInt32(uint tag, int defaultValue) { - return new FieldCodec(input => input.ReadSInt32(), (output, value) => output.WriteSInt32(value), CodedOutputStream.ComputeSInt32Size, tag, defaultValue); + return new FieldCodec( + input => input.ReadSInt32(), + (output, value) => output.WriteSInt32(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadSInt32(), + (ref CodedOutputWriter output, int value) => output.WriteSInt32(value), +#endif + CodedOutputStream.ComputeSInt32Size, + tag, + defaultValue); } /// @@ -273,7 +318,16 @@ public static FieldCodec ForSInt32(uint tag, int defaultValue) /// A codec for the given tag. public static FieldCodec ForFixed32(uint tag, uint defaultValue) { - return new FieldCodec(input => input.ReadFixed32(), (output, value) => output.WriteFixed32(value), 4, tag, defaultValue); + return new FieldCodec( + input => input.ReadFixed32(), + (output, value) => output.WriteFixed32(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadFixed32(), + (ref CodedOutputWriter output, uint value) => output.WriteFixed32(value), +#endif + 4, + tag, + defaultValue); } /// @@ -284,7 +338,16 @@ public static FieldCodec ForFixed32(uint tag, uint defaultValue) /// A codec for the given tag. public static FieldCodec ForSFixed32(uint tag, int defaultValue) { - return new FieldCodec(input => input.ReadSFixed32(), (output, value) => output.WriteSFixed32(value), 4, tag, defaultValue); + return new FieldCodec( + input => input.ReadSFixed32(), + (output, value) => output.WriteSFixed32(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadSFixed32(), + (ref CodedOutputWriter output, int value) => output.WriteSFixed32(value), +#endif + 4, + tag, + defaultValue); } /// @@ -295,7 +358,16 @@ public static FieldCodec ForSFixed32(uint tag, int defaultValue) /// A codec for the given tag. public static FieldCodec ForUInt32(uint tag, uint defaultValue) { - return new FieldCodec(input => input.ReadUInt32(), (output, value) => output.WriteUInt32(value), CodedOutputStream.ComputeUInt32Size, tag, defaultValue); + return new FieldCodec( + input => input.ReadUInt32(), + (output, value) => output.WriteUInt32(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadUInt32(), + (ref CodedOutputWriter output, uint value) => output.WriteUInt32(value), +#endif + CodedOutputStream.ComputeUInt32Size, + tag, + defaultValue); } /// @@ -306,7 +378,16 @@ public static FieldCodec ForUInt32(uint tag, uint defaultValue) /// A codec for the given tag. public static FieldCodec ForInt64(uint tag, long defaultValue) { - return new FieldCodec(input => input.ReadInt64(), (output, value) => output.WriteInt64(value), CodedOutputStream.ComputeInt64Size, tag, defaultValue); + return new FieldCodec( + input => input.ReadInt64(), + (output, value) => output.WriteInt64(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadInt64(), + (ref CodedOutputWriter output, long value) => output.WriteInt64(value), +#endif + CodedOutputStream.ComputeInt64Size, + tag, + defaultValue); } /// @@ -317,7 +398,16 @@ public static FieldCodec ForInt64(uint tag, long defaultValue) /// A codec for the given tag. public static FieldCodec ForSInt64(uint tag, long defaultValue) { - return new FieldCodec(input => input.ReadSInt64(), (output, value) => output.WriteSInt64(value), CodedOutputStream.ComputeSInt64Size, tag, defaultValue); + return new FieldCodec( + input => input.ReadSInt64(), + (output, value) => output.WriteSInt64(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadSInt64(), + (ref CodedOutputWriter output, long value) => output.WriteSInt64(value), +#endif + CodedOutputStream.ComputeSInt64Size, + tag, + defaultValue); } /// @@ -328,7 +418,16 @@ public static FieldCodec ForSInt64(uint tag, long defaultValue) /// A codec for the given tag. public static FieldCodec ForFixed64(uint tag, ulong defaultValue) { - return new FieldCodec(input => input.ReadFixed64(), (output, value) => output.WriteFixed64(value), 8, tag, defaultValue); + return new FieldCodec( + input => input.ReadFixed64(), + (output, value) => output.WriteFixed64(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadFixed64(), + (ref CodedOutputWriter output, ulong value) => output.WriteFixed64(value), +#endif + 8, + tag, + defaultValue); } /// @@ -339,7 +438,16 @@ public static FieldCodec ForFixed64(uint tag, ulong defaultValue) /// A codec for the given tag. public static FieldCodec ForSFixed64(uint tag, long defaultValue) { - return new FieldCodec(input => input.ReadSFixed64(), (output, value) => output.WriteSFixed64(value), 8, tag, defaultValue); + return new FieldCodec( + input => input.ReadSFixed64(), + (output, value) => output.WriteSFixed64(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadSFixed64(), + (ref CodedOutputWriter output, long value) => output.WriteSFixed64(value), +#endif + 8, + tag, + defaultValue); } /// @@ -350,7 +458,16 @@ public static FieldCodec ForSFixed64(uint tag, long defaultValue) /// A codec for the given tag. public static FieldCodec ForUInt64(uint tag, ulong defaultValue) { - return new FieldCodec(input => input.ReadUInt64(), (output, value) => output.WriteUInt64(value), CodedOutputStream.ComputeUInt64Size, tag, defaultValue); + return new FieldCodec( + input => input.ReadUInt64(), + (output, value) => output.WriteUInt64(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadUInt64(), + (ref CodedOutputWriter output, ulong value) => output.WriteUInt64(value), +#endif + CodedOutputStream.ComputeUInt64Size, + tag, + defaultValue); } /// @@ -361,7 +478,16 @@ public static FieldCodec ForUInt64(uint tag, ulong defaultValue) /// A codec for the given tag. public static FieldCodec ForFloat(uint tag, float defaultValue) { - return new FieldCodec(input => input.ReadFloat(), (output, value) => output.WriteFloat(value), CodedOutputStream.FloatSize, tag, defaultValue); + return new FieldCodec( + input => input.ReadFloat(), + (output, value) => output.WriteFloat(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadFloat(), + (ref CodedOutputWriter output, float value) => output.WriteFloat(value), +#endif + CodedOutputStream.FloatSize, + tag, + defaultValue); } /// @@ -372,7 +498,16 @@ public static FieldCodec ForFloat(uint tag, float defaultValue) /// A codec for the given tag. public static FieldCodec ForDouble(uint tag, double defaultValue) { - return new FieldCodec(input => input.ReadDouble(), (output, value) => output.WriteDouble(value), CodedOutputStream.DoubleSize, tag, defaultValue); + return new FieldCodec( + input => input.ReadDouble(), + (output, value) => output.WriteDouble(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => input.ReadDouble(), + (ref CodedOutputWriter output, double value) => output.WriteDouble(value), +#endif + CodedOutputStream.DoubleSize, + tag, + defaultValue); } // Enums are tricky. We can probably use expression trees to build these delegates automatically, @@ -388,9 +523,13 @@ public static FieldCodec ForDouble(uint tag, double defaultValue) /// A codec for the given tag. public static FieldCodec ForEnum(uint tag, Func toInt32, Func fromInt32, T defaultValue) { - return new FieldCodec(input => fromInt32( - input.ReadEnum()), + return new FieldCodec( + input => fromInt32(input.ReadEnum()), (output, value) => output.WriteEnum(toInt32(value)), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => fromInt32(input.ReadEnum()), + (ref CodedOutputWriter output, T value) => output.WriteEnum(toInt32(value)), +#endif value => CodedOutputStream.ComputeEnumSize(toInt32(value)), tag, defaultValue); } @@ -403,13 +542,20 @@ public static FieldCodec ForEnum(uint tag, Func toInt32, Func ForMessage(uint tag, MessageParser parser) where T : class, IMessage { return new FieldCodec( - input => - { - T message = parser.CreateTemplate(); - input.ReadMessage(message); - return message; + input => + { + T message = parser.CreateTemplate(); + input.ReadMessage(message); + return message; }, (output, value) => output.WriteMessage(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => + { + throw new NotSupportedException(); + }, + (ref CodedOutputWriter output, T value) => throw new NotSupportedException(), +#endif (CodedInputStream i, ref T v) => { if (v == null) @@ -454,7 +600,14 @@ public static FieldCodec ForGroup(uint startTag, uint endTag, MessageParse input.ReadGroup(message); return message; }, - (output, value) => output.WriteGroup(value), + (output, value) => output.WriteGroup(value), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => + { + throw new NotSupportedException(); + }, + (ref CodedOutputWriter output, T value) => throw new NotSupportedException(), +#endif (CodedInputStream i, ref T v) => { if (v == null) @@ -492,6 +645,13 @@ public static FieldCodec ForClassWrapper(uint tag) where T : class return new FieldCodec( input => WrapperCodecs.Read(input, nestedCodec), (output, value) => WrapperCodecs.Write(output, value, nestedCodec), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => + { + throw new NotSupportedException(); + }, + (ref CodedOutputWriter output, T value) => throw new NotSupportedException(), +#endif (CodedInputStream i, ref T v) => v = WrapperCodecs.Read(i, nestedCodec), (ref T v, T v2) => { v = v2; return v == null; }, value => WrapperCodecs.CalculateSize(value, nestedCodec), @@ -509,6 +669,13 @@ public static FieldCodec ForClassWrapper(uint tag) where T : class return new FieldCodec( input => WrapperCodecs.Read(input, nestedCodec), (output, value) => WrapperCodecs.Write(output, value.Value, nestedCodec), +#if NETSTANDARD2_0 + (ref CodedInputReader input) => + { + throw new NotSupportedException(); + }, + (ref CodedOutputWriter output, T? value) => throw new NotSupportedException(), +#endif (CodedInputStream i, ref T? v) => v = WrapperCodecs.Read(i, nestedCodec), (ref T? v, T? v2) => { if (v2.HasValue) { v = v2; } return v.HasValue; }, value => value == null ? 0 : WrapperCodecs.CalculateSize(value.Value, nestedCodec), @@ -592,6 +759,11 @@ internal static int CalculateSize(T value, FieldCodec codec) } } +#if NETSTANDARD2_0 + internal delegate void WriterAction(ref CodedOutputWriter writer, TValue value); + internal delegate TValue ReaderFunc(ref CodedInputReader reader); +#endif + /// /// /// An encode/decode pair for a single field. This effectively encapsulates @@ -647,6 +819,13 @@ internal static bool IsPackedRepeatedField(uint tag) => /// internal Action ValueWriter { get; } +#if NETSTANDARD2_0 + /// + /// Returns a delegate to write a value (unconditionally) to a coded output writer. + /// + internal WriterAction ValueSpanWriter { get; } +#endif + /// /// Returns the size calculator for just a value. /// @@ -658,6 +837,14 @@ internal static bool IsPackedRepeatedField(uint tag) => /// internal Func ValueReader { get; } +#if NETSTANDARD2_0 + /// + /// Returns a delegate to read a value from a coded input reader. It is assumed that + /// the reader is already positioned on the appropriate tag. + /// + internal ReaderFunc ValueSpanReader { get; } +#endif + /// /// Returns a delegate to merge a value from a coded input stream. /// It is assumed that the stream is already positioned on the appropriate tag @@ -705,9 +892,22 @@ internal static bool IsPackedRepeatedField(uint tag) => internal FieldCodec( Func reader, Action writer, +#if NETSTANDARD2_0 + ReaderFunc spanReader, + WriterAction spanWriter, +#endif int fixedSize, uint tag, - T defaultValue) : this(reader, writer, _ => fixedSize, tag, defaultValue) + T defaultValue) : this( + reader, + writer, +#if NETSTANDARD2_0 + spanReader, + spanWriter, +#endif + _ => fixedSize, + tag, + defaultValue) { FixedSize = fixedSize; } @@ -715,26 +915,62 @@ internal FieldCodec( internal FieldCodec( Func reader, Action writer, +#if NETSTANDARD2_0 + ReaderFunc spanReader, + WriterAction spanWriter, +#endif Func sizeCalculator, uint tag, - T defaultValue) : this(reader, writer, (CodedInputStream i, ref T v) => v = reader(i), (ref T v, T v2) => { v = v2; return true; }, sizeCalculator, tag, 0, defaultValue) + T defaultValue) : this( + reader, + writer, +#if NETSTANDARD2_0 + spanReader, + spanWriter, +#endif + (CodedInputStream i, ref T v) => v = reader(i), + (ref T v, T v2) => { v = v2; return true; }, + sizeCalculator, + tag, + 0, + defaultValue) { } internal FieldCodec( Func reader, Action writer, +#if NETSTANDARD2_0 + ReaderFunc spanReader, + WriterAction spanWriter, +#endif InputMerger inputMerger, ValuesMerger valuesMerger, Func sizeCalculator, uint tag, - uint endTag = 0) : this(reader, writer, inputMerger, valuesMerger, sizeCalculator, tag, endTag, DefaultDefault) + uint endTag = 0) : this( + reader, + writer, +#if NETSTANDARD2_0 + spanReader, + spanWriter, +#endif + inputMerger, + valuesMerger, + sizeCalculator, + tag, + endTag, + DefaultDefault) { } internal FieldCodec( Func reader, Action writer, +#if NETSTANDARD2_0 + ReaderFunc spanReader, + WriterAction spanWriter, +#endif InputMerger inputMerger, ValuesMerger valuesMerger, Func sizeCalculator, @@ -744,6 +980,10 @@ internal FieldCodec( { ValueReader = reader; ValueWriter = writer; +#if NETSTANDARD2_0 + ValueSpanReader = spanReader; + ValueSpanWriter = spanWriter; +#endif ValueMerger = inputMerger; FieldMerger = valuesMerger; ValueSizeCalculator = sizeCalculator; diff --git a/csharp/src/Google.Protobuf/Google.Protobuf.csproj b/csharp/src/Google.Protobuf/Google.Protobuf.csproj index a81a52f76988..8c12c0338120 100644 --- a/csharp/src/Google.Protobuf/Google.Protobuf.csproj +++ b/csharp/src/Google.Protobuf/Google.Protobuf.csproj @@ -1,11 +1,11 @@ - + C# runtime library for Protocol Buffers - Google's data interchange format. Copyright 2015, Google Inc. Google Protocol Buffers 3.11.0-rc0 - 6 + 7.3 Google Inc. netstandard1.0;netstandard2.0;net45 true @@ -20,6 +20,7 @@ https://github.com/protocolbuffers/protobuf.git $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb + True diff --git a/csharp/src/Google.Protobuf/ISpanMessage.cs b/csharp/src/Google.Protobuf/ISpanMessage.cs new file mode 100644 index 000000000000..404a8132148f --- /dev/null +++ b/csharp/src/Google.Protobuf/ISpanMessage.cs @@ -0,0 +1,69 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using System; +using Google.Protobuf.Reflection; + +namespace Google.Protobuf +{ +#if NETSTANDARD2_0 + public interface ISpanMessage + { + void MergeFrom(ref CodedInputReader input); + void WriteTo(ref CodedOutputWriter output); + int CalculateSize(); + + /// + /// Descriptor for this message. All instances are expected to return the same descriptor, + /// and for generated types this will be an explicitly-implemented member, returning the + /// same value as the static property declared on the type. + /// + MessageDescriptor Descriptor { get; } + } + + /// + /// Generic interface for a Protocol Buffers message, + /// where the type parameter is expected to be the same type as + /// the implementation class. + /// + /// The message type. + public interface ISpanMessage : ISpanMessage, IEquatable, IDeepCloneable where T : ISpanMessage + { + /// + /// Merges the given message into this one. + /// + /// See the user guide for precise merge semantics. + /// The message to merge with this one. Must not be null. + void MergeFrom(T message); + } +#endif +} diff --git a/csharp/src/Google.Protobuf/SequenceReader.cs b/csharp/src/Google.Protobuf/SequenceReader.cs new file mode 100644 index 000000000000..fed94e6830f6 --- /dev/null +++ b/csharp/src/Google.Protobuf/SequenceReader.cs @@ -0,0 +1,410 @@ +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +#if NETSTANDARD2_0 +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text; + +namespace System.Buffers +{ + internal ref partial struct SequenceReader where T : unmanaged, IEquatable + { + private SequencePosition _currentPosition; + private SequencePosition _nextPosition; + private bool _moreData; + private long _length; + + /// + /// Create a over the given . + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public SequenceReader(ReadOnlySequence sequence) + { + CurrentSpanIndex = 0; + Consumed = 0; + Sequence = sequence; + _currentPosition = sequence.Start; + _length = -1; + + var first = sequence.First.Span; + _nextPosition = sequence.GetPosition(first.Length); + CurrentSpan = first; + _moreData = first.Length > 0; + + if (!_moreData && !sequence.IsSingleSegment) + { + _moreData = true; + GetNextSpan(); + } + } + + /// + /// True when there is no more data in the . + /// + public bool End => !_moreData; + + /// + /// The underlying for the reader. + /// + public ReadOnlySequence Sequence { get; } + + /// + /// The current position in the . + /// + public SequencePosition Position + => Sequence.GetPosition(CurrentSpanIndex, _currentPosition); + + /// + /// The current segment in the as a span. + /// + public ReadOnlySpan CurrentSpan { get; private set; } + + /// + /// The index in the . + /// + public int CurrentSpanIndex { get; private set; } + + /// + /// The unread portion of the . + /// + public ReadOnlySpan UnreadSpan + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get => CurrentSpan.Slice(CurrentSpanIndex); + } + + /// + /// The total number of 's processed by the reader. + /// + public long Consumed { get; private set; } + + /// + /// Remaining 's in the reader's . + /// + public long Remaining => Length - Consumed; + + /// + /// Count of in the reader's . + /// + public long Length + { + [MethodImpl(MethodImplOptions.AggressiveInlining)] + get + { + if (_length < 0) + { + // Cache the length + _length = Sequence.Length; + } + return _length; + } + } + + /// + /// Peeks at the next value without advancing the reader. + /// + /// The next value or default if at the end. + /// False if at the end of the reader. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryPeek(out T value) + { + if (_moreData) + { + value = CurrentSpan[CurrentSpanIndex]; + return true; + } + else + { + value = default; + return false; + } + } + + /// + /// Read the next value and advance the reader. + /// + /// The next value or default if at the end. + /// False if at the end of the reader. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryRead(out T value) + { + if (End) + { + value = default; + return false; + } + + value = CurrentSpan[CurrentSpanIndex]; + CurrentSpanIndex++; + Consumed++; + + if (CurrentSpanIndex >= CurrentSpan.Length) + { + GetNextSpan(); + } + + return true; + } + + /// + /// Move the reader back the specified number of items. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Rewind(long count) + { + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + Consumed -= count; + + if (CurrentSpanIndex >= count) + { + CurrentSpanIndex -= (int)count; + _moreData = true; + } + else + { + // Current segment doesn't have enough data, scan backward through segments + RetreatToPreviousSpan(Consumed); + } + } + + [MethodImpl(MethodImplOptions.NoInlining)] + private void RetreatToPreviousSpan(long consumed) + { + ResetReader(); + Advance(consumed); + } + + private void ResetReader() + { + CurrentSpanIndex = 0; + Consumed = 0; + _currentPosition = Sequence.Start; + _nextPosition = _currentPosition; + + if (Sequence.TryGet(ref _nextPosition, out ReadOnlyMemory memory, advance: true)) + { + _moreData = true; + + if (memory.Length == 0) + { + CurrentSpan = default; + // No data in the first span, move to one with data + GetNextSpan(); + } + else + { + CurrentSpan = memory.Span; + } + } + else + { + // No data in any spans and at end of sequence + _moreData = false; + CurrentSpan = default; + } + } + + /// + /// Get the next segment with available data, if any. + /// + private void GetNextSpan() + { + if (!Sequence.IsSingleSegment) + { + SequencePosition previousNextPosition = _nextPosition; + while (Sequence.TryGet(ref _nextPosition, out ReadOnlyMemory memory, advance: true)) + { + _currentPosition = previousNextPosition; + if (memory.Length > 0) + { + CurrentSpan = memory.Span; + CurrentSpanIndex = 0; + return; + } + else + { + CurrentSpan = default; + CurrentSpanIndex = 0; + previousNextPosition = _nextPosition; + } + } + } + _moreData = false; + } + + /// + /// Move the reader ahead the specified number of items. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public void Advance(long count) + { + const long TooBigOrNegative = unchecked((long)0xFFFFFFFF80000000); + if ((count & TooBigOrNegative) == 0 && CurrentSpan.Length - CurrentSpanIndex > (int)count) + { + CurrentSpanIndex += (int)count; + Consumed += count; + } + else + { + // Can't satisfy from the current span + AdvanceToNextSpan(count); + } + } + + /// + /// Unchecked helper to avoid unnecessary checks where you know count is valid. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void AdvanceCurrentSpan(long count) + { + Debug.Assert(count >= 0); + + Consumed += count; + CurrentSpanIndex += (int)count; + if (CurrentSpanIndex >= CurrentSpan.Length) + GetNextSpan(); + } + + /// + /// Only call this helper if you know that you are advancing in the current span + /// with valid count and there is no need to fetch the next one. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal void AdvanceWithinSpan(long count) + { + Debug.Assert(count >= 0); + + Consumed += count; + CurrentSpanIndex += (int)count; + + Debug.Assert(CurrentSpanIndex < CurrentSpan.Length); + } + + private void AdvanceToNextSpan(long count) + { + if (count < 0) + { + throw new ArgumentOutOfRangeException(nameof(count)); + } + + Consumed += count; + while (_moreData) + { + int remaining = CurrentSpan.Length - CurrentSpanIndex; + + if (remaining > count) + { + CurrentSpanIndex += (int)count; + count = 0; + break; + } + + // As there may not be any further segments we need to + // push the current index to the end of the span. + CurrentSpanIndex += remaining; + count -= remaining; + Debug.Assert(count >= 0); + + GetNextSpan(); + + if (count == 0) + { + break; + } + } + + if (count != 0) + { + // Not enough data left- adjust for where we actually ended and throw + Consumed -= count; + throw new ArgumentOutOfRangeException(nameof(count)); + } + } + + /// + /// Copies data from the current to the given span. + /// + /// Destination to copy to. + /// True if there is enough data to copy to the . + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool TryCopyTo(Span destination) + { + ReadOnlySpan firstSpan = UnreadSpan; + if (firstSpan.Length >= destination.Length) + { + firstSpan.Slice(0, destination.Length).CopyTo(destination); + return true; + } + + return TryCopyMultisegment(destination); + } + + internal bool TryCopyMultisegment(Span destination) + { + if (Remaining < destination.Length) + return false; + + ReadOnlySpan firstSpan = UnreadSpan; + Debug.Assert(firstSpan.Length < destination.Length); + firstSpan.CopyTo(destination); + int copied = firstSpan.Length; + + SequencePosition next = _nextPosition; + while (Sequence.TryGet(ref next, out ReadOnlyMemory nextSegment, true)) + { + if (nextSegment.Length > 0) + { + ReadOnlySpan nextSpan = nextSegment.Span; + int toCopy = Math.Min(nextSpan.Length, destination.Length - copied); + nextSpan.Slice(0, toCopy).CopyTo(destination.Slice(copied)); + copied += toCopy; + if (copied >= destination.Length) + { + break; + } + } + } + + return true; + } + } +} +#endif \ No newline at end of file diff --git a/csharp/src/Google.Protobuf/UnknownField.cs b/csharp/src/Google.Protobuf/UnknownField.cs index e3ce0e86efce..36aa9a8ada3f 100644 --- a/csharp/src/Google.Protobuf/UnknownField.cs +++ b/csharp/src/Google.Protobuf/UnknownField.cs @@ -147,6 +147,59 @@ internal void WriteTo(int fieldNumber, CodedOutputStream output) } } +#if NETSTANDARD2_0 + /// + /// Serializes the field, including the field number, and writes it to + /// + /// + /// The unknown field number. + /// The CodedOutputWriter to write to. + internal void WriteTo(int fieldNumber, ref CodedOutputWriter output) + { + if (varintList != null) + { + foreach (ulong value in varintList) + { + output.WriteTag(fieldNumber, WireFormat.WireType.Varint); + output.WriteUInt64(value); + } + } + if (fixed32List != null) + { + foreach (uint value in fixed32List) + { + output.WriteTag(fieldNumber, WireFormat.WireType.Fixed32); + output.WriteFixed32(value); + } + } + if (fixed64List != null) + { + foreach (ulong value in fixed64List) + { + output.WriteTag(fieldNumber, WireFormat.WireType.Fixed64); + output.WriteFixed64(value); + } + } + if (lengthDelimitedList != null) + { + foreach (ByteString value in lengthDelimitedList) + { + output.WriteTag(fieldNumber, WireFormat.WireType.LengthDelimited); + output.WriteBytes(value); + } + } + if (groupList != null) + { + foreach (UnknownFieldSet value in groupList) + { + output.WriteTag(fieldNumber, WireFormat.WireType.StartGroup); + value.WriteTo(ref output); + output.WriteTag(fieldNumber, WireFormat.WireType.EndGroup); + } + } + } +#endif + /// /// Computes the number of bytes required to encode this field, including field /// number. diff --git a/csharp/src/Google.Protobuf/UnknownFieldSet.cs b/csharp/src/Google.Protobuf/UnknownFieldSet.cs index d136cf1e6572..a3e95016f7a7 100644 --- a/csharp/src/Google.Protobuf/UnknownFieldSet.cs +++ b/csharp/src/Google.Protobuf/UnknownFieldSet.cs @@ -1,339 +1,440 @@ -#region Copyright notice and license -// Protocol Buffers - Google's data interchange format -// Copyright 2015 Google Inc. All rights reserved. -// https://developers.google.com/protocol-buffers/ -// -// Redistribution and use in source and binary forms, with or without -// modification, are permitted provided that the following conditions are -// met: -// -// * Redistributions of source code must retain the above copyright -// notice, this list of conditions and the following disclaimer. -// * Redistributions in binary form must reproduce the above -// copyright notice, this list of conditions and the following disclaimer -// in the documentation and/or other materials provided with the -// distribution. -// * Neither the name of Google Inc. nor the names of its -// contributors may be used to endorse or promote products derived from -// this software without specific prior written permission. -// -// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -#endregion - -using System; -using System.Collections.Generic; -using System.IO; -using Google.Protobuf.Reflection; - -namespace Google.Protobuf -{ - /// - /// Used to keep track of fields which were seen when parsing a protocol message - /// but whose field numbers or types are unrecognized. This most frequently - /// occurs when new fields are added to a message type and then messages containing - /// those fields are read by old software that was built before the new types were - /// added. - /// - /// Most users will never need to use this class directly. - /// - public sealed partial class UnknownFieldSet - { - private readonly IDictionary fields; - - /// - /// Creates a new UnknownFieldSet. - /// - internal UnknownFieldSet() - { - this.fields = new Dictionary(); - } - - /// - /// Checks whether or not the given field number is present in the set. - /// - internal bool HasField(int field) - { - return fields.ContainsKey(field); - } - - /// - /// Serializes the set and writes it to . - /// - public void WriteTo(CodedOutputStream output) - { - foreach (KeyValuePair entry in fields) - { - entry.Value.WriteTo(entry.Key, output); - } - } - - /// - /// Gets the number of bytes required to encode this set. - /// - public int CalculateSize() - { - int result = 0; - foreach (KeyValuePair entry in fields) - { - result += entry.Value.GetSerializedSize(entry.Key); - } - return result; - } - - /// - /// Checks if two unknown field sets are equal. - /// - public override bool Equals(object other) - { - if (ReferenceEquals(this, other)) - { - return true; - } - UnknownFieldSet otherSet = other as UnknownFieldSet; - IDictionary otherFields = otherSet.fields; - if (fields.Count != otherFields.Count) - { - return false; - } - foreach (KeyValuePair leftEntry in fields) - { - UnknownField rightValue; - if (!otherFields.TryGetValue(leftEntry.Key, out rightValue)) - { - return false; - } - if (!leftEntry.Value.Equals(rightValue)) - { - return false; - } - } - return true; - } - - /// - /// Gets the unknown field set's hash code. - /// - public override int GetHashCode() - { - int ret = 1; - foreach (KeyValuePair field in fields) - { - // Use ^ here to make the field order irrelevant. - int hash = field.Key.GetHashCode() ^ field.Value.GetHashCode(); - ret ^= hash; - } - return ret; - } - - // Optimization: We keep around the last field that was - // modified so that we can efficiently add to it multiple times in a - // row (important when parsing an unknown repeated field). - private int lastFieldNumber; - private UnknownField lastField; - - private UnknownField GetOrAddField(int number) - { - if (lastField != null && number == lastFieldNumber) - { - return lastField; - } - if (number == 0) - { - return null; - } - - UnknownField existing; - if (fields.TryGetValue(number, out existing)) - { - return existing; - } - lastField = new UnknownField(); - AddOrReplaceField(number, lastField); - lastFieldNumber = number; - return lastField; - } - - /// - /// Adds a field to the set. If a field with the same number already exists, it - /// is replaced. - /// - internal UnknownFieldSet AddOrReplaceField(int number, UnknownField field) - { - if (number == 0) - { - throw new ArgumentOutOfRangeException("number", "Zero is not a valid field number."); - } - fields[number] = field; - return this; - } - - /// - /// Parse a single field from and merge it - /// into this set. - /// - /// The coded input stream containing the field - /// false if the tag is an "end group" tag, true otherwise - private bool MergeFieldFrom(CodedInputStream input) - { - uint tag = input.LastTag; - int number = WireFormat.GetTagFieldNumber(tag); - switch (WireFormat.GetTagWireType(tag)) - { - case WireFormat.WireType.Varint: - { - ulong uint64 = input.ReadUInt64(); - GetOrAddField(number).AddVarint(uint64); - return true; - } - case WireFormat.WireType.Fixed32: - { - uint uint32 = input.ReadFixed32(); - GetOrAddField(number).AddFixed32(uint32); - return true; - } - case WireFormat.WireType.Fixed64: - { - ulong uint64 = input.ReadFixed64(); - GetOrAddField(number).AddFixed64(uint64); - return true; - } - case WireFormat.WireType.LengthDelimited: - { - ByteString bytes = input.ReadBytes(); - GetOrAddField(number).AddLengthDelimited(bytes); - return true; - } - case WireFormat.WireType.StartGroup: - { - uint endTag = WireFormat.MakeTag(number, WireFormat.WireType.EndGroup); - UnknownFieldSet set = new UnknownFieldSet(); - while (input.ReadTag() != endTag) - { - set.MergeFieldFrom(input); - } - GetOrAddField(number).AddGroup(set); - return true; - } - case WireFormat.WireType.EndGroup: - { - return false; - } - default: - throw InvalidProtocolBufferException.InvalidWireType(); - } - } - - /// - /// Create a new UnknownFieldSet if unknownFields is null. - /// Parse a single field from and merge it - /// into unknownFields. If is configured to discard unknown fields, - /// will be returned as-is and the field will be skipped. - /// - /// The UnknownFieldSet which need to be merged - /// The coded input stream containing the field - /// The merged UnknownFieldSet - public static UnknownFieldSet MergeFieldFrom(UnknownFieldSet unknownFields, - CodedInputStream input) - { - if (input.DiscardUnknownFields) - { - input.SkipLastField(); - return unknownFields; - } - if (unknownFields == null) - { - unknownFields = new UnknownFieldSet(); - } +#region Copyright notice and license +// Protocol Buffers - Google's data interchange format +// Copyright 2015 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +#endregion + +using System; +using System.Collections.Generic; +using System.IO; +using Google.Protobuf.Reflection; + +namespace Google.Protobuf +{ + /// + /// Used to keep track of fields which were seen when parsing a protocol message + /// but whose field numbers or types are unrecognized. This most frequently + /// occurs when new fields are added to a message type and then messages containing + /// those fields are read by old software that was built before the new types were + /// added. + /// + /// Most users will never need to use this class directly. + /// + public sealed partial class UnknownFieldSet + { + private readonly IDictionary fields; + + /// + /// Creates a new UnknownFieldSet. + /// + internal UnknownFieldSet() + { + this.fields = new Dictionary(); + } + + /// + /// Checks whether or not the given field number is present in the set. + /// + internal bool HasField(int field) + { + return fields.ContainsKey(field); + } + + /// + /// Serializes the set and writes it to . + /// + public void WriteTo(CodedOutputStream output) + { + foreach (KeyValuePair entry in fields) + { + entry.Value.WriteTo(entry.Key, output); + } + } + +#if NETSTANDARD2_0 + /// + /// Serializes the set and writes it to . + /// + public void WriteTo(ref CodedOutputWriter output) + { + foreach (KeyValuePair entry in fields) + { + entry.Value.WriteTo(entry.Key, ref output); + } + } +#endif + + /// + /// Gets the number of bytes required to encode this set. + /// + public int CalculateSize() + { + int result = 0; + foreach (KeyValuePair entry in fields) + { + result += entry.Value.GetSerializedSize(entry.Key); + } + return result; + } + + /// + /// Checks if two unknown field sets are equal. + /// + public override bool Equals(object other) + { + if (ReferenceEquals(this, other)) + { + return true; + } + UnknownFieldSet otherSet = other as UnknownFieldSet; + IDictionary otherFields = otherSet.fields; + if (fields.Count != otherFields.Count) + { + return false; + } + foreach (KeyValuePair leftEntry in fields) + { + UnknownField rightValue; + if (!otherFields.TryGetValue(leftEntry.Key, out rightValue)) + { + return false; + } + if (!leftEntry.Value.Equals(rightValue)) + { + return false; + } + } + return true; + } + + /// + /// Gets the unknown field set's hash code. + /// + public override int GetHashCode() + { + int ret = 1; + foreach (KeyValuePair field in fields) + { + // Use ^ here to make the field order irrelevant. + int hash = field.Key.GetHashCode() ^ field.Value.GetHashCode(); + ret ^= hash; + } + return ret; + } + + // Optimization: We keep around the last field that was + // modified so that we can efficiently add to it multiple times in a + // row (important when parsing an unknown repeated field). + private int lastFieldNumber; + private UnknownField lastField; + + private UnknownField GetOrAddField(int number) + { + if (lastField != null && number == lastFieldNumber) + { + return lastField; + } + if (number == 0) + { + return null; + } + + UnknownField existing; + if (fields.TryGetValue(number, out existing)) + { + return existing; + } + lastField = new UnknownField(); + AddOrReplaceField(number, lastField); + lastFieldNumber = number; + return lastField; + } + + /// + /// Adds a field to the set. If a field with the same number already exists, it + /// is replaced. + /// + internal UnknownFieldSet AddOrReplaceField(int number, UnknownField field) + { + if (number == 0) + { + throw new ArgumentOutOfRangeException("number", "Zero is not a valid field number."); + } + fields[number] = field; + return this; + } + + /// + /// Parse a single field from and merge it + /// into this set. + /// + /// The coded input stream containing the field + /// false if the tag is an "end group" tag, true otherwise + private bool MergeFieldFrom(CodedInputStream input) + { + uint tag = input.LastTag; + int number = WireFormat.GetTagFieldNumber(tag); + switch (WireFormat.GetTagWireType(tag)) + { + case WireFormat.WireType.Varint: + { + ulong uint64 = input.ReadUInt64(); + GetOrAddField(number).AddVarint(uint64); + return true; + } + case WireFormat.WireType.Fixed32: + { + uint uint32 = input.ReadFixed32(); + GetOrAddField(number).AddFixed32(uint32); + return true; + } + case WireFormat.WireType.Fixed64: + { + ulong uint64 = input.ReadFixed64(); + GetOrAddField(number).AddFixed64(uint64); + return true; + } + case WireFormat.WireType.LengthDelimited: + { + ByteString bytes = input.ReadBytes(); + GetOrAddField(number).AddLengthDelimited(bytes); + return true; + } + case WireFormat.WireType.StartGroup: + { + uint endTag = WireFormat.MakeTag(number, WireFormat.WireType.EndGroup); + UnknownFieldSet set = new UnknownFieldSet(); + while (input.ReadTag() != endTag) + { + set.MergeFieldFrom(input); + } + GetOrAddField(number).AddGroup(set); + return true; + } + case WireFormat.WireType.EndGroup: + { + return false; + } + default: + throw InvalidProtocolBufferException.InvalidWireType(); + } + } + +#if NETSTANDARD2_0 + /// + /// Parse a single field from and merge it + /// into this set. + /// + /// The coded input reader containing the field + /// false if the tag is an "end group" tag, true otherwise + private bool MergeFieldFrom(ref CodedInputReader input) + { + uint tag = input.LastTag; + int number = WireFormat.GetTagFieldNumber(tag); + switch (WireFormat.GetTagWireType(tag)) + { + case WireFormat.WireType.Varint: + { + ulong uint64 = input.ReadUInt64(); + GetOrAddField(number).AddVarint(uint64); + return true; + } + case WireFormat.WireType.Fixed32: + { + uint uint32 = input.ReadFixed32(); + GetOrAddField(number).AddFixed32(uint32); + return true; + } + case WireFormat.WireType.Fixed64: + { + ulong uint64 = input.ReadFixed64(); + GetOrAddField(number).AddFixed64(uint64); + return true; + } + case WireFormat.WireType.LengthDelimited: + { + ByteString bytes = input.ReadBytes(); + GetOrAddField(number).AddLengthDelimited(bytes); + return true; + } + case WireFormat.WireType.StartGroup: + { + uint endTag = WireFormat.MakeTag(number, WireFormat.WireType.EndGroup); + UnknownFieldSet set = new UnknownFieldSet(); + while (input.ReadTag() != endTag) + { + set.MergeFieldFrom(ref input); + } + GetOrAddField(number).AddGroup(set); + return true; + } + case WireFormat.WireType.EndGroup: + { + return false; + } + default: + throw InvalidProtocolBufferException.InvalidWireType(); + } + } +#endif + + /// + /// Create a new UnknownFieldSet if unknownFields is null. + /// Parse a single field from and merge it + /// into unknownFields. If is configured to discard unknown fields, + /// will be returned as-is and the field will be skipped. + /// + /// The UnknownFieldSet which need to be merged + /// The coded input stream containing the field + /// The merged UnknownFieldSet + public static UnknownFieldSet MergeFieldFrom(UnknownFieldSet unknownFields, + CodedInputStream input) + { + if (input.DiscardUnknownFields) + { + input.SkipLastField(); + return unknownFields; + } + if (unknownFields == null) + { + unknownFields = new UnknownFieldSet(); + } if (!unknownFields.MergeFieldFrom(input)) { throw new InvalidProtocolBufferException("Merge an unknown field of end-group tag, indicating that the corresponding start-group was missing."); // match the old code-gen - } - return unknownFields; + } + return unknownFields; + } + +#if NETSTANDARD2_0 + /// + /// Create a new UnknownFieldSet if unknownFields is null. + /// Parse a single field from and merge it + /// into unknownFields. If is configured to discard unknown fields, + /// will be returned as-is and the field will be skipped. + /// + /// The UnknownFieldSet which need to be merged + /// The coded input reader containing the field + /// The merged UnknownFieldSet + public static UnknownFieldSet MergeFieldFrom(UnknownFieldSet unknownFields, + ref CodedInputReader input) + { + if (input.DiscardUnknownFields) + { + input.SkipLastField(); + return unknownFields; + } + if (unknownFields == null) + { + unknownFields = new UnknownFieldSet(); + } + if (!unknownFields.MergeFieldFrom(ref input)) + { + throw new InvalidProtocolBufferException("Merge an unknown field of end-group tag, indicating that the corresponding start-group was missing."); // match the old code-gen + } + return unknownFields; + } +#endif + + /// + /// Merges the fields from into this set. + /// If a field number exists in both sets, the values in + /// will be appended to the values in this set. + /// + private UnknownFieldSet MergeFrom(UnknownFieldSet other) + { + if (other != null) + { + foreach (KeyValuePair entry in other.fields) + { + MergeField(entry.Key, entry.Value); + } + } + return this; + } + + /// + /// Created a new UnknownFieldSet to if + /// needed and merges the fields from into the first set. + /// If a field number exists in both sets, the values in + /// will be appended to the values in this set. + /// + public static UnknownFieldSet MergeFrom(UnknownFieldSet unknownFields, + UnknownFieldSet other) + { + if (other == null) + { + return unknownFields; + } + if (unknownFields == null) + { + unknownFields = new UnknownFieldSet(); + } + unknownFields.MergeFrom(other); + return unknownFields; + } + + + /// + /// Adds a field to the unknown field set. If a field with the same + /// number already exists, the two are merged. + /// + private UnknownFieldSet MergeField(int number, UnknownField field) + { + if (number == 0) + { + throw new ArgumentOutOfRangeException("number", "Zero is not a valid field number."); + } + if (HasField(number)) + { + GetOrAddField(number).MergeFrom(field); + } + else + { + AddOrReplaceField(number, field); + } + return this; + } + + /// + /// Clone an unknown field set from . + /// + public static UnknownFieldSet Clone(UnknownFieldSet other) + { + if (other == null) + { + return null; + } + UnknownFieldSet unknownFields = new UnknownFieldSet(); + unknownFields.MergeFrom(other); + return unknownFields; } - - /// - /// Merges the fields from into this set. - /// If a field number exists in both sets, the values in - /// will be appended to the values in this set. - /// - private UnknownFieldSet MergeFrom(UnknownFieldSet other) - { - if (other != null) - { - foreach (KeyValuePair entry in other.fields) - { - MergeField(entry.Key, entry.Value); - } - } - return this; - } - - /// - /// Created a new UnknownFieldSet to if - /// needed and merges the fields from into the first set. - /// If a field number exists in both sets, the values in - /// will be appended to the values in this set. - /// - public static UnknownFieldSet MergeFrom(UnknownFieldSet unknownFields, - UnknownFieldSet other) - { - if (other == null) - { - return unknownFields; - } - if (unknownFields == null) - { - unknownFields = new UnknownFieldSet(); - } - unknownFields.MergeFrom(other); - return unknownFields; - } - - - /// - /// Adds a field to the unknown field set. If a field with the same - /// number already exists, the two are merged. - /// - private UnknownFieldSet MergeField(int number, UnknownField field) - { - if (number == 0) - { - throw new ArgumentOutOfRangeException("number", "Zero is not a valid field number."); - } - if (HasField(number)) - { - GetOrAddField(number).MergeFrom(field); - } - else - { - AddOrReplaceField(number, field); - } - return this; - } - - /// - /// Clone an unknown field set from . - /// - public static UnknownFieldSet Clone(UnknownFieldSet other) - { - if (other == null) - { - return null; - } - UnknownFieldSet unknownFields = new UnknownFieldSet(); - unknownFields.MergeFrom(other); - return unknownFields; - } - } -} - + } +} +