Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reduce the amount of memory allocated by System.IO.Tests #66387

Merged
merged 4 commits into from
Mar 9, 2022
Merged

Reduce the amount of memory allocated by System.IO.Tests #66387

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
2e80621
use 2.1 GB instead 6.5 GB to verify lack of Int32 overflow:
adamsitnik Mar 9, 2022
e701a67
use SkipTestException to indicate that the test has been skipped
adamsitnik Mar 9, 2022
2079807
don't run WriteChars_VeryLargeArray_DoesNotOverflow with other tests …
adamsitnik Mar 9, 2022
c762134
move the test to Outerloop as suggested by Stephen
adamsitnik Mar 9, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions src/libraries/System.IO/tests/BinaryWriter/BinaryWriter.EncodingTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@
using System.Reflection;
using System.Runtime.InteropServices;
using System.Text;
using Microsoft.DotNet.XUnitExtensions;
using Xunit;

namespace System.IO.Tests
{
// WriteChars_VeryLargeArray_DoesNotOverflow allocates a lot of memory and can cause OOM,
// it should not be executed in parallel with other tests
[Collection(nameof(DisableParallelization))]
public class BinaryWriter_EncodingTests
{
[Fact]
Expand Down Expand Up @@ -187,40 +191,40 @@ public void WriteString_NotUtf8(int stringLengthInChars)
Assert.Equal(expectedBytes, stream.GetBuffer()[Get7BitEncodedIntByteLength((uint)expectedBytes.Length)..(int)stream.Length]);
}

[OuterLoop("Allocates a lot of memory")]
[Fact]
[SkipOnPlatform(TestPlatforms.Android, "OOM on Android could be uncatchable & kill the test runner")]
public unsafe void WriteChars_VeryLargeArray_DoesNotOverflow()
adamsitnik marked this conversation as resolved.
Show resolved Hide resolved
{
const nuint INPUT_LEN_IN_CHARS = 1_500_000_000;
const nuint OUTPUT_LEN_IN_BYTES = 3_500_000_000; // overallocate
const nuint INT32_OVERFLOW_SIZE = (nuint)int.MaxValue + 3;

SafeBuffer unmanagedInputBuffer = null;
SafeBuffer unmanagedOutputBufer = null;
SafeBuffer unmanagedBuffer = null;
try
{
try
{
unmanagedInputBuffer = SafeBufferUtil.CreateSafeBuffer(INPUT_LEN_IN_CHARS * sizeof(char));
unmanagedOutputBufer = SafeBufferUtil.CreateSafeBuffer(OUTPUT_LEN_IN_BYTES * sizeof(byte));
unmanagedBuffer = SafeBufferUtil.CreateSafeBuffer(INT32_OVERFLOW_SIZE * sizeof(byte));
}
catch (OutOfMemoryException)
{
return; // skip test in low-mem conditions
throw new SkipTestException($"Unable to execute {nameof(WriteChars_VeryLargeArray_DoesNotOverflow)} due to OOM"); // skip test in low-mem conditions
}

Span<char> inputSpan = new Span<char>((char*)unmanagedInputBuffer.DangerousGetHandle(), (int)INPUT_LEN_IN_CHARS);
inputSpan.Fill('\u0224'); // LATIN CAPITAL LETTER Z WITH HOOK
Stream outStream = new UnmanagedMemoryStream(unmanagedOutputBufer, 0, (long)unmanagedOutputBufer.ByteLength, FileAccess.ReadWrite);
Assert.True((long)unmanagedBuffer.ByteLength > int.MaxValue);

// reuse same memory for input and output to avoid allocating more memory and OOMs
Span<char> span = new Span<char>((char*)unmanagedBuffer.DangerousGetHandle(), (int)(INT32_OVERFLOW_SIZE / sizeof(char)));
span.Fill('\u0224'); // LATIN CAPITAL LETTER Z WITH HOOK
Stream outStream = new UnmanagedMemoryStream(unmanagedBuffer, 0, (long)unmanagedBuffer.ByteLength, FileAccess.ReadWrite);
BinaryWriter writer = new BinaryWriter(outStream);

writer.Write(inputSpan); // will write 3 billion bytes to the output
writer.Write(span); // will write slightly more than int.MaxValue bytes to the output

Assert.Equal(3_000_000_000, outStream.Position);
Assert.Equal((long)INT32_OVERFLOW_SIZE, outStream.Position);
}
finally
{
unmanagedInputBuffer?.Dispose();
unmanagedOutputBufer?.Dispose();
unmanagedBuffer?.Dispose();
}
}

Expand Down
3 changes: 2 additions & 1 deletion src/libraries/System.IO/tests/System.IO.Tests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<Project Sdk="Microsoft.NET.Sdk">
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<RootNamespace>System.IO</RootNamespace>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
Expand Down Expand Up @@ -44,6 +44,7 @@
<Compile Include="Stream\Stream.ValidateArguments.cs" />
<Compile Include="StringReader\StringReader.CtorTests.cs" />
<Compile Include="StringWriter\StringWriterTests.cs" />
<Compile Include="$(CommonTestPath)TestUtilities\System\DisableParallelization.cs" Link="Common\TestUtilities\System\DisableParallelization.cs" />
<Compile Include="$(CommonTestPath)System\Buffers\NativeMemoryManager.cs" Link="Common\System\Buffers\NativeMemoryManager.cs" />
<Compile Include="$(CommonTestPath)System\IO\CallTrackingStream.cs" Link="Common\System\IO\CallTrackingStream.cs" />
<Compile Include="$(CommonTestPath)System\IO\DelegateStream.cs" Link="Common\System\IO\DelegateStream.cs" />
Expand Down