Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Ensure that aggregation is consistent regardless of data alignment #106166

Merged
merged 7 commits into from
Aug 13, 2024
Merged

Ensure that aggregation is consistent regardless of data alignment #106166

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
23ed7eb
Ensure that aggregation is consistent regardless of data alignment
tannergooding Aug 8, 2024
b3d460e
Ensure we handle for all aggregation helpers
tannergooding Aug 9, 2024
864920d
Ensure we don't process beg twice
tannergooding Aug 9, 2024
238c947
Ensure that we properly track in the case we can't align
tannergooding Aug 12, 2024
23c043c
Add missing semicolon
tannergooding Aug 12, 2024
1f8f4a8
Fix the handling on .NET Framework
tannergooding Aug 12, 2024
211d70f
Ensure yptr on .NET Framework is incremented as well
tannergooding Aug 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
112 changes: 99 additions & 13 deletions ...nsors/src/System/Numerics/Tensors/netcore/Common/TensorPrimitives.IAggregationOperator.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -141,9 +141,12 @@ static T Vectorized128(ref T xRef, nuint remainder)

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// can never achieve the required SIMD alignment. This cannot be done for
// float or double since that changes how results compound together.

bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0;
bool canAlign = (typeof(T) != typeof(float)) &&
(typeof(T) != typeof(double)) &&
((nuint)xPtr % (nuint)sizeof(T)) == 0;

if (canAlign)
{
Expand All @@ -156,11 +159,20 @@ static T Vectorized128(ref T xRef, nuint remainder)
misalignment = ((uint)sizeof(Vector128<T>) - ((nuint)xPtr % (uint)sizeof(Vector128<T>))) / (uint)sizeof(T);

xPtr += misalignment;

Debug.Assert(((nuint)xPtr % (uint)sizeof(Vector128<T>)) == 0);

remainder -= misalignment;
}
else
{
// We can't align, but this also means we're processing the full data from beg
// so account for that to ensure we don't double process and include them in the
// aggregate twice.

misalignment = (uint)Vector128<T>.Count;
xPtr += misalignment;
remainder -= misalignment;
}

Vector128<T> vector1;
Vector128<T> vector2;
Expand Down Expand Up @@ -310,9 +322,12 @@ static T Vectorized256(ref T xRef, nuint remainder)

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// can never achieve the required SIMD alignment. This cannot be done for
// float or double since that changes how results compound together.

bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0;
bool canAlign = (typeof(T) != typeof(float)) &&
(typeof(T) != typeof(double)) &&
((nuint)xPtr % (nuint)sizeof(T)) == 0;

if (canAlign)
{
Expand All @@ -330,6 +345,16 @@ static T Vectorized256(ref T xRef, nuint remainder)

remainder -= misalignment;
}
else
{
// We can't align, but this also means we're processing the full data from beg
// so account for that to ensure we don't double process and include them in the
// aggregate twice.

misalignment = (uint)Vector256<T>.Count;
xPtr += misalignment;
remainder -= misalignment;
}

Vector256<T> vector1;
Vector256<T> vector2;
Expand Down Expand Up @@ -479,9 +504,12 @@ static T Vectorized512(ref T xRef, nuint remainder)

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// can never achieve the required SIMD alignment. This cannot be done for
// float or double since that changes how results compound together.

bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0;
bool canAlign = (typeof(T) != typeof(float)) &&
(typeof(T) != typeof(double)) &&
((nuint)xPtr % (nuint)sizeof(T)) == 0;

if (canAlign)
{
Expand All @@ -499,6 +527,16 @@ static T Vectorized512(ref T xRef, nuint remainder)

remainder -= misalignment;
}
else
{
// We can't align, but this also means we're processing the full data from beg
// so account for that to ensure we don't double process and include them in the
// aggregate twice.

misalignment = (uint)Vector512<T>.Count;
xPtr += misalignment;
remainder -= misalignment;
}

Vector512<T> vector1;
Vector512<T> vector2;
Expand Down Expand Up @@ -1227,9 +1265,12 @@ static T Vectorized128(ref T xRef, ref T yRef, nuint remainder)

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// can never achieve the required SIMD alignment. This cannot be done for
tannergooding marked this conversation as resolved.
Show resolved Hide resolved
// float or double since that changes how results compound together.

bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0;
bool canAlign = (typeof(T) != typeof(float)) &&
(typeof(T) != typeof(double)) &&
((nuint)xPtr % (nuint)sizeof(T)) == 0;

if (canAlign)
{
Expand All @@ -1248,6 +1289,19 @@ static T Vectorized128(ref T xRef, ref T yRef, nuint remainder)

remainder -= misalignment;
}
else
{
// We can't align, but this also means we're processing the full data from beg
// so account for that to ensure we don't double process and include them in the
// aggregate twice.

misalignment = (uint)Vector128<T>.Count;

xPtr += misalignment;
yPtr += misalignment;

remainder -= misalignment;
}

Vector128<T> vector1;
Vector128<T> vector2;
Expand Down Expand Up @@ -1418,9 +1472,12 @@ static T Vectorized256(ref T xRef, ref T yRef, nuint remainder)

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// can never achieve the required SIMD alignment. This cannot be done for
// float or double since that changes how results compound together.

bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0;
bool canAlign = (typeof(T) != typeof(float)) &&
(typeof(T) != typeof(double)) &&
((nuint)xPtr % (nuint)sizeof(T)) == 0;

if (canAlign)
{
Expand All @@ -1439,6 +1496,19 @@ static T Vectorized256(ref T xRef, ref T yRef, nuint remainder)

remainder -= misalignment;
}
else
{
// We can't align, but this also means we're processing the full data from beg
// so account for that to ensure we don't double process and include them in the
// aggregate twice.

misalignment = (uint)Vector256<T>.Count;

xPtr += misalignment;
yPtr += misalignment;

remainder -= misalignment;
}

Vector256<T> vector1;
Vector256<T> vector2;
Expand Down Expand Up @@ -1609,9 +1679,12 @@ static T Vectorized512(ref T xRef, ref T yRef, nuint remainder)

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// can never achieve the required SIMD alignment. This cannot be done for
// float or double since that changes how results compound together.

bool canAlign = ((nuint)xPtr % (nuint)sizeof(T)) == 0;
bool canAlign = (typeof(T) != typeof(float)) &&
(typeof(T) != typeof(double)) &&
((nuint)xPtr % (nuint)sizeof(T)) == 0;

if (canAlign)
{
Expand All @@ -1630,6 +1703,19 @@ static T Vectorized512(ref T xRef, ref T yRef, nuint remainder)

remainder -= misalignment;
}
else
{
// We can't align, but this also means we're processing the full data from beg
// so account for that to ensure we don't double process and include them in the
// aggregate twice.

misalignment = (uint)Vector512<T>.Count;

xPtr += misalignment;
yPtr += misalignment;

remainder -= misalignment;
}

Vector512<T> vector1;
Vector512<T> vector2;
Expand Down
58 changes: 17 additions & 41 deletions ...cs.Tensors/src/System/Numerics/Tensors/netstandard/TensorPrimitives.Single.netstandard.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,28 +175,15 @@ static float Vectorized(ref float xRef, nuint remainder, TTransformOperator tran
{
float* xPtr = px;

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.
// Unlike many other vectorization algorithms, we cannot align for aggregation
// because that changes how results compound together and can cause a significant
// difference in the output. This also means we're processing the full data from beg
// so account for that to ensure we don't double process and include them in the
// aggregate twice.

bool canAlign = ((nuint)(xPtr) % sizeof(float)) == 0;

if (canAlign)
{
// Compute by how many elements we're misaligned and adjust the pointers accordingly
//
// Noting that we are only actually aligning dPtr. This is because unaligned stores
// are more expensive than unaligned loads and aligning both is significantly more
// complex.

misalignment = ((uint)(sizeof(Vector<float>)) - ((nuint)(xPtr) % (uint)(sizeof(Vector<float>)))) / sizeof(float);

xPtr += misalignment;

Debug.Assert(((nuint)(xPtr) % (uint)(sizeof(Vector<float>))) == 0);

remainder -= misalignment;
}
misalignment = (uint)Vector<float>.Count;
xPtr += misalignment;
remainder -= misalignment;

Vector<float> vector1;
Vector<float> vector2;
Expand Down Expand Up @@ -480,29 +467,18 @@ static float Vectorized(ref float xRef, ref float yRef, nuint remainder, TBinary
float* xPtr = px;
float* yPtr = py;

// We need to the ensure the underlying data can be aligned and only align
// it if it can. It is possible we have an unaligned ref, in which case we
// can never achieve the required SIMD alignment.

bool canAlign = ((nuint)(xPtr) % sizeof(float)) == 0;

if (canAlign)
{
// Compute by how many elements we're misaligned and adjust the pointers accordingly
//
// Noting that we are only actually aligning dPtr. This is because unaligned stores
// are more expensive than unaligned loads and aligning both is significantly more
// complex.

misalignment = ((uint)(sizeof(Vector<float>)) - ((nuint)(xPtr) % (uint)(sizeof(Vector<float>)))) / sizeof(float);
// Unlike many other vectorization algorithms, we cannot align for aggregation
// because that changes how results compound together and can cause a significant
// difference in the output. This also means we're processing the full data from beg
// so account for that to ensure we don't double process and include them in the
// aggregate twice.

xPtr += misalignment;
yPtr += misalignment;
misalignment = (uint)Vector<float>.Count;

Debug.Assert(((nuint)(xPtr) % (uint)(sizeof(Vector<float>))) == 0);
xPtr += misalignment;
yPtr += misalignment;

remainder -= misalignment;
}
remainder -= misalignment;

Vector<float> vector1;
Vector<float> vector2;
Expand Down
Loading