Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Datareader errorhandling improvements #48

Merged
merged 3 commits into from
Nov 13, 2024
Merged

Datareader errorhandling improvements #48

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
41f592c
Datareader now throws exceptions if the imput maps are not valid for …
jsaarimaa Nov 12, 2024
55e3c2e
Updated the README to match the new interfaces
jsaarimaa Nov 13, 2024
68fba91
Fixed issues found by sonar
jsaarimaa Nov 13, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 25 additions & 25 deletions Px.Utils.TestingApp/Commands/DataReadBenchmark.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ internal sealed class DataReadBenchmark : FileBenchmark

internal override string Description => "Benchmarks the data reading capabilities of the PxFileStreamDataReader.";

private DataIndexer? Indexer { get; set; }
private IMatrixMap? Target { get; set; }

private int _numberOfCells = 1000000;

Expand Down Expand Up @@ -53,81 +53,81 @@ protected override void OneTimeBenchmarkSetup()
private void RunReadDoubleDataValuesBenchmarks()
{
if(MetaData is null) throw new InvalidOperationException(metadataNotFoundMessage);
Indexer = GenerateBenchmarkIndexer(MetaData, _numberOfCells);
Target = GenerateBenchmarkTargetMap(MetaData, _numberOfCells);

DoubleDataValue[] buffer = new DoubleDataValue[Indexer.DataLength];
DoubleDataValue[] buffer = new DoubleDataValue[Target.GetSize()];

using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read);
using PxFileStreamDataReader reader = new(stream);

reader.ReadDoubleDataValues(buffer, 0, Indexer);
reader.ReadDoubleDataValues(buffer, 0, Target, MetaData);
}

private void RunReadDecimalDataValuesBenchmarks()
{
if (MetaData is null) throw new InvalidOperationException(metadataNotFoundMessage);
Indexer = GenerateBenchmarkIndexer(MetaData, _numberOfCells);
Target = GenerateBenchmarkTargetMap(MetaData, _numberOfCells);

DecimalDataValue[] buffer = new DecimalDataValue[Indexer.DataLength];
DecimalDataValue[] buffer = new DecimalDataValue[Target.GetSize()];

using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read);
using PxFileStreamDataReader reader = new(stream);

reader.ReadDecimalDataValues(buffer, 0, Indexer);
reader.ReadDecimalDataValues(buffer, 0, Target, MetaData);
}

private void RunReadUnsafeDoubleBenchmarks()
{
if (MetaData is null) throw new InvalidOperationException(metadataNotFoundMessage);
Indexer = GenerateBenchmarkIndexer(MetaData, _numberOfCells);
Target = GenerateBenchmarkTargetMap(MetaData, _numberOfCells);

double[] buffer = new double[Indexer.DataLength];
double[] buffer = new double[Target.GetSize()];
double[] missingValueEncodings = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0];

using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read);
using PxFileStreamDataReader reader = new(stream);

reader.ReadUnsafeDoubles(buffer, 0, Indexer, missingValueEncodings);
reader.ReadUnsafeDoubles(buffer, 0, Target, MetaData, missingValueEncodings);
}

private async Task RunReadDoubleDataValuesBenchmarksAsync()
{
if (MetaData is null) throw new InvalidOperationException(metadataNotFoundMessage);
Indexer = GenerateBenchmarkIndexer(MetaData, _numberOfCells);
Target = GenerateBenchmarkTargetMap(MetaData, _numberOfCells);

DoubleDataValue[] buffer = new DoubleDataValue[Indexer.DataLength];
DoubleDataValue[] buffer = new DoubleDataValue[Target.GetSize()];

using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read);
using PxFileStreamDataReader reader = new(stream);

await reader.ReadDoubleDataValuesAsync(buffer, 0, Indexer);
await reader.ReadDoubleDataValuesAsync(buffer, 0, Target, MetaData);
}

private async Task RunReadDecimalDataValuesBenchmarksAsync()
{
if (MetaData is null) throw new InvalidOperationException(metadataNotFoundMessage);
Indexer = GenerateBenchmarkIndexer(MetaData, _numberOfCells);
Target = GenerateBenchmarkTargetMap(MetaData, _numberOfCells);

DecimalDataValue[] buffer = new DecimalDataValue[Indexer.DataLength];
DecimalDataValue[] buffer = new DecimalDataValue[Target.GetSize()];

using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read);
using PxFileStreamDataReader reader = new(stream);

await reader.ReadDecimalDataValuesAsync(buffer, 0, Indexer);
await reader.ReadDecimalDataValuesAsync(buffer, 0, Target, MetaData);
}

private async Task RunReadUnsafeDoubleBenchmarksAsync()
{
if (MetaData is null) throw new InvalidOperationException(metadataNotFoundMessage);
Indexer = GenerateBenchmarkIndexer(MetaData, _numberOfCells);
Target = GenerateBenchmarkTargetMap(MetaData, _numberOfCells);

double[] buffer = new double[Indexer.DataLength];
double[] buffer = new double[Target.GetSize()];
double[] missingValueEncodings = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0];

using Stream stream = new FileStream(TestFilePath, FileMode.Open, FileAccess.Read);
using PxFileStreamDataReader reader = new(stream);

await reader.ReadUnsafeDoublesAsync(buffer, 0, Indexer, missingValueEncodings);
await reader.ReadUnsafeDoublesAsync(buffer, 0, Target, MetaData, missingValueEncodings);
}

protected override void SetRunParameters()
Expand All @@ -143,12 +143,12 @@ protected override void SetRunParameters()
}
}

private static DataIndexer GenerateBenchmarkIndexer(IMatrixMap map, int targetSize)
private static IMatrixMap GenerateBenchmarkTargetMap(IMatrixMap complete, int targetSize)
{
int size = map.GetSize();
if (size < targetSize) return new DataIndexer(map, map);
int size = complete.GetSize();
if (size < targetSize) return complete;

List<IDimensionMap> sortedDimensions = [.. map.DimensionMaps];
List<IDimensionMap> sortedDimensions = [.. complete.DimensionMaps];
while (size > targetSize)
{
sortedDimensions = [.. sortedDimensions.OrderByDescending(x => x.ValueCodes.Count)];
Expand All @@ -157,15 +157,15 @@ private static DataIndexer GenerateBenchmarkIndexer(IMatrixMap map, int targetSi
size = sortedDimensions.Aggregate(1, (acc, x) => acc * x.ValueCodes.Count);
}

List<IDimensionMap> dimList = map.DimensionMaps
List<IDimensionMap> dimList = complete.DimensionMaps
.Select(dim => dim.Code)
.Select(dimCode => new DimensionMap(
dimCode,
[.. sortedDimensions.First(dim => dim.Code == dimCode).ValueCodes]))
.Cast<IDimensionMap>()
.ToList();

return new DataIndexer(map, new MatrixMap(dimList));
return new MatrixMap(dimList);
}
}
}
219 changes: 219 additions & 0 deletions Px.Utils.UnitTests/ModelTests/ExtensionTests/MatrixMapExtensionTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,219 @@
using Px.Utils.Models.Metadata;
using Px.Utils.Models.Metadata.ExtensionMethods;

namespace Px.Utils.UnitTests.ModelTests.ExtensionTests
{
[TestClass]
public class MatrixMapExtensionTests
{
[TestMethod]
public void WhenCalledWithItselfShouldReturnTrue()
{
// Arrange
List<IDimensionMap> dimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
];

MatrixMap map = new(dimensions);

// Act and Assert
Assert.IsTrue(map.IsSubmapOf(map));
}

[TestMethod]
public void WhenCalledWithSubmapShouldReturnTrue()
{
// Arrange
List<IDimensionMap> dimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
];

MatrixMap map = new(dimensions);

List<IDimensionMap> subDimensions = [
new DimensionMap("11", ["aa11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33"]),
];

MatrixMap subMap = new(subDimensions);

// Act and Assert
Assert.IsTrue(subMap.IsSubmapOf(map));
}

[TestMethod]
public void WhenCalledWithSubmapWithoutfirstValuesShouldReturnTrue()
{
// Arrange
List<IDimensionMap> dimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
];

MatrixMap map = new(dimensions);

List<IDimensionMap> subDimensions = [
new DimensionMap("11", ["bb11", "cc11"]),
new DimensionMap("22", ["bb22"]),
new DimensionMap("33", ["bb33", "cc33", "dd33"]),

];

MatrixMap subMap = new(subDimensions);

// Act and Assert
Assert.IsTrue(subMap.IsSubmapOf(map));
}

[TestMethod]
public void WhenCalledWithSizeOneSubmapShouldReturnTrue()
{
// Arrange
List<IDimensionMap> dimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
];

MatrixMap map = new(dimensions);

List<IDimensionMap> subDimensions = [
new DimensionMap("11", ["cc11"]),
new DimensionMap("22", ["bb22"]),
new DimensionMap("33", ["cc33"]),

];

MatrixMap subMap = new(subDimensions);

// Act and Assert
Assert.IsTrue(subMap.IsSubmapOf(map));
}

[TestMethod]
public void WhenCalledWithDifferentDimensionOrderShouldReturnFalse()
{
// Arrange
List<IDimensionMap> dimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
];

MatrixMap map = new(dimensions);

List<IDimensionMap> subDimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
new DimensionMap("22", ["aa22", "bb22"]),
];

MatrixMap subMap = new(subDimensions);

// Act and Assert
Assert.IsFalse(subMap.IsSubmapOf(map));
}

[TestMethod]
public void WhenCalledWithDifferentValuesShouldReturnFalse()
{
// Arrange
List<IDimensionMap> dimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
];

MatrixMap map = new(dimensions);

List<IDimensionMap> subDimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33", "ee33"]),
];

MatrixMap subMap = new(subDimensions);

// Act and Assert
Assert.IsFalse(subMap.IsSubmapOf(map));
}

[TestMethod]
public void WhenCalledWithDifferentNumberOfDimensionsShouldReturnFalse()
{
// Arrange
List<IDimensionMap> dimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
];

MatrixMap map = new(dimensions);

List<IDimensionMap> subDimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
];

MatrixMap subMap = new(subDimensions);

// Act and Assert
Assert.IsFalse(subMap.IsSubmapOf(map));
}

[TestMethod]
public void WhenCalledWithDifferenValueOrderShouldReturnFalse()
{
// Arrange
List<IDimensionMap> dimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
];

MatrixMap map = new(dimensions);

List<IDimensionMap> subDimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "cc33", "bb33"]),
];

MatrixMap subMap = new(subDimensions);

// Act and Assert
Assert.IsFalse(subMap.IsSubmapOf(map));
}

[TestMethod]
public void WhenCalledWithNotMatchingValueCodesShouldReturnFalse()
{
// Arrange
List<IDimensionMap> dimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["aa22", "bb22"]),
new DimensionMap("33", ["aa33", "bb33", "cc33", "dd33"]),
];

MatrixMap map = new(dimensions);

List<IDimensionMap> subDimensions = [
new DimensionMap("11", ["aa11", "bb11", "cc11"]),
new DimensionMap("22", ["foo", "bb22"]),
new DimensionMap("33", ["aa33", "bar"]),
];

MatrixMap subMap = new(subDimensions);

// Act and Assert
Assert.IsFalse(subMap.IsSubmapOf(map));
}

}
}
27 changes: 27 additions & 0 deletions Px.Utils.UnitTests/PxFileTests/DataTests/DataIndexerTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -174,5 +174,32 @@ public void ThreeDimensions3val2val5valMultidimensionalReorderingTestWithDimOrde
}
while (generator.Next());
}

[TestMethod]
public void MismatchingValueCodesShouldCauseArgumentException()
{
MatrixMetadata testMeta = TestModelBuilder.BuildTestMetadata([3, 2, 5]);
MatrixMap matrixMap = new(
[
new DimensionMap("var0", ["var0_val2", "var0_val1"]),
new DimensionMap("var1", ["var2_val0", "var1_val1"]),
new DimensionMap("var2", ["var2_val4", "var2_val3"])
]);

Assert.ThrowsException<ArgumentException>(() => new DataIndexer(testMeta, matrixMap));
}

[TestMethod]
public void MissingDimensionShouldCauseArgumentException()
{
MatrixMetadata testMeta = TestModelBuilder.BuildTestMetadata([3, 2, 5]);
MatrixMap matrixMap = new(
[
new DimensionMap("var0", ["var0_val2", "var0_val1"]),
new DimensionMap("var1", ["var1_val0", "var1_val1"])
]);

Assert.ThrowsException<ArgumentException>(() => new DataIndexer(testMeta, matrixMap));
}
}
}
Loading
Loading