Skip to content

Commit

Permalink
Integration testing based imprevements (#47)
Browse files Browse the repository at this point in the history 
Co-authored-by: Sakari Malkki <[email protected]>
  • Loading branch information
@jsaarimaa @sakari-malkki
jsaarimaa and sakari-malkki authored Nov 13, 2024
1 parent e2b5a58 commit f572368
Show file tree
Hide file tree
Showing 17 changed files with 439 additions and 71 deletions.
66 changes: 66 additions & 0 deletions .../ModelBuilderTests/Fixtures/PxFileMetaEntries_Robust_1_Language_With_Table_Level_Units.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Px.Utils.UnitTests.ModelBuilderTests.Fixtures
{
internal static class PxFileMetaEntries_Robust_1_Language_With_Table_Level_Units_And_Precision
{
public static List<KeyValuePair<string, string>> Entries =
[
new("CHARSET", "\"ANSI\""),
new("AXIS-VERSION", "\"2013\""),
new("CODEPAGE", "\"iso-8859-15\""),
new("LANGUAGE", "\"fi\""),
new("CREATION-DATE", "\"20200121 09:00\""),
new("NEXT-UPDATE", "\"20240131 08:00\""),
new("TABLEID", "\"example_table_id_for_testing\""),
new("DECIMALS", "0"),
new("SHOWDECIMALS", "1"),
new("MATRIX", "\"001_12ab_2022\""),
new("SUBJECT-CODE", "\"ABCD\""),
new("SUBJECT-AREA", "\"abcd\""),
new("COPYRIGHT", "YES"),
new("DESCRIPTION", "\"test_description_fi\""),
new("TITLE", "\"test_title_fi\""),
new("CONTENTS", "\"test_contents_fi\""),
new("UNITS", "\"test_unit_fi\""),
new("STUB", "\"Vuosi\",\"Alue\",\"Talotyyppi\""),
new("HEADING", "\"Tiedot\""),
new("CONTVARIABLE", "\"Tiedot\""),
new("VALUES(\"Vuosi\")", "\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\""),
new("VALUES(\"Alue\")", "\"Koko maa\",\"Pääkaupunkiseutu (PKS)\",\"Muu Suomi (koko maa pl. PKS)\",\"Helsinki\", \"Espoo-Kauniainen\",\"Vantaa\",\"Turku\""),
new("VALUES(\"Talotyyppi\")", "\"Talotyypit yhteensä\",\"Rivitalot\",\"Kerrostalot\""),
new("VALUES(\"Tiedot\")", "\"Indeksi (2015=100)\",\"Muutos edelliseen vuoteen (indeksi 2015=100)\",\"Kauppojen lukumäärä\""),
new("TIMEVAL(\"Vuosi\")", "TLIST(A1),\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\""),
new("CODES(\"Vuosi\")", "\"2015\",\"2016\",\"2017\",\"2018\",\"2019\",\"2020\",\"2021\",\"2022\""),
new("CODES(\"Alue\")", "\"ksu\",\"pks\",\"msu\",\"091\",\"049\",\"092\",\"853\""),
new("CODES(\"Talotyyppi\")", "\"0\",\"1\",\"3\""),
new("CODES(\"Tiedot\")", "\"ketjutettu_lv\",\"vmuutos_lv\",\"lkm_julk_uudet\""),
new("VARIABLE-TYPE(\"Vuosi\")", "\"Time\""),
new("VARIABLE-TYPE(\"Alue\")", "\"Classificatory\""),
new("VARIABLE-TYPE(\"Talotyyppi\")", "\"Classificatory\""),
new("MAP(\"Alue\")", "\"Alue 2018\""),
new("ELIMINATION(\"Talotyyppi\")", "\"Talotyypit yhteensä\""),
new("PRECISION(\"Tiedot\",\"Muutos edelliseen vuoteen (indeksi 2015=100)\")", "1"),
new("LAST-UPDATED(\"Indeksi (2015=100)\")", "\"20230131 08:00\""),
new("LAST-UPDATED(\"Muutos edelliseen vuoteen (indeksi 2015=100)\")", "\"20230131 09:00\""),
new("LAST-UPDATED(\"Kauppojen lukumäärä\")", "\"20230131 10:00\""),
new("UNITS(\"Indeksi (2015=100)\")", "\"indeksipisteluku\""),
new("UNITS(\"Muutos edelliseen vuoteen (indeksi 2015=100)\")", "\"%\""),
new("UNITS", "\"lukumäärä\""), // table level units
new("CONTACT(\"Indeksi (2015=100)\")", "\"test_contact1_fi\""),
new("CONTACT(\"Muutos edelliseen vuoteen (indeksi 2015=100)\")", "\"test_contact2_fi\""),
new("CONTACT(\"Kauppojen lukumäärä\")", "\"test_contact3_fi\""),
new("SOURCE", "\"test_source_fi\""),
new("OFFICIAL-STATISTICS", "YES"),
new("NOTE", "\"test_note_fi\""),
new("NOTE(\"Talotyyppi\")", "\"test_note_talotyyppi\""),
new("VALUENOTE(\"Tiedot\",\"Indeksi (2015=100)\")", "\"test_value_note_tiedot_indeksi\""),
new("VALUENOTE(\"Tiedot\",\"Muutos edelliseen vuoteen (indeksi 2015=100)\")", "\"test_value_note_tiedot_muutos\""),
new("VALUENOTE(\"Tiedot\",\"Kauppojen lukumäärä\")", "\"test_value_note_tiedot_kauppojen_lukumäärä\"")
];
}
}
27 changes: 27 additions & 0 deletions Px.Utils.UnitTests/ModelBuilderTests/MatrixMetadataBuilderTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
using Px.Utils.Models.Metadata.Dimensions;
using Px.Utils.Models.Metadata.Enums;
using Px.Utils.Models.Metadata.MetaProperties;
using Px.Utils.PxFile;
using Px.Utils.UnitTests.ModelBuilderTests.Fixtures;
using System.Globalization;
using Px.Utils.PxFile;

Check warning on line 11 in Px.Utils.UnitTests/ModelBuilderTests/MatrixMetadataBuilderTests.cs

View workflow job for this annotation

GitHub Actions / sonar 

The using directive for 'Px.Utils.PxFile' appeared previously in this namespace
using Px.Utils.UnitTests.ModelBuilderTests.Fixtures;

Check warning on line 12 in Px.Utils.UnitTests/ModelBuilderTests/MatrixMetadataBuilderTests.cs

View workflow job for this annotation

GitHub Actions / sonar 

The using directive for 'Px.Utils.UnitTests.ModelBuilderTests.Fixtures' appeared previously in this namespace
Expand All @@ -17,6 +19,8 @@ public class MatrixMetadataBuilderTests
private MatrixMetadata Actual_3Lang { get; } = new MatrixMetadataBuilder().Build(PxFileMetaEntries_Robust_3_Languages.Entries);
private MatrixMetadata Actual_1Lang { get; } = new MatrixMetadataBuilder().Build(PxFileMetaEntries_Robust_1_Language.Entries);
private MatrixMetadata Actual_Recommended_3Lang { get; } = new MatrixMetadataBuilder().Build(PxFileMetaEntries_Recommended_3_Langs.Entries);
private MatrixMetadata Actual_1Lang_With_Table_Level_Units_And_Precision { get; } =
new MatrixMetadataBuilder().Build(PxFileMetaEntries_Robust_1_Language_With_Table_Level_Units_And_Precision.Entries);

[TestMethod]
public void IEnumerableBuildTest()
Expand Down Expand Up @@ -206,6 +210,29 @@ public void SingleLangVariableBuildTest()
CollectionAssert.AreEqual(expectedNames, Actual_1Lang.Dimensions.Select(d => d.Name).ToList());
}

[TestMethod]
public void SingleLangWithTableLevelUnitsAndPrecisionBuildTest()
{
ContentDimension? contentDimension = (ContentDimension?)Actual_1Lang_With_Table_Level_Units_And_Precision.Dimensions.Find(d => d.Type == DimensionType.Content);
Assert.IsNotNull(contentDimension);
MultilanguageString[] expectedUnits = [
new("fi", "indeksipisteluku"),
new("fi", "%"),
new("fi", "lukumäärä")
];
for(int i = 0; i < contentDimension.Values.Count; i++)
{
Assert.AreEqual(expectedUnits[i], contentDimension.Values[i].Unit);
}
for(int i = 0; i < contentDimension.Values.Count; i++)
{
Assert.AreEqual(1, contentDimension.Values[i].Precision);
}
Assert.IsFalse(Actual_1Lang_With_Table_Level_Units_And_Precision.AdditionalProperties.ContainsKey(PxFileSyntaxConf.Default.Tokens.KeyWords.Units));
Assert.IsFalse(Actual_1Lang_With_Table_Level_Units_And_Precision.AdditionalProperties.ContainsKey(PxFileSyntaxConf.Default.Tokens.KeyWords.Decimals));
Assert.IsFalse(Actual_1Lang_With_Table_Level_Units_And_Precision.AdditionalProperties.ContainsKey(PxFileSyntaxConf.Default.Tokens.KeyWords.ShowDecimals));
}

#region Content Dimension Tests

[TestMethod]
Expand Down
91 changes: 91 additions & 0 deletions Px.Utils.UnitTests/SerializerTests/Fixtures/Json/MatrixMetadataJson.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,5 +95,96 @@ internal static class MatrixMetadataJson

public static string SimpleMetaWithoutWhitespace =
@"{""DefaultLanguage"":""foo"",""AvailableLanguages"":[""foo"",""bar""],""Dimensions"":[{""Type"":""Nominal"",""Code"":""dimension_code_1"",""Name"":{""foo"":""dimension_name_1_foo"",""bar"":""dimension_name_1_bar""},""Values"":[{""Code"":""value_code_1_1"",""Name"":{""foo"":""value_name_1_1_foo"",""bar"":""value_name_1_1_bar""},""AdditionalProperties"":{},""IsVirtual"":false}],""AdditionalProperties"":{}},{""Type"":""Content"",""Code"":""dimension_code_2"",""Name"":{""foo"":""dimension_name_2_foo"",""bar"":""dimension_name_2_bar""},""Values"":[{""Unit"":{""foo"":""unit_name_2_1_foo"",""bar"":""unit_name_2_1_bar""},""LastUpdated"":""0001-01-01T00:00:00"",""Precision"":1,""Code"":""value_code_2_1"",""Name"":{""foo"":""value_name_2_1_foo"",""bar"":""value_name_2_1_bar""},""AdditionalProperties"":{},""IsVirtual"":false}],""AdditionalProperties"":{}},{""Type"":""Time"",""Code"":""dimension_code_3"",""Name"":{""foo"":""dimension_name_3_foo"",""bar"":""dimension_name_3_bar""},""Interval"":""Year"",""Values"":[{""Code"":""value_code_3_1"",""Name"":{""foo"":""value_name_3_1_foo"",""bar"":""value_name_3_1_bar""},""AdditionalProperties"":{},""IsVirtual"":false}],""AdditionalProperties"":{}}],""AdditionalProperties"":{""property_1"":{""Type"":""Text"",""Value"":""property_value_1""},""property_2"":{""Type"":""MultilanguageText"",""Value"":{""foo"":""property_value_foo"",""bar"":""property_value_bar""}}}}";

public static string SimpleMetaPascalCase =
@"{
""DefaultLanguage"": ""foo"",
""AvailableLanguages"": [
""foo"",
""bar""
],
""Dimensions"": [
{
""Type"": ""Nominal"",
""Code"": ""dimension_code_1"",
""Name"": {
""foo"": ""dimension_name_1_foo"",
""bar"": ""dimension_name_1_bar""
},
""Values"": [
{
""Code"": ""value_code_1_1"",
""Name"": {
""foo"": ""value_name_1_1_foo"",
""bar"": ""value_name_1_1_bar""
},
""AdditionalProperties"": {},
""IsVirtual"": false
}
],
""AdditionalProperties"": {}
},
{
""Type"": ""Content"",
""Code"": ""dimension_code_2"",
""Name"": {
""foo"": ""dimension_name_2_foo"",
""bar"": ""dimension_name_2_bar""
},
""Values"": [
{
""Unit"": {
""foo"": ""unit_name_2_1_foo"",
""bar"": ""unit_name_2_1_bar""
},
""LastUpdated"": ""0001-01-01T00:00:00"",
""Precision"": 1,
""Code"": ""value_code_2_1"",
""Name"": {
""foo"": ""value_name_2_1_foo"",
""bar"": ""value_name_2_1_bar""
},
""AdditionalProperties"": {},
""IsVirtual"": false
}
],
""AdditionalProperties"": {}
},
{
""Type"": ""Time"",
""Code"": ""dimension_code_3"",
""Name"": {
""foo"": ""dimension_name_3_foo"",
""bar"": ""dimension_name_3_bar""
},
""Interval"": ""Year"",
""Values"": [
{
""Code"": ""value_code_3_1"",
""Name"": {
""foo"": ""value_name_3_1_foo"",
""bar"": ""value_name_3_1_bar""
},
""AdditionalProperties"": {},
""IsVirtual"": false
}
],
""AdditionalProperties"": {}
}
],
""additionalProperties"": {
""property_1"": {
""type"": ""Text"",
""value"": ""property_value_1""
},
""property_2"": {
""type"": ""MultilanguageText"",
""value"": {
""foo"": ""property_value_foo"",
""bar"": ""property_value_bar""
}
}
}
}";
}
}
12 changes: 11 additions & 1 deletion Px.Utils.UnitTests/SerializerTests/MatrixMetadataConverterSerializeTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ public class MatrixMetadataConverterSerializeTests
private static readonly JsonSerializerOptions IndentedCachedOptions = new()
{
WriteIndented = true,
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = true
};

private static readonly JsonSerializerOptions OneLineCachedOptions = new()
Expand Down Expand Up @@ -57,5 +58,14 @@ public void SerializeMetadataToOneLineMatchesFixture()
string output = JsonSerializer.Serialize(MatrixMetadataFixture.SimpleMeta, OneLineCachedOptions);
Assert.AreEqual(MatrixMetadataJson.SimpleMetaWithoutWhitespace, output);
}

[TestMethod]
public void DeserializeMetadataFromPascalCaseJsonReturnsCorrectObject()
{
MatrixMetadata? deserialized = JsonSerializer.Deserialize<MatrixMetadata>(MatrixMetadataJson.SimpleMetaPascalCase, IndentedCachedOptions);
string output = JsonSerializer.Serialize(deserialized, IndentedCachedOptions);
string expected = JsonSerializer.Serialize(MatrixMetadataFixture.SimpleMeta, IndentedCachedOptions);
Assert.AreEqual(expected, output);
}
}
}
13 changes: 13 additions & 0 deletions Px.Utils.UnitTests/SerializerTests/MultilanguageStringConverterTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,5 +107,18 @@ public void DeserializeMultilanguageStringWithNoLanguages()
Assert.IsNotNull(deserialized);
Assert.AreEqual(0, deserialized.Languages.Count());
}

[TestMethod]
public void DeserializeMultilanguageStringWithNull()
{
// Arrange
string serialized = "null";

// Act
MultilanguageString? deserialized = JsonSerializer.Deserialize<MultilanguageString>(serialized);

// Assert
Assert.IsNull(deserialized);
}
}
}
63 changes: 51 additions & 12 deletions Px.Utils/ModelBuilders/MatrixMetadataBuilder.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -226,7 +226,19 @@ private ContentDimension BuildContentDimension(Dictionary<MetadataEntryKey, stri
{
string code = GetDimensionCode(entries, langs, dimensionName);
ContentValueList values = BuildContentDimensionValues(entries, langs, dimensionName);

// Table level UNIT, SHOWDECIMALS and DECIMALS properties are not needed after building the content dimension, so they're removed here
if (TryGetEntries(entries, _pxFileSyntaxConf.Tokens.KeyWords.Units, langs, out Dictionary<MetadataEntryKey, string>? unitEntries))
{
foreach (MetadataEntryKey key in unitEntries.Keys) entries.Remove(key);
}
if (TryGetEntries(entries, _pxFileSyntaxConf.Tokens.KeyWords.Decimals, langs, out Dictionary<MetadataEntryKey, string>? decimalEntries))
{
foreach (MetadataEntryKey key in decimalEntries.Keys) entries.Remove(key);
}
if (TryGetEntries(entries, _pxFileSyntaxConf.Tokens.KeyWords.ShowDecimals, langs, out Dictionary<MetadataEntryKey, string>? showDecimalEntries))
{
foreach (MetadataEntryKey key in showDecimalEntries.Keys) entries.Remove(key);
}
return new ContentDimension(code, dimensionName, [], values);
}

Expand Down Expand Up @@ -391,19 +403,32 @@ private string GetDimensionCode(Dictionary<MetadataEntryKey, string> entries, Px
private MultilanguageString GetUnit(Dictionary<MetadataEntryKey, string> entries, PxFileLanguages langs, MultilanguageString dimName, MultilanguageString valName)
{
string unitKey = _pxFileSyntaxConf.Tokens.KeyWords.Units;
// If table level unit is used, the unit key is not associated with a specific dimension value and is removed after building the content dimension
bool tableLevelUnitUsed;
if (TryGetEntries(entries, unitKey, langs, out Dictionary<MetadataEntryKey, string>? unitEntries, dimName, valName) || // Both identifiers
TryGetEntries(entries, unitKey, langs, out unitEntries, valName) || // Only value identifier
TryGetEntries(entries, unitKey, langs, out unitEntries)) // No identifiers
TryGetEntries(entries, unitKey, langs, out unitEntries, valName)) // One identifier
{
tableLevelUnitUsed = false;
}
else if (TryGetEntries(entries, unitKey, langs, out unitEntries)) // No identifiers
{
tableLevelUnitUsed = true;
}
else
{
Dictionary<string, string> translations = [];
foreach (KeyValuePair<MetadataEntryKey, string> kvp in unitEntries)
throw new ArgumentException("Unit information not found");
}

Dictionary<string, string> translations = [];
foreach (KeyValuePair<MetadataEntryKey, string> kvp in unitEntries)
{
translations[kvp.Key.Language ?? langs.DefaultLanguage] = kvp.Value.CleanStringDelimeters(_stringDelimeter);
if (!tableLevelUnitUsed)
{
translations[kvp.Key.Language ?? langs.DefaultLanguage] = kvp.Value.CleanStringDelimeters(_stringDelimeter);
entries.Remove(kvp.Key);
}
return new MultilanguageString(translations);
}
throw new ArgumentException("Unit information not found");
return new MultilanguageString(translations);
}

private DateTime GetLastUpdated(Dictionary<MetadataEntryKey, string> entries, PxFileLanguages langs, MultilanguageString dimName, MultilanguageString valName)
Expand All @@ -425,14 +450,28 @@ private DateTime GetLastUpdated(Dictionary<MetadataEntryKey, string> entries, Px
private int GetPrecision(Dictionary<MetadataEntryKey, string> entries, PxFileLanguages langs, MultilanguageString dimName, MultilanguageString valName)
{
string precisionKey = _pxFileSyntaxConf.Tokens.KeyWords.Precision;
if ((TryGetEntries(entries, precisionKey, langs, out Dictionary<MetadataEntryKey, string>? precisionEntries, dimName, valName) || // Both identifiers
// If table level precision is used, the precision key is not associated with a specific dimension value and is removed after building the content dimension
bool tableLevelPrecisionUsed = false;
if (TryGetEntries(entries, precisionKey, langs, out Dictionary<MetadataEntryKey, string>? precisionEntries, dimName, valName) || // Both identifiers
TryGetEntries(entries, precisionKey, langs, out precisionEntries, valName)) // Only value identifier
&& int.TryParse(precisionEntries.Values.First(), out int result))
{
foreach (MetadataEntryKey key in precisionEntries.Keys) entries.Remove(key);
return result;
tableLevelPrecisionUsed = false;
}
// No identifiers, table level precision using SHOWDECIMALS and DECIMALS keyword
else if (TryGetEntries(entries, _pxFileSyntaxConf.Tokens.KeyWords.ShowDecimals, langs, out precisionEntries) ||
TryGetEntries(entries, _pxFileSyntaxConf.Tokens.KeyWords.Decimals, langs, out precisionEntries))
{
tableLevelPrecisionUsed = true;
}

if (precisionEntries is not null && int.TryParse(precisionEntries.Values.First(), out int result))
{
if (!tableLevelPrecisionUsed)
{
foreach (MetadataEntryKey key in precisionEntries.Keys) entries.Remove(key);
}
return result;
}
return 0; // Default value
}

Expand Down
Loading

0 comments on commit f572368

Please sign in to comment.