Skip to content

Commit

Permalink
Merge pull request #23 from SasinduDilshara/development-revamp
Browse files Browse the repository at this point in the history 
Allow empty strings to represent nil values using user configs
  • Loading branch information
@SasinduDilshara
SasinduDilshara authored Aug 12, 2024
2 parents 268decc + 1928804 commit 34bea43
Show file tree
Hide file tree
Showing 11 changed files with 187 additions and 41 deletions.
10 changes: 4 additions & 6 deletions ballerina-tests/parse-string-array-types-tests/tests/parse_string_to_array_test.bal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -437,14 +437,12 @@ function testParseStringArrayAsExpectedTypeWithOutputHeaders() {
]);

string[][]|csv:Error cv1baa_5 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: true, header: 2});
test:assertEquals(cv1baa_5, [
["true", "false", "true", "false"],
["true", "false", "true", "false"]
]);
test:assertTrue(cv1baa_5 is csv:Error);
test:assertEquals((<csv:Error>cv1baa_5).message(), "Duplicate header found: 'true'");

string[][]|csv:Error cv1baa_6 = csv:parseString(csvStringWithBooleanValues1, {outputWithHeaders: false, header: 2});
string[][]|csv:Error cv1baa_6 = csv:parseString(csvStringWithBooleanValues8, {outputWithHeaders: false, header: 2});
test:assertEquals(cv1baa_6, [
["true", "false", "true", "false"]
["true", "false", "true1", "false1"]
]);

[string, string, string, string, string][]|csv:Error cv2baa_7 = csv:parseString(csvStringWithBooleanValues2, {outputWithHeaders: true});
Expand Down
6 changes: 6 additions & 0 deletions ballerina-tests/parse-string-array-types-tests/tests/test_data_values.bal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,3 +60,9 @@ string csvStringWithBooleanValues6 = string `b2,b3
string csvStringWithBooleanValues7 = string `b1,b2,b3,b4
${b1},${b2},(),${b4}
`;

string csvStringWithBooleanValues8 = string `b1,b2,b3,b4
true,false,true1,false1
true,false, true1,false1
true,false,true1,false1
`;
4 changes: 4 additions & 0 deletions ballerina-tests/type-compatible-tests/tests/csv_content_2.txt
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
a, b, c d, e
"Hello World1", \"Hello World2\", Hello World3, 21
"Hello World1", \"Hello World2\", Hello World3, 22
"Hello World1", \"Hello World2\", Hello World3, 23
58 changes: 58 additions & 0 deletions ballerina-tests/type-compatible-tests/tests/nill_type_test.bal
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
import ballerina/data.csv;
import ballerina/test;

type Book1 record {
()|string name;
()|string author;
()|string year;
};

string csvString1 = string `name,author,year
,b,c
a,,c
a,b,`;

@test:Config{}
function testEmptyStringWithNilConfig() {
Book1[]|error books1 = csv:parseString(csvString1, {nilValue: ""});
test:assertEquals(books1, [
{name: null, author: "b", year: "c"},
{name: "a", author: null, year: "c"},
{name: "a", author: "b", year: null}
]);

Book1[]|error books2 = csv:parseString(csvString1);
test:assertEquals(books2, [
{name: "", author: "b", year: "c"},
{name: "a", author: "", year: "c"},
{name: "a", author: "b", year: ""}
]);

(boolean|()|string|int)[][]|error arrbooks1 = csv:parseString(csvString1, {nilValue: ""});
test:assertEquals(arrbooks1, [
[null, "b", "c"],
["a", null, "c"],
["a", "b", null]
]);

(boolean|()|string|int)[][2]|error arrbooks2 = csv:parseString(csvString1, {nilValue: ""});
test:assertEquals(arrbooks2, [
[null, "b"],
["a", null],
["a", "b"]
]);

(boolean|()|string|int)[][]|error arrbooks3 = csv:parseString(csvString1);
test:assertEquals(arrbooks3, [
["", "b", "c"],
["a", "", "c"],
["a", "b", ""]
]);

(boolean|()|string|int)[][2]|error arrbooks4 = csv:parseString(csvString1);
test:assertEquals(arrbooks4, [
["", "b"],
["a", ""],
["a", "b"]
]);
}
33 changes: 18 additions & 15 deletions ballerina-tests/type-compatible-tests/tests/parse_string_compatibality_test.bal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import ballerina/io;
import ballerina/test;

const string filepath = "tests/csv_content.txt";
const string filepath2 = "tests/csv_content_2.txt";
const string errorFilepath = "tests/csv_error_content.txt";

@test:Config
Expand Down Expand Up @@ -215,6 +216,7 @@ function testSpaceBetweendData() {
@test:Config
function testParseBytes() returns error? {
byte[] csvBytes = check io:fileReadBytes(filepath);
byte[] csvBytes2 = check io:fileReadBytes(filepath2);

record{}[]|csv:Error rec = csv:parseBytes(csvBytes, {});
test:assertEquals(rec, [
Expand Down Expand Up @@ -245,17 +247,17 @@ function testParseBytes() returns error? {
["Hello World", "\"Hello World\"", "Hello World", "2"]
]);

rec2 = csv:parseBytes(csvBytes, {outputWithHeaders: true, header: 1});
rec2 = csv:parseBytes(csvBytes2, {outputWithHeaders: true, header: 1});
test:assertEquals(rec2, [
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"]
["Hello World1", "\"Hello World2\"", "Hello World3", "21"],
["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
]);

rec2 = csv:parseBytes(csvBytes, { header: 1});
rec2 = csv:parseBytes(csvBytes2, { header: 1});
test:assertEquals(rec2, [
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"]
["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
]);

int[][]|csv:Error rec3 = csv:parseBytes(csvBytes, {});
Expand All @@ -270,6 +272,8 @@ function testParseBytes() returns error? {
@test:Config
function testParseStream() returns error? {
stream<byte[], io:Error?> csvByteStream = check io:fileReadBlocksAsStream(filepath);
stream<byte[], io:Error?> csvByteStream2 = check io:fileReadBlocksAsStream(filepath2);

record{}[]|csv:Error rec = csv:parseStream(csvByteStream, {});
test:assertEquals(rec, [
{"a":"Hello World","b":"\"Hello World\"","c d":"Hello World","e":2},
Expand All @@ -293,12 +297,11 @@ function testParseStream() returns error? {
["Hello World", "\"Hello World\"", "Hello World", "2"]
]);

csvByteStream = check io:fileReadBlocksAsStream(filepath);
rec2 = csv:parseStream(csvByteStream, {header: 1, outputWithHeaders: true});
rec2 = csv:parseStream(csvByteStream2, {header: 1, outputWithHeaders: true});
test:assertEquals(rec2, [
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"]
["Hello World1", "\"Hello World2\"", "Hello World3", "21"],
["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
]);

csvByteStream = check io:fileReadBlocksAsStream(filepath);
Expand All @@ -310,11 +313,11 @@ function testParseStream() returns error? {
["Hello World", "\"Hello World\"", "Hello World", "2"]
]);

csvByteStream = check io:fileReadBlocksAsStream(filepath);
csvByteStream = check io:fileReadBlocksAsStream(filepath2);
rec2 = csv:parseStream(csvByteStream, {header: 1});
test:assertEquals(rec2, [
["Hello World", "\"Hello World\"", "Hello World", "2"],
["Hello World", "\"Hello World\"", "Hello World", "2"]
["Hello World1", "\"Hello World2\"", "Hello World3", "22"],
["Hello World1", "\"Hello World2\"", "Hello World3", "23"]
]);

csvByteStream = check io:fileReadBlocksAsStream(filepath);
Expand Down
25 changes: 25 additions & 0 deletions ballerina-tests/user-config-tests/tests/test_data_values.bal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,3 +121,28 @@ string csvStringData9 = string `
4@ string@ true@ 2.234@ ()@-3.21
5@ string@ true@ 2.234@ -3.21@ null`;

string csvStringData10 = string `
hello, hello, (), 12, true, 12.34
// comment
a, b, c, d, e, f
1, string1, true, 2.234, 2.234, ()
2, string2, false, 0, 0, null
3, string3, false, 1.23, 1.23, ()
4, string4, true, -6.51, -6.52, ()
5, string5, true, 3, 31, ()`;

string csvStringData11 = string `
a, b, c, d, e, f
1, string1, true, 2.234, 2.234, ()
2, string2, false, 0, 0, null
3, string3, false, 1.23, 1.23, ()
4, string4, true, -6.51, -6.52, ()
5, string5, true, 3, 3, ()`;
50 changes: 42 additions & 8 deletions ballerina-tests/user-config-tests/tests/user_config_with_parser_options_test.bal
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,9 +125,9 @@ function testFromCsvStringWithParserOptions() {
{a: 5, b: "string5", c: true, d: 3, e: 3, f: "()"}
]);

record {}[]|csv:Error csv3op3_4 = csv:parseString(csvStringData3, {header: 9, skipLines: "2-10"});
record {}[]|csv:Error csv3op3_4 = csv:parseString(csvStringData11, {header: 9, skipLines: "2-10"});
test:assertEquals(csv3op3_4, [
{'4: 5, string4: "string5", "true": true, "-6.51": 3, "()": null}
{'4: 5, string4: "string5", "true": true, "-6.51": 3, "-6.52": 3, "()": null}
]);
}

Expand Down Expand Up @@ -203,15 +203,19 @@ function testHeaderOption() {
{a: 5, b: "string5", c: true, d: 3, e: 3, f: ()}
]);

record {}[]|csv:Error csv2cop2 = csv:parseString(csvStringData2, {header: 100});
record {}[]|csv:Error csv2cop2 = csv:parseString(csvStringData10, {header: 100});
test:assertTrue(csv2cop2 is csv:Error);
test:assertEquals((<error>csv2cop2).message(), "The provided header row is empty");

record {}[]|csv:Error csv2cop3 = csv:parseString(csvStringData2, {header: 11});
record {}[]|csv:Error csv2cop3 = csv:parseString(csvStringData10, {header: 11});
test:assertEquals(csv2cop3, []);

record {}[]|csv:Error csv2cop4 = csv:parseString(csvStringData2, {header: 10});
test:assertEquals(csv2cop4, [{'4: 5, string4: "string5", "true": true, "-6.51": 3, "()": ()}]);
record {}[]|csv:Error csv2cop3_2 = csv:parseString(csvStringData10, {header: 9});
test:assertTrue(csv2cop3_2 is csv:Error);
test:assertEquals((<error>csv2cop3_2).message(), "Duplicate header found: '1.23'");

record {}[]|csv:Error csv2cop4 = csv:parseString(csvStringData10, {header: 10});
test:assertEquals(csv2cop4, [{'4: 5, string4: "string5", "true": true, "-6.51": 3, "-6.52": 31, "()": ()}]);

record {}[]|csv:Error csv1cop5 = csv:parseString(csvStringData1, {});
test:assertTrue(csv1cop5 is csv:Error);
Expand Down Expand Up @@ -313,6 +317,9 @@ function testCommentConfigOption() {
string csvValue9 = string `a,# b
1 ,#2 # comment
# comment`;
string csvValue10 = string `a# b
1 ,#2 # comment
# comment`;

record {int a;}[]|csv:Error cn;

Expand Down Expand Up @@ -345,8 +352,13 @@ function testCommentConfigOption() {
// TODO:Fix the error message
// test:assertEquals((<error> cn).message(), common:generateErrorMessageForInvalidCast("1, 2", "int"));

cn = csv:parseString(csvValue10);
test:assertTrue(cn is csv:Error);
test:assertEquals((<error> cn).message(), "Invalid length for the headers");

cn = csv:parseString(csvValue9);
test:assertEquals(cn, [{a: 1}]);
test:assertTrue(cn is csv:Error);
test:assertEquals((<error> cn).message(), "The provided header row is empty");
}

@test:Config {dependsOn: [testCSVLocale]}
Expand Down Expand Up @@ -375,6 +387,20 @@ function testCommentConfigOption2() {
& comment`;

string csvValue7 = string `
a& b
1 ,&2 & comment
& comment`;

string csvValue8 = string `
a,e& b
1 ,&2 & comment
& comment`;

record {int a; int b;}[]|csv:Error cn;
record {int c;}[]|csv:Error cn2;

Expand All @@ -401,8 +427,16 @@ function testCommentConfigOption2() {

cn = csv:parseString(csvValue6, {comment: "&", header: 2});
test:assertTrue(cn is csv:Error);
test:assertEquals((<error>cn).message(), "The provided header row is empty");

cn = csv:parseString(csvValue8, {comment: "&", header: 2});
test:assertTrue(cn is csv:Error);
test:assertEquals((<error>cn).message(), common:generateErrorMessageForMissingRequiredField("b"));

cn = csv:parseString(csvValue7, {comment: "&", header: 2});
test:assertTrue(cn is csv:Error);
test:assertEquals((<error>cn).message(), "Invalid length for the headers");

cn2 = csv:parseString(csvValue1, {comment: "&"});
test:assertTrue(cn2 is csv:Error);
test:assertEquals((<error>cn2).message(), common:generateErrorMessageForMissingRequiredField("c"));
Expand All @@ -426,7 +460,7 @@ function testCommentConfigOption2() {

cn2 = csv:parseString(csvValue6, {header: 2, comment: "&"});
test:assertTrue(cn2 is csv:Error);
test:assertEquals((<error>cn2).message(), common:generateErrorMessageForMissingRequiredField("c"));
test:assertEquals((<error>cn2).message(), "The provided header row is empty");
}

@test:Config {dependsOn: [testCSVLocale]}
Expand Down
9 changes: 6 additions & 3 deletions native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,9 @@ static Object initRowValue(Type expectedType) {
};
}

static void convertAndUpdateCurrentJsonNode(CsvParser.StateMachine sm,
String value, Type type, CsvConfig config, Type exptype,
Field currentField) {
static void convertAndUpdateCurrentCsvNode(CsvParser.StateMachine sm,
String value, Type type, CsvConfig config, Type exptype,
Field currentField) {
Object currentCsv = sm.currentCsvNode;
Object nilValue = config.nilValue;
if (sm.config.nilAsOptionalField && !type.isNilable()
Expand All @@ -91,6 +91,7 @@ static void convertAndUpdateCurrentJsonNode(CsvParser.StateMachine sm,
case TypeTags.RECORD_TYPE_TAG:
((BMap<BString, Object>) currentCsv).put(StringUtils.fromString(getHeaderValueForColumnIndex(sm)),
convertedValue);
sm.currentCsvNodeLength++;
return;
case TypeTags.ARRAY_TAG:
ArrayType arrayType = (ArrayType) currentCsvNodeType;
Expand All @@ -99,9 +100,11 @@ static void convertAndUpdateCurrentJsonNode(CsvParser.StateMachine sm,
return;

Check warning on line 100 in native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java

View check run for this annotation

Codecov / codecov/patch

native/src/main/java/io/ballerina/lib/data/csvdata/csv/CsvCreator.java#L100 

Added line #L100 was not covered by tests
}
((BArray) currentCsv).add(sm.columnIndex, convertedValue);
sm.currentCsvNodeLength++;
return;
case TypeTags.TUPLE_TAG:
((BArray) currentCsv).add(sm.columnIndex, convertedValue);
sm.currentCsvNodeLength++;
return;
default:
}
Expand Down
Loading

0 comments on commit 34bea43

Please sign in to comment.