From 8ab9d8908bf656260ac3431911508426cd8a69c4 Mon Sep 17 00:00:00 2001 From: Nipuna Fernando Date: Fri, 8 Mar 2024 17:09:49 +0530 Subject: [PATCH 1/4] [Automated] Update the native jar versions --- ballerina/Ballerina.toml | 2 +- ballerina/Dependencies.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml index 9b20fc6..d2e874e 100644 --- a/ballerina/Ballerina.toml +++ b/ballerina/Ballerina.toml @@ -1,7 +1,7 @@ [package] org = "ballerina" name = "yaml" -version = "0.5.2" +version = "0.5.3" authors = ["Ballerina"] keywords = ["yaml"] repository = "https://github.com/ballerina-platform/module-ballerina-yaml" diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml index 8f2add7..0bcb4cc 100644 --- a/ballerina/Dependencies.toml +++ b/ballerina/Dependencies.toml @@ -147,7 +147,7 @@ dependencies = [ [[package]] org = "ballerina" name = "yaml" -version = "0.5.2" +version = "0.5.3" dependencies = [ {org = "ballerina", name = "file"}, {org = "ballerina", name = "io"}, From 1b2bbcdc5d2c7af01d8f566b2d9675b91ac9c142 Mon Sep 17 00:00:00 2001 From: Nipuna Fernando Date: Fri, 8 Mar 2024 23:03:27 +0530 Subject: [PATCH 2/4] Add integration tests to the readString api --- ballerina/modules/common/utils.bal | 16 +++++++++++ ballerina/modules/composer/state.bal | 4 +-- ballerina/modules/lexer/lexer.bal | 6 ---- ballerina/modules/lexer/state.bal | 5 ---- ballerina/modules/parser/parser.bal | 5 ++-- ballerina/modules/parser/state.bal | 41 ++++++---------------------- ballerina/tests/it.bal | 25 ++++++++++++++--- ballerina/utils.bal | 15 +++++++++- ballerina/yaml.bal | 12 +++----- 9 files changed, 69 insertions(+), 60 deletions(-) diff --git a/ballerina/modules/common/utils.bal b/ballerina/modules/common/utils.bal index 33eb79b..666e6c3 100644 --- a/ballerina/modules/common/utils.bal +++ b/ballerina/modules/common/utils.bal @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +import ballerina/io; + # Generate an error for conversion fault between Ballerina and YAML. # # + message - Cause of the error message @@ -40,3 +42,17 @@ public isolated function processTypeCastingError(json|error value) returns json| # + return - Formatted error message as a string public isolated function generateExpectedEndEventErrorMessage(string actualEvent, string expectedEvent) returns string => string `Expected '-${expectedEvent}' before '-${actualEvent}'`; + +# Convert the string to a string array. +# +# + inputStr - Input string to be converted +# + return - String array of the input string +public isolated function convertStringToLines(string inputStr) returns string[] { + do { + io:ReadableByteChannel readableChannel = check io:createReadableChannel(inputStr.toBytes()); + io:ReadableCharacterChannel readableCharChannel = new (readableChannel, io:DEFAULT_ENCODING); + return check readableCharChannel.readAllLines(); + } on fail { + return [inputStr]; + } +} diff --git a/ballerina/modules/composer/state.bal b/ballerina/modules/composer/state.bal index b015d10..ddfc251 100644 --- a/ballerina/modules/composer/state.bal +++ b/ballerina/modules/composer/state.bal @@ -36,10 +36,10 @@ public class ComposerState { # Flag is set if same map keys are allowed in a mapping readonly & boolean allowMapEntryRedefinition; - public isolated function init(string[]|string yamlInput, map tagSchema, + public isolated function init(string[] lines, map tagSchema, boolean allowAnchorRedefinition, boolean allowMapEntryRedefinition) returns parser:ParsingError? { - self.parserState = check new (yamlInput); + self.parserState = check new (lines); self.tagSchema = tagSchema; self.allowAnchorRedefinition = allowAnchorRedefinition; self.allowMapEntryRedefinition = allowMapEntryRedefinition; diff --git a/ballerina/modules/lexer/lexer.bal b/ballerina/modules/lexer/lexer.bal index cc27183..34f87b1 100644 --- a/ballerina/modules/lexer/lexer.bal +++ b/ballerina/modules/lexer/lexer.bal @@ -62,12 +62,6 @@ public isolated function scan(LexerState state) returns LexerState|LexicalError return state.index == 0 ? state.tokenize(EMPTY_LINE) : state.tokenize(EOL); } - // Check for line breaks when reading from string - if state.peek() == "\n" && state.context != LEXER_DOUBLE_QUOTE { - state.isNewLine = true; - return state.tokenize(EOL); - } - match state.context { LEXER_START => { return contextStart(state); diff --git a/ballerina/modules/lexer/state.bal b/ballerina/modules/lexer/state.bal index 1272266..b104021 100644 --- a/ballerina/modules/lexer/state.bal +++ b/ballerina/modules/lexer/state.bal @@ -70,8 +70,6 @@ public class LexerState { public boolean firstLine = true; - public boolean isNewLine = false; - int mappingKeyColumn = -1; # Output YAML token @@ -150,7 +148,6 @@ public class LexerState { self.indentStartIndex = -1; self.tokensForMappingValue = []; self.tabInWhitespace = -1; - self.isNewLine = false; self.keyDefinedForLine = false; } @@ -167,6 +164,4 @@ public class LexerState { } public isolated function isFlowCollection() returns boolean => self.numOpenedFlowCollections > 0; - - public isolated function isEndOfStream() returns boolean => self.index >= self.line.length(); } diff --git a/ballerina/modules/parser/parser.bal b/ballerina/modules/parser/parser.bal index 5c07cd2..18d2e9f 100644 --- a/ballerina/modules/parser/parser.bal +++ b/ballerina/modules/parser/parser.bal @@ -250,8 +250,9 @@ public isolated function parse(ParserState state, ParserOption option = DEFAULT, # + return - True if the string is a valid planar scalar. Else, false. public isolated function isValidPlanarScalar(string value) returns boolean { string? planarScalarResult = (); - do { - ParserState parserState = check new (value); + do { + string[] lines = common:convertStringToLines(value); + ParserState parserState = check new (lines); planarScalarResult = check planarScalar(parserState, false); } on fail { return false; diff --git a/ballerina/modules/parser/state.bal b/ballerina/modules/parser/state.bal index 04e653f..b71e4db 100644 --- a/ballerina/modules/parser/state.bal +++ b/ballerina/modules/parser/state.bal @@ -11,13 +11,12 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - import yaml.lexer; import yaml.common; public class ParserState { # Properties for the YAML lines - string[]|string yamlInput; + string[] lines; int numLines; int lineIndex = -1; @@ -62,12 +61,9 @@ public class ParserState { common:Event[] eventBuffer = []; - boolean isStringInput; - - public isolated function init(string[]|string yamlInput) returns ParsingError? { - self.yamlInput = yamlInput; - self.isStringInput = yamlInput is string; - self.numLines = self.isStringInput ? 1 : (yamlInput).length(); + public isolated function init(string[] lines) returns ParsingError? { + self.lines = lines; + self.numLines = lines.length(); ParsingError? err = self.initLexer(); if err is ParsingError { self.eventBuffer.push({endType: common:STREAM}); @@ -90,29 +86,11 @@ public class ParserState { self.lineIndex += 1; string line; - if self.isStringInput { - string currentLine = self.lexerState.line; - if self.lexerState.isNewLine { - line = currentLine.substring(self.lexerState.index); - } else { - if self.lineIndex == 0 { - line = self.yamlInput; - } else { - int? index = currentLine.indexOf("\n"); - if index is int { - line = currentLine.substring(index + 1); - } else { - return generateGrammarError(self, message); - } - } - } - } else { - if self.lineIndex >= self.numLines { - return generateGrammarError(self, message); - } - string[] lines = self.yamlInput; - line = lines[self.lineIndex]; + if self.lineIndex >= self.numLines { + return generateGrammarError(self, message); } + string[] lines = self.lines; + line = lines[self.lineIndex]; self.explicitDoc = false; self.expectBlockSequenceValue = false; @@ -120,6 +98,5 @@ public class ParserState { self.lexerState.setLine(line, self.lineIndex); } - isolated function isEndOfFile() returns boolean => - self.isStringInput ? self.lexerState.isEndOfStream() : self.lineIndex >= self.numLines - 1; + isolated function isEndOfFile() returns boolean => self.lineIndex >= self.numLines - 1; } diff --git a/ballerina/tests/it.bal b/ballerina/tests/it.bal index 77edb8c..7da4c34 100644 --- a/ballerina/tests/it.bal +++ b/ballerina/tests/it.bal @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import ballerina/io; import ballerina/test; YamlType[] customYamlTypes = []; @@ -35,7 +36,7 @@ map customTags = { }; @test:BeforeSuite -function initYamlCustomeTypes() { +function initYamlCustomTypes() { customTags.entries().forEach(function([string, FailSafeSchema] entry) { customYamlTypes.push({ tag: entry[0], @@ -50,11 +51,27 @@ function initYamlCustomeTypes() { @test:Config { dataProvider: yamlDataGen } -function testYAMLIntegrationTest(string filePath, json expectedOutput, boolean isStream, boolean isError) returns error? { +function testYAMLIntegrationTestForReadFile(string filePath, json expectedOutput, boolean isStream, boolean isError) returns error? { json|Error output = readFile(filePath, yamlTypes = customYamlTypes, isStream = isStream); + assertOutput(output, expectedOutput, isError); +} + +@test:Config { + dataProvider: yamlDataGen +} +function testYAMLIntegrationTestForReadString(string filePath, json expectedOutput, boolean isStream, boolean isError) returns error? { + io:ReadableByteChannel byteChannel = check io:openReadableFile(filePath); + (byte[] & readonly) readBytes = check byteChannel.readAll(); + string yamlContent = check string:fromBytes(readBytes); + + json|Error output = readString(yamlContent, yamlTypes = customYamlTypes, isStream = isStream); + assertOutput(output, expectedOutput, isError); +} + +function assertOutput(json|Error actualOut, json expectedOut, boolean isError) { if isError { - test:assertTrue(output is Error); + test:assertTrue(actualOut is Error); } else { - test:assertEquals(output, expectedOutput); + test:assertEquals(actualOut, expectedOut); } } diff --git a/ballerina/utils.bal b/ballerina/utils.bal index b9f5371..27c9de4 100644 --- a/ballerina/utils.bal +++ b/ballerina/utils.bal @@ -11,8 +11,9 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - import yaml.schema; +import yaml.composer; + import ballerina/file; # Checks if the file exists. If not, creates a new file. @@ -61,3 +62,15 @@ isolated function generateTagHandlesMap(YamlType[] yamlTypes, YAMLSchema yamlSch return tagHandles; } + +# Parses the given lines of YAML and returns the JSON representation. +# +# + lines - Lines of the YAML document +# + config - Configuration for the YAML parser +# + return - JSON representation of the YAML document +isolated function readLines(string[] lines, ReadConfig config) returns json|Error { + composer:ComposerState composerState = check new (lines, generateTagHandlesMap(config.yamlTypes, config.schema), + config.allowAnchorRedefinition, config.allowMapEntryRedefinition + ); + return config.isStream ? composer:composeStream(composerState) : composer:composeDocument(composerState); +} diff --git a/ballerina/yaml.bal b/ballerina/yaml.bal index 45f39c0..16e6a55 100644 --- a/ballerina/yaml.bal +++ b/ballerina/yaml.bal @@ -15,7 +15,7 @@ import ballerina/io; import yaml.emitter; import yaml.serializer; -import yaml.composer; +import yaml.common; # Parses a Ballerina string of YAML content into a Ballerina map object. # @@ -23,10 +23,8 @@ import yaml.composer; # + config - Configuration for reading a YAML file # + return - YAML map object on success. Else, returns an error public isolated function readString(string yamlString, *ReadConfig config) returns json|Error { - composer:ComposerState composerState = check new (yamlString, - generateTagHandlesMap(config.yamlTypes, config.schema), config.allowAnchorRedefinition, - config.allowMapEntryRedefinition); - return composer:composeDocument(composerState); + string[] lines = common:convertStringToLines(yamlString); + return readLines(lines, config); } # Parses a YAML file into a Ballerina json object. @@ -36,9 +34,7 @@ public isolated function readString(string yamlString, *ReadConfig config) retur # + return - YAML map object on success. Else, returns an error public isolated function readFile(string filePath, *ReadConfig config) returns json|Error { string[] lines = check io:fileReadLines(filePath); - composer:ComposerState composerState = check new (lines, generateTagHandlesMap(config.yamlTypes, config.schema), - config.allowAnchorRedefinition, config.allowMapEntryRedefinition); - return config.isStream ? composer:composeStream(composerState) : composer:composeDocument(composerState); + return readLines(lines, config); } # Converts the YAML structure to an array of strings. From fd6b3bb530cda4beb3fc3b1bcd691877285a9183 Mon Sep 17 00:00:00 2001 From: Nipuna Fernando Date: Sat, 9 Mar 2024 00:07:35 +0530 Subject: [PATCH 3/4] Remove convert string to lines from common --- ballerina/modules/common/utils.bal | 16 ---------------- ballerina/modules/parser/parser.bal | 3 +-- ballerina/yaml.bal | 5 +++-- 3 files changed, 4 insertions(+), 20 deletions(-) diff --git a/ballerina/modules/common/utils.bal b/ballerina/modules/common/utils.bal index 666e6c3..33eb79b 100644 --- a/ballerina/modules/common/utils.bal +++ b/ballerina/modules/common/utils.bal @@ -12,8 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -import ballerina/io; - # Generate an error for conversion fault between Ballerina and YAML. # # + message - Cause of the error message @@ -42,17 +40,3 @@ public isolated function processTypeCastingError(json|error value) returns json| # + return - Formatted error message as a string public isolated function generateExpectedEndEventErrorMessage(string actualEvent, string expectedEvent) returns string => string `Expected '-${expectedEvent}' before '-${actualEvent}'`; - -# Convert the string to a string array. -# -# + inputStr - Input string to be converted -# + return - String array of the input string -public isolated function convertStringToLines(string inputStr) returns string[] { - do { - io:ReadableByteChannel readableChannel = check io:createReadableChannel(inputStr.toBytes()); - io:ReadableCharacterChannel readableCharChannel = new (readableChannel, io:DEFAULT_ENCODING); - return check readableCharChannel.readAllLines(); - } on fail { - return [inputStr]; - } -} diff --git a/ballerina/modules/parser/parser.bal b/ballerina/modules/parser/parser.bal index 18d2e9f..80cd6c4 100644 --- a/ballerina/modules/parser/parser.bal +++ b/ballerina/modules/parser/parser.bal @@ -251,8 +251,7 @@ public isolated function parse(ParserState state, ParserOption option = DEFAULT, public isolated function isValidPlanarScalar(string value) returns boolean { string? planarScalarResult = (); do { - string[] lines = common:convertStringToLines(value); - ParserState parserState = check new (lines); + ParserState parserState = check new ([value]); planarScalarResult = check planarScalar(parserState, false); } on fail { return false; diff --git a/ballerina/yaml.bal b/ballerina/yaml.bal index 16e6a55..efbe2f4 100644 --- a/ballerina/yaml.bal +++ b/ballerina/yaml.bal @@ -15,7 +15,6 @@ import ballerina/io; import yaml.emitter; import yaml.serializer; -import yaml.common; # Parses a Ballerina string of YAML content into a Ballerina map object. # @@ -23,7 +22,9 @@ import yaml.common; # + config - Configuration for reading a YAML file # + return - YAML map object on success. Else, returns an error public isolated function readString(string yamlString, *ReadConfig config) returns json|Error { - string[] lines = common:convertStringToLines(yamlString); + io:ReadableByteChannel readableChannel = check io:createReadableChannel(yamlString.toBytes()); + io:ReadableCharacterChannel readableCharChannel = new (readableChannel, io:DEFAULT_ENCODING); + string[] lines = check readableCharChannel.readAllLines(); return readLines(lines, config); } From e10169ddb60f02135bfe1cf6867b4a9855be1dbd Mon Sep 17 00:00:00 2001 From: Nipuna Fernando Date: Sat, 9 Mar 2024 00:10:32 +0530 Subject: [PATCH 4/4] Support the new line character in windows --- ballerina/modules/lexer/scan.bal | 2 +- ballerina/modules/lexer/state.bal | 6 ++++++ ballerina/modules/parser/state.bal | 3 +-- ballerina/tests/api_test.bal | 6 ++---- ballerina/yaml.bal | 6 +++--- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/ballerina/modules/lexer/scan.bal b/ballerina/modules/lexer/scan.bal index 3424148..fddad37 100644 --- a/ballerina/modules/lexer/scan.bal +++ b/ballerina/modules/lexer/scan.bal @@ -158,7 +158,7 @@ isolated function scanPlanarChar(LexerState state) returns boolean|LexicalError } // Step back from the white spaces if EOL or ':' is reached - if state.peek() == () || state.peek() == "\n" { + if state.peek() == () || state.isNewLine() { state.forward(-numWhitespace); return true; } diff --git a/ballerina/modules/lexer/state.bal b/ballerina/modules/lexer/state.bal index b104021..d3620e0 100644 --- a/ballerina/modules/lexer/state.bal +++ b/ballerina/modules/lexer/state.bal @@ -164,4 +164,10 @@ public class LexerState { } public isolated function isFlowCollection() returns boolean => self.numOpenedFlowCollections > 0; + + # Check if the current character is a new line. + # This should be replaced by the os module once it supports an API: #4931. + # + # + return - True if the current character is a new line + public isolated function isNewLine() returns boolean => self.peek() == "\n" || self.peek() == "\r\n"; } diff --git a/ballerina/modules/parser/state.bal b/ballerina/modules/parser/state.bal index b71e4db..ec213eb 100644 --- a/ballerina/modules/parser/state.bal +++ b/ballerina/modules/parser/state.bal @@ -89,8 +89,7 @@ public class ParserState { if self.lineIndex >= self.numLines { return generateGrammarError(self, message); } - string[] lines = self.lines; - line = lines[self.lineIndex]; + line = self.lines[self.lineIndex]; self.explicitDoc = false; self.expectBlockSequenceValue = false; diff --git a/ballerina/tests/api_test.bal b/ballerina/tests/api_test.bal index 00b4af5..0a43bb1 100644 --- a/ballerina/tests/api_test.bal +++ b/ballerina/tests/api_test.bal @@ -17,8 +17,7 @@ import ballerina/io; import ballerina/test; @test:Config { - groups: ["api"], - enable: false + groups: ["api"] } function testReadYamlString() returns error? { string input = string ` @@ -76,8 +75,7 @@ function testWriteYamlFile() returns error? { @test:Config { dataProvider: yamlSchemaDataGen, - groups: ["api"], - enable: false + groups: ["api"] } function testReadYAMLSchema(YAMLSchema schema, json expectedOutput) returns error? { string input = string ` diff --git a/ballerina/yaml.bal b/ballerina/yaml.bal index efbe2f4..3e75f90 100644 --- a/ballerina/yaml.bal +++ b/ballerina/yaml.bal @@ -22,9 +22,9 @@ import yaml.serializer; # + config - Configuration for reading a YAML file # + return - YAML map object on success. Else, returns an error public isolated function readString(string yamlString, *ReadConfig config) returns json|Error { - io:ReadableByteChannel readableChannel = check io:createReadableChannel(yamlString.toBytes()); - io:ReadableCharacterChannel readableCharChannel = new (readableChannel, io:DEFAULT_ENCODING); - string[] lines = check readableCharChannel.readAllLines(); + io:ReadableByteChannel byteChannel = check io:createReadableChannel(yamlString.toBytes()); + io:ReadableCharacterChannel charChannel = new (byteChannel, io:DEFAULT_ENCODING); + string[] lines = check charChannel.readAllLines(); return readLines(lines, config); }