diff --git a/ballerina/byte_reader.bal b/ballerina/byte_reader.bal new file mode 100644 index 0000000..55815e3 --- /dev/null +++ b/ballerina/byte_reader.bal @@ -0,0 +1,138 @@ +// Copyright (c) 2024 WSO2 LLC. (http://www.wso2.com) All Rights Reserved. +// +// WSO2 LLC. licenses this file to you under the Apache License, +// Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +class BytesReader { + *Visitor; + + private final GroupValue value = {}; + private final map redfinedValues = {}; + private final map redefinedItems; + private ByteIterator copybookIterator; + private final string? targetRecordName; + + isolated function init(byte[] copybookData, Schema schema, string? targetRecordName = ()) { + self.copybookIterator = new (copybookData); + self.redefinedItems = schema.getRedefinedItems(); + self.targetRecordName = targetRecordName; + } + + isolated function visitSchema(Schema schema, anydata data = ()) { + string? targetRecordName = self.targetRecordName; + if targetRecordName is string { + Node typeDef = getTypeDefinition(schema, targetRecordName); + typeDef.accept(self); + return; + } + foreach Node typeDef in schema.getTypeDefinitions() { + typeDef.accept(self); + } + } + + isolated function visitGroupItem(GroupItem groupItem, anydata data = ()) { + ByteIterator temp = self.copybookIterator; + self.copybookIterator = self.getIteratorForItem(groupItem); + + if isArray(groupItem) { + GroupValue[] elements = []; + foreach int i in 0 ..< groupItem.getElementCount() { + GroupValue groupValue = {}; + foreach var child in groupItem.getChildren() { + child.accept(self, groupValue); + } + elements.push(groupValue); + } + self.addValue(groupItem.getName(), elements, data); + } else { + GroupValue groupValue = {}; + foreach var child in groupItem.getChildren() { + child.accept(self, groupValue); + } + self.addValue(groupItem.getName(), groupValue, data); + } + + // Reset the iterator to previous text iterator + self.copybookIterator = temp; + } + + isolated function visitDataItem(DataItem dataItem, anydata data = ()) { + ByteIterator temp = self.copybookIterator; + self.copybookIterator = self.getIteratorForItem(dataItem); + if isArray(dataItem) { + string[] elements = []; + foreach int i in 0 ..< dataItem.getElementCount() { + elements.push(self.read(dataItem)); + } + self.addValue(dataItem.getName(), elements, data); + } else { + self.addValue(dataItem.getName(), self.read(dataItem), data); + } + + // Reset the iterator to previous text iterator + self.copybookIterator = temp; + } + + private isolated function getIteratorForItem(DataItem|GroupItem item) returns ByteIterator { + string? redefinedItemName = (); + if item is GroupItem { + redefinedItemName = item.getRedefinedItemName(); + } + if item is DataItem { + redefinedItemName = item.getRedefinedItemName(); + } + if redefinedItemName is string { + // Obtain the iterator from redfinedValues map if the provided item is a redefining item + ByteIterator iterator = new (self.redfinedValues.get(redefinedItemName).toBytes()); + return iterator; + } + return self.copybookIterator; + } + + private isolated function read(DataItem dataItem) returns string { + byte[] bytes = []; + int readLength = dataItem.isBinary() ? dataItem.getPackLength() : dataItem.getReadLength(); + foreach int i in 0 ..< readLength { + var data = self.copybookIterator.next(); + if data is () { + break; + } + bytes.push(data.value); + } + if bytes.length() == 0 { + return ""; + } + if dataItem.isBinary() { + int intValue = checkpanic decodeBinaryValue(bytes); + return intValue.toString(); + } + return checkpanic string:fromBytes(bytes); + } + + private isolated function addValue(string fieldName, FieldValue fieldValue, anydata parent) { + if parent is GroupValue { + parent[fieldName] = fieldValue; + } else if parent is () { + self.value[fieldName] = fieldValue; + } + + if self.redefinedItems.hasKey(fieldName) { + self.redfinedValues[fieldName] = stringify(fieldValue); + } + } + + isolated function getValue() returns GroupValue { + return sanitize(self.value); + } +} diff --git a/ballerina/convertor.bal b/ballerina/convertor.bal index 3826b47..984388e 100644 --- a/ballerina/convertor.bal +++ b/ballerina/convertor.bal @@ -63,13 +63,59 @@ public isolated class Converter { check self.validateTargetRecordName(targetRecordName); JsonToCopybookConverter converter = new (self.schema, targetRecordName); converter.visitSchema(self.schema, readonlyJson); - return converter.getValue(); + return converter.getStringValue(); } } on fail error err { return createError(err); } } + # Converts the provided record or map value to bytes. + # + input - The JSON value that needs to be converted as copybook data + # + targetRecordName - The name of the copybook record definition in the copybook. This parameter must be a string + # if the provided schema file contains more than one copybook record type definition + # + encoding - The encoding of the output bytes array. Default is ASCII + # + return - The converted byte array. In case of an error, a `copybook:Error` is is returned + public isolated function toBytes(record {} input, string? targetRecordName = (), Encoding encoding = ASCII) returns byte[]|Error { + do { + readonly & map readonlyJson = check input.cloneWithType(); + lock { + check self.validateTargetRecordName(targetRecordName); + JsonToCopybookConverter converter = new (self.schema, targetRecordName); + converter.visitSchema(self.schema, readonlyJson); + byte[] bytes = check converter.getByteValue(); + bytes = encoding is EBCDIC ? toEbcdicBytes(bytes) : bytes; + return bytes.clone(); + } + } on fail error err { + return createError(err); + } + } + + # Converts the given copybook bytes to a Ballerina record. + # + bytes - Bytes array that needs to be converted to a record value + # + targetRecordName - The name of the copybook record definition in the copybook. This parameter must be a string + # if the provided schema file contains more than one copybook record type definition + # + encoding - The encoding of the input bytes array. Default is ASCII + # + t - The type of the target record type + # + return - A record value on success, a `copybook:Error` in case of coercion errors + public isolated function fromBytes(byte[] bytes, string? targetRecordName = (), Encoding encoding = ASCII, + typedesc t = <>) returns t|Error = @java:Method { + 'class: "io.ballerina.lib.copybook.runtime.converter.Utils" + } external; + + private isolated function fromBytesToRecord(byte[] bytes, string? targetRecordName = (), Encoding encoding = ASCII) + returns record {}|Error { + lock { + check self.validateTargetRecordName(targetRecordName); + byte[] byteArray = encoding is EBCDIC ? toAsciiBytes(bytes.clone()) : bytes.clone(); + BytesReader copybookReader = new (byteArray, self.schema, targetRecordName); + self.schema.accept(copybookReader); + DataCoercer dataCoercer = new (self.schema, targetRecordName); + return dataCoercer.coerce(copybookReader.getValue()).clone(); + } + } + private isolated function validateTargetRecordName(string? targetRecordName) returns Error? { lock { if targetRecordName is () { diff --git a/ballerina/copybook_reader.bal b/ballerina/copybook_reader.bal index 94b3673..d01f06f 100644 --- a/ballerina/copybook_reader.bal +++ b/ballerina/copybook_reader.bal @@ -58,7 +58,7 @@ class CopybookReader { } else { GroupValue groupValue = {}; foreach var child in groupItem.getChildren() { - child.accept(self, groupValue); + child.accept(self, groupValue); } self.addValue(groupItem.getName(), groupValue, data); } diff --git a/ballerina/data_item.bal b/ballerina/data_item.bal index 43db18b..3ce82af 100644 --- a/ballerina/data_item.bal +++ b/ballerina/data_item.bal @@ -67,6 +67,14 @@ isolated distinct class DataItem { 'class: "io.ballerina.lib.copybook.runtime.converter.Utils" } external; + isolated function isBinary() returns boolean = @java:Method { + 'class: "io.ballerina.lib.copybook.runtime.converter.Utils" + } external; + + isolated function getPackLength() returns int = @java:Method { + 'class: "io.ballerina.lib.copybook.runtime.converter.Utils" + } external; + isolated function accept(Visitor visitor, anydata data = ()) { visitor.visitDataItem(self, data); } diff --git a/ballerina/default_value_creator.bal b/ballerina/default_value_creator.bal index afc8c0b..4d36dcf 100644 --- a/ballerina/default_value_creator.bal +++ b/ballerina/default_value_creator.bal @@ -40,7 +40,7 @@ class DefaultValueCreator { return; } string dataItemDefaultValue = dataItem.getDefaultValue() ?: ""; - if (dataItem.isNumeric() && !dataItem.isDecimal()) || (dataItem.isDecimal() && dataItemDefaultValue != "") { + if (dataItem.isNumeric() && !dataItem.isDecimal()) || (dataItem.isDecimal() && dataItemDefaultValue != "") { dataItemDefaultValue = dataItemDefaultValue.padZero(dataItem.getReadLength()); } else { dataItemDefaultValue = dataItemDefaultValue.padEnd(dataItem.getReadLength()); @@ -59,7 +59,7 @@ class DefaultValueCreator { return string:'join("", ...values); } - isolated function getDefaultValue() returns string { - return string:'join("", ...self.defaultValueFragments); + isolated function getDefaultValue() returns byte[] { + return string:'join("", ...self.defaultValueFragments).toBytes(); } } diff --git a/ballerina/json_to_copybook_convertor.bal b/ballerina/json_to_copybook_convertor.bal index 85df51d..5fe1bc8 100644 --- a/ballerina/json_to_copybook_convertor.bal +++ b/ballerina/json_to_copybook_convertor.bal @@ -17,7 +17,7 @@ class JsonToCopybookConverter { *Visitor; - private final string[] value = []; + private final byte[] value = []; private final Error[] errors = []; private final string[] path = []; private final map redefinedItems; @@ -52,7 +52,7 @@ class JsonToCopybookConverter { if data.hasKey(copybookRootRecord.getName()) { copybookRootRecord.accept(self, data.get(copybookRootRecord.getName())); } else { - self.value.push(self.getDefaultValue(copybookRootRecord)); + self.value.push(...self.getDefaultValue(copybookRootRecord)); } } @@ -76,7 +76,7 @@ class JsonToCopybookConverter { int elementSize = computeSize(groupItem, false); int remainingElements = (groupItem.getElementCount() - data.length()); int paddLength = remainingElements * elementSize; - self.value.push("".padEnd(paddLength)); + self.value.push(..."".padEnd(paddLength).toBytes()); } else { self.errors.push(error Error(string `Found an invalid value '${data is () ? "null" : data.toString()}'` + string `at ${self.getPath()}. A '${groupItem.getElementCount() < 0 ? "map" : "map[]"}'` @@ -95,7 +95,7 @@ class JsonToCopybookConverter { if !value.hasKey(child.getName()) { redefiningItemNameWithValue = self.findRedefiningItemNameWithValue(value, redefiningItems); if redefiningItemNameWithValue is () { - self.value.push(self.getDefaultValue(child)); + self.value.push(...self.getDefaultValue(child)); return; } self.visitAllowedRedefiningItems[redefiningItemNameWithValue] = (); @@ -113,7 +113,7 @@ class JsonToCopybookConverter { } } - private isolated function getDefaultValue(Node node) returns string { + private isolated function getDefaultValue(Node node) returns byte[] { DefaultValueCreator defaultValueCreator = new; node.accept(defaultValueCreator); return defaultValueCreator.getDefaultValue(); @@ -135,28 +135,28 @@ class JsonToCopybookConverter { } self.path.push(dataItem.getName()); do { - string? primitiveValue = (); + byte[] primitiveValue = []; if dataItem.getElementCount() < 0 && data is PrimitiveType { primitiveValue = check self.handlePrimitive(data, dataItem); } else if dataItem.getElementCount() > 0 && data is PrimitiveArrayType { primitiveValue = check self.handlePrimitiveArray(data, dataItem); } - if primitiveValue is () { + if primitiveValue.length() == 0 { check error Error(string `Found an invalid value '${data.toString()}' at ${self.getPath()}.` + string `A ${dataItem.getElementCount() < 0 ? "primitive" : "array"} value is expected`); return; } - self.value.push(primitiveValue); + self.value.push(...primitiveValue); } on fail Error e { self.errors.push(e); } _ = self.path.pop(); } - private isolated function handlePrimitive(PrimitiveType value, DataItem dataItem) returns string|Error { - string primitiveValue; + private isolated function handlePrimitive(PrimitiveType value, DataItem dataItem) returns byte[]|Error { + byte[] primitiveValue; if value is string { - primitiveValue = check self.handleStringValue(value, dataItem); + primitiveValue = (check self.handleStringValue(value, dataItem)); } else if value is int { primitiveValue = check self.handleIntValue(value, dataItem); } else { @@ -166,7 +166,7 @@ class JsonToCopybookConverter { return primitiveValue; } - private isolated function handleStringValue(string value, DataItem dataItem) returns string|Error { + private isolated function handleStringValue(string value, DataItem dataItem) returns byte[]|Error { int maxLength = dataItem.getReadLength(); if dataItem.isNumeric() { return error Error(string `Expected a numeric value at ${self.getPath()} but found string "${value}"`); @@ -174,10 +174,10 @@ class JsonToCopybookConverter { if value.length() > dataItem.getReadLength() { return error Error(string `Value '${value}' exceeds the max length ${maxLength} at ${self.getPath()}`); } - return value.padEnd(maxLength); + return value.padEnd(maxLength).toBytes(); } - private isolated function handleIntValue(int value, DataItem dataItem) returns string|Error { + private isolated function handleIntValue(int value, DataItem dataItem) returns byte[]|Error { if !dataItem.isNumeric() { return error Error(string `Numeric value ${value} found at ${self.getPath()}.` + " Expecting a non-numeric value"); @@ -192,27 +192,38 @@ class JsonToCopybookConverter { } if dataItem.getPicture().startsWith("-") { // Add " " in the beginning of the string if the data has is positive - return value < 0 ? value.toString().padZero(maxByteSize) - : value.toString().padZero(maxByteSize - 1).padStart(maxByteSize); + return value < 0 ? value.toString().padZero(maxByteSize).toBytes() + : value.toString().padZero(maxByteSize - 1).padStart(maxByteSize).toBytes(); } else if dataItem.getPicture().startsWith("+") { // Add "+" in the beginning of the string if the data has is positive - return value < 0 ? value.toString().padZero(maxByteSize) - : "+" + value.toString().padZero(maxByteSize - 1).padStart(maxByteSize - 1); + return value < 0 ? value.toString().padZero(maxByteSize).toBytes() + : ("+" + value.toString().padZero(maxByteSize - 1).padStart(maxByteSize - 1)).toBytes(); } // handle S9(9) + if dataItem.isBinary() { + return check self.handleBinaryValue(value, dataItem); + } if dataItem.isSigned() && value < 0 { - return value.toString().padZero(maxByteSize + 1); // +1 for sign byte + return value.toString().padZero(maxByteSize + 1).toBytes(); // +1 for sign byte + } + return value.toString().padZero(maxByteSize).toBytes(); + } + + private isolated function handleBinaryValue(int value, DataItem dataItem) returns byte[]|Error { + do { + return check getEncodedCopybookBinaryValue(value, dataItem.getPackLength()); + } on fail error err { + return error Error(string `Failed to encode the value ${value} to binary: ${err.message()}`); } - return value.toString().padZero(maxByteSize); } private isolated function handlePrimitiveArray(PrimitiveArrayType array, DataItem dataItem) - returns string|Error { - string[] elements = []; + returns byte[]|Error { + byte[][] elements = []; PrimitiveType[] primitiveArray = array; // This is allowed by covariance foreach int i in 0 ..< primitiveArray.length() { self.path.push(string `[${i}]`); - string|Error primitiveValue = self.handlePrimitive(primitiveArray[i], dataItem); + byte[]|Error primitiveValue = self.handlePrimitive(primitiveArray[i], dataItem); if primitiveValue is Error { self.errors.push(primitiveValue); continue; @@ -224,10 +235,16 @@ class JsonToCopybookConverter { if array.length() > maxElementCount { return error Error(string `Number of elements exceeds the max size ${maxElementCount} at ${self.getPath()}`); } - return "".'join(...elements).padEnd(computeSize(dataItem)); + byte[] result = []; + foreach byte[] item in elements { + result.push(...item); + } + string paddingString = "".padEnd(computeSize(dataItem) - result.length()); + result.push(...paddingString.toBytes()); + return result; } - private isolated function handleDecimalValue(decimal input, DataItem dataItem) returns string|Error { + private isolated function handleDecimalValue(decimal input, DataItem dataItem) returns byte[]|Error { // TODO: skipped decimal with V, implment seperately for decimal containing V // TODO: handle special case Z for fraction if !dataItem.isDecimal() && !dataItem.isNumeric() { @@ -255,15 +272,15 @@ class JsonToCopybookConverter { string decimalString = wholeNumber + "." + fraction.padEnd(dataItem.getFloatingPointLength(), "0"); if dataItem.getPicture().startsWith("-") && input >= 0d { // A deducted of 1 made from readLength for sign place holder " " - return " " + decimalString.padZero(dataItem.getReadLength() - 1 - supressZeroCount) - .padStart(dataItem.getReadLength() - 1); + return (" " + decimalString.padZero(dataItem.getReadLength() - 1 - supressZeroCount) + .padStart(dataItem.getReadLength() - 1)).toBytes(); } if dataItem.getPicture().startsWith("+") && input > 0d { // A deducted of 1 made from readLength for sign char "+" - return "+" + decimalString.padZero(dataItem.getReadLength() - 1 - supressZeroCount) - .padStart(dataItem.getReadLength() - 1); + return ("+" + decimalString.padZero(dataItem.getReadLength() - 1 - supressZeroCount) + .padStart(dataItem.getReadLength() - 1)).toBytes(); } - return decimalString.padZero(dataItem.getReadLength() - supressZeroCount).padStart(dataItem.getReadLength()); + return decimalString.padZero(dataItem.getReadLength() - supressZeroCount).padStart(dataItem.getReadLength()).toBytes(); } private isolated function checkDecimalLength(string wholeNumber, string fraction, decimal input, @@ -292,41 +309,52 @@ class JsonToCopybookConverter { return; } - private isolated function validateEnumValue(string value, DataItem dataItem) returns Error? { - string[]? possibleEnumValues = dataItem.getPossibleEnumValues(); - if possibleEnumValues is () { - return; - } - anydata providedValue = value; - anydata[] possibleValues = possibleEnumValues; + private isolated function validateEnumValue(byte[] value, DataItem dataItem) returns Error? { + string decodedValue = ""; do { + string[]? possibleEnumValues = dataItem.getPossibleEnumValues(); + if possibleEnumValues is () { + return; + } + anydata[] possibleValues = possibleEnumValues; + decodedValue = dataItem.isBinary() ? (check decodeBinaryValue(value)).toString() : check string:fromBytes(value); + anydata providedValue = decodedValue; if dataItem.isDecimal() { - providedValue = check decimal:fromString(value); + providedValue = check decimal:fromString(decodedValue); possibleValues = check toDecimalArray(possibleEnumValues); } else if dataItem.isNumeric() { - providedValue = check int:fromString(value); + providedValue = check int:fromString(decodedValue); possibleValues = check toIntArray(possibleEnumValues); } - } on fail error e { - return error Error(string `Failed to validate enum value '${value}': ${e.message()}`, e.cause()); - } - if possibleValues.indexOf(providedValue) is int { + if possibleValues.indexOf(providedValue) !is int { + string[] formattedEnumValues = possibleEnumValues.'map(posibleValue => string `'${posibleValue.toString()}'`); + return error Error(string `Value '${decodedValue}' is invalid for field '${dataItem.getName()}'. ` + + string `Allowed values are: ${string:'join(", ", ...formattedEnumValues)}.`); + } return; + } on fail error e { + return error Error(string `Failed to validate enum value '${decodedValue}': ${e.message()}`, e.cause()); } - string[] formattedEnumValues = possibleEnumValues.'map(posibleValue => string `'${posibleValue.toString()}'`); - return error Error(string `Value '${value}' is invalid for field '${dataItem.getName()}'. ` - + string `Allowed values are: ${string:'join(", ", ...formattedEnumValues)}.`); } private isolated function getPath() returns string { return string `'${".".'join(...self.path)}'`; } - isolated function getValue() returns string|Error { + isolated function getStringValue() returns string|Error { + string|error stringValue = string:fromBytes(self.value); + if self.errors.length() > 0 || stringValue is error { + string[] errorMsgs = self.errors.'map(err => err.message()); + return error Error("JSON to copybook data conversion failed.", errors = errorMsgs); + } + return stringValue; + } + + isolated function getByteValue() returns byte[]|Error { if self.errors.length() > 0 { string[] errorMsgs = self.errors.'map(err => err.message()); return error Error("JSON to copybook data conversion failed.", errors = errorMsgs); } - return "".'join(...self.value); + return self.value; } } diff --git a/ballerina/tests/resources/copybook-json/copybook-19.json b/ballerina/tests/resources/copybook-json/copybook-19.json new file mode 100644 index 0000000..db9724a --- /dev/null +++ b/ballerina/tests/resources/copybook-json/copybook-19.json @@ -0,0 +1,18 @@ +{ + "MQCIH": { + "MQCIH-CID": "CIH", + "MQCIH-VERSION": 180, + "MQCIH-LENGTH": 2, + "MQCIH-CODING": 0, + "MQCIH-CODE": 546, + "MQCIH-FORMATID": "RQMSTR", + "MQCIH-FLAGSID": 0, + "MQCIH-RETURNIDCODE": 0, + "MQCIH-COMPIDCODE": 1, + "MQCIH-REASONID": 0, + "MQCIH-CONTROL": -1, + "MQCIH-INTERVAL": 273, + "MQCIH-FACILITY": "00000001", + "MQCIH-AUTHENTICATOR": "CLT01" + } +} diff --git a/ballerina/tests/resources/copybook-json/copybook-20.json b/ballerina/tests/resources/copybook-json/copybook-20.json new file mode 100644 index 0000000..de88684 --- /dev/null +++ b/ballerina/tests/resources/copybook-json/copybook-20.json @@ -0,0 +1,30 @@ +{ + "MQCIH": { + "TRUCID": "C4H", + "ERSION": 150, + "TRUCLENGTH": 5, + "NCODING": 0, + "ODEDCHARSETID": 55, + "ORMAT": "STR", + "LAGS": 0, + "ETURNCODE": 0, + "OMPCODE": 1, + "EASON": 0, + "OWCONTROL": -1, + "ETWAITINTERVAL": 293, + "INKTYPE": 1, + "UTPUTDATALENGTH": -1, + "ACILITYKEEPTIME": 0, + "DSDESCRIPTOR": 0, + "ONVERSATIONALTASK": 0, + "ASKENDSTATUS": 0, + "ACILITY": "0000", + "UTHENTICATOR": "C1", + "EPLYTOFORMAT": "RQ", + "RANSACTIONID": "XT", + "URSORPOSITION": 0, + "RROROFFSET": 0, + "NPUTITEM": -8, + "ESERVED4": 0 + } +} diff --git a/ballerina/tests/resources/copybooks/copybook-19.cpy b/ballerina/tests/resources/copybooks/copybook-19.cpy new file mode 100644 index 0000000..d1f4a02 --- /dev/null +++ b/ballerina/tests/resources/copybooks/copybook-19.cpy @@ -0,0 +1,15 @@ +01 MQCIH. + 15 MQCIH-CID PIC X(4) VALUE 'CIH '. + 15 MQCIH-VERSION PIC S9(9) BINARY VALUE 2. + 15 MQCIH-LENGTH PIC S9(9) BINARY VALUE 180. + 15 MQCIH-CODING PIC S9(9) BINARY VALUE 0. + 15 MQCIH-CODE PIC S9(9) BINARY VALUE 0. + 15 MQCIH-FORMATID PIC X(8) VALUE SPACES. + 15 MQCIH-FLAGSID PIC S9(9) BINARY VALUE 0. + 15 MQCIH-RETURNIDCODE PIC S9(9) BINARY VALUE 0. + 15 MQCIH-COMPIDCODE PIC S9(9) BINARY VALUE 0. + 15 MQCIH-REASONID PIC S9(9) BINARY VALUE 0. + 15 MQCIH-CONTROL PIC S9(9) BINARY VALUE 273. + 15 MQCIH-INTERVAL PIC S9(9) BINARY VALUE -2. + 15 MQCIH-FACILITY PIC X(8) VALUE LOW-VALUES. + 15 MQCIH-AUTHENTICATOR PIC X(8) VALUE SPACES. diff --git a/ballerina/tests/resources/copybooks/copybook-20.cpy b/ballerina/tests/resources/copybooks/copybook-20.cpy new file mode 100644 index 0000000..cabae81 --- /dev/null +++ b/ballerina/tests/resources/copybooks/copybook-20.cpy @@ -0,0 +1,40 @@ + +01 MQCIH. +15 TRUCID PIC X(4) VALUE 'C4H '. +15 ERSION PIC S9(9) BINARY VALUE 4. +15 TRUCLENGTH PIC S9(9) BINARY VALUE 190. +15 NCODING PIC S9(9) BINARY VALUE 0. +15 ODEDCHARSETID PIC S9(9) BINARY VALUE 0. +15 ORMAT PIC X(8) VALUE SPACES. +15 LAGS PIC S9(9) BINARY VALUE 0. +15 ETURNCODE PIC S9(9) BINARY VALUE 0. +15 OMPCODE PIC S9(9) BINARY VALUE 0. +15 EASON PIC S9(9) BINARY VALUE 0. +15 OWCONTROL PIC S9(9) BINARY VALUE 208. +15 ETWAITINTERVAL PIC S9(9) BINARY VALUE -6. +15 INKTYPE PIC S9(9) BINARY VALUE 1. +15 UTPUTDATALENGTH PIC S9(9) BINARY VALUE -3. +15 ACILITYKEEPTIME PIC S9(9) BINARY VALUE 1. +15 DSDESCRIPTOR PIC S9(9) BINARY VALUE 0. +15 ONVERSATIONALTASK PIC S9(9) BINARY VALUE 2. +15 ASKENDSTATUS PIC S9(9) BINARY VALUE 0. +15 ACILITY PIC X(8) VALUE LOW-VALUES. +15 UNCTION PIC X(4) VALUE SPACES. +15 BENDCODE PIC X(4) VALUE SPACES. +15 UTHENTICATOR PIC X(8) VALUE SPACES. +15 ESERVED1 PIC X(8) VALUE SPACES. +15 EPLYTOFORMAT PIC X(8) VALUE SPACES. +15 EMOTESYSID PIC X(4) VALUE SPACES. +15 EMOTETRANSID PIC X(4) VALUE SPACES. +15 RANSACTIONID PIC X(4) VALUE SPACES. +15 ACILITYLIKE PIC X(4) VALUE SPACES. +15 TTENTIONID PIC X(4) VALUE SPACES. +15 TARTCODE PIC X(4) VALUE SPACES. +15 ANCELCODE PIC X(4) VALUE SPACES. +15 EXTTRANSACTIONID PIC X(4) VALUE SPACES. +15 ESERVED2 PIC X(8) VALUE SPACES. +15 ESERVED3 PIC X(8) VALUE SPACES. +15 URSORPOSITION PIC S9(9) BINARY VALUE 0. +15 RROROFFSET PIC S9(9) BINARY VALUE 0. +15 NPUTITEM PIC S9(9) BINARY VALUE 0. +15 ESERVED4 PIC S9(9) BINARY VALUE 0. diff --git a/ballerina/tests/test.bal b/ballerina/tests/test.bal index 27745df..297c07a 100644 --- a/ballerina/tests/test.bal +++ b/ballerina/tests/test.bal @@ -198,3 +198,60 @@ isolated function testCopybookWithMultipleRootRecords() returns error? { map jsonOutput = check converter.fromCopybook(ascii); test:assertEquals(jsonOutput, data); } + +@test:Config { + dataProvider: dataProviderBytes +} +isolated function testToByteAndFromBytes(string fileName, string targetRecord) returns error? { + Converter converter = check new (getCopybookPath(fileName)); + json jsonInput = check io:fileReadJson(getCopybookJsonPath(fileName)); + byte[] bytes = check converter.toBytes(check jsonInput.cloneWithType(), targetRecord); + map toJson = check converter.fromBytes(bytes, targetRecord); + test:assertEquals(check toJson.data, jsonInput); +} + +function dataProviderBytes() returns string[][] { + return [ + ["copybook-19", "MQCIH"], + ["copybook-20", "MQCIH"] + ]; +} + +@test:Config { + dataProvider: testByteConverterDataProvider +} +isolated function testByteConverter(string copybookName) returns error? { + Converter converter = check new (getCopybookPath(copybookName)); + string input = check io:fileReadString(getAsciiFilePath(copybookName)); + map jsonInput = check (check converter.toJson(input)).get(DATA).ensureType(); + byte[] bytes = check converter.toBytes(check jsonInput.cloneWithType()); + map toJson = check converter.fromBytes(bytes); + test:assertEquals(check toJson.data, jsonInput); +} + +isolated function testByteConverterDataProvider() returns map<[string]> { + map<[string]> filePaths = {}; + foreach int i in 1 ... 5 { + filePaths[i.toString()] = [string `copybook-${i}`]; + } + return filePaths; +} + +@test:Config { + enable: false, + dataProvider: dataProviderBytesEncoding +} +isolated function testToByteAndFromBytesWithEncoding(string fileName, string targetRecord) returns error? { + Converter converter = check new (getCopybookPath(fileName)); + json jsonInput = check io:fileReadJson(getCopybookJsonPath(fileName)); + byte[] bytes = check converter.toBytes(check jsonInput.cloneWithType(), targetRecord, EBCDIC); + map toJson = check converter.fromBytes(bytes, targetRecord, EBCDIC); + test:assertEquals(jsonInput, check toJson.data); +} + +function dataProviderBytesEncoding() returns string[][] { + return [ + ["copybook-19", "MQCIH"], + ["copybook-20", "MQCIH"] + ]; +} diff --git a/ballerina/types.bal b/ballerina/types.bal index a543eee..e0b709d 100644 --- a/ballerina/types.bal +++ b/ballerina/types.bal @@ -21,6 +21,24 @@ type Iterator object { public isolated function next() returns record {|string:Char value;|}?; }; +// Added as a workaround for https://github.com/ballerina-platform/ballerina-lang/issues/43301 +isolated class ByteIterator { + + private final object { + public isolated function next() returns record {|byte value;|}?; + } iterator; + + isolated function init(byte[] bytes) { + self.iterator = bytes.cloneReadOnly().iterator(); + } + + isolated function next() returns record {|byte value;|}? { + lock { + return self.iterator.next().cloneReadOnly(); + } + } +}; + type GroupValue record { }; @@ -34,3 +52,8 @@ type PrimitiveArrayType string[]|int[]|float[]|decimal[]; const ROOT_JSON_PATH = "$"; const ERRORS = "errors"; const DATA = "data"; + +public enum Encoding { + ASCII, + EBCDIC +} diff --git a/ballerina/utils.bal b/ballerina/utils.bal index a229c2f..443d70b 100644 --- a/ballerina/utils.bal +++ b/ballerina/utils.bal @@ -207,6 +207,107 @@ isolated function createError(error err) returns Error { isolated function toDecimalArray(string[] possibleEnumValues) returns decimal[]|error => possibleEnumValues.'map(possibleValue => check decimal:fromString(possibleValue)); - isolated function toIntArray(string[] possibleEnumValues) returns int[]|error => possibleEnumValues.'map(possibleValue => check int:fromString(possibleValue)); + +isolated function toByteArray(int[] possibleEnumValues, int length) returns byte[][]|error { + return possibleEnumValues.'map(intValue => check encodeCopybookBinaryValue(intValue, length)); +} + +isolated function fromByteToStringArray(string[] possibleEnumValues, int length) returns int[]|error { + return possibleEnumValues.'map(possibleValue => check int:fromString(possibleValue)); +} + +isolated function getEncodedCopybookBinaryValue(int value, int length) returns byte[]|error { + if value < 0 { + return encodeNegativeValue(value, length); + } + return encodeCopybookBinaryValue(value, length); +} + +isolated function encodeNegativeValue(int value, int length) returns byte[]|error { + string binaryString = decimalToBinary(value.abs()); + binaryString = check findTwosComplement(binaryString.padZero(32)); + int complementValue = binaryToDecimal(binaryString); + return encodeCopybookBinaryValue(complementValue, length); +} + +isolated function encodeCopybookBinaryValue(int value, int length) returns byte[]|error { + string hexString = int:toHexString(value).padZero(length * 2); + string[] doubles = splitAs2Chars(hexString); + int[] bytes = doubles.reverse().'map(r => check int:fromHexString(r)); + return bytes.cloneWithType(); +} + +isolated function decodeBinaryValue(int[] bytes) returns int|error { + string[] binaryValues = bytes.reverse().'map(r => decimalToBinary(r).padStart(8, "0")); + string value = ""; + foreach string name in binaryValues { + value += name; + } + if value.startsWith("1") { + value = check findTwosComplement(value); + return -binaryToDecimal(value); + } + return binaryToDecimal(value); +} + +isolated function splitAs2Chars(string hexString) returns string[] { + string[] doubles = []; + int i = 0; + while i < hexString.length() - 1 { + string double = hexString.substring(i, i + 2); + doubles.push(double); + i += 2; + } + return doubles; +} + +isolated function decimalToBinary(int decimalValue) returns string { + if decimalValue == 0 { + return "0"; + } + string binary = ""; + int val = decimalValue; + int i = 0; + while val > 0 { + binary = (val % 2).toString() + binary; + val = val / 2; + i += 1; + } + return binary; +} + +isolated function binaryToDecimal(string binary) returns int { + int decimalValue = 0; + int length = binary.length(); + foreach int i in 0 ..< length { + if binary[length - i - 1] == "1" { + decimalValue += 1 << i; + } + } + return decimalValue; +} + +isolated function findTwosComplement(string binary) returns string|error { + string onesComplement = ""; + foreach string:Char char in binary { + onesComplement += (char == "0" ? "1" : "0"); + } + int[] twosComplement = onesComplement.toCodePointInts(); + int codePointInt0 = string:toCodePointInt("0"); + int codePointInt1 = string:toCodePointInt("1"); + boolean carry = true; + foreach int i in int:range(onesComplement.length() - 1, 0, -1) { + if twosComplement[i] == codePointInt1 && carry { + twosComplement[i] = codePointInt0; + } else if carry { + twosComplement[i] = codePointInt1; + carry = false; + } + } + if carry { + twosComplement.unshift(codePointInt1); + } + return check string:fromCodePointInts(twosComplement); +} diff --git a/changelog.md b/changelog.md index 7bdec6e..b79856c 100644 --- a/changelog.md +++ b/changelog.md @@ -7,12 +7,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## [Unreleased] ### Added + - [[#6861] Add Support to Generate Zero Values for Picture Strings of Integer Type](https://github.com/ballerina-platform/ballerina-library/issues/6861) - [[#6862] Add Support for Enum Value Validation](https://github.com/ballerina-platform/ballerina-library/issues/6862) - [[#6863] Add Support for the Value Clause](https://github.com/ballerina-platform/ballerina-library/issues/6863) - [[#6892] ADd Helper Functions to Convert ASCII Bytes To EBCDIC and Vice Versa](https://github.com/ballerina-platform/ballerina-library/issues/6892) +- [[#6886] Add Support for 'BINARY' Value Encoding](https://github.com/ballerina-platform/ballerina-library/issues/6886) ### Changed + - [[#6895] Add Support to Handle Copybook with Multiple Root Records](https://github.com/ballerina-platform/ballerina-library/issues/6895) diff --git a/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/DataItem.java b/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/DataItem.java index 6de6130..b88d0f6 100644 --- a/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/DataItem.java +++ b/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/DataItem.java @@ -32,10 +32,13 @@ public class DataItem implements CopybookNode { private final int floatingPointLength; private final String redefinedItemName; private final String defaultValue; + private final boolean isBinary; + private final int packedLength; private final ArrayList possibleEnumValues = new ArrayList<>(); public DataItem(int level, String name, String picture, boolean isNumeric, int readLength, int occurs, - int floatingPointLength, String redefinedItemName, String defaultValue, GroupItem parent) { + int floatingPointLength, String redefinedItemName, String defaultValue, GroupItem parent, + boolean isBinary, int packedLength) { this.level = level; this.name = name; this.picture = picture; @@ -46,6 +49,8 @@ public DataItem(int level, String name, String picture, boolean isNumeric, int r this.floatingPointLength = floatingPointLength; this.redefinedItemName = redefinedItemName; this.defaultValue = defaultValue; + this.isBinary = isBinary; + this.packedLength = packedLength; if (parent != null) { parent.addChild(this); } @@ -100,6 +105,14 @@ public String getDefaultValue() { return defaultValue; } + public boolean isBinary() { + return isBinary; + } + + public int getPackLength() { + return packedLength; + } + void addEnumValues(String enumValue) { this.possibleEnumValues.add(enumValue); } diff --git a/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/SchemaBuilder.java b/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/SchemaBuilder.java index 88aa22f..9dc2132 100644 --- a/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/SchemaBuilder.java +++ b/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/SchemaBuilder.java @@ -173,9 +173,10 @@ public CopybookNode visitDataDescriptionEntryFormat1(DataDescriptionEntryFormat1 int readLength = Utils.getReadLength(pictureType); var valueClause = ctx.dataDescriptionEntryClauses().dataValueClause(0); String defaultValue = this.getDataValue(valueClause, readLength); + boolean isBinary = Utils.isBinary(ctx.dataDescriptionEntryClauses().dataUsageClause(0)); DataItem dataItem = new DataItem(level, name, Utils.getPictureString(pictureType), Utils.isNumeric(pictureType), readLength, occurs, Utils.getFloatingPointLength(pictureType), redefinedItemName, defaultValue, - getParent(level)); + getParent(level), isBinary, isBinary ? Utils.getBinaryPackLength(readLength) : 0); this.possibleEnum = dataItem; return dataItem; } diff --git a/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/Utils.java b/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/Utils.java index 3978f80..782ef2b 100644 --- a/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/Utils.java +++ b/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/Utils.java @@ -21,6 +21,7 @@ import java.util.regex.Pattern; import static io.ballerina.lib.copybook.commons.generated.CopybookParser.DataOccursClauseContext; +import static io.ballerina.lib.copybook.commons.generated.CopybookParser.DataUsageClauseContext; import static io.ballerina.lib.copybook.commons.generated.CopybookParser.PictureStringContext; public final class Utils { @@ -56,6 +57,19 @@ static boolean isArray(DataOccursClauseContext occursClause) { return occursClause != null; } + static boolean isBinary(DataUsageClauseContext usageClause) { + return usageClause != null && (usageClause.BINARY() != null || usageClause.COMP() != null); + } + + static int getBinaryPackLength(int readLength) { + if (readLength <= 4) { + return 2; + } else if (readLength <= 9) { + return 4; + } + return 18; + } + static int getFloatingPointLength(PictureStringContext pictureType) { return LengthCalculator.calculateFractionLength(getPictureString(pictureType)); } diff --git a/native/src/main/java/io/ballerina/lib/copybook/runtime/converter/Utils.java b/native/src/main/java/io/ballerina/lib/copybook/runtime/converter/Utils.java index fab6f62..95d665a 100644 --- a/native/src/main/java/io/ballerina/lib/copybook/runtime/converter/Utils.java +++ b/native/src/main/java/io/ballerina/lib/copybook/runtime/converter/Utils.java @@ -55,6 +55,7 @@ public final class Utils { private static final String GROUP_ITEM_TYPE_NAME = "GroupItem"; private static final String DATA_ITEM_TYPE_NAME = "DataItem"; private static final String ERROR_TYPE_NAME = "Error"; + private static final String FROM_BYTES_TO_RECORD_METHOD_NAME = "fromBytesToRecord"; private Utils() { } @@ -123,6 +124,16 @@ public static boolean isEnum(BObject bObject) { return dataItem.isEnum(); } + public static boolean isBinary(BObject bObject) { + DataItem dataItem = (DataItem) bObject.getNativeData(NATIVE_VALUE); + return dataItem.isBinary(); + } + + public static int getPackLength(BObject bObject) { + DataItem dataItem = (DataItem) bObject.getNativeData(NATIVE_VALUE); + return dataItem.getPackLength(); + } + public static BString getPicture(BObject bObject) { DataItem dataItem = (DataItem) bObject.getNativeData(NATIVE_VALUE); return StringUtils.fromString(dataItem.getPicture()); @@ -235,4 +246,14 @@ public static Object fromCopybook(Environment env, BObject converter, BString co PredefinedTypes.TYPE_NULL, paramFeed); return null; } + + public static Object fromBytes(Environment env, BObject converter, BArray bytes, Object targetRecordName, + BString encoding, BTypedesc typedesc) { + Future balFuture = env.markAsync(); + ExecutionCallback callback = new ExecutionCallback(balFuture); + Object[] paramFeed = {bytes, true, targetRecordName, true, encoding, true}; + env.getRuntime().invokeMethodAsyncConcurrently(converter, FROM_BYTES_TO_RECORD_METHOD_NAME, null, null, + callback, null, PredefinedTypes.TYPE_NULL, paramFeed); + return null; + } }