Merge pull request #51 from MohamedSabthar/s9v99

Add Support for Implied Decimal
ballerina-platform · Oct 18, 2024 · ca14672 · ca14672
2 parents 19434b1 + 1dd1c5a
commit ca14672
Show file tree

Hide file tree

Showing 15 changed files with 133 additions and 17 deletions.
diff --git a/ballerina/Ballerina.toml b/ballerina/Ballerina.toml
@@ -1,7 +1,7 @@
 [package]
 org = "ballerinax"
 name = "copybook"
-version = "1.0.1"
+version = "1.0.2"
 authors = ["Ballerina"]
 keywords = ["copybook", "serdes", "cobol", "mainframe"]
 repository = "https://github.com/ballerina-platform/module-ballerinax-copybook"
@@ -15,14 +15,14 @@ graalvmCompatible = true
 [[platform.java17.dependency]]
 groupId = "io.ballerina.lib"
 artifactId = "copybook-native"
-version = "1.0.1"
-path="../native/build/libs/copybook-native-1.0.1.jar"
+version = "1.0.2-SNAPSHOT"
+path="../native/build/libs/copybook-native-1.0.2-SNAPSHOT.jar"
 
 [[platform.java17.dependency]]
 groupId = "io.ballerina.lib"
 artifactId = "copybook-commons"
-version = "1.0.1"
-path = "../commons/build/libs/copybook-commons-1.0.1.jar"
+version = "1.0.2"
+path = "../commons/build/libs/copybook-commons-1.0.2-SNAPSHOT.jar"
 
 [[platform.java17.dependency]]
 groupId = "org.antlr"

diff --git a/ballerina/Dependencies.toml b/ballerina/Dependencies.toml
@@ -102,7 +102,7 @@ dependencies = [
 [[package]]
 org = "ballerinax"
 name = "copybook"
-version = "1.0.1"
+version = "1.0.2"
 dependencies = [
 	{org = "ballerina", name = "file"},
 	{org = "ballerina", name = "io"},

diff --git a/ballerina/copybook_reader.bal b/ballerina/copybook_reader.bal
@@ -119,7 +119,7 @@ class CopybookReader {
                 int intValue = check decodeBinaryValue(bytes, self.encoding);
                 return intValue.toString();
             }
-            if self.encoding == EBCDIC {
+            if self.encoding == EBCDIC && dataItem.getRedefinedItemName() is () {
                 bytes = toAsciiBytes(bytes);
             }
             string token = check string:fromBytes(bytes);

diff --git a/ballerina/data_coercer.bal b/ballerina/data_coercer.bal
@@ -111,6 +111,12 @@ class DataCoercer {
 
     private isolated function coerceDecimal(string data, DataItem dataItem) returns decimal? {
         string decimalString = data.trim();
+        if dataItem.hasImpliedSeperator() {
+            int floatingPointLength = dataItem.getFloatingPointLength();
+            int decimalIndex = decimalString.length() - floatingPointLength;
+            // Handle the implied decimal by adding the decimal separator manually to the string value.
+            decimalString = data.substring(0, decimalIndex) + "." + data.substring(decimalIndex);
+        }
         error|decimal coercedValue = trap decimal:fromString(decimalString);
         if coercedValue is error {
             self.errors.push(error Error(string `Failed to convert the value '${data}' to a 'decimal' `
@@ -152,7 +158,17 @@ class DataCoercer {
     private isolated function validateMaxByte(string value, DataItem dataItem) returns Error? {
         if dataItem.isDecimal() {
             int? seperatorIndex = value.indexOf(".");
-            int wholeNumberMaxLength = dataItem.getReadLength() - dataItem.getFloatingPointLength() - 1;
+            int wholeNumberMaxLength = dataItem.getReadLength() - dataItem.getFloatingPointLength();
+            if !dataItem.hasImpliedSeperator() {
+                // Made a reduction of 1 for the decimal separator since the floating-point length includes 
+                // the decimal separator for data item that don't have an implied decimal.
+                wholeNumberMaxLength -= 1;
+            }
+            boolean valueHasSign = value.startsWith("+") || value.startsWith("-");
+            // Handle sing remembered decimal S9(9)V9
+            if valueHasSign && dataItem.isSigned() {
+                wholeNumberMaxLength += 1;
+            }
             if (seperatorIndex is int && seperatorIndex > wholeNumberMaxLength)
                 || (seperatorIndex is () && value.length() > wholeNumberMaxLength) {
                 return error Error(string `The integral part of the decimal value` +
@@ -170,6 +186,11 @@ class DataCoercer {
             boolean valueHasSign = value.startsWith("+") || value.startsWith("-");
             maxReadBytes = dataItem.getReadLength() + (dataItem.isSigned() && valueHasSign ? 1 : 0);
         }
+        if dataItem.hasImpliedSeperator() {
+            // An addition of 1 is made to account for the manually included decimal separator
+            // in the value for an implied decimal.
+            maxReadBytes += 1;
+        }
         if value.length() > maxReadBytes {
             return error Error(string `The value '${value}' exceeds the maximum byte size of ${maxReadBytes} ` +
                 string `at ${self.getPath()}.`);

diff --git a/ballerina/data_item.bal b/ballerina/data_item.bal
@@ -82,4 +82,8 @@ isolated distinct class DataItem {
     isolated function toString() returns string {
         return externToString(self);
     }
+
+    isolated function hasImpliedSeperator() returns boolean {
+        return self.getPicture().includes("V");
+    }
 }
diff --git a/ballerina/json_to_copybook_convertor.bal b/ballerina/json_to_copybook_convertor.bal
@@ -116,7 +116,7 @@ class JsonToCopybookConverter {
     }
 
     private isolated function getDefaultValue(Node node) returns byte[] {
-        DefaultValueCreator defaultValueCreator = new(self.encoding);
+        DefaultValueCreator defaultValueCreator = new (self.encoding);
         node.accept(defaultValueCreator);
         Error[]? errors = defaultValueCreator.getErrors();
         if errors is Error[] {
@@ -288,16 +288,20 @@ class JsonToCopybookConverter {
             return ("+" + decimalString.padZero(dataItem.getReadLength() - 1 - supressZeroCount)
                 .padStart(dataItem.getReadLength() - 1)).toBytes();
         }
-        return decimalString.padZero(dataItem.getReadLength() - supressZeroCount).padStart(dataItem.getReadLength()).toBytes();
+        decimalString = decimalString.padZero(dataItem.getReadLength() - supressZeroCount).padStart(dataItem.getReadLength());
+        return dataItem.hasImpliedSeperator() ? re `\.`.replace(decimalString, "").toBytes() : decimalString.toBytes();
     }
 
     private isolated function checkDecimalLength(string wholeNumber, string fraction, decimal input,
             DataItem dataItem) returns Error? {
-        // A deducted of 1 made from readLength for decimal seperator "."
-        int expectedWholeNumberLength = dataItem.getReadLength() - dataItem.getFloatingPointLength() - 1;
+        int expectedWholeNumberLength = dataItem.getReadLength() - dataItem.getFloatingPointLength();
+        if !dataItem.hasImpliedSeperator() {
+            // A deducted of 1 made from readLength for decimal seperator "."
+            expectedWholeNumberLength -= 1;
+        }
         // If PIC has + or -, then remove the space allocated for the sign
         string picture = dataItem.getPicture();
-        expectedWholeNumberLength -= (picture.startsWith("+") || picture.startsWith("-")) is true ? 1 : 0;
+        expectedWholeNumberLength -= picture.startsWith("+") || picture.startsWith("-") ? 1 : 0;
         string integerPart = wholeNumber.startsWith("-") ? wholeNumber.substring(1) : wholeNumber;
         if integerPart.length() > expectedWholeNumberLength {
             return error Error(string `Value '${input}' exceeds the maximum number of integer digits `

diff --git a/ballerina/tests/resources/copybook-ascii/copybook-21.txt b/ballerina/tests/resources/copybook-ascii/copybook-21.txt
@@ -0,0 +1 @@
+TEST-1234567890123451234-12341234-2341221
diff --git a/ballerina/tests/resources/copybook-ebcdic/copybook-21.txt b/ballerina/tests/resources/copybook-ebcdic/copybook-21.txt
@@ -0,0 +1 @@
+����`�������������������`��������`�������
diff --git a/ballerina/tests/resources/copybook-json/copybook-21.json b/ballerina/tests/resources/copybook-json/copybook-21.json
@@ -0,0 +1,9 @@
+{
+  "STJ002-COMMAREA": {
+    "STJ002-STRING": "TEST",
+    "STJ002-SALDIS": -1234567890123.45,
+    "STJ002-SALDIS-RED": "-123456789012345",
+    "STJ002-ARRAY": [12.34, -12.34, 12.34, -23.41],
+    "STJ002-DATA": 2.21
+  }
+}
diff --git a/ballerina/tests/resources/copybooks/copybook-21.cpy b/ballerina/tests/resources/copybooks/copybook-21.cpy
@@ -0,0 +1,6 @@
+01  STJ002-COMMAREA.
+    02 STJ002-STRING           PIC X(4).
+    02  STJ002-SALDIS          PIC S9(13)V99.
+    02  STJ002-SALDIS-RED      REDEFINES STJ002-SALDIS PIC X(16).
+    02  STJ002-ARRAY           PIC S9(2)V99 OCCURS 4 TIMES.
+    02  STJ002-DATA            PIC 9(2)V99.
diff --git a/ballerina/tests/test.bal b/ballerina/tests/test.bal
@@ -246,3 +246,37 @@ isolated function testEbcidiValueHavingOptionalSignedInteger() returns error? {
         test:assertEquals(jsonFromEbcdic, jsonData);
     }
 }
+
+@test:Config
+isolated function testImpliedDecimal() returns error? {
+    Converter converter = check new (getCopybookPath("copybook-21"));
+    byte[] ascii = check io:fileReadBytes(getAsciiFilePath("copybook-21"));
+
+    json actual = check converter.fromBytes(ascii);
+    json jsonVal = check io:fileReadJson(getCopybookJsonPath("copybook-21"));
+    test:assertEquals(actual.toJsonString(), {data: jsonVal}.toJsonString());
+
+    byte[] bytes = check converter.toBytes(check jsonVal.ensureType());
+    test:assertEquals(string:fromBytes(bytes), check string:fromBytes(ascii));
+}
+
+@test:Config
+isolated function testImpliedDecimalForDefaultValue() returns error? {
+    Converter converter = check new (getCopybookPath("copybook-21"));
+    byte[] bytes = check converter.toBytes({});
+    test:assertEquals(string:fromBytes(bytes), "                                       ");
+}
+
+@test:Config
+isolated function testImpliedDecimalWithEbcidic() returns error? {
+    Converter converter = check new (getCopybookPath("copybook-21"));
+    json jsonVal = check io:fileReadJson(getCopybookJsonPath("copybook-21"));
+
+    byte[] bytes = check converter.toBytes(check jsonVal.ensureType(), encoding = EBCDIC);
+    check io:fileWriteBytes(getEbcdicFilePath("copybook-21"), bytes);
+    byte[] ebcdic = check io:fileReadBytes(getEbcdicFilePath("copybook-21"));
+    test:assertEquals(bytes, ebcdic);
+
+    json actual = check converter.fromBytes(ebcdic, encoding = EBCDIC);
+    test:assertEquals(actual.toJsonString(), {data: jsonVal}.toJsonString());
+}
diff --git a/changelog.md b/changelog.md
@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ## [Unreleased]
 
+### Added
+- [[#7275] Add Support for Implied Decimal](https://github.com/ballerina-platform/ballerina-library/issues/7275)
+
 ## [1.0.1] - 2024-09-11
 
 ### Fixed

diff --git a/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/LengthCalculator.java b/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/LengthCalculator.java
@@ -26,8 +26,10 @@
 import static io.ballerina.lib.copybook.commons.schema.PictureStringValidator.isDecimal;
 import static io.ballerina.lib.copybook.commons.schema.PictureStringValidator.isDecimalWithCardinality;
 import static io.ballerina.lib.copybook.commons.schema.PictureStringValidator.isDecimalWithSuppressedZeros;
+import static io.ballerina.lib.copybook.commons.schema.PictureStringValidator.isImpliedDecimalWithCardinality;
 import static io.ballerina.lib.copybook.commons.schema.PictureStringValidator.isInt;
 import static io.ballerina.lib.copybook.commons.schema.PictureStringValidator.isIntWithCardinality;
+import static io.ballerina.lib.copybook.commons.schema.PictureStringValidator.isSignRememberedImpliedDecimalWithCardinality;
 import static io.ballerina.lib.copybook.commons.schema.PictureStringValidator.isSignRememberedIntWithCardinality;
 
 class LengthCalculator {
@@ -36,11 +38,12 @@ private LengthCalculator() {
     }
 
     static int calculateFractionLength(String pictureString) {
-        if (!isDecimal(pictureString) && !isDecimalWithCardinality(pictureString) && !isDecimalWithSuppressedZeros(
-                pictureString)) {
+        if (!isDecimal(pictureString) && !isDecimalWithCardinality(pictureString)
+                && !isDecimalWithSuppressedZeros(pictureString) && !isImpliedDecimalWithCardinality(pictureString)
+                && !isSignRememberedImpliedDecimalWithCardinality(pictureString)) {
             return 0;
         }
-        Matcher matcher = Pattern.compile("^.*\\.(?<fraction>9+)$").matcher(pictureString);
+        Matcher matcher = Pattern.compile("^.*([.V])(?<fraction>9+)$").matcher(pictureString);
         if (matcher.find()) {
             return matcher.group("fraction").length();
         }
@@ -64,6 +67,11 @@ static int calculateReadLength(String pictureString) {
             return getReadLengthSignRememberedIntWithCardinality(pictureString);
         }
 
+        if (isImpliedDecimalWithCardinality(pictureString)
+                || isSignRememberedImpliedDecimalWithCardinality(pictureString)) {
+            return getImpliedDecimalWithCardinality(pictureString);
+        }
+
         if (isDecimalWithCardinality(pictureString)) {
             return getReadLengthDecimalWithCardinality(pictureString);
         }
@@ -99,6 +107,18 @@ private static int getReadLengthSignRememberedIntWithCardinality(String pictureS
         return 0;
     }
 
+    private static int getImpliedDecimalWithCardinality(String pictureString) {
+        Matcher matcher = Pattern.compile("^S?9\\((?<cardinality>\\d+)\\)V(?<fraction>9+)$")
+                .matcher(pictureString);
+        if (matcher.find()) {
+            int integerCardinality = Integer.parseInt(matcher.group("cardinality"));
+            // This fraction doesn't includes decimal separator "."
+            int fractionLength = matcher.group("fraction").length();
+            return integerCardinality + fractionLength;
+        }
+        return 0;
+    }
+
     private static int getReadLengthDecimalWithCardinality(String pictureString) {
         Matcher matcher = Pattern.compile("^(?<sign>[+-])?9\\((?<cardinality>\\d+)\\)(?<fraction>\\.9+)$")
                 .matcher(pictureString);

diff --git a/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/PictureStringValidator.java b/commons/src/main/java/io/ballerina/lib/copybook/commons/schema/PictureStringValidator.java
@@ -55,6 +55,16 @@ static boolean isSignRememberedIntWithCardinality(String pictureString) {
         return Pattern.compile("^S9\\(\\d+\\)$").matcher(pictureString).find();
     }
 
+    static boolean isImpliedDecimalWithCardinality(String pictureString) {
+        // ex: PIC 9(2)V99
+        return Pattern.compile("^9\\(\\d+\\)V9+$").matcher(pictureString).find();
+    }
+
+    static boolean isSignRememberedImpliedDecimalWithCardinality(String pictureString) {
+        // ex: PIC S9(2)V99
+        return Pattern.compile("^S9\\(\\d+\\)V9+$").matcher(pictureString).find();
+    }
+
     static boolean isDecimalWithCardinality(String pictureString) {
         // ex: PIC 9(9).333 or -9(9).333 or +9(9).333
         return Pattern.compile("^[+-]?9\\(\\d+\\)\\.9+$").matcher(pictureString).find();
@@ -69,6 +79,7 @@ static boolean isSupportedPictureString(String pictureString) {
         return isAlphaNumeric(pictureString) || isInt(pictureString) || isDecimal(pictureString)
                 || isAlphaNumericWithCardinality(pictureString) || isIntWithCardinality(pictureString)
                 || isSignRememberedIntWithCardinality(pictureString) || isDecimalWithCardinality(pictureString)
-                || isDecimalWithSuppressedZeros(pictureString);
+                || isDecimalWithSuppressedZeros(pictureString) || isImpliedDecimalWithCardinality(pictureString)
+                || isSignRememberedImpliedDecimalWithCardinality(pictureString);
     }
 }
diff --git a/docs/spec/spec.md b/docs/spec/spec.md
@@ -161,3 +161,5 @@ The `copybook:Error` type represents all the errors related to the Copybook modu
 | PIC  9(_) | PIC 9(07), PIC 9(14) |
 | PIC  -9(_).__ | PIC -9(9).99, PIC -9(2).999 |
 | PIC  Z(_)9.__ | PIC Z(9)9.99, PIC Z(2)9.999 |
+| PIC  9(_)V__  | PIC 9(2)V99, PIC 9(13)V999  |
+| PIC  S9(_)V__ | PIC S9(2)V99, PIC S9(13)V999|