[INLONG-10931][Manager] Data preview supports data containing escape …

…characters
fuweng11 · Aug 28, 2024 · f4f1cd2 · f4f1cd2
1 parent 89f43c6
commit f4f1cd2
Show file tree

Hide file tree

Showing 2 changed files with 17 additions and 4 deletions.
diff --git a/inlong-manager/manager-service/pom.xml b/inlong-manager/manager-service/pom.xml
@@ -32,6 +32,11 @@
     </properties>
 
     <dependencies>
+        <dependency>
+            <groupId>org.apache.inlong</groupId>
+            <artifactId>transform-sdk</artifactId>
+            <version>${project.version}</version>
+        </dependency>
         <dependency>
             <groupId>org.apache.inlong</groupId>
             <artifactId>manager-common</artifactId>

diff --git a/...service/src/main/java/org/apache/inlong/manager/service/datatype/CsvDataTypeOperator.java b/...service/src/main/java/org/apache/inlong/manager/service/datatype/CsvDataTypeOperator.java
@@ -23,6 +23,7 @@
 import org.apache.inlong.manager.common.util.CommonBeanUtils;
 import org.apache.inlong.manager.pojo.consume.BriefMQMessage.FieldInfo;
 import org.apache.inlong.manager.pojo.stream.InlongStreamInfo;
+import org.apache.inlong.sdk.transform.decode.SplitUtils;
 
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.StringUtils;
@@ -47,10 +48,17 @@ public List<FieldInfo> parseFields(String str, InlongStreamInfo streamInfo) thro
             if (StringUtils.isNotBlank(streamInfo.getDataSeparator())) {
                 separator = (char) Integer.parseInt(streamInfo.getDataSeparator());
             }
-            String[] bodys = StringUtils.split(str, separator);
-            for (int i = 0; i < bodys.length; i++) {
-                if (i < fields.size()) {
-                    fields.get(i).setFieldValue(bodys[i]);
+            Character escapeChar = null;
+            if (StringUtils.isNotBlank(streamInfo.getDataEscapeChar())) {
+                escapeChar = streamInfo.getDataEscapeChar().charAt(0);
+            }
+            String[][] rowValues = SplitUtils.splitCsv(str, separator, escapeChar, '\"', '\n', true);
+            for (int i = 0; i < rowValues.length; i++) {
+                String[] fieldValues = rowValues[i];
+                for (int j = 0; j < fieldValues.length; j++) {
+                    if (i + j < fields.size()) {
+                        fields.get(i + j).setFieldValue(fieldValues[j]);
+                    }
                 }
             }
         } catch (Exception e) {