Support config regex to skip specified line

Implement #42 Change-Id: Ib27d2cfdaa05599761024ddd5c913e47cccb6eb6
apache · Mar 18, 2019 · 8516c81 · 8516c81
1 parent 3948170
commit 8516c81
Show file tree

Hide file tree

Showing 5 changed files with 18 additions and 20 deletions.
diff --git a/src/main/java/com/baidu/hugegraph/loader/reader/file/AbstractFileReader.java b/src/main/java/com/baidu/hugegraph/loader/reader/file/AbstractFileReader.java
@@ -123,16 +123,16 @@ protected Line fetch() {
             return null;
         }
 
-        // Skip the comment line
-        if (this.isCommentLine(rawLine)) {
+        // Skip the line matched specified regex
+        if (this.needSkipLine(rawLine)) {
             return this.fetch();
         } else {
             return this.parser.parse(rawLine);
         }
     }
 
-    private boolean isCommentLine(String line) {
-        return this.source.commentSymbols().stream().anyMatch(line::startsWith);
+    private boolean needSkipLine(String line) {
+        return line.matches(this.source.skippedLineRegex());
     }
 
     private boolean isDuplicateHeader(String line) {

diff --git a/src/main/java/com/baidu/hugegraph/loader/source/file/FileSource.java b/src/main/java/com/baidu/hugegraph/loader/source/file/FileSource.java
@@ -20,9 +20,7 @@
 package com.baidu.hugegraph.loader.source.file;
 
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 
 import com.baidu.hugegraph.loader.source.InputSource;
 import com.baidu.hugegraph.loader.source.SourceType;
@@ -32,6 +30,7 @@ public class FileSource implements InputSource {
 
     private static final String DEFAULT_CHARSET = "UTF-8";
     private static final String DEFAULT_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss";
+    private static final String DEFAULT_SKIPPED_LINE_REGEX = "";
 
     @JsonProperty("path")
     private String path;
@@ -45,16 +44,16 @@ public class FileSource implements InputSource {
     private String charset;
     @JsonProperty("date_format")
     private String dateFormat;
+    @JsonProperty("skipped_line_regex")
+    private String skippedLineRegex;
     @JsonProperty("compression")
     private Compression compression;
-    @JsonProperty("comment_symbols")
-    private Set<String> commentSymbols;
 
     public FileSource() {
         this.charset = DEFAULT_CHARSET;
         this.dateFormat = DEFAULT_DATE_FORMAT;
+        this.skippedLineRegex = DEFAULT_SKIPPED_LINE_REGEX;
         this.compression = Compression.NONE;
-        this.commentSymbols = new HashSet<>();
     }
 
     @Override
@@ -90,13 +89,12 @@ public String dateFormat() {
         return this.dateFormat;
     }
 
-    public Compression compression() {
-        return this.compression;
+    public String skippedLineRegex() {
+        return this.skippedLineRegex;
     }
 
-    public Set<String> commentSymbols() {
-        assert this.commentSymbols != null;
-        return Collections.unmodifiableSet(this.commentSymbols);
+    public Compression compression() {
+        return this.compression;
     }
 
     @Override

diff --git a/src/test/java/com/baidu/hugegraph/loader/test/functional/FileLoadTest.java b/src/test/java/com/baidu/hugegraph/loader/test/functional/FileLoadTest.java
@@ -628,18 +628,18 @@ public void testMultiFilesHaveHeader() {
     }
 
     @Test
-    public void testFileHasCommentLine() {
+    public void testFileHasSkipLine() {
         ioUtil.write("vertex_person.csv",
                      "name,age,city",
                      "# This is a comment",
-                     "marko,29,Beijing",
+                     "marko,29,#Beijing",
                      "// This is also a comment",
                      "# This is still a comment",
-                     "vadas,27,Hongkong");
+                     "vadas,27,//Hongkong");
 
         String[] args = new String[]{
-                "-f", configPath("file_has_comment_line/struct.json"),
-                "-s", configPath("file_has_comment_line/schema.groovy"),
+                "-f", configPath("file_has_skipped_line/struct.json"),
+                "-s", configPath("file_has_skipped_line/schema.groovy"),
                 "-g", GRAPH,
                 "-h", SERVER,
                 "--test-mode", "true"

diff --git a/...urces/file_has_comment_line/schema.groovy → ...urces/file_has_skipped_line/schema.groovy b/...urces/file_has_comment_line/schema.groovy → ...urces/file_has_skipped_line/schema.groovy
diff --git a/...sources/file_has_comment_line/struct.json → ...sources/file_has_skipped_line/struct.json b/...sources/file_has_comment_line/struct.json → ...sources/file_has_skipped_line/struct.json
@@ -7,7 +7,7 @@
         "path": "${store_path}/vertex_person.csv",
         "format": "CSV",
         "charset": "UTF-8",
-        "comment_symbols": ["#", "//"]
+        "skipped_line_regex": "(^#|^//).*"
       },
       "mapping": {
         "name": "name",