feat: support for nullable prop (vesoft-inc#239)

README.md

@@ -203,6 +203,13 @@ schema:
             index: 1
           - name: gender
             type: string
+          - name: phone
+            type: string
+            nullable: true
+          - name: wechat
+            type: string
+            nullable: true
+            nullValue: "__NULL__"
 ```
 
 ##### `schema.vertex.vid`
@@ -225,6 +232,8 @@ Each tag contains the following two properties:
   * `name`: **Required**. The property name, must be the same with the tag property in Nebula Graph.
   * `type`: **Optional**. The property type, currently  `bool`, `int`, `float`, `double`, `string`, `time`, `timestamp`, `date`, `datetime`, `geography`, `geography(point)`, `geography(linestring)` and `geography(polygon)` are supported.
   * `index`: **Optional**. The column number in the CSV file.
+  * `nullable`: **Optional**. Whether this prop property can be `NULL`, optional values is `true` or `false`, default `false`.
+  * `nullValue`: **Optional**. If `nullable` is set to `true`, the property will be set to `NULL` when the value is equal to `nullValue`, default `""`.
 
 > **NOTE**: The properties in the preceding `prop` parameter must be sorted in the **same** way as in the CSV data file.
 

README_zh-CN.md

@@ -198,6 +198,8 @@ schema:
   - `name`：**必填**。属性名称，同 Nebula Graph 中创建的 TAG 的属性名称一致。
   - `type`：**必填**。属性类型，目前支持 `bool`、`int`、`float`、`double`、`timestamp`、`string`、`geography`、`geography(point)`、`geography(linestring)`和`geography(polygon)` 几种类型。
   - `index`：**可选**。在 CSV 文件中的列标。
+  - `nullable`：**可选**。此属性是否可以为 `NULL`，可选 `true` 或者 `false`，默认值为 `false` 。
+  - `nullValue`：**可选**。如果 `nullable` 设置为 `true`，则当值等于 `nullValue` 的时候属性将被设置为 `NULL` ，默认值为 `""`。
 
 > **注意**：上述 `props` 中的属性描述**顺序**必须同数据文件中的对应数据排列顺序一致。
 

examples/v1/example.yaml

@@ -24,7 +24,7 @@ clientSettings:
       CREATE TAG course_no_props();
       CREATE TAG building_no_props();
       CREATE EDGE follow_no_props();
-    afterPeriod: 8s
+    afterPeriod: 10s
   preStop:
     commands: |
       UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = false };

examples/v2/basic_type_test.csv

@@ -3,3 +3,5 @@ b2,false,0,0,0.0,0
 b3,true,1,2.0,3.3,abc
 b4,false,3,2.0,3.3,0a bd
 b5,true,-3,2,3,abcd efg
+bnull1,,,,,
+bnull2,,,,,__NULL__

examples/v2/date_test.csv

@@ -1,2 +1,3 @@
 d1,2020-01-01,18:28:23.284,2020-01-01T18:28:23.284,2020-01-01T18:28:23
 d2,2020-01-02,18:38:23.284,2020-01-11T19:28:23.284,1578770903
+dnull,,,,

examples/v2/example.yaml

@@ -24,13 +24,14 @@ clientSettings:
       CREATE TAG course_no_props();
       CREATE TAG building_no_props();
       CREATE EDGE follow_no_props();
-      CREATE TAG basic_type_test(b bool, i int, f float, d double, s string);
-      CREATE EDGE edge_basic_type_test(b bool, i int, f float, d double, s string);
-      CREATE TAG date_test(c1 date, c2 time, c3 datetime, c4 timestamp);
-      CREATE EDGE edge_date_test(c1 date, c2 time, c3 datetime, c4 timestamp);
-      CREATE TAG geography_test(any_shape geography, only_point geography(point), only_linestring geography(linestring), only_polygon geography(polygon));
-      CREATE EDGE edge_geography_test(any_shape geography, only_point geography(point), only_linestring geography(linestring), only_polygon geography(polygon))
-    afterPeriod: 8s
+      CREATE TAG basic_type_test(b bool NULL, i int NULL, f float NULL, d double NULL, s string NULL);
+      CREATE EDGE edge_basic_type_test(b bool NULL, i int NULL, f float NULL, d double NULL, s string NULL);
+      CREATE TAG date_test(c1 date NULL, c2 time NULL, c3 datetime NULL, c4 timestamp NULL);
+      CREATE EDGE edge_date_test(c1 date NULL, c2 time NULL, c3 datetime NULL, c4 timestamp NULL);
+      CREATE TAG geography_test(any_shape geography NULL, only_point geography(point) NULL, only_linestring geography(linestring) NULL, only_polygon geography(polygon) NULL);
+      CREATE EDGE edge_geography_test(any_shape geography NULL, only_point geography(point) NULL, only_linestring geography(linestring) NULL, only_polygon geography(polygon) NULL)
+
+    afterPeriod: 10s
   preStop:
     commands: |
       UPDATE CONFIGS storage:rocksdb_column_family_options = { disable_auto_compactions = false };
@@ -478,18 +479,23 @@ files:
               - name: b
                 type: bool
                 index: 1
+                nullable: true
               - name: i
                 type: int
                 index: 2
+                nullable: true
               - name: f
                 type: float
                 index: 3
+                nullable: true
               - name: d
                 type: double
                 index: 4
+                nullable: true
               - name: s
                 type: string
                 index: 5
+                nullable: true
 
   - path: ./basic_type_test.csv
     failDataPath: ./err/edge_basic_type_test.csv
@@ -513,18 +519,24 @@ files:
           - name: b
             type: bool
             index: 1
+            nullable: true
           - name: i
             type: int
             index: 2
+            nullable: true
           - name: f
             type: float
             index: 3
+            nullable: true
           - name: d
             type: double
             index: 4
+            nullable: true
           - name: s
             type: string
             index: 5
+            nullable: true
+            nullValue: "__NULL__"
 
   - path: ./date_test.csv
     failDataPath: ./err/date_test.csv
@@ -546,15 +558,19 @@ files:
               - name: c1
                 type: date
                 index: 1
+                nullable: true
               - name: c2
                 type: time
                 index: 2
+                nullable: true
               - name: c3
                 type: datetime
                 index: 3
+                nullable: true
               - name: c4
                 type: timestamp
                 index: 4
+                nullable: true
 
   - path: ./date_test.csv
     failDataPath: ./err/edge_date_test.csv
@@ -578,15 +594,19 @@ files:
           - name: c1
             type: date
             index: 1
+            nullable: true
           - name: c2
             type: time
             index: 2
+            nullable: true
           - name: c3
             type: datetime
             index: 3
+            nullable: true
           - name: c4
             type: timestamp
             index: 4
+            nullable: true
 
   - path: ./geography_test.csv
     failDataPath: ./err/geography_test.csv
@@ -608,15 +628,19 @@ files:
               - name: any_shape
                 type: geography
                 index: 1
+                nullable: true
               - name: only_point
                 type: geography(point)
                 index: 2
+                nullable: true
               - name: only_linestring
                 type: geography(linestring)
                 index: 3
+                nullable: true
               - name: only_polygon
                 type: geography(polygon)
                 index: 4
+                nullable: true
 
   - path: ./geography_test.csv
     failDataPath: ./err/edge_geography_test.csv
@@ -640,12 +664,16 @@ files:
           - name: any_shape
             type: geography
             index: 1
+            nullable: true
           - name: only_point
             type: geography(point)
             index: 2
+            nullable: true
           - name: only_linestring
             type: geography(linestring)
             index: 3
+            nullable: true
           - name: only_polygon
             type: geography(polygon)
             index: 4
+            nullable: true

examples/v2/geography_test.csv

@@ -1,3 +1,4 @@
 g1,POINT (-82.3764154 42.6452196),"Point(3 8)","LINEstring (-108.7 35.0,-100.0 46.5,-90.7 34.9,-108.7 35.0)","POlygon ( (-100.1 41.4,-102.9 37.6,-96.8 37.5,-100.1 41.4))"
 g2,"LineString(0 1, 1 2, 2 3)","point(4.6 5.7 )","LINESTRING(43.8 52.6, -78.99 84.323)","POLYGON ((-108.7 35.0,-100.0 46.5,-90.7 34.9,-108.7 35.0))"
 g3,"Polygon((-85.1 34.8,-80.7 28.4,-76.9 34.9,-85.1 34.8))", Point(0.0 0.0),"linestring(0 1, 179.99 89.99)","polygon((0 1, 2 4, 3 5, 4 9, 0 1))"
+gnull,,,,

pkg/config/config.go

@@ -17,6 +17,10 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+const (
+	dbNULL = "NULL"
+)
+
 var (
 	reTimestampInteger = regexp.MustCompile(`^(0[xX][0-9a-fA-F]+|0[0-7]+|\d+)$`)
 )
@@ -47,9 +51,11 @@ type NebulaClientSettings struct {
 }
 
 type Prop struct {
-	Name  *string `json:"name" yaml:"name"`
-	Type  *string `json:"type" yaml:"type"`
-	Index *int    `json:"index" yaml:"index"`
+	Name      *string `json:"name" yaml:"name"`
+	Type      *string `json:"type" yaml:"type"`
+	Index     *int    `json:"index" yaml:"index"`
+	Nullable  bool    `json:"nullable" yaml:"nullable"`
+	NullValue string  `json:"nullValue" yaml:"nullValue"`
 }
 
 type VID struct {
@@ -466,17 +472,17 @@ func (s *Schema) validateAndReset(prefix string) error {
 	var err error = nil
 	switch strings.ToLower(*s.Type) {
 	case "edge":
-		if s.Edge != nil {
-			err = s.Edge.validateAndReset(fmt.Sprintf("%s.edge", prefix))
-		} else {
+		if s.Edge == nil {
 			logger.Log.Infof("%s.edge is nil", prefix)
+			s.Edge = &Edge{}
 		}
+		err = s.Edge.validateAndReset(fmt.Sprintf("%s.edge", prefix))
 	case "vertex":
-		if s.Vertex != nil {
-			err = s.Vertex.validateAndReset(fmt.Sprintf("%s.vertex", prefix))
-		} else {
+		if s.Vertex == nil {
 			logger.Log.Infof("%s.vertex is nil", prefix)
+			s.Vertex = &Vertex{}
 		}
+		err = s.Vertex.validateAndReset(fmt.Sprintf("%s.vertex", prefix))
 	default:
 		err = fmt.Errorf("Error schema type(%s) in %s.type only edge and vertex are supported", *s.Type, prefix)
 	}
@@ -830,6 +836,9 @@ func (p *Prop) FormatValue(record base.Record) (string, error) {
 		return "", fmt.Errorf("Prop index %d out range %d of record(%v)", *p.Index, len(record), record)
 	}
 	r := record[*p.Index]
+	if p.Nullable && r == p.NullValue {
+		return dbNULL, nil
+	}
 	if p.IsStringType() {
 		return fmt.Sprintf("%q", r), nil
 	}