exchange supports batch update and delete (#2213)

vesoft-inc · Aug 10, 2023 · c539a6e · c539a6e
1 parent 3d7e95a
commit c539a6e
Show file tree

Hide file tree

Showing 15 changed files with 128 additions and 0 deletions.
diff --git a/docs-2.0/nebula-exchange/parameter-reference/ex-ug-parameter.md b/docs-2.0/nebula-exchange/parameter-reference/ex-ug-parameter.md
@@ -98,6 +98,8 @@ For different data sources, the vertex configurations are different. There are m
 |`tags.name`|string|-|Yes|The tag name defined in NebulaGraph.|
 |`tags.type.source`|string|-|Yes|Specify a data source. For example, `csv`.|
 |`tags.type.sink`|string|`client`|Yes|Specify an import method. Optional values are `client` and `SST`.|
+|`tags.writeMode`|string|`INSERT`|No|Types of batch operations on data, including batch inserts, updates, and deletes. Optional values are `INSERT`, `UPDATE`, `DELETE`.|
+|`tags.deleteEdge`|string|`false`|No|Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `tags.writeMode` is `DELETE`.|
 |`tags.fields`|list\[string\]|-|Yes|The header or column name of the column corresponding to properties. If there is a header or a column name, please use that name directly. If a CSV file does not have a header, use the form of `[_c0, _c1, _c2]` to represent the first column, the second column, the third column, and so on.|
 |`tags.nebula.fields`|list\[string\]|-|Yes|Property names defined in NebulaGraph, the order of which must correspond to `tags.fields`. For example, `[_c1, _c2]` corresponds to `[name, age]`, which means that values in the second column are the values of the property `name`, and values in the third column are the values of the property `age`.|
 |`tags.vertex.field`|string|-|Yes|The column of vertex IDs. For example, when a CSV file has no header, users can use `_c0` to indicate values in the first column are vertex IDs.|
@@ -245,6 +247,7 @@ For the specific parameters of different data sources for edge configurations, p
 |`edges.name`| string|-|Yes|The edge type name defined in NebulaGraph.|
 |`edges.type.source`|string|-|Yes|The data source of edges. For example, `csv`.|
 |`edges.type.sink`|string|`client`|Yes|The method specified to import data. Optional values are `client` and `SST`.|
+|`edges.writeMode`|string|`INSERT`|No|Types of batch operations on data, including batch inserts, updates, and deletes. Optional values are `INSERT`, `UPDATE`, `DELETE`.|
 |`edges.fields`|list\[string\]|-|Yes|The header or column name of the column corresponding to properties. If there is a header or column name, please use that name directly. If a CSV file does not have a header, use the form of `[_c0, _c1, _c2]` to represent the first column, the second column, the third column, and so on.|
 |`edges.nebula.fields`|list\[string\]|-|Yes|Edge names defined in NebulaGraph, the order of which must correspond to `edges.fields`. For example, `[_c2, _c3]` corresponds to `[start_year, end_year]`, which means that values in the third column are the values of the start year, and values in the fourth column are the values of the end year.|
 |`edges.source.field`|string|-|Yes|The column of source vertices of edges. For example, `_c0` indicates a value in the first column that is used as the source vertex of an edge.|

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-clickhouse.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-clickhouse.md
@@ -174,6 +174,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
         # policy:hash
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -258,6 +264,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-csv.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-csv.md
@@ -202,6 +202,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # If the CSV file does not have a header, set the header to false. The default value is false.
       header: false
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -317,6 +323,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # If the CSV file does not have a header, set the header to false. The default value is false.
       header: false
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-hbase.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-hbase.md
@@ -207,6 +207,11 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
 
       # Number of pieces of data written to NebulaGraph in a single batch.
       batch: 256
@@ -287,6 +292,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-hive.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-hive.md
@@ -245,6 +245,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -318,6 +324,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-jdbc.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-jdbc.md
@@ -225,6 +225,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -319,6 +325,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-json.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-json.md
@@ -222,6 +222,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -323,6 +329,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-kafka.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-kafka.md
@@ -186,6 +186,11 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
 
       # The number of data written to NebulaGraph in a single batch.
       batch: 10
@@ -252,6 +257,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
   #    # (Optional) Specify a column as the source of the rank.
   #    #ranking: rank
 
+  #    # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+  #    #writeMode: INSERT
+
   #    # The number of data written to NebulaGraph in a single batch.
   #    batch: 10
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-maxcompute.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-maxcompute.md
@@ -183,6 +183,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -281,6 +287,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of Spark partitions.
       partition:10
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-mysql.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-mysql.md
@@ -213,6 +213,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -299,6 +305,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-neo4j.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-neo4j.md
@@ -190,6 +190,13 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #            newColName:new-field
       #        }
       }
+
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       partition: 10
       batch: 1000
       check_point_path: /tmp/test
@@ -250,6 +257,10 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
       #ranking: rank
+
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       partition: 10
       batch: 1000
       check_point_path: /tmp/test

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-oracle.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-oracle.md
@@ -212,6 +212,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -296,6 +302,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-orc.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-orc.md
@@ -189,6 +189,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -290,6 +296,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-parquet.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-parquet.md
@@ -190,6 +190,12 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 
@@ -290,6 +296,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 256
 

diff --git a/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-pulsar.md b/docs-2.0/nebula-exchange/use-exchange/ex-ug-import-from-pulsar.md
@@ -170,6 +170,11 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       #        }
       }
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+      
+      # Whether or not to delete the related incoming and outgoing edges of the vertices when performing a batch delete operation. This parameter takes effect when `writeMode` is `DELETE`.
+      #deleteEdge: false
 
       # The number of data written to NebulaGraph in a single batch.
       batch: 10
@@ -257,6 +262,9 @@ After Exchange is compiled, copy the conf file `target/classes/application.conf`
       # (Optional) Specify a column as the source of the rank.
       #ranking: rank
 
+      # Batch operation types, including INSERT, UPDATE, and DELETE. defaults to INSERT.
+      #writeMode: INSERT
+
       # The number of data written to NebulaGraph in a single batch.
       batch: 10