From bac4691fd01340fc193d3fd8d23ec50510c900d9 Mon Sep 17 00:00:00 2001 From: knwng Date: Tue, 21 Dec 2021 23:48:42 +0800 Subject: [PATCH 1/2] add working directory to handle the data in different directories --- pkg/config/config.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index 591714cd..780aed40 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -106,6 +106,7 @@ type YAMLConfig struct { RemoveTempFiles *bool `json:"removeTempFiles" yaml:"removeTempFiles"` // from v1 NebulaClientSettings *NebulaClientSettings `json:"clientSettings" yaml:"clientSettings"` LogPath *string `json:"logPath" yaml:"logPath"` + WorkingDirectory *string `json:"workingDir" yaml:"workingDir"` Files []*File `json:"files" yaml:"files"` } @@ -147,6 +148,15 @@ func Parse(filename string) (*YAMLConfig, error) { return nil, ierrors.Wrap(ierrors.InvalidConfigPathOrFormat, err) } path := filepath.Dir(abs) + + if workingDir := *conf.WorkingDirectory; len(workingDir) > 0 { + if !filepath.IsAbs(workingDir) { + path = filepath.Join(path, workingDir) + } else { + path = workingDir + } + } + if err = conf.ValidateAndReset(path); err != nil { return nil, ierrors.Wrap(ierrors.ConfigError, err) } From a5771d510d914a2331196563f6cbb4f9d17268f3 Mon Sep 17 00:00:00 2001 From: knwng Date: Sat, 25 Dec 2021 00:46:31 +0800 Subject: [PATCH 2/2] update README; add testing example --- README.md | 4 ++- README_zh-CN.md | 4 ++- examples/v2/data/course.csv | 7 ++++ examples/v2/example_with_working_dir.yaml | 44 +++++++++++++++++++++++ pkg/config/config.go | 8 ++--- 5 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 examples/v2/data/course.csv create mode 100644 examples/v2/example_with_working_dir.yaml diff --git a/README.md b/README.md index e2771046..cfe47535 100644 --- a/README.md +++ b/README.md @@ -157,12 +157,14 @@ clientSettings: ### Files -The following two configurations are related to the log and data files: +The following three configurations are related to the log and data files: +* `workingDir`: **Optional**. If you have multiple directories containing data with the same file structure, you can use this parameter to switch between them. For example, the value of `path` and `failDataPath` of the configuration below will be automatically changed to `./data/student.csv` and `./data/err/student.csv`. If you change workingDir to `./data1`, the path will be changed accordingly. The param can be either absolute or relative. * `logPath`: **Optional**. Specifies the log path when importing data. The default path is `/tmp/nebula-importer-{timestamp}.log`. * `files`: **Required**. It is an array type to configure different data files. You can also import data from a HTTP link by inputting the link in the file path. ```yaml +workingDir: ./data/ logPath: ./err/test.log files: - path: ./student.csv diff --git a/README_zh-CN.md b/README_zh-CN.md index a145607b..7d3fb1da 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -111,12 +111,14 @@ clientSettings: ### 文件 -跟日志和数据文件相关的配置跟以下两个选项有关: +跟日志和数据文件相关的配置跟以下三个选项有关: +- `workingDir`: **可选**。如果有多个文件夹,里面有相同文件结构的数据,可以使用这个参数在多个文件夹中切换。比如对于下面代码块的配置来说,`path`和`failDataPath`的值会被自动替换成`./data/student.csv`和`./data/err/student.csv`,如果把`workingDir`换成`./data1`,这两个值也会做相应改变。这个参数可以是绝对路径,也可以是相对路径。 - `logPath`:**可选**。指定导入过程中的错误等日志信息输出的文件路径,默认输出到 `/tmp/nebula-importer-{timestamp}.log` 中。 - `files`:**必填**。数组类型,用来配置不同的数据文件。您也可以从 HTTP 链接导入数据,在文件路径中输入链接即可。 ```yaml +workingDir: ./data/ logPath: ./err/test.log files: - path: ./student.csv diff --git a/examples/v2/data/course.csv b/examples/v2/data/course.csv new file mode 100644 index 00000000..b0335b68 --- /dev/null +++ b/examples/v2/data/course.csv @@ -0,0 +1,7 @@ +x101,Math,3,No5 +y102,English,6,No11 +"z103",Chinese,1,No1 +0test,Test,2,No2 +00test,Test2,4,No3 +"000test",中国(  ),5,No10 +"0000test",中国( ),7,No10 diff --git a/examples/v2/example_with_working_dir.yaml b/examples/v2/example_with_working_dir.yaml new file mode 100644 index 00000000..f102f780 --- /dev/null +++ b/examples/v2/example_with_working_dir.yaml @@ -0,0 +1,44 @@ +version: v2 +description: example +removeTempFiles: false +clientSettings: + retry: 3 + concurrency: 2 # number of graph clients + channelBufferSize: 1 + space: importer_test_working_dir + connection: + user: root + password: nebula + address: graphd1:9669,graphd2:9669 + postStart: + commands: | + DROP SPACE IF EXISTS importer_test_working_dir; + CREATE SPACE IF NOT EXISTS importer_test_working_dir(partition_num=1, replica_factor=1, vid_type=FIXED_STRING(10)); + USE importer_test_working_dir; + CREATE TAG course(name string, credits int); + afterPeriod: 8s +workingDir: ./data/ +logPath: ./err/test.log +files: + - path: ./course.csv + failDataPath: ./err/course.csv + batchSize: 2 + inOrder: true + type: csv + csv: + withHeader: false + withLabel: false + schema: + type: vertex + vertex: + tags: + - name: course + props: + - name: name + type: string + - name: credits + type: int + - name: building + props: + - name: name + type: string diff --git a/pkg/config/config.go b/pkg/config/config.go index 780aed40..15435f0d 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -149,11 +149,11 @@ func Parse(filename string) (*YAMLConfig, error) { } path := filepath.Dir(abs) - if workingDir := *conf.WorkingDirectory; len(workingDir) > 0 { - if !filepath.IsAbs(workingDir) { - path = filepath.Join(path, workingDir) + if workingDir := conf.WorkingDirectory; workingDir != nil && len(*workingDir) > 0 { + if !filepath.IsAbs(*workingDir) { + path = filepath.Join(path, *workingDir) } else { - path = workingDir + path = *workingDir } }