From 7ed4573cda53cb33de4e39fba6581e83109f1110 Mon Sep 17 00:00:00 2001 From: Vee Zhang Date: Fri, 16 Dec 2022 14:42:13 +0800 Subject: [PATCH] fix: failDataPath is directory --- README.md | 6 +-- README_zh-CN.md | 6 +-- examples/sf/sf.yaml | 62 +++++++++++------------ examples/v1/example.yaml | 36 ++++++------- examples/v2/example.yaml | 54 ++++++++++---------- examples/v2/example_with_working_dir.yaml | 2 +- pkg/config/config.go | 23 +++++++-- 7 files changed, 102 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 3bce89ca..406713ca 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ clientSettings: The following three configurations are related to the log and data files: -* `workingDir`: **Optional**. If you have multiple directories containing data with the same file structure, you can use this parameter to switch between them. For example, the value of `path` and `failDataPath` of the configuration below will be automatically changed to `./data/student.csv` and `./data/err/student.csv`. If you change workingDir to `./data1`, the path will be changed accordingly. The param can be either absolute or relative. +* `workingDir`: **Optional**. If you have multiple directories containing data with the same file structure, you can use this parameter to switch between them. For example, the value of `path` and `failDataPath` of the configuration below will be automatically changed to `./data/student.csv` and `./data/err/student`. If you change workingDir to `./data1`, the path will be changed accordingly. The param can be either absolute or relative. * `logPath`: **Optional**. Specifies the log path when importing data. The default path is `/tmp/nebula-importer-{timestamp}.log`. * `files`: **Required**. It is an array type to configure different data files. You can also import data from a HTTP link by inputting the link in the file path. @@ -147,7 +147,7 @@ workingDir: ./data/ logPath: ./err/test.log files: - path: ./student.csv - failDataPath: ./err/student.csv + failDataPath: ./err/student batchSize: 128 limit: 10 inOrder: false @@ -163,7 +163,7 @@ files: One CSV file can only store one type of vertex or edge. Vertices and edges of the different schema must be stored in different files. * `path`: **Required**. Specifies the path where the data files are stored. If a relative path is used, the `path` and current configuration file directory are spliced. Wildcard filename is also supported, for example: `./follower-*.csv`, please make sure that all matching files with the same schema. -* `failDataPath`: **Required**. Specifies the path for data that failed in inserting so that the failed data are reinserted. +* `failDataPath`: **Required**. Specifies the directory for data that failed in inserting so that the failed data are reinserted. * `batchSize`: **Optional**. Specifies the batch size of the inserted data. The default value is 128. * `limit`: **Optional**. Limits the max data reading rows. * `inOrder`: **Optional**. Whether to insert the data rows in the file in order. If you do not specify it, you avoid the decrease in importing rate caused by the data skew. diff --git a/README_zh-CN.md b/README_zh-CN.md index 036d374f..ebe0ad80 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -113,7 +113,7 @@ clientSettings: 跟日志和数据文件相关的配置跟以下三个选项有关： -- `workingDir`: **可选**。如果有多个文件夹，里面有相同文件结构的数据，可以使用这个参数在多个文件夹中切换。比如对于下面代码块的配置来说，`path`和`failDataPath`的值会被自动替换成`./data/student.csv`和`./data/err/student.csv`，如果把`workingDir`换成`./data1`，这两个值也会做相应改变。这个参数可以是绝对路径，也可以是相对路径。 +- `workingDir`: **可选**。如果有多个文件夹，里面有相同文件结构的数据，可以使用这个参数在多个文件夹中切换。比如对于下面代码块的配置来说，`path`和`failDataPath`的值会被自动替换成`./data/student.csv`和`./data/err/student`，如果把`workingDir`换成`./data1`，这两个值也会做相应改变。这个参数可以是绝对路径，也可以是相对路径。 - `logPath`：**可选**。指定导入过程中的错误等日志信息输出的文件路径，默认输出到 `/tmp/nebula-importer-{timestamp}.log` 中。 - `files`：**必填**。数组类型，用来配置不同的数据文件。您也可以从 HTTP 链接导入数据，在文件路径中输入链接即可。 @@ -122,7 +122,7 @@ workingDir: ./data/ logPath: ./err/test.log files: - path: ./student.csv - failDataPath: ./err/student.csv + failDataPath: ./err/student batchSize: 128 limit: 10 inOrder: false @@ -138,7 +138,7 @@ files: 一个数据文件中只能存放一种点或者边，不同 schema 的点或者边数据需要放置在不同的文件中。 - `path`：**必填**。指定数据文件的存放路径，如果使用相对路径，则会拼接当前配置文件的目录和 `path`。 -- `failDataPath`：**必填**。指定插入失败的数据输出的文件，以便后面补写出错数据。 +- `failDataPath`：**必填**。指定插入失败的数据输出的文件目录，以便后面补写出错数据。 - `batchSize`：**可选**。批量插入数据的条数，默认 128。 - `limit`：**可选**。限制读取文件的行数。 - `inOrder`：**可选**。是否按序插入文件中的每一行。如果不指定，可以避免数据倾斜导致的导入速率的下降。 diff --git a/examples/sf/sf.yaml b/examples/sf/sf.yaml index 42f13956..9808d911 100644 --- a/examples/sf/sf.yaml +++ b/examples/sf/sf.yaml @@ -45,7 +45,7 @@ clientSettings: logPath: ./err/test.log files: - path: ./social_network/dynamic/person_final.csv - failDataPath: ./err/data/Person.csv + failDataPath: ./err/data/Person batchSize: 100 type: csv csv: @@ -91,7 +91,7 @@ files: index: 11 - path: ./social_network/dynamic/forum.csv - failDataPath: ./err/data/Forum.csv + failDataPath: ./err/data/Forum batchSize: 100 type: csv csv: @@ -116,7 +116,7 @@ files: index: 2 - path: ./social_network/dynamic/comment.csv - failDataPath: ./err/data/Comment.csv + failDataPath: ./err/data/Comment batchSize: 100 type: csv csv: @@ -150,7 +150,7 @@ files: index: 5 - path: ./social_network/dynamic/post.csv - failDataPath: ./err/data/Post.csv + failDataPath: ./err/data/Post batchSize: 100 type: csv csv: @@ -190,7 +190,7 @@ files: index: 7 - path: ./social_network/static/tagclass.csv - failDataPath: ./err/data/Tagclass.csv + failDataPath: ./err/data/Tagclass batchSize: 100 type: csv csv: @@ -215,7 +215,7 @@ files: index: 2 - path: ./social_network/static/organisation.csv - failDataPath: ./err/data/Organisation.csv + failDataPath: ./err/data/Organisation batchSize: 100 type: csv csv: @@ -243,7 +243,7 @@ files: index: 3 - path: ./social_network/static/place.csv - failDataPath: ./err/data/Place.csv + failDataPath: ./err/data/Place batchSize: 100 type: csv csv: @@ -271,7 +271,7 @@ files: index: 3 - path: ./social_network/static/tag.csv - failDataPath: ./err/data/Tag.csv + failDataPath: ./err/data/Tag batchSize: 100 type: csv csv: @@ -296,7 +296,7 @@ files: index: 2 - path: ./social_network/dynamic/forum_hasModerator_person.csv - failDataPath: ./err/data/HAS_MODERATOR.csv + failDataPath: ./err/data/HAS_MODERATOR batchSize: 100 type: csv csv: @@ -319,7 +319,7 @@ files: props: - path: ./social_network/dynamic/person_likes_comment.csv - failDataPath: ./err/data/LIKES_COMMENT.csv + failDataPath: ./err/data/LIKES_COMMENT batchSize: 100 type: csv csv: @@ -345,7 +345,7 @@ files: index: 2 - path: ./social_network/dynamic/forum_hasMember_person.csv - failDataPath: ./err/data/HAS_MEMBER.csv + failDataPath: ./err/data/HAS_MEMBER batchSize: 100 type: csv csv: @@ -371,7 +371,7 @@ files: index: 2 - path: ./social_network/dynamic/person_likes_post.csv - failDataPath: ./err/data/LIKES_POST.csv + failDataPath: ./err/data/LIKES_POST batchSize: 100 type: csv csv: @@ -397,7 +397,7 @@ files: index: 2 - path: ./social_network/dynamic/post_hasTag_tag.csv - failDataPath: ./err/data/HAS_TAG.csv + failDataPath: ./err/data/HAS_TAG batchSize: 100 type: csv csv: @@ -420,7 +420,7 @@ files: props: - path: ./social_network/dynamic/comment_hasTag_tag.csv - failDataPath: ./err/data/HAS_TAG.csv + failDataPath: ./err/data/HAS_TAG batchSize: 100 type: csv csv: @@ -443,7 +443,7 @@ files: props: - path: ./social_network/dynamic/forum_containerOf_post.csv - failDataPath: ./err/data/CONTAINER_OF.csv + failDataPath: ./err/data/CONTAINER_OF batchSize: 100 type: csv csv: @@ -466,7 +466,7 @@ files: props: - path: ./social_network/dynamic/person_knows_person.csv - failDataPath: ./err/data/KNOWS.csv + failDataPath: ./err/data/KNOWS batchSize: 100 type: csv csv: @@ -492,7 +492,7 @@ files: index: 2 - path: ./social_network/dynamic/person_hasInterest_tag.csv - failDataPath: ./err/data/HAS_INTEREST.csv + failDataPath: ./err/data/HAS_INTEREST batchSize: 100 type: csv csv: @@ -515,7 +515,7 @@ files: props: - path: ./social_network/dynamic/person_workAt_organisation.csv - failDataPath: ./err/data/WORK_AT.csv + failDataPath: ./err/data/WORK_AT batchSize: 100 type: csv csv: @@ -541,7 +541,7 @@ files: index: 2 - path: ./social_network/dynamic/person_isLocatedIn_place.csv - failDataPath: ./err/data/IS_LOCATED_IN.csv + failDataPath: ./err/data/IS_LOCATED_IN batchSize: 100 type: csv csv: @@ -564,7 +564,7 @@ files: props: - path: ./social_network/dynamic/forum_hasTag_tag.csv - failDataPath: ./err/data/HAS_TAG.csv + failDataPath: ./err/data/HAS_TAG batchSize: 100 type: csv csv: @@ -587,7 +587,7 @@ files: props: - path: ./social_network/dynamic/comment_replyOf_post.csv - failDataPath: ./err/data/REPLY_OF_POST.csv + failDataPath: ./err/data/REPLY_OF_POST batchSize: 100 type: csv csv: @@ -610,7 +610,7 @@ files: props: - path: ./social_network/dynamic/post_isLocatedIn_place.csv - failDataPath: ./err/data/IS_LOCATED_IN.csv + failDataPath: ./err/data/IS_LOCATED_IN batchSize: 100 type: csv csv: @@ -633,7 +633,7 @@ files: props: - path: ./social_network/dynamic/comment_replyOf_comment.csv - failDataPath: ./err/data/REPLY_OF_COMMENT.csv + failDataPath: ./err/data/REPLY_OF_COMMENT batchSize: 100 type: csv csv: @@ -656,7 +656,7 @@ files: props: - path: ./social_network/dynamic/comment_isLocatedIn_place.csv - failDataPath: ./err/data/IS_LOCATED_IN.csv + failDataPath: ./err/data/IS_LOCATED_IN batchSize: 100 type: csv csv: @@ -679,7 +679,7 @@ files: props: - path: ./social_network/dynamic/person_studyAt_organisation.csv - failDataPath: ./err/data/STUDY_AT.csv + failDataPath: ./err/data/STUDY_AT batchSize: 100 type: csv csv: @@ -705,7 +705,7 @@ files: index: 2 - path: ./social_network/dynamic/comment_hasCreator_person_new.csv - failDataPath: ./err/data/COMMENT_HAS_CREATOR.csv + failDataPath: ./err/data/COMMENT_HAS_CREATOR batchSize: 100 type: csv csv: @@ -743,7 +743,7 @@ files: index: 7 - path: ./social_network/dynamic/post_hasCreator_person_new.csv - failDataPath: ./err/data/POST_HAS_CREATOR.csv + failDataPath: ./err/data/POST_HAS_CREATOR batchSize: 100 type: csv csv: @@ -787,7 +787,7 @@ files: index: 9 - path: ./social_network/static/tagclass_isSubclassOf_tagclass.csv - failDataPath: ./err/data/IS_SUBCLASS_OF.csv + failDataPath: ./err/data/IS_SUBCLASS_OF batchSize: 100 type: csv csv: @@ -810,7 +810,7 @@ files: props: - path: ./social_network/static/place_isPartOf_place.csv - failDataPath: ./err/data/IS_PART_OF.csv + failDataPath: ./err/data/IS_PART_OF batchSize: 100 type: csv csv: @@ -833,7 +833,7 @@ files: props: - path: ./social_network/static/tag_hasType_tagclass.csv - failDataPath: ./err/data/HAS_TYPE.csv + failDataPath: ./err/data/HAS_TYPE batchSize: 100 type: csv csv: @@ -856,7 +856,7 @@ files: props: - path: ./social_network/static/organisation_isLocatedIn_place.csv - failDataPath: ./err/data/IS_LOCATED_IN.csv + failDataPath: ./err/data/IS_LOCATED_IN batchSize: 100 type: csv csv: diff --git a/examples/v1/example.yaml b/examples/v1/example.yaml index 46a6113f..6e80bb85 100644 --- a/examples/v1/example.yaml +++ b/examples/v1/example.yaml @@ -52,7 +52,7 @@ files: type: int - path: ./course.csv - failDataPath: ./err/course.csv + failDataPath: ./err/course batchSize: 2 inOrder: true type: csv @@ -77,7 +77,7 @@ files: type: string - path: ./course.csv - failDataPath: ./err/course-concat.csv + failDataPath: ./err/course-concat batchSize: 2 inOrder: true type: csv @@ -109,7 +109,7 @@ files: type: string - path: ./course-with-header.csv - failDataPath: ./err/course-with-header.csv + failDataPath: ./err/course-with-header batchSize: 2 inOrder: true type: csv @@ -120,7 +120,7 @@ files: type: vertex - path: ./follow-with-label.csv - failDataPath: ./err/follow-with-label.csv + failDataPath: ./err/follow-with-label batchSize: 2 inOrder: true type: csv @@ -146,7 +146,7 @@ files: index: 1 - path: ./follow-with-label-and-str-vid.csv - failDataPath: ./err/follow-with-label-and-str-vid.csv + failDataPath: ./err/follow-with-label-and-str-vid batchSize: 2 inOrder: true type: csv @@ -174,7 +174,7 @@ files: index: 1 - path: ./follow.csv - failDataPath: ./err/follow.csv + failDataPath: ./err/follow batchSize: 2 type: csv csv: @@ -194,7 +194,7 @@ files: type: double - path: ./follow-with-header.csv - failDataPath: ./err/follow-with-header.csv + failDataPath: ./err/follow-with-header batchSize: 2 type: csv csv: @@ -207,7 +207,7 @@ files: withRanking: true - path: ./student.csv - failDataPath: ./err/student.csv + failDataPath: ./err/student batchSize: 2 type: csv csv: @@ -229,7 +229,7 @@ files: type: string - path: ./student.csv - failDataPath: ./err/student_index.csv + failDataPath: ./err/student_index batchSize: 2 type: csv csv: @@ -254,7 +254,7 @@ files: type: string - path: ./student-with-label-and-str-vid.csv - failDataPath: ./err/student_label_str_vid.csv + failDataPath: ./err/student_label_str_vid batchSize: 2 type: csv csv: @@ -280,7 +280,7 @@ files: type: string - path: ./follow.csv - failDataPath: ./err/follow_index.csv + failDataPath: ./err/follow_index batchSize: 2 limit: 3 type: csv @@ -307,7 +307,7 @@ files: index: 3 - path: ./follow-delimiter.csv - failDataPath: ./err/follow-delimiter.csv + failDataPath: ./err/follow-delimiter batchSize: 2 type: csv csv: @@ -321,7 +321,7 @@ files: withRanking: true - path: https://raw.githubusercontent.com/vesoft-inc/nebula-importer/master/examples/v2/follow.csv - failDataPath: ./err/follow_http.csv + failDataPath: ./err/follow_http batchSize: 2 limit: 3 type: csv @@ -348,7 +348,7 @@ files: index: 3 - path: ./course.csv - failDataPath: ./err/course-empty-props.csv + failDataPath: ./err/course-empty-props batchSize: 2 inOrder: true type: csv @@ -366,7 +366,7 @@ files: - name: course_no_props - path: ./course.csv - failDataPath: ./err/course-multi-empty-props.csv + failDataPath: ./err/course-multi-empty-props batchSize: 2 inOrder: true type: csv @@ -385,7 +385,7 @@ files: - name: building_no_props - path: ./course.csv - failDataPath: ./err/course-mix-empty-props.csv + failDataPath: ./err/course-mix-empty-props batchSize: 2 inOrder: true type: csv @@ -408,7 +408,7 @@ files: index: 3 - path: ./course.csv - failDataPath: ./err/course-mix-empty-props-2.csv + failDataPath: ./err/course-mix-empty-props-2 batchSize: 2 inOrder: true type: csv @@ -432,7 +432,7 @@ files: - path: ./follow.csv - failDataPath: ./err/follow-empty-props.csv + failDataPath: ./err/follow-empty-props batchSize: 2 type: csv csv: diff --git a/examples/v2/example.yaml b/examples/v2/example.yaml index 70d675e5..509c40d7 100644 --- a/examples/v2/example.yaml +++ b/examples/v2/example.yaml @@ -55,7 +55,7 @@ files: type: int - path: ./course.csv - failDataPath: ./err/course.csv + failDataPath: ./err/course batchSize: 2 inOrder: true type: csv @@ -78,7 +78,7 @@ files: type: string - path: ./course.csv - failDataPath: ./err/course-concat.csv + failDataPath: ./err/course-concat batchSize: 2 inOrder: true type: csv @@ -109,7 +109,7 @@ files: type: string - path: ./course-with-header.csv - failDataPath: ./err/course-with-header.csv + failDataPath: ./err/course-with-header batchSize: 2 inOrder: true type: csv @@ -120,7 +120,7 @@ files: type: vertex - path: ./follow-with-label.csv - failDataPath: ./err/follow-with-label.csv + failDataPath: ./err/follow-with-label batchSize: 2 inOrder: true type: csv @@ -144,7 +144,7 @@ files: index: 1 - path: ./follow-with-label-and-str-vid.csv - failDataPath: ./err/follow-with-label-and-str-vid.csv + failDataPath: ./err/follow-with-label-and-str-vid batchSize: 2 inOrder: true type: csv @@ -170,7 +170,7 @@ files: index: 1 - path: ./follow.csv - failDataPath: ./err/follow.csv + failDataPath: ./err/follow batchSize: 2 type: csv csv: @@ -186,7 +186,7 @@ files: type: double - path: ./follow.csv - failDataPath: ./err/follow_test_prefix.csv + failDataPath: ./err/follow_test_prefix batchSize: 2 type: csv csv: @@ -206,7 +206,7 @@ files: type: double - path: ./glob-follow-*.csv - failDataPath: ./err/follow-glob.csv + failDataPath: ./err/follow-glob batchSize: 2 type: csv csv: @@ -223,7 +223,7 @@ files: - path: ./follow-with-header.csv - failDataPath: ./err/follow-with-header.csv + failDataPath: ./err/follow-with-header batchSize: 2 type: csv csv: @@ -236,7 +236,7 @@ files: withRanking: true - path: ./student.csv - failDataPath: ./err/student.csv + failDataPath: ./err/student batchSize: 2 type: csv csv: @@ -256,7 +256,7 @@ files: type: string - path: ./student.csv - failDataPath: ./err/student_test_prefix.csv + failDataPath: ./err/student_test_prefix batchSize: 2 type: csv csv: @@ -278,7 +278,7 @@ files: type: string - path: ./student.csv - failDataPath: ./err/student_index.csv + failDataPath: ./err/student_index batchSize: 2 type: csv csv: @@ -303,7 +303,7 @@ files: type: string - path: ./student-with-label-and-str-vid.csv - failDataPath: ./err/student_label_str_vid.csv + failDataPath: ./err/student_label_str_vid batchSize: 2 type: csv csv: @@ -327,7 +327,7 @@ files: type: string - path: ./follow.csv - failDataPath: ./err/follow_index.csv + failDataPath: ./err/follow_index batchSize: 2 limit: 3 type: csv @@ -351,7 +351,7 @@ files: index: 3 - path: ./follow-delimiter.csv - failDataPath: ./err/follow-delimiter.csv + failDataPath: ./err/follow-delimiter batchSize: 2 type: csv csv: @@ -365,7 +365,7 @@ files: withRanking: true - path: https://raw.githubusercontent.com/vesoft-inc/nebula-importer/master/examples/v2/follow.csv - failDataPath: ./err/follow_http.csv + failDataPath: ./err/follow_http batchSize: 2 limit: 3 type: csv @@ -389,7 +389,7 @@ files: index: 3 - path: ./course.csv - failDataPath: ./err/course-empty-props.csv + failDataPath: ./err/course-empty-props batchSize: 2 inOrder: true type: csv @@ -406,7 +406,7 @@ files: - name: course_no_props - path: ./course.csv - failDataPath: ./err/course-multi-empty-props.csv + failDataPath: ./err/course-multi-empty-props batchSize: 2 inOrder: true type: csv @@ -424,7 +424,7 @@ files: - name: building_no_props - path: ./course.csv - failDataPath: ./err/course-mix-empty-props.csv + failDataPath: ./err/course-mix-empty-props batchSize: 2 inOrder: true type: csv @@ -446,7 +446,7 @@ files: index: 3 - path: ./course.csv - failDataPath: ./err/course-mix-empty-props-2.csv + failDataPath: ./err/course-mix-empty-props-2 batchSize: 2 inOrder: true type: csv @@ -470,7 +470,7 @@ files: - path: ./follow.csv - failDataPath: ./err/follow-empty-props.csv + failDataPath: ./err/follow-empty-props batchSize: 2 type: csv csv: @@ -488,7 +488,7 @@ files: index: 0 - path: ./basic_type_test.csv - failDataPath: ./err/basic_type_test.csv + failDataPath: ./err/basic_type_test batchSize: 2 inOrder: true type: csv @@ -533,7 +533,7 @@ files: nullable: true - path: ./basic_type_test.csv - failDataPath: ./err/edge_basic_type_test.csv + failDataPath: ./err/edge_basic_type_test batchSize: 2 inOrder: true type: csv @@ -580,7 +580,7 @@ files: nullValue: "__NULL__" - path: ./date_test.csv - failDataPath: ./err/date_test.csv + failDataPath: ./err/date_test batchSize: 2 inOrder: true type: csv @@ -614,7 +614,7 @@ files: nullable: true - path: ./date_test.csv - failDataPath: ./err/edge_date_test.csv + failDataPath: ./err/edge_date_test batchSize: 2 inOrder: true type: csv @@ -650,7 +650,7 @@ files: nullable: true - path: ./geography_test.csv - failDataPath: ./err/geography_test.csv + failDataPath: ./err/geography_test batchSize: 2 inOrder: true type: csv @@ -684,7 +684,7 @@ files: nullable: true - path: ./geography_test.csv - failDataPath: ./err/edge_geography_test.csv + failDataPath: ./err/edge_geography_test batchSize: 2 inOrder: true type: csv diff --git a/examples/v2/example_with_working_dir.yaml b/examples/v2/example_with_working_dir.yaml index f102f780..1208706d 100644 --- a/examples/v2/example_with_working_dir.yaml +++ b/examples/v2/example_with_working_dir.yaml @@ -21,7 +21,7 @@ workingDir: ./data/ logPath: ./err/test.log files: - path: ./course.csv - failDataPath: ./err/course.csv + failDataPath: ./err/course batchSize: 2 inOrder: true type: csv diff --git a/pkg/config/config.go b/pkg/config/config.go index 7de3a45c..4db8b896 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -226,7 +226,7 @@ func (config *YAMLConfig) expandDirectoryToFiles(dir string) (err error) { var newFiles []*File for _, file := range config.Files { - err, files := file.expandFiles(dir) + files, err := file.expandFiles(dir) if err != nil { logger.Log.Errorf("error when expand file: %s", err) return err @@ -331,6 +331,11 @@ func (f *File) validateAndReset(dir, prefix string) error { f.FailDataPath = &failDataPath logger.Log.Warnf("You have not configured the failed data output file path in: %s.failDataPath, reset to tmp path: %s", prefix, *f.FailDataPath) + } else { + if !filepath.IsAbs(*f.FailDataPath) { + absPath := filepath.Join(dir, *f.FailDataPath) + f.FailDataPath = &absPath + } } } else { if !filepath.IsAbs(*f.Path) { @@ -382,8 +387,18 @@ func (f *File) validateAndReset(dir, prefix string) error { return f.Schema.validateAndReset(fmt.Sprintf("%s.schema", prefix)) } -func (f *File) expandFiles(dir string) (err error, files []*File) { +func (f *File) expandFiles(dir string) ([]*File, error) { + var files []*File if base.HasHttpPrefix(*f.Path) { + if f.FailDataPath != nil { + _, filename, err := base.ExtractFilename(*f.Path) + if err != nil { + return nil, err + } + failedDataPath := filepath.Join(*f.FailDataPath, filename) + f.FailDataPath = &failedDataPath + logger.Log.Infof("Failed data path: %v", failedDataPath) + } files = append(files, f) } else { if !filepath.IsAbs(*f.Path) { @@ -394,7 +409,7 @@ func (f *File) expandFiles(dir string) (err error, files []*File) { fileNames, err := filepath.Glob(*f.Path) if err != nil || len(fileNames) == 0 { logger.Log.Errorf("error file path: %s", *f.Path) - return err, files + return files, err } for i := range fileNames { @@ -413,7 +428,7 @@ func (f *File) expandFiles(dir string) (err error, files []*File) { } } - return err, files + return files, nil } func (c *CSVConfig) validateAndReset(prefix string) error {