lightning: introduce param to skip CSV header parsing

br/pkg/lightning/config/config.go

@@ -484,6 +484,7 @@ type CSVConfig struct {
 	Terminator      string `toml:"terminator" json:"terminator"`
 	Null            string `toml:"null" json:"null"`
 	Header          bool   `toml:"header" json:"header"`
+	SkipHeader      bool   `toml:"skip-header" json:"skip-header"`
 	TrimLastSep     bool   `toml:"trim-last-separator" json:"trim-last-separator"`
 	NotNull         bool   `toml:"not-null" json:"not-null"`
 	BackslashEscape bool   `toml:"backslash-escape" json:"backslash-escape"`
@@ -746,6 +747,7 @@ func NewConfig() *Config {
 				Separator:       ",",
 				Delimiter:       `"`,
 				Header:          true,
+				SkipHeader:      false,
 				NotNull:         false,
 				Null:            `\N`,
 				BackslashEscape: true,

br/pkg/lightning/mydump/csv_parser.go

@@ -607,6 +607,9 @@ func (parser *CSVParser) ReadColumns() error {
 	if err != nil {
 		return errors.Trace(err)
 	}
+	if parser.cfg.SkipHeader {
+		return nil
+	}
 	parser.columns = make([]string, 0, len(columns))
 	for _, colName := range columns {
 		colName, _, err = parser.unescapeString(colName)

br/pkg/lightning/mydump/region.go

@@ -404,7 +404,9 @@ func SplitLargeFile(
 		if err = parser.ReadColumns(); err != nil {
 			return 0, nil, nil, err
 		}
-		columns = parser.Columns()
+		if !cfg.Mydumper.CSV.SkipHeader {
+			columns = parser.Columns()
+		}
 		startOffset, _ = parser.Pos()
 		endOffset = startOffset + maxRegionSize
 		if endOffset > dataFile.FileMeta.FileSize {

br/tests/lightning_config_skip_csv_header/data/mytest.testtbl-schema.sql

@@ -0,0 +1,5 @@
+CREATE TABLE testtbl (
+  id INTEGER PRIMARY KEY,
+  val1 VARCHAR(40) NOT NULL,
+  INDEX `idx_val1` (`val1`)
+);

br/tests/lightning_config_skip_csv_header/data/mytest.testtbl.csv

@@ -0,0 +1,6 @@
+aaa,bbb
+1,"aaa01"
+2,"aaa01"
+3,"aaa02"
+4,"aaa02"
+5,"aaa05"

br/tests/lightning_config_skip_csv_header/err_config.toml

@@ -0,0 +1,9 @@
+[lightning]
+check-requirements=true
+
+[mydumper.csv]
+header = true
+skip-header = false
+
+[tikv-importer]
+duplicate-resolution = 'remove'

br/tests/lightning_config_skip_csv_header/normal_config.toml

@@ -0,0 +1,9 @@
+[lightning]
+check-requirements=true
+
+[mydumper.csv]
+header = true
+skip-header = true
+
+[tikv-importer]
+duplicate-resolution = 'remove'

br/tests/lightning_config_skip_csv_header/run.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright 2023 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+mydir=$(dirname "${BASH_SOURCE[0]}")
+
+data_file="${mydir}/data/mytest.testtbl.csv"
+
+total_row_count=$( sed '1d' "${data_file}" | wc -l | xargs echo )
+
+run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
+
+console_output_file="/tmp/${TEST_NAME}.out"
+
+run_lightning --backend tidb --config "${mydir}/err_config.toml" 2>&1 | tee "${console_output_file}"
+if [[ ${PIPESTATUS[0]} -eq 0 ]]; then
+    echo "The lightning import doesn't fail as expected" >&2
+    exit 1
+fi
+
+grep -q "Lightning:Restore:ErrUnknownColumns" "${console_output_file}"
+
+# import a second time
+
+run_sql 'DROP TABLE IF EXISTS mytest.testtbl'
+
+run_lightning --backend tidb --config "${mydir}/normal_config.toml"
+
+run_sql 'SELECT * FROM mytest.testtbl'
+run_sql 'SELECT COUNT(*) FROM mytest.testtbl'
+check_contains "COUNT(*): ${total_row_count}"
+run_sql 'SELECT COUNT(*) FROM mytest.testtbl WHERE id > 0'
+check_contains "COUNT(*): ${total_row_count}"