lightning: support column starts with slash/number/non-ascii for parq…

…uet (pingcap#36985) (pingcap#37092)

close pingcap#36980, close pingcap#36981

br/pkg/lightning/mydump/parquet_parser.go

@@ -186,11 +186,10 @@ func NewParquetParser(
 
 	columns := make([]string, 0, len(reader.Footer.Schema)-1)
 	columnMetas := make([]*parquet.SchemaElement, 0, len(reader.Footer.Schema)-1)
-	for _, c := range reader.SchemaHandler.SchemaElements {
+	for i, c := range reader.SchemaHandler.SchemaElements {
 		if c.GetNumChildren() == 0 {
-			// NOTE: the SchemaElement.Name is capitalized, SchemaHandler.Infos.ExName is the raw column name
-			// though in this context, there is no difference between these two fields
-			columns = append(columns, strings.ToLower(c.Name))
+			// we need to use the raw name, SchemaElement.Name might be prefixed with PARGO_PERFIX_
+			columns = append(columns, strings.ToLower(reader.SchemaHandler.GetExName(i)))
 			// transfer old ConvertedType to LogicalType
 			columnMeta := c
 			if c.ConvertedType != nil && c.LogicalType == nil {

br/tests/lightning_parquet/db.sql

@@ -217,3 +217,11 @@ CREATE TABLE `warehouse` (
   PRIMARY KEY (`w_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;
 /*!40101 SET character_set_client = @saved_cs_client */;
+
+DROP TABLE IF EXISTS `special_col_name`;
+CREATE TABLE `special_col_name` (
+    `c1` varchar(128) DEFAULT NULL,
+    `_c2` timestamp NULL DEFAULT NULL,
+    `123_c3` timestamp NULL DEFAULT NULL,
+    `中_c4` timestamp NULL DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;

br/tests/lightning_parquet/run.sh

@@ -40,6 +40,7 @@ for BACKEND in local importer tidb; do
     check_row_count orders 100
     check_row_count stock 50
     check_row_count warehouse 1
+    check_row_count special_col_name 1
 
     run_sql 'select sum(c_id) from test.customer;'
     check_contains "sum(c_id): 210"