fix(pkg/csv2lp): don't allow duplicate tags #19453

influxdata · Aug 31, 2020 · 638fc38 · 638fc38
1 parent c187ff2
commit 638fc38
Show file tree

Hide file tree

Showing 2 changed files with 21 additions and 3 deletions.
diff --git a/pkg/csv2lp/csv_table.go b/pkg/csv2lp/csv_table.go
@@ -328,6 +328,8 @@ func (t *CsvTable) recomputeLineProtocolColumns() {
 	t.cachedFieldValue = nil
 	t.cachedTags = nil
 	t.cachedFields = nil
+	// collect unique tag names (#19453)
+	var tags = make(map[string]*CsvTableColumn)
 
 	// having a _field column indicates fields without a line type are ignored
 	defaultIsField := t.Column(labelFieldName) == nil
@@ -353,8 +355,11 @@ func (t *CsvTable) recomputeLineProtocolColumns() {
 		case col.Label == labelFieldValue:
 			t.cachedFieldValue = col
 		case col.LinePart == linePartTag:
+			if val, found := tags[col.Label]; found {
+				log.Printf("WARNING: ignoring duplicate tag '%s' at column index %d, using column at index %d\n", col.Label, val.Index, col.Index)
+			}
 			col.escapedLabel = escapeTag(col.Label)
-			t.cachedTags = append(t.cachedTags, col)
+			tags[col.Label] = col
 		case col.LinePart == linePartField:
 			col.escapedLabel = escapeTag(col.Label)
 			t.cachedFields = append(t.cachedFields, col)
@@ -365,8 +370,12 @@ func (t *CsvTable) recomputeLineProtocolColumns() {
 			}
 		}
 	}
-	// line protocol requires sorted tags
-	if t.cachedTags != nil && len(t.cachedTags) > 0 {
+	// line protocol requires sorted unique tags
+	if len(tags) > 0 {
+		t.cachedTags = make([]*CsvTableColumn, 0, len(tags))
+		for _, v := range tags {
+			t.cachedTags = append(t.cachedTags, v)
+		}
 		sort.Slice(t.cachedTags, func(i, j int) bool {
 			return t.cachedTags[i].Label < t.cachedTags[j].Label
 		})

diff --git a/pkg/csv2lp/csv_table_test.go b/pkg/csv2lp/csv_table_test.go
@@ -255,6 +255,15 @@ func Test_CsvTableProcessing(t *testing.T) {
 			"#default cpu,yes,0,1\n#datatype ,tag,,\n_measurement,test,col1,_time\n,,,",
 			"cpu,test=yes col1=0 1",
 		},
+		{
+			"no duplicate tags", // duplicate tags are ignored, the last column wins, https://github.com/influxdata/influxdb/issues/19453
+			"#datatype,string,long,dateTime:RFC3339,dateTime:RFC3339,dateTime:RFC3339,double,string,string,string,string,string,string,string,string,string,string\n" +
+				"#group,true,true,false,false,false,false,true,true,true,true,true,true,true,true,true,true\n" +
+				"#default,_result,,,,,,,,,,,,,,,\n" +
+				",result,table,_start,_stop,_time,_value,_field,_measurement,env,host,hostname,nodename,org,result,table,url\n" +
+				",,0,2020-08-26T23:10:54.023607624Z,2020-08-26T23:15:54.023607624Z,2020-08-26T23:11:00Z,0,0.001,something,host,pod,node,host,,success,role,http://127.0.0.1:8099/metrics\n",
+			"something,env=host,host=pod,hostname=node,nodename=host,result=success,table=role,url=http://127.0.0.1:8099/metrics 0.001=0 1598483460000000000",
+		},
 	}
 
 	for _, test := range tests {