NETOBSERV-223: filter by empty names

pkg/loki/filter.go

@@ -5,6 +5,9 @@ import (
 	"strings"
 )
 
+// remove quotes and replace * by regex any
+var valueReplacer = strings.NewReplacer(`*`, `.*`, `"`, "")
+
 type labelMatcher string
 
 const (
@@ -30,6 +33,17 @@ type labelFilter struct {
 	valueType valueType
 }
 
+// lineFilter represents a condition based on a JSON raw text match.
+type lineFilter struct {
+	key    string
+	values []lineMatch
+}
+
+type lineMatch struct {
+	value     string
+	valueType valueType
+}
+
 func stringLabelFilter(labelKey string, value string) labelFilter {
 	return labelFilter{
 		key:       labelKey,
@@ -72,10 +86,67 @@ func (f *labelFilter) writeInto(sb *strings.Builder) {
 		sb.WriteString(f.value)
 		sb.WriteString(`")`)
 	case typeRegex:
-		sb.WriteByte('`')
+		sb.WriteString("`(?i).*")
 		sb.WriteString(f.value)
-		sb.WriteByte('`')
+		sb.WriteString(".*`")
 	default:
 		panic(fmt.Sprint("wrong filter value type", int(f.valueType)))
 	}
 }
+
+// asLabelFilters transforms a lineFilter (raw text match) into a group of
+// labelFilters (attributes match)
+func (f *lineFilter) asLabelFilters() []labelFilter {
+	lfs := make([]labelFilter, 0, len(f.values))
+	for _, v := range f.values {
+		lf := labelFilter{
+			key:       f.key,
+			valueType: v.valueType,
+			value:     v.value,
+		}
+		if v.valueType == typeRegex {
+			lf.matcher = labelMatches
+		} else {
+			lf.matcher = labelEqual
+		}
+		lfs = append(lfs, lf)
+	}
+	return lfs
+}
+
+// writeInto transforms a lineFilter to its corresponding part of a LogQL query
+// under construction (contained in the provided strings.Builder)
+func (f *lineFilter) writeInto(sb *strings.Builder) {
+	for i, v := range f.values {
+		if i > 0 {
+			sb.WriteByte('|')
+		}
+		// match end of KEY + regex VALUE:
+		// if numeric, KEY":VALUE,
+		// if string KEY":"VALUE"
+		// ie 'Port' key will match both 'SrcPort":"XXX"' and 'DstPort":"XXX"
+		// VALUE can be quoted for exact match or contains * to inject regex any
+		// For numeric values, exact match is implicit
+		// 	(the trick is to match for the ending coma; it works as long as the filtered field
+		// 	is not the last one (they're in alphabetic order); a less performant alternative
+		// 	but more future-proof/less hacky could be to move that to a json filter, if needed)
+		sb.WriteString(f.key)
+		sb.WriteString(`":`)
+		switch v.valueType {
+		case typeNumber:
+			sb.WriteString(v.value)
+			// a number can be followed by } if it's the last property of a JSON document
+			sb.WriteString("[,}]")
+		case typeString, typeIP:
+			// exact matches are specified as just strings
+			sb.WriteByte('"')
+			sb.WriteString(valueReplacer.Replace(v.value))
+			sb.WriteByte('"')
+		// contains-match are specified as regular expressions
+		case typeRegex:
+			sb.WriteString(`"(?i)[^"]*`)
+			sb.WriteString(valueReplacer.Replace(v.value))
+			sb.WriteString(`.*"`)
+		}
+	}
+}

pkg/loki/flow_query.go

@@ -21,17 +21,14 @@ const (
 // can contains only alphanumeric / '-' / '_' / '.' / ',' / '"' / '*' / ':' / '/' characteres
 var filterRegexpValidation = regexp.MustCompile(`^[\w-_.,\"*:/]*$`)
 
-// remove quotes and replace * by regex any
-var valueReplacer = strings.NewReplacer(`*`, `.*`, `"`, "")
-
 // FlowQueryBuilder stores a state to build a LogQL query
 type FlowQueryBuilder struct {
 	config       *Config
 	startTime    string
 	endTime      string
 	limit        string
 	labelFilters []labelFilter
-	lineFilters  []string
+	lineFilters  []lineFilter
 	jsonFilters  [][]labelFilter
 }
 
@@ -123,44 +120,33 @@ func (q *FlowQueryBuilder) addLabelRegex(key string, values []string) {
 }
 
 func (q *FlowQueryBuilder) addLineFilters(key string, values []string) {
-	regexStr := strings.Builder{}
-	for i, value := range values {
-		if i > 0 {
-			regexStr.WriteByte('|')
-		}
-		// match end of KEY + regex VALUE:
-		// if numeric, KEY":VALUE,
-		// if string KEY":"VALUE"
-		// ie 'Port' key will match both 'SrcPort":"XXX"' and 'DstPort":"XXX"
-		// VALUE can be quoted for exact match or contains * to inject regex any
-		// For numeric values, exact match is implicit
-		// 	(the trick is to match for the ending coma; it works as long as the filtered field
-		// 	is not the last one (they're in alphabetic order); a less performant alternative
-		// 	but more future-proof/less hacky could be to move that to a json filter, if needed)
-		regexStr.WriteString(key)
-		regexStr.WriteString(`":`)
-		if fields.IsNumeric(key) {
-			regexStr.WriteString(value)
-			regexStr.WriteByte(',')
-		} else {
-			regexStr.WriteByte('"')
-			// match start any if not quoted
-			// and case insensitive
-			if !strings.HasPrefix(value, `"`) {
-				regexStr.WriteString("(?i)[^\"]*")
-			}
-			//inject value with regex
-			regexStr.WriteString(valueReplacer.Replace(value))
-			// match end any if not quoted
-			if !strings.HasSuffix(value, `"`) {
-				regexStr.WriteString(".*")
-			}
-			regexStr.WriteByte('"')
+	if len(values) == 0 {
+		return
+	}
+	lf := lineFilter{
+		key: key,
+	}
+	isNumeric := fields.IsNumeric(key)
+	emptyMatches := false
+	for _, value := range values {
+		lm := lineMatch{}
+		switch {
+		case isNumeric:
+			lm = lineMatch{valueType: typeNumber, value: value}
+		case isExactMatch(value):
+			lm = lineMatch{valueType: typeString, value: trimExactMatch(value)}
+			emptyMatches = emptyMatches || len(lm.value) == 0
+		default:
+			lm = lineMatch{valueType: typeRegex, value: value}
 		}
+		lf.values = append(lf.values, lm)
 	}
-
-	if regexStr.Len() > 0 {
-		q.lineFilters = append(q.lineFilters, regexStr.String())
+	// if there is at least an empty exact match, there is no uniform/safe way to filter by text,
+	// so we should use JSON label matchers instead of text line matchers
+	if emptyMatches {
+		q.jsonFilters = append(q.jsonFilters, lf.asLabelFilters())
+	} else {
+		q.lineFilters = append(q.lineFilters, lf)
 	}
 }
 
@@ -195,7 +181,7 @@ func (q *FlowQueryBuilder) appendLabels(sb *strings.Builder) {
 func (q *FlowQueryBuilder) appendLineFilters(sb *strings.Builder) {
 	for _, lf := range q.lineFilters {
 		sb.WriteString("|~`")
-		sb.WriteString(lf)
+		lf.writeInto(sb)
 		sb.WriteByte('`')
 	}
 }

pkg/server/server_flows_test.go

@@ -38,19 +38,19 @@ func TestLokiFiltering(t *testing.T) {
 		inputPath: "?filters=" + url.QueryEscape("Proto=6&SrcK8S_Name=test"),
 		outputQueryParts: []string{
 			"?query={app=\"netobserv-flowcollector\"}",
-			"|~`Proto\":6,`",
+			"|~`Proto\":6[,}]`",
 			"|~`SrcK8S_Name\":\"(?i)[^\"]*test.*\"`",
 		},
 	}, {
 		inputPath: "?filters=" + url.QueryEscape("Proto=6|SrcK8S_Name=test"),
 		outputQueries: []string{
-			"?query={app=\"netobserv-flowcollector\"}|~`Proto\":6,`",
+			"?query={app=\"netobserv-flowcollector\"}|~`Proto\":6[,}]`",
 			"?query={app=\"netobserv-flowcollector\"}|~`SrcK8S_Name\":\"(?i)[^\"]*test.*\"`",
 		},
 	}, {
 		inputPath: "?filters=" + url.QueryEscape("Proto=6|SrcK8S_Name=test") + "&reporter=source",
 		outputQueries: []string{
-			"?query={app=\"netobserv-flowcollector\",FlowDirection=\"1\"}|~`Proto\":6,`",
+			"?query={app=\"netobserv-flowcollector\",FlowDirection=\"1\"}|~`Proto\":6[,}]`",
 			"?query={app=\"netobserv-flowcollector\",FlowDirection=\"1\"}|~`SrcK8S_Name\":\"(?i)[^\"]*test.*\"`",
 		},
 	}, {
@@ -71,7 +71,7 @@ func TestLokiFiltering(t *testing.T) {
 	}, {
 		inputPath: "?filters=" + url.QueryEscape("SrcPort=8080&SrcAddr=10.128.0.1&SrcK8S_Namespace=default"),
 		outputQueries: []string{
-			"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}|~`SrcPort\":8080,`|json|SrcAddr=ip(\"10.128.0.1\")",
+			"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}|~`SrcPort\":8080[,}]`|json|SrcAddr=ip(\"10.128.0.1\")",
 		},
 	}, {
 		inputPath: "?filters=" + url.QueryEscape("SrcAddr=10.128.0.1&DstAddr=10.128.0.2"),
@@ -90,14 +90,14 @@ func TestLokiFiltering(t *testing.T) {
 		outputQueries: []string{
 			"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*default.*\"}",
 			"?query={app=\"netobserv-flowcollector\"}|json|SrcAddr=ip(\"10.128.0.1\")",
-			"?query={app=\"netobserv-flowcollector\"}|~`SrcPort\":8080,`",
+			"?query={app=\"netobserv-flowcollector\"}|~`SrcPort\":8080[,}]`",
 		},
 	}, {
 		inputPath: "?filters=" + url.QueryEscape("SrcPort=8080|SrcAddr=10.128.0.1|SrcK8S_Namespace=default") + "&reporter=destination",
 		outputQueries: []string{
 			"?query={app=\"netobserv-flowcollector\",FlowDirection=\"0\",SrcK8S_Namespace=~\"(?i).*default.*\"}",
 			"?query={app=\"netobserv-flowcollector\",FlowDirection=\"0\"}|json|SrcAddr=ip(\"10.128.0.1\")",
-			"?query={app=\"netobserv-flowcollector\",FlowDirection=\"0\"}|~`SrcPort\":8080,`",
+			"?query={app=\"netobserv-flowcollector\",FlowDirection=\"0\"}|~`SrcPort\":8080[,}]`",
 		},
 	}, {
 		inputPath:     "?startTime=1640991600",
@@ -125,27 +125,65 @@ func TestLokiFiltering(t *testing.T) {
 		inputPath: "?filters=" + url.QueryEscape("Port=8080&K8S_Name=test"),
 		outputQueryParts: []string{
 			"?query={app=\"netobserv-flowcollector\"}",
-			"|~`Port\":8080,`",
+			"|~`Port\":8080[,}]`",
 			"|~`K8S_Name\":\"(?i)[^\"]*test.*\"`",
 		},
 	}, {
 		inputPath: "?filters=" + url.QueryEscape("Port=8080|K8S_Name=test"),
 		outputQueries: []string{
 			"?query={app=\"netobserv-flowcollector\"}|~`K8S_Name\":\"(?i)[^\"]*test.*\"`",
-			"?query={app=\"netobserv-flowcollector\"}|~`Port\":8080,`",
+			"?query={app=\"netobserv-flowcollector\"}|~`Port\":8080[,}]`",
 		},
 	}, {
 		inputPath: "?filters=" + url.QueryEscape("Port=8080&SrcK8S_Namespace=test|Port=8080&DstK8S_Namespace=test"),
 		outputQueries: []string{
-			"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*test.*\"}|~`Port\":8080,`",
-			"?query={app=\"netobserv-flowcollector\",DstK8S_Namespace=~\"(?i).*test.*\"}|~`Port\":8080,`",
+			"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*test.*\"}|~`Port\":8080[,}]`",
+			"?query={app=\"netobserv-flowcollector\",DstK8S_Namespace=~\"(?i).*test.*\"}|~`Port\":8080[,}]`",
 		},
 	}, {
 		inputPath: "?filters=" + url.QueryEscape("Port=8080|SrcK8S_Namespace=test|DstK8S_Namespace=test"),
 		outputQueries: []string{
 			"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=~\"(?i).*test.*\"}",
 			"?query={app=\"netobserv-flowcollector\",DstK8S_Namespace=~\"(?i).*test.*\"}",
-			"?query={app=\"netobserv-flowcollector\"}|~`Port\":8080,`",
+			"?query={app=\"netobserv-flowcollector\"}|~`Port\":8080[,}]`",
+		},
+	}, {
+		inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Namespace=""&DstPort=70`),
+		outputQueries: []string{
+			"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=\"\"}|~`DstPort\":70[,}]`",
+		},
+	}, {
+		inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Name=""&DstPort=70`),
+		outputQueries: []string{
+			"?query={app=\"netobserv-flowcollector\"}|~`DstPort\":70[,}]`|json|SrcK8S_Name=\"\"",
+		},
+	}, {
+		inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Name="",foo&DstK8S_Name="hello"`),
+		outputQueries: []string{
+			"?query={app=\"netobserv-flowcollector\"}|~`DstK8S_Name\":\"hello\"`|json|SrcK8S_Name=\"\"+or+SrcK8S_Name=~`(?i).*foo.*`",
+		},
+	}, {
+		inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Namespace=""|DstPort=70`),
+		outputQueries: []string{
+			"?query={app=\"netobserv-flowcollector\",SrcK8S_Namespace=\"\"}",
+			"?query={app=\"netobserv-flowcollector\"}|~`DstPort\":70[,}]`",
+		},
+	}, {
+		inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Name=""|DstPort=70`),
+		outputQueries: []string{
+			"?query={app=\"netobserv-flowcollector\"}|~`DstPort\":70[,}]`",
+			"?query={app=\"netobserv-flowcollector\"}|json|SrcK8S_Name=\"\"",
+		},
+	}, {
+		inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Name="",foo|DstK8S_Name="hello"`),
+		outputQueries: []string{
+			"?query={app=\"netobserv-flowcollector\"}|~`DstK8S_Name\":\"hello\"`",
+			"?query={app=\"netobserv-flowcollector\"}|json|SrcK8S_Name=\"\"+or+SrcK8S_Name=~`(?i).*foo.*`",
+		},
+	}, {
+		inputPath: "?filters=" + url.QueryEscape(`SrcK8S_Type="","Pod"`),
+		outputQueries: []string{
+			"?query={app=\"netobserv-flowcollector\"}|json|SrcK8S_Type=\"\"+or+SrcK8S_Type=\"Pod\"",
 		},
 	}}