Add a flag for including only certain fields

cmd/ecslog/main.go

@@ -48,6 +48,8 @@ var flagColorScheme = flags.StringP("color-scheme", "c", "default",
 	"Color scheme to use, if colorizing.") // hidden
 var flagExcludeFields = flags.StringP("exclude-fields", "x", "",
 	"Comma-separated list of fields to exclude from the output.")
+var flagIncludeFields = flags.StringP("include-fields", "i", "",
+	"Comma-separated list of fields to include in the output.")
 
 func printError(msg string) {
 	fmt.Fprintf(os.Stderr, "ecslog: error: %s\n", msg)
@@ -144,6 +146,7 @@ func main() {
 
 	commaSplitter := regexp.MustCompile(`\s*,\s*`)
 	excludeFields := commaSplitter.Split(*flagExcludeFields, -1)
+	includeFields := commaSplitter.Split(*flagIncludeFields, -1)
 
 	ecsLenient := false
 	if cfgECSLenient, ok := cfg.GetBool("ecsLenient"); ok {
@@ -161,6 +164,7 @@ func main() {
 		formatName,
 		maxLineLen,
 		excludeFields,
+		includeFields,
 		ecsLenient,
 		timestampShowDiff,
 	)

cmd/ecslog/main_test.go

@@ -96,9 +96,16 @@ var mainTestCases = []mainTestCase{
 		regexp.MustCompile(`^\n    foo: "bar"\n    spam: "eggs"\n$`),
 		nil,
 	},
+	{
+		"ecslog --include-fields ...all the title fields...",
+		[]string{"ecslog", "-i", "foo", "./testdata/exclude-fields.log"},
+		0,
+		regexp.MustCompile(`^\[2021-01-19T22:51:12.142Z\]  INFO: hi\n    foo: "bar"\n$`),
+		nil,
+	},
 }
 
-func TestMain(t *testing.T) {
+func TestFlags(t *testing.T) {
 	for _, tc := range mainTestCases {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Logf("-- `ecslog` test case %q\n", tc.name)

internal/ecslog/ecslog.go

@@ -31,6 +31,7 @@ type Renderer struct {
 	formatter         Formatter
 	maxLineLen        int
 	excludeFields     []string
+	includeFields     []string
 	ecsLenient        bool
 	timestampShowDiff bool
 	levelFilter       string
@@ -59,7 +60,7 @@ type Renderer struct {
 //   fields to exist.
 // - `timestampShowDiff` is a bool indicating if the @timestamp diff from the
 //   preceding log record should be styled.
-func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int, excludeFields []string, ecsLenient, timestampShowDiff bool) (*Renderer, error) {
+func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int, excludeFields, includeFields []string, ecsLenient, timestampShowDiff bool) (*Renderer, error) {
 	// Get appropriate "painter" for terminal coloring.
 	var painter *ansipainter.ANSIPainter
 	if shouldColorize == "auto" {
@@ -116,6 +117,7 @@ func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int,
 		formatter:         formatter,
 		maxLineLen:        maxLineLen,
 		excludeFields:     excludeFields,
+		includeFields:     includeFields,
 		ecsLenient:        ecsLenient,
 		timestampShowDiff: timestampShowDiff,
 

internal/ecslog/ecslog_test.go

@@ -19,6 +19,7 @@ type renderFileTestCase struct {
 	levelFilter       string
 	kqlFilter         string
 	timestampShowDiff bool
+	includeFields     []string
 
 	input  string
 	output string
@@ -28,41 +29,41 @@ var renderFileTestCases = []renderFileTestCase{
 	// Non-ecs-logging lines
 	{
 		"empty object",
-		"no", "", "default", false, "", "", false,
+		"no", "", "default", false, "", "", false, []string{},
 		"{}",
 		"{}\n",
 	},
 
 	// Basics
 	{
 		"basic",
-		"no", "", "default", false, "", "", false,
+		"no", "", "default", false, "", "", false, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
 		"[2021-01-19T22:51:12.142Z]  INFO: hi\n",
 	},
 	{
 		"basic, extra var",
-		"no", "", "default", false, "", "", false,
+		"no", "", "default", false, "", "", false, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":"bar"}`,
 		"[2021-01-19T22:51:12.142Z]  INFO: hi\n    foo: \"bar\"\n",
 	},
 	{
 		"no message is allowed",
-		"no", "", "default", false, "", "", false,
+		"no", "", "default", false, "", "", false, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"foo":"bar"}`,
 		"[2021-01-19T22:51:12.142Z]  INFO:\n    foo: \"bar\"\n",
 	},
 
 	// Coloring
 	{
 		"coloring 1",
-		"yes", "default", "default", false, "", "", false,
+		"yes", "default", "default", false, "", "", false, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
 		"[2021-01-19T22:51:12.142Z] \x1b[32m INFO\x1b[0m: \x1b[36mhi\x1b[0m\n",
 	},
 	{
 		"timestamp diff highlighting 1",
-		"yes", "default", "default", false, "", "", true,
+		"yes", "default", "default", false, "", "", true, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}
 {"log.level":"info","@timestamp":"2021-01-19T22:51:23.456Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
 		"[2021-01-19T22:51:12.142Z] \x1b[32m INFO\x1b[0m: \x1b[36mhi\x1b[0m\n" +
@@ -72,54 +73,79 @@ var renderFileTestCases = []renderFileTestCase{
 	// KQL filtering
 	{
 		"kql filtering, yep",
-		"no", "", "default", false, "", "foo:bar", false,
+		"no", "", "default", false, "", "foo:bar", false, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":"bar"}`,
 		"[2021-01-19T22:51:12.142Z]  INFO: hi\n    foo: \"bar\"\n",
 	},
 	{
 		"kql filtering, nope",
-		"no", "", "default", false, "", "foo:baz", false,
+		"no", "", "default", false, "", "foo:baz", false, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":"bar"}`,
 		"",
 	},
 	{
 		"kql filtering, log.level range query, yep",
-		"no", "", "default", false, "", "log.level > debug", false,
+		"no", "", "default", false, "", "log.level > debug", false, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
 		"[2021-01-19T22:51:12.142Z]  INFO: hi\n",
 	},
 	{
 		"kql filtering, log.level range query, nope",
-		"no", "", "default", false, "", "log.level > warn", false,
+		"no", "", "default", false, "", "log.level > warn", false, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
 		"",
 	},
 
 	// ecsLenient
 	{
 		"lenient: missing log.level",
-		"no", "", "default", true, "", "", false,
+		"no", "", "default", true, "", "", false, []string{},
 		`{"@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
 		"[2021-01-19T22:51:12.142Z] : hi\n",
 	},
 	{
 		"lenient: missing @timestamp",
-		"no", "", "default", true, "", "", false,
+		"no", "", "default", true, "", "", false, []string{},
 		`{"log.level":"info","ecs":{"version":"1.5.0"},"message":"hi"}`,
 		" INFO: hi\n",
 	},
 	{
 		"lenient: missing ecs.version",
-		"no", "", "default", true, "", "", false,
+		"no", "", "default", true, "", "", false, []string{},
 		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","message":"hi"}`,
 		"[2021-01-19T22:51:12.142Z]  INFO: hi\n",
 	},
 	{
 		"lenient: @timestamp only",
-		"no", "", "default", true, "", "", false,
+		"no", "", "default", true, "", "", false, []string{},
 		`{"@timestamp":"2021-01-19T22:51:12.142Z","message":"hi"}`,
 		"[2021-01-19T22:51:12.142Z] : hi\n",
 	},
+	// Include fields
+	{
+		"include fields: only log",
+		"no", "", "default", true, "", "", false, []string{"log"},
+		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","log.origin":{"foo":"bar","file.name":"main.go","file.line":"42"}}`,
+		"[2021-01-19T22:51:12.142Z]  INFO: hi\n    log.origin: {\n        \"foo\": \"bar\",\n        \"file.name\": \"main.go\",\n        \"file.line\": \"42\"\n    }\n",
+	},
+	{
+		"include fields: only log.origin.file",
+		"no", "", "default", true, "", "", false, []string{"log.origin.file"},
+		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","log.origin":{"foo":"bar","file.name":"main.go","file.line":"42"}}`,
+		"[2021-01-19T22:51:12.142Z]  INFO: hi\n    log.origin: {\n        \"file.name\": \"main.go\",\n        \"file.line\": \"42\"\n    }\n",
+	},
+	{
+		"include fields: only log.origin.file.name",
+		"no", "", "default", true, "", "", false, []string{"log.origin.file.name"},
+		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","log.origin":{"foo":"bar","file.name":"main.go","file.line":"42"}}`,
+		"[2021-01-19T22:51:12.142Z]  INFO: hi\n    log.origin: {\n        \"file.name\": \"main.go\"\n    }\n",
+	},
+	{
+		"include fields: only foo",
+		"no", "", "default", true, "", "", false, []string{"foo"},
+		`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":0,"bar":1}`,
+		"[2021-01-19T22:51:12.142Z]  INFO: hi\n    foo: 0\n",
+	},
 }
 
 func TestRenderFile(t *testing.T) {
@@ -131,6 +157,7 @@ func TestRenderFile(t *testing.T) {
 				tc.formatName,
 				-1,
 				[]string{},
+				tc.includeFields,
 				tc.ecsLenient,
 				tc.timestampShowDiff,
 			)

internal/ecslog/formatters.go

@@ -32,12 +32,16 @@ func (f *defaultFormatter) formatRecord(r *Renderer, rec *fastjson.Value, b *str
 	//   fields as a HTTP request/response text representation
 	obj := rec.GetObject()
 	obj.Visit(func(k []byte, v *fastjson.Value) {
+		includeFields, ok := anyIsPrefix(r.includeFields, string(k))
+		if !ok {
+			return
+		}
 		b.WriteString("\n    ")
 		r.painter.Paint(b, "extraField")
 		b.Write(k)
 		r.painter.Reset(b)
 		b.WriteString(": ")
-		formatJSONValue(b, v, "    ", "    ", r.painter, false)
+		formatJSONValue(b, v, "    ", "    ", r.painter, false, includeFields)
 	})
 }
 
@@ -58,6 +62,10 @@ func (f *compactFormatter) formatRecord(r *Renderer, rec *fastjson.Value, b *str
 	//   fields as a HTTP request/response text representation
 	obj := rec.GetObject()
 	obj.Visit(func(k []byte, v *fastjson.Value) {
+		includeFields, ok := anyIsPrefix(r.includeFields, string(k))
+		if !ok {
+			return
+		}
 		b.WriteString("\n    ")
 		r.painter.Paint(b, "extraField")
 		b.Write(k)
@@ -72,9 +80,9 @@ func (f *compactFormatter) formatRecord(r *Renderer, rec *fastjson.Value, b *str
 		vStr := v.String()
 		// 80 (quotable width) - 8 (indentation) - length of `k` - len(": ")
 		if len(vStr) < 80-8-len(k)-2 {
-			formatJSONValue(b, v, "    ", "    ", r.painter, true)
+			formatJSONValue(b, v, "    ", "    ", r.painter, true, includeFields)
 		} else {
-			formatJSONValue(b, v, "    ", "    ", r.painter, false)
+			formatJSONValue(b, v, "    ", "    ", r.painter, false, includeFields)
 		}
 	})
 }
@@ -294,7 +302,7 @@ func formatDefaultTitleLine(r *Renderer, rec *fastjson.Value, b *strings.Builder
 	}
 }
 
-func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent string, painter *ansipainter.ANSIPainter, compact bool) {
+func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent string, painter *ansipainter.ANSIPainter, compact bool, includeFields []string) {
 	var i uint
 
 	switch v.Type() {
@@ -303,6 +311,10 @@ func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent s
 		obj := v.GetObject()
 		i = 0
 		obj.Visit(func(subk []byte, subv *fastjson.Value) {
+			nestedIncludeFields, ok := anyIsPrefix(includeFields, string(subk))
+			if !ok {
+				return
+			}
 			if i != 0 {
 				b.WriteByte(',')
 				if compact {
@@ -320,7 +332,7 @@ func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent s
 			b.WriteByte('"')
 			painter.Reset(b)
 			b.WriteString(": ")
-			formatJSONValue(b, subv, currIndent+indent, indent, painter, compact)
+			formatJSONValue(b, subv, currIndent+indent, indent, painter, compact, nestedIncludeFields)
 			i++
 		})
 		if !compact {
@@ -342,7 +354,7 @@ func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent s
 				b.WriteString(currIndent)
 				b.WriteString(indent)
 			}
-			formatJSONValue(b, subv, currIndent+indent, indent, painter, compact)
+			formatJSONValue(b, subv, currIndent+indent, indent, painter, compact, includeFields)
 		}
 		if !compact {
 			b.WriteByte('\n')
@@ -437,3 +449,29 @@ var formatterFromName = map[string]Formatter{
 	"simple":  &simpleFormatter{},
 	"compact": &compactFormatter{},
 }
+
+// returns whether any of the include items is a prefix of key, along with the postfix (if any)
+func anyIsPrefix(includes []string, key string) ([]string, bool) {
+	if len(includes) == 0 || (len(includes) == 1 && includes[0] == "") {
+		return includes, true
+	}
+	subKeys := strings.Split(key, ".")
+	for _, include := range includes {
+		match := true
+		includeSubKeys := strings.Split(include, ".")
+		for i, subKey := range subKeys {
+			if len(includeSubKeys) > i && includeSubKeys[i] != subKey {
+				match = false
+				break
+			}
+		}
+		if match {
+			if len(includeSubKeys) > len(subKeys) {
+				// we didn't match the whole key, return the reminder to match in the next recursion step
+				return []string{strings.Join(includeSubKeys[len(subKeys):], ".")}, true
+			}
+			return []string{}, true
+		}
+	}
+	return includes, false
+}