diff --git a/cmd/ecslog/main.go b/cmd/ecslog/main.go index c3eaa18..77c8e4a 100644 --- a/cmd/ecslog/main.go +++ b/cmd/ecslog/main.go @@ -48,6 +48,8 @@ var flagColorScheme = flags.StringP("color-scheme", "c", "default", "Color scheme to use, if colorizing.") // hidden var flagExcludeFields = flags.StringP("exclude-fields", "x", "", "Comma-separated list of fields to exclude from the output.") +var flagIncludeFields = flags.StringP("include-fields", "i", "", + "Comma-separated list of fields to include in the output.") func printError(msg string) { fmt.Fprintf(os.Stderr, "ecslog: error: %s\n", msg) @@ -144,6 +146,7 @@ func main() { commaSplitter := regexp.MustCompile(`\s*,\s*`) excludeFields := commaSplitter.Split(*flagExcludeFields, -1) + includeFields := commaSplitter.Split(*flagIncludeFields, -1) ecsLenient := false if cfgECSLenient, ok := cfg.GetBool("ecsLenient"); ok { @@ -161,6 +164,7 @@ func main() { formatName, maxLineLen, excludeFields, + includeFields, ecsLenient, timestampShowDiff, ) diff --git a/cmd/ecslog/main_test.go b/cmd/ecslog/main_test.go index 6e0425c..11ef8ab 100644 --- a/cmd/ecslog/main_test.go +++ b/cmd/ecslog/main_test.go @@ -96,9 +96,16 @@ var mainTestCases = []mainTestCase{ regexp.MustCompile(`^\n foo: "bar"\n spam: "eggs"\n$`), nil, }, + { + "ecslog --include-fields foo", + []string{"ecslog", "-i", "foo", "./testdata/exclude-fields.log"}, + 0, + regexp.MustCompile(`^\[2021-01-19T22:51:12.142Z\] INFO: hi\n foo: "bar"\n$`), + nil, + }, } -func TestMain(t *testing.T) { +func TestFlags(t *testing.T) { for _, tc := range mainTestCases { t.Run(tc.name, func(t *testing.T) { t.Logf("-- `ecslog` test case %q\n", tc.name) diff --git a/internal/ecslog/ecslog.go b/internal/ecslog/ecslog.go index 2b655db..336feb3 100644 --- a/internal/ecslog/ecslog.go +++ b/internal/ecslog/ecslog.go @@ -31,6 +31,7 @@ type Renderer struct { formatter Formatter maxLineLen int excludeFields []string + includeFields []string ecsLenient bool timestampShowDiff bool levelFilter string @@ -59,7 +60,7 @@ type Renderer struct { // fields to exist. // - `timestampShowDiff` is a bool indicating if the @timestamp diff from the // preceding log record should be styled. -func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int, excludeFields []string, ecsLenient, timestampShowDiff bool) (*Renderer, error) { +func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int, excludeFields, includeFields []string, ecsLenient, timestampShowDiff bool) (*Renderer, error) { // Get appropriate "painter" for terminal coloring. var painter *ansipainter.ANSIPainter if shouldColorize == "auto" { @@ -116,6 +117,7 @@ func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int, formatter: formatter, maxLineLen: maxLineLen, excludeFields: excludeFields, + includeFields: includeFields, ecsLenient: ecsLenient, timestampShowDiff: timestampShowDiff, diff --git a/internal/ecslog/ecslog_test.go b/internal/ecslog/ecslog_test.go index dad6930..477b2f7 100644 --- a/internal/ecslog/ecslog_test.go +++ b/internal/ecslog/ecslog_test.go @@ -19,6 +19,7 @@ type renderFileTestCase struct { levelFilter string kqlFilter string timestampShowDiff bool + includeFields []string input string output string @@ -28,7 +29,7 @@ var renderFileTestCases = []renderFileTestCase{ // Non-ecs-logging lines { "empty object", - "no", "", "default", false, "", "", false, + "no", "", "default", false, "", "", false, []string{}, "{}", "{}\n", }, @@ -36,19 +37,19 @@ var renderFileTestCases = []renderFileTestCase{ // Basics { "basic", - "no", "", "default", false, "", "", false, + "no", "", "default", false, "", "", false, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`, "[2021-01-19T22:51:12.142Z] INFO: hi\n", }, { "basic, extra var", - "no", "", "default", false, "", "", false, + "no", "", "default", false, "", "", false, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":"bar"}`, "[2021-01-19T22:51:12.142Z] INFO: hi\n foo: \"bar\"\n", }, { "no message is allowed", - "no", "", "default", false, "", "", false, + "no", "", "default", false, "", "", false, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"foo":"bar"}`, "[2021-01-19T22:51:12.142Z] INFO:\n foo: \"bar\"\n", }, @@ -56,13 +57,13 @@ var renderFileTestCases = []renderFileTestCase{ // Coloring { "coloring 1", - "yes", "default", "default", false, "", "", false, + "yes", "default", "default", false, "", "", false, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`, "[2021-01-19T22:51:12.142Z] \x1b[32m INFO\x1b[0m: \x1b[36mhi\x1b[0m\n", }, { "timestamp diff highlighting 1", - "yes", "default", "default", false, "", "", true, + "yes", "default", "default", false, "", "", true, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"} {"log.level":"info","@timestamp":"2021-01-19T22:51:23.456Z","ecs":{"version":"1.5.0"},"message":"hi"}`, "[2021-01-19T22:51:12.142Z] \x1b[32m INFO\x1b[0m: \x1b[36mhi\x1b[0m\n" + @@ -72,25 +73,25 @@ var renderFileTestCases = []renderFileTestCase{ // KQL filtering { "kql filtering, yep", - "no", "", "default", false, "", "foo:bar", false, + "no", "", "default", false, "", "foo:bar", false, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":"bar"}`, "[2021-01-19T22:51:12.142Z] INFO: hi\n foo: \"bar\"\n", }, { "kql filtering, nope", - "no", "", "default", false, "", "foo:baz", false, + "no", "", "default", false, "", "foo:baz", false, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":"bar"}`, "", }, { "kql filtering, log.level range query, yep", - "no", "", "default", false, "", "log.level > debug", false, + "no", "", "default", false, "", "log.level > debug", false, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`, "[2021-01-19T22:51:12.142Z] INFO: hi\n", }, { "kql filtering, log.level range query, nope", - "no", "", "default", false, "", "log.level > warn", false, + "no", "", "default", false, "", "log.level > warn", false, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`, "", }, @@ -98,28 +99,53 @@ var renderFileTestCases = []renderFileTestCase{ // ecsLenient { "lenient: missing log.level", - "no", "", "default", true, "", "", false, + "no", "", "default", true, "", "", false, []string{}, `{"@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`, "[2021-01-19T22:51:12.142Z] : hi\n", }, { "lenient: missing @timestamp", - "no", "", "default", true, "", "", false, + "no", "", "default", true, "", "", false, []string{}, `{"log.level":"info","ecs":{"version":"1.5.0"},"message":"hi"}`, " INFO: hi\n", }, { "lenient: missing ecs.version", - "no", "", "default", true, "", "", false, + "no", "", "default", true, "", "", false, []string{}, `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","message":"hi"}`, "[2021-01-19T22:51:12.142Z] INFO: hi\n", }, { "lenient: @timestamp only", - "no", "", "default", true, "", "", false, + "no", "", "default", true, "", "", false, []string{}, `{"@timestamp":"2021-01-19T22:51:12.142Z","message":"hi"}`, "[2021-01-19T22:51:12.142Z] : hi\n", }, + // Include fields + { + "include fields: only log", + "no", "", "default", true, "", "", false, []string{"log"}, + `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","log.origin":{"foo":"bar","file.name":"main.go","file.line":"42"}}`, + "[2021-01-19T22:51:12.142Z] INFO: hi\n log.origin: {\n \"foo\": \"bar\",\n \"file.name\": \"main.go\",\n \"file.line\": \"42\"\n }\n", + }, + { + "include fields: only log.origin.file", + "no", "", "default", true, "", "", false, []string{"log.origin.file"}, + `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","log.origin":{"foo":"bar","file.name":"main.go","file.line":"42"}}`, + "[2021-01-19T22:51:12.142Z] INFO: hi\n log.origin: {\n \"file.name\": \"main.go\",\n \"file.line\": \"42\"\n }\n", + }, + { + "include fields: only log.origin.file.name", + "no", "", "default", true, "", "", false, []string{"log.origin.file.name"}, + `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","log.origin":{"foo":"bar","file.name":"main.go","file.line":"42"}}`, + "[2021-01-19T22:51:12.142Z] INFO: hi\n log.origin: {\n \"file.name\": \"main.go\"\n }\n", + }, + { + "include fields: only foo", + "no", "", "default", true, "", "", false, []string{"foo"}, + `{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":0,"bar":1}`, + "[2021-01-19T22:51:12.142Z] INFO: hi\n foo: 0\n", + }, } func TestRenderFile(t *testing.T) { @@ -131,6 +157,7 @@ func TestRenderFile(t *testing.T) { tc.formatName, -1, []string{}, + tc.includeFields, tc.ecsLenient, tc.timestampShowDiff, ) diff --git a/internal/ecslog/formatters.go b/internal/ecslog/formatters.go index f5b9b86..5f60bd4 100644 --- a/internal/ecslog/formatters.go +++ b/internal/ecslog/formatters.go @@ -32,12 +32,16 @@ func (f *defaultFormatter) formatRecord(r *Renderer, rec *fastjson.Value, b *str // fields as a HTTP request/response text representation obj := rec.GetObject() obj.Visit(func(k []byte, v *fastjson.Value) { + includeFields, ok := anyIsPrefix(r.includeFields, string(k)) + if !ok { + return + } b.WriteString("\n ") r.painter.Paint(b, "extraField") b.Write(k) r.painter.Reset(b) b.WriteString(": ") - formatJSONValue(b, v, " ", " ", r.painter, false) + formatJSONValue(b, v, " ", " ", r.painter, false, includeFields) }) } @@ -58,6 +62,10 @@ func (f *compactFormatter) formatRecord(r *Renderer, rec *fastjson.Value, b *str // fields as a HTTP request/response text representation obj := rec.GetObject() obj.Visit(func(k []byte, v *fastjson.Value) { + includeFields, ok := anyIsPrefix(r.includeFields, string(k)) + if !ok { + return + } b.WriteString("\n ") r.painter.Paint(b, "extraField") b.Write(k) @@ -72,9 +80,9 @@ func (f *compactFormatter) formatRecord(r *Renderer, rec *fastjson.Value, b *str vStr := v.String() // 80 (quotable width) - 8 (indentation) - length of `k` - len(": ") if len(vStr) < 80-8-len(k)-2 { - formatJSONValue(b, v, " ", " ", r.painter, true) + formatJSONValue(b, v, " ", " ", r.painter, true, includeFields) } else { - formatJSONValue(b, v, " ", " ", r.painter, false) + formatJSONValue(b, v, " ", " ", r.painter, false, includeFields) } }) } @@ -294,7 +302,7 @@ func formatDefaultTitleLine(r *Renderer, rec *fastjson.Value, b *strings.Builder } } -func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent string, painter *ansipainter.ANSIPainter, compact bool) { +func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent string, painter *ansipainter.ANSIPainter, compact bool, includeFields []string) { var i uint switch v.Type() { @@ -303,6 +311,10 @@ func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent s obj := v.GetObject() i = 0 obj.Visit(func(subk []byte, subv *fastjson.Value) { + nestedIncludeFields, ok := anyIsPrefix(includeFields, string(subk)) + if !ok { + return + } if i != 0 { b.WriteByte(',') if compact { @@ -320,7 +332,7 @@ func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent s b.WriteByte('"') painter.Reset(b) b.WriteString(": ") - formatJSONValue(b, subv, currIndent+indent, indent, painter, compact) + formatJSONValue(b, subv, currIndent+indent, indent, painter, compact, nestedIncludeFields) i++ }) if !compact { @@ -342,7 +354,7 @@ func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent s b.WriteString(currIndent) b.WriteString(indent) } - formatJSONValue(b, subv, currIndent+indent, indent, painter, compact) + formatJSONValue(b, subv, currIndent+indent, indent, painter, compact, includeFields) } if !compact { b.WriteByte('\n') @@ -437,3 +449,29 @@ var formatterFromName = map[string]Formatter{ "simple": &simpleFormatter{}, "compact": &compactFormatter{}, } + +// returns whether any of the include items is a prefix of key, along with the postfix (if any) +func anyIsPrefix(includes []string, key string) ([]string, bool) { + if len(includes) == 0 || (len(includes) == 1 && includes[0] == "") { + return includes, true + } + subKeys := strings.Split(key, ".") + for _, include := range includes { + match := true + includeSubKeys := strings.Split(include, ".") + for i, subKey := range subKeys { + if len(includeSubKeys) > i && includeSubKeys[i] != subKey { + match = false + break + } + } + if match { + if len(includeSubKeys) > len(subKeys) { + // we didn't match the whole key, return the reminder to match in the next recursion step + return []string{strings.Join(includeSubKeys[len(subKeys):], ".")}, true + } + return []string{}, true + } + } + return includes, false +}