Skip to content

Commit

Permalink
feat: add -i, --include-fields FIELDS flag for including only certa…
Browse files Browse the repository at this point in the history 
…in fields (#15)

This complements `-x FIELDS`. If included field are given, then *only*
those extra fields are shown; with the exception that the title line
is not affected. I.e., `-i FIELDS` do not apply to title line fields. This
feels incongruous with `-x FIELDS`, which *does* apply to
title line fields, but I think it makes for more natural usage.
  • Loading branch information
@jalvz
jalvz authored Jun 18, 2021
1 parent 0582acb commit 0c711ae
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 22 deletions.
4 changes: 4 additions & 0 deletions cmd/ecslog/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ var flagColorScheme = flags.StringP("color-scheme", "c", "default",
"Color scheme to use, if colorizing.") // hidden
var flagExcludeFields = flags.StringP("exclude-fields", "x", "",
"Comma-separated list of fields to exclude from the output.")
var flagIncludeFields = flags.StringP("include-fields", "i", "",
"Comma-separated list of fields to include in the output.")

func printError(msg string) {
fmt.Fprintf(os.Stderr, "ecslog: error: %s\n", msg)
Expand Down Expand Up @@ -144,6 +146,7 @@ func main() {

commaSplitter := regexp.MustCompile(`\s*,\s*`)
excludeFields := commaSplitter.Split(*flagExcludeFields, -1)
includeFields := commaSplitter.Split(*flagIncludeFields, -1)

ecsLenient := false
if cfgECSLenient, ok := cfg.GetBool("ecsLenient"); ok {
Expand All @@ -161,6 +164,7 @@ func main() {
formatName,
maxLineLen,
excludeFields,
includeFields,
ecsLenient,
timestampShowDiff,
)
Expand Down
9 changes: 8 additions & 1 deletion cmd/ecslog/main_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,9 +96,16 @@ var mainTestCases = []mainTestCase{
regexp.MustCompile(`^\n foo: "bar"\n spam: "eggs"\n$`),
nil,
},
{
"ecslog --include-fields foo",
[]string{"ecslog", "-i", "foo", "./testdata/exclude-fields.log"},
0,
regexp.MustCompile(`^\[2021-01-19T22:51:12.142Z\] INFO: hi\n foo: "bar"\n$`),
nil,
},
}

func TestMain(t *testing.T) {
func TestFlags(t *testing.T) {
for _, tc := range mainTestCases {
t.Run(tc.name, func(t *testing.T) {
t.Logf("-- `ecslog` test case %q\n", tc.name)
Expand Down
4 changes: 3 additions & 1 deletion internal/ecslog/ecslog.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ type Renderer struct {
formatter Formatter
maxLineLen int
excludeFields []string
includeFields []string
ecsLenient bool
timestampShowDiff bool
levelFilter string
Expand Down Expand Up @@ -59,7 +60,7 @@ type Renderer struct {
// fields to exist.
// - `timestampShowDiff` is a bool indicating if the @timestamp diff from the
// preceding log record should be styled.
func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int, excludeFields []string, ecsLenient, timestampShowDiff bool) (*Renderer, error) {
func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int, excludeFields, includeFields []string, ecsLenient, timestampShowDiff bool) (*Renderer, error) {
// Get appropriate "painter" for terminal coloring.
var painter *ansipainter.ANSIPainter
if shouldColorize == "auto" {
Expand Down Expand Up @@ -116,6 +117,7 @@ func NewRenderer(shouldColorize, colorScheme, formatName string, maxLineLen int,
formatter: formatter,
maxLineLen: maxLineLen,
excludeFields: excludeFields,
includeFields: includeFields,
ecsLenient: ecsLenient,
timestampShowDiff: timestampShowDiff,

Expand Down
55 changes: 41 additions & 14 deletions internal/ecslog/ecslog_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ type renderFileTestCase struct {
levelFilter string
kqlFilter string
timestampShowDiff bool
includeFields []string

input string
output string
Expand All @@ -28,41 +29,41 @@ var renderFileTestCases = []renderFileTestCase{
// Non-ecs-logging lines
{
"empty object",
"no", "", "default", false, "", "", false,
"no", "", "default", false, "", "", false, []string{},
"{}",
"{}\n",
},

// Basics
{
"basic",
"no", "", "default", false, "", "", false,
"no", "", "default", false, "", "", false, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
"[2021-01-19T22:51:12.142Z] INFO: hi\n",
},
{
"basic, extra var",
"no", "", "default", false, "", "", false,
"no", "", "default", false, "", "", false, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":"bar"}`,
"[2021-01-19T22:51:12.142Z] INFO: hi\n foo: \"bar\"\n",
},
{
"no message is allowed",
"no", "", "default", false, "", "", false,
"no", "", "default", false, "", "", false, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"foo":"bar"}`,
"[2021-01-19T22:51:12.142Z] INFO:\n foo: \"bar\"\n",
},

// Coloring
{
"coloring 1",
"yes", "default", "default", false, "", "", false,
"yes", "default", "default", false, "", "", false, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
"[2021-01-19T22:51:12.142Z] \x1b[32m INFO\x1b[0m: \x1b[36mhi\x1b[0m\n",
},
{
"timestamp diff highlighting 1",
"yes", "default", "default", false, "", "", true,
"yes", "default", "default", false, "", "", true, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}
{"log.level":"info","@timestamp":"2021-01-19T22:51:23.456Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
"[2021-01-19T22:51:12.142Z] \x1b[32m INFO\x1b[0m: \x1b[36mhi\x1b[0m\n" +
Expand All @@ -72,54 +73,79 @@ var renderFileTestCases = []renderFileTestCase{
// KQL filtering
{
"kql filtering, yep",
"no", "", "default", false, "", "foo:bar", false,
"no", "", "default", false, "", "foo:bar", false, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":"bar"}`,
"[2021-01-19T22:51:12.142Z] INFO: hi\n foo: \"bar\"\n",
},
{
"kql filtering, nope",
"no", "", "default", false, "", "foo:baz", false,
"no", "", "default", false, "", "foo:baz", false, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":"bar"}`,
"",
},
{
"kql filtering, log.level range query, yep",
"no", "", "default", false, "", "log.level > debug", false,
"no", "", "default", false, "", "log.level > debug", false, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
"[2021-01-19T22:51:12.142Z] INFO: hi\n",
},
{
"kql filtering, log.level range query, nope",
"no", "", "default", false, "", "log.level > warn", false,
"no", "", "default", false, "", "log.level > warn", false, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
"",
},

// ecsLenient
{
"lenient: missing log.level",
"no", "", "default", true, "", "", false,
"no", "", "default", true, "", "", false, []string{},
`{"@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi"}`,
"[2021-01-19T22:51:12.142Z] : hi\n",
},
{
"lenient: missing @timestamp",
"no", "", "default", true, "", "", false,
"no", "", "default", true, "", "", false, []string{},
`{"log.level":"info","ecs":{"version":"1.5.0"},"message":"hi"}`,
" INFO: hi\n",
},
{
"lenient: missing ecs.version",
"no", "", "default", true, "", "", false,
"no", "", "default", true, "", "", false, []string{},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","message":"hi"}`,
"[2021-01-19T22:51:12.142Z] INFO: hi\n",
},
{
"lenient: @timestamp only",
"no", "", "default", true, "", "", false,
"no", "", "default", true, "", "", false, []string{},
`{"@timestamp":"2021-01-19T22:51:12.142Z","message":"hi"}`,
"[2021-01-19T22:51:12.142Z] : hi\n",
},
// Include fields
{
"include fields: only log",
"no", "", "default", true, "", "", false, []string{"log"},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","log.origin":{"foo":"bar","file.name":"main.go","file.line":"42"}}`,
"[2021-01-19T22:51:12.142Z] INFO: hi\n log.origin: {\n \"foo\": \"bar\",\n \"file.name\": \"main.go\",\n \"file.line\": \"42\"\n }\n",
},
{
"include fields: only log.origin.file",
"no", "", "default", true, "", "", false, []string{"log.origin.file"},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","log.origin":{"foo":"bar","file.name":"main.go","file.line":"42"}}`,
"[2021-01-19T22:51:12.142Z] INFO: hi\n log.origin: {\n \"file.name\": \"main.go\",\n \"file.line\": \"42\"\n }\n",
},
{
"include fields: only log.origin.file.name",
"no", "", "default", true, "", "", false, []string{"log.origin.file.name"},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","log.origin":{"foo":"bar","file.name":"main.go","file.line":"42"}}`,
"[2021-01-19T22:51:12.142Z] INFO: hi\n log.origin: {\n \"file.name\": \"main.go\"\n }\n",
},
{
"include fields: only foo",
"no", "", "default", true, "", "", false, []string{"foo"},
`{"log.level":"info","@timestamp":"2021-01-19T22:51:12.142Z","ecs":{"version":"1.5.0"},"message":"hi","foo":0,"bar":1}`,
"[2021-01-19T22:51:12.142Z] INFO: hi\n foo: 0\n",
},
}

func TestRenderFile(t *testing.T) {
Expand All @@ -131,6 +157,7 @@ func TestRenderFile(t *testing.T) {
tc.formatName,
-1,
[]string{},
tc.includeFields,
tc.ecsLenient,
tc.timestampShowDiff,
)
Expand Down
50 changes: 44 additions & 6 deletions internal/ecslog/formatters.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,16 @@ func (f *defaultFormatter) formatRecord(r *Renderer, rec *fastjson.Value, b *str
// fields as a HTTP request/response text representation
obj := rec.GetObject()
obj.Visit(func(k []byte, v *fastjson.Value) {
includeFields, ok := anyIsPrefix(r.includeFields, string(k))
if !ok {
return
}
b.WriteString("\n ")
r.painter.Paint(b, "extraField")
b.Write(k)
r.painter.Reset(b)
b.WriteString(": ")
formatJSONValue(b, v, " ", " ", r.painter, false)
formatJSONValue(b, v, " ", " ", r.painter, false, includeFields)
})
}

Expand All @@ -58,6 +62,10 @@ func (f *compactFormatter) formatRecord(r *Renderer, rec *fastjson.Value, b *str
// fields as a HTTP request/response text representation
obj := rec.GetObject()
obj.Visit(func(k []byte, v *fastjson.Value) {
includeFields, ok := anyIsPrefix(r.includeFields, string(k))
if !ok {
return
}
b.WriteString("\n ")
r.painter.Paint(b, "extraField")
b.Write(k)
Expand All @@ -72,9 +80,9 @@ func (f *compactFormatter) formatRecord(r *Renderer, rec *fastjson.Value, b *str
vStr := v.String()
// 80 (quotable width) - 8 (indentation) - length of `k` - len(": ")
if len(vStr) < 80-8-len(k)-2 {
formatJSONValue(b, v, " ", " ", r.painter, true)
formatJSONValue(b, v, " ", " ", r.painter, true, includeFields)
} else {
formatJSONValue(b, v, " ", " ", r.painter, false)
formatJSONValue(b, v, " ", " ", r.painter, false, includeFields)
}
})
}
Expand Down Expand Up @@ -294,7 +302,7 @@ func formatDefaultTitleLine(r *Renderer, rec *fastjson.Value, b *strings.Builder
}
}

func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent string, painter *ansipainter.ANSIPainter, compact bool) {
func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent string, painter *ansipainter.ANSIPainter, compact bool, includeFields []string) {
var i uint

switch v.Type() {
Expand All @@ -303,6 +311,10 @@ func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent s
obj := v.GetObject()
i = 0
obj.Visit(func(subk []byte, subv *fastjson.Value) {
nestedIncludeFields, ok := anyIsPrefix(includeFields, string(subk))
if !ok {
return
}
if i != 0 {
b.WriteByte(',')
if compact {
Expand All @@ -320,7 +332,7 @@ func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent s
b.WriteByte('"')
painter.Reset(b)
b.WriteString(": ")
formatJSONValue(b, subv, currIndent+indent, indent, painter, compact)
formatJSONValue(b, subv, currIndent+indent, indent, painter, compact, nestedIncludeFields)
i++
})
if !compact {
Expand All @@ -342,7 +354,7 @@ func formatJSONValue(b *strings.Builder, v *fastjson.Value, currIndent, indent s
b.WriteString(currIndent)
b.WriteString(indent)
}
formatJSONValue(b, subv, currIndent+indent, indent, painter, compact)
formatJSONValue(b, subv, currIndent+indent, indent, painter, compact, includeFields)
}
if !compact {
b.WriteByte('\n')
Expand Down Expand Up @@ -437,3 +449,29 @@ var formatterFromName = map[string]Formatter{
"simple": &simpleFormatter{},
"compact": &compactFormatter{},
}

// returns whether any of the include items is a prefix of key, along with the postfix (if any)
func anyIsPrefix(includes []string, key string) ([]string, bool) {
if len(includes) == 0 || (len(includes) == 1 && includes[0] == "") {
return includes, true
}
subKeys := strings.Split(key, ".")
for _, include := range includes {
match := true
includeSubKeys := strings.Split(include, ".")
for i, subKey := range subKeys {
if len(includeSubKeys) > i && includeSubKeys[i] != subKey {
match = false
break
}
}
if match {
if len(includeSubKeys) > len(subKeys) {
// we didn't match the whole key, return the reminder to match in the next recursion step
return []string{strings.Join(includeSubKeys[len(subKeys):], ".")}, true
}
return []string{}, true
}
}
return includes, false
}

0 comments on commit 0c711ae

Please sign in to comment.