Merge pull request #131 from andrewkroh/bugfix/omit-null-fields

Removing 'fields' from JSON output when the field is null.

CHANGELOG.md

@@ -11,6 +11,7 @@ All documentation about Filebeat can be found here.
 ### Backward Compatibility Breaks
 
 ### Bugfixes
+- Omit 'fields' from event JSON when null. #126
 
 ### Added
 

beat/filebeat.go

@@ -3,7 +3,6 @@ package beat
 import (
 	"fmt"
 	"os"
-	"time"
 
 	"github.com/elastic/libbeat/beat"
 	"github.com/elastic/libbeat/cfgfile"
@@ -126,18 +125,7 @@ func Publish(beat *beat.Beat, fb *Filebeat) {
 
 		pubEvents := make([]common.MapStr, 0, len(events))
 		for _, event := range events {
-			bEvent := common.MapStr{
-				"timestamp": common.Time(time.Now()),
-				"source":    event.Source,
-				"offset":    event.Offset,
-				"line":      event.Line,
-				"message":   event.Text,
-				"fields":    event.Fields,
-				"fileinfo":  event.Fileinfo,
-				"type":      "log",
-			}
-
-			pubEvents = append(pubEvents, bEvent)
+			pubEvents = append(pubEvents, event.ToMapStr())
 		}
 
 		beat.Events.PublishEvents(pubEvents, publisher.Sync)

etc/fields.yml

@@ -23,8 +23,8 @@ env:
       format: YYYY-MM-DDTHH:MM:SS.milliZ
       example: 2015-01-24T14:06:05.071Z
       description: >
-        The timestamp when the log line is published. The precision is in milliseconds.
-        The timezone is UTC.
+        The timestamp when the log line was read. The precision is in
+        milliseconds. The timezone is UTC.
 
     - name: type
       description: >

harvester/log.go

@@ -83,6 +83,7 @@ func (h *Harvester) Harvest() {
 
 		line++
 		event := &input.FileEvent{
+			ReadTime: lastReadTime,
 			Source:   &h.Path,
 			Offset:   h.Offset,
 			Line:     line,

input/file.go

@@ -2,7 +2,9 @@ package input
 
 import (
 	"os"
+	"time"
 
+	"github.com/elastic/libbeat/common"
 	"github.com/elastic/libbeat/logp"
 )
 
@@ -15,6 +17,7 @@ type File struct {
 
 // FileEvent is sent to the output and must contain all relevant information
 type FileEvent struct {
+	ReadTime time.Time
 	Source   *string
 	Offset   int64
 	Line     uint64
@@ -44,6 +47,24 @@ func (f *FileEvent) GetState() *FileState {
 	return state
 }
 
+func (f *FileEvent) ToMapStr() common.MapStr {
+	event := common.MapStr{
+		"timestamp": common.Time(f.ReadTime),
+		"source":    f.Source,
+		"offset":    f.Offset,
+		"line":      f.Line,
+		"message":   f.Text,
+		"fileinfo":  f.Fileinfo,
+		"type":      "log",
+	}
+
+	if f.Fields != nil {
+		event["fields"] = f.Fields
+	}
+
+	return event
+}
+
 // Check that the file isn't a symlink, mode is regular or file is nil
 func (f *File) IsRegularFile() bool {
 	if f.File == nil {

input/file_test.go

@@ -103,3 +103,11 @@ func TestSafeFileRotateExistingFile(t *testing.T) {
 	assert.NoError(t, err)
 	assert.Equal(t, []byte("new filebeat 2"), contents)
 }
+
+func TestFileEventToMapStr(t *testing.T) {
+	// Test 'fields' is not present when it is nil.
+	event := FileEvent{}
+	mapStr := event.ToMapStr()
+	_, found := mapStr["fields"]
+	assert.False(t, found)
+}