diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index 34f19eb141db..3d5c0d65ca3e 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -10,6 +10,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
*Affecting all Beats*
+- Add @timestamp support nanoseconds. {pull}9818[9818]
+
*Auditbeat*
diff --git a/libbeat/common/datetime.go b/libbeat/common/datetime.go
index c0eac0cc6f53..2e9c769bec3b 100644
--- a/libbeat/common/datetime.go
+++ b/libbeat/common/datetime.go
@@ -21,21 +21,37 @@ import (
"encoding/binary"
"encoding/json"
"errors"
+ "fmt"
"hash"
"time"
)
-// TsLayout is the layout to be used in the timestamp marshaling/unmarshaling everywhere.
-// The timezone must always be UTC.
-const TsLayout = "2006-01-02T15:04:05.000Z"
+const (
+ // TsLayout is the seconds layout to be used in the timestamp marshaling/unmarshaling everywhere.
+ // The timezone must always be UTC.
+ TsLayout = "2006-01-02T15:04:05"
+)
// Time is an abstraction for the time.Time type
type Time time.Time
+func (t Time) generateTsLayout() string {
+ nanoTime := time.Time(t).UTC().UnixNano()
+ trailZero := "000000000"
+ for i := 0; i < 2; i++ {
+ if nanoTime%1000 != 0 {
+ break
+ }
+ trailZero = trailZero[:len(trailZero)-3]
+ nanoTime = nanoTime / 1000
+ }
+ return fmt.Sprintf("%s.%sZ", TsLayout, trailZero)
+}
+
// MarshalJSON implements json.Marshaler interface.
// The time is a quoted string in the JsTsLayout format.
func (t Time) MarshalJSON() ([]byte, error) {
- return json.Marshal(time.Time(t).UTC().Format(TsLayout))
+ return json.Marshal(time.Time(t).UTC().Format(t.generateTsLayout()))
}
// UnmarshalJSON implements js.Unmarshaler interface.
@@ -54,14 +70,28 @@ func (t Time) Hash32(h hash.Hash32) error {
return err
}
-// ParseTime parses a time in the TsLayout format.
+// ParseTime parses a time in the NanoTsLayout format first, then use millisTsLayout format
func ParseTime(timespec string) (Time, error) {
- t, err := time.Parse(TsLayout, timespec)
+ var (
+ t time.Time
+ err error
+ tsLayout string
+ trailZero string
+ )
+
+ for i := 0; i < 3; i++ {
+ trailZero += "000"
+ tsLayout = fmt.Sprintf("%s.%sZ", TsLayout, trailZero)
+ t, err = time.Parse(tsLayout, timespec)
+ if err == nil {
+ break
+ }
+ }
return Time(t), err
}
func (t Time) String() string {
- return time.Time(t).Format(TsLayout)
+ return time.Time(t).Format(t.generateTsLayout())
}
// MustParseTime is a convenience equivalent of the ParseTime function
diff --git a/libbeat/common/datetime_test.go b/libbeat/common/datetime_test.go
index ff9b433b958d..2d04a98ad0ab 100644
--- a/libbeat/common/datetime_test.go
+++ b/libbeat/common/datetime_test.go
@@ -64,7 +64,7 @@ func TestParseTimeNegative(t *testing.T) {
tests := []inputOutput{
{
Input: "2015-02-29TT14:06:05.071Z",
- Err: "parsing time \"2015-02-29TT14:06:05.071Z\" as \"2006-01-02T15:04:05.000Z\": cannot parse \"T14:06:05.071Z\" as \"15\"",
+ Err: "parsing time \"2015-02-29TT14:06:05.071Z\" as \"2006-01-02T15:04:05.000000000Z\": cannot parse \"T14:06:05.071Z\" as \"15\"",
},
}
diff --git a/libbeat/common/dtfmt/builder.go b/libbeat/common/dtfmt/builder.go
index 458878ffcc98..85af091e95f4 100644
--- a/libbeat/common/dtfmt/builder.go
+++ b/libbeat/common/dtfmt/builder.go
@@ -45,7 +45,6 @@ func (b *builder) createConfig() (ctxConfig, error) {
func (b *builder) compile() (prog, error) {
p := prog{}
-
for _, e := range b.elements {
tmp, err := e.compile()
if err != nil {
@@ -97,28 +96,19 @@ func (b *builder) add(e element) {
b.elements = append(b.elements, e)
}
-func (b *builder) millisOfSecond(digits int) {
+func (b *builder) nanoOfSecond(digits int) {
if digits <= 0 {
return
}
- switch digits {
- case 1:
- b.appendExtDecimal(ftMillisOfSecond, 100, 1, 1)
- case 2:
- b.appendExtDecimal(ftMillisOfSecond, 10, 2, 2)
- case 3:
- b.appendExtDecimal(ftMillisOfSecond, 0, 3, 3)
- default:
- b.appendExtDecimal(ftMillisOfSecond, 0, 3, 3)
- b.appendZeros(digits - 3)
+ if digits <= 9 {
+ b.appendExtDecimal(ftNanoOfSecond, 9-digits, digits, digits)
+ } else {
+ b.appendExtDecimal(ftNanoOfSecond, 0, 9, 9)
+ b.appendZeros(digits - 9)
}
}
-func (b *builder) millisOfDay(digits int) {
- b.appendDecimal(ftMillisOfDay, digits, 8)
-}
-
func (b *builder) secondOfMinute(digits int) {
b.appendDecimal(ftSecondOfMinute, digits, 2)
}
@@ -237,8 +227,8 @@ func (b *builder) appendDecimalValue(ft fieldType, minDigits, maxDigits int, sig
}
}
-func (b *builder) appendExtDecimal(ft fieldType, div, minDigits, maxDigits int) {
- b.add(paddedNumber{ft, div, minDigits, maxDigits, false})
+func (b *builder) appendExtDecimal(ft fieldType, divExp, minDigits, maxDigits int) {
+ b.add(paddedNumber{ft, divExp, minDigits, maxDigits, false})
}
func (b *builder) appendDecimal(ft fieldType, minDigits, maxDigits int) {
diff --git a/libbeat/common/dtfmt/ctx.go b/libbeat/common/dtfmt/ctx.go
index faa405e050a1..a88f1edabc2f 100644
--- a/libbeat/common/dtfmt/ctx.go
+++ b/libbeat/common/dtfmt/ctx.go
@@ -31,7 +31,7 @@ type ctx struct {
isoWeek, isoYear int
hour, min, sec int
- millis int
+ nano int
tzOffset int
@@ -44,6 +44,7 @@ type ctxConfig struct {
weekday bool
yearday bool
millis bool
+ nano bool
iso bool
tzOffset bool
}
@@ -59,8 +60,8 @@ func (c *ctx) initTime(config *ctxConfig, t time.Time) {
c.isoYear, c.isoWeek = t.ISOWeek()
}
- if config.millis {
- c.millis = t.Nanosecond() / 1000000
+ if config.nano {
+ c.nano = t.Nanosecond()
}
if config.yearday {
@@ -84,8 +85,8 @@ func (c *ctxConfig) enableClock() {
c.clock = true
}
-func (c *ctxConfig) enableMillis() {
- c.millis = true
+func (c *ctxConfig) enableNano() {
+ c.nano = true
}
func (c *ctxConfig) enableWeekday() {
diff --git a/libbeat/common/dtfmt/dtfmt_test.go b/libbeat/common/dtfmt/dtfmt_test.go
index 61af8ab17cc4..d017bdbec173 100644
--- a/libbeat/common/dtfmt/dtfmt_test.go
+++ b/libbeat/common/dtfmt/dtfmt_test.go
@@ -103,6 +103,15 @@ func TestFormat(t *testing.T) {
{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123, time.FixedZone("PST", -8*60*60)),
"yyyy-MM-dd'T'HH:mm:ss.SSSz",
"2017-01-02T04:06:07.123-08:00"},
+
+ // beats nanoseconds timestamp
+ {mkDateTime(2017, 1, 2, 4, 6, 7, 123),
+ "yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnn'Z'",
+ "2017-01-02T04:06:07.123000000Z"},
+
+ {mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123, time.FixedZone("PST", -8*60*60)),
+ "yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnnz",
+ "2017-01-02T04:06:07.123000000-08:00"},
}
for i, test := range tests {
diff --git a/libbeat/common/dtfmt/elems.go b/libbeat/common/dtfmt/elems.go
index a86f67a7fc5d..5582f48c6ff0 100644
--- a/libbeat/common/dtfmt/elems.go
+++ b/libbeat/common/dtfmt/elems.go
@@ -45,7 +45,7 @@ type unpaddedNumber struct {
type paddedNumber struct {
ft fieldType
- div int
+ divExp int
minDigits, maxDigits int
signed bool
}
@@ -123,12 +123,9 @@ func numRequires(c *ctxConfig, ft fieldType) error {
ftSecondOfMinute:
c.enableClock()
- case ftMillisOfDay:
- c.enableClock()
- c.enableMillis()
+ case ftNanoOfSecond:
+ c.enableNano()
- case ftMillisOfSecond:
- c.enableMillis()
}
return nil
@@ -191,10 +188,10 @@ func (n unpaddedNumber) compile() (prog, error) {
}
func (n paddedNumber) compile() (prog, error) {
- if n.div == 0 {
+ if n.divExp == 0 {
return makeProg(opNumPadded, byte(n.ft), byte(n.maxDigits))
}
- return makeProg(opExtNumPadded, byte(n.ft), byte(n.div), byte(n.maxDigits))
+ return makeProg(opExtNumPadded, byte(n.ft), byte(n.divExp), byte(n.maxDigits))
}
func (n twoDigitYear) compile() (prog, error) {
diff --git a/libbeat/common/dtfmt/fields.go b/libbeat/common/dtfmt/fields.go
index 08b0d1c6dc10..fbf0122b8541 100644
--- a/libbeat/common/dtfmt/fields.go
+++ b/libbeat/common/dtfmt/fields.go
@@ -41,9 +41,8 @@ const (
ftMinuteOfHour
ftSecondOfDay
ftSecondOfMinute
- ftMillisOfDay
- ftMillisOfSecond
ftTimeZoneOffset
+ ftNanoOfSecond
)
func getIntField(ft fieldType, ctx *ctx, t time.Time) (int, error) {
@@ -105,11 +104,8 @@ func getIntField(ft fieldType, ctx *ctx, t time.Time) (int, error) {
case ftSecondOfMinute:
return ctx.sec, nil
- case ftMillisOfDay:
- return ((ctx.hour*60+ctx.min)*60+ctx.sec)*1000 + ctx.millis, nil
-
- case ftMillisOfSecond:
- return ctx.millis, nil
+ case ftNanoOfSecond:
+ return ctx.nano, nil
}
return 0, nil
diff --git a/libbeat/common/dtfmt/fmt.go b/libbeat/common/dtfmt/fmt.go
index acc4989acd9a..c8243022d98b 100644
--- a/libbeat/common/dtfmt/fmt.go
+++ b/libbeat/common/dtfmt/fmt.go
@@ -60,6 +60,7 @@ func releaseCtx(c *ctx) {
func NewFormatter(pattern string) (*Formatter, error) {
b := newBuilder()
+ // pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
err := parsePatternTo(b, pattern)
if err != nil {
return nil, err
@@ -135,6 +136,7 @@ func (f *Formatter) Format(t time.Time) (string, error) {
}
func parsePatternTo(b *builder, pattern string) error {
+ // pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
for i := 0; i < len(pattern); {
tok, tokText, err := parseToken(pattern, &i)
if err != nil {
@@ -209,11 +211,18 @@ func parsePatternTo(b *builder, pattern string) error {
b.secondOfMinute(tokLen)
case 'S': // fraction of second
- b.millisOfSecond(tokLen)
+ b.nanoOfSecond(tokLen)
case 'z': // timezone offset
b.timeZoneOffsetText()
+ case 'n': // nano second
+ // if timestamp layout use `n`, it always return 9 digits nanoseconds.
+ if tokLen != 9 {
+ tokLen = 9
+ }
+ b.nanoOfSecond(tokLen)
+
case '\'': // literal
if tokLen == 1 {
b.appendRune(rune(tokText[0]))
@@ -234,7 +243,7 @@ func parseToken(pattern string, i *int) (rune, string, error) {
start := *i
idx := start
length := len(pattern)
-
+ // pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
r, w := utf8.DecodeRuneInString(pattern[idx:])
idx += w
if ('A' <= r && r <= 'Z') || ('a' <= r && r <= 'z') {
diff --git a/libbeat/common/dtfmt/prog.go b/libbeat/common/dtfmt/prog.go
index cee6f3d473db..a2d4517c83ec 100644
--- a/libbeat/common/dtfmt/prog.go
+++ b/libbeat/common/dtfmt/prog.go
@@ -36,13 +36,23 @@ const (
opCopyLong // [op, len1, len, content[len1<<8 + len]]
opNum // [op, ft]
opNumPadded // [op, ft, digits]
- opExtNumPadded // [op, ft, div, digits]
+ opExtNumPadded // [op, ft, divExp, digits]
opZeros // [op, count]
opTwoDigit // [op, ft]
opTextShort // [op, ft]
opTextLong // [op, ft]
)
+var pow10Table [10]int
+
+func init() {
+ x := 1
+ for i := range pow10Table {
+ pow10Table[i] = x
+ x *= 10
+ }
+}
+
func (p prog) eval(bytes []byte, ctx *ctx, t time.Time) ([]byte, error) {
for i := 0; i < len(p.p); {
op := p.p[i]
@@ -90,7 +100,8 @@ func (p prog) eval(bytes []byte, ctx *ctx, t time.Time) ([]byte, error) {
}
bytes = appendPadded(bytes, v, digits)
case opExtNumPadded:
- ft, div, digits := fieldType(p.p[i]), int(p.p[i+1]), int(p.p[i+2])
+ ft, divExp, digits := fieldType(p.p[i]), int(p.p[i+1]), int(p.p[i+2])
+ div := pow10Table[divExp]
i += 3
v, err := getIntField(ft, ctx, t)
if err != nil {
diff --git a/libbeat/outputs/codec/common.go b/libbeat/outputs/codec/common.go
index d3df85dc5346..a168f352e446 100644
--- a/libbeat/outputs/codec/common.go
+++ b/libbeat/outputs/codec/common.go
@@ -36,9 +36,9 @@ func MakeTimestampEncoder() func(*time.Time, structform.ExtVisitor) error {
func MakeUTCOrLocalTimestampEncoder(localTime bool) func(*time.Time, structform.ExtVisitor) error {
var dtPattern string
if localTime {
- dtPattern = "yyyy-MM-dd'T'HH:mm:ss.SSSz"
+ dtPattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSz"
} else {
- dtPattern = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
+ dtPattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"
}
formatter, err := dtfmt.NewFormatter(dtPattern)
diff --git a/libbeat/outputs/codec/json/json_test.go b/libbeat/outputs/codec/json/json_test.go
index dc01e397e0bd..33432858d102 100644
--- a/libbeat/outputs/codec/json/json_test.go
+++ b/libbeat/outputs/codec/json/json_test.go
@@ -37,13 +37,13 @@ func TestJsonCodec(t *testing.T) {
"default json": testCase{
config: defaultConfig,
in: common.MapStr{"msg": "message"},
- expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"message"}`,
+ expected: `{"@timestamp":"0001-01-01T00:00:00.000000000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"message"}`,
},
"pretty enabled": testCase{
config: Config{Pretty: true},
in: common.MapStr{"msg": "message"},
expected: `{
- "@timestamp": "0001-01-01T00:00:00.000Z",
+ "@timestamp": "0001-01-01T00:00:00.000000000Z",
"@metadata": {
"beat": "test",
"type": "_doc",
@@ -55,23 +55,23 @@ func TestJsonCodec(t *testing.T) {
"html escaping enabled": testCase{
config: Config{EscapeHTML: true},
in: common.MapStr{"msg": "world"},
- expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"\u003chello\u003eworld\u003c/hello\u003e"}`,
+ expected: `{"@timestamp":"0001-01-01T00:00:00.000000000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"\u003chello\u003eworld\u003c/hello\u003e"}`,
},
"html escaping disabled": testCase{
config: Config{EscapeHTML: false},
in: common.MapStr{"msg": "world"},
- expected: `{"@timestamp":"0001-01-01T00:00:00.000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"world"}`,
+ expected: `{"@timestamp":"0001-01-01T00:00:00.000000000Z","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"world"}`,
},
"UTC timezone offset": testCase{
config: Config{LocalTime: true},
in: common.MapStr{"msg": "message"},
- expected: `{"@timestamp":"0001-01-01T00:00:00.000+00:00","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"message"}`,
+ expected: `{"@timestamp":"0001-01-01T00:00:00.000000000+00:00","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"message"}`,
},
"PST timezone offset": testCase{
config: Config{LocalTime: true},
ts: time.Time{}.In(time.FixedZone("PST", -8*60*60)),
in: common.MapStr{"msg": "message"},
- expected: `{"@timestamp":"0000-12-31T16:00:00.000-08:00","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"message"}`,
+ expected: `{"@timestamp":"0000-12-31T16:00:00.000000000-08:00","@metadata":{"beat":"test","type":"_doc","version":"1.2.3"},"msg":"message"}`,
},
}
diff --git a/libbeat/outputs/console/console_test.go b/libbeat/outputs/console/console_test.go
index 46c655094ab0..bdc5e36709f6 100644
--- a/libbeat/outputs/console/console_test.go
+++ b/libbeat/outputs/console/console_test.go
@@ -85,7 +85,7 @@ func TestConsoleOutput(t *testing.T) {
[]beat.Event{
{Fields: event("field", "value")},
},
- "{\"@timestamp\":\"0001-01-01T00:00:00.000Z\",\"@metadata\":{\"beat\":\"test\",\"type\":\"_doc\",\"version\":\"1.2.3\"},\"field\":\"value\"}\n",
+ "{\"@timestamp\":\"0001-01-01T00:00:00.000000000Z\",\"@metadata\":{\"beat\":\"test\",\"type\":\"_doc\",\"version\":\"1.2.3\"},\"field\":\"value\"}\n",
},
{
"single json event (pretty=true)",
@@ -96,7 +96,7 @@ func TestConsoleOutput(t *testing.T) {
[]beat.Event{
{Fields: event("field", "value")},
},
- "{\n \"@timestamp\": \"0001-01-01T00:00:00.000Z\",\n \"@metadata\": {\n \"beat\": \"test\",\n \"type\": \"_doc\",\n \"version\": \"1.2.3\"\n },\n \"field\": \"value\"\n}\n",
+ "{\n \"@timestamp\": \"0001-01-01T00:00:00.000000000Z\",\n \"@metadata\": {\n \"beat\": \"test\",\n \"type\": \"_doc\",\n \"version\": \"1.2.3\"\n },\n \"field\": \"value\"\n}\n",
},
// TODO: enable test after update fmtstr support to beat.Event
{
diff --git a/libbeat/outputs/elasticsearch/enc_test.go b/libbeat/outputs/elasticsearch/enc_test.go
index 135bdf0d5f58..846b546b4450 100644
--- a/libbeat/outputs/elasticsearch/enc_test.go
+++ b/libbeat/outputs/elasticsearch/enc_test.go
@@ -31,7 +31,7 @@ import (
func TestJSONEncoderMarshalBeatEvent(t *testing.T) {
encoder := newJSONEncoder(nil, true)
event := beat.Event{
- Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 0, time.UTC),
+ Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 15, time.UTC),
Fields: common.MapStr{
"field1": "value1",
},
@@ -41,14 +41,14 @@ func TestJSONEncoderMarshalBeatEvent(t *testing.T) {
if err != nil {
t.Errorf("Error while marshaling beat.Event using JSONEncoder: %v", err)
}
- assert.Equal(t, encoder.buf.String(), "{\"@timestamp\":\"2017-11-07T12:00:00.000Z\",\"field1\":\"value1\"}\n",
+ assert.Equal(t, encoder.buf.String(), "{\"@timestamp\":\"2017-11-07T12:00:00.000000015Z\",\"field1\":\"value1\"}\n",
"Unexpected marshaled format of beat.Event")
}
func TestJSONEncoderMarshalMonitoringEvent(t *testing.T) {
encoder := newJSONEncoder(nil, true)
event := report.Event{
- Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 0, time.UTC),
+ Timestamp: time.Date(2017, time.November, 7, 12, 0, 0, 15, time.UTC),
Fields: common.MapStr{
"field1": "value1",
},
@@ -58,6 +58,6 @@ func TestJSONEncoderMarshalMonitoringEvent(t *testing.T) {
if err != nil {
t.Errorf("Error while marshaling report.Event using JSONEncoder: %v", err)
}
- assert.Equal(t, encoder.buf.String(), "{\"timestamp\":\"2017-11-07T12:00:00.000Z\",\"field1\":\"value1\"}\n",
+ assert.Equal(t, encoder.buf.String(), "{\"timestamp\":\"2017-11-07T12:00:00.000000015Z\",\"field1\":\"value1\"}\n",
"Unexpected marshaled format of report.Event")
}