[WIP]beats @timestamp support nanoseconds

CHANGELOG.next.asciidoc

@@ -30,6 +30,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Libbeat: Cleanup the x-pack licenser code to use the new license endpoint and the new format. {pull}15091[15091]
 - Users can now specify `monitoring.cloud.*` to override `monitoring.elasticsearch.*` settings. {issue}14399[14399] {pull}15254[15254]
 - Update to ECS 1.4.0. {pull}14844[14844]
+- Update add_cloud_metadata fields to adjust to ECS. {pull}9265[9265]
+- Automaticall cap signed integers to 63bits. {pull}8991[8991]
+- Rename beat.timezone to event.timezone. {pull}9458[9458]
+- Use _doc as document type. {pull}9056[9056]{pull}9573[9573]
+- Update to Golang 1.11.3. {pull}9560[9560]
+- Add @timestamp support nanoseconds. {pull}9818[9818]
 
 *Auditbeat*
 

libbeat/common/datetime.go

@@ -21,21 +21,37 @@ import (
 	"encoding/binary"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"hash"
 	"time"
 )
 
-// TsLayout is the layout to be used in the timestamp marshaling/unmarshaling everywhere.
-// The timezone must always be UTC.
-const TsLayout = "2006-01-02T15:04:05.000Z"
+const (
+	// TsLayout is the seconds layout to be used in the timestamp marshaling/unmarshaling everywhere.
+	// The timezone must always be UTC.
+	TsLayout = "2006-01-02T15:04:05"
+)
 
 // Time is an abstraction for the time.Time type
 type Time time.Time
 
+func (t Time) generateTsLayout() string {
+	nanoTime := time.Time(t).UTC().UnixNano()
+	trailZero := "000000000"
+	for i := 0; i < 2; i++ {
+		if nanoTime%1000 != 0 {
+			break
+		}
+		trailZero = trailZero[:len(trailZero)-3]
+		nanoTime = nanoTime / 1000
+	}
+	return fmt.Sprintf("%s.%sZ", TsLayout, trailZero)
+}
+
 // MarshalJSON implements json.Marshaler interface.
 // The time is a quoted string in the JsTsLayout format.
 func (t Time) MarshalJSON() ([]byte, error) {
-	return json.Marshal(time.Time(t).UTC().Format(TsLayout))
+	return json.Marshal(time.Time(t).UTC().Format(t.generateTsLayout()))
 }
 
 // UnmarshalJSON implements js.Unmarshaler interface.
@@ -54,14 +70,28 @@ func (t Time) Hash32(h hash.Hash32) error {
 	return err
 }
 
-// ParseTime parses a time in the TsLayout format.
+// ParseTime parses a time in the NanoTsLayout format first, then use millisTsLayout format
 func ParseTime(timespec string) (Time, error) {
-	t, err := time.Parse(TsLayout, timespec)
+	var (
+		t         time.Time
+		err       error
+		tsLayout  string
+		trailZero string
+	)
+
+	for i := 0; i < 3; i++ {
+		trailZero += "000"
+		tsLayout = fmt.Sprintf("%s.%sZ", TsLayout, trailZero)
+		t, err = time.Parse(tsLayout, timespec)
+		if err == nil {
+			break
+		}
+	}
 	return Time(t), err
 }
 
 func (t Time) String() string {
-	return time.Time(t).Format(TsLayout)
+	return time.Time(t).Format(t.generateTsLayout())
 }
 
 // MustParseTime is a convenience equivalent of the ParseTime function

libbeat/common/datetime_test.go

@@ -64,7 +64,7 @@ func TestParseTimeNegative(t *testing.T) {
 	tests := []inputOutput{
 		{
 			Input: "2015-02-29TT14:06:05.071Z",
-			Err:   "parsing time \"2015-02-29TT14:06:05.071Z\" as \"2006-01-02T15:04:05.000Z\": cannot parse \"T14:06:05.071Z\" as \"15\"",
+			Err:   "parsing time \"2015-02-29TT14:06:05.071Z\" as \"2006-01-02T15:04:05.000000000Z\": cannot parse \"T14:06:05.071Z\" as \"15\"",
 		},
 	}
 

libbeat/common/dtfmt/builder.go

@@ -45,7 +45,6 @@ func (b *builder) createConfig() (ctxConfig, error) {
 
 func (b *builder) compile() (prog, error) {
 	p := prog{}
-
 	for _, e := range b.elements {
 		tmp, err := e.compile()
 		if err != nil {
@@ -97,28 +96,19 @@ func (b *builder) add(e element) {
 	b.elements = append(b.elements, e)
 }
 
-func (b *builder) millisOfSecond(digits int) {
+func (b *builder) nanoOfSecond(digits int) {
 	if digits <= 0 {
 		return
 	}
 
-	switch digits {
-	case 1:
-		b.appendExtDecimal(ftMillisOfSecond, 100, 1, 1)
-	case 2:
-		b.appendExtDecimal(ftMillisOfSecond, 10, 2, 2)
-	case 3:
-		b.appendExtDecimal(ftMillisOfSecond, 0, 3, 3)
-	default:
-		b.appendExtDecimal(ftMillisOfSecond, 0, 3, 3)
-		b.appendZeros(digits - 3)
+	if digits <= 9 {
+		b.appendExtDecimal(ftNanoOfSecond, 9-digits, digits, digits)
+	} else {
+		b.appendExtDecimal(ftNanoOfSecond, 0, 9, 9)
+		b.appendZeros(digits - 9)
 	}
 }
 
-func (b *builder) millisOfDay(digits int) {
-	b.appendDecimal(ftMillisOfDay, digits, 8)
-}
-
 func (b *builder) secondOfMinute(digits int) {
 	b.appendDecimal(ftSecondOfMinute, digits, 2)
 }
@@ -237,8 +227,8 @@ func (b *builder) appendDecimalValue(ft fieldType, minDigits, maxDigits int, sig
 	}
 }
 
-func (b *builder) appendExtDecimal(ft fieldType, div, minDigits, maxDigits int) {
-	b.add(paddedNumber{ft, div, minDigits, maxDigits, false})
+func (b *builder) appendExtDecimal(ft fieldType, divExp, minDigits, maxDigits int) {
+	b.add(paddedNumber{ft, divExp, minDigits, maxDigits, false})
 }
 
 func (b *builder) appendDecimal(ft fieldType, minDigits, maxDigits int) {

libbeat/common/dtfmt/ctx.go

@@ -31,7 +31,7 @@ type ctx struct {
 	isoWeek, isoYear int
 
 	hour, min, sec int
-	millis         int
+	nano           int
 
 	tzOffset int
 
@@ -44,6 +44,7 @@ type ctxConfig struct {
 	weekday  bool
 	yearday  bool
 	millis   bool
+	nano     bool
 	iso      bool
 	tzOffset bool
 }
@@ -59,8 +60,8 @@ func (c *ctx) initTime(config *ctxConfig, t time.Time) {
 		c.isoYear, c.isoWeek = t.ISOWeek()
 	}
 
-	if config.millis {
-		c.millis = t.Nanosecond() / 1000000
+	if config.nano {
+		c.nano = t.Nanosecond()
 	}
 
 	if config.yearday {
@@ -84,8 +85,8 @@ func (c *ctxConfig) enableClock() {
 	c.clock = true
 }
 
-func (c *ctxConfig) enableMillis() {
-	c.millis = true
+func (c *ctxConfig) enableNano() {
+	c.nano = true
 }
 
 func (c *ctxConfig) enableWeekday() {

libbeat/common/dtfmt/dtfmt_test.go

@@ -103,6 +103,15 @@ func TestFormat(t *testing.T) {
 		{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123, time.FixedZone("PST", -8*60*60)),
 			"yyyy-MM-dd'T'HH:mm:ss.SSSz",
 			"2017-01-02T04:06:07.123-08:00"},
+
+		// beats nanoseconds timestamp
+		{mkDateTime(2017, 1, 2, 4, 6, 7, 123),
+			"yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnn'Z'",
+			"2017-01-02T04:06:07.123000000Z"},
+
+		{mkDateTimeWithLocation(2017, 1, 2, 4, 6, 7, 123, time.FixedZone("PST", -8*60*60)),
+			"yyyy-MM-dd'T'HH:mm:ss.nnnnnnnnnz",
+			"2017-01-02T04:06:07.123000000-08:00"},
 	}
 
 	for i, test := range tests {

libbeat/common/dtfmt/elems.go

@@ -45,7 +45,7 @@ type unpaddedNumber struct {
 
 type paddedNumber struct {
 	ft                   fieldType
-	div                  int
+	divExp               int
 	minDigits, maxDigits int
 	signed               bool
 }
@@ -123,12 +123,9 @@ func numRequires(c *ctxConfig, ft fieldType) error {
 		ftSecondOfMinute:
 		c.enableClock()
 
-	case ftMillisOfDay:
-		c.enableClock()
-		c.enableMillis()
+	case ftNanoOfSecond:
+		c.enableNano()
 
-	case ftMillisOfSecond:
-		c.enableMillis()
 	}
 
 	return nil
@@ -191,10 +188,10 @@ func (n unpaddedNumber) compile() (prog, error) {
 }
 
 func (n paddedNumber) compile() (prog, error) {
-	if n.div == 0 {
+	if n.divExp == 0 {
 		return makeProg(opNumPadded, byte(n.ft), byte(n.maxDigits))
 	}
-	return makeProg(opExtNumPadded, byte(n.ft), byte(n.div), byte(n.maxDigits))
+	return makeProg(opExtNumPadded, byte(n.ft), byte(n.divExp), byte(n.maxDigits))
 }
 
 func (n twoDigitYear) compile() (prog, error) {

libbeat/common/dtfmt/fields.go

@@ -41,9 +41,8 @@ const (
 	ftMinuteOfHour
 	ftSecondOfDay
 	ftSecondOfMinute
-	ftMillisOfDay
-	ftMillisOfSecond
 	ftTimeZoneOffset
+	ftNanoOfSecond
 )
 
 func getIntField(ft fieldType, ctx *ctx, t time.Time) (int, error) {
@@ -105,11 +104,8 @@ func getIntField(ft fieldType, ctx *ctx, t time.Time) (int, error) {
 	case ftSecondOfMinute:
 		return ctx.sec, nil
 
-	case ftMillisOfDay:
-		return ((ctx.hour*60+ctx.min)*60+ctx.sec)*1000 + ctx.millis, nil
-
-	case ftMillisOfSecond:
-		return ctx.millis, nil
+	case ftNanoOfSecond:
+		return ctx.nano, nil
 	}
 
 	return 0, nil

libbeat/common/dtfmt/fmt.go

@@ -60,6 +60,7 @@ func releaseCtx(c *ctx) {
 func NewFormatter(pattern string) (*Formatter, error) {
 	b := newBuilder()
 
+	// pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
 	err := parsePatternTo(b, pattern)
 	if err != nil {
 		return nil, err
@@ -135,6 +136,7 @@ func (f *Formatter) Format(t time.Time) (string, error) {
 }
 
 func parsePatternTo(b *builder, pattern string) error {
+	// pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
 	for i := 0; i < len(pattern); {
 		tok, tokText, err := parseToken(pattern, &i)
 		if err != nil {
@@ -209,11 +211,18 @@ func parsePatternTo(b *builder, pattern string) error {
 			b.secondOfMinute(tokLen)
 
 		case 'S': // fraction of second
-			b.millisOfSecond(tokLen)
+			b.nanoOfSecond(tokLen)
 
 		case 'z': // timezone offset
 			b.timeZoneOffsetText()
 
+		case 'n': // nano second
+			// if timestamp layout use `n`, it always return 9 digits nanoseconds.
+			if tokLen != 9 {
+				tokLen = 9
+			}
+			b.nanoOfSecond(tokLen)
+
 		case '\'': // literal
 			if tokLen == 1 {
 				b.appendRune(rune(tokText[0]))
@@ -234,7 +243,7 @@ func parseToken(pattern string, i *int) (rune, string, error) {
 	start := *i
 	idx := start
 	length := len(pattern)
-
+	// pattern: yyyy-MM-dd'T'HH:mm:ss.fffffffff'Z'
 	r, w := utf8.DecodeRuneInString(pattern[idx:])
 	idx += w
 	if ('A' <= r && r <= 'Z') || ('a' <= r && r <= 'z') {

libbeat/common/dtfmt/prog.go

@@ -36,13 +36,23 @@ const (
 	opCopyLong          // [op, len1, len, content[len1<<8 + len]]
 	opNum               // [op, ft]
 	opNumPadded         // [op, ft, digits]
-	opExtNumPadded      // [op, ft, div, digits]
+	opExtNumPadded      // [op, ft, divExp, digits]
 	opZeros             // [op, count]
 	opTwoDigit          // [op, ft]
 	opTextShort         // [op, ft]
 	opTextLong          // [op, ft]
 )
 
+var pow10Table [10]int
+
+func init() {
+	x := 1
+	for i := range pow10Table {
+		pow10Table[i] = x
+		x *= 10
+	}
+}
+
 func (p prog) eval(bytes []byte, ctx *ctx, t time.Time) ([]byte, error) {
 	for i := 0; i < len(p.p); {
 		op := p.p[i]
@@ -90,7 +100,8 @@ func (p prog) eval(bytes []byte, ctx *ctx, t time.Time) ([]byte, error) {
 			}
 			bytes = appendPadded(bytes, v, digits)
 		case opExtNumPadded:
-			ft, div, digits := fieldType(p.p[i]), int(p.p[i+1]), int(p.p[i+2])
+			ft, divExp, digits := fieldType(p.p[i]), int(p.p[i+1]), int(p.p[i+2])
+			div := pow10Table[divExp]
 			i += 3
 			v, err := getIntField(ft, ctx, t)
 			if err != nil {

libbeat/outputs/codec/common.go

@@ -36,9 +36,9 @@ func MakeTimestampEncoder() func(*time.Time, structform.ExtVisitor) error {
 func MakeUTCOrLocalTimestampEncoder(localTime bool) func(*time.Time, structform.ExtVisitor) error {
 	var dtPattern string
 	if localTime {
-		dtPattern = "yyyy-MM-dd'T'HH:mm:ss.SSSz"
+		dtPattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSSz"
 	} else {
-		dtPattern = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
+		dtPattern = "yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'"
 	}
 
 	formatter, err := dtfmt.NewFormatter(dtPattern)