From baaef679088265284bfeeb3f36c666489ee90e5e Mon Sep 17 00:00:00 2001 From: Pucci Date: Mon, 25 Mar 2019 16:12:46 -0300 Subject: [PATCH 1/5] Attention: This mod changes the behaviour of interval log rotation After: Log name based on file change Before: Log name based on rotation timestamp. This patch solves the problem of filenames containing 00-00-00 and adds the possibility of log rotations of 2,3,4,5 m or hours (not only 1m,1h) Important to mention that log rotation is based on logging.metrics.period. So if you put a time of 10s but logging.metrics.period default is 30s, it will rotate logs each 30s. --- libbeat/common/file/interval_rotator.go | 73 +++++++------------------ libbeat/common/file/rotator.go | 5 +- 2 files changed, 22 insertions(+), 56 deletions(-) diff --git a/libbeat/common/file/interval_rotator.go b/libbeat/common/file/interval_rotator.go index b531acca237..a7ff8fccb4d 100644 --- a/libbeat/common/file/interval_rotator.go +++ b/libbeat/common/file/interval_rotator.go @@ -32,7 +32,7 @@ type intervalRotator struct { clock clock weekly bool arbitrary bool - newInterval func(lastTime time.Time, currentTime time.Time) bool + newInterval func(lastTime time.Time, currentTime time.Time, step time.Duration) bool } type clock interface { @@ -61,48 +61,40 @@ func newIntervalRotator(interval time.Duration) (*intervalRotator, error) { func (r *intervalRotator) initialize() error { r.clock = realClock{} - switch r.interval { - case time.Second: + r.newInterval = newStep + if r.interval < time.Minute { r.fileFormat = "2006-01-02-15-04-05" - r.newInterval = newSecond - case time.Minute: + } else if r.interval < time.Hour { r.fileFormat = "2006-01-02-15-04" - r.newInterval = newMinute - case time.Hour: + } else if r.interval < 24*time.Hour { r.fileFormat = "2006-01-02-15" - r.newInterval = newHour - case 24 * time.Hour: // calendar day + } else if r.interval < 7*24*time.Hour { r.fileFormat = "2006-01-02" - r.newInterval = newDay - case 7 * 24 * time.Hour: // calendar week + } else if r.interval < 30*24*time.Hour { r.fileFormat = "" - r.newInterval = newWeek r.weekly = true - case 30 * 24 * time.Hour: // calendar month + } else if r.interval < 365*24*time.Hour { r.fileFormat = "2006-01" - r.newInterval = newMonth - case 365 * 24 * time.Hour: // calendar year - r.fileFormat = "2006" - r.newInterval = newYear - default: + } else if r.interval >= 365*24*time.Hour { + r.fileFormat = "2006-01" + } else { r.arbitrary = true r.fileFormat = "2006-01-02-15-04-05" - r.newInterval = func(lastTime time.Time, currentTime time.Time) bool { + r.newInterval = func(lastTime time.Time, currentTime time.Time, step time.Duration) bool { lastInterval := lastTime.Unix() / (int64(r.interval) / int64(time.Second)) currentInterval := currentTime.Unix() / (int64(r.interval) / int64(time.Second)) return lastInterval != currentInterval } + } + return nil } func (r *intervalRotator) LogPrefix(filename string, modTime time.Time) string { var t time.Time - if r.lastRotate.IsZero() { - t = modTime - } else { - t = r.lastRotate - } + + t = modTime if r.weekly { y, w := t.ISOWeek() @@ -118,7 +110,7 @@ func (r *intervalRotator) LogPrefix(filename string, modTime time.Time) string { func (r *intervalRotator) NewInterval() bool { now := r.clock.Now() - newInterval := r.newInterval(r.lastRotate, now) + newInterval := r.newInterval(r.lastRotate, now, r.interval) return newInterval } @@ -162,33 +154,6 @@ func IntervalLogIndex(filename string) (uint64, int, error) { return u64, i, err } -func newSecond(lastTime time.Time, currentTime time.Time) bool { - return lastTime.Second() != currentTime.Second() || newMinute(lastTime, currentTime) -} - -func newMinute(lastTime time.Time, currentTime time.Time) bool { - return lastTime.Minute() != currentTime.Minute() || newHour(lastTime, currentTime) -} - -func newHour(lastTime time.Time, currentTime time.Time) bool { - return lastTime.Hour() != currentTime.Hour() || newDay(lastTime, currentTime) -} - -func newDay(lastTime time.Time, currentTime time.Time) bool { - return lastTime.Day() != currentTime.Day() || newMonth(lastTime, currentTime) -} - -func newWeek(lastTime time.Time, currentTime time.Time) bool { - lastYear, lastWeek := lastTime.ISOWeek() - currentYear, currentWeek := currentTime.ISOWeek() - return lastWeek != currentWeek || - lastYear != currentYear -} - -func newMonth(lastTime time.Time, currentTime time.Time) bool { - return lastTime.Month() != currentTime.Month() || newYear(lastTime, currentTime) -} - -func newYear(lastTime time.Time, currentTime time.Time) bool { - return lastTime.Year() != currentTime.Year() +func newStep(lastTime time.Time, currentTime time.Time, step time.Duration) bool { + return !currentTime.Before(lastTime.Add(step)) } diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index fb88481fd59..7a3ade1e13c 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -378,14 +378,15 @@ func (r *Rotator) rotate(reason rotateReason) error { } func (r *Rotator) rotateByInterval(reason rotateReason) error { - fi, err := os.Stat(r.filename) + _, err := os.Stat(r.filename) + if os.IsNotExist(err) { return nil } else if err != nil { return errors.Wrap(err, "failed to rotate backups") } - logPrefix := r.intervalRotator.LogPrefix(r.filename, fi.ModTime()) + logPrefix := r.intervalRotator.LogPrefix(r.filename, time.Now()) files, err := filepath.Glob(logPrefix + "*") if err != nil { return errors.Wrap(err, "failed to list logs during rotation") From 3bec275d70377b8f40a31af2d0875ad19b82909e Mon Sep 17 00:00:00 2001 From: Leonardo Pucci <38509299+leopucci@users.noreply.github.com> Date: Tue, 26 Mar 2019 11:10:10 -0300 Subject: [PATCH 2/5] Adjusted yearly rotation t.Format calls AppendFormat that breaks time into year/month/day/hour..etc.. Now yearly rotation will rotate with fixed format filebeat-2019-1 --- libbeat/common/file/interval_rotator.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libbeat/common/file/interval_rotator.go b/libbeat/common/file/interval_rotator.go index a7ff8fccb4d..a48ca71e421 100644 --- a/libbeat/common/file/interval_rotator.go +++ b/libbeat/common/file/interval_rotator.go @@ -31,6 +31,7 @@ type intervalRotator struct { fileFormat string clock clock weekly bool + yearly bool arbitrary bool newInterval func(lastTime time.Time, currentTime time.Time, step time.Duration) bool } @@ -77,6 +78,7 @@ func (r *intervalRotator) initialize() error { r.fileFormat = "2006-01" } else if r.interval >= 365*24*time.Hour { r.fileFormat = "2006-01" + r.yearly = true } else { r.arbitrary = true r.fileFormat = "2006-01-02-15-04-05" @@ -100,6 +102,10 @@ func (r *intervalRotator) LogPrefix(filename string, modTime time.Time) string { y, w := t.ISOWeek() return fmt.Sprintf("%s-%04d-%02d-", filename, y, w) } + if r.yearly { + y, _ := t.ISOWeek() + return fmt.Sprintf("%s-%04d-", filename, y) + } if r.arbitrary { intervalNumber := t.Unix() / (int64(r.interval) / int64(time.Second)) intervalStart := time.Unix(0, intervalNumber*int64(r.interval)) From ceb3d7b9d578583588d6de080a0268de90622d5e Mon Sep 17 00:00:00 2001 From: Leonardo Pucci <38509299+leopucci@users.noreply.github.com> Date: Fri, 29 Mar 2019 09:29:44 -0300 Subject: [PATCH 3/5] Update libbeat/common/file/rotator.go --- libbeat/common/file/rotator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index 7a3ade1e13c..37e61a895ac 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -386,7 +386,7 @@ func (r *Rotator) rotateByInterval(reason rotateReason) error { return errors.Wrap(err, "failed to rotate backups") } - logPrefix := r.intervalRotator.LogPrefix(r.filename, time.Now()) + logPrefix := r.intervalRotator.LogPrefix(r.filename, fi.ModTime()) files, err := filepath.Glob(logPrefix + "*") if err != nil { return errors.Wrap(err, "failed to list logs during rotation") From 83e2b05809331d11f17ebe89855155c531f60cb6 Mon Sep 17 00:00:00 2001 From: Leonardo Pucci <38509299+leopucci@users.noreply.github.com> Date: Fri, 29 Mar 2019 11:17:31 -0300 Subject: [PATCH 4/5] Update rotator.go --- libbeat/common/file/rotator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libbeat/common/file/rotator.go b/libbeat/common/file/rotator.go index 37e61a895ac..8123b3ed459 100644 --- a/libbeat/common/file/rotator.go +++ b/libbeat/common/file/rotator.go @@ -378,7 +378,7 @@ func (r *Rotator) rotate(reason rotateReason) error { } func (r *Rotator) rotateByInterval(reason rotateReason) error { - _, err := os.Stat(r.filename) + fi, err := os.Stat(r.filename) if os.IsNotExist(err) { return nil From a5b1ae15c64c5a672e7a998b77244f6710924aa5 Mon Sep 17 00:00:00 2001 From: Leonardo Pucci Date: Tue, 2 Apr 2019 09:45:28 -0300 Subject: [PATCH 5/5] Removed arbitrary time - fixed tests --- libbeat/common/file/interval_rotator.go | 70 ++++++++++++-------- libbeat/common/file/interval_rotator_test.go | 43 ------------ 2 files changed, 44 insertions(+), 69 deletions(-) diff --git a/libbeat/common/file/interval_rotator.go b/libbeat/common/file/interval_rotator.go index a48ca71e421..3f09b1521ae 100644 --- a/libbeat/common/file/interval_rotator.go +++ b/libbeat/common/file/interval_rotator.go @@ -33,7 +33,7 @@ type intervalRotator struct { weekly bool yearly bool arbitrary bool - newInterval func(lastTime time.Time, currentTime time.Time, step time.Duration) bool + newInterval func(lastTime time.Time, currentTime time.Time) bool } type clock interface { @@ -62,32 +62,29 @@ func newIntervalRotator(interval time.Duration) (*intervalRotator, error) { func (r *intervalRotator) initialize() error { r.clock = realClock{} - r.newInterval = newStep if r.interval < time.Minute { r.fileFormat = "2006-01-02-15-04-05" + r.newInterval = newSecond } else if r.interval < time.Hour { r.fileFormat = "2006-01-02-15-04" + r.newInterval = newMinute } else if r.interval < 24*time.Hour { r.fileFormat = "2006-01-02-15" + r.newInterval = newHour } else if r.interval < 7*24*time.Hour { r.fileFormat = "2006-01-02" + r.newInterval = newDay } else if r.interval < 30*24*time.Hour { r.fileFormat = "" r.weekly = true + r.newInterval = newWeek } else if r.interval < 365*24*time.Hour { r.fileFormat = "2006-01" + r.newInterval = newMonth } else if r.interval >= 365*24*time.Hour { - r.fileFormat = "2006-01" + r.fileFormat = "2006" r.yearly = true - } else { - r.arbitrary = true - r.fileFormat = "2006-01-02-15-04-05" - r.newInterval = func(lastTime time.Time, currentTime time.Time, step time.Duration) bool { - lastInterval := lastTime.Unix() / (int64(r.interval) / int64(time.Second)) - currentInterval := currentTime.Unix() / (int64(r.interval) / int64(time.Second)) - return lastInterval != currentInterval - } - + r.newInterval = newYear } return nil @@ -95,28 +92,22 @@ func (r *intervalRotator) initialize() error { func (r *intervalRotator) LogPrefix(filename string, modTime time.Time) string { var t time.Time - - t = modTime + if r.lastRotate.IsZero() { + t = modTime + } else { + t = r.lastRotate + } if r.weekly { y, w := t.ISOWeek() return fmt.Sprintf("%s-%04d-%02d-", filename, y, w) } - if r.yearly { - y, _ := t.ISOWeek() - return fmt.Sprintf("%s-%04d-", filename, y) - } - if r.arbitrary { - intervalNumber := t.Unix() / (int64(r.interval) / int64(time.Second)) - intervalStart := time.Unix(0, intervalNumber*int64(r.interval)) - return fmt.Sprintf("%s-%s-", filename, intervalStart.Format(r.fileFormat)) - } return fmt.Sprintf("%s-%s-", filename, t.Format(r.fileFormat)) } func (r *intervalRotator) NewInterval() bool { now := r.clock.Now() - newInterval := r.newInterval(r.lastRotate, now, r.interval) + newInterval := r.newInterval(r.lastRotate, now) return newInterval } @@ -160,6 +151,33 @@ func IntervalLogIndex(filename string) (uint64, int, error) { return u64, i, err } -func newStep(lastTime time.Time, currentTime time.Time, step time.Duration) bool { - return !currentTime.Before(lastTime.Add(step)) +func newSecond(lastTime time.Time, currentTime time.Time) bool { + return lastTime.Second() != currentTime.Second() || newMinute(lastTime, currentTime) +} + +func newMinute(lastTime time.Time, currentTime time.Time) bool { + return lastTime.Minute() != currentTime.Minute() || newHour(lastTime, currentTime) +} + +func newHour(lastTime time.Time, currentTime time.Time) bool { + return lastTime.Hour() != currentTime.Hour() || newDay(lastTime, currentTime) +} + +func newDay(lastTime time.Time, currentTime time.Time) bool { + return lastTime.Day() != currentTime.Day() || newMonth(lastTime, currentTime) +} + +func newWeek(lastTime time.Time, currentTime time.Time) bool { + lastYear, lastWeek := lastTime.ISOWeek() + currentYear, currentWeek := currentTime.ISOWeek() + return lastWeek != currentWeek || + lastYear != currentYear +} + +func newMonth(lastTime time.Time, currentTime time.Time) bool { + return lastTime.Month() != currentTime.Month() || newYear(lastTime, currentTime) +} + +func newYear(lastTime time.Time, currentTime time.Time) bool { + return lastTime.Year() != currentTime.Year() } diff --git a/libbeat/common/file/interval_rotator_test.go b/libbeat/common/file/interval_rotator_test.go index fda516d8627..6415b5cd541 100644 --- a/libbeat/common/file/interval_rotator_test.go +++ b/libbeat/common/file/interval_rotator_test.go @@ -209,49 +209,6 @@ func TestYearlyRotator(t *testing.T) { assert.Equal(t, "foo-2019-", a.LogPrefix("foo", time.Now())) } -func TestArbitraryIntervalRotator(t *testing.T) { - a, err := newIntervalRotator(3 * time.Second) - if err != nil { - t.Fatal(err) - } - - // Monday, 2018-Dec-31 - clock := &testClock{time.Date(2018, 12, 31, 0, 0, 1, 0, time.Local)} - a.clock = clock - assert.Equal(t, "foo-2018-12-30-00-00-00-", a.LogPrefix("foo", time.Date(2018, 12, 30, 0, 0, 0, 0, time.Local))) - a.Rotate() - n := a.NewInterval() - assert.False(t, n) - assert.Equal(t, "foo-2018-12-31-00-00-00-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - n = a.NewInterval() - assert.False(t, n) - assert.Equal(t, "foo-2018-12-31-00-00-00-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - n = a.NewInterval() - assert.True(t, n) - a.Rotate() - assert.Equal(t, "foo-2018-12-31-00-00-03-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - n = a.NewInterval() - assert.False(t, n) - assert.Equal(t, "foo-2018-12-31-00-00-03-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - n = a.NewInterval() - assert.False(t, n) - assert.Equal(t, "foo-2018-12-31-00-00-03-", a.LogPrefix("foo", time.Now())) - - clock.time = clock.time.Add(time.Second) - n = a.NewInterval() - assert.True(t, n) - a.Rotate() - assert.Equal(t, "foo-2018-12-31-00-00-06-", a.LogPrefix("foo", time.Now())) -} - func TestIntervalIsTruncatedToSeconds(t *testing.T) { a, err := newIntervalRotator(2345 * time.Millisecond) if err != nil {