From d9dd0814b969adb584cc7b52e87cbe6c0eee605a Mon Sep 17 00:00:00 2001 From: Tiago Queiroz Date: Fri, 31 May 2024 11:51:06 -0400 Subject: [PATCH] Fix closerCore With behaviour (#210) This commit fixes the behaviour of `closerCore.With` and add tests to ensure it works as expected. The tests are a modification of existing tests that calls the configuration functions like Beats does and ensures both closerCore and typedCore are used. The method calls get proxyed from one core to another, so both are tested together. The race detector is enabled when testing on Linux and MacOS. Windows requires CGO and installation of gcc on CI, so it is not covered by this commit. --- .buildkite/scripts/test.sh | 2 +- NOTICE.txt | 8 +- go.mod | 2 +- go.sum | 8 +- logp/core.go | 12 +- logp/core_test.go | 64 +++++++++ logp/defaults_test.go | 270 +++++++++++++++++++++++++++++++++++++ logp/selective_test.go | 77 ++++++++--- 8 files changed, 405 insertions(+), 38 deletions(-) diff --git a/.buildkite/scripts/test.sh b/.buildkite/scripts/test.sh index 44287b61..c3a5a443 100755 --- a/.buildkite/scripts/test.sh +++ b/.buildkite/scripts/test.sh @@ -11,7 +11,7 @@ with_go_junit_report echo "--- Go Test" set +e -go test -v ./... > tests-report.txt +go test -race -v ./... > tests-report.txt exit_code=$? set -e diff --git a/NOTICE.txt b/NOTICE.txt index cd90ad8d..32d48354 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1252,11 +1252,11 @@ Contents of probable licence file $GOMODCACHE/github.com/spf13/cobra@v1.7.0/LICE -------------------------------------------------------------------------------- Dependency : github.com/stretchr/testify -Version: v1.8.2 +Version: v1.9.0 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.8.2/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/stretchr/testify@v1.9.0/LICENSE: MIT License @@ -4562,11 +4562,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- Dependency : github.com/stretchr/objx -Version: v0.5.0 +Version: v0.5.2 Licence type (autodetected): MIT -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/stretchr/objx@v0.5.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/stretchr/objx@v0.5.2/LICENSE: The MIT License diff --git a/go.mod b/go.mod index e1ba0328..1842e75b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/mitchellh/hashstructure v1.1.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 github.com/spf13/cobra v1.7.0 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.9.0 go.elastic.co/apm/module/apmhttp/v2 v2.0.0 go.elastic.co/ecszap v1.0.1 go.elastic.co/go-licence-detector v0.5.0 diff --git a/go.sum b/go.sum index 482b4907..98bf07ed 100644 --- a/go.sum +++ b/go.sum @@ -90,17 +90,13 @@ github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRM github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.0/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= diff --git a/logp/core.go b/logp/core.go index bfda3278..04884b84 100644 --- a/logp/core.go +++ b/logp/core.go @@ -26,7 +26,6 @@ import ( "os" "path/filepath" "strings" - "sync" "sync/atomic" "unsafe" @@ -67,14 +66,13 @@ type coreLogger struct { type closerCore struct { zapcore.Core io.Closer - mutex sync.Mutex } -func (c *closerCore) With(fields []zapcore.Field) zapcore.Core { - c.mutex.Lock() - c.Core = c.Core.With(fields) - c.mutex.Unlock() - return c +func (c closerCore) With(fields []zapcore.Field) zapcore.Core { + return closerCore{ + Core: c.Core.With(fields), + Closer: c.Closer, + } } // Configure configures the logp package. diff --git a/logp/core_test.go b/logp/core_test.go index 26176fd8..f3555195 100644 --- a/logp/core_test.go +++ b/logp/core_test.go @@ -635,6 +635,70 @@ func TestCoresCanBeClosed(t *testing.T) { } } +func TestCloserLoggerCoreWith(t *testing.T) { + defaultWriter := writeSyncer{} + + cfg := zap.NewProductionEncoderConfig() + cfg.TimeKey = "" // remove the time to make the log entry consistent + + core := closerCore{ + Core: zapcore.NewCore( + zapcore.NewJSONEncoder(cfg), + &defaultWriter, + zapcore.InfoLevel, + ), + } + + expectedLines := []string{ + // First/Default logger + `{"level":"info","msg":"Very first message"}`, + + // Two messages after calling With + `{"level":"info","msg":"a message with extra fields","foo":"bar"}`, + `{"level":"info","msg":"another message with extra fields","foo":"bar"}`, + + // A message with the default logger + `{"level":"info","msg":"a message without extra fields"}`, + + // Two more messages with a different field + `{"level":"info","msg":"a message with an answer","answer":"42"}`, + `{"level":"info","msg":"another message with an answer","answer":"42"}`, + + // One last message with the default logger + `{"level":"info","msg":"another message without any extra fields"}`, + } + + // The default logger, it should not be modified by any call to With. + logger := zap.New(&core) + logger.Info("Very first message") + + // Add a field and write messages + loggerWithFields := logger.With(strField("foo", "bar")) + loggerWithFields.Info("a message with extra fields") + loggerWithFields.Info("another message with extra fields") + + // Use the default logger again + logger.Info("a message without extra fields") + + // New logger with other fields + loggerWithFields = logger.With(strField("answer", "42")) + loggerWithFields.Info("a message with an answer") + loggerWithFields.Info("another message with an answer") + + // One last message with the default logger + logger.Info("another message without any extra fields") + + scanner := bufio.NewScanner(strings.NewReader(defaultWriter.String())) + count := 0 + for scanner.Scan() { + l := scanner.Text() + if l != expectedLines[count] { + t.Error("Expecting:\n", l, "\nGot:\n", expectedLines[count]) + } + count++ + } +} + func strField(key, val string) zapcore.Field { return zapcore.Field{Type: zapcore.StringType, Key: key, String: val} } diff --git a/logp/defaults_test.go b/logp/defaults_test.go index d3f02237..aeec9bc8 100644 --- a/logp/defaults_test.go +++ b/logp/defaults_test.go @@ -18,12 +18,19 @@ package logp_test import ( + "bufio" "encoding/json" "fmt" "os" "path/filepath" "runtime" + "sort" + "sync" "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap/zapcore" "github.com/elastic/elastic-agent-libs/logp" ) @@ -109,3 +116,266 @@ func TestDefaultConfig(t *testing.T) { t.Log(string(data)) } } + +func TestWith(t *testing.T) { + tempDir := t.TempDir() + + logSelector := "enabled-log-selector" + + defaultCfg := logp.DefaultConfig(logp.DefaultEnvironment) + eventsCfg := logp.DefaultEventConfig(logp.DefaultEnvironment) + + defaultCfg.Level = logp.DebugLevel + defaultCfg.Beat = t.Name() + defaultCfg.Selectors = []string{logSelector} + defaultCfg.ToStderr = false + defaultCfg.ToFiles = true + defaultCfg.Files.Path = tempDir + + eventsCfg.Level = defaultCfg.Level + eventsCfg.Beat = defaultCfg.Beat + eventsCfg.Selectors = defaultCfg.Selectors + eventsCfg.ToStderr = defaultCfg.ToStderr + eventsCfg.ToFiles = defaultCfg.ToFiles + eventsCfg.Files.Path = tempDir + + if err := logp.ConfigureWithTypedOutput(defaultCfg, eventsCfg, "log.type", "event"); err != nil { + t.Fatalf("could not configure logger: %s", err) + } + + expectedLines := []string{ + // First/Default logger + `{"log.level":"info","message":"Very first message"}`, + + // Two messages after calling With + `{"log.level":"info","message":"a message with extra fields","foo":"bar"}`, + `{"log.level":"info","message":"another message with extra fields","foo":"bar"}`, + + // A message with the default logger + `{"log.level":"info","message":"a message without extra fields"}`, + + // Two more messages with a different field + `{"log.level":"info","message":"a message with an answer","answer":"42"}`, + `{"log.level":"info","message":"another message with an answer","answer":"42"}`, + + // One last message with the default logger + `{"log.level":"info","message":"another message without any extra fields"}`, + } + + logger := logp.L() + defer logger.Close() + + logger.Info("Very first message") + + // Add a field and write messages + loggerWithFields := logger.With(strField("foo", "bar")) + loggerWithFields.Info("a message with extra fields") + loggerWithFields.Info("another message with extra fields") + + // Use the default logger again + logger.Info("a message without extra fields") + + // New logger with other fields + loggerWithFields = logger.With(strField("answer", "42")) + loggerWithFields.Info("a message with an answer") + loggerWithFields.Info("another message with an answer") + + // One last message with the default logger + logger.Info("another message without any extra fields") + + entries := takeAllLogsFromPath(t, tempDir) + + if len(expectedLines) != len(entries) { + t.Fatalf("expecting %d lines got %d", len(expectedLines), len(entries)) + } + + sort.Slice(entries, func(i, j int) bool { + t1 := entries[i]["@timestamp"].(string) //nolint: errcheck // We know it's a sting and it is a test + t2 := entries[j]["@timestamp"].(string) //nolint: errcheck // We know it's a sting and it is a test + return t1 < t2 + }) + + // Now that the slice is sorted, remove some fields, including + // the @timestamp we used to sort it + for i := range entries { + delete(entries[i], "@timestamp") + delete(entries[i], "log.origin") + delete(entries[i], "ecs.version") + delete(entries[i], "service.name") + } + + strEntries := []string{} + for _, e := range entries { + data, _ := json.Marshal(e) + strEntries = append(strEntries, string(data)) + } + + for i := range strEntries { + assert.JSONEq(t, strEntries[i], expectedLines[i], "Some log entries are different than expected") + } +} + +func TestConcurrency(t *testing.T) { + tempDir := t.TempDir() + + logSelector := "enabled-log-selector" + + defaultCfg := logp.DefaultConfig(logp.DefaultEnvironment) + eventsCfg := logp.DefaultEventConfig(logp.DefaultEnvironment) + + defaultCfg.Level = logp.DebugLevel + defaultCfg.Beat = t.Name() + defaultCfg.Selectors = []string{logSelector} + defaultCfg.ToStderr = false + defaultCfg.ToFiles = true + defaultCfg.Files.Path = tempDir + + eventsCfg.Level = defaultCfg.Level + eventsCfg.Beat = defaultCfg.Beat + eventsCfg.Selectors = defaultCfg.Selectors + eventsCfg.ToStderr = defaultCfg.ToStderr + eventsCfg.ToFiles = defaultCfg.ToFiles + eventsCfg.Files.Path = tempDir + + expectedLines := []string{ + `{"id":0,"log.level":"info","message":"count: 000","sort_field":0}`, + `{"id":0,"log.level":"info","message":"count: 001","sort_field":10000}`, + `{"id":0,"log.level":"info","message":"count: 002","sort_field":20000}`, + `{"id":0,"log.level":"info","message":"count: 003","sort_field":30000}`, + `{"id":0,"log.level":"info","message":"count: 004","sort_field":40000}`, + `{"id":0,"log.level":"info","message":"count: 005","sort_field":50000}`, + `{"id":0,"log.level":"info","message":"count: 006","sort_field":60000}`, + `{"id":0,"log.level":"info","message":"count: 007","sort_field":70000}`, + `{"id":0,"log.level":"info","message":"count: 008","sort_field":80000}`, + `{"id":0,"log.level":"info","message":"count: 009","sort_field":90000}`, + `{"id":100,"log.level":"info","message":"count: 100","sort_field":1000000}`, + `{"id":100,"log.level":"info","message":"count: 101","sort_field":1010000}`, + `{"id":100,"log.level":"info","message":"count: 102","sort_field":1020000}`, + `{"id":100,"log.level":"info","message":"count: 103","sort_field":1030000}`, + `{"id":100,"log.level":"info","message":"count: 104","sort_field":1040000}`, + `{"id":100,"log.level":"info","message":"count: 105","sort_field":1050000}`, + `{"id":100,"log.level":"info","message":"count: 106","sort_field":1060000}`, + `{"id":100,"log.level":"info","message":"count: 107","sort_field":1070000}`, + `{"id":100,"log.level":"info","message":"count: 108","sort_field":1080000}`, + `{"id":100,"log.level":"info","message":"count: 109","sort_field":1090000}`, + `{"id":200,"log.level":"info","message":"count: 200","sort_field":2000000}`, + `{"id":200,"log.level":"info","message":"count: 201","sort_field":2010000}`, + `{"id":200,"log.level":"info","message":"count: 202","sort_field":2020000}`, + `{"id":200,"log.level":"info","message":"count: 203","sort_field":2030000}`, + `{"id":200,"log.level":"info","message":"count: 204","sort_field":2040000}`, + `{"id":200,"log.level":"info","message":"count: 205","sort_field":2050000}`, + `{"id":200,"log.level":"info","message":"count: 206","sort_field":2060000}`, + `{"id":200,"log.level":"info","message":"count: 207","sort_field":2070000}`, + `{"id":200,"log.level":"info","message":"count: 208","sort_field":2080000}`, + `{"id":200,"log.level":"info","message":"count: 209","sort_field":2090000}`, + `{"id":300,"log.level":"info","message":"count: 300","sort_field":3000000}`, + `{"id":300,"log.level":"info","message":"count: 301","sort_field":3010000}`, + `{"id":300,"log.level":"info","message":"count: 302","sort_field":3020000}`, + `{"id":300,"log.level":"info","message":"count: 303","sort_field":3030000}`, + `{"id":300,"log.level":"info","message":"count: 304","sort_field":3040000}`, + `{"id":300,"log.level":"info","message":"count: 305","sort_field":3050000}`, + `{"id":300,"log.level":"info","message":"count: 306","sort_field":3060000}`, + `{"id":300,"log.level":"info","message":"count: 307","sort_field":3070000}`, + `{"id":300,"log.level":"info","message":"count: 308","sort_field":3080000}`, + `{"id":300,"log.level":"info","message":"count: 309","sort_field":3090000}`, + `{"id":400,"log.level":"info","message":"count: 400","sort_field":4000000}`, + `{"id":400,"log.level":"info","message":"count: 401","sort_field":4010000}`, + `{"id":400,"log.level":"info","message":"count: 402","sort_field":4020000}`, + `{"id":400,"log.level":"info","message":"count: 403","sort_field":4030000}`, + `{"id":400,"log.level":"info","message":"count: 404","sort_field":4040000}`, + `{"id":400,"log.level":"info","message":"count: 405","sort_field":4050000}`, + `{"id":400,"log.level":"info","message":"count: 406","sort_field":4060000}`, + `{"id":400,"log.level":"info","message":"count: 407","sort_field":4070000}`, + `{"id":400,"log.level":"info","message":"count: 408","sort_field":4080000}`, + `{"id":400,"log.level":"info","message":"count: 409","sort_field":4090000}`, + } + + if err := logp.ConfigureWithTypedOutput(defaultCfg, eventsCfg, "log.type", "event"); err != nil { + t.Fatalf("could not configure logger: %s", err) + } + + wg := sync.WaitGroup{} + + for i := 0; i < 500; i += 100 { + wg.Add(1) + go func(id int) { + logger := logp.L().With("id", id) + defer wg.Done() + time.Sleep(time.Millisecond * 100) + for j := 0; j < 10; j++ { + logger.Infow(fmt.Sprintf("count: %03d", id+j), "sort_field", (id+j)*10000) + } + }(i) + } + + wg.Wait() + + entries := takeAllLogsFromPath(t, tempDir) + if len(expectedLines) != len(entries) { + t.Fatalf("expecting %d lines got %d", len(expectedLines), len(entries)) + } + + sort.Slice(entries, func(i, j int) bool { + t1 := entries[i]["sort_field"].(float64) //nolint: errcheck // We know it's a float64 and it is a test + t2 := entries[j]["sort_field"].(float64) //nolint: errcheck // We know it's a float64 and it is a test + return t1 < t2 + }) + + // Now that the slice is sorted, remove some fields, including + // the @timestamp we used to sort it + for i := range entries { + delete(entries[i], "@timestamp") + delete(entries[i], "log.origin") + delete(entries[i], "ecs.version") + delete(entries[i], "service.name") + } + + strEntries := []string{} + for _, e := range entries { + data, _ := json.Marshal(e) + strEntries = append(strEntries, string(data)) + } + + for i := range strEntries { + assert.JSONEq(t, strEntries[i], expectedLines[i], "Some log entries are different than expected") + } + + // Get a logger and close it so the file descriptors are released. + // This is specially important on Windows + logp.L().Close() +} + +func strField(key, val string) zapcore.Field { + return zapcore.Field{Type: zapcore.StringType, Key: key, String: val} +} + +func takeAllLogsFromPath(t *testing.T, path string) []map[string]any { + entries := []map[string]any{} + + glob := filepath.Join(path, "*.ndjson") + files, err := filepath.Glob(glob) + if err != nil { + t.Fatalf("cannot get files for glob '%s': %s", glob, err) + } + + for _, filePath := range files { + f, err := os.Open(filePath) + if err != nil { + t.Fatalf("cannot open file '%s': %s", filePath, err) + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + m := map[string]any{} + bytes := sc.Bytes() + if err := json.Unmarshal(bytes, &m); err != nil { + t.Fatalf("cannot unmarshal log entry: '%s', err: %s", string(bytes), err) + } + + entries = append(entries, m) + } + } + + return entries +} diff --git a/logp/selective_test.go b/logp/selective_test.go index 16c07d87..e4deaac3 100644 --- a/logp/selective_test.go +++ b/logp/selective_test.go @@ -18,6 +18,11 @@ package logp import ( + "bufio" + "encoding/json" + "os" + "path/filepath" + "sort" "testing" "github.com/stretchr/testify/assert" @@ -55,7 +60,9 @@ func TestLoggerSelectors(t *testing.T) { assert.Len(t, logs, 1) } -func TestTypedCoreSelectors(t *testing.T) { +func TestTypedAndCloserCoreSelectors(t *testing.T) { + tempDir := t.TempDir() + logSelector := "enabled-log-selector" expectedMsg := "this should be logged" @@ -63,59 +70,91 @@ func TestTypedCoreSelectors(t *testing.T) { eventsCfg := DefaultEventConfig(DefaultEnvironment) defaultCfg.Level = DebugLevel - defaultCfg.toObserver = true - defaultCfg.ToStderr = false - defaultCfg.ToFiles = false + defaultCfg.Beat = t.Name() defaultCfg.Selectors = []string{logSelector} + defaultCfg.ToStderr = false + defaultCfg.ToFiles = true + defaultCfg.Files.Path = tempDir eventsCfg.Level = defaultCfg.Level - eventsCfg.toObserver = defaultCfg.toObserver + eventsCfg.Beat = defaultCfg.Beat + eventsCfg.Selectors = defaultCfg.Selectors eventsCfg.ToStderr = defaultCfg.ToStderr eventsCfg.ToFiles = defaultCfg.ToFiles - eventsCfg.Selectors = defaultCfg.Selectors + eventsCfg.Files.Path = tempDir if err := ConfigureWithTypedOutput(defaultCfg, eventsCfg, "log.type", "event"); err != nil { t.Fatalf("could not configure logger: %s", err) } enabledSelector := NewLogger(logSelector) + defer enabledSelector.Close() disabledSelector := NewLogger("foo-selector") + defer disabledSelector.Close() enabledSelector.Debugw(expectedMsg) enabledSelector.Debugw(expectedMsg, "log.type", "event") disabledSelector.Debug("this should not be logged") - logEntries := ObserverLogs().TakeAll() + logEntries := takeAllLogsFromPath(t, tempDir) if len(logEntries) != 2 { t.Errorf("expecting 2 log entries, got %d", len(logEntries)) t.Log("Log entries:") for _, e := range logEntries { - t.Log("Message:", e.Message, "Fields:", e.Context) + t.Log(e) } t.FailNow() } for i, logEntry := range logEntries { - msg := logEntry.Message + msg := logEntry["message"].(string) //nolint: errcheck // We know it's a string and it is a test if msg != expectedMsg { t.Fatalf("[%d] expecting log message '%s', got '%s'", i, expectedMsg, msg) } // The second entry should also contain `log.type: event` if i == 1 { - fields := logEntry.Context - if len(fields) != 1 { - t.Errorf("expecting one field, got %d", len(fields)) + logType := logEntry["log.type"].(string) //nolint: errcheck // We know it's a string and it is a test + if logType != "event" { + t.Errorf("expecting value 'event', got '%s'", logType) } + } + } +} - k := fields[0].Key - v := fields[0].String - if k != "log.type" { - t.Errorf("expecting key 'log.type', got '%s'", k) - } - if v != "event" { - t.Errorf("expecting value 'event', got '%s'", v) +func takeAllLogsFromPath(t *testing.T, path string) []map[string]any { + entries := []map[string]any{} + + glob := filepath.Join(path, "*.ndjson") + files, err := filepath.Glob(glob) + if err != nil { + t.Fatalf("cannot get files for glob '%s': %s", glob, err) + } + + for _, filePath := range files { + f, err := os.Open(filePath) + if err != nil { + t.Fatalf("cannot open file '%s': %s", filePath, err) + } + defer f.Close() + + sc := bufio.NewScanner(f) + for sc.Scan() { + m := map[string]any{} + bytes := sc.Bytes() + if err := json.Unmarshal(bytes, &m); err != nil { + t.Fatalf("cannot unmarshal log entry: '%s', err: %s", string(bytes), err) } + + entries = append(entries, m) } } + + sort.Slice(entries, func(i, j int) bool { + t1 := entries[i]["@timestamp"].(string) //nolint: errcheck // We know it's a string and it is a test + t2 := entries[j]["@timestamp"].(string) //nolint: errcheck // We know it's a string and it is a test + return t1 < t2 + }) + + return entries }