diff --git a/auditbeat/module/auditd/audit_linux_test.go b/auditbeat/module/auditd/audit_linux_test.go index 39eef5fecbb..c4e657f54ed 100644 --- a/auditbeat/module/auditd/audit_linux_test.go +++ b/auditbeat/module/auditd/audit_linux_test.go @@ -77,7 +77,7 @@ func TestData(t *testing.T) { } beatEvent := mbtest.StandardizeEvent(ms, events[0], core.AddDatasetToEvent) - mbtest.WriteEventToDataJSON(t, beatEvent) + mbtest.WriteEventToDataJSON(t, beatEvent, "") } func getConfig() map[string]interface{} { diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index d3c8e673fbe..b879b1e425a 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -39,7 +39,7 @@ func TestData(t *testing.T) { } fullEvent := mbtest.StandardizeEvent(ms, events[len(events)-1], core.AddDatasetToEvent) - mbtest.WriteEventToDataJSON(t, fullEvent) + mbtest.WriteEventToDataJSON(t, fullEvent, "") } func TestDetectDeletedFiles(t *testing.T) { diff --git a/metricbeat/mb/testing/data_generator.go b/metricbeat/mb/testing/data_generator.go index a80953f39a2..b6902de7ba6 100644 --- a/metricbeat/mb/testing/data_generator.go +++ b/metricbeat/mb/testing/data_generator.go @@ -32,7 +32,7 @@ func WriteEvent(f mb.EventFetcher, t testing.TB) error { } fullEvent := CreateFullEvent(f, event) - WriteEventToDataJSON(t, fullEvent) + WriteEventToDataJSON(t, fullEvent, "") return nil } @@ -75,13 +75,13 @@ func WriteEventsCond(f mb.EventsFetcher, t testing.TB, cond func(e common.MapStr } fullEvent := CreateFullEvent(f, *event) - WriteEventToDataJSON(t, fullEvent) + WriteEventToDataJSON(t, fullEvent, "") return nil } // WriteEventsReporterV2 fetches events and writes the first event to a ./_meta/data.json // file. -func WriteEventsReporterV2(f mb.ReportingMetricSetV2, t testing.TB) error { +func WriteEventsReporterV2(f mb.ReportingMetricSetV2, t testing.TB, path string) error { if !*dataFlag { t.Skip("skip data generation tests") } @@ -97,7 +97,7 @@ func WriteEventsReporterV2(f mb.ReportingMetricSetV2, t testing.TB) error { e := StandardizeEvent(f, events[0], mb.AddMetricSetInfo) - WriteEventToDataJSON(t, e) + WriteEventToDataJSON(t, e, path) return nil } @@ -141,7 +141,7 @@ func StandardizeEvent(ms mb.MetricSet, e mb.Event, modifiers ...mb.EventModifier // WriteEventToDataJSON writes the given event as "pretty" JSON to // a ./_meta/data.json file. If the -data CLI flag is unset or false then the // method is a no-op. -func WriteEventToDataJSON(t testing.TB, fullEvent beat.Event) { +func WriteEventToDataJSON(t testing.TB, fullEvent beat.Event, pathPostfix string) { if !*dataFlag { return } @@ -151,6 +151,8 @@ func WriteEventToDataJSON(t testing.TB, fullEvent beat.Event) { t.Fatal(err) } + path = path + pathPostfix + fields := fullEvent.Fields fields["@timestamp"] = fullEvent.Timestamp diff --git a/metricbeat/module/elasticsearch/elasticsearch_integration_test.go b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go new file mode 100644 index 00000000000..db3eb01a48a --- /dev/null +++ b/metricbeat/module/elasticsearch/elasticsearch_integration_test.go @@ -0,0 +1,119 @@ +// +build integration + +package elasticsearch_test + +import ( + "fmt" + "net/http" + "os" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/elastic/beats/libbeat/tests/compose" + mbtest "github.com/elastic/beats/metricbeat/mb/testing" + + _ "github.com/elastic/beats/metricbeat/module/elasticsearch/index" + _ "github.com/elastic/beats/metricbeat/module/elasticsearch/node" + _ "github.com/elastic/beats/metricbeat/module/elasticsearch/node_stats" +) + +var metricSets = []string{ + "index", + "node", + "node_stats", +} + +func TestFetch(t *testing.T) { + compose.EnsureUp(t, "elasticsearch") + + err := createIndex(getEnvHost() + ":" + getEnvPort()) + assert.NoError(t, err) + + for _, metricSet := range metricSets { + t.Run(metricSet, func(t *testing.T) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet)) + events, errs := mbtest.ReportingFetchV2(f) + + assert.NotNil(t, events) + assert.Nil(t, errs) + t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0].BeatEvent("elasticsearch", metricSet).Fields.StringToPrint()) + }) + } +} + +func TestData(t *testing.T) { + compose.EnsureUp(t, "elasticsearch") + + for _, metricSet := range metricSets { + t.Run(metricSet, func(t *testing.T) { + f := mbtest.NewReportingMetricSetV2(t, getConfig(metricSet)) + err := mbtest.WriteEventsReporterV2(f, t, "/"+metricSet) + if err != nil { + t.Fatal("write", err) + } + }) + } +} + +// GetEnvHost returns host for Elasticsearch +func getEnvHost() string { + host := os.Getenv("ES_HOST") + + if len(host) == 0 { + host = "127.0.0.1" + } + return host +} + +// GetEnvPort returns port for Elasticsearch +func getEnvPort() string { + port := os.Getenv("ES_PORT") + + if len(port) == 0 { + port = "9200" + } + return port +} + +// GetConfig returns config for elasticsearch module +func getConfig(metricset string) map[string]interface{} { + return map[string]interface{}{ + "module": "elasticsearch", + "metricsets": []string{metricset}, + "hosts": []string{getEnvHost() + ":" + getEnvPort()}, + } +} + +// createIndex creates and elasticsearch index in case it does not exit yet +func createIndex(host string) error { + client := &http.Client{} + + resp, err := http.Get("http://" + host + "/testindex") + if err != nil { + return err + } + resp.Body.Close() + + // This means index already exists + if resp.StatusCode == 200 { + return nil + } + + req, err := http.NewRequest("PUT", "http://"+host+"/testindex", nil) + if err != nil { + return err + } + + resp, err = client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return fmt.Errorf("HTTP error %d: %s", resp.StatusCode, resp.Status) + } + + return nil +} diff --git a/metricbeat/module/elasticsearch/index/_meta/data.json b/metricbeat/module/elasticsearch/index/_meta/data.json index 01d1bb4a00e..0d66c4fec42 100644 --- a/metricbeat/module/elasticsearch/index/_meta/data.json +++ b/metricbeat/module/elasticsearch/index/_meta/data.json @@ -6,25 +6,25 @@ }, "elasticsearch": { "cluster": { - "id": "AI8XrG4wTLKQg3lbRGgQYQ", + "id": "EFdYsDSoSZmU7fXE2wvJ0g", "name": "docker-cluster" }, "index": { - "name": ".monitoring-es-6-2018.04.22", + "name": ".ml-anomalies-shared", "total": { "docs": { - "count": 241933, - "deleted": 525 + "count": 7, + "deleted": 2 }, "segments": { - "count": 16, + "count": 6, "memory": { - "bytes": 565573 + "bytes": 10886 } }, "store": { "size": { - "bytes": 135541290 + "bytes": 44505 } } } diff --git a/metricbeat/module/elasticsearch/index/index_integration_test.go b/metricbeat/module/elasticsearch/index/index_integration_test.go deleted file mode 100644 index 2694a46f1e2..00000000000 --- a/metricbeat/module/elasticsearch/index/index_integration_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// +build integration - -package index - -import ( - "fmt" - "net/http" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/elastic/beats/libbeat/tests/compose" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/elasticsearch" -) - -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "elasticsearch") - - err := createIndex(elasticsearch.GetEnvHost() + ":" + elasticsearch.GetEnvPort()) - assert.NoError(t, err) - - f := mbtest.NewReportingMetricSetV2(t, elasticsearch.GetConfig("index")) - events, errs := mbtest.ReportingFetchV2(f) - - assert.NotNil(t, events) - assert.Nil(t, errs) - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) -} - -func TestData(t *testing.T) { - compose.EnsureUp(t, "elasticsearch") - - err := createIndex(elasticsearch.GetEnvHost() + ":" + elasticsearch.GetEnvPort()) - assert.NoError(t, err) - - f := mbtest.NewReportingMetricSetV2(t, elasticsearch.GetConfig("index")) - err = mbtest.WriteEventsReporterV2(f, t) - if err != nil { - t.Fatal("write", err) - } -} - -// createIndex creates and elasticsearch index in case it does not exit yet -func createIndex(host string) error { - client := &http.Client{} - - resp, err := http.Get("http://" + host + "/testindex") - if err != nil { - return err - } - resp.Body.Close() - - // This means index already exists - if resp.StatusCode == 200 { - return nil - } - - req, err := http.NewRequest("PUT", "http://"+host+"/testindex", nil) - if err != nil { - return err - } - - resp, err = client.Do(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != 200 { - return fmt.Errorf("HTTP error %d: %s", resp.StatusCode, resp.Status) - } - - return nil -} diff --git a/metricbeat/module/elasticsearch/node/_meta/data.json b/metricbeat/module/elasticsearch/node/_meta/data.json index 827e6a2503e..49ddc7ac000 100644 --- a/metricbeat/module/elasticsearch/node/_meta/data.json +++ b/metricbeat/module/elasticsearch/node/_meta/data.json @@ -13,10 +13,10 @@ "memory": { "heap": { "init": { - "bytes": 1073741824 + "bytes": 536870912 }, "max": { - "bytes": 1056309248 + "bytes": 519438336 } }, "nonheap": { @@ -30,7 +30,7 @@ }, "version": "1.8.0_161" }, - "name": "xMLdnuCzSoempTmcPqdtfw", + "name": "q7ya3ngBTS2cxzYi0duQdA", "process": { "mlockall": false }, diff --git a/metricbeat/module/elasticsearch/node/node_integration_test.go b/metricbeat/module/elasticsearch/node/node_integration_test.go deleted file mode 100644 index 221249ea792..00000000000 --- a/metricbeat/module/elasticsearch/node/node_integration_test.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build integration - -package node - -import ( - "testing" - - "github.com/elastic/beats/libbeat/tests/compose" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/elasticsearch" - - "github.com/stretchr/testify/assert" -) - -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "elasticsearch") - - f := mbtest.NewReportingMetricSetV2(t, elasticsearch.GetConfig("node")) - events, errs := mbtest.ReportingFetchV2(f) - - assert.NotNil(t, events) - assert.Nil(t, errs) - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) -} - -func TestData(t *testing.T) { - compose.EnsureUp(t, "elasticsearch") - - f := mbtest.NewReportingMetricSetV2(t, elasticsearch.GetConfig("node")) - err := mbtest.WriteEventsReporterV2(f, t) - if err != nil { - t.Fatal("write", err) - } -} diff --git a/metricbeat/module/elasticsearch/node_stats/_meta/data.json b/metricbeat/module/elasticsearch/node_stats/_meta/data.json index a974350d3f3..37594fd9778 100644 --- a/metricbeat/module/elasticsearch/node_stats/_meta/data.json +++ b/metricbeat/module/elasticsearch/node_stats/_meta/data.json @@ -9,15 +9,15 @@ "name": "docker-cluster" }, "node": { - "name": "VnS118cKS7-8mz8l2jZZKQ", + "name": "q7ya3ngBTS2cxzYi0duQdA", "stats": { "fs": { "summary": { "available": { - "bytes": 36334804992 + "bytes": 31770497024 }, "free": { - "bytes": 39551520768 + "bytes": 34987212800 }, "total": { "bytes": 62725623808 @@ -26,18 +26,18 @@ }, "indices": { "docs": { - "count": 1141228, - "deleted": 42042 + "count": 23809, + "deleted": 34 }, "segments": { - "count": 154, + "count": 54, "memory": { - "bytes": 2862018 + "bytes": 319028 } }, "store": { "size": { - "bytes": 642756442 + "bytes": 8625187 } } }, @@ -46,14 +46,14 @@ "collectors": { "old": { "collection": { - "count": 105, - "ms": 14100 + "count": 5, + "ms": 854 } }, "young": { "collection": { - "count": 43895, - "ms": 423308 + "count": 673, + "ms": 19783 } } } @@ -65,13 +65,13 @@ "bytes": 362414080 }, "peak": { - "bytes": 275023840 + "bytes": 273524720 }, "peak_max": { "bytes": 362414080 }, "used": { - "bytes": 255660936 + "bytes": 138590056 } }, "survivor": { @@ -85,7 +85,7 @@ "bytes": 17432576 }, "used": { - "bytes": 4414672 + "bytes": 5442216 } }, "young": { @@ -99,7 +99,7 @@ "bytes": 139591680 }, "used": { - "bytes": 91758824 + "bytes": 22574552 } } } @@ -108,14 +108,14 @@ } } }, - "event": { - "service": "elasticsearch" - }, "metricset": { "host": "127.0.0.1:9200", "module": "elasticsearch", "name": "node_stats", "namespace": "elasticsearch.node.stats", "rtt": 115 + }, + "service": { + "name": "elasticsearch" } } \ No newline at end of file diff --git a/metricbeat/module/elasticsearch/node_stats/node_stats_integration_test.go b/metricbeat/module/elasticsearch/node_stats/node_stats_integration_test.go deleted file mode 100644 index b9f544aed17..00000000000 --- a/metricbeat/module/elasticsearch/node_stats/node_stats_integration_test.go +++ /dev/null @@ -1,34 +0,0 @@ -// +build integration - -package node_stats - -import ( - "testing" - - "github.com/elastic/beats/libbeat/tests/compose" - mbtest "github.com/elastic/beats/metricbeat/mb/testing" - "github.com/elastic/beats/metricbeat/module/elasticsearch" - - "github.com/stretchr/testify/assert" -) - -func TestFetch(t *testing.T) { - compose.EnsureUp(t, "elasticsearch") - - f := mbtest.NewReportingMetricSetV2(t, elasticsearch.GetConfig("node_stats")) - events, errs := mbtest.ReportingFetchV2(f) - - assert.NotNil(t, events) - assert.Nil(t, errs) - t.Logf("%s/%s event: %+v", f.Module().Name(), f.Name(), events[0]) -} - -func TestData(t *testing.T) { - compose.EnsureUp(t, "elasticsearch") - - f := mbtest.NewReportingMetricSetV2(t, elasticsearch.GetConfig("node_stats")) - err := mbtest.WriteEventsReporterV2(f, t) - if err != nil { - t.Fatal("write", err) - } -} diff --git a/metricbeat/module/elasticsearch/testing.go b/metricbeat/module/elasticsearch/testing.go deleted file mode 100644 index 1631a533521..00000000000 --- a/metricbeat/module/elasticsearch/testing.go +++ /dev/null @@ -1,32 +0,0 @@ -package elasticsearch - -import "os" - -// GetEnvHost returns host for Elasticsearch -func GetEnvHost() string { - host := os.Getenv("ES_HOST") - - if len(host) == 0 { - host = "127.0.0.1" - } - return host -} - -// GetEnvPort returns port for Elasticsearch -func GetEnvPort() string { - port := os.Getenv("ES_PORT") - - if len(port) == 0 { - port = "9200" - } - return port -} - -// GetConfig returns config for elasticsearch module -func GetConfig(metricset string) map[string]interface{} { - return map[string]interface{}{ - "module": "elasticsearch", - "metricsets": []string{metricset}, - "hosts": []string{GetEnvHost() + ":" + GetEnvPort()}, - } -} diff --git a/metricbeat/module/system/core/core_test.go b/metricbeat/module/system/core/core_test.go index 8d4b6f7d69b..718a0c23929 100644 --- a/metricbeat/module/system/core/core_test.go +++ b/metricbeat/module/system/core/core_test.go @@ -24,7 +24,7 @@ func TestData(t *testing.T) { } event := mbtest.CreateFullEvent(f, events[1]) - mbtest.WriteEventToDataJSON(t, event) + mbtest.WriteEventToDataJSON(t, event, "") } func getConfig() map[string]interface{} { diff --git a/metricbeat/module/system/cpu/cpu_test.go b/metricbeat/module/system/cpu/cpu_test.go index 66efa36d6ab..281cdf46402 100644 --- a/metricbeat/module/system/cpu/cpu_test.go +++ b/metricbeat/module/system/cpu/cpu_test.go @@ -23,7 +23,7 @@ func TestData(t *testing.T) { } event := mbtest.CreateFullEvent(f, fields) - mbtest.WriteEventToDataJSON(t, event) + mbtest.WriteEventToDataJSON(t, event, "") } func getConfig() map[string]interface{} { diff --git a/metricbeat/module/system/load/load_test.go b/metricbeat/module/system/load/load_test.go index 6aec866e75b..5ee03112d7d 100644 --- a/metricbeat/module/system/load/load_test.go +++ b/metricbeat/module/system/load/load_test.go @@ -18,7 +18,7 @@ func TestData(t *testing.T) { } event := mbtest.CreateFullEvent(f, load) - mbtest.WriteEventToDataJSON(t, event) + mbtest.WriteEventToDataJSON(t, event, "") } func getConfig() map[string]interface{} { diff --git a/metricbeat/module/system/uptime/metricset_test.go b/metricbeat/module/system/uptime/metricset_test.go index 524bf871c58..96660567a98 100644 --- a/metricbeat/module/system/uptime/metricset_test.go +++ b/metricbeat/module/system/uptime/metricset_test.go @@ -17,7 +17,7 @@ func TestData(t *testing.T) { } event := mbtest.CreateFullEvent(f, uptime) - mbtest.WriteEventToDataJSON(t, event) + mbtest.WriteEventToDataJSON(t, event, "") } func getConfig() map[string]interface{} { diff --git a/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go b/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go index f6f101f92d1..d9a6a615ab5 100644 --- a/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go +++ b/metricbeat/module/windows/perfmon/pdh_integration_windows_test.go @@ -52,7 +52,7 @@ func TestData(t *testing.T) { } beatEvent := mbtest.StandardizeEvent(ms, events[0], mb.AddMetricSetInfo) - mbtest.WriteEventToDataJSON(t, beatEvent) + mbtest.WriteEventToDataJSON(t, beatEvent, "") } func TestQuery(t *testing.T) {