diff --git a/dev-tools/packaging/templates/docker/Dockerfile.tmpl b/dev-tools/packaging/templates/docker/Dockerfile.tmpl index 880aac57ccdc..dd8bd021a8c4 100644 --- a/dev-tools/packaging/templates/docker/Dockerfile.tmpl +++ b/dev-tools/packaging/templates/docker/Dockerfile.tmpl @@ -47,7 +47,7 @@ ENV NODE_PATH={{ $beatHome }}/.node RUN echo \ $NODE_PATH \ {{ $beatHome }}/.config \ - {{ $beatHome }}/suites \ + {{ $beatHome }}/.synthetics \ {{ $beatHome }}/.npm \ {{ $beatHome }}/.cache \ | xargs -IDIR sh -c 'mkdir -p DIR && chmod 0770 DIR' diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go index 037dd3f402eb..4fb86cd9e18a 100644 --- a/filebeat/autodiscover/builder/hints/logs.go +++ b/filebeat/autodiscover/builder/hints/logs.go @@ -52,15 +52,14 @@ var validModuleNames = regexp.MustCompile("[^a-zA-Z0-9\\_\\-]+") type logHints struct { config *config registry *fileset.ModuleRegistry + log *logp.Logger } // NewLogHints builds a log hints builder func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) { config := defaultConfig() - err := cfg.Unpack(&config) - - if err != nil { - return nil, fmt.Errorf("unable to unpack hints config due to error: %v", err) + if err := cfg.Unpack(&config); err != nil { + return nil, fmt.Errorf("unable to unpack hints config due to error: %w", err) } moduleRegistry, err := fileset.NewModuleRegistry(nil, beat.Info{}, false) @@ -68,39 +67,33 @@ func NewLogHints(cfg *common.Config) (autodiscover.Builder, error) { return nil, err } - return &logHints{&config, moduleRegistry}, nil + return &logHints{&config, moduleRegistry, logp.NewLogger("hints.builder")}, nil } // Create config based on input hints in the bus event func (l *logHints) CreateConfig(event bus.Event, options ...ucfg.Option) []*common.Config { var hints common.MapStr - hIface, ok := event["hints"] - if ok { - hints, _ = hIface.(common.MapStr) - } - - inputConfig := l.getInputsConfigs(hints) - - // If default config is disabled return nothing unless it's explicty enabled - if !l.config.DefaultConfig.Enabled() && !builder.IsEnabled(hints, l.config.Key) { - logp.Debug("hints.builder", "default config is disabled: %+v", event) - return []*common.Config{} + if hintsIfc, found := event["hints"]; found { + hints, _ = hintsIfc.(common.MapStr) } - // If explictly disabled, return nothing - if builder.IsDisabled(hints, l.config.Key) { - logp.Debug("hints.builder", "logs disabled by hint: %+v", event) - return []*common.Config{} + // Hint must be explicitly enabled when default_config sets enabled=false. + if !l.config.DefaultConfig.Enabled() && !builder.IsEnabled(hints, l.config.Key) || + builder.IsDisabled(hints, l.config.Key) { + l.log.Debugw("Hints config is not enabled.", "autodiscover.event", event) + return nil } - if inputConfig != nil { - configs := []*common.Config{} + if inputConfig := l.getInputsConfigs(hints); inputConfig != nil { + var configs []*common.Config for _, cfg := range inputConfig { if config, err := common.NewConfigFrom(cfg); err == nil { configs = append(configs, config) + } else { + l.log.Warnw("Failed to create config from input.", "error", err) } } - logp.Debug("hints.builder", "generated config %+v", configs) + l.log.Debugf("Generated %d input configs from hint.", len(configs)) // Apply information in event to the template to generate the final config return template.ApplyConfigTemplate(event, configs) } diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go index 0dadfe54798c..e00ec39920eb 100644 --- a/filebeat/autodiscover/builder/hints/logs_test.go +++ b/filebeat/autodiscover/builder/hints/logs_test.go @@ -22,6 +22,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/common/bus" @@ -692,9 +693,9 @@ func TestGenerateHints(t *testing.T) { for _, test := range tests { // Configure path for modules access abs, _ := filepath.Abs("../../..") - err := paths.InitPaths(&paths.Path{ + require.NoError(t, paths.InitPaths(&paths.Path{ Home: abs, - }) + })) l, err := NewLogHints(test.config) if err != nil { @@ -927,9 +928,9 @@ func TestGenerateHintsWithPaths(t *testing.T) { // Configure path for modules access abs, _ := filepath.Abs("../../..") - err := paths.InitPaths(&paths.Path{ + require.NoError(t, paths.InitPaths(&paths.Path{ Home: abs, - }) + })) l, err := NewLogHints(cfg) if err != nil { @@ -937,7 +938,7 @@ func TestGenerateHintsWithPaths(t *testing.T) { } cfgs := l.CreateConfig(test.event) - assert.Equal(t, test.len, len(cfgs), test.msg) + require.Equal(t, test.len, len(cfgs), test.msg) if test.len != 0 { config := common.MapStr{} err := cfgs[0].Unpack(&config) diff --git a/filebeat/docs/autodiscover-hints.asciidoc b/filebeat/docs/autodiscover-hints.asciidoc index b63fb6979bdf..55b8f4adb5fa 100644 --- a/filebeat/docs/autodiscover-hints.asciidoc +++ b/filebeat/docs/autodiscover-hints.asciidoc @@ -259,7 +259,9 @@ You can label Docker containers with useful info to decode logs structured as JS [float] ==== Nomad -Nomad autodiscover provider supports hints using the https://www.nomadproject.io/docs/job-specification/meta.html[`meta` stanza]. To enable it just set `hints.enabled`: +Nomad autodiscover provider supports hints using the +https://www.nomadproject.io/docs/job-specification/meta.html[`meta` stanza]. To +enable it just set `hints.enabled`: [source,yaml] ----- @@ -269,7 +271,8 @@ filebeat.autodiscover: hints.enabled: true ----- -You can configure the default config that will be launched when a new job is seen, like this: +You can configure the default config that will be launched when a new job is +seen, like this: [source,yaml] ----- @@ -278,13 +281,13 @@ filebeat.autodiscover: - type: nomad hints.enabled: true hints.default_config: - type: nomad + type: log paths: - - /var/lib/nomad/alloc/${data.nomad.allocation.id}/alloc/logs/${data.nomad.task.name}.* + - /opt/nomad/alloc/${data.nomad.allocation.id}/alloc/logs/${data.nomad.task.name}.* ----- -You can also disable default settings entirely, so only Jobs annotated like `co.elastic.logs/enabled: true` -will be retrieved: +You can also disable the default config such that only logs from jobs explicitly +annotated with `"co.elastic.logs/enabled" = "true"` will be collected: [source,yaml] ----- @@ -292,17 +295,67 @@ filebeat.autodiscover: providers: - type: nomad hints.enabled: true - hints.default_config.enabled: false + hints.default_config: + enabled: false + type: log + paths: + - /opt/nomad/alloc/${data.nomad.allocation.id}/alloc/logs/${data.nomad.task.name}.* ----- -You can annotate Nomad Jobs using the `meta` stanza with useful info to spin up {beatname_uc} inputs -or modules: +You can annotate Nomad Jobs using the `meta` stanza with useful info to spin up +{beatname_uc} inputs or modules: [source,hcl] ----- meta { - "co.elastic.logs/multiline.pattern" = "^\[" - "co.elastic.logs/multiline.negate" = true - "co.elastic.logs/multiline.match" = after + "co.elastic.logs/enabled" = "true" + "co.elastic.logs/multiline.pattern" = "^\\[" + "co.elastic.logs/multiline.negate" = "true" + "co.elastic.logs/multiline.match" = "after" } ----- + +If you are using autodiscover then in most cases you will want to use the +<> processor to enrich events with +Nomad metadata. This example configures {{beatname_uc}} to connect to the local +Nomad agent over HTTPS and adds the Nomad allocation ID to all events from the +input. Later in the pipeline the `add_nomad_metadata` processor will use that ID +to enrich the event. + +[source,yaml] +----- +filebeat.autodiscover: + providers: + - type: nomad + address: https://localhost:4646 + hints.enabled: true + hints.default_config: + enabled: false <1> + type: log + paths: + - /opt/nomad/alloc/${data.nomad.allocation.id}/alloc/logs/${data.nomad.task.name}.* + processors: + - add_fields: <2> + target: nomad + fields: + allocation.id: ${data.nomad.allocation.id} + +processors: + - add_nomad_metadata: <3> + when.has_fields.fields: [nomad.allocation.id] + address: https://localhost:4646 + default_indexers.enabled: false + default_matchers.enabled: false + indexers: + - allocation_uuid: + matchers: + - fields: + lookup_fields: + - 'nomad.allocation.id' +----- +<1> The default config is disabled meaning any task without the +`"co.elastic.logs/enabled" = "true"` metadata will be ignored. +<2> The `add_fields` processor populates the `nomad.allocation.id` field with +the Nomad allocation UUID. +<3> The `add_nomad_metadata` processor is configured at the global level so +that it is only instantiated one time which saves resources. diff --git a/filebeat/docs/autodiscover-nomad-config.asciidoc b/filebeat/docs/autodiscover-nomad-config.asciidoc index 2e72ae7fae71..b04255223ab9 100644 --- a/filebeat/docs/autodiscover-nomad-config.asciidoc +++ b/filebeat/docs/autodiscover-nomad-config.asciidoc @@ -43,6 +43,6 @@ filebeat.autodiscover: - /var/lib/nomad/alloc/${data.nomad.allocation.id}/alloc/logs/${data.nomad.task.name}.* ------------------------------------------------------------------------------------- -WARNING: The `docker` input is currently not supported. Nomad doesn't expose the container id -associated with the allocation. Without the container id, there is no way of generating the proper +WARNING: The `docker` input is currently not supported. Nomad doesn't expose the container ID +associated with the allocation. Without the container ID, there is no way of generating the proper path for reading the container's logs. diff --git a/heartbeat/tests/system/test_autodiscovery.py b/heartbeat/tests/system/test_autodiscovery.py index 7f5060ace81a..cbc1b6799819 100644 --- a/heartbeat/tests/system/test_autodiscovery.py +++ b/heartbeat/tests/system/test_autodiscovery.py @@ -40,7 +40,7 @@ def test_docker(self): proc = self.start_beat() self.wait_until(lambda: self.log_contains( - re.compile('autodiscover.+Got a start event:', re.I))) + re.compile('autodiscover.+Got a start event', re.I))) self.wait_until(lambda: self.output_count(lambda x: x >= 1)) diff --git a/libbeat/autodiscover/autodiscover.go b/libbeat/autodiscover/autodiscover.go index f36faa6b8f25..8c6977e8362b 100644 --- a/libbeat/autodiscover/autodiscover.go +++ b/libbeat/autodiscover/autodiscover.go @@ -169,7 +169,7 @@ func (a *Autodiscover) worker() { } func (a *Autodiscover) handleStart(event bus.Event) bool { - a.logger.Debugf("Got a start event: %v", event) + a.logger.Debugw("Got a start event.", "autodiscover.event", event) eventID := getID(event) if eventID == "" { diff --git a/libbeat/autodiscover/builder/helper.go b/libbeat/autodiscover/builder/helper.go index faaa112c65f3..6f78973e6fbf 100644 --- a/libbeat/autodiscover/builder/helper.go +++ b/libbeat/autodiscover/builder/helper.go @@ -29,6 +29,8 @@ import ( "github.com/elastic/beats/v7/libbeat/logp" ) +const logName = "autodiscover.builder" + // GetContainerID returns the id of a container func GetContainerID(container common.MapStr) string { id, _ := container["id"].(string) @@ -92,7 +94,7 @@ func GetProcessors(hints common.MapStr, key string) []common.MapStr { if str, ok := value.(string); ok { cfg := common.MapStr{} if err := json.Unmarshal([]byte(str), &cfg); err != nil { - logp.Debug("autodiscover.builder", "unable to unmarshal json due to error: %v", err) + logp.NewLogger(logName).Debugw("Unable to unmarshal json due to error", "error", err) continue } proc[key] = cfg @@ -124,7 +126,7 @@ func GetConfigs(hints common.MapStr, key, name string) []common.MapStr { var configs []common.MapStr for _, key := range nums { - rawCfg, _ := raw[key] + rawCfg := raw[key] if config, ok := rawCfg.(common.MapStr); ok { configs = append(configs, config) } @@ -159,15 +161,15 @@ func GetHintAsConfigs(hints common.MapStr, key string) []common.MapStr { if str[0] != '[' { cfg := common.MapStr{} if err := json.Unmarshal([]byte(str), &cfg); err != nil { - logp.Debug("autodiscover.builder", "unable to unmarshal json due to error: %v", err) + logp.NewLogger(logName).Debugw("Unable to unmarshal json due to error", "error", err) return nil } return []common.MapStr{cfg} } - cfg := []common.MapStr{} + var cfg []common.MapStr if err := json.Unmarshal([]byte(str), &cfg); err != nil { - logp.Debug("autodiscover.builder", "unable to unmarshal json due to error: %v", err) + logp.NewLogger(logName).Debugw("Unable to unmarshal json due to error", "error", err) return nil } return cfg @@ -175,7 +177,7 @@ func GetHintAsConfigs(hints common.MapStr, key string) []common.MapStr { return nil } -// IsEnabled will return true when 'enabled' is **explicity** set to true +// IsEnabled will return true when 'enabled' is **explicitly** set to true. func IsEnabled(hints common.MapStr, key string) bool { if value, err := hints.GetValue(fmt.Sprintf("%s.enabled", key)); err == nil { enabled, _ := strconv.ParseBool(value.(string)) @@ -185,14 +187,16 @@ func IsEnabled(hints common.MapStr, key string) bool { return false } -// IsDisabled will return true when 'enabled' key is **explicity** set to false +// IsDisabled will return true when 'enabled' is **explicitly** set to false. func IsDisabled(hints common.MapStr, key string) bool { if value, err := hints.GetValue(fmt.Sprintf("%s.enabled", key)); err == nil { enabled, err := strconv.ParseBool(value.(string)) - if err == nil { - logp.Debug("autodiscover.builder", "error parsing 'enabled' hint from: %+v", hints) - return !enabled + if err != nil { + logp.NewLogger(logName).Debugw("Error parsing 'enabled' hint.", + "error", err, "autodiscover.hints", hints) + return false } + return !enabled } // keep reading disable (deprecated) for backwards compatibility @@ -271,7 +275,7 @@ func GetHintsAsList(hints common.MapStr, key string) []common.MapStr { var configs []common.MapStr for _, key := range nums { - rawCfg, _ := raw[key] + rawCfg := raw[key] if config, ok := rawCfg.(common.MapStr); ok { configs = append(configs, config) } diff --git a/libbeat/autodiscover/builder/plugin.go b/libbeat/autodiscover/builder/plugin.go index 83abccc2097c..8ad423da13fb 100644 --- a/libbeat/autodiscover/builder/plugin.go +++ b/libbeat/autodiscover/builder/plugin.go @@ -29,7 +29,7 @@ type builderPlugin struct { builder autodiscover.BuilderConstructor } -var pluginKey = "libbeat.autodiscover.builder" +const pluginKey = "libbeat.autodiscover.builder" // Plugin accepts a BuilderConstructor to be registered as a plugin func Plugin(name string, b autodiscover.BuilderConstructor) map[string][]interface{} { diff --git a/libbeat/docs/shared-autodiscover.asciidoc b/libbeat/docs/shared-autodiscover.asciidoc index 7bfed6f8f742..1d19a0f4bac1 100644 --- a/libbeat/docs/shared-autodiscover.asciidoc +++ b/libbeat/docs/shared-autodiscover.asciidoc @@ -482,7 +482,21 @@ The `nomad` autodiscover provider has the following configuration settings: `namespace`:: (Optional) Namespace to use. If not provided the `default` namespace is used. -`secret_id`:: (Optional) SecretID to use if ACL is enabled in Nomad. +`secret_id`:: (Optional) SecretID to use if ACL is enabled in Nomad. This is an +example ACL policy to apply to the token. + +[source,hcl] +---- +namespace "*" { + policy = "read" +} +node { + policy = "read" +} +agent { + policy = "read" +} +---- `node`:: (Optional) Specify the node to scope {beatname_lc} to in case it cannot be accurately detected when `node` scope is used. @@ -496,6 +510,7 @@ The `nomad` autodiscover provider has the following configuration settings: `allow_stale`:: (Optional) allows any Nomad server (non-leader) to service a read. This normally means that the local node where filebeat is allocated will service filebeat's requests. + Defaults to `true`. include::../../{beatname_lc}/docs/autodiscover-nomad-config.asciidoc[] diff --git a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go index 5a511d96b465..974b8f85bf9d 100644 --- a/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go +++ b/libbeat/processors/add_cloud_metadata/add_cloud_metadata.go @@ -118,25 +118,10 @@ func (p *addCloudMetadata) Run(event *beat.Event) (*beat.Event, error) { return event, nil } - // If cloud key exists in event already and overwrite flag is set to false, this processor will not overwrite the - // cloud fields. For example aws module writes cloud.instance.* to events already, with overwrite=false, - // add_cloud_metadata should not overwrite these fields with new values. - if !p.initData.overwrite { - cloudValue, _ := event.GetValue("cloud") - if cloudValue != nil { - err := p.extractECSMeta(event, meta) - if err != nil { - return nil, err - } - return event, nil - } - } - - err := p.extractECSMeta(event, meta) + err := p.addMeta(event, meta) if err != nil { return nil, err } - _, err = event.PutValue("cloud", meta) return event, err } @@ -144,23 +129,21 @@ func (p *addCloudMetadata) String() string { return "add_cloud_metadata=" + p.getMeta().String() } -func (p *addCloudMetadata) extractECSMeta(event *beat.Event, meta common.MapStr) error { - // handle ECS fields first - if !p.initData.overwrite { - orchestratorValue, _ := event.GetValue("orchestrator") - if orchestratorValue != nil { - meta.Delete("orchestrator") - return nil +func (p *addCloudMetadata) addMeta(event *beat.Event, meta common.MapStr) error { + for key, metaVal := range meta { + // If key exists in event already and overwrite flag is set to false, this processor will not overwrite the + // meta fields. For example aws module writes cloud.instance.* to events already, with overwrite=false, + // add_cloud_metadata should not overwrite these fields with new values. + if !p.initData.overwrite { + v, _ := event.GetValue(key) + if v != nil { + continue + } } - } - orchestratorFields, err := meta.GetValue("orchestrator") - if err == nil { - _, err = event.PutValue("orchestrator", orchestratorFields) + _, err := event.PutValue(key, metaVal) if err != nil { return err } } - meta.Delete("orchestrator") - return nil } diff --git a/libbeat/processors/add_cloud_metadata/http_fetcher.go b/libbeat/processors/add_cloud_metadata/http_fetcher.go index e337edd5be9b..0af5693526a5 100644 --- a/libbeat/processors/add_cloud_metadata/http_fetcher.go +++ b/libbeat/processors/add_cloud_metadata/http_fetcher.go @@ -75,7 +75,7 @@ func (f *httpMetadataFetcher) fetchMetadata(ctx context.Context, client http.Cli // Apply schema. res.metadata = f.conv(res.metadata) - res.metadata["provider"] = f.provider + res.metadata.Put("cloud.provider", f.provider) return res } diff --git a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go index 65dd9c232868..cf82c2d621ae 100644 --- a/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go +++ b/libbeat/processors/add_cloud_metadata/provider_alibaba_cloud.go @@ -36,7 +36,7 @@ var alibabaCloudMetadataFetcher = provider{ m["service"] = common.MapStr{ "name": "ECS", } - return common.MapStr(m) + return common.MapStr{"cloud": m} } urls, err := getMetadataURLs(c, ecsMetadataHost, []string{ diff --git a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go index cbd825714681..350ec63b8c6a 100644 --- a/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go +++ b/libbeat/processors/add_cloud_metadata/provider_aws_ec2.go @@ -45,7 +45,7 @@ var ec2MetadataFetcher = provider{ "account": s.Object{"id": c.Str("accountId")}, "image": s.Object{"id": c.Str("imageId")}, }.Apply(m) - return out + return common.MapStr{"cloud": out} } fetcher, err := newMetadataFetcher(config, "aws", nil, metadataHost, ec2Schema, ec2InstanceIdentityURI) diff --git a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go index b3f2a0b3222b..8ba8a2284b35 100644 --- a/libbeat/processors/add_cloud_metadata/provider_azure_vm.go +++ b/libbeat/processors/add_cloud_metadata/provider_azure_vm.go @@ -50,7 +50,7 @@ var azureVMMetadataFetcher = provider{ }, "region": c.Str("location"), }.Apply(m) - return out + return common.MapStr{"cloud": out} } fetcher, err := newMetadataFetcher(config, "azure", azHeaders, metadataHost, azSchema, azMetadataURI) diff --git a/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go b/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go index 04da6228378f..63c90bc5a96e 100644 --- a/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go +++ b/libbeat/processors/add_cloud_metadata/provider_digital_ocean.go @@ -41,7 +41,7 @@ var doMetadataFetcher = provider{ "name": c.Str("serviceName"), }, }.Apply(m) - return out + return common.MapStr{"cloud": out} } doMetadataURI := "/metadata/v1.json" diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce.go b/libbeat/processors/add_cloud_metadata/provider_google_gce.go index b9d6d4cc203d..d6cb4bb74c13 100644 --- a/libbeat/processors/add_cloud_metadata/provider_google_gce.go +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce.go @@ -49,14 +49,15 @@ var gceMetadataFetcher = provider{ gceMetadataURI := "/computeMetadata/v1/?recursive=true&alt=json" gceHeaders := map[string]string{"Metadata-Flavor": "Google"} gceSchema := func(m map[string]interface{}) common.MapStr { - out := common.MapStr{ + cloud := common.MapStr{ "service": common.MapStr{ "name": "GCE", }, } + meta := common.MapStr{} trimLeadingPath := func(key string) { - v, err := out.GetValue(key) + v, err := cloud.GetValue(key) if err != nil { return } @@ -64,7 +65,7 @@ var gceMetadataFetcher = provider{ if !ok { return } - out.Put(key, path.Base(p)) + cloud.Put(key, path.Base(p)) } if instance, ok := m["instance"].(map[string]interface{}); ok { @@ -77,6 +78,10 @@ var gceMetadataFetcher = provider{ "type": c.Str("machineType"), }, "availability_zone": c.Str("zone"), + }.ApplyTo(cloud, instance) + trimLeadingPath("machine.type") + trimLeadingPath("availability_zone") + s.Schema{ "orchestrator": s.Object{ "cluster": c.Dict( "attributes", @@ -85,29 +90,27 @@ var gceMetadataFetcher = provider{ "kubeconfig": c.Str("kubeconfig"), }), }, - }.ApplyTo(out, instance) - trimLeadingPath("machine.type") - trimLeadingPath("availability_zone") + }.ApplyTo(meta, instance) } - if kubeconfig, err := out.GetValue("orchestrator.cluster.kubeconfig"); err == nil { + if kubeconfig, err := meta.GetValue("orchestrator.cluster.kubeconfig"); err == nil { kubeConfig, ok := kubeconfig.(string) if !ok { - out.Delete("orchestrator.cluster.kubeconfig") + meta.Delete("orchestrator.cluster.kubeconfig") } cc := &KubeConfig{} err := yaml.Unmarshal([]byte(kubeConfig), cc) if err != nil { - out.Delete("orchestrator.cluster.kubeconfig") + meta.Delete("orchestrator.cluster.kubeconfig") } if len(cc.Clusters) > 0 { if cc.Clusters[0].Cluster.Server != "" { - out.Delete("orchestrator.cluster.kubeconfig") - out.Put("orchestrator.cluster.url", cc.Clusters[0].Cluster.Server) + meta.Delete("orchestrator.cluster.kubeconfig") + meta.Put("orchestrator.cluster.url", cc.Clusters[0].Cluster.Server) } } } else { - out.Delete("orchestrator") + meta.Delete("orchestrator") } if project, ok := m["project"].(map[string]interface{}); ok { @@ -118,10 +121,11 @@ var gceMetadataFetcher = provider{ "account": s.Object{ "id": c.Str("projectId"), }, - }.ApplyTo(out, project) + }.ApplyTo(cloud, project) } - return out + meta.DeepUpdate(common.MapStr{"cloud": cloud}) + return meta } fetcher, err := newMetadataFetcher(config, provider, gceHeaders, metadataHost, gceSchema, gceMetadataURI) diff --git a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go index 01ada43cfb3e..9922e853d2ee 100644 --- a/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go +++ b/libbeat/processors/add_cloud_metadata/provider_openstack_nova.go @@ -49,7 +49,7 @@ func buildOpenstackNovaCreate(scheme string) func(provider string, c *common.Con m["service"] = common.MapStr{ "name": "Nova", } - return common.MapStr(m) + return common.MapStr{"cloud": m} } urls, err := getMetadataURLsWithScheme(c, scheme, metadataHost, []string{ diff --git a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go index 0f09f4944ae3..f562e2e9609b 100644 --- a/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go +++ b/libbeat/processors/add_cloud_metadata/provider_tencent_cloud.go @@ -36,7 +36,7 @@ var qcloudMetadataFetcher = provider{ m["service"] = common.MapStr{ "name": "CVM", } - return common.MapStr(m) + return common.MapStr{"cloud": m} } urls, err := getMetadataURLs(c, qcloudMetadataHost, []string{ diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index 6accbbe91dd7..6dda8fcef330 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -75,6 +75,7 @@ grouped in the following categories: * <> * <> * <> +* <> * <> * <> * <> @@ -49252,6 +49253,1210 @@ type: object -- +[[exported-fields-syncgateway]] +== SyncGateway fields + +SyncGateway metrics + + +[float] +=== syncgateway + +`syncgateway` contains the information and statistics from SyncGateway. + + + +[float] +=== syncgateway + +Couchbase Sync Gateway metrics. + + + +*`syncgateway.syncgateway.name`*:: ++ +-- +Name of the database on when field `couchbase.syncgateway.type` is `db_stats`. + + +type: keyword + +-- + +[float] +=== metrics + +Metrics of all databases contained in the config file of the SyncGateway instance. + + + + +*`syncgateway.syncgateway.metrics.docs.writes.conflict.count`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.metrics.docs.writes.count`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.metrics.docs.writes.bytes`*:: ++ +-- +type: long + +-- + + +*`syncgateway.syncgateway.metrics.replications.active`*:: ++ +-- +Number of active replications + +type: long + +-- + +*`syncgateway.syncgateway.metrics.replications.total`*:: ++ +-- +Total number of replications (active or not) + +type: long + +-- + + + + + +*`syncgateway.syncgateway.gsi.views.tombstones.query.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.tombstones.query.time`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.tombstones.query.error.count`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.access.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.access.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.access.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.channels.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.channels.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.channels.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.channels.star.query.time`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.channels.star.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.channels.star.query.error.count`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.role_access.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.role_access.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.role_access.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.sequences.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.sequences.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.sequences.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.all_docs.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.all_docs.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.all_docs.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.principals.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.principals.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.principals.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.resync.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.resync.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.resync.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.gsi.views.sessions.query.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.gsi.views.sessions.query.error.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.gsi.views.sessions.query.time`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.security.access_errors.count`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.security.auth.failed.count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.security.docs_rejected.count`*:: ++ +-- +type: double + +-- + + + + +*`syncgateway.syncgateway.cache.channel.revs.active`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.channel.revs.removal`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.channel.revs.tombstone`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.channel.hits`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.channel.misses`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cache.revs.hits`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cache.revs.misses`*:: ++ +-- +type: double + +-- + + + + +*`syncgateway.syncgateway.cbl.replication.pull.caught_up`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.since_zero`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.total.continuous`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.total.one_shot`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.active.continuous`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.active.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.active.one_shot`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.attachment.bytes`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.attachment.count`*:: ++ +-- +type: long + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.request_changes.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.request_changes.time`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.rev.processing_time`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.pull.rev.send.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.pull.rev.send.latency`*:: ++ +-- +type: double + +-- + + + +*`syncgateway.syncgateway.cbl.replication.push.attachment.bytes`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.attachment.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.doc_push_count`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.push.propose_change.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.propose_change.time`*:: ++ +-- +type: double + +-- + + +*`syncgateway.syncgateway.cbl.replication.push.sync_function.count`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.sync_function.time`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.cbl.replication.push.write_processing_time`*:: ++ +-- +type: double + +-- + +[float] +=== memstats + +Dumps a large amount of information about the memory heap and garbage collector + + +*`syncgateway.syncgateway.memstats.BuckHashSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Mallocs`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.PauseTotalNs`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.TotalAlloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Alloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.GCSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.LastGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.MSpanSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.GCCPUFraction`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.HeapReleased`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.HeapSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.DebugGC`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.memstats.HeapIdle`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Lookups`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.HeapObjects`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.MSpanInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.NumForcedGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.OtherSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Frees`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.NextGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.StackInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.Sys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.NumGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.EnableGC`*:: ++ +-- +type: long + +-- + +*`syncgateway.syncgateway.memstats.HeapAlloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.MCacheInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.MCacheSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.HeapInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.syncgateway.memstats.StackSys`*:: ++ +-- +type: double + +-- + +[float] +=== memory + +SyncGateway memory metrics. It dumps a large amount of information about the memory heap and garbage collector + + + +*`syncgateway.memory.BuckHashSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Mallocs`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.PauseTotalNs`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.TotalAlloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Alloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.GCSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.LastGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.MSpanSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.GCCPUFraction`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.HeapReleased`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.HeapSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.DebugGC`*:: ++ +-- +type: long + +-- + +*`syncgateway.memory.HeapIdle`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Lookups`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.HeapObjects`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.MSpanInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.NumForcedGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.OtherSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Frees`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.NextGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.StackInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.Sys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.NumGC`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.EnableGC`*:: ++ +-- +type: long + +-- + +*`syncgateway.memory.HeapAlloc`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.MCacheInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.MCacheSys`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.HeapInuse`*:: ++ +-- +type: double + +-- + +*`syncgateway.memory.StackSys`*:: ++ +-- +type: double + +-- + +[float] +=== replication + +SyncGateway per replication metrics. + + + +[float] +=== metrics + +Metrics related with data replication. + + + + +*`syncgateway.replication.metrics.attachment.transferred.bytes`*:: ++ +-- +Number of attachment bytes transferred for this replica. + +type: long + +-- + +*`syncgateway.replication.metrics.attachment.transferred.count`*:: ++ +-- +The total number of attachments transferred since replication started. + +type: long + +-- + + +*`syncgateway.replication.metrics.docs.checked_sent`*:: ++ +-- +The total number of documents checked for changes since replication started. + +type: double + +-- + + +*`syncgateway.replication.metrics.docs.pushed.count`*:: ++ +-- +The total number of documents checked for changes since replication started. + +type: long + +-- + +*`syncgateway.replication.metrics.docs.pushed.failed`*:: ++ +-- +The total number of documents that failed to be pushed since replication started. + +type: long + +-- + +*`syncgateway.replication.id`*:: ++ +-- +ID of the replica. + +type: keyword + +-- + +[float] +=== resources + +SyncGateway global resource utilization + + + +*`syncgateway.resources.error_count`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.goroutines_high_watermark`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.num_goroutines`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.process.cpu_percent_utilization`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.process.memory_resident`*:: ++ +-- +type: long + +-- + + + +*`syncgateway.resources.pub_net.recv.bytes`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.pub_net.sent.bytes`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.admin_net_bytes.recv`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.admin_net_bytes.sent`*:: ++ +-- +type: long + +-- + + + +*`syncgateway.resources.go_memstats.heap.alloc`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.heap.idle`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.heap.inuse`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.heap.released`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.go_memstats.pause.ns`*:: ++ +-- +type: long + +-- + + +*`syncgateway.resources.go_memstats.stack.inuse`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.stack.sys`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.go_memstats.sys`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.system_memory_total`*:: ++ +-- +type: long + +-- + +*`syncgateway.resources.warn_count`*:: ++ +-- +type: long + +-- + [[exported-fields-system]] == System fields diff --git a/metricbeat/docs/modules/syncgateway.asciidoc b/metricbeat/docs/modules/syncgateway.asciidoc new file mode 100644 index 000000000000..a0688f60d229 --- /dev/null +++ b/metricbeat/docs/modules/syncgateway.asciidoc @@ -0,0 +1,57 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-module-syncgateway]] +[role="xpack"] +== SyncGateway module + +beta[] + +Sync Gateway is the synchronization server in a Couchbase for Mobile and Edge deployment. This metricset allows to monitor a Sync Gateway instance by using its REST API. + +Sync Gateway access `[host]:[port]/_expvar` on Sync Gateway nodes to fetch metrics data, ensure that the URL is accessible from the host where Metricbeat is running. + + +[float] +=== Example configuration + +The SyncGateway module supports the standard configuration options that are described +in <>. Here is an example configuration: + +[source,yaml] +---- +metricbeat.modules: +- module: syncgateway + metricsets: + - db +# - memory +# - replication +# - resources + period: 10s + + # SyncGateway hosts + hosts: ["127.0.0.1:4985"] +---- + +[float] +=== Metricsets + +The following metricsets are available: + +* <> + +* <> + +* <> + +* <> + +include::syncgateway/db.asciidoc[] + +include::syncgateway/memory.asciidoc[] + +include::syncgateway/replication.asciidoc[] + +include::syncgateway/resources.asciidoc[] + diff --git a/metricbeat/docs/modules/syncgateway/db.asciidoc b/metricbeat/docs/modules/syncgateway/db.asciidoc new file mode 100644 index 000000000000..4bc7a2964e7f --- /dev/null +++ b/metricbeat/docs/modules/syncgateway/db.asciidoc @@ -0,0 +1,25 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-syncgateway-db]] +[role="xpack"] +=== SyncGateway db metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/syncgateway/db/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/syncgateway/db/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/syncgateway/memory.asciidoc b/metricbeat/docs/modules/syncgateway/memory.asciidoc new file mode 100644 index 000000000000..6ae4f0fedb61 --- /dev/null +++ b/metricbeat/docs/modules/syncgateway/memory.asciidoc @@ -0,0 +1,24 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-syncgateway-memory]] +[role="xpack"] +=== SyncGateway memory metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/syncgateway/memory/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/syncgateway/memory/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules/syncgateway/replication.asciidoc b/metricbeat/docs/modules/syncgateway/replication.asciidoc new file mode 100644 index 000000000000..0d73b130cfef --- /dev/null +++ b/metricbeat/docs/modules/syncgateway/replication.asciidoc @@ -0,0 +1,18 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-syncgateway-replication]] +[role="xpack"] +=== SyncGateway replication metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/syncgateway/replication/_meta/docs.asciidoc[] + + +==== Fields + +For a description of each field in the metricset, see the +<> section. + diff --git a/metricbeat/docs/modules/syncgateway/resources.asciidoc b/metricbeat/docs/modules/syncgateway/resources.asciidoc new file mode 100644 index 000000000000..1860fe198142 --- /dev/null +++ b/metricbeat/docs/modules/syncgateway/resources.asciidoc @@ -0,0 +1,25 @@ +//// +This file is generated! See scripts/mage/docs_collector.go +//// + +[[metricbeat-metricset-syncgateway-resources]] +[role="xpack"] +=== SyncGateway resources metricset + +beta[] + +include::../../../../x-pack/metricbeat/module/syncgateway/resources/_meta/docs.asciidoc[] + +This is a default metricset. If the host module is unconfigured, this metricset is enabled by default. + +==== Fields + +For a description of each field in the metricset, see the +<> section. + +Here is an example document generated by this metricset: + +[source,json] +---- +include::../../../../x-pack/metricbeat/module/syncgateway/resources/_meta/data.json[] +---- diff --git a/metricbeat/docs/modules_list.asciidoc b/metricbeat/docs/modules_list.asciidoc index cd1e738c9e43..7917a95594c7 100644 --- a/metricbeat/docs/modules_list.asciidoc +++ b/metricbeat/docs/modules_list.asciidoc @@ -254,6 +254,11 @@ This file is generated! See scripts/mage/docs_collector.go |<> |<> |image:./images/icon-no.png[No prebuilt dashboards] | .1+| .1+| |<> +|<> beta[] |image:./images/icon-no.png[No prebuilt dashboards] | +.4+| .4+| |<> beta[] +|<> beta[] +|<> beta[] +|<> beta[] |<> |image:./images/icon-yes.png[Prebuilt dashboards are available] | .18+| .18+| |<> |<> @@ -350,6 +355,7 @@ include::modules/redisenterprise.asciidoc[] include::modules/sql.asciidoc[] include::modules/stan.asciidoc[] include::modules/statsd.asciidoc[] +include::modules/syncgateway.asciidoc[] include::modules/system.asciidoc[] include::modules/tomcat.asciidoc[] include::modules/traefik.asciidoc[] diff --git a/x-pack/elastic-agent/CHANGELOG.next.asciidoc b/x-pack/elastic-agent/CHANGELOG.next.asciidoc index 4672765b0195..bbc394f54727 100644 --- a/x-pack/elastic-agent/CHANGELOG.next.asciidoc +++ b/x-pack/elastic-agent/CHANGELOG.next.asciidoc @@ -115,4 +115,5 @@ - Use `filestream` input for internal log collection. {pull}25660[25660] - Enable agent to send custom headers to kibana/ES {pull}26275[26275] - Set `agent.id` to the Fleet Agent ID in events published from inputs backed by Beats. {issue}21121[21121] {pull}26394[26394] {pull}26548[26548] +- Enable configuring monitoring namespace {issue}26439[26439] - Communicate with Fleet Server over HTTP2. {pull}26474[26474] diff --git a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go index 8c3eb1c7d439..b31220f93ded 100644 --- a/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go +++ b/x-pack/elastic-agent/pkg/agent/application/pipeline/emitter/modifiers/monitoring_decorator.go @@ -49,6 +49,7 @@ func InjectMonitoring(agentInfo *info.AgentInfo, outputGroup string, rootAst *tr transpiler.NewKey("logs", transpiler.NewBoolVal(true)), transpiler.NewKey("metrics", transpiler.NewBoolVal(true)), transpiler.NewKey("use_output", transpiler.NewStrVal("default")), + transpiler.NewKey("namespace", transpiler.NewStrVal("default")), }) transpiler.Insert(rootAst, transpiler.NewKey("monitoring", monitoringNode), "settings") diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go index 8ab5b7d38f77..4712759ab70c 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring.go @@ -161,11 +161,12 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in var steps []configrequest.Step watchLogs := o.monitor.WatchLogs() watchMetrics := o.monitor.WatchMetrics() + monitoringNamespace := o.monitor.MonitoringNamespace() // generate only when monitoring is running (for config refresh) or // state changes (turning on/off) if watchLogs != o.isMonitoringLogs() || watchLogs { - fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output) + fbConfig, any := o.getMonitoringFilebeatConfig(outputType, output, monitoringNamespace) stepID := configrequest.StepRun if !watchLogs || !any { stepID = configrequest.StepRemove @@ -182,7 +183,7 @@ func (o *Operator) generateMonitoringSteps(version, outputType string, output in steps = append(steps, filebeatStep) } if watchMetrics != o.isMonitoringMetrics() || watchMetrics { - mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output) + mbConfig, any := o.getMonitoringMetricbeatConfig(outputType, output, monitoringNamespace) stepID := configrequest.StepRun if !watchMetrics || !any { stepID = configrequest.StepRemove @@ -215,12 +216,12 @@ func loadSpecFromSupported(processName string) program.Spec { } } -func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) { +func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) { inputs := []interface{}{ map[string]interface{}{ "type": "filestream", "parsers": []map[string]interface{}{ - map[string]interface{}{ + { "ndjson": map[string]interface{}{ "overwrite_keys": true, "message_key": "message", @@ -233,7 +234,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log"), filepath.Join(paths.Home(), "logs", "elastic-agent-watcher-json.log*"), }, - "index": "logs-elastic_agent-default", + "index": fmt.Sprintf("logs-elastic_agent-%s", monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -241,7 +242,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa "fields": map[string]interface{}{ "type": "logs", "dataset": "elastic_agent", - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -280,7 +281,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa inputs = append(inputs, map[string]interface{}{ "type": "filestream", "parsers": []map[string]interface{}{ - map[string]interface{}{ + { "ndjson": map[string]interface{}{ "overwrite_keys": true, "message_key": "message", @@ -288,7 +289,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa }, }, "paths": paths, - "index": fmt.Sprintf("logs-elastic_agent.%s-default", name), + "index": fmt.Sprintf("logs-elastic_agent.%s-%s", name, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -296,7 +297,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa "fields": map[string]interface{}{ "type": "logs", "dataset": fmt.Sprintf("elastic_agent.%s", name), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -355,7 +356,7 @@ func (o *Operator) getMonitoringFilebeatConfig(outputType string, output interfa return result, true } -func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}) (map[string]interface{}, bool) { +func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output interface{}, monitoringNamespace string) (map[string]interface{}, bool) { hosts := o.getMetricbeatEndpoints() if len(hosts) == 0 { return nil, false @@ -369,7 +370,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "metricsets": []string{"stats", "state"}, "period": "10s", "hosts": endpoints, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", name), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", name, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -377,7 +378,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", name), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -407,7 +408,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "period": "10s", "path": "/stats", "hosts": endpoints, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -415,7 +416,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, @@ -490,7 +491,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "period": "10s", "path": "/stats", "hosts": []string{beats.AgentPrefixedMonitoringEndpoint(o.config.DownloadConfig.OS(), o.config.MonitoringConfig.HTTP)}, - "index": fmt.Sprintf("metrics-elastic_agent.%s-default", fixedAgentName), + "index": fmt.Sprintf("metrics-elastic_agent.%s-%s", fixedAgentName, monitoringNamespace), "processors": []map[string]interface{}{ { "add_fields": map[string]interface{}{ @@ -498,7 +499,7 @@ func (o *Operator) getMonitoringMetricbeatConfig(outputType string, output inter "fields": map[string]interface{}{ "type": "metrics", "dataset": fmt.Sprintf("elastic_agent.%s", fixedAgentName), - "namespace": "default", + "namespace": monitoringNamespace, }, }, }, diff --git a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go index 136c9e485b12..c23248ff2d93 100644 --- a/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go +++ b/x-pack/elastic-agent/pkg/agent/operation/monitoring_test.go @@ -215,6 +215,9 @@ func (b *testMonitor) Reload(cfg *config.Config) error { return nil } // IsMonitoringEnabled returns true if monitoring is configured. func (b *testMonitor) IsMonitoringEnabled() bool { return b.monitorLogs || b.monitorMetrics } +// MonitoringNamespace returns monitoring namespace configured. +func (b *testMonitor) MonitoringNamespace() string { return "default" } + // WatchLogs return true if monitoring is configured and monitoring logs is enabled. func (b *testMonitor) WatchLogs() bool { return b.monitorLogs } diff --git a/x-pack/elastic-agent/pkg/agent/program/program_test.go b/x-pack/elastic-agent/pkg/agent/program/program_test.go index 4498f7e52360..5ca35de01362 100644 --- a/x-pack/elastic-agent/pkg/agent/program/program_test.go +++ b/x-pack/elastic-agent/pkg/agent/program/program_test.go @@ -383,6 +383,10 @@ func TestConfiguration(t *testing.T) { empty bool err bool }{ + "namespace": { + programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, + expected: 6, + }, "single_config": { programs: []string{"filebeat", "fleet-server", "heartbeat", "metricbeat", "endpoint", "packetbeat"}, expected: 6, diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml new file mode 100644 index 000000000000..7e9f04dc4119 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-endpoint-security.yml @@ -0,0 +1,114 @@ +fleet: + enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + +output: + elasticsearch: + hosts: + - "127.0.0.1:9200" + - "127.0.0.1:9300" + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + +inputs: +- id: endpoint-id + type: endpoint + name: endpoint-1 + enabled: true + package: + name: endpoint + version: 0.3.0 + data_stream: + namespace: default + artifact_manifest: + schema_version: v22 + manifest_version: v21 + artifacts: + - endpoint-allowlist-windows: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-windows + - endpoint-allowlist-macos: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-macos + - endpoint-allowlist-linux: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-linux + policy: + linux: + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + windows: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + registry: logs-endpoint.events.registry-default + process: logs-endpoint.events.process-default + driver: logs-endpoint.events.driver-default + library: logs-endpoint.events.library-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + registry: true + process: true + security: true + file: true + dns: false + dll_and_driver_load: false + network: true + mac: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml new file mode 100644 index 000000000000..83df83e56e0c --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-filebeat.yml @@ -0,0 +1,68 @@ +filebeat: + inputs: + - type: log + paths: + - /var/log/hello1.log + - /var/log/hello2.log + index: logs-generic-default + vars: + var: value + processors: + - add_fields: + target: "data_stream" + fields: + type: logs + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - type: log + paths: + - /var/log/hello3.log + - /var/log/hello4.log + index: testtype-generic-default + vars: + var: value + processors: + - add_fields: + target: "data_stream" + fields: + type: testtype + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml new file mode 100644 index 000000000000..c03696aff1f0 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-fleet-server.yml @@ -0,0 +1,16 @@ +fleet: + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + +output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +inputs: + - id: fleet-server-id + type: fleet-server diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml new file mode 100644 index 000000000000..f34b204f5fae --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-heartbeat.yml @@ -0,0 +1,30 @@ +inputs: +- type: synthetics/http + id: unique-http-id + name: my-http + schedule: '*/5 * * * * * *' + host: "http://localhost:80/service/status" + timeout: 16s + wait: 1s + data_stream.namespace: default + processors: + - add_fields: + target: 'elastic_agent' + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: 'agent' + fields: + id: agent-id +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml new file mode 100644 index 000000000000..3f16a9d9e210 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-metricbeat.yml @@ -0,0 +1,88 @@ +metricbeat: + modules: + - module: docker + metricsets: [status] + index: metrics-docker.status-default + hosts: ["http://127.0.0.1:8080"] + processors: + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: docker.status + namespace: default + - add_fields: + target: "event" + fields: + dataset: docker.status + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - module: docker + metricsets: [info] + index: metrics-generic-default + hosts: ["http://127.0.0.1:8080"] + processors: + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: generic + namespace: default + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id + - module: apache + metricsets: [info] + index: metrics-generic-testing + hosts: ["http://apache.remote"] + processors: + - add_fields: + fields: + should_be: first + - add_fields: + target: "data_stream" + fields: + type: metrics + dataset: generic + namespace: testing + - add_fields: + target: "event" + fields: + dataset: generic + - add_fields: + target: "elastic_agent" + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: "agent" + fields: + id: agent-id +output: + elasticsearch: + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml new file mode 100644 index 000000000000..d71499bdef4d --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace-packetbeat.yml @@ -0,0 +1,35 @@ +inputs: +- type: packet + processors: + - add_fields: + target: 'elastic_agent' + fields: + id: agent-id + version: 8.0.0 + snapshot: false + - add_fields: + target: 'agent' + fields: + id: agent-id + streams: + - type: flow + timeout: 10s + period: 10s + keep_null: false + data_stream: + dataset: packet.flow + type: logs + - type: icmp + data_stream: + dataset: packet.icmp + type: logs +output: + elasticsearch: + hosts: + - 127.0.0.1:9200 + - 127.0.0.1:9300 + namespace: test_namespace + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= diff --git a/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml new file mode 100644 index 000000000000..c2f83a9abf08 --- /dev/null +++ b/x-pack/elastic-agent/pkg/agent/program/testdata/namespace.yml @@ -0,0 +1,201 @@ +name: Production Website DB Servers +fleet: + enabled: true + access_api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + protocol: https + hosts: [ localhost:5601 ] + timeout: 30s + agent: + id: fleet-agent-id + logging.level: error + host: + id: host-agent-id + server: + output: + elasticsearch: + hosts: [ 127.0.0.1:9200, 127.0.0.1:9300 ] + username: fleet + password: fleetpassword + +outputs: + default: + type: elasticsearch + namespace: test_namespace + hosts: [127.0.0.1:9200, 127.0.0.1:9300] + username: elastic + password: changeme + api_key: TiNAGG4BaaMdaH1tRfuU:KnR6yE41RrSowb0kQ0HWoA + ca_sha256: 7HIpactkIAq2Y49orFOOQKurWxmmSFZhBCoQYcRhJ3Y= + + monitoring: + type: elasticsearch + api_key: VuaCfGcBCdbkQm-e5aOx:ui2lp2axTNmsyakw9tvNnw + hosts: ["monitoring:9200"] + ca_sha256: "7lHLiyp4J8m9kw38SJ7SURJP4bXRZv/BNxyyXkCcE/M=" + +inputs: +- id: fleet-server-id + type: fleet-server + use_output: default + data_stream: + type: default +- type: docker/metrics + use_output: default + streams: + - metricset: status + processors: + - null + data_stream: + dataset: docker.status + - metricset: info + data_stream: + dataset: "" + hosts: ["http://127.0.0.1:8080"] +- type: logfile + use_output: default + streams: + - paths: + - /var/log/hello1.log + - /var/log/hello2.log + vars: + var: value +- type: logfile + data_stream: + type: testtype + use_output: default + streams: + - paths: + - /var/log/hello3.log + - /var/log/hello4.log + vars: + var: value +- id: apache-metrics-id + type: apache/metrics + data_stream: + namespace: testing + use_output: default + processors: + - add_fields: + fields: + should_be: first + streams: + - enabled: true + metricset: info + hosts: ["http://apache.remote"] + hosts: ["http://apache.local"] +- type: synthetics/http + id: unique-http-id + name: my-http + schedule: '*/5 * * * * * *' + host: "http://localhost:80/service/status" + timeout: 16s + wait: 1s +- type: packet + streams: + - type: flow + timeout: 10s + period: 10s + keep_null: false + data_stream: + dataset: packet.flow + type: logs + - type: icmp + data_stream: + dataset: packet.icmp + type: logs +- id: endpoint-id + type: endpoint + name: endpoint-1 + enabled: true + package: + name: endpoint + version: 0.3.0 + data_stream: + namespace: default + artifact_manifest: + schema_version: v22 + manifest_version: v21 + artifacts: + - endpoint-allowlist-windows: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-windows + - endpoint-allowlist-macos: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-macos + - endpoint-allowlist-linux: + sha256: 1234 + size: 2 + url: /relative/path/to/endpoint-allowlist-linux + policy: + linux: + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + windows: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + registry: logs-endpoint.events.registry-default + process: logs-endpoint.events.process-default + driver: logs-endpoint.events.driver-default + library: logs-endpoint.events.library-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + registry: true + process: true + security: true + file: true + dns: false + dll_and_driver_load: false + network: true + mac: + malware: + mode: prevent + advanced: + free-form: free-form-value + indices: + network: logs-endpoint.events.network-default + file: logs-endpoint.events.file-default + process: logs-endpoint.events.process-default + alerts: logs-endpoint.alerts-default + metadata: metrics-endpoint.metadata-default + policy: metrics-endpoint.policy-default + telemetry: metrics-endpoint.telemetry-default + logging: + file: info + stdout: debug + events: + process: true + file: true + network: true + +agent.monitoring: + use_output: monitoring + +agent: + reload: 123 diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go index 100d8a462a02..31bb2faaa7c1 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/ast.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/ast.go @@ -716,8 +716,10 @@ func load(val reflect.Value) (Node, error) { return loadSliceOrArray(val) case reflect.String: return &StrVal{value: val.Interface().(string)}, nil - case reflect.Int, reflect.Int64: + case reflect.Int: return &IntVal{value: val.Interface().(int)}, nil + case reflect.Int64: + return &IntVal{value: int(val.Interface().(int64))}, nil case reflect.Uint: return &UIntVal{value: uint64(val.Interface().(uint))}, nil case reflect.Uint64: diff --git a/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go b/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go index e1b22c390edc..1e52d6c6f47e 100644 --- a/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go +++ b/x-pack/elastic-agent/pkg/agent/transpiler/ast_test.go @@ -105,6 +105,7 @@ func TestAST(t *testing.T) { "support integers": { hashmap: map[string]interface{}{ "timeout": 12, + "zero": int64(0), "range": []int{20, 30, 40}, }, ast: &AST{ @@ -121,6 +122,7 @@ func TestAST(t *testing.T) { ), }, &Key{name: "timeout", value: &IntVal{value: 12}}, + &Key{name: "zero", value: &IntVal{value: 0}}, }, }, }, diff --git a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go index 743d44118d6b..1c0c4ba61ad7 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/beats/beats_monitor.go @@ -21,6 +21,7 @@ import ( ) const httpPlusPrefix = "http+" +const defaultMonitoringNamespace = "default" // Monitor is a monitoring interface providing information about the way // how beat is monitored @@ -69,6 +70,14 @@ func (b *Monitor) Close() { // IsMonitoringEnabled returns true if monitoring is enabled. func (b *Monitor) IsMonitoringEnabled() bool { return b.config.Enabled } +// MonitoringNamespace returns monitoring namespace configured. +func (b *Monitor) MonitoringNamespace() string { + if b.config.Namespace == "" { + return defaultMonitoringNamespace + } + return b.config.Namespace +} + // WatchLogs returns true if monitoring is enabled and monitor should watch logs. func (b *Monitor) WatchLogs() bool { return b.config.Enabled && b.config.MonitorLogs } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go index 2ce067d4e192..fe18b0fb73eb 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/config/config.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/config/config.go @@ -5,6 +5,7 @@ package config const defaultPort = 6791 +const defaultNamespace = "default" // MonitoringConfig describes a configuration of a monitoring type MonitoringConfig struct { @@ -12,6 +13,7 @@ type MonitoringConfig struct { MonitorLogs bool `yaml:"logs" config:"logs"` MonitorMetrics bool `yaml:"metrics" config:"metrics"` HTTP *MonitoringHTTPConfig `yaml:"http" config:"http"` + Namespace string `yaml:"namespace" config:"namespace"` } // MonitoringHTTPConfig is a config defining HTTP endpoint published by agent @@ -33,5 +35,6 @@ func DefaultConfig() *MonitoringConfig { Enabled: false, Port: defaultPort, }, + Namespace: defaultNamespace, } } diff --git a/x-pack/elastic-agent/pkg/core/monitoring/monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/monitor.go index 00c7a50003ae..6c71f4f65fc0 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/monitor.go @@ -23,6 +23,7 @@ type Monitor interface { Cleanup(spec program.Spec, pipelineID string) error Reload(cfg *config.Config) error IsMonitoringEnabled() bool + MonitoringNamespace() string WatchLogs() bool WatchMetrics() bool Close() diff --git a/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go b/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go index 9ea8f08a7885..d98deb90888b 100644 --- a/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go +++ b/x-pack/elastic-agent/pkg/core/monitoring/noop/noop_monitor.go @@ -66,3 +66,6 @@ func (b *Monitor) WatchLogs() bool { return false } // WatchMetrics return true if monitoring is configured and monitoring metrics is enabled. func (b *Monitor) WatchMetrics() bool { return false } + +// MonitoringNamespace returns monitoring namespace configured. +func (b *Monitor) MonitoringNamespace() string { return "default" } diff --git a/x-pack/libbeat/autodiscover/providers/nomad/config.go b/x-pack/libbeat/autodiscover/providers/nomad/config.go index 1a611aceee35..5ce13556d91d 100644 --- a/x-pack/libbeat/autodiscover/providers/nomad/config.go +++ b/x-pack/libbeat/autodiscover/providers/nomad/config.go @@ -41,9 +41,6 @@ type Config struct { func defaultConfig() *Config { return &Config{ Address: "http://127.0.0.1:4646", - Region: "", - Namespace: "", - SecretID: "", Scope: ScopeNode, allowStale: true, waitTime: 15 * time.Second, @@ -53,7 +50,7 @@ func defaultConfig() *Config { } } -// Validate ensures correctness of config +// Validate ensures correctness of config. func (c *Config) Validate() error { // Make sure that prefix doesn't ends with a '.' if c.Prefix[len(c.Prefix)-1] == '.' && c.Prefix != "." { diff --git a/x-pack/libbeat/autodiscover/providers/nomad/nomad.go b/x-pack/libbeat/autodiscover/providers/nomad/nomad.go index 6b3d9bc70fa0..17284c49fd7f 100644 --- a/x-pack/libbeat/autodiscover/providers/nomad/nomad.go +++ b/x-pack/libbeat/autodiscover/providers/nomad/nomad.go @@ -50,12 +50,10 @@ func AutodiscoverBuilder( c *common.Config, keystore keystore.Keystore, ) (autodiscover.Provider, error) { - cfgwarn.Experimental("The nomad autodiscover is experimental") + cfgwarn.Experimental("The nomad autodiscover provider is experimental.") config := defaultConfig() - - err := c.Unpack(&config) - if err != nil { + if err := c.Unpack(&config); err != nil { return nil, err } @@ -130,18 +128,18 @@ func AutodiscoverBuilder( watcher.AddEventHandler(nomad.ResourceEventHandlerFuncs{ AddFunc: func(obj nomad.Resource) { - logger.Debugf("Watcher Allocation add: %+v", obj.ID) + logger.Debugw("Nomad allocation added", "nomad.allocation.id", obj.ID) p.emit(&obj, "start") }, UpdateFunc: func(obj nomad.Resource) { - logger.Debugf("nomad", "Watcher Allocation update: %+v", obj.ID) + logger.Debugw("Nomad allocation updated", "nomad.allocation.id", obj.ID) p.emit(&obj, "stop") // We have a CleanupTimeout grace period (defaults to 15s) to wait for the stop event // to be processed time.AfterFunc(config.CleanupTimeout, func() { p.emit(&obj, "start") }) }, DeleteFunc: func(obj nomad.Resource) { - logger.Debugf("Watcher Allocation delete: %+v", obj.ID) + logger.Debugw("Nomad allocation deleted", "nomad.allocation.id", obj.ID) p.emit(&obj, "stop") }, }) @@ -152,7 +150,7 @@ func AutodiscoverBuilder( // Start for Runner interface. func (p *Provider) Start() { if err := p.watcher.Start(); err != nil { - p.logger.Errorf("Error starting nomad autodiscover provider: %s", err) + p.logger.Errorw("Error starting nomad autodiscover provider", "error", err) } } @@ -175,7 +173,7 @@ func (p *Provider) emit(obj *nomad.Resource, flag string) { // the NodeID host, err := p.metagen.AllocationNodeName(obj.NodeID) if err != nil { - p.logger.Errorf("Error fetching node information: %s", err) + p.logger.Errorw("Error fetching node information", "error", err) } // If we cannot get a host, we assume that the allocation was stopped @@ -189,7 +187,7 @@ func (p *Provider) emit(obj *nomad.Resource, flag string) { // common metadata from the entire allocation allocMeta := p.metagen.ResourceMetadata(*obj) - // job metadatata merged with the task metadata + // job metadata merged with the task metadata tasks := p.metagen.GroupMeta(obj.Job) // emit per-task separated events @@ -225,7 +223,7 @@ func (p *Provider) publish(event bus.Event) { // Call all appenders to append any extra configuration p.appenders.Append(event) - p.logger.Debugf("nomad", "Publishing event: %+v", event) + p.logger.Debugw("Publishing nomad autodiscover event.", "autodiscover.event", event) p.bus.Publish(event) } @@ -275,12 +273,9 @@ func (p *Provider) generateHints(event bus.Event) bus.Event { cname := builder.GetContainerName(container) hints := builder.GenerateHints(tasks, cname, p.config.Prefix) - - p.logger.Debugf("Generated hints from %+v %+v", tasks, hints) - if len(hints) != 0 { + if len(hints) > 0 { e["hints"] = hints } - p.logger.Debugf("nomad", "Generated builder event %+v", e) prefix := strings.Split(p.config.Prefix, ".")[0] tasks.Delete(prefix) diff --git a/x-pack/libbeat/common/nomad/metadata.go b/x-pack/libbeat/common/nomad/metadata.go index b0648b6e3e7e..51e2fe93ebc7 100644 --- a/x-pack/libbeat/common/nomad/metadata.go +++ b/x-pack/libbeat/common/nomad/metadata.go @@ -82,7 +82,7 @@ func (g *metaGenerator) ResourceMetadata(obj Resource) common.MapStr { // Returns an array of per-task metadata aggregating the group metadata into the // task metadata func (g *metaGenerator) GroupMeta(job *Job) []common.MapStr { - tasksMeta := []common.MapStr{} + var tasksMeta []common.MapStr for _, group := range job.TaskGroups { meta := make(map[string]string, len(job.Meta)) @@ -131,7 +131,7 @@ func (g *metaGenerator) GroupMeta(job *Job) []common.MapStr { // Returns per-task metadata func (g *metaGenerator) tasksMeta(group *TaskGroup) []common.MapStr { - tasks := []common.MapStr{} + var tasks []common.MapStr for _, task := range group.Tasks { var svcNames, svcTags, svcCanaryTags []string @@ -201,7 +201,7 @@ func generateMapSubset(input common.MapStr, keys []string, dedot bool) common.Ma } // AllocationNodeName returns Name of the node where the task is allocated. It -// does one additional API call to circunvent the empty NodeName property of +// does one additional API call to circumvent the empty NodeName property of // older Nomad versions (up to v0.8) func (g *metaGenerator) AllocationNodeName(id string) (string, error) { if name, ok := g.nodesCache[id]; ok { diff --git a/x-pack/libbeat/common/nomad/watcher.go b/x-pack/libbeat/common/nomad/watcher.go index eba77e764ad7..31cb5590ff9b 100644 --- a/x-pack/libbeat/common/nomad/watcher.go +++ b/x-pack/libbeat/common/nomad/watcher.go @@ -45,7 +45,7 @@ type WatchOptions struct { SyncTimeout time.Duration // Node is used for filtering events Node string - // Namespace is used for filtering events on specified namespacesx, + // Namespace is used for filtering events on specified namespaces. Namespace string // RefreshInterval is the time interval that the Nomad API will be queried RefreshInterval time.Duration @@ -103,7 +103,7 @@ func (w *watcher) AddEventHandler(h ResourceEventHandlerFuncs) { // Sync the allocations on the given node and update the local metadata func (w *watcher) sync() error { - w.logger.Debugf("Syncing allocations and metadata") + w.logger.Debug("Syncing allocations and metadata") w.logger.Debugf("Starting with WaitIndex=%v", w.waitIndex) queryOpts := &api.QueryOptions{ @@ -114,21 +114,20 @@ func (w *watcher) sync() error { allocations, meta, err := w.getAllocations(queryOpts) if err != nil { - return fmt.Errorf("listing allocations: %w", err) + return fmt.Errorf("failed listing allocations: %w", err) } - w.logger.Debugf("Found %d allocations", len(allocations)) - remoteWaitIndex := meta.LastIndex localWaitIndex := queryOpts.WaitIndex // Only emit updated metadata if the WaitIndex have changed if remoteWaitIndex <= localWaitIndex { w.logger.Debugf("Allocations index is unchanged remoteWaitIndex=%v == localWaitIndex=%v", - fmt.Sprint(remoteWaitIndex), fmt.Sprint(localWaitIndex)) + remoteWaitIndex, localWaitIndex) return nil } + w.logger.Debugf("Found %d allocations", len(allocations)) for _, alloc := range allocations { // the allocation has not changed since last seen, ignore if w.waitIndex > alloc.AllocModifyIndex { @@ -184,7 +183,7 @@ func (w *watcher) sync() error { func (w *watcher) watch() { // Failures counter, do exponential backoff on retries var failures uint - logp.Info("Nomad: %s", "Watching API for resource events") + w.logger.Info("Watching API for resource events") ticker := time.NewTicker(w.options.RefreshInterval) defer ticker.Stop() @@ -193,9 +192,8 @@ func (w *watcher) watch() { case <-w.done: return case <-ticker.C: - err := w.sync() - if err != nil { - logp.Err("Nomad: Error while watching for allocation changes %v", err) + if err := w.sync(); err != nil { + w.logger.Warnw("Error while watching for Nomad allocation changes. Backing off and continuing.", "error", err) backoff(failures) failures++ } @@ -215,11 +213,14 @@ func (w *watcher) getAllocations(queryOpts *api.QueryOptions) ([]*api.Allocation if err != nil { return nil, meta, err } + var allocations []*api.Allocation for _, stub := range stubs { allocation, _, err := w.client.Allocations().Info(stub.ID, queryOpts) if err != nil { - w.logger.Warnf("Failed to get details of allocation '%s'", stub.ID) + w.logger.Warnw("Failed to get details of an allocation.", + "nomad.allocation.id", stub.ID) + continue } allocations = append(allocations, allocation) } @@ -232,12 +233,11 @@ func (w *watcher) fetchNodeID() (string, error) { AllowStale: w.options.AllowStale, } - // Fetch the nodeId from the node name, used to filter the allocations - // If for some reason the NodeID changes filebeat will have to be restarted + // Fetch the nodeId from the node name, used to filter the allocations. + // If for some reason the NodeID changes filebeat will have to be restarted. nodes, _, err := w.client.Nodes().List(queryOpts) - if err != nil { - w.logger.Fatalf("Nomad: Fetching node list err %s", err.Error()) + w.logger.Errorw("Failed fetching Nomad node list.", "error", err) return "", err } diff --git a/x-pack/libbeat/processors/add_nomad_metadata/docs/add_nomad_metadata.asciidoc b/x-pack/libbeat/processors/add_nomad_metadata/docs/add_nomad_metadata.asciidoc index 1ab6b8cab038..31c4d2ede20e 100644 --- a/x-pack/libbeat/processors/add_nomad_metadata/docs/add_nomad_metadata.asciidoc +++ b/x-pack/libbeat/processors/add_nomad_metadata/docs/add_nomad_metadata.asciidoc @@ -32,7 +32,21 @@ uses `http://127.0.0.1:4646` by default. in this namespace will be annotated. `region`:: (Optional) Region to watch. If set, only events for allocations in this region will be annotated. -`secretID`:: (Optional) SecretID to use when connecting with the agent API. +`secret_id`:: (Optional) SecretID to use when connecting with the agent API. +This is an example ACL policy to apply to the token. + +[source,hcl] +---- +namespace "*" { + policy = "read" +} +node { + policy = "read" +} +agent { + policy = "read" +} +---- `refresh_interval`:: (Optional) Interval used to updated the cached metadata. It defaults to 30 seconds. `cleanup_timeout`:: (Optional) After an allocation has been removed, time to diff --git a/x-pack/metricbeat/include/list.go b/x-pack/metricbeat/include/list.go index 856c33577462..9b0f402423a2 100644 --- a/x-pack/metricbeat/include/list.go +++ b/x-pack/metricbeat/include/list.go @@ -60,5 +60,10 @@ import ( _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/stan/subscriptions" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/statsd/server" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway/db" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway/memory" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway/replication" + _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway/resources" _ "github.com/elastic/beats/v7/x-pack/metricbeat/module/tomcat" ) diff --git a/x-pack/metricbeat/metricbeat.reference.yml b/x-pack/metricbeat/metricbeat.reference.yml index 4bd416f78835..0a8bc00d276e 100644 --- a/x-pack/metricbeat/metricbeat.reference.yml +++ b/x-pack/metricbeat/metricbeat.reference.yml @@ -1352,6 +1352,18 @@ metricbeat.modules: enabled: false #ttl: "30s" +#----------------------------- SyncGateway Module ----------------------------- +- module: syncgateway + metricsets: + - db +# - memory +# - replication +# - resources + period: 10s + + # SyncGateway hosts + hosts: ["127.0.0.1:4985"] + #-------------------------------- Tomcat Module -------------------------------- - module: tomcat metricsets: ['threading', 'cache', 'memory', 'requests'] diff --git a/x-pack/metricbeat/module/syncgateway/_meta/config.yml b/x-pack/metricbeat/module/syncgateway/_meta/config.yml new file mode 100644 index 000000000000..d47955d6da56 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/config.yml @@ -0,0 +1,10 @@ +- module: syncgateway + metricsets: + - db +# - memory +# - replication +# - resources + period: 10s + + # SyncGateway hosts + hosts: ["127.0.0.1:4985"] diff --git a/x-pack/metricbeat/module/syncgateway/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/_meta/docs.asciidoc new file mode 100644 index 000000000000..ce6c60d06826 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/docs.asciidoc @@ -0,0 +1,3 @@ +Sync Gateway is the synchronization server in a Couchbase for Mobile and Edge deployment. This metricset allows to monitor a Sync Gateway instance by using its REST API. + +Sync Gateway access `[host]:[port]/_expvar` on Sync Gateway nodes to fetch metrics data, ensure that the URL is accessible from the host where Metricbeat is running. diff --git a/x-pack/metricbeat/module/syncgateway/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/_meta/fields.yml new file mode 100644 index 000000000000..1cba880d6d6d --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/fields.yml @@ -0,0 +1,10 @@ +- key: syncgateway + title: "SyncGateway" + description: SyncGateway metrics + release: beta + fields: + - name: syncgateway + type: group + description: > + `syncgateway` contains the information and statistics from SyncGateway. + fields: diff --git a/x-pack/metricbeat/module/syncgateway/_meta/testdata/config.json b/x-pack/metricbeat/module/syncgateway/_meta/testdata/config.json new file mode 100644 index 000000000000..c118d4641fba --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/testdata/config.json @@ -0,0 +1,76 @@ +{ + "interface": ":4984", + "logging": { + "log_file_path": "/var/tmp/sglogs", + "console": { + "log_level": "debug", + "log_keys": [ + "*" + ] + }, + "error": { + "enabled": true, + "rotation": { + "max_size": 20, + "max_age": 180 + } + }, + "warn": { + "enabled": true, + "rotation": { + "max_size": 20, + "max_age": 90 + } + }, + "info": { + "enabled": false + }, + "debug": { + "enabled": false + } + }, + "databases": { + "beer-sample": { + "import_docs": "continuous", + "enable_shared_bucket_access": true, + "bucket": "beer-sample", + "server": "http://172.17.0.2:8091", + "username": "admin", + "password": "123456", + "num_index_replicas": 0, + "users": { + "GUEST": { + "disabled": true + }, + "admin": { + "password": "123456", + "admin_channels": [ + "*" + ] + } + }, + "revs_limit": 20 + }, + "travel-sample": { + "import_docs": "continuous", + "enable_shared_bucket_access": true, + "bucket": "travel-sample", + "server": "http://172.17.0.2:8091", + "username": "admin", + "password": "123456", + "num_index_replicas": 0, + "users": { + "GUEST": { + "disabled": true + }, + "admin": { + "password": "123456", + "admin_channels": [ + "*" + ] + } + }, + "revs_limit": 20 + } + } +} diff --git a/x-pack/metricbeat/module/syncgateway/_meta/testdata/expvar.282c.json b/x-pack/metricbeat/module/syncgateway/_meta/testdata/expvar.282c.json new file mode 100644 index 000000000000..8b1f71fde745 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/_meta/testdata/expvar.282c.json @@ -0,0 +1,1224 @@ +{ + "cb": { + "ops": {}, + "pools": {} + }, + "cmdline": [ + "sync_gateway", + "--defaultLogFilePath=/var/log/sync_gateway", + "-adminInterface", + ":4985", + "/etc/sync_gateway/sync_gateway.json" + ], + "goblip": {}, + "mc": { + "recv": { + "bytes": { + "0x1f": 1008, + "0x89": 864, + "SASL_AUTH": 864, + "SASL_LIST_MECHS": 2376, + "UPR_OPEN": 864, + "total": 5976 + }, + "errs": {}, + "ops": { + "0x1f": 36, + "0x89": 36, + "SASL_AUTH": 36, + "SASL_LIST_MECHS": 36, + "UPR_OPEN": 36, + "total": 180 + } + }, + "tap": { + "bytes": {}, + "errs": {}, + "ops": {} + }, + "xmit": { + "bytes": { + "0x1f": 3566, + "0x89": 1296, + "NOOP": 720, + "SASL_AUTH": 1512, + "SASL_LIST_MECHS": 864, + "UPR_BUFFERACK": 336, + "UPR_CONTROL": 4932, + "UPR_OPEN": 3170, + "UPR_STREAMREQ": 294912, + "total": 311308 + }, + "errs": {}, + "ops": { + "0x1f": 36, + "0x89": 36, + "NOOP": 30, + "SASL_AUTH": 36, + "SASL_LIST_MECHS": 36, + "UPR_BUFFERACK": 12, + "UPR_CONTROL": 108, + "UPR_OPEN": 36, + "UPR_STREAMREQ": 4096, + "total": 4426 + } + } + }, + "memstats": { + "Alloc": 159398816, + "TotalAlloc": 2719416432, + "Sys": 357708024, + "Lookups": 0, + "Mallocs": 57667615, + "Frees": 55790048, + "HeapAlloc": 159398816, + "HeapSys": 332169216, + "HeapIdle": 67780608, + "HeapInuse": 264388608, + "HeapReleased": 26927104, + "HeapObjects": 1877567, + "StackInuse": 3375104, + "StackSys": 3375104, + "MSpanInuse": 4030632, + "MSpanSys": 4358144, + "MCacheInuse": 20832, + "MCacheSys": 32768, + "BuckHashSys": 1759627, + "GCSys": 13555712, + "OtherSys": 2457453, + "NextGC": 265304208, + "LastGC": 1620310398040906500, + "PauseTotalNs": 88693194, + "PauseNs": [ + 67131, + 17720, + 49699, + 15125, + 26870, + 11396, + 17540, + 28606, + 64253, + 95491, + 26325, + 18928, + 255476, + 622721, + 378524, + 7860579, + 2189850, + 312269, + 160370, + 1127927, + 226473, + 714888, + 399169, + 3320206, + 7245241, + 2639040, + 1084283, + 3470662, + 975960, + 952710, + 166017, + 733559, + 6388460, + 276382, + 4632588, + 640706, + 146091, + 195839, + 4782434, + 79438, + 559445, + 1344800, + 9485458, + 2707508, + 1547026, + 220774, + 2629126, + 9999615, + 226927, + 7472682, + 82887, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "PauseEnd": [ + 1620310338920958200, + 1620310351426572000, + 1620310352681769500, + 1620310352686500000, + 1620310352693075200, + 1620310352700478000, + 1620310352745081600, + 1620310352759565600, + 1620310352763341800, + 1620310352767747600, + 1620310352793954800, + 1620310352846218000, + 1620310352905687000, + 1620310352982015700, + 1620310353051933000, + 1620310353138877000, + 1620310353255282200, + 1620310353399916800, + 1620310353720879600, + 1620310354061829600, + 1620310354374070800, + 1620310354696978400, + 1620310355152804000, + 1620310355628574500, + 1620310356137813000, + 1620310356627671300, + 1620310357016540400, + 1620310374534675200, + 1620310374809924900, + 1620310375047449300, + 1620310375474444000, + 1620310375924982000, + 1620310376440657000, + 1620310377146911000, + 1620310377906382000, + 1620310378854651000, + 1620310379714263000, + 1620310380508387800, + 1620310381220176100, + 1620310382398691600, + 1620310383426419200, + 1620310384538631700, + 1620310385624432000, + 1620310387400379100, + 1620310388572275200, + 1620310389941691000, + 1620310391488313600, + 1620310393008522000, + 1620310394713452500, + 1620310396453674500, + 1620310398040906500, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "NumGC": 51, + "NumForcedGC": 0, + "GCCPUFraction": 0.008328526565497294, + "EnableGC": true, + "DebugGC": false, + "BySize": [ + { + "Size": 0, + "Mallocs": 0, + "Frees": 0 + }, + { + "Size": 8, + "Mallocs": 901392, + "Frees": 896474 + }, + { + "Size": 16, + "Mallocs": 29531337, + "Frees": 28456937 + }, + { + "Size": 32, + "Mallocs": 6033284, + "Frees": 5643846 + }, + { + "Size": 48, + "Mallocs": 3201493, + "Frees": 3012460 + }, + { + "Size": 64, + "Mallocs": 797155, + "Frees": 786367 + }, + { + "Size": 80, + "Mallocs": 544250, + "Frees": 538869 + }, + { + "Size": 96, + "Mallocs": 712994, + "Frees": 706044 + }, + { + "Size": 112, + "Mallocs": 300043, + "Frees": 296594 + }, + { + "Size": 128, + "Mallocs": 454667, + "Frees": 453674 + }, + { + "Size": 144, + "Mallocs": 258222, + "Frees": 257167 + }, + { + "Size": 160, + "Mallocs": 197323, + "Frees": 185378 + }, + { + "Size": 176, + "Mallocs": 118510, + "Frees": 117754 + }, + { + "Size": 192, + "Mallocs": 44105, + "Frees": 43694 + }, + { + "Size": 208, + "Mallocs": 130830, + "Frees": 129313 + }, + { + "Size": 224, + "Mallocs": 42617, + "Frees": 42049 + }, + { + "Size": 240, + "Mallocs": 89888, + "Frees": 89150 + }, + { + "Size": 256, + "Mallocs": 300134, + "Frees": 297243 + }, + { + "Size": 288, + "Mallocs": 1747420, + "Frees": 1617371 + }, + { + "Size": 320, + "Mallocs": 4829, + "Frees": 3282 + }, + { + "Size": 352, + "Mallocs": 83681, + "Frees": 82264 + }, + { + "Size": 384, + "Mallocs": 3024, + "Frees": 2009 + }, + { + "Size": 416, + "Mallocs": 2534, + "Frees": 2134 + }, + { + "Size": 448, + "Mallocs": 2288, + "Frees": 1784 + }, + { + "Size": 480, + "Mallocs": 2686, + "Frees": 2434 + }, + { + "Size": 512, + "Mallocs": 277265, + "Frees": 272027 + }, + { + "Size": 576, + "Mallocs": 164320, + "Frees": 152902 + }, + { + "Size": 640, + "Mallocs": 14575, + "Frees": 14272 + }, + { + "Size": 704, + "Mallocs": 8716, + "Frees": 8364 + }, + { + "Size": 768, + "Mallocs": 5289, + "Frees": 5072 + }, + { + "Size": 896, + "Mallocs": 15547, + "Frees": 14356 + }, + { + "Size": 1024, + "Mallocs": 73044, + "Frees": 69817 + }, + { + "Size": 1152, + "Mallocs": 73955, + "Frees": 72310 + }, + { + "Size": 1280, + "Mallocs": 62537, + "Frees": 60565 + }, + { + "Size": 1408, + "Mallocs": 37251, + "Frees": 35700 + }, + { + "Size": 1536, + "Mallocs": 33999, + "Frees": 33097 + }, + { + "Size": 1792, + "Mallocs": 45383, + "Frees": 44658 + }, + { + "Size": 2048, + "Mallocs": 3753, + "Frees": 3688 + }, + { + "Size": 2304, + "Mallocs": 12732, + "Frees": 7985 + }, + { + "Size": 2688, + "Mallocs": 838, + "Frees": 761 + }, + { + "Size": 3072, + "Mallocs": 2908, + "Frees": 2903 + }, + { + "Size": 3200, + "Mallocs": 146, + "Frees": 143 + }, + { + "Size": 3456, + "Mallocs": 567, + "Frees": 492 + }, + { + "Size": 4096, + "Mallocs": 2241, + "Frees": 2136 + }, + { + "Size": 4864, + "Mallocs": 481, + "Frees": 432 + }, + { + "Size": 5376, + "Mallocs": 1494, + "Frees": 1410 + }, + { + "Size": 6144, + "Mallocs": 448, + "Frees": 440 + }, + { + "Size": 6528, + "Mallocs": 161, + "Frees": 158 + }, + { + "Size": 6784, + "Mallocs": 778, + "Frees": 778 + }, + { + "Size": 6912, + "Mallocs": 86, + "Frees": 83 + }, + { + "Size": 8192, + "Mallocs": 1377, + "Frees": 981 + }, + { + "Size": 9472, + "Mallocs": 995, + "Frees": 981 + }, + { + "Size": 9728, + "Mallocs": 2122, + "Frees": 72 + }, + { + "Size": 10240, + "Mallocs": 132, + "Frees": 130 + }, + { + "Size": 10880, + "Mallocs": 137, + "Frees": 137 + }, + { + "Size": 12288, + "Mallocs": 704, + "Frees": 629 + }, + { + "Size": 13568, + "Mallocs": 156, + "Frees": 147 + }, + { + "Size": 14336, + "Mallocs": 62, + "Frees": 60 + }, + { + "Size": 16384, + "Mallocs": 279, + "Frees": 264 + }, + { + "Size": 18432, + "Mallocs": 489, + "Frees": 222 + }, + { + "Size": 19072, + "Mallocs": 15, + "Frees": 15 + } + ] + }, + "syncGateway_changeCache": { + "maxPending": 0 + }, + "syncgateway": { + "global": { + "resource_utilization": { + "admin_net_bytes_recv": 186313933, + "admin_net_bytes_sent": 67278955, + "error_count": 0, + "go_memstats_heapalloc": 143386592, + "go_memstats_heapidle": 81616896, + "go_memstats_heapinuse": 250585088, + "go_memstats_heapreleased": 26927104, + "go_memstats_pausetotalns": 88693194, + "go_memstats_stackinuse": 3342336, + "go_memstats_stacksys": 3342336, + "go_memstats_sys": 357708024, + "goroutines_high_watermark": 311, + "num_goroutines": 308, + "process_cpu_percent_utilization": 0.48708528167446, + "process_memory_resident": 335306752, + "pub_net_bytes_recv": 186313933, + "pub_net_bytes_sent": 67278955, + "system_memory_total": 33291907072, + "warn_count": 6 + } + }, + "per_db": { + "beer-sample": { + "cache": { + "abandoned_seqs": 0, + "chan_cache_active_revs": 0, + "chan_cache_bypass_count": 0, + "chan_cache_channels_added": 0, + "chan_cache_channels_evicted_inactive": 0, + "chan_cache_channels_evicted_nru": 0, + "chan_cache_compact_count": 0, + "chan_cache_compact_time": 0, + "chan_cache_hits": 0, + "chan_cache_max_entries": 0, + "chan_cache_misses": 0, + "chan_cache_num_channels": 0, + "chan_cache_pending_queries": 0, + "chan_cache_removal_revs": 0, + "chan_cache_tombstone_revs": 0, + "high_seq_cached": 7304, + "high_seq_stable": 7305, + "num_active_channels": 0, + "num_skipped_seqs": 0, + "pending_seq_len": 0, + "rev_cache_bypass": 0, + "rev_cache_hits": 0, + "rev_cache_misses": 0, + "skipped_seq_len": 0 + }, + "cbl_replication_pull": { + "attachment_pull_bytes": 0, + "attachment_pull_count": 0, + "max_pending": 222, + "num_replications_active": 0, + "num_pull_repl_active_continuous": 0, + "num_pull_repl_active_one_shot": 0, + "num_pull_repl_caught_up": 0, + "num_pull_repl_since_zero": 0, + "num_pull_repl_total_continuous": 0, + "num_pull_repl_total_one_shot": 0, + "request_changes_count": 0, + "request_changes_time": 0, + "rev_processing_time": 0, + "rev_send_count": 0, + "rev_send_latency": 0 + }, + "cbl_replication_push": { + "attachment_push_bytes": 0, + "attachment_push_count": 0, + "doc_push_count": 0, + "propose_change_count": 0, + "propose_change_time": 0, + "sync_function_count": 0, + "sync_function_time": 0, + "write_processing_time": 0 + }, + "database": { + "abandoned_seqs": 0, + "conflict_write_count": 0, + "crc32c_match_count": 0, + "dcp_caching_count": 7303, + "dcp_caching_time": 47545220448, + "dcp_received_count": 7303, + "dcp_received_time": 71654930936, + "doc_reads_bytes_blip": 0, + "doc_writes_bytes": 2542924, + "doc_writes_bytes_blip": 0, + "doc_writes_xattr_bytes": 1822707, + "high_seq_feed": 14610, + "num_doc_reads_blip": 0, + "num_doc_reads_rest": 0, + "num_doc_writes": 7303, + "num_replications_active": 0, + "num_replications_total": 0, + "num_tombstones_compacted": 0, + "sequence_assigned_count": 7304, + "sequence_get_count": 2, + "sequence_incr_count": 733, + "sequence_released_count": 1, + "sequence_reserved_count": 7305, + "warn_channels_per_doc_count": 0, + "warn_grants_per_doc_count": 0, + "warn_xattr_size_count": 0, + "cache_feed": {}, + "import_feed": {} + }, + "gsi_views": { + "access_query_count": 1, + "access_query_error_count": 0, + "access_query_time": 214030402, + "allDocs_query_count": 0, + "allDocs_query_error_count": 0, + "allDocs_query_time": 0, + "channelsStar_query_count": 0, + "channelsStar_query_error_count": 0, + "channelsStar_query_time": 0, + "channels_query_count": 0, + "channels_query_error_count": 0, + "channels_query_time": 0, + "principals_query_count": 0, + "principals_query_error_count": 0, + "principals_query_time": 0, + "resync_query_count": 0, + "resync_query_error_count": 0, + "resync_query_time": 0, + "roleAccess_query_count": 1, + "roleAccess_query_error_count": 0, + "roleAccess_query_time": 197753953, + "sequences_query_count": 0, + "sequences_query_error_count": 0, + "sequences_query_time": 0, + "sessions_query_count": 0, + "sessions_query_error_count": 0, + "sessions_query_time": 0, + "tombstones_query_count": 0, + "tombstones_query_error_count": 0, + "tombstones_query_time": 0 + }, + "security": { + "auth_failed_count": 0, + "auth_success_count": 0, + "num_access_errors": 0, + "num_docs_rejected": 0, + "total_auth_time": 0 + }, + "shared_bucket_import": { + "import_count": 7303, + "import_cancel_cas": 0, + "import_error_count": 0, + "import_processing_time": 58213371539, + "import_high_seq": 7304, + "import_partitions": 16 + } + }, + "travel-sample": { + "cache": { + "abandoned_seqs": 0, + "chan_cache_active_revs": 0, + "chan_cache_bypass_count": 0, + "chan_cache_channels_added": 0, + "chan_cache_channels_evicted_inactive": 0, + "chan_cache_channels_evicted_nru": 0, + "chan_cache_compact_count": 0, + "chan_cache_compact_time": 0, + "chan_cache_hits": 0, + "chan_cache_max_entries": 0, + "chan_cache_misses": 0, + "chan_cache_num_channels": 0, + "chan_cache_pending_queries": 0, + "chan_cache_removal_revs": 0, + "chan_cache_tombstone_revs": 0, + "high_seq_cached": 31592, + "high_seq_stable": 31595, + "num_active_channels": 0, + "num_skipped_seqs": 0, + "pending_seq_len": 0, + "rev_cache_bypass": 0, + "rev_cache_hits": 0, + "rev_cache_misses": 0, + "skipped_seq_len": 0 + }, + "cbl_replication_pull": { + "attachment_pull_bytes": 0, + "attachment_pull_count": 0, + "max_pending": 260, + "num_replications_active": 0, + "num_pull_repl_active_continuous": 0, + "num_pull_repl_active_one_shot": 0, + "num_pull_repl_caught_up": 0, + "num_pull_repl_since_zero": 0, + "num_pull_repl_total_continuous": 0, + "num_pull_repl_total_one_shot": 0, + "request_changes_count": 0, + "request_changes_time": 0, + "rev_processing_time": 0, + "rev_send_count": 0, + "rev_send_latency": 0 + }, + "cbl_replication_push": { + "attachment_push_bytes": 0, + "attachment_push_count": 0, + "doc_push_count": 0, + "propose_change_count": 0, + "propose_change_time": 0, + "sync_function_count": 0, + "sync_function_time": 0, + "write_processing_time": 0 + }, + "database": { + "abandoned_seqs": 0, + "conflict_write_count": 0, + "crc32c_match_count": 0, + "dcp_caching_count": 31591, + "dcp_caching_time": 157384807512, + "dcp_received_count": 31591, + "dcp_received_time": 292348176079, + "doc_reads_bytes_blip": 0, + "doc_writes_bytes": 36196306, + "doc_writes_bytes_blip": 0, + "doc_writes_xattr_bytes": 7935238, + "high_seq_feed": 63187, + "num_doc_reads_blip": 0, + "num_doc_reads_rest": 0, + "num_doc_writes": 31591, + "num_replications_active": 0, + "num_replications_total": 0, + "num_tombstones_compacted": 0, + "sequence_assigned_count": 31592, + "sequence_get_count": 2, + "sequence_incr_count": 3162, + "sequence_released_count": 3, + "sequence_reserved_count": 31595, + "warn_channels_per_doc_count": 0, + "warn_grants_per_doc_count": 0, + "warn_xattr_size_count": 0, + "cache_feed": {}, + "import_feed": {} + }, + "gsi_views": { + "access_query_count": 1, + "access_query_error_count": 0, + "access_query_time": 116702828, + "allDocs_query_count": 0, + "allDocs_query_error_count": 0, + "allDocs_query_time": 0, + "channelsStar_query_count": 0, + "channelsStar_query_error_count": 0, + "channelsStar_query_time": 0, + "channels_query_count": 0, + "channels_query_error_count": 0, + "channels_query_time": 0, + "principals_query_count": 0, + "principals_query_error_count": 0, + "principals_query_time": 0, + "resync_query_count": 0, + "resync_query_error_count": 0, + "resync_query_time": 0, + "roleAccess_query_count": 1, + "roleAccess_query_error_count": 0, + "roleAccess_query_time": 120046831, + "sequences_query_count": 0, + "sequences_query_error_count": 0, + "sequences_query_time": 0, + "sessions_query_count": 0, + "sessions_query_error_count": 0, + "sessions_query_time": 0, + "tombstones_query_count": 0, + "tombstones_query_error_count": 0, + "tombstones_query_time": 0 + }, + "security": { + "auth_failed_count": 0, + "auth_success_count": 0, + "num_access_errors": 0, + "num_docs_rejected": 0, + "total_auth_time": 0 + }, + "shared_bucket_import": { + "import_count": 31591, + "import_cancel_cas": 0, + "import_error_count": 0, + "import_processing_time": 345830590715, + "import_high_seq": 31592, + "import_partitions": 16 + } + } + }, + "per_replication": { + "rep1": { + "sgr_num_docs_pushed": 0, + "sgr_num_attachment_bytes_transferred": 0, + "sgr_num_attachments_transferred": 0, + "sgr_docs_checked_sent": 0, + "sgr_num_docs_failed_to_push": 0 + } + } + } +} diff --git a/x-pack/metricbeat/module/syncgateway/db/_meta/data.json b/x-pack/metricbeat/module/syncgateway/db/_meta/data.json new file mode 100644 index 000000000000..b9abb0610641 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/_meta/data.json @@ -0,0 +1,206 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "syncgateway.db", + "duration": 115000, + "module": "syncgateway" + }, + "metricset": { + "name": "db", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:44949", + "type": "syncgateway" + }, + "syncgateway": { + "db": { + "cache": { + "channel": { + "hits": 0, + "misses": 0, + "revs": { + "active": 0, + "removal": 0, + "tombstone": 0 + } + }, + "revs": { + "hits": 0, + "misses": 0 + } + }, + "cbl": { + "replication": { + "pull": { + "active": { + "continuous": 0, + "count": 0, + "one_shot": 0 + }, + "attachment": { + "bytes": 0, + "count": 0 + }, + "caught_up": 0, + "request_changes": { + "count": 0, + "time": 0 + }, + "rev": { + "processing_time": 0, + "send": { + "count": 0, + "latency": 0 + } + }, + "since_zero": 0, + "total": { + "continuous": 0, + "one_shot": 0 + } + }, + "push": { + "attachment": { + "bytes": 0, + "count": 0 + }, + "doc_push_count": 0, + "propose_change": { + "count": 0, + "time": 0 + }, + "sync_function": { + "count": 0, + "time": 0 + }, + "write_processing_time": 0 + } + } + }, + "gsi": { + "views": { + "access": { + "query": { + "count": 1, + "error": { + "count": 0 + }, + "time": 214030402 + } + }, + "all_docs": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "channels": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + }, + "star": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + } + }, + "principals": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "resync": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "role_access": { + "query": { + "count": 1, + "error": { + "count": 0 + }, + "time": 197753953 + } + }, + "sequences": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "sessions": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + }, + "tombstones": { + "query": { + "count": 0, + "error": { + "count": 0 + }, + "time": 0 + } + } + } + }, + "metrics": { + "docs": { + "writes": { + "bytes": 2542924, + "conflict": { + "count": 0 + }, + "count": 7303 + } + }, + "replications": { + "active": 0, + "total": 0 + } + }, + "name": "beer-sample", + "security": { + "access_errors": { + "count": 0 + }, + "auth": { + "failed": { + "count": 0 + } + }, + "docs_rejected": { + "count": 0 + } + } + } + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/syncgateway/db/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/db/_meta/docs.asciidoc new file mode 100644 index 000000000000..f3185e37d4da --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/_meta/docs.asciidoc @@ -0,0 +1 @@ +SyncGateway `db` metriset contains the most relevant information of the module about the db metrics diff --git a/x-pack/metricbeat/module/syncgateway/db/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/db/_meta/fields.yml new file mode 100644 index 000000000000..4d7d4393e190 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/_meta/fields.yml @@ -0,0 +1,380 @@ +- name: syncgateway + type: group + description: > + Couchbase Sync Gateway metrics. + release: beta + fields: + - name: name + type: keyword + description: > + Name of the database on when field `couchbase.syncgateway.type` is `db_stats`. + - name: metrics + type: group + description: Metrics of all databases contained in the config file of the SyncGateway instance. + fields: + - name: docs + type: group + fields: + - name: writes + type: group + fields: + - name: conflict.count + type: long + - name: count + type: long + - name: bytes + type: long + - name: replications + type: group + fields: + - name: active + type: long + description: Number of active replications + - name: total + type: long + description: Total number of replications (active or not) + - name: gsi + type: group + fields: + - name: views + type: group + fields: + - name: tombstones + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: error.count + type: double + - name: access + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: channels + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: star + type: group + fields: + - name: query + type: group + fields: + - name: time + type: double + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: role_access + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: sequences + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: all_docs + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: principals + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: resync + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: sessions + type: group + fields: + - name: query + type: group + fields: + - name: count + type: double + - name: error + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: security + type: group + fields: + - name: access_errors + type: group + fields: + - name: count + type: double + - name: auth + type: group + fields: + - name: failed + type: group + fields: + - name: count + type: double + - name: docs_rejected + type: group + fields: + - name: count + type: double + - name: cache + type: group + fields: + - name: channel + type: group + fields: + - name: revs + type: group + fields: + - name: active + type: double + - name: removal + type: double + - name: tombstone + type: double + - name: hits + type: double + - name: misses + type: double + - name: revs + type: group + fields: + - name: hits + type: double + - name: misses + type: double + - name: cbl + type: group + fields: + - name: replication + type: group + fields: + - name: pull + type: group + fields: + - name: caught_up + type: double + - name: since_zero + type: double + - name: total + type: group + fields: + - name: continuous + type: double + - name: one_shot + type: double + - name: active + type: group + fields: + - name: continuous + type: double + - name: count + type: double + - name: one_shot + type: double + - name: attachment + type: group + fields: + - name: bytes + type: long + - name: count + type: long + - name: request_changes + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: rev + type: group + fields: + - name: processing_time + type: double + - name: send + type: group + fields: + - name: count + type: double + - name: latency + type: double + - name: push + type: group + fields: + - name: attachment + type: group + fields: + - name: bytes + type: double + - name: count + type: double + - name: doc_push_count + type: double + - name: propose_change + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: sync_function + type: group + fields: + - name: count + type: double + - name: time + type: double + - name: write_processing_time + type: double + - name: memstats + type: group + description: Dumps a large amount of information about the memory heap and garbage collector + fields: + - name: BuckHashSys + type: double + - name: Mallocs + type: double + - name: PauseTotalNs + type: double + - name: TotalAlloc + type: double + - name: Alloc + type: double + - name: GCSys + type: double + - name: LastGC + type: double + - name: MSpanSys + type: double + - name: GCCPUFraction + type: double + - name: HeapReleased + type: double + - name: HeapSys + type: double + - name: DebugGC + type: long + - name: HeapIdle + type: double + - name: Lookups + type: double + - name: HeapObjects + type: double + - name: MSpanInuse + type: double + - name: NumForcedGC + type: double + - name: OtherSys + type: double + - name: Frees + type: double + - name: NextGC + type: double + - name: StackInuse + type: double + - name: Sys + type: double + - name: NumGC + type: double + - name: EnableGC + type: long + - name: HeapAlloc + type: double + - name: MCacheInuse + type: double + - name: MCacheSys + type: double + - name: HeapInuse + type: double + - name: StackSys + type: double diff --git a/x-pack/metricbeat/module/syncgateway/db/data.go b/x-pack/metricbeat/module/syncgateway/db/data.go new file mode 100644 index 000000000000..d8fb8669ffce --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/data.go @@ -0,0 +1,200 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package db + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +type SgResponse struct { + SyncgatewayChangeCache struct { + MaxPending float64 `json:"maxPending"` + } `json:"syncGateway_changeCache"` + Syncgateway Syncgateway `json:"syncgateway"` + MemStats map[string]interface{} `json:"memstats"` +} + +type Syncgateway struct { + Global struct { + ResourceUtilization map[string]interface{} `json:"resource_utilization"` + } `json:"global"` + PerDb map[string]map[string]interface{} `json:"per_db"` + PerReplication map[string]map[string]interface{} `json:"per_replication"` +} + +var ( + dbSchema = s.Schema{ + "cache": c.Dict("cache", s.Schema{ + "channel": s.Object{ + "revs": s.Object{ + "active": c.Float("chan_cache_active_revs"), + "removal": c.Float("chan_cache_removal_revs"), + "tombstone": c.Float("chan_cache_tombstone_revs"), + }, + "hits": c.Float("chan_cache_hits"), + "misses": c.Float("chan_cache_misses"), + }, + "revs": s.Object{ + "hits": c.Float("rev_cache_hits"), + "misses": c.Float("rev_cache_misses"), + }, + }), + "metrics": c.Dict("database", s.Schema{ + "replications": s.Object{ + "active": c.Float("num_replications_active"), + "total": c.Float("num_replications_total"), + }, + "docs": s.Object{ + "writes": s.Object{ + "count": c.Float("num_doc_writes"), + "bytes": c.Float("doc_writes_bytes"), + "conflict": s.Object{"count": c.Float("conflict_write_count")}, + }, + }, + }), + "security": c.Dict("security", s.Schema{ + "auth": s.Object{ + "failed": s.Object{"count": c.Float("auth_failed_count")}, + }, + "access_errors": s.Object{"count": c.Float("num_access_errors")}, + "docs_rejected": s.Object{"count": c.Float("num_docs_rejected")}, + }), + "cbl": s.Object{ + "replication": s.Object{ + "pull": c.Dict("cbl_replication_pull", s.Schema{ + "attachment": s.Object{ + "bytes": c.Float("attachment_pull_bytes"), + "count": c.Float("attachment_pull_count"), + }, + "active": s.Object{ + "count": c.Float("num_replications_active"), + "continuous": c.Float("num_pull_repl_active_continuous"), + "one_shot": c.Float("num_pull_repl_active_one_shot"), + }, + "total": s.Object{ + "continuous": c.Float("num_pull_repl_total_continuous"), + "one_shot": c.Float("num_pull_repl_total_one_shot"), + }, + "caught_up": c.Float("num_pull_repl_caught_up"), + "since_zero": c.Float("num_pull_repl_since_zero"), + "request_changes": s.Object{ + "count": c.Float("request_changes_count"), + "time": c.Float("request_changes_time"), + }, + "rev": s.Object{ + "processing_time": c.Float("rev_processing_time"), + "send": s.Object{ + "count": c.Float("rev_send_count"), + "latency": c.Float("rev_send_latency"), + }, + }, + }), + "push": c.Dict("cbl_replication_push", s.Schema{ + "write_processing_time": c.Float("write_processing_time"), + "doc_push_count": c.Float("doc_push_count"), + "attachment": s.Object{ + "bytes": c.Float("attachment_push_bytes"), + "count": c.Float("attachment_push_count"), + }, + "propose_change": s.Object{ + "count": c.Float("propose_change_count"), + "time": c.Float("propose_change_time"), + }, + "sync_function": s.Object{ + "count": c.Float("sync_function_count"), + "time": c.Float("sync_function_time"), + }, + }), + }, + }, + "gsi": s.Object{ + "views": c.Dict("gsi_views", s.Schema{ + "access": s.Object{ + "query": s.Object{ + "count": c.Float("access_query_count"), + "error": s.Object{"count": c.Float("access_query_error_count")}, + "time": c.Float("access_query_time"), + }, + }, + "all_docs": s.Object{ + "query": s.Object{ + "count": c.Float("allDocs_query_count"), + "error": s.Object{"count": c.Float("allDocs_query_error_count")}, + "time": c.Float("allDocs_query_time"), + }, + }, + "channels": s.Object{ + "star": s.Object{ + "query": s.Object{ + "count": c.Float("channelsStar_query_count"), + "error": s.Object{"count": c.Float("channelsStar_query_error_count")}, + "time": c.Float("channelsStar_query_time"), + }, + }, + "query": s.Object{ + "count": c.Float("channels_query_count"), + "error": s.Object{"count": c.Float("channels_query_error_count")}, + "time": c.Float("channels_query_time"), + }, + }, + "principals": s.Object{ + "query": s.Object{ + "count": c.Float("principals_query_count"), + "error": s.Object{"count": c.Float("principals_query_error_count")}, + "time": c.Float("principals_query_time"), + }, + }, + "resync": s.Object{ + "query": s.Object{ + "count": c.Float("resync_query_count"), + "error": s.Object{"count": c.Float("resync_query_error_count")}, + "time": c.Float("resync_query_time"), + }, + }, + "role_access": s.Object{ + "query": s.Object{ + "count": c.Float("roleAccess_query_count"), + "error": s.Object{"count": c.Float("roleAccess_query_error_count")}, + "time": c.Float("roleAccess_query_time"), + }, + }, + "sequences": s.Object{ + "query": s.Object{ + "count": c.Float("sequences_query_count"), + "error": s.Object{"count": c.Float("sequences_query_error_count")}, + "time": c.Float("sequences_query_time"), + }, + }, + "sessions": s.Object{ + "query": s.Object{ + "count": c.Float("sessions_query_count"), + "error": s.Object{"count": c.Float("sessions_query_error_count")}, + "time": c.Float("sessions_query_time"), + }, + }, + "tombstones": s.Object{ + "query": s.Object{ + "count": c.Float("tombstones_query_count"), + "error": s.Object{"count": c.Float("tombstones_query_error_count")}, + "time": c.Float("tombstones_query_time"), + }, + }, + }), + }, + } +) + +func eventMapping(r mb.ReporterV2, content *syncgateway.SgResponse) { + for dbName, db := range content.Syncgateway.PerDb { + dbData, _ := dbSchema.Apply(db) + dbData.Put("name", dbName) + r.Event(mb.Event{ + MetricSetFields: dbData, + }) + } +} diff --git a/x-pack/metricbeat/module/syncgateway/db/data_test.go b/x-pack/metricbeat/module/syncgateway/db/data_test.go new file mode 100644 index 000000000000..847d2a622595 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/data_test.go @@ -0,0 +1,44 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package db + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func TestData(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + f := mbtest.NewReportingMetricSetV2Error(t, syncgateway.GetConfig([]string{"db"}, server.URL)) + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + t.Skip("Skipping test because it seems like 'TestMetricsetFieldsDocumented' function, used here in this test; has some kind of bug") + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"db"}, server.URL) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} diff --git a/x-pack/metricbeat/module/syncgateway/db/db.go b/x-pack/metricbeat/module/syncgateway/db/db.go new file mode 100644 index 000000000000..eae750f9badf --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/db/db.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package db + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +const ( + defaultScheme = "http" + defaultPath = "/_expvar" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("syncgateway", "db", New, + mb.WithHostParser(hostParser), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + mod syncgateway.Module +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(syncgateway.Module) + if !ok { + return nil, fmt.Errorf("error trying to type assert syncgateway.Module. Base module must be composed of syncgateway.Module") + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + http: http, + mod: mod, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + response, err := m.mod.GetSyncgatewayResponse(m.http) + if err != nil { + return err + } + + eventMapping(reporter, response) + + return nil +} diff --git a/x-pack/metricbeat/module/syncgateway/doc.go b/x-pack/metricbeat/module/syncgateway/doc.go new file mode 100644 index 000000000000..2654d21ebd35 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/doc.go @@ -0,0 +1,5 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package syncgateway diff --git a/x-pack/metricbeat/module/syncgateway/fields.go b/x-pack/metricbeat/module/syncgateway/fields.go new file mode 100644 index 000000000000..365091180c47 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/fields.go @@ -0,0 +1,23 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +// Code generated by beats/dev-tools/cmd/asset/asset.go - DO NOT EDIT. + +package syncgateway + +import ( + "github.com/elastic/beats/v7/libbeat/asset" +) + +func init() { + if err := asset.SetFields("metricbeat", "syncgateway", asset.ModuleFieldsPri, AssetSyncgateway); err != nil { + panic(err) + } +} + +// AssetSyncgateway returns asset data. +// This is the base64 encoded gzipped contents of module/syncgateway. +func AssetSyncgateway() string { + return "eJzsXMty27gS3fsrurK6dxF9gBe36o4zcVw1cVLjmTUNgi0SEQkweFijfP0UqRdFAQT4ij011CILKTg4fbqBbrz8Hja4uwW14zQlGrdkdwOgmc7xFt497Ti933/77gYgQUUlKzUT/BYav0GBWjKqbgAk5kgU3kKMmtwArBnmibq9AQB4D5wU2O6r+uhdibeQSmHKwzcXXf3v8CXAc6PtM1DBNWFcgc4QGF8LWZCqBRCegNJEM6UZVbCWomjyXR3wmuS6CdpJdhCtPnfC0CwmCuu+oSXWqvF/26odP22CTZLVvxc/HBlucLcVMmn91sGz+jySAkGsayUToklNW3DYZsj3NOCZHu1ZNQRaVZ0+A1PwnMRRJbp6XlkJn4PkmnNb1SvGn/eNK4okz08U1TEGMAHGa/ZU8DVLYc3yk0HNUGVcacIprlq92aRusk/EFfVu+l2gTeCtZBpt0H54XxfNbipZckb1igrDtfO/H7vMBU8DMCeDinduETxQRwiJZc5oPQHM4ylCNXtpj7lgWy/C+dEUMco6mmtQH/kzCS00ySfh8EeFBPzEpEkB/nPgJSRwof9rHdCpYsGD2Te8Xhhu5/GaFkWstOA/YYx9Nyh33iDu6imkN+gxAs+9JsLEuSt4L0E1u8osozFRSiG9c08Q9Hk4UlSLU/s4IAjUZwoEmgM9TYLXitUTyYxwjvkSUktITSHVaTWhSZdK0zvbF17hvYb2DP3kgz5ugTl9Dj3DGXpJBz3kgwFmQp/ZTYocoyVnLhPcpDlT4XeDnC7F9RJTk8UUyfPIsd0CS0hdgy4h5Q2pUjJOWUmW4n4JqsmCSqLacboE1BJQIzHPxZRSzg3gJaSuQJeQusI8hxI1kum2k4dv0O9XjlEt+Twb9V16dmp4omh0NguzNWE5tg9Uw+B9XUDv4zSPCFXhHEn8hlRbKb+Sn06NCc3sx9ZDgvKwYzuLmRJf5p+IO48Uoe9Oo8RCvDhPB3vDnU7OxgAewTKmu+UMwCiYUp6dBs/ocDh1fLS8nnmnsRC3HT98XDWOgmdRqzR59xn2JNMpMWmmo868328jn3GK0Q+UYroR5j7Mh5nqMK4ZN8J0XbYItqOJLDhGKhOTFHk9p8d/hkCzlMCzqK41oVmBAcXIlMr7rgBB2I0iGCR50CUlid8NKh1VZUcacF3pzS+gpj7qlPjyU0UppahWJYyn0RyXVhRyV9XfzyZ4C8vLJnhONHLqOyQOgj/ndGVbfcG09fJbn5peIx80ln5R5YVo1EKyjVpKUQqFh0lvmfPaeGrHabQ2nDqq5UvQf5089fXmKHyi9q5zCizqW+bBi52LS68fTFEqIJATmSKQopIQxPry9UAsjK4vjhdYCLmDDElZvylIiYxJikBFniPVV5uPvnXVL4ZuPhGVPe3cq1DPCvYzyfOuq+ie5l+JUVjf/H0cjFE3/39FYyjCqMb3dyPk+40ofX83WPynkvARvd/f3X3986MkrrkiCOQTkvL3/YMR9wZfAMYIOz5gbNIOGTvv6Vd9PyTWqSPMhUJsTDmYe9X9l/gbUuuGTXgYPHCjBtvwaIqPQlJMhsfiF52hHOHDjxKtNU0Yf/xrxDB60oRuRuk3wu5HUwxn/isncY5jIn/U1Pf5jtAMRym3hxihXz18R/mu8n4PAo3EL+SoB3mXTxbr1H58jAcPGpLZaoMh7/vctYK3RLLXCN5mHbWBt62zJvC2HNTIVgN4G1lzv19Me84PoOjO9d7GHTk+qO0Avvac7pjVPLnc7wprDg8yzZ67w9xom7e8Ld252tvUkaO97Wy52c/TlpO9rZy52N+yv1223Ott5Mi5AZE5aGpx59jApgN0ceXUMO8Fdth9uDcqmZYom7BTPXGf5MW4xJxoTGDLdFa/G28y7fsKvHP/cfy5qJaEqzVK+RNum4x4cX3+uB41n2Tad9M0DNZCgs6YOvqh7YNrqhO8M2+/fc5wf/jaeP98Jn3Jtz72vQhvpYnUmFwTb17CmSVAaIZ0g0mk3DvgAbtzXikSQc1eiEOHtdMOx159BIHWAcGbuUU1NlgmVQhC75rNwV9nRB96BS0gxoOjQo04Emf2QjXgr5E8fDj+oY6rGeGcs5Qw8vKZ0aiMleYiJvkJF4xmOfvRTopDklZ9Q9J6BOOpWFIhhdGMo4oylmbRlmiUBZGbfjDcFNEZql/bwzZ9cML13s8rTVSipMh1ZFfYQwyu1v+RRMWSrgTsts7EEUe7V4bdkqK24+7xs3xXag4QypEcXoXXqXBKCsYr9SMbyEw+GK5SxyCNeh9A+WzIkNg8Mt5fxLHX2GFnG4LZN+r7IDj2CntASPdphwflNPRJ14blGI09rzW6Q7Bavc3CagLRlXWHNtSyju1d59hSO6WxiA7zvO1upAdgSyT3Zd6/AwAA//8fni1d" +} diff --git a/x-pack/metricbeat/module/syncgateway/memory/_meta/data.json b/x-pack/metricbeat/module/syncgateway/memory/_meta/data.json new file mode 100644 index 000000000000..d10b990f9673 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/_meta/data.json @@ -0,0 +1,49 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "syncgateway.memory", + "duration": 115000, + "module": "syncgateway" + }, + "metricset": { + "name": "memory", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:39195", + "type": "syncgateway" + }, + "syncgateway": { + "memory": { + "Alloc": 159398816, + "BuckHashSys": 1759627, + "DebugGC": false, + "EnableGC": true, + "Frees": 55790048, + "GCCPUFraction": 0.008328526565497294, + "GCSys": 13555712, + "HeapAlloc": 159398816, + "HeapIdle": 67780608, + "HeapInuse": 264388608, + "HeapObjects": 1877567, + "HeapReleased": 26927104, + "HeapSys": 332169216, + "LastGC": 1620310398040906500, + "Lookups": 0, + "MCacheInuse": 20832, + "MCacheSys": 32768, + "MSpanInuse": 4030632, + "MSpanSys": 4358144, + "Mallocs": 57667615, + "NextGC": 265304208, + "NumForcedGC": 0, + "NumGC": 51, + "OtherSys": 2457453, + "PauseTotalNs": 88693194, + "StackInuse": 3375104, + "StackSys": 3375104, + "Sys": 357708024, + "TotalAlloc": 2719416432 + } + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/syncgateway/memory/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/memory/_meta/docs.asciidoc new file mode 100644 index 000000000000..69772dee2ca3 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/_meta/docs.asciidoc @@ -0,0 +1 @@ +SyncGateway `memory` metriset contains detailed information about the memory usage of the SyncGateway Go's process. diff --git a/x-pack/metricbeat/module/syncgateway/memory/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/memory/_meta/fields.yml new file mode 100644 index 000000000000..824a590fbdb5 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/_meta/fields.yml @@ -0,0 +1,64 @@ +- name: memory + type: group + description: > + SyncGateway memory metrics. It dumps a large amount of information about the memory heap and garbage collector + release: beta + fields: + - name: BuckHashSys + type: double + - name: Mallocs + type: double + - name: PauseTotalNs + type: double + - name: TotalAlloc + type: double + - name: Alloc + type: double + - name: GCSys + type: double + - name: LastGC + type: double + - name: MSpanSys + type: double + - name: GCCPUFraction + type: double + - name: HeapReleased + type: double + - name: HeapSys + type: double + - name: DebugGC + type: long + - name: HeapIdle + type: double + - name: Lookups + type: double + - name: HeapObjects + type: double + - name: MSpanInuse + type: double + - name: NumForcedGC + type: double + - name: OtherSys + type: double + - name: Frees + type: double + - name: NextGC + type: double + - name: StackInuse + type: double + - name: Sys + type: double + - name: NumGC + type: double + - name: EnableGC + type: long + - name: HeapAlloc + type: double + - name: MCacheInuse + type: double + - name: MCacheSys + type: double + - name: HeapInuse + type: double + - name: StackSys + type: double diff --git a/x-pack/metricbeat/module/syncgateway/memory/data.go b/x-pack/metricbeat/module/syncgateway/memory/data.go new file mode 100644 index 000000000000..b51f91f30067 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/data.go @@ -0,0 +1,20 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package memory + +import ( + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func eventMapping(r mb.ReporterV2, content *syncgateway.SgResponse) { + delete(content.MemStats, "BySize") + delete(content.MemStats, "PauseNs") + delete(content.MemStats, "PauseEnd") + + r.Event(mb.Event{ + MetricSetFields: content.MemStats, + }) +} diff --git a/x-pack/metricbeat/module/syncgateway/memory/data_test.go b/x-pack/metricbeat/module/syncgateway/memory/data_test.go new file mode 100644 index 000000000000..ad6ab9d98ff0 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/data_test.go @@ -0,0 +1,45 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package memory + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func TestData(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"memory"}, server.URL) + + f := mbtest.NewReportingMetricSetV2Error(t, config) + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"memory"}, server.URL) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} diff --git a/x-pack/metricbeat/module/syncgateway/memory/memory.go b/x-pack/metricbeat/module/syncgateway/memory/memory.go new file mode 100644 index 000000000000..0edc000d0f35 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/memory/memory.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package memory + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +const ( + defaultScheme = "http" + defaultPath = "/_expvar" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("syncgateway", "memory", New, + mb.WithHostParser(hostParser), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + mod syncgateway.Module +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(syncgateway.Module) + if !ok { + return nil, fmt.Errorf("error trying to type assert syncgateway.Module. Base module must be composed of syncgateway.Module") + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + http: http, + mod: mod, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + response, err := m.mod.GetSyncgatewayResponse(m.http) + if err != nil { + return err + } + + eventMapping(reporter, response) + + return nil +} diff --git a/x-pack/metricbeat/module/syncgateway/replication/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/replication/_meta/docs.asciidoc new file mode 100644 index 000000000000..691e7df6b779 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/_meta/docs.asciidoc @@ -0,0 +1 @@ +SyncGateway `replication` metriset contains information about the replicas. Note that you won't see any data if you don't have any replicas configured on your Couchbase buckets. diff --git a/x-pack/metricbeat/module/syncgateway/replication/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/replication/_meta/fields.yml new file mode 100644 index 000000000000..79f4e909efbb --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/_meta/fields.yml @@ -0,0 +1,40 @@ +- name: replication + type: group + description: > + SyncGateway per replication metrics. + release: beta + fields: + - name: metrics + type: group + description: Metrics related with data replication. + fields: + - name: attachment + type: group + fields: + - name: transferred + type: group + fields: + - name: bytes + type: long + description: Number of attachment bytes transferred for this replica. + - name: count + type: long + description: The total number of attachments transferred since replication started. + - name: docs + type: group + fields: + - name: checked_sent + type: double + description: The total number of documents checked for changes since replication started. + - name: pushed + type: group + fields: + - name: count + type: long + description: The total number of documents checked for changes since replication started. + - name: failed + type: long + description: The total number of documents that failed to be pushed since replication started. + - name: id + type: keyword + description: ID of the replica. diff --git a/x-pack/metricbeat/module/syncgateway/replication/data.go b/x-pack/metricbeat/module/syncgateway/replication/data.go new file mode 100644 index 000000000000..55b68b7c213e --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/data.go @@ -0,0 +1,41 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package replication + +import ( + "github.com/elastic/beats/v7/libbeat/common" + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +var replicationSchema = s.Schema{ + "docs": s.Object{ + "pushed": s.Object{ + "count": c.Int("sgr_num_docs_pushed"), + "failed": c.Int("sgr_num_docs_failed_to_push"), + }, + "checked_sent": c.Int("sgr_docs_checked_sent"), + }, + "attachment": s.Object{ + "transferred": s.Object{ + "bytes": c.Int("sgr_num_attachment_bytes_transferred"), + "count": c.Int("sgr_num_attachments_transferred"), + }, + }, +} + +func eventMapping(r mb.ReporterV2, content *syncgateway.SgResponse) { + for replID, replData := range content.Syncgateway.PerReplication { + replData, _ := replicationSchema.Apply(replData) + r.Event(mb.Event{ + MetricSetFields: common.MapStr{ + "id": replID, + "metrics": replData, + }, + }) + } +} diff --git a/x-pack/metricbeat/module/syncgateway/replication/data_test.go b/x-pack/metricbeat/module/syncgateway/replication/data_test.go new file mode 100644 index 000000000000..a622bb656e3a --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/data_test.go @@ -0,0 +1,43 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package replication + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func TestData(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + f := mbtest.NewReportingMetricSetV2Error(t, syncgateway.GetConfig([]string{"replication"}, server.URL)) + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"replication"}, server.URL) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} diff --git a/x-pack/metricbeat/module/syncgateway/replication/replication.go b/x-pack/metricbeat/module/syncgateway/replication/replication.go new file mode 100644 index 000000000000..3ce9be667054 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/replication/replication.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package replication + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +const ( + defaultScheme = "http" + defaultPath = "/_expvar" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("syncgateway", "replication", New, + mb.WithHostParser(hostParser), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + mod syncgateway.Module +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(syncgateway.Module) + if !ok { + return nil, fmt.Errorf("error trying to type assert syncgateway.Module. Base module must be composed of syncgateway.Module") + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + http: http, + mod: mod, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + response, err := m.mod.GetSyncgatewayResponse(m.http) + if err != nil { + return err + } + + eventMapping(reporter, response) + + return nil +} diff --git a/x-pack/metricbeat/module/syncgateway/resources/_meta/data.json b/x-pack/metricbeat/module/syncgateway/resources/_meta/data.json new file mode 100644 index 000000000000..d9968ce35239 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/_meta/data.json @@ -0,0 +1,57 @@ +{ + "@timestamp": "2017-10-12T08:05:34.853Z", + "event": { + "dataset": "syncgateway.resources", + "duration": 115000, + "module": "syncgateway" + }, + "metricset": { + "name": "resources", + "period": 10000 + }, + "service": { + "address": "127.0.0.1:37571", + "type": "syncgateway" + }, + "syncgateway": { + "resources": { + "admin_net_bytes": { + "recv": 186313933, + "sent": 67278955 + }, + "error_count": 0, + "go_memstats": { + "heap": { + "alloc": 143386592, + "idle": 81616896, + "inuse": 250585088, + "released": 26927104 + }, + "pause": { + "ns": 88693194 + }, + "stack": { + "inuse": 3342336, + "sys": 3342336 + }, + "sys": 357708024 + }, + "goroutines_high_watermark": 311, + "num_goroutines": 308, + "process": { + "cpu_percent_utilization": 0, + "memory_resident": 335306752 + }, + "pub_net": { + "recv": { + "bytes": 186313933 + }, + "sent": { + "bytes": 67278955 + } + }, + "system_memory_total": 33291907072, + "warn_count": 6 + } + } +} \ No newline at end of file diff --git a/x-pack/metricbeat/module/syncgateway/resources/_meta/docs.asciidoc b/x-pack/metricbeat/module/syncgateway/resources/_meta/docs.asciidoc new file mode 100644 index 000000000000..7f252b51dfd6 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/_meta/docs.asciidoc @@ -0,0 +1 @@ +SyncGateway `resources` metriset contains about the global resource utilization of the SyncGateway instance. diff --git a/x-pack/metricbeat/module/syncgateway/resources/_meta/fields.yml b/x-pack/metricbeat/module/syncgateway/resources/_meta/fields.yml new file mode 100644 index 000000000000..0de117b6fc1c --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/_meta/fields.yml @@ -0,0 +1,71 @@ +- name: resources + type: group + description: > + SyncGateway global resource utilization + release: beta + fields: + - name: error_count + type: long + - name: goroutines_high_watermark + type: long + - name: num_goroutines + type: long + - name: process + type: group + fields: + - name: cpu_percent_utilization + type: long + - name: memory_resident + type: long + - name: pub_net + type: group + fields: + - name: recv + type: group + fields: + - name: bytes + type: long + - name: sent + type: group + fields: + - name: bytes + type: long + - name: admin_net_bytes + type: group + fields: + - name: recv + type: long + - name: sent + type: long + - name: go_memstats + type: group + fields: + - name: heap + type: group + fields: + - name: alloc + type: long + - name: idle + type: long + - name: inuse + type: long + - name: released + type: long + - name: pause + type: group + fields: + - name: ns + type: long + - name: stack + type: group + fields: + - name: inuse + type: long + - name: sys + type: long + - name: sys + type: long + - name: system_memory_total + type: long + - name: warn_count + type: long diff --git a/x-pack/metricbeat/module/syncgateway/resources/data.go b/x-pack/metricbeat/module/syncgateway/resources/data.go new file mode 100644 index 000000000000..46fcd2781daf --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/data.go @@ -0,0 +1,51 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package resources + +import ( + s "github.com/elastic/beats/v7/libbeat/common/schema" + c "github.com/elastic/beats/v7/libbeat/common/schema/mapstriface" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +var globalSchema = s.Schema{ + "go_memstats": s.Object{ + "heap": s.Object{ + "alloc": c.Int("go_memstats_heapalloc"), + "idle": c.Int("go_memstats_heapidle"), + "inuse": c.Int("go_memstats_heapinuse"), + "released": c.Int("go_memstats_heapreleased"), + }, + "pause": s.Object{"ns": c.Int("go_memstats_pausetotalns")}, + "stack": s.Object{ + "inuse": c.Int("go_memstats_stackinuse"), + "sys": c.Int("go_memstats_stacksys"), + }, + "sys": c.Int("go_memstats_sys"), + }, + "admin_net_bytes": s.Object{ + "recv": c.Int("admin_net_bytes_recv"), + "sent": c.Int("admin_net_bytes_sent"), + }, + "error_count": c.Int("error_count"), + "goroutines_high_watermark": c.Int("goroutines_high_watermark"), + "num_goroutines": c.Int("num_goroutines"), + "process": s.Object{ + "cpu_percent_utilization": c.Int("process_cpu_percent_utilization"), + "memory_resident": c.Int("process_memory_resident"), + }, + "pub_net": s.Object{ + "recv": s.Object{"bytes": c.Int("pub_net_bytes_recv")}, + "sent": s.Object{"bytes": c.Int("pub_net_bytes_sent")}, + }, + "system_memory_total": c.Int("system_memory_total"), + "warn_count": c.Int("warn_count"), +} + +func eventMapping(r mb.ReporterV2, content *syncgateway.SgResponse) { + globalData, _ := globalSchema.Apply(content.Syncgateway.Global.ResourceUtilization) + r.Event(mb.Event{MetricSetFields: globalData}) +} diff --git a/x-pack/metricbeat/module/syncgateway/resources/data_test.go b/x-pack/metricbeat/module/syncgateway/resources/data_test.go new file mode 100644 index 000000000000..7c0824770f47 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/data_test.go @@ -0,0 +1,43 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package resources + +import ( + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + mbtest "github.com/elastic/beats/v7/metricbeat/mb/testing" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +func TestData(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + f := mbtest.NewReportingMetricSetV2Error(t, syncgateway.GetConfig([]string{"resources"}, server.URL)) + if err := mbtest.WriteEventsReporterV2Error(f, t, ""); err != nil { + t.Fatal("write", err) + } +} + +func TestFetch(t *testing.T) { + mux := syncgateway.CreateTestMuxer() + server := httptest.NewServer(mux) + defer server.Close() + + config := syncgateway.GetConfig([]string{"resources"}, server.URL) + metricSet := mbtest.NewReportingMetricSetV2Error(t, config) + + events, errs := mbtest.ReportingFetchV2Error(metricSet) + if len(errs) > 0 { + t.Fatalf("Expected 0 error, had %d. %v\n", len(errs), errs) + } + + assert.NotEmpty(t, events) + mbtest.TestMetricsetFieldsDocumented(t, metricSet, events) +} diff --git a/x-pack/metricbeat/module/syncgateway/resources/resources.go b/x-pack/metricbeat/module/syncgateway/resources/resources.go new file mode 100644 index 000000000000..2a4d8446ea7d --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/resources/resources.go @@ -0,0 +1,75 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package resources + +import ( + "fmt" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/beats/v7/metricbeat/mb/parse" + "github.com/elastic/beats/v7/x-pack/metricbeat/module/syncgateway" +) + +const ( + defaultScheme = "http" + defaultPath = "/_expvar" +) + +var ( + hostParser = parse.URLHostParserBuilder{ + DefaultScheme: defaultScheme, + DefaultPath: defaultPath, + }.Build() +) + +// init registers the MetricSet with the central registry. +// The New method will be called after the setup of the module and before starting to fetch data +func init() { + mb.Registry.MustAddMetricSet("syncgateway", "resources", New, + mb.WithHostParser(hostParser), + mb.DefaultMetricSet(), + ) +} + +// MetricSet type defines all fields of the MetricSet +type MetricSet struct { + mb.BaseMetricSet + http *helper.HTTP + mod syncgateway.Module +} + +// New create a new instance of the MetricSet +func New(base mb.BaseMetricSet) (mb.MetricSet, error) { + mod, ok := base.Module().(syncgateway.Module) + if !ok { + return nil, fmt.Errorf("error trying to type assert syncgateway.Module. Base module must be composed of syncgateway.Module") + } + + http, err := helper.NewHTTP(base) + if err != nil { + return nil, err + } + + return &MetricSet{ + BaseMetricSet: base, + http: http, + mod: mod, + }, nil +} + +// Fetch methods implements the data gathering and data conversion to the right +// format. It publishes the event which is then forwarded to the output. In case +// of an error set the Error field of mb.Event or simply call report.Error(). +func (m *MetricSet) Fetch(reporter mb.ReporterV2) error { + response, err := m.mod.GetSyncgatewayResponse(m.http) + if err != nil { + return err + } + + eventMapping(reporter, response) + + return nil +} diff --git a/x-pack/metricbeat/module/syncgateway/response.go b/x-pack/metricbeat/module/syncgateway/response.go new file mode 100644 index 000000000000..a11b092f6298 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/response.go @@ -0,0 +1,21 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package syncgateway + +type SgResponse struct { + SyncgatewayChangeCache struct { + MaxPending float64 `json:"maxPending"` + } `json:"syncGateway_changeCache"` + Syncgateway Syncgateway `json:"syncgateway"` + MemStats map[string]interface{} `json:"memstats"` +} + +type Syncgateway struct { + Global struct { + ResourceUtilization map[string]interface{} `json:"resource_utilization"` + } `json:"global"` + PerDb map[string]map[string]interface{} `json:"per_db"` + PerReplication map[string]map[string]interface{} `json:"per_replication"` +} diff --git a/x-pack/metricbeat/module/syncgateway/syncgateway.go b/x-pack/metricbeat/module/syncgateway/syncgateway.go new file mode 100644 index 000000000000..dfd2e16cb84e --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/syncgateway.go @@ -0,0 +1,87 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package syncgateway + +import ( + "encoding/json" + "sync" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/beats/v7/metricbeat/helper" + "github.com/elastic/beats/v7/metricbeat/mb" +) + +func init() { + // Register the ModuleFactory function for the "kubernetes" module. + if err := mb.Registry.AddModule("syncgateway", ModuleBuilder()); err != nil { + panic(err) + } +} + +// FetchPeriodThreshold is the threshold to use when checking if the cluster state call in this cache should be +// fetched again. It's used as a percentage so 0.9 means that cache is invalidated if at least 90% of period time has +// passed. This small threshold is to avoid race conditions where one of the X number of calls that happen after each +// period do not fetch new data while other does, becoming unsync by [period] seconds. It does not fully solve the +// problem because the period waits aren't guaranteed but it should be mostly enough. +const FetchPeriodThreshold = 0.9 + +type Module interface { + mb.Module + GetSyncgatewayResponse(http *helper.HTTP) (*SgResponse, error) +} + +type exprVarCache struct { + lock sync.Mutex + lastFetchTimestamp time.Time + + cachedData SgResponse +} + +type module struct { + mb.BaseModule + + expvarCache *exprVarCache + cacheHash uint64 +} + +func ModuleBuilder() func(base mb.BaseModule) (mb.Module, error) { + expvarCache := &exprVarCache{} + + return func(base mb.BaseModule) (mb.Module, error) { + m := module{ + BaseModule: base, + expvarCache: expvarCache, + } + return &m, nil + } +} + +func (m *module) GetSyncgatewayResponse(http *helper.HTTP) (*SgResponse, error) { + m.expvarCache.lock.Lock() + defer m.expvarCache.lock.Unlock() + + elapsedFromLastFetch := time.Since(m.expvarCache.lastFetchTimestamp) + + //Check the lifetime of current response data + if elapsedFromLastFetch == 0 || elapsedFromLastFetch >= (time.Duration(float64(m.Config().Period)*FetchPeriodThreshold)) { + //Fetch data again + byt, err := http.FetchContent() + if err != nil { + return nil, err + } + + input := SgResponse{} + if err = json.Unmarshal(byt, &input); err != nil { + return nil, errors.Wrap(err, "error unmarshalling JSON of SyncGateway expvar response") + } + + m.expvarCache.cachedData = input + m.expvarCache.lastFetchTimestamp = time.Now() + } + + return &m.expvarCache.cachedData, nil +} diff --git a/x-pack/metricbeat/module/syncgateway/testing.go b/x-pack/metricbeat/module/syncgateway/testing.go new file mode 100644 index 000000000000..88c2d309c2a8 --- /dev/null +++ b/x-pack/metricbeat/module/syncgateway/testing.go @@ -0,0 +1,33 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License; +// you may not use this file except in compliance with the Elastic License. + +package syncgateway + +import ( + "fmt" + "io/ioutil" + "net/http" +) + +func CreateTestMuxer() *http.ServeMux { + mux := http.NewServeMux() + + mux.Handle("/_expvar", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + input, _ := ioutil.ReadFile("../_meta/testdata/expvar.282c.json") + _, err := w.Write(input) + if err != nil { + fmt.Println("error writing response on mock server") + } + })) + + return mux +} + +func GetConfig(metricsets []string, host string) map[string]interface{} { + return map[string]interface{}{ + "module": "syncgateway", + "metricsets": metricsets, + "hosts": []string{host}, + } +} diff --git a/x-pack/metricbeat/modules.d/syncgateway.yml.disabled b/x-pack/metricbeat/modules.d/syncgateway.yml.disabled new file mode 100644 index 000000000000..54c42a11809c --- /dev/null +++ b/x-pack/metricbeat/modules.d/syncgateway.yml.disabled @@ -0,0 +1,13 @@ +# Module: syncgateway +# Docs: https://www.elastic.co/guide/en/beats/metricbeat/master/metricbeat-module-syncgateway.html + +- module: syncgateway + metricsets: + - db +# - memory +# - replication +# - resources + period: 10s + + # SyncGateway hosts + hosts: ["127.0.0.1:4985"] diff --git a/x-pack/osquerybeat/beater/osquerybeat.go b/x-pack/osquerybeat/beater/osquerybeat.go index fef59f975ec8..24a24623c08c 100644 --- a/x-pack/osquerybeat/beater/osquerybeat.go +++ b/x-pack/osquerybeat/beater/osquerybeat.go @@ -19,6 +19,7 @@ import ( "golang.org/x/sync/errgroup" "github.com/elastic/beats/v7/libbeat/beat" + "github.com/elastic/beats/v7/libbeat/beat/events" "github.com/elastic/beats/v7/libbeat/common" "github.com/elastic/beats/v7/libbeat/logp" "github.com/elastic/beats/v7/libbeat/processors" @@ -430,7 +431,7 @@ func (bt *osquerybeat) publishEvents(index, actionID, responseID string, hits [] event.Fields["response_id"] = responseID } if index != "" { - event.Meta = common.MapStr{"index": index} + event.Meta = common.MapStr{events.FieldMetaRawIndex: index} } bt.client.Publish(event)